prefetch: Introduce basic readahead prefetch

Signed-off-by: Michael Lyulko <michael.lyulko@huawei.com>
Signed-off-by: Shai Fultheim <shai.fultheim@huawei.com>
Signed-off-by: Ido Ben Tsion <ido.ben.tsion@huawei.com>
Signed-off-by: Robert Baldyga <robert.baldyga@unvertical.com>

Author: robertbaldyga

inc/ocf_cache.h

@@ -80,6 +80,9 @@ struct ocf_cache_info {
 	ocf_promotion_t promotion_policy;
 		/*!< Promotion policy selected */
 
+	ocf_pf_mask_t prefetch_mask;
+		/*!< Prefetchers selected (bitmask) */
+
 	ocf_cache_line_size_t cache_line_size;
 		/*!< Cache line size in KiB */
 

inc/ocf_def.h

@@ -346,7 +346,7 @@ typedef enum {
 /**
  * Maximum numbers of IO classes per cache instance
  */
-#define OCF_USER_IO_CLASS_MAX 33
+#define OCF_USER_IO_CLASS_MAX 34
 /**
  * Minimum value of a valid IO class ID
  */

inc/ocf_err.h

@@ -28,6 +28,9 @@ typedef enum {
 	/** Operation interrupted */
 	OCF_ERR_INTR,
 
+	/** Busy */
+	OCF_ERR_BUSY,
+
 	/** Operation not supported */
 	OCF_ERR_NOT_SUPP,
 

inc/ocf_mngt.h

@@ -10,6 +10,7 @@
 
 #include "ocf_cache.h"
 #include "ocf_core.h"
+#include "ocf_prefetch.h"
 
 /**
  * @file
@@ -1005,6 +1006,31 @@ int ocf_mngt_cache_promotion_set_param(ocf_cache_t cache, ocf_promotion_t type,
 int ocf_mngt_cache_promotion_get_param(ocf_cache_t cache, ocf_promotion_t type,
 		uint8_t param_id, uint32_t *param_value);
 
+/**
+ * @brief Set prefetch policy in given cache
+ *
+ * @attention This changes only runtime state. To make changes persistent
+ *            use function ocf_mngt_cache_save().
+ *
+ * @param[in] cache Cache handle
+ * @param[in] mask Bitmask of prefetch policies to enable
+ *
+ * @retval 0 Policy has been set successfully
+ * @retval Non-zero Error occurred and policy has not been set
+ */
+int ocf_mngt_cache_prefetch_set_policy(ocf_cache_t cache, ocf_pf_mask_t mask);
+
+/**
+ * @brief Get prefetch policy in given cache
+ *
+ * @param[in] cache Cache handle
+ * @param[out] mask Bitmask of enabled prefetch policies
+ *
+ * @retval 0 success
+ * @retval Non-zero Error occurred and policy could not be retrieved
+ */
+int ocf_mngt_cache_prefetch_get_policy(ocf_cache_t cache, ocf_pf_mask_t *mask);
+
 /**
  * @brief IO class configuration
  */

inc/ocf_prefetch.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright(c) 2021-2024 Huawei Technologies Co., Ltd.
+ * Copyright(c) 2026 Unvertical
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __OCF_PREFETCH_H__
+#define __OCF_PREFETCH_H__
+
+#include "ocf_def.h"
+
+/*
+ * Prefetch Algorithm ID
+ */
+typedef enum {
+	ocf_pf_none = -1,
+	ocf_pf_readahead = 0,
+	ocf_pf_num,
+} ocf_pf_id_t;
+
+typedef uint8_t ocf_pf_mask_t;
+
+/* The bitmask must fit all the values of ocf_pf_id_t */
+_Static_assert(OCF_BITWIDTH(ocf_pf_mask_t) >= ocf_pf_num);
+
+#define OCF_PF_MASK_DEFAULT 0
+
+#endif /* __OCF_PREFETCH_H__ */

inc/ocf_stats.h

@@ -15,6 +15,8 @@
 #ifndef __OCF_STATS_H__
 #define __OCF_STATS_H__
 
+#include "ocf_prefetch.h"
+
 /**
  * Entire row of statistcs
  */
@@ -71,7 +73,11 @@ struct ocf_stats_usage {
  * ║ Pass-Through writes  │     0 │   0.0 │ Requests ║
  * ║ Serviced requests    │   222 │ 100.0 │ Requests ║
  * ╟──────────────────────┼───────┼───────┼──────────╢
- * ║ Total requests       │   222 │ 100.0 │ Requests ║
+ * ║ Prefetch: readahead  │     2 │   0.0 │ Requests ║
+ * ║ Prefetch total       │     2 │   0.0 │ Requests ║
+ * ╟──────────────────────┼───────┼───────┼──────────╢
+ * ║ User requests        │   222 │ 100.0 │ Requests ║
+ * ║ Total requests       │   224 │ 100.0 │ Requests ║
  * ╚══════════════════════╧═══════╧═══════╧══════════╝
  * </pre>
  */
@@ -89,6 +95,8 @@ struct ocf_stats_requests {
 	struct ocf_stat rd_pt;
 	struct ocf_stat wr_pt;
 	struct ocf_stat serviced;
+	struct ocf_stat prefetch[ocf_pf_num];
+	struct ocf_stat user;
 	struct ocf_stat total;
 };
 
@@ -127,6 +135,8 @@ struct ocf_stats_blocks {
 	struct ocf_stat pass_through_rd;
 	struct ocf_stat pass_through_wr;
 	struct ocf_stat pass_through_total;
+	struct ocf_stat prefetch_core_rd[ocf_pf_num];
+	struct ocf_stat prefetch_cache_wr[ocf_pf_num];
 };
 
 /**

src/engine/engine_common.c

@@ -442,7 +442,7 @@ static int lock_clines(struct ocf_request *req)
  * Caller must assure that request map info is up to date (request
  * is traversed).
  */
-static inline void ocf_prepare_clines_miss(struct ocf_request *req)
+void ocf_prepare_clines_miss(struct ocf_request *req)
 {
 	bool part_has_space;
 
@@ -599,7 +599,7 @@ void ocf_engine_update_request_stats(struct ocf_request *req)
 {
 	ocf_core_stats_request_update(req->core, req->part_id, req->rw,
 			req->info.hit_no, req->core_line_count,
-			req->is_deferred);
+			req->io.pf_id, req->is_deferred);
 }
 
 void inc_fallback_pt_error_counter(ocf_cache_t cache)

src/engine/engine_common.h

@@ -223,6 +223,13 @@ struct ocf_engine_callbacks
 	ocf_req_async_lock_cb resume;
 };
 
+/**
+ * @brief Prepare cache lines for miss request
+ *
+ * @param req OCF request
+ */
+void ocf_prepare_clines_miss(struct ocf_request *req);
+
 /**
  * @brief Map and lock cachelines
  *

src/engine/engine_io.c

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2024 Huawei Technologies
+ * Copyright(c) 2026 Unvertical
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -131,7 +132,7 @@ void ocf_engine_forward_core_io_req(struct ocf_request *req,
 		ocf_req_end_t callback)
 {
 	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
-			req->bytes);
+			req->bytes, req->io.pf_id);
 
 	ocf_req_forward_core_init(req, callback);
 
@@ -143,7 +144,7 @@ void ocf_engine_forward_core_flush_req(struct ocf_request *req,
 		ocf_req_end_t callback)
 {
 	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
-			req->bytes);
+			req->bytes, req->io.pf_id);
 
 	ocf_req_forward_core_init(req, callback);
 
@@ -154,7 +155,7 @@ void ocf_engine_forward_core_discard_req(struct ocf_request *req,
 		ocf_req_end_t callback)
 {
 	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
-			req->bytes);
+			req->bytes, req->io.pf_id);
 
 	ocf_req_forward_core_init(req, callback);
 

src/engine/engine_prefetch.c

@@ -0,0 +1,164 @@
+/*
+ * Copyright(c) 2012-2022 Intel Corporation
+ * Copyright(c) 2022-2024 Huawei Technologies
+ * Copyright(c) 2026 Unvertical
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "ocf/ocf.h"
+#include "engine_prefetch.h"
+#include "engine_inv.h"
+#include "engine_bf.h"
+#include "engine_common.h"
+#include "engine_io.h"
+#include "../ocf_cache_priv.h"
+#include "../concurrency/ocf_concurrency.h"
+#include "../ocf_request.h"
+#include "../utils/utils_cache_line.h"
+#include "../utils/utils_user_part.h"
+#include "../metadata/metadata.h"
+#include "../ocf_def_priv.h"
+
+#define OCF_ENGINE_DEBUG_IO_NAME "prefetch"
+#include "engine_debug.h"
+
+static void _ocf_prefetch_read_complete(struct ocf_request *req, int error)
+{
+	OCF_DEBUG_RQ(req, "Prefetch read completion");
+
+	ocf_req_get(req);
+
+	if (error) {
+		req->complete(req, error);
+
+		ctx_data_free(req->cache->owner, req->data);
+
+		ocf_core_stats_core_error_update(req->core, OCF_READ);
+
+		/* Invalidate metadata */
+		ocf_engine_invalidate(req);
+
+		return;
+	}
+
+	/* Pretend the data is the copy, so that it's used by the backfill */
+	req->cp_data = req->data;
+	req->data = NULL;
+
+	/* Complete request */
+	req->complete(req, error);
+
+	ocf_engine_backfill(req);
+}
+
+static int _ocf_prefetch_read_do(struct ocf_request *req)
+{
+	struct ocf_alock *c = ocf_cache_line_concurrency(req->cache);
+
+	if (unlikely(ocf_engine_is_hit(req))) {
+		ocf_req_unlock(c, req);
+		req->complete(req, 0);
+		return 0;
+	}
+
+	if (req->info.dirty_any) {
+		ocf_hb_req_prot_lock_rd(req);
+
+		/* Request is dirty need to clean request */
+		ocf_engine_clean(req);
+
+		ocf_hb_req_prot_unlock_rd(req);
+		return 0;
+	}
+
+	req->data = ctx_data_alloc(ocf_cache_get_ctx(req->cache),
+			OCF_DIV_ROUND_UP(req->bytes, PAGE_SIZE));
+	if (!req->data) {
+		ocf_req_unlock(c, req);
+		req->complete(req, 0);
+		return 0;
+	}
+
+	ocf_hb_req_prot_lock_wr(req);
+
+	/* Set valid status bits map */
+	ocf_set_valid_map_info(req);
+
+	if (ocf_engine_needs_repart(req)) {
+		OCF_DEBUG_RQ(req, "Re-Part");
+
+		/* Probably some cache lines are assigned into wrong
+		 * partition. Need to move it to new one
+		 */
+		ocf_user_part_move(req);
+	}
+
+	ocf_hb_req_prot_unlock_wr(req);
+
+	OCF_DEBUG_RQ(req, "Submit");
+
+	ocf_req_get(req);
+	ocf_engine_forward_core_io_req(req, _ocf_prefetch_read_complete);
+
+	/* Update statistics */
+	ocf_engine_update_request_stats(req);
+	ocf_engine_update_block_stats(req);
+
+	/* Put OCF request - decrease reference counter */
+	ocf_req_put(req);
+
+	return 0;
+}
+
+int ocf_prefetch_read(struct ocf_request *req)
+{
+	struct ocf_user_part *user_part = &req->cache->user_parts[req->part_id];
+	struct ocf_alock *c = ocf_cache_line_concurrency(req->cache);
+	ocf_cache_t cache = req->cache;
+	int lock;
+
+	if (env_atomic_read(&cache->pending_read_misses_list_blocked)) {
+		req->complete(req, -OCF_ERR_BUSY);
+		return 0;
+	}
+
+	req->engine_handler = _ocf_prefetch_read_do;
+
+	if (!ocf_user_part_is_enabled(user_part)) {
+		req->complete(req, -OCF_ERR_BUSY);
+		return 0;
+	}
+
+	ocf_req_hash(req);
+
+	ocf_hb_req_prot_lock_wr(req);
+
+	ocf_engine_lookup(req);
+
+	if (unlikely(ocf_engine_is_hit(req))) {
+		ocf_hb_req_prot_unlock_wr(req);
+		req->complete(req, 0);
+		return 0;
+	}
+
+	ocf_prepare_clines_miss(req);
+	if (unlikely(ocf_req_test_mapping_error(req))) {
+		ocf_hb_req_prot_unlock_wr(req);
+		req->complete(req, 0);
+		return 0;
+	}
+
+	lock = ocf_req_async_lock_wr(c, req, ocf_engine_on_resume);
+	if (unlikely(lock < 0)) {
+		ocf_hb_req_prot_unlock_wr(req);
+		req->complete(req, 0);
+		return 0;
+	}
+
+	ocf_hb_req_prot_unlock_wr(req);
+
+	if (lock == OCF_LOCK_ACQUIRED)
+		_ocf_prefetch_read_do(req);
+
+	return 0;
+}