From 4d22de68c074cea2a9aa19374b05636b2f0d4e4c Mon Sep 17 00:00:00 2001 From: Sara Merzel Date: Tue, 12 Sep 2023 15:54:18 +0300 Subject: [PATCH 1/3] Introduce LOOKUP_HIT_INVALID status A new status for cache lines that are partial hits Signed-off-by: Sara Merzel Signed-off-by: Robert Baldyga Signed-off-by: Michal Mielewczyk --- src/cleaning/acp.c | 2 +- src/engine/cache_engine.h | 1 + src/engine/engine_common.c | 11 +++++++---- src/utils/utils_user_part.c | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 26725626..6d31ab20 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -549,7 +549,7 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, ocf_engine_lookup_map_entry(cache, &info, core_id, core_line); - if (info.status == LOOKUP_HIT && + if ((info.status == LOOKUP_HIT || info.status == LOOKUP_HIT_INVALID) && metadata_test_dirty(cache, info.coll_idx)) { locked = ocf_cache_line_try_lock_rd( ocf_cache_line_concurrency(cache), diff --git a/src/engine/cache_engine.h b/src/engine/cache_engine.h index f9c7c7d2..4afb231d 100644 --- a/src/engine/cache_engine.h +++ b/src/engine/cache_engine.h @@ -14,6 +14,7 @@ struct ocf_thread_priv; #define LOOKUP_HIT 5 #define LOOKUP_MISS 6 #define LOOKUP_REMAPPED 8 +#define LOOKUP_HIT_INVALID 9 static inline ocf_req_cache_mode_t ocf_cache_mode_to_req_cache_mode( ocf_cache_mode_t mode) diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index e25eb4f7..d48759f2 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -143,7 +143,8 @@ static void ocf_engine_update_req_info(struct ocf_cache *cache, ENV_BUG_ON(entry->status != LOOKUP_HIT && entry->status != LOOKUP_MISS && - entry->status != LOOKUP_REMAPPED); + entry->status != LOOKUP_REMAPPED && + entry->status != LOOKUP_HIT_INVALID); /* Handle return value */ if (entry->status == LOOKUP_HIT) { @@ -152,6 +153,7 @@ static void ocf_engine_update_req_info(struct ocf_cache *cache, req->info.hit_no++; } else { req->info.invalid_no++; + entry->status = LOOKUP_HIT_INVALID; } /* Check request is dirty */ @@ -165,7 +167,7 @@ static void ocf_engine_update_req_info(struct ocf_cache *cache, } } - if (entry->status == LOOKUP_HIT || entry->status == LOOKUP_REMAPPED) { + if (entry->status == LOOKUP_HIT || entry->status == LOOKUP_HIT_INVALID || entry->status == LOOKUP_REMAPPED) { if (req->part_id != ocf_metadata_get_partition_id(cache, entry->coll_idx)) { /* @@ -201,7 +203,7 @@ void ocf_engine_set_hot(struct ocf_request *req) entry = &(req->map[i]); status = entry->status; - if (status == LOOKUP_HIT) { + if (status == LOOKUP_HIT || status == LOOKUP_HIT_INVALID) { /* Update eviction (LRU) */ ocf_lru_hot_cline(cache, entry->coll_idx); } @@ -330,6 +332,7 @@ static void ocf_engine_map_hndl_error(struct ocf_cache *cache, switch (entry->status) { case LOOKUP_HIT: + case LOOKUP_HIT_INVALID: case LOOKUP_MISS: break; @@ -537,7 +540,7 @@ static int _ocf_engine_clean_getter(struct ocf_cache *cache, entry = &req->map[item]; - if (entry->status != LOOKUP_HIT) + if (entry->status != LOOKUP_HIT && entry->status != LOOKUP_HIT_INVALID) return -1; if (!metadata_test_dirty(cache, entry->coll_idx)) diff --git a/src/utils/utils_user_part.c b/src/utils/utils_user_part.c index 6cd5a402..4dd5ce38 100644 --- a/src/utils/utils_user_part.c +++ b/src/utils/utils_user_part.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -112,7 +113,7 @@ void ocf_user_part_move(struct ocf_request *req) * cachelines are assigned to target partition during eviction. * So only hit cachelines are interesting. */ - if (entry->status != LOOKUP_HIT) { + if (entry->status != LOOKUP_HIT && entry->status != LOOKUP_HIT_INVALID) { /* No HIT */ continue; } From 8fbc8c4201f223d05f2ff2cd6fc486c518f4d49c Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 2 Sep 2024 15:16:34 +0200 Subject: [PATCH 2/3] engine_bf: don't backfill read hits Signed-off-by: Michal Mielewczyk --- src/engine/engine_bf.c | 77 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index 2d80d78a..6c7affa5 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -70,13 +70,88 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) static int _ocf_backfill_do(struct ocf_request *req) { + ocf_cache_t cache = req->cache; + uint64_t addr, bytes, total_bytes = 0, addr_next = 0; + uint64_t seek, skip; + uint32_t i; + req->data = req->cp_data; if (unlikely(req->data == NULL)) { _ocf_backfill_complete(req, -OCF_ERR_NO_MEM); return 0; } - ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_backfill_complete); + req->cache_forward_end = _ocf_backfill_complete; + + if (ocf_engine_is_sequential(req)) { + addr = cache->device->metadata_offset; + addr += req->map[0].coll_idx * ocf_line_size(cache); + addr += req->addr % ocf_line_size(cache); + + ocf_core_stats_cache_block_update(req->core, req->part_id, + OCF_WRITE, req->bytes); + + ocf_req_forward_cache_io(req, OCF_WRITE, addr, req->bytes, + req->offset); + return 0; + } + + ocf_req_forward_cache_get(req); + for (i = 0; i < req->core_line_count; i++) { + if (addr_next) { + addr = addr_next; + } else { + addr = req->map[i].coll_idx; + addr *= ocf_line_size(cache); + addr += cache->device->metadata_offset; + } + bytes = ocf_line_size(cache); + + if (i == 0) { + seek = req->addr % ocf_line_size(cache); + addr += seek; + bytes -= seek; + } + + if (req->map[i].status == LOOKUP_HIT) { + /* This is the 1st cache line in the interval, + * and it's a hit. Don't write it to the cache. + */ + addr_next = 0; + total_bytes += bytes; + continue; + } + + for (; i < (req->core_line_count - 1); i++) { + addr_next = req->map[i + 1].coll_idx; + addr_next *= ocf_line_size(cache); + addr_next += cache->device->metadata_offset; + + if (addr_next != (addr + bytes)) + break; + + bytes += ocf_line_size(cache); + } + + if (i == (req->core_line_count - 1)) { + skip = (ocf_line_size(cache) - + ((req->addr + req->bytes) % + ocf_line_size(cache))) % ocf_line_size(cache); + bytes -= skip; + } + + bytes = OCF_MIN(bytes, req->bytes - total_bytes); + + ocf_core_stats_cache_block_update(req->core, req->part_id, + OCF_WRITE, bytes); + + ocf_req_forward_cache_io(req, OCF_WRITE, addr, bytes, + req->offset + total_bytes); + + total_bytes += bytes; + } + + ocf_req_forward_cache_put(req); return 0; } From b1c57afede9bd36eaf77ae705bba64e929d59d47 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 3 Sep 2024 14:57:56 +0200 Subject: [PATCH 3/3] fixup: refactor the backfill optimization Simplify calculating addr and size. Getting rid of a few ifs shouldn't impair the performance as well! Signed-off-by: Michal Mielewczyk --- src/engine/engine_bf.c | 75 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index 6c7affa5..7de2f71b 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -68,11 +68,21 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) } } +#define __entries_not_adjacent(__req, __i, __j) \ + __req->map[__i].coll_idx + 1 != __req->map[__j].coll_idx + +#define __is_the_last_chunk(__req, __i) (__i == (__req->core_line_count - 1)) + +#define __skip_on_the_last_entry(__skip, __req, __i) \ + __skip * (int)__is_the_last_chunk(__req, __i) + static int _ocf_backfill_do(struct ocf_request *req) { ocf_cache_t cache = req->cache; - uint64_t addr, bytes, total_bytes = 0, addr_next = 0; - uint64_t seek, skip; + uint64_t metadata_offset = cache->device->metadata_offset; + ocf_cache_line_size_t cache_line_size = ocf_line_size(cache); + uint64_t addr, bytes, total_bytes = 0; + uint64_t seek, skip, last_chunk_size; uint32_t i; req->data = req->cp_data; @@ -84,9 +94,9 @@ static int _ocf_backfill_do(struct ocf_request *req) req->cache_forward_end = _ocf_backfill_complete; if (ocf_engine_is_sequential(req)) { - addr = cache->device->metadata_offset; - addr += req->map[0].coll_idx * ocf_line_size(cache); - addr += req->addr % ocf_line_size(cache); + addr = metadata_offset; + addr += req->map[0].coll_idx * cache_line_size; + addr += req->addr % cache_line_size; ocf_core_stats_cache_block_update(req->core, req->part_id, OCF_WRITE, req->bytes); @@ -96,49 +106,40 @@ static int _ocf_backfill_do(struct ocf_request *req) return 0; } + seek = req->addr % cache_line_size; + last_chunk_size = (req->addr + req->bytes) % cache_line_size; + skip = (cache_line_size - last_chunk_size) % cache_line_size; + ocf_req_forward_cache_get(req); for (i = 0; i < req->core_line_count; i++) { - if (addr_next) { - addr = addr_next; - } else { - addr = req->map[i].coll_idx; - addr *= ocf_line_size(cache); - addr += cache->device->metadata_offset; - } - bytes = ocf_line_size(cache); - - if (i == 0) { - seek = req->addr % ocf_line_size(cache); - addr += seek; - bytes -= seek; - } - if (req->map[i].status == LOOKUP_HIT) { - /* This is the 1st cache line in the interval, - * and it's a hit. Don't write it to the cache. - */ - addr_next = 0; - total_bytes += bytes; + /* This is the 1st cache line in the interval, and it's + a hit. Don't write it to the cache */ + total_bytes += cache_line_size; + total_bytes -= seek; + /* Seek should be taken into account for the first chunk + only */ + seek = 0; continue; } - for (; i < (req->core_line_count - 1); i++) { - addr_next = req->map[i + 1].coll_idx; - addr_next *= ocf_line_size(cache); - addr_next += cache->device->metadata_offset; + addr = metadata_offset; + addr += req->map[i].coll_idx * cache_line_size; - if (addr_next != (addr + bytes)) + bytes = cache_line_size; + for (; i < (req->core_line_count - 1); i++) { + if (__entries_not_adjacent(req, i, i + 1)) break; - bytes += ocf_line_size(cache); + bytes += cache_line_size; } - if (i == (req->core_line_count - 1)) { - skip = (ocf_line_size(cache) - - ((req->addr + req->bytes) % - ocf_line_size(cache))) % ocf_line_size(cache); - bytes -= skip; - } + /* Seek should be taken into account for the first chunk only */ + addr += seek; + bytes -= seek; + seek = 0; + + bytes -= __skip_on_the_last_entry(skip, req, i); bytes = OCF_MIN(bytes, req->bytes - total_bytes);