Fix race condition when adding cores in examine

Change-Id: I3607fd8a261755440fdd2948a027a37aaf66d67e
Signed-off-by: Rafal Stefanowski <rafal.stefanowski@huawei.com>

Author: Rafal Stefanowski

module/bdev/ocf/vbdev_ocf.c

@@ -585,7 +585,7 @@ _core_add_examine_add_cb(ocf_cache_t cache, ocf_core_t core, void *cb_arg, int e
 	SPDK_NOTICELOG("OCF core '%s': added to cache '%s'\n",
 		       ocf_core_get_name(core), ocf_cache_get_name(cache));
 
-	/* If core was taken from waitlist, remove it from there. */
+	/* If core was taken from wait list, remove it from there. */
 	if (vbdev_ocf_core_waitlist_get_by_name(vbdev_ocf_core_get_name(core_ctx))) {
 		vbdev_ocf_core_waitlist_remove(core_ctx);
 	}
@@ -612,6 +612,13 @@ _core_add_examine_lock_cb(ocf_cache_t cache, void *cb_arg, int error)
 		return;
 	}
 
+	/* Check if core is loaded from metadata only if its try_add flag was not
+	 * set to true already during examine_config stage. That would mean that
+	 * this core was hot removed before and now it is being attached back. */
+	if (!core_ctx->core_cfg.try_add) {
+		core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx));
+	}
+
 	ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_examine_add_cb, core_ctx);
 }
 
@@ -717,7 +724,7 @@ _examine_disk_core_visitor(ocf_core_t core, void *cb_arg)
 		return 0;
 	}
 
-	/* Get cache once to be in sync with adding core from waitlist scenario. */
+	/* Get cache once to be in sync with adding core from wait list scenario. */
 	ocf_mngt_cache_get(cache);
 	ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx);
 
@@ -754,6 +761,9 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev)
 			continue;
 		}
 
+		SPDK_NOTICELOG("OCF core '%s': adding from wait list to cache '%s'\n",
+			       vbdev_ocf_core_get_name(core_ctx), core_ctx->cache_name);
+
 		if (ocf_mngt_cache_get_by_name(vbdev_ocf_ctx, core_ctx->cache_name,
 					       OCF_CACHE_NAME_SIZE, &cache)) {
 			SPDK_NOTICELOG("OCF core '%s': add deferred - waiting for OCF cache '%s'\n",
@@ -771,8 +781,6 @@ vbdev_ocf_module_examine_disk(struct spdk_bdev *bdev)
 			return;
 		}
 
-		core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx));
-
 		ocf_mngt_cache_lock(cache, _core_add_examine_lock_cb, core_ctx);
 		return;
 	}

module/bdev/ocf/vbdev_ocf_core.c

@@ -67,7 +67,7 @@ _core_is_loaded_cache_visitor(ocf_cache_t cache, void *cb_arg)
 	if (!rc && !ocf_core_get_priv(core)) {
 		/* Core context is assigned only after manual core add (either right
 		 * away if all devices are present, after corresponding cache start,
-		 * or base bdev appearance).
+		 * or base bdev showing up).
 		 * If there is no context, it means that this core was just added from
 		 * metadata during cache load and that's what we're looking for here. */
 
@@ -518,24 +518,35 @@ _core_add_from_waitlist_lock_cb(ocf_cache_t cache, void *cb_arg, int error)
 		return;
 	}
 
+	core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx));
+
 	ocf_mngt_cache_add_core(cache, &core_ctx->core_cfg, _core_add_from_waitlist_add_cb, core_ctx);
 }
 
-void
-vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache)
+static void
+_core_add_from_waitlist_read_lock_cb(ocf_cache_t cache, void *cb_arg, int error)
 {
 	struct vbdev_ocf_cache *cache_ctx = ocf_cache_get_priv(cache);
 	struct vbdev_ocf_core *core_ctx;
 	uint32_t cache_block_size = spdk_bdev_get_block_size(cache_ctx->base.bdev);
 	uint32_t core_block_size;
 
+	SPDK_DEBUGLOG(vbdev_ocf, "OCF cache '%s': looking for its cores in wait list\n",
+		      ocf_cache_get_name(cache));
+
+	if (error) {
+		SPDK_ERRLOG("OCF cache '%s': failed to acquire OCF cache lock (OCF error: %d)\n",
+			    ocf_cache_get_name(cache), error);
+		return;
+	}
+
 	vbdev_ocf_foreach_core_in_waitlist(core_ctx) {
 		if (strcmp(ocf_cache_get_name(cache), core_ctx->cache_name) ||
 		    !vbdev_ocf_core_is_base_attached(core_ctx)) {
 			continue;
 		}
 
-		SPDK_NOTICELOG("OCF core '%s': adding from waitlist to cache '%s'\n",
+		SPDK_NOTICELOG("OCF core '%s': adding from wait list to cache '%s'\n",
 			       vbdev_ocf_core_get_name(core_ctx), ocf_cache_get_name(cache));
 
 		core_block_size = spdk_bdev_get_block_size(core_ctx->base.bdev);
@@ -546,10 +557,16 @@ vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache)
 			continue;
 		}
 
-		core_ctx->core_cfg.try_add = vbdev_ocf_core_is_loaded(vbdev_ocf_core_get_name(core_ctx));
-
 		ocf_mngt_cache_lock(cache, _core_add_from_waitlist_lock_cb, core_ctx);
 	}
+
+	ocf_mngt_cache_read_unlock(cache);
+}
+
+void
+vbdev_ocf_core_add_from_waitlist(ocf_cache_t cache)
+{
+	ocf_mngt_cache_read_lock(cache, _core_add_from_waitlist_read_lock_cb, NULL);
 }
 
 static void