refactor: usage of is_anchor flag instead of keep and is_latest

Author: andreishket

server/workers/base/src/base.py

@@ -235,11 +235,10 @@ def handle_contentproviders(self, request_id, params):
 
 def filter_duplicates(df):
     df.drop_duplicates("id", inplace=True, keep="first")
-    df["is_latest"] = True
+    df["is_anchor"] = False
     df["doi_duplicate"] = False
     df["has_relations"] = False
     df["link_duplicate"] = False
-    df["keep"] = False
     df["duplicates"] = df.apply(
         lambda x: ",".join([x["id"], x["duplicates"]])
         if len(x["duplicates"].split(",")) >= 1
@@ -266,17 +265,16 @@ def filter_duplicates(df):
     non_datasets = df.loc[df.index.difference(pure_datasets.index)]
     non_datasets = prioritize_OA_and_latest(non_datasets, dupind)
     pure_datasets = mark_latest_doi(pure_datasets, dupind)
-    filtered_non_datasets = non_datasets[non_datasets.is_latest == True]
+    filtered_non_datasets = non_datasets[non_datasets.is_anchor == True]
     filtered_datasets = pure_datasets[
-        (pure_datasets.keep == True) | (pure_datasets.is_duplicate == False)
+        (pure_datasets.is_anchor == True) | (pure_datasets.is_duplicate == False)
     ]
     filtered = pd.concat([filtered_non_datasets, filtered_datasets])
     filtered.sort_index(inplace=True)
     for c in [
         "doi_duplicate",
         "link_duplicate",
-        "is_latest",
-        "keep",
+        "is_anchor",
         "duplicates",
         "doi_version",
         "unversioned_doi",

server/workers/base/tests/unit/test_base.py

@@ -193,8 +193,7 @@ def test_filter_duplicates():
     # Add extra columns that filter_duplicates is supposed to drop.
     df["doi_duplicate"] = False
     df["link_duplicate"] = False
-    df["is_latest"] = True
-    df["keep"] = False
+    df["is_anchor"] = False
     df["doi_version"] = ["v1", "v1", "v2"]
     df["unversioned_doi"] = ["doi1", "doi1", "doi2"]
     df["publisher_doi"] = ["pub1", "pub1", "pub2"]
@@ -203,7 +202,7 @@ def test_filter_duplicates():
     filtered = filter_duplicates(df.copy())
     # Verify that the dropped columns are not present.
     for col in [
-        "doi_duplicate", "link_duplicate", "is_latest", "keep",
+        "doi_duplicate", "link_duplicate", "is_anchor",
         "doi_version", "unversioned_doi", "publisher_doi", "has_relations"
     ]:
         assert col not in filtered.columns

server/workers/common/common/deduplication.py

@@ -72,16 +72,13 @@ def remove_textual_duplicates_from_different_sources(df, dupind):
         if len(idx) > 1:
             tmp = df.loc[idx]
             df.loc[tmp.index, "is_duplicate"] = True
-            df.loc[tmp.index, "is_latest"] = False
+            df.loc[tmp.index, "is_anchor"] = False
             publisher_dois = list(filter(None, tmp.publisher_doi.unique().tolist()))
             if len(publisher_dois) > 0:
                 # keep entry with doi
-                df.loc[idx, "keep"] = False
-                df.loc[tmp[tmp.publisher_doi!=""].index, "is_latest"] = True
-                df.loc[tmp[tmp.publisher_doi!=""].index, "keep"] = True
+                df.loc[tmp[tmp.publisher_doi!=""].index, "is_anchor"] = True
             else:
-                df.loc[tmp.sort_values(["doi", "year"], ascending=[False, False]).head(1).index, "is_latest"] = True
-                df.loc[tmp.sort_values(["doi", "year"], ascending=[False, False]).head(1).index, "keep"] = True
+                df.loc[tmp.sort_values(["doi", "year"], ascending=[False, False]).head(1).index, "is_anchor"] = True
     return df
 
 def mark_latest_doi(df, dupind):
@@ -91,29 +88,24 @@ def mark_latest_doi(df, dupind):
         for udoi in list(filter(None, tmp.unversioned_doi.unique().tolist())):
             tmp2 = tmp[tmp.unversioned_doi == udoi]
             if len(tmp2) > 0:
-                df.loc[tmp2.index, "is_latest"] = False
-                df.loc[tmp2.index, "keep"] = False
+                df.loc[tmp2.index, "is_anchor"] = False
                 versions = tmp2.id
                 latest = tmp2.sort_values("doi_version", ascending=False).head(1).id
                 v = [{"versions": versions.values.tolist(), "latest": latest.values.tolist()}]*len(tmp2)
                 df.loc[versions.index, "versions"] = v
-                df.loc[latest.index, "is_latest"] = True
-                df.loc[latest.index, "keep"] = True
+                df.loc[latest.index, "is_anchor"] = True
     return df
 
 def prioritize_OA_and_latest(df, dupind):
     for _, idx in dupind.items():
         idx = df.index.intersection(idx)
         if len(idx) > 1:
             tmp = df.loc[idx]
-            df.loc[idx, "keep"] = False
-            df.loc[idx, "is_latest"] = False
+            df.loc[idx, "is_anchor"] = False
             if len(tmp[tmp.oa_state=="1"]) > 0:
-                df.loc[tmp[tmp.oa_state=="1"].sort_values("year", ascending=False).head(1).index, "keep"] = True
-                df.loc[tmp[tmp.oa_state=="1"].sort_values("year", ascending=False).head(1).index, "is_latest"] = True
+                df.loc[tmp[tmp.oa_state=="1"].sort_values("year", ascending=False).head(1).index, "is_anchor"] = True
             else:
-                df.loc[tmp.sort_values("year", ascending=False).head(1).index, "keep"] = True
-                df.loc[tmp.sort_values("year", ascending=False).head(1).index, "is_latest"] = True
+                df.loc[tmp.sort_values("year", ascending=False).head(1).index, "is_anchor"] = True
     return df
 
 def mark_duplicates(metadata):