refactor: usage of doi and provider prioritization only for ORCID

Author: andreishket

server/workers/base/src/base.py

@@ -2,6 +2,7 @@
 import json
 import subprocess
 import pandas as pd
+import logging
 from common.r_wrapper import RWrapper
 from common.deduplication import (
     find_version_in_doi,
@@ -29,6 +30,7 @@
 import sys
 from common.rate_limiter import RateLimiter
 
+logger = logging.getLogger(__name__)
 
 class BaseClient(RWrapper):
     def __init__(self, *args):
@@ -50,14 +52,18 @@ def next_item(self):
         message = json.loads(message.decode("utf-8"))
         request_id = message.get("id")
         params = self.add_default_params(message.get("params"))
+        original_service = params.get("original_service")
         params["service"] = "base"
+        if original_service:
+            params["original_service"] = original_service
         endpoint = message.get("endpoint")
         self.logger.debug(f"Request ID: {request_id}, Params: {params}, Endpoint: {endpoint}")
         return request_id, params, endpoint
 
     def execute_search(self, params):
         q = params.get("q")
         service = params.get("service")
+        original_service = params.get("original_service", service)
         data = {}
         data["params"] = params
         cmd = [self.command, self.runner, self.wd, q, service]
@@ -80,7 +86,7 @@ def execute_search(self, params):
             else:
                 metadata = pd.DataFrame(raw_metadata)
                 metadata = self.sanitize_metadata(metadata)
-                metadata = filter_duplicates(metadata)
+                metadata = filter_duplicates(metadata, original_service)
                 metadata = pd.concat(
                     [metadata, parse_annotations_for_all(metadata, "subject_orig")],
                     axis=1,
@@ -234,7 +240,7 @@ def handle_contentproviders(self, request_id, params):
 pattern_annotations = re.compile(r"([A-Za-z]+:[\w'\- ]+);?")
 
 
-def filter_duplicates(df):
+def filter_duplicates(df, service):
     df.drop_duplicates("id", inplace=True, keep="first")
     df["is_anchor"] = False
     df["doi_duplicate"] = False
@@ -262,10 +268,13 @@ def filter_duplicates(df):
     df = remove_textual_duplicates_from_different_sources(df, dupind)
     df = add_false_negatives(df)
     df = mark_latest_doi(df, dupind)
+
     pure_datasets = df[df.typenorm == "7"]
     non_datasets = df.loc[df.index.difference(pure_datasets.index)]
+
     non_datasets = prioritize_OA_and_latest(non_datasets, dupind)
-    non_datasets = prioritize_doi_and_provider(non_datasets, dupind)
+    if service == "orcid":
+        non_datasets = prioritize_doi_and_provider(non_datasets, dupind)
     pure_datasets = mark_latest_doi(pure_datasets, dupind)
 
     pure_datasets_condition_mask = (pure_datasets.is_anchor == True) | (pure_datasets.is_duplicate == False)

server/workers/orcid/src/orcid_service.py

@@ -177,6 +177,7 @@ def request_base_metadata(self, dois: List[str], params: Dict[str, str]) -> pd.D
                     'today': '2024-10-21', 
                     'unique_id': 'abf2625e2d84eb4367fb443e2cb6f4a1', 
                     'service': 'base', 
+                    'original_service': 'orcid',
                     'embed': 'false', 
                     'vis_id': 'abf2625e2d84eb4367fb443e2cb6f4a1', 
                     'limit': 120,