Filter NA/empty values from metadata when reading tabular and MSP files

Co-authored-by: hechth <12066490+hechth@users.noreply.github.com>

Author: Copilot

MSMetaEnhancer/libs/data/DataFrame.py

@@ -2,6 +2,7 @@
 
 from MSMetaEnhancer.libs.data.Data import Data
 from MSMetaEnhancer.libs.utils.Errors import UnknownFileFormat
+from MSMetaEnhancer.libs.utils.Generic import is_na_value
 
 
 class DataFrame(Data):
@@ -45,7 +46,8 @@ def save_data(self, filename: str, file_format: str):
             raise UnknownFileFormat(f'Format {file_format} not supported.')
 
     def get_metadata(self):
-        return self.df.to_dict('records')
+        records = self.df.to_dict('records')
+        return [{k: v for k, v in record.items() if not is_na_value(v)} for record in records]
 
     def fuse_metadata(self, metadata_list):
         self.df = pandas.DataFrame.from_dict(metadata_list)

MSMetaEnhancer/libs/data/Spectra.py

@@ -5,6 +5,7 @@
 
 from MSMetaEnhancer.libs.data.Data import Data
 from MSMetaEnhancer.libs.utils.Errors import UnknownFileFormat
+from MSMetaEnhancer.libs.utils.Generic import is_na_value
 
 
 class Spectra(Data):
@@ -48,7 +49,8 @@ def save_data(self, filename: str, file_format: str):
             raise UnknownFileFormat(f'Format {file_format} not supported.')
 
     def get_metadata(self):
-        return [spectra.metadata for spectra in self.spectrums]
+        return [{k: v for k, v in spectra.metadata.items() if not is_na_value(v)}
+                for spectra in self.spectrums]
 
     def fuse_metadata(self, metadata):
         for i in range(len(metadata)):

MSMetaEnhancer/libs/utils/Generic.py

@@ -1,3 +1,20 @@
+import math
+
+
+NA_STRING_VALUES = {'na', 'n/a', 'nan', 'none', ''}
+
+
+def is_na_value(value) -> bool:
+    """Check if a value should be treated as NA/missing (e.g. empty, None, NaN, 'NA')."""
+    if value is None:
+        return True
+    if isinstance(value, float) and math.isnan(value):
+        return True
+    if isinstance(value, str) and value.strip().lower() in NA_STRING_VALUES:
+        return True
+    return False
+
+
 def escape_single_quotes(f):
     async def wrapper(self, arg):
         return await f(self, arg.replace("'", "\\'"))

tests/test_data/sample_metadata_with_na.csv

@@ -0,0 +1,4 @@
+formula,mw,casno,inchikey,smiles
+H2,2,1333740,NA,
+D2,4,7782390,nan,None
+CH4,16,74828,N/A,n/a

tests/test_data/sample_with_na.msp

@@ -0,0 +1,30 @@
+NAME: Hydrogen
+FORMULA: H2
+MW: 2
+INCHIKEY: NA
+SMILES: n/a
+NUM PEAKS: 2
+1.0	20.98
+2.0	999.0
+
+NAME: Deuterium
+FORMULA: D2
+MW: 4
+INCHIKEY: nan
+SMILES: None
+NUM PEAKS: 2
+2.0	14.99
+4.0	999.0
+
+NAME: Methane
+FORMULA: CH4
+MW: 16
+INCHIKEY: N/A
+SMILES: 
+NUM PEAKS: 6
+12.0	37.97
+13.0	105.9
+14.0	203.82
+15.0	886.2
+16.0	999.0
+17.0	15.99

tests/test_io.py

@@ -83,3 +83,22 @@ def test_tabular_data():
                 assert meta_item[key] == data_item[key], (
                     f"Value mismatch for key '{key}' at index {i}: {meta_item[key]} != {data_item[key]}"
                 )
+
+
+@pytest.mark.parametrize('backend, file_type, filename, absent_keys', [
+    [DataFrame(), 'csv', 'tests/test_data/sample_metadata_with_na.csv', ['inchikey', 'smiles']],
+    [Spectra(), 'msp', 'tests/test_data/sample_with_na.msp', ['inchikey', 'smiles']],
+])
+def test_na_values_filtered_from_metadata(backend, file_type, filename, absent_keys):
+    """NA and empty values in data files should be excluded from metadata dicts."""
+    backend.load_data(filename, file_type)
+    metadata = backend.get_metadata()
+
+    assert len(metadata) == 3
+
+    for i, meta_item in enumerate(metadata):
+        # Keys that had NA values must be absent
+        for key in absent_keys:
+            assert key not in meta_item, (
+                f"NA key '{key}' should not be present at index {i}, got {meta_item.get(key)}"
+            )