Merge branch 'di_workflow_bjm' into 'master'

DI FTICR workflow improvements and version bump

See merge request mass-spectrometry/enviroms!12

Author: bmeluch

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 5.0.0
+current_version = 5.1.0
 commit = False
 tag = False
 
@@ -21,5 +21,3 @@ values =
 [bumpversion:file:wdl/lc_fticr_ms.wdl]
 
 [bumpversion:file:wdl/di_fticr_ms.wdl]
-
-

.github/workflows/di_workflow_tests.yaml

@@ -0,0 +1,41 @@
+name: Test DI Workflow
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  run-miniwdl:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+
+    - name: Install Docker
+      run: |
+        curl -fsSL https://get.docker.com -o get-docker.sh
+        sh get-docker.sh
+        sudo usermod -aG docker $USER
+
+    - name: Install MiniWDL
+      run: |
+        python -m pip install --upgrade pip
+        pip install miniwdl
+
+    - name: Build Docker Image
+      run: |
+        make docker-build-local
+
+    - name: Run di WDL with local Docker Image based on Dockerfile
+      run: |
+        miniwdl run wdl/di_fticr_ms.wdl -i wdl/di_fticr_wdl_input_local_docker.json --verbose --no-cache --copy-input-files
+
+    - name: Run di WDL with pushed Docker Image
+      run: |
+        make wdl-run-di

.gitignore

@@ -39,4 +39,12 @@ share/python-wheels/
 *.egg
 
 # Raw data files
-*.raw
+*.raw
+*.d
+data/raw_data/example_data/
+
+# Processed data
+output
+
+# Generated molecular formula database
+db/molformulas.sqlite

.gitlab-ci.yml

@@ -0,0 +1,14 @@
+## enviroMS Contributing Guidelines
+
+A work in progress.
+
+#### Steps to create a new version
+
+1. Protect feature branch.
+2. Set up Github mirroring (work with Yuri).
+3. The next push to feature branch will trigger Github Actions to run on that branch. Ensure those checks are passing.
+4. Bump enviroMS version using Makefile command (see semantic versioning guidelines).
+5. Merge feature branch into main on Gitlab.
+6. Use Github Actions to manually push the new Docker image to Dockerhub.
+7. Ensure Github checks now pass on main, using the latest Dockerhub image.
+8. Create a new release tag on Github.

CONTRIBUTING.md

@@ -1,8 +1,8 @@
 app_name = enviroms
 parameters_path = data/enviroms.toml
 # change the path to your data path /Users/eber373/Desenvolvimento/enviroms
-data_dir = /Users/eber373/Development/enviroms/data
-configuration_dir = /Users/eber373/Development/enviroms/configuration
+data_dir = /home/bmeluch/enviroMS_work/example_data
+configuration_dir = /home/bmeluch/enviroMS_work/enviroms/configuration
 version := $(shell cat .bumpversion.cfg | grep current_version | cut -d= -f2 | tr -d ' ')
 stage := $(shell cat .bumpversion.cfg | grep optional_value | cut -d= -f2 | tr -d ' ') 
 
@@ -72,14 +72,26 @@ docker-build:
 
 docker-build-local:
 
-	docker build -t local-enviroms:$(version) .
-
+	docker build --pull --no-cache -t local-enviroms:latest .
 
 docker-run-di:
 
 	@echo $(data_dir)
 	@echo $(configuration_dir)
-	docker run -v $(data_dir):/enviroms/data -v $(configuration_dir):/enviroms/configuration microbiomedata/enviroms:$(version) run-di /enviroms/configuration/enviroms.toml
+	docker run -v $(data_dir):/enviroms/data -v $(configuration_dir):/enviroms/configuration microbiomedata/enviroms:$(version) run_di /enviroms/configuration/di_enviroms.toml
+
+docker-run-di-local:
+
+	@make docker-build-local
+	@echo $(data_dir)
+	@echo $(configuration_dir)
+	docker run \
+	-v $(data_dir):/enviroms/data/raw_data \
+	-v ./data/output:/enviroms/data/output \
+	-v ./data/reference:/enviroms/data/reference \
+	-v $(configuration_dir):/enviroms/configuration \
+	local-enviroms:latest \
+	enviroMS run_di /enviroms/configuration/di_enviroms.toml
 
 docker-run-lc:
 
@@ -91,14 +103,25 @@ cascade-run:
 
 	srun -A mscms -t 240 -N 1 -n time enviroMS run-di -r 2 --mpi  /dtemp/mscms/enviroms/data/configuration/enviroms.toml
 
+cli-run-di :
+	enviroMS run_di configuration/di_enviroms.toml --jobs 1 --replicas 1 --tasks 1
+
 wdl-run-di :
 
 	miniwdl run wdl/di_fticr_ms.wdl -i wdl/di_fticr_wdl_input.json --verbose --no-cache --copy-input-files
 
+wdl-run-di-local :
+	@make docker-build-local
+	miniwdl run wdl/di_fticr_ms.wdl -i wdl/di_fticr_wdl_input_local_docker.json --verbose --no-cache --copy-input-files
+
 wdl-run-lc :
 
 	miniwdl run wdl/lc_fticr_ms.wdl -i wdl/lc_fticr_wdl_input.json --verbose --no-cache --copy-input-files
 
+wdl-run-lc-local:
+
+	miniwdl run wdl/lc_fticr_ms.wdl -i wdl/lc_fticr_wdl_input_local_docker.json --verbose --no-cache --copy-input-files
+
 get-lcms-fticr-test-data:
 
 	@echo "Downloading test files for LC-MS FT-ICR workflow"
@@ -132,6 +155,4 @@ get-lcms-fticr-test-data:
 	else echo "Reference file exists"; fi
 	@echo "LC-MS FT-ICR test files complete"
 
-wdl-run-lc-local:
 
-	miniwdl run wdl/lc_fticr_ms.wdl -i wdl/lc_fticr_wdl_input_local_docker.json --verbose --no-cache --copy-input-files

Makefile

@@ -20,7 +20,7 @@
 
 ## Current Version
 
-### `5.0.0`
+### `5.1.0`
 
 ### Data input formats
 
@@ -199,7 +199,7 @@ A docker image containing the EnviroMS command line as code entry-point
 - Run Workflow from Container:
 
     $(data_dir) = dir_containing the FT-ICR MS data
-    $(configuration_dir) = dir_containing the enviroms.toml, corems.toml and nmdc_metadata.json
+    $(configuration_dir) = dir_containing the enviroms.toml and corems.toml
     
     ```bash
     docker run -v $(data_dir):/enviroms/data \

README.md

@@ -5,7 +5,7 @@ isotopologue_filter_atoms = [ "Cl", "Br",]
 use_runtime_kendrick_filter = false
 use_min_peaks_filter = true
 min_peaks_per_class = 15
-url_database = "sqlite:///db/molformulas.db"
+url_database = ""
 db_jobs = 3
 db_chunk_size = 300
 ion_charge = -1
@@ -29,8 +29,8 @@ isRadical = false
 isProtonated = true
 isAdduct = false
 ionization_type = "ESI"
-min_ppm_error = -10.0
-max_ppm_error = 10.0
+min_ppm_error = -10
+max_ppm_error = 10
 min_abun_error = -100.0
 max_abun_error = 100.0
 mz_error_range = 1.5
@@ -62,8 +62,8 @@ max_picking_mz = 1200.0
 picking_point_extrapolate = 3
 calib_minimize_method = "Powell"
 calib_pol_order = 2
-max_calib_ppm_error = 1.0
-min_calib_ppm_error = -1.0
+max_calib_ppm_error = -1.0
+min_calib_ppm_error = 1.0
 calib_sn_threshold = 2.0
 calibration_ref_match_method = "legacy"
 calibration_ref_match_method_implemented = [ "legacy", "merged",]
@@ -77,14 +77,17 @@ implemented_kendrick_rounding_methods = [ "floor", "ceil", "round",]
 peak_derivative_threshold = 0.0
 peak_min_prominence_percent = 0.1
 min_peak_datapoints = 5.0
-peak_max_prominence_percent = 0.1
+peak_max_prominence_percent = 1
 peak_height_max_percent = 10.0
 legacy_resolving_power = true
 centroid_legacy_polyfit = false
 
 [MolecularFormulaSearch.usedAtoms]
 C = [ 1, 100,]
-H = [ 1, 200,]
+H = [ 4, 200,]
+O = [ 0, 22,]
+N = [ 0, 1,]
+S = [ 0, 1,]
 
 [MolecularFormulaSearch.used_atom_valences]
 C = 4

configuration/di_corems.toml

@@ -1,18 +1,19 @@
 raw_file_start_scan = 1
 raw_file_final_scan = 7
-file_paths = [ "data/raw_data/20190709_WK_CADY_Auto_S16_H1_Post_O5_1_01_36.tsv" ]
-output_directory = "output"
+file_paths = "data/raw_data/*"
+output_directory = "data/output"
 output_group_name = "..."
 output_type = "csv"
 polarity = -1
 is_centroid = true
-corems_toml_path = "configuration/corems.toml"
-nmdc_metadata_path = "configuration/nmdc_metadata.json"
+corems_toml_path = "configuration/di_corems.toml"
 calibrate = true
-calibration_ref_file_path = "data/raw_data/SRFA.ref"
+batch_calibrate = true
+calibration_ref_file_path = "data/reference/Hawkes_neg.ref"
 plot_mz_error = true
 plot_ms_assigned_unassigned = true
 plot_c_dbe = true
 plot_van_krevelen = true
 plot_ms_classes = true
-plot_mz_error_classes = true
+plot_mz_error_classes = true
+plot_qc = true