2.0.0 Merge

.github/workflows/codecov.yml

@@ -2,14 +2,7 @@ name: codecov.io
 
 # Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the master branch
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
+  workflow_call:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -21,22 +14,23 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v5
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          python-version: 3.9
+          python-version: '3.x'
 
       - name: Install dependencies
         run: |
           pip install -r requirements.txt
           pip install pytest-cov
-          python setup.py install
+          pip install pytest-mock
+          pip install .
 
       - name: Run tests and collect coverage
         run: pytest --cov=./mwtab --cov-report=xml
 
       # codecov
       - name: "Upload coverage to Codecov"
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v5

.github/workflows/documentation.yml

@@ -0,0 +1,48 @@
+name: Publish Documentation
+
+on:
+  workflow_call:
+
+jobs:
+  publish-documentation:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v5
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.x'
+    - name: Upgrade pip, install package, install requirements, build docs
+      run: |
+        pip install --upgrade pip
+        pip install .
+        if [ -f ./docs/requirements.txt ]; then pip install -r ./docs/requirements.txt; fi
+        pip install sphinx
+        sphinx-build docs ./docs/_build/html/
+    # Create an artifact of the html output.
+    - uses: actions/upload-artifact@v4
+      with:
+        name: DocumentationHTML
+        path: docs/_build/html/
+    # Publish built docs to gh-pages branch.
+    # ===============================
+    - name: Commit documentation changes
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${GITHUB_ACTOR}"
+        git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+        git clone "https://token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" --branch gh-pages --single-branch gh-pages
+        cp -r docs/_build/html/* gh-pages/
+        cd gh-pages
+        touch .nojekyll
+        git add .
+        git commit -m "Update documentation." -a || true
+        # The above command will fail if no changes were present, so we ignore
+        # that.
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        branch: gh-pages
+        directory: gh-pages
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+    # ===============================

.github/workflows/package_release.yml

@@ -0,0 +1,47 @@
+name: Package and Documentation Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release-version:
+    runs-on: ubuntu-latest
+    steps:
+      - id: parse-version
+        name: Parse release version
+        run: |
+          echo "version=${RELEASE_VERSION/v/}" >> "$GITHUB_OUTPUT"
+    env:
+      RELEASE_VERSION: ${{ github.event.release.tag_name }}
+    outputs:
+      version: ${{ steps.parse-version.outputs.version }}
+  publish-test-pypi:
+    uses: ./.github/workflows/pypi.yml
+    with:
+      repository_url: https://test.pypi.org/legacy/
+    secrets:
+      API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
+  test-test-pypi:
+    needs: [release-version, publish-test-pypi]
+    uses: ./.github/workflows/tests.yml
+    with:
+      install_command: "python3 -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple mwtab==${{ needs.release-version.outputs.version }}"
+  publish-pypi:
+    needs: test-test-pypi
+    uses: ./.github/workflows/pypi.yml
+    with:
+      repository_url: https://upload.pypi.org/legacy/
+    secrets:
+      API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+  test-pypi:
+    needs: [release-version, publish-pypi]
+    uses: ./.github/workflows/tests.yml
+    with:
+      install_command: "python3 -m pip install mwtab==${{ needs.release-version.outputs.version }}"
+  publish-documentation:
+    needs: test-pypi
+    uses: ./.github/workflows/documentation.yml
+  upload-coverage:
+    needs: test-pypi
+    uses: ./.github/workflows/codecov.yml

.github/workflows/pull_request.yml

@@ -0,0 +1,16 @@
+name: Pull request
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  pull-request:
+      uses: ./.github/workflows/tests.yml
+      with:
+        install_command: "python3 -m pip install -e ."

.github/workflows/pypi.yml

@@ -0,0 +1,32 @@
+name: Publish package
+
+on:
+  workflow_call:
+    inputs:
+      repository_url:
+        description: The URL of the PyPi distribution
+        required: true
+        type: string
+    secrets:
+      API_TOKEN:
+        required: true
+
+jobs:
+  publish-package:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install build
+      - name: Build package
+        run: python3 -m build
+      - name: Publish package to a PyPi distribution
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.API_TOKEN }}
+          repository-url: ${{ inputs.repository_url }}

.github/workflows/tests.yml

@@ -0,0 +1,40 @@
+name: Run Tests
+
+on:
+  workflow_call:
+    inputs:
+      install_command:
+        description: The command for installing the package to test.
+        required: true
+        type: string
+
+jobs:
+  run-tests:
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        os: [ ubuntu-latest, windows-latest, macOS-latest ]
+    runs-on: ${{matrix.os}}
+    steps:
+    - uses: actions/checkout@v5
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install testing environment
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install pytest pytest-mock pytest-cov deepdiff
+    - name: Install package
+      uses: Wandalen/wretry.action@master
+      with:
+        command: ${{ inputs.install_command }}
+        attempt_limit: 10
+        attempt_delay: 10000
+    - name: Run tests on package
+      run: pytest
+    # - name: Debug with tmate on failure
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+    #   with:
+    #     limit-access-to-actor: true

.gitignore

@@ -4,4 +4,14 @@ venv/*
 data/*
 .DS_Store
 .ipynb_checkpoints
-.idea
+.idea
+/build
+.coverage
+/*.json
+/htmlcov
+/mwtab.egg-info
+/src/mwtab.egg-info
+src/mwtab/_version.py
+/scratch
+docs/standard_column_names.json
+/docs/_contents

CHANGELOG.rst

@@ -1,6 +1,23 @@
 Release History
 ===============
 
+2.0.0
+~~~~~
+-Can now read duplicate keys in "Additional sample data" and reproduce it in write, will validate it as an error.
+-Writing out now ensures correct key ordering for JSON.
+-Validation now validates the main sections not just the internals of them.
+-Validate now checks that metabolites in the Data section are in the Metabolites section and vice versa.
+-Batch processing from the command line is more fault tolerant and won't stop the batch for 1 bad file.
+-Improved tokenizer so more files can be read in without error.
+-Changed schema validation to use jsonschema instead of schema.
+-Added validations for METABOLITES columns that try to give warnings for bad values, for example 'kegg_id' column should all be something like C00000.
+-Expanded the standard column name functionality to look for many more column names than in the previous version and do it in a much more robust way.
+-Changed validation error messages to be aware of whether the input file is JSON or mwTab and print accordingly.
+-Reduced many spurious validation messages.
+-Added a validation for when certain columns are found in METABOLITES, to look for the implied pair and warn if it isn't there. For example, retention_index and retention_index_type.
+-Added validations on some values, such as gender.
+-Many more various minor validations were added.
+
 
 1.2.5.post1 (2022-05-11)
 ~~~~~~~~~~~~~~~~~~~~~~~~

LICENSE

@@ -1,22 +1,22 @@
 The Clear BSD License
 
-Copyright (c) 2020, Christian D. Powell, Andrey Smelter, Hunter N.B. Moseley
+Copyright (c) 2025, P. Travis Thompson, Christian D. Powell, Andrey Smelter, Hunter N.B. Moseley
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted (subject to the limitations in the disclaimer
 below) provided that the following conditions are met:
 
      * Redistributions of source code must retain the above copyright notice,
-     this list of conditions and the following disclaimer.
+       this list of conditions and the following disclaimer.
 
      * Redistributions in binary form must reproduce the above copyright
-     notice, this list of conditions and the following disclaimer in the
-     documentation and/or other materials provided with the distribution.
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
 
      * Neither the name of the copyright holder nor the names of its
-     contributors may be used to endorse or promote products derived from this
-     software without specific prior written permission.
+       contributors may be used to endorse or promote products derived from this
+       software without specific prior written permission.
 
 NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
 THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND