diff --git a/.cookiecutter.json b/.cookiecutter.json index 2e6fd32e..4981aea8 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -14,6 +14,17 @@ "project_with_config_settings": "no", "generate_docs": "yes", "version": "2.10.0", - "original_publish_year": "2021" + "original_publish_year": "2021", + "_drift_manager": { + "template": "https://github.com/networktocode-llc/cookiecutter-ntc.git", + "template_dir": "python", + "template_ref": "main", + "cookie_dir": "", + "pull_request_strategy": "create", + "post_actions": [], + "draft": false, + "baked_commit_ref": "6e239e9b2ba0888f80cf1693318f007a2085d79a", + "drift_managed_branch": "develop" + } } } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 585befea..b2100354 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,28 +1,24 @@ --- name: 🐛 Bug Report -about: Report a reproducible bug in the current release of circuit_maintenance_parser +about: Report a reproducible bug in the current release of circuit-maintenance-parser --- ### Environment - -- Python version: -- circuit_maintenance_parser version: +* Python version: +* circuit-maintenance-parser version: - ### Expected Behavior - + ### Observed Behavior - ### Steps to Reproduce - 1. 2. 3. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 82e36d60..f83b4c17 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,16 +1,15 @@ --- name: ✨ Feature Request about: Propose a new feature or enhancement + --- ### Environment - -- circuit_maintenance_parser version: +* circuit-maintenance-parser version: - ### Proposed Functionality - ### Use Case diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20edb902..4ad6c2c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,15 +1,13 @@ --- name: "CI" -concurrency: # Cancel any existing runs of this workflow for this same PR - group: "${{ '{{ github.workflow }}' }}-${{ '{{ github.ref }}' }}" +concurrency: # Cancel any existing runs of this workflow for this same PR + group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: true -on: # yamllint disable-line rule:truthy rule:comments +on: # yamllint disable-line rule:truthy rule:comments push: branches: - "main" - "develop" - tags: - - "v*" pull_request: ~ env: @@ -18,7 +16,7 @@ env: jobs: ruff-format: - runs-on: "ubuntu-24.04" + runs-on: "ubuntu-latest" env: INVOKE_PARSER_LOCAL: "True" steps: @@ -27,11 +25,11 @@ jobs: - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" with: - poetry-version: "1.8.5" + poetry-version: "2.1.3" - name: "Linting: ruff format" run: "poetry run invoke ruff --action format" ruff-lint: - runs-on: "ubuntu-24.04" + runs-on: "ubuntu-latest" env: INVOKE_PARSER_LOCAL: "True" steps: @@ -40,25 +38,25 @@ jobs: - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" with: - poetry-version: "1.8.5" + poetry-version: "2.1.3" - name: "Linting: ruff" run: "poetry run invoke ruff --action lint" - # Temporarily disabled due to issues with the docs build and needing best practices for NTC python builds. - # check-docs-build: - # runs-on: "ubuntu-24.04" - # env: - # INVOKE_PARSER_LOCAL: "True" - # steps: - # - name: "Check out repository code" - # uses: "actions/checkout@v4" - # - name: "Setup environment" - # uses: "networktocode/gh-action-setup-poetry-environment@v6" - # with: - # poetry-version: "1.8.5" - # - name: "Check Docs Build" - # run: "poetry run invoke build-and-check-docs" + check-docs-build: + runs-on: "ubuntu-latest" + env: + INVOKE_PARSER_LOCAL: "True" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v6" + with: + poetry-version: "2.1.3" + poetry-install-options: "--only dev,docs" + - name: "Check Docs Build" + run: "poetry run invoke build-and-check-docs" poetry: - runs-on: "ubuntu-24.04" + runs-on: "ubuntu-latest" env: INVOKE_PARSER_LOCAL: "True" steps: @@ -67,15 +65,11 @@ jobs: - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" with: - poetry-version: "1.8.5" + poetry-version: "2.1.3" - name: "Checking: poetry lock file" - run: "poetry lock --check" - needs: - - "ruff-format" - - "ruff-lint" - - "yamllint" + run: "poetry run invoke lock --check" yamllint: - runs-on: "ubuntu-24.04" + runs-on: "ubuntu-latest" env: INVOKE_PARSER_LOCAL: "True" steps: @@ -84,32 +78,36 @@ jobs: - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" with: - poetry-version: "1.8.5" + poetry-version: "2.1.3" - name: "Linting: yamllint" run: "poetry run invoke yamllint" + check-in-docker: needs: - "ruff-format" - "ruff-lint" - pylint: - runs-on: "ubuntu-24.04" + - "poetry" + - "yamllint" + runs-on: "ubuntu-latest" strategy: fail-fast: true matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] env: - PYTHON_VER: "${{ matrix.python-version }}" + INVOKE_PARSER_PYTHON_VER: "${{ matrix.python-version }}" steps: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" + with: + poetry-version: "2.1.3" - name: "Get image version" - run: "echo INVOKE_PARSER_IMAGE_VER=`poetry version -s`-py$${{ matrix.python-version }} >> $GITHUB_ENV" + run: "echo INVOKE_PARSER_IMAGE_VER=`poetry version -s`-py${{ matrix.python-version }} >> $GITHUB_ENV" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" # v3.10.0 + uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" # v3.10.0 - name: "Build" - uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25" # v5.4.0 + uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25" # v5.4.0 with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -120,33 +118,33 @@ jobs: cache-from: "type=gha,scope=${{ env.INVOKE_PARSER_IMAGE_NAME }}-${{ env.INVOKE_PARSER_IMAGE_VER }}-py${{ matrix.python-version }}" cache-to: "type=gha,scope=${{ env.INVOKE_PARSER_IMAGE_NAME }}-${{ env.INVOKE_PARSER_IMAGE_VER }}-py${{ matrix.python-version }}" build-args: | - PYTHON_VER=${{ env.PYTHON_VER }} - - name: "Debug: Show docker images" - run: "docker image ls" + PYTHON_VER=${{ matrix.python-version }} - name: "Linting: Pylint" run: "poetry run invoke pylint" - needs: - - "poetry" pytest: + needs: + - "check-in-docker" strategy: fail-fast: true matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] - runs-on: "ubuntu-24.04" + runs-on: "ubuntu-latest" env: - PYTHON_VER: "${{ matrix.python-version }}" + INVOKE_PARSER_PYTHON_VER: "${{ matrix.python-version }}" steps: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" + with: + poetry-version: "2.1.3" - name: "Get image version" run: "echo INVOKE_PARSER_IMAGE_VER=`poetry version -s`-py${{ matrix.python-version }} >> $GITHUB_ENV" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" # v3.10.0 + uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" # v3.10.0 - name: "Build" - uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25" # v5.4.0 + uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25" # v5.4.0 with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -157,46 +155,14 @@ jobs: cache-from: "type=gha,scope=${{ env.INVOKE_PARSER_IMAGE_NAME }}-${{ env.INVOKE_PARSER_IMAGE_VER }}-py${{ matrix.python-version }}" cache-to: "type=gha,scope=${{ env.INVOKE_PARSER_IMAGE_NAME }}-${{ env.INVOKE_PARSER_IMAGE_VER }}-py${{ matrix.python-version }}" build-args: | - PYTHON_VER=${{ env.PYTHON_VER }} - - name: "Debug: Show docker images" - run: "docker image ls" + PYTHON_VER=${{ matrix.python-version }} - name: "Run Tests" run: "poetry run invoke pytest" - needs: - - "poetry" - publish_gh: - name: "Publish to GitHub" - runs-on: "ubuntu-24.04" - # yamllint disable-line rule:quoted-strings - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: "Check out repository code" - uses: "actions/checkout@v4" - - name: "Set up Python" - uses: "actions/setup-python@v5" - with: - python-version: "3.10" - - name: "Install Python Packages" - run: "pip install poetry" - - name: "Set env" - run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV" - - name: "Run Poetry Version" - run: "poetry version $RELEASE_VERSION" - - name: "Build Documentation" - run: "poetry run invoke build-and-check-docs" - - name: "Run Poetry Build" - run: "poetry build" - - name: "Upload binaries to release" - run: "gh release upload ${{ github.ref_name }} dist/*.{tar.gz,whl}" - env: - GH_TOKEN: "${{ secrets.NTC_GITHUB_TOKEN }}" - needs: - - "pytest" changelog: if: > contains(fromJson('["develop"]'), github.base_ref) && (github.head_ref != 'main') && (!startsWith(github.head_ref, 'release')) - runs-on: "ubuntu-22.04" + runs-on: "ubuntu-latest" steps: - name: "Check out repository code" uses: "actions/checkout@v4" @@ -205,70 +171,8 @@ jobs: - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" with: - poetry-version: "1.8.5" + poetry-version: "2.1.3" - name: "Check for changelog entry" run: | git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} poetry run towncrier check --compare-with origin/${{ github.base_ref }} - publish_pypi: - name: "Push Package to PyPI" - runs-on: "ubuntu-24.04" - # yamllint disable-line rule:quoted-strings - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: "Check out repository code" - uses: "actions/checkout@v4" - - name: "Set up Python" - uses: "actions/setup-python@v5" - with: - python-version: "3.10" - - name: "Install Python Packages" - run: "pip install poetry" - - name: "Set env" - run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV" - - name: "Run Poetry Version" - run: "poetry version $RELEASE_VERSION" - - name: "Run Poetry Build" - run: "poetry build" - - name: "Push to PyPI" - uses: "pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc" # v1.12.4 - with: - user: "__token__" - password: "${{ secrets.PYPI_API_TOKEN }}" - needs: - - "pytest" - slack-notify: - needs: - - "publish_gh" - - "publish_pypi" - runs-on: "ubuntu-24.04" - env: - SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" - SLACK_MESSAGE: >- - *NOTIFICATION: NEW-RELEASE-PUBLISHED*\n - Repository: <${{ github.server_url }}/${{ github.repository }}|${{ github.repository }}>\n - Release: <${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>\n - Published by: <${{ github.server_url }}/${{ github.actor }}|${{ github.actor }}> - steps: - - name: "Send a notification to Slack" - # ENVs cannot be used directly in job.if. This is a workaround to check - # if SLACK_WEBHOOK_URL is present. - if: "env.SLACK_WEBHOOK_URL != ''" - uses: "slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3" # v1.27.1 - with: - payload: | - { - "text": "${{ env.SLACK_MESSAGE }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "${{ env.SLACK_MESSAGE }}" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" - SLACK_WEBHOOK_TYPE: "INCOMING_WEBHOOK" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..e31b6a74 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,106 @@ +--- +name: "Release" +on: # yamllint disable-line rule:truthy rule:comments + release: + types: ["published"] + +jobs: + build: + name: "Build package with poetry" + runs-on: "ubuntu-latest" + if: "startsWith(github.ref, 'refs/tags/v')" + steps: + - uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v6" + with: + poetry-version: "2.1.3" + python-version: "3.13" + poetry-install-options: "--no-root" + - name: "Run Poetry Build" + run: "poetry build" + + - name: "Check that the release tag matches the version in pyproject.toml" + run: | + if [ "${{ github.ref_name }}" != "v$(poetry version -s)" ]; then exit 1; fi + + - uses: "actions/upload-artifact@v4" + with: + name: "distfiles" + path: "dist/" + if-no-files-found: "error" + + publish-github: + name: "Publish to GitHub" + runs-on: "ubuntu-latest" + if: "startsWith(github.ref, 'refs/tags/v')" + permissions: + contents: "write" + needs: "build" + steps: + - uses: "actions/checkout@v4" + - name: "Retrieve built package from cache" + uses: "actions/download-artifact@v4" + with: + name: "distfiles" + path: "dist/" + + - name: "Upload binaries to release" + run: "gh release upload ${{ github.ref_name }} dist/*.{tar.gz,whl}" + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + publish-pypi: + name: "Push Package to PyPI" + runs-on: "ubuntu-latest" + if: "startsWith(github.ref, 'refs/tags/v')" + needs: "build" + environment: "pypi" + # Steps to publish to PyPI. + steps: + - name: "Retrieve built package from cache" + uses: "actions/download-artifact@v4" + with: + name: "distfiles" + path: "dist/" + - name: "Publish package distributions to PyPI" + uses: "pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e" # v1.13.0 + ## Used for networktocode org since trusted publisher isn't supported for GitHub Plan. + with: + user: "__token__" + password: "${{ secrets.PYPI_API_TOKEN }}" + # End publish to PyPI job. + + slack-notify: + needs: + - "publish-github" + - "publish-pypi" + runs-on: "ubuntu-latest" + env: + # Secrets cannot be directly referenced in if: conditionals. They must be set as a job env var first. + # Ref: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#example-using-secrets + SLACK_WEBHOOK_URL: "${{ secrets.OSS_PYPI_SLACK_WEBHOOK_URL }}" + SLACK_WEBHOOK_TYPE: "INCOMING_WEBHOOK" + SLACK_MESSAGE: >- + *NOTIFICATION: NEW-RELEASE-PUBLISHED*\n + Repository: <${{ github.server_url }}/${{ github.repository }}|${{ github.repository }}>\n + Release: <${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>\n + Published by: <${{ github.server_url }}/${{ github.actor }}|${{ github.actor }}> + steps: + - name: "Send a notification to Slack" + if: "${{ env.SLACK_WEBHOOK_URL != '' }}" + uses: "slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3" # v1.27.1 + with: + payload: | + { + "text": "${{ env.SLACK_MESSAGE }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ env.SLACK_MESSAGE }}" + } + } + ] + } diff --git a/.gitignore b/.gitignore index f539d853..bb324eae 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,7 @@ venv.bak/ # mkdocs documentation /site +/circuit-maintenance-parser/static/ # mypy .mypy_cache/ @@ -294,3 +295,17 @@ fabric.properties ### vscode ### .vscode/* *.code-workspace + +# Rando +creds.env +development/*.txt + +# Invoke overrides +invoke.yml + +# Docs +docs/README.md +docs/CHANGELOG.md +public +/compose.yaml +/dump.sql diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..36f32f00 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,27 @@ +--- +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Setup the build environment. +build: + os: "ubuntu-lts-latest" + tools: + python: "3.13" + jobs: + post_install: + # Install poetry + # https://python-poetry.org/docs/#installing-manually + - "pip install poetry" + # Install dependencies 'docs' dependency group + # https://python-poetry.org/docs/managing-dependencies/#dependency-groups + # VIRTUAL_ENV needs to be set manually for now. + # See https://github.com/readthedocs/readthedocs.org/pull/11152/ + - "VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --only docs" + +mkdocs: + configuration: "mkdocs.yml" + fail_on_warning: true diff --git a/Dockerfile b/Dockerfile index 06cdf467..3c1fdca0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VER="3.9" +ARG PYTHON_VER="3.10" FROM python:${PYTHON_VER}-slim @@ -8,7 +8,7 @@ FROM python:${PYTHON_VER}-slim # This also makes it so that Poetry will *not* be included in the "final" image since it's not installed to /usr/local/ ARG POETRY_HOME=/opt/poetry ARG POETRY_INSTALLER_PARALLEL=true -ARG POETRY_VERSION=1.8.2 +ARG POETRY_VERSION=2.1.3 ARG POETRY_VIRTUALENVS_CREATE=false ADD https://install.python-poetry.org /tmp/install-poetry.py RUN python /tmp/install-poetry.py diff --git a/LICENSE b/LICENSE index f433b1a5..9e40eee1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,177 +1,15 @@ +Apache Software License 2.0 - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Copyright (c) 2021-2026, Network to Code, LLC - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - 1. Definitions. +http://www.apache.org/licenses/LICENSE-2.0 - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 71ef76fd..ee5dcae2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,24 @@ # circuit-maintenance-parser +

+ + + + +

+ `circuit-maintenance-parser` is a Python library that parses circuit maintenance notifications from Network Service Providers (NSPs), converting heterogeneous formats to a well-defined structured format. +## Documentation + +Full documentation for this library can be found over on the [Circuit-Maintenance-Parser Docs](https://circuit-maintenance-parser.readthedocs.io/) website: + +- [User Guide](https://circuit-maintenance-parser.readthedocs.io/en/latest/user/lib_overview/) - Overview, Using the Library, Getting Started. +- [Administrator Guide](https://circuit-maintenance-parser.readthedocs.io/en/latest/admin/install/) - How to Install, Configure, Upgrade, or Uninstall the Library. +- [Developer Guide](https://circuit-maintenance-parser.readthedocs.io/en/latest/dev/contributing/) - Extending the Library, Code Reference, Contribution Guide. +- [Release Notes / Changelog](https://circuit-maintenance-parser.readthedocs.io/en/latest/admin/release_notes/). +- [Frequently Asked Questions](https://circuit-maintenance-parser.readthedocs.io/en/latest/user/faq/). + ## Context Every network depends on external circuits provided by NSPs who interconnect them to the Internet, to office branches or to @@ -280,158 +297,21 @@ Circuit Maintenance Notification #0 } ``` -## How to Extend the Library? - -Even though the library aims to include support for as many providers as possible, it's likely that not all the thousands of NSP are supported and you may need to add support for some new one. Adding a new `Provider` is quite straightforward, and in the following example we are adding support for an imaginary provider, ABCDE, that uses HTML notifications. - -The first step is creating a new file: `circuit_maintenance_parser/parsers/abcde.py`. This file will contain all the custom parsers needed for the provider and it will import the base classes for each parser type from `circuit_maintenance_parser.parser`. In the example, we only need to import `Html` and in the child class implement the methods required by the class, in this case `parse_html()` which will return a `dict` with all the data that this `Parser` can extract. In this case, we have to helper methods, `_parse_bs` and `_parse_tables` that implement the logic to navigate the notification data. - -```python -from typing import Dict -import bs4 # type: ignore -from bs4.element import ResultSet # type: ignore -from circuit_maintenance_parser.parser import Html - -class HtmlParserABCDE1(Html): - def parse_html(self, soup: ResultSet) -> Dict: - data = {} - self._parse_bs(soup.find_all("b"), data) - self._parse_tables(soup.find_all("table"), data) - return [data] - - def _parse_bs(self, btags: ResultSet, data: Dict): - ... - - def _parse_tables(self, tables: ResultSet, data: Dict): - ... -``` - -The next step is to create the new `Provider` by defining a new class in `circuit_maintenance_parser/provider.py`. This class that inherits from `GenericProvider` only needs to define two attributes: - -- `_processors`: is a `list` of `Processor` instances that uses several data `Parsers`. In this example, we don't need to create a new custom `Processor` because the combined logic serves well (the most likely case), and we only need to use the newly defined `HtmlParserABCDE1` and also the generic `EmailDateParser` that extracts the email date. Also notice that you could have multiple `Processors` with different `Parsers` in this list, supporting several formats. -- `_default_organizer`: This is a default helper to fill the `organizer` attribute in the `Maintenance` if the information is not part of the original notification. - -```python -class ABCDE(GenericProvider): - _processors: List[GenericProcessor] = [ - CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserABCDE1]), - ] - _default_organizer = "noc@abcde.com" -``` - -And expose the new `Provider` in `circuit_maintenance_parser/__init__.py`: - -```python -from .provider import ( - GenericProvider, - ABCDE, - ... -) - -SUPPORTED_PROVIDERS = ( - GenericProvider, - ABCDE, - ... -) -``` +## Contributing -Last, but not least, you should update the tests! - -- Test the new `Parser` in `tests/unit/test_parsers.py` -- Test the new `Provider` logic in `tests/unit/test_e2e.py` - -... adding the necessary data samples in `tests/unit/data/abcde/`. - -> You can anonymize your IPv4 and IPv6 addresses using the `invoke anonymize-ips`. Keep in mind that only IPv4 addresses for documentation purposes (RFC5737: "192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24") are preserved, in case you need to check these IPs in your test output (unlikely) - -# Contributing - -Pull requests are welcomed and automatically built and tested against multiple versions of Python through Travis CI. +Pull requests are welcomed and automatically built and tested through GitHub Actions. The project is following Network to Code software development guidelines and is leveraging: -- Black, Pylint, Mypy, Bandit and pydocstyle for Python linting and formatting. +- Ruff for Python linting and formatting. +- Pylint for additional static analysis. - Unit and integration tests to ensure the library is working properly. -## Local Development - -### Requirements - -- Install `poetry` -- Install dependencies and library locally: `poetry install` -- Run CI tests locally: `invoke tests` - -> Note: you can run the tasks without Docker by setting the environment variable `INVOKE_PARSER_LOCAL=True`. This will run the tasks directly on your local machine instead of inside a Docker container. - -### How to add a new Circuit Maintenance provider? - -1. Define the `Parsers`(inheriting from some of the generic `Parsers` or a new one) that will extract the data from the notification, which could contain multiple `DataParts`. The `data_type` of the `Parser` and the `DataPart` have to match. The custom `Parsers` will be placed in the `parsers` folder. -2. Update the `unit/test_parsers.py` with the new parsers, providing some data to test and validate the extracted data. -3. Define a new `Provider` inheriting from the `GenericProvider`, defining the `Processors` and the respective `Parsers` to be used. Maybe you can reuse some of the generic `Processors` or maybe you will need to create a custom one. If this is the case, place it in the `processors` folder. - - The `Provider` also supports the definition of a `_include_filter` and a `_exclude_filter` to limit the notifications that are actually processed, avoiding false positive errors for notification that are not relevant. -4. Update the `unit/test_e2e.py` with the new provider, providing some data to test and validate the final `Maintenances` created. -5. **Expose the new `Provider` class** updating the map `SUPPORTED_PROVIDERS` in `circuit_maintenance_parser/__init__.py` to officially expose the `Provider`. -6. You can run some tests here to verify that your new unit tests do not cause issues with existing tests, and in general they work as expected. You can do this by running `pytest --log-cli-level=DEBUG --capture=tee-sys`. You can narrow down the tests that you want to execute with the `-k` flag. If successful, your results should look similar to the following: - -``` --> % pytest --log-cli-level=DEBUG --capture=tee-sys -k test_parsers -...omitted debug logs... -====================================================== 99 passed, 174 deselected, 17 warnings in 10.35s ====================================================== -``` - -7. Run some final CI tests locally to ensure that there is no linting/formatting issues with your changes. You should look to get a code score of 10/10. See the example below: `invoke tests` - -``` --> % poetry run invoke tests -DOCKER - Running command: ruff format --check . container: circuit_maintenance_parser:latest -52 files already formatted -DOCKER - Running command: ruff check --output-format concise . container: circuit_maintenance_parser:latest -All checks passed! -DOCKER - Running command: find . -name "*.py" | grep -vE "tests/unit" | xargs pylint container: circuit_maintenance_parser:latest - ------------------------------------- -Your code has been rated at 10.00/10 -``` - -### How to debug circuit-maintenance-parser library locally - -1. `poetry install` updates the library and its dependencies locally. -2. `circuit-maintenance-parser` is now built with your recent local changes. - -If you were to add loggers or debuggers to one of the classes: - -```python -class HtmlParserZayo1(Html): - def parse_bs(self, btags: ResultSet, data: dict): - """Parse B tag.""" - raise Exception('Debugging exception') -``` - -After running `poetry install`: - -``` --> % circuit-maintenance-parser --data-file ~/Downloads/zayo.eml --data-type email --provider-type zayo -Provider processing failed: Failed creating Maintenance notification for Zayo. -Details: -- Processor CombinedProcessor from Zayo failed due to: Debugging exception -``` - -> Note: `invoke build` will result in an error due to no Dockerfile. This is expected as the library runs simple pytest testing without a container. - -``` --> % invoke build -Building image circuit-maintenance-parser:2.2.2-py3.8 -#1 [internal] load build definition from Dockerfile -#1 transferring dockerfile: 2B done -#1 DONE 0.0s -WARNING: failed to get git remote url: fatal: No remote configured to list refs from. -ERROR: failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to read dockerfile: open /var/lib/docker/tmp/buildkit-mount1243547759/Dockerfile: no such file or directory -``` +For more details, see the [Contributing Guide](https://circuit-maintenance-parser.readthedocs.io/en/latest/dev/contributing/) and [Development Environment Guide](https://circuit-maintenance-parser.readthedocs.io/en/latest/dev/dev_environment/). ## Questions -For any questions or comments, please check the [FAQ](FAQ.md) first and feel free to swing by the [Network to Code slack channel](https://networktocode.slack.com/) (channel #networktocode). -Sign up [here](http://slack.networktocode.com/) +For any questions or comments, please check the [FAQ](https://circuit-maintenance-parser.readthedocs.io/en/latest/user/faq/) first. Feel free to also swing by the [Network to Code Slack](https://networktocode.slack.com/) (channel `#networktocode`), sign up [here](http://slack.networktocode.com/) if you don't have an account. ## License notes diff --git a/changes/+main.housekeeping b/changes/+main.housekeeping new file mode 100644 index 00000000..3433adf6 --- /dev/null +++ b/changes/+main.housekeeping @@ -0,0 +1 @@ +Rebaked from the cookie `main`. diff --git a/changes/.gitignore b/changes/.gitignore new file mode 100644 index 00000000..f935021a --- /dev/null +++ b/changes/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/circuit_maintenance_parser/__init__.py b/circuit_maintenance_parser/__init__.py index 0d7dc878..bc994864 100644 --- a/circuit_maintenance_parser/__init__.py +++ b/circuit_maintenance_parser/__init__.py @@ -1,5 +1,6 @@ """Circuit-maintenance-parser init.""" +from importlib import metadata from typing import Optional, Type from .data import NotificationData @@ -75,6 +76,9 @@ Zayo, ) + +__version__ = metadata.version(__name__) + SUPPORTED_PROVIDER_NAMES = [provider.get_provider_type() for provider in SUPPORTED_PROVIDERS] SUPPORTED_ORGANIZER_EMAILS = [provider.get_default_organizer() for provider in SUPPORTED_PROVIDERS] diff --git a/circuit_maintenance_parser/cli.py b/circuit_maintenance_parser/cli.py index 69fbe9a4..08274682 100644 --- a/circuit_maintenance_parser/cli.py +++ b/circuit_maintenance_parser/cli.py @@ -13,7 +13,12 @@ @click.command() @click.option("--data-file", required=True, help="File containing raw data to parse.") -@click.option("--data-type", required=False, help="Type of notification data. Default: Icalendar", default="ical") +@click.option( + "--data-type", + required=False, + help="Type of notification data. Default: Icalendar", + default="ical", +) @click.option( "--provider-type", type=click.Choice([provider.get_provider_type() for provider in SUPPORTED_PROVIDERS]), diff --git a/docs/admin/install.md b/docs/admin/install.md new file mode 100644 index 00000000..12922649 --- /dev/null +++ b/docs/admin/install.md @@ -0,0 +1,22 @@ +# Installation + +Option 1: Install from PyPI. + +```bash +pip install circuit-maintenance-parser +``` + +Option 2: Manually install via Poetry. + +```bash +git clone https://github.com/networktocode/circuit-maintenance-parser.git +cd circuit-maintenance-parser +curl -sSL https://install.python-poetry.org | python3 - +poetry install +``` + +Option 3: Install from a GitHub branch, such as develop as shown below. + +```bash +pip install git+https://github.com/networktocode/circuit-maintenance-parser.git@develop +``` diff --git a/docs/admin/release_notes/index.md b/docs/admin/release_notes/index.md new file mode 100644 index 00000000..12cb5169 --- /dev/null +++ b/docs/admin/release_notes/index.md @@ -0,0 +1,3 @@ +# Release Notes + +All the published release notes can be found via the navigation menu. All patch releases are included in the same minor release (e.g. `v1.2`) document. diff --git a/docs/admin/release_notes/version_1.0.md b/docs/admin/release_notes/version_1.0.md new file mode 100644 index 00000000..d87d2b1b --- /dev/null +++ b/docs/admin/release_notes/version_1.0.md @@ -0,0 +1,15 @@ +# v1.0 Release Notes + +This document describes all new features and changes in the `1.x` release series. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +For a complete history of all releases, see the [full changelog](../../release_notes.md). + +## v1.0.2 - 2021-05-05 + +### Added + +- [#10](https://github.com/networktocode/circuit-maintenance-parser/pull/10) - Added `cli` command to run as a script. + +## v1.0.0 - 2021-04-29 + +Initial release of the circuit-maintenance-parser library with support for parsing circuit maintenance notifications from Network Service Providers using the iCalendar BCOP standard format. diff --git a/docs/admin/release_notes/version_2.10.md b/docs/admin/release_notes/version_2.10.md new file mode 100644 index 00000000..e0b2ea78 --- /dev/null +++ b/docs/admin/release_notes/version_2.10.md @@ -0,0 +1,19 @@ +# v2.10 Release Notes + +This document describes all new features and changes in the `2.10` release series. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [v2.10.0 (2026-01-27)](https://github.com/networktocode/circuit-maintenance-parser/releases/tag/v2.10.0) + +### Added + +- [#330](https://github.com/networktocode/circuit-maintenance-parser/issues/330) - Add support for parsing AWS HTML format maintenance notification emails + +### Dependencies + +- [#360](https://github.com/networktocode/circuit-maintenance-parser/issues/360) - Updated lxml to include version 6 +- [#361](https://github.com/networktocode/circuit-maintenance-parser/issues/361) - Updated timezonefinder to v8.2.0 +- [#345](https://github.com/networktocode/circuit_maintenance_parser/issues/345) - Updated minimum Python version to 3.10 and upgraded dependencies including pytest (9.0), pylint (4.0), towncrier (25.8), backoff (2.2), and type stubs + +### Housekeeping + +- [#344](https://github.com/networktocode/circuit-maintenance-parser/issues/344) - Updated test coverage for the codebase diff --git a/docs/admin/uninstall.md b/docs/admin/uninstall.md new file mode 100644 index 00000000..0d90822d --- /dev/null +++ b/docs/admin/uninstall.md @@ -0,0 +1,7 @@ +# Uninstall + +Uninstall from environment. + +```bash +pip uninstall circuit-maintenance-parser +``` diff --git a/docs/admin/upgrade.md b/docs/admin/upgrade.md new file mode 100644 index 00000000..e30f6919 --- /dev/null +++ b/docs/admin/upgrade.md @@ -0,0 +1,7 @@ +# Upgrading the Library + +Upgrade from PyPI. + +```bash +pip install circuit-maintenance-parser --upgrade +``` diff --git a/docs/assets/extra.css b/docs/assets/extra.css new file mode 100644 index 00000000..50884f4a --- /dev/null +++ b/docs/assets/extra.css @@ -0,0 +1,152 @@ +:root>* { + --md-accent-fg-color: #ff8504; + --md-primary-fg-color: #ff8504; + --md-typeset-a-color: #0097ff; +} + +[data-md-color-scheme="slate"] { + --md-default-bg-color: hsla(var(--md-hue), 0%, 15%, 1); + --md-typeset-a-color: #0097ff; +} + +/* Accessibility: Increase fonts for dark theme */ +[data-md-color-scheme="slate"] .md-typeset { + font-size: 0.9rem; +} + +[data-md-color-scheme="slate"] .md-typeset table:not([class]) { + font-size: 0.7rem; +} + +.md-tabs__link { + font-size: 0.8rem; +} + +.md-tabs__link--active { + color: var(--md-primary-fg-color); +} + +.md-header__button.md-logo :is(img, svg) { + height: 2rem; +} + +.md-header__button.md-logo :-webkit-any(img, svg) { + height: 2rem; +} + +.md-header__title { + font-size: 1.2rem; +} + +img.logo { + height: 100px; +} + +img.copyright-logo { + height: 24px; + vertical-align: middle; +} + +[data-md-color-primary=black] .md-header { + background-color: #212121; +} + +@media screen and (min-width: 76.25em) { + [data-md-color-primary=black] .md-tabs { + background-color: #212121; + } +} + +/* Customization for mkdocstrings */ +/* Indentation. */ +div.doc-contents:not(.first) { + padding-left: 25px; + border-left: .2rem solid var(--md-typeset-table-color); +} + +/* Mark external links as such. */ +a.autorefs-external::after { + /* https://primer.style/octicons/arrow-up-right-24 */ + background-image: url('data:image/svg+xml,'); + content: ' '; + + display: inline-block; + position: relative; + top: 0.1em; + margin-left: 0.2em; + margin-right: 0.1em; + + height: 1em; + width: 1em; + border-radius: 100%; + background-color: var(--md-typeset-a-color); +} + +a.autorefs-external:hover::after { + background-color: var(--md-accent-fg-color); +} + + +/* Customization for mkdocs-version-annotations */ +:root { + /* Icon for "version-added" admonition: Material Design Icons "plus-box-outline" */ + --md-admonition-icon--version-added: url('data:image/svg+xml;charset=utf-8,'); + /* Icon for "version-changed" admonition: Material Design Icons "delta" */ + --md-admonition-icon--version-changed: url('data:image/svg+xml;charset=utf-8,'); + /* Icon for "version-removed" admonition: Material Design Icons "minus-circle-outline" */ + --md-admonition-icon--version-removed: url('data:image/svg+xml;charset=utf-8,'); +} + +/* "version-added" admonition in green */ +.md-typeset .admonition.version-added, +.md-typeset details.version-added { + border-color: rgb(0, 200, 83); +} + +.md-typeset .version-added>.admonition-title, +.md-typeset .version-added>summary { + background-color: rgba(0, 200, 83, .1); +} + +.md-typeset .version-added>.admonition-title::before, +.md-typeset .version-added>summary::before { + background-color: rgb(0, 200, 83); + -webkit-mask-image: var(--md-admonition-icon--version-added); + mask-image: var(--md-admonition-icon--version-added); +} + +/* "version-changed" admonition in orange */ +.md-typeset .admonition.version-changed, +.md-typeset details.version-changed { + border-color: rgb(255, 145, 0); +} + +.md-typeset .version-changed>.admonition-title, +.md-typeset .version-changed>summary { + background-color: rgba(255, 145, 0, .1); +} + +.md-typeset .version-changed>.admonition-title::before, +.md-typeset .version-changed>summary::before { + background-color: rgb(255, 145, 0); + -webkit-mask-image: var(--md-admonition-icon--version-changed); + mask-image: var(--md-admonition-icon--version-changed); +} + +/* "version-removed" admonition in red */ +.md-typeset .admonition.version-removed, +.md-typeset details.version-removed { + border-color: rgb(255, 82, 82); +} + +.md-typeset .version-removed>.admonition-title, +.md-typeset .version-removed>summary { + background-color: rgba(255, 82, 82, .1); +} + +.md-typeset .version-removed>.admonition-title::before, +.md-typeset .version-removed>summary::before { + background-color: rgb(255, 82, 82); + -webkit-mask-image: var(--md-admonition-icon--version-removed); + mask-image: var(--md-admonition-icon--version-removed); +} diff --git a/docs/assets/favicon.ico b/docs/assets/favicon.ico new file mode 100644 index 00000000..9685c83b Binary files /dev/null and b/docs/assets/favicon.ico differ diff --git a/docs/assets/networktocode_bw.png b/docs/assets/networktocode_bw.png new file mode 100644 index 00000000..075c4926 Binary files /dev/null and b/docs/assets/networktocode_bw.png differ diff --git a/docs/assets/networktocode_logo.png b/docs/assets/networktocode_logo.png new file mode 100644 index 00000000..b2a3cba2 Binary files /dev/null and b/docs/assets/networktocode_logo.png differ diff --git a/docs/assets/overrides/partials/copyright.html b/docs/assets/overrides/partials/copyright.html new file mode 100644 index 00000000..cbf6bde5 --- /dev/null +++ b/docs/assets/overrides/partials/copyright.html @@ -0,0 +1,21 @@ + + + + + + +
diff --git a/docs/dev/arch_decision.md b/docs/dev/arch_decision.md new file mode 100644 index 00000000..cbe4d49d --- /dev/null +++ b/docs/dev/arch_decision.md @@ -0,0 +1,3 @@ +# Architecture Decision Records + +The intention is to document deviations from a standard pattern. diff --git a/docs/dev/contributing.md b/docs/dev/contributing.md new file mode 100644 index 00000000..67dd6cc3 --- /dev/null +++ b/docs/dev/contributing.md @@ -0,0 +1,77 @@ +# Contributing + +Pull requests are welcomed and automatically built and tested against multiple versions of Python through GitHub Actions. + +Except for unit tests, testing is only supported on Python 3.13. + +The project is packaged with a light development environment based on `Docker` to help with the local development of the project and to run tests within GitHub Actions. + +The project is following Network to Code software development guidelines and is leveraging the following: + +- Python linting and formatting: `pylint` and `ruff`. +- YAML linting is done with `yamllint`. + +Documentation is built using [mkdocs](https://www.mkdocs.org/). The [Docker based development environment](dev_environment.md#docker-development-environment) can be started by running `invoke docs` [http://localhost:8001](http://localhost:8001) that auto-refreshes when you make any changes to your local files. + +## Creating Changelog Fragments + +All pull requests to `next` or `develop` must include a changelog fragment file in the `./changes` directory. To create a fragment, use your GitHub issue number and fragment type as the filename. For example, `2362.added`. Valid fragment types are `added`, `changed`, `deprecated`, `fixed`, `removed`, and `security`. The change summary is added to the file in plain text. Change summaries should be complete sentences, starting with a capital letter and ending with a period, and be in past tense. Each line of the change fragment will generate a single change entry in the release notes. Use multiple lines in the same file if your change needs to generate multiple release notes in the same category. If the change needs to create multiple entries in separate categories, create multiple files. + +!!! example + + **Wrong** + ```plaintext title="changes/1234.fixed" + fix critical bug in documentation + ``` + + **Right** + ```plaintext title="changes/1234.fixed" + Fixed critical bug in documentation. + ``` + +!!! example "Multiple Entry Example" + + This will generate 2 entries in the `fixed` category and one entry in the `changed` category. + + ```plaintext title="changes/1234.fixed" + Fixed critical bug in documentation. + Fixed release notes generation. + ``` + + ```plaintext title="changes/1234.changed" + Changed release notes generation. + ``` + +## Branching Policy + +The branching policy includes the following tenets: + +- The develop branch is the primary branch to develop off of. +- If there is a reason to have a patch version, the maintainers may use cherry-picking strategy. +- PRs intended to add new features should be sourced from the develop branch. +- PRs intended to address bug fixes and security patches should be sourced from the develop branch. +- PRs intended to add new features that break backward compatibility should be discussed before a PR is created. + +Circuit-Maintenance-Parser will observe Semantic Versioning, as of 1.0. This may result in an quick turn around in minor versions to keep pace with an ever growing feature set. + +## Release Policy + +Circuit-Maintenance-Parser has currently no intended scheduled release schedule, and will release new features in minor versions. + +When a new release is created the following should happen. + +- A release PR is created with: + - Update to the changelog in `docs/admin/release_notes/version_..md` file to reflect the changes. + - Change the version from `..-beta` to `..` in pyproject.toml. + - Set the PR to the main +- Ensure the tests for the PR pass. +- Merge the PR. +- Create a new tag: + - The tag should be in the form of `v..`. + - The title should be in the form of `v..`. + - The description should be the changes that were added to the `version_..md` document. +- If merged into `main`, then push from `main` to `develop`, in order to retain the merge commit created when the PR was merged +- A post release PR is created with. + - Change the version from `..` to `..-beta` pyproject.toml. + - Set the PR to the `develop`. + - Once tests pass, merge. diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md new file mode 100644 index 00000000..0bd0e062 --- /dev/null +++ b/docs/dev/dev_environment.md @@ -0,0 +1,108 @@ +# Building Your Development Environment + +## Quickstart + +The development environment can be used in two ways: + +1. `Recommended` All services are spun up using Docker and a local mount so you can develop locally, but circuit-maintenance-parser is spun up within the Docker container. +2. With a local poetry environment if you wish to develop outside of Docker. + +This is a quick reference guide if you're already familiar with the development environment provided, which you can read more about later in this document. + +### Invoke + +The [Invoke](http://www.pyinvoke.org/) library is used to provide some helper commands based on the environment. There are a few configuration parameters which can be passed to Invoke to override the default configuration: + +- `local`: a boolean flag indicating if invoke tasks should be run on the host or inside the docker containers (default: False, commands will be run in docker containers) + +Using **Invoke** these configuration options can be overridden using [several methods](https://docs.pyinvoke.org/en/stable/concepts/configuration.html). Perhaps the simplest is setting an environment variable `INVOKE_CIRCUIT-MAINTENANCE-PARSER_VARIABLE_NAME` where `VARIABLE_NAME` is the variable you are trying to override. There is an example `invoke.yml` (`invoke.example.yml`) in this directory which can be used as a starting point. + +### Docker Development Environment + +!!! tip + This is the recommended option for development. + +This project is managed by [Python Poetry](https://python-poetry.org/) and has a few requirements to setup your development environment: + +1. Install Poetry, see the [Poetry Documentation](https://python-poetry.org/docs/#installation) for your operating system. +2. Install Docker, see the [Docker documentation](https://docs.docker.com/get-docker/) for your operating system. + +Once you have Poetry and Docker installed you can run the following commands (in the root of the repository) to install all other development dependencies in an isolated Python virtual environment: + +```shell +poetry shell +poetry install +invoke build +invoke start +``` + +Live documentation can be viewed at [http://localhost:8001](http://localhost:8001). + +To either stop or destroy the development environment use the following options. + +- **invoke stop** - Stop the containers, but keep all underlying systems intact +- **invoke destroy** - Stop and remove all containers, volumes, etc. (This results in data loss due to the volume being deleted) + +## Poetry + +Poetry is used in lieu of the "virtualenv" commands and is leveraged in both environments. The virtual environment will provide all of the Python packages required to manage the development environment such as **Invoke**. See the [Local Development Environment](#full-docker-development-environment) section to see how to install circuit-maintenance-parser if you're going to be developing locally (i.e. not using the Docker container). + +The `pyproject.toml` file outlines all of the relevant dependencies for the project: + +- `tool.poetry.dependencies` - the main list of dependencies. +- `tool.poetry.group.dev.dependencies` - development dependencies, to facilitate linting, testing, and documentation building. + +The `poetry shell` command is used to create and enable a virtual environment managed by Poetry, so all commands ran going forward are executed within the virtual environment. This is similar to running the `source venv/bin/activate` command with virtualenvs. To install project dependencies in the virtual environment, you should run `poetry install` - this will install **both** project and development dependencies. + +For more details about Poetry and its commands please check out its [online documentation](https://python-poetry.org/docs/). + +## Full Docker Development Environment + +This project is set up with a number of **Invoke** tasks consumed as simple CLI commands to get developing fast. You'll use a few `invoke` commands to get your environment up and running. + +## CLI Helper Commands + +The project features a CLI helper based on [invoke](http://www.pyinvoke.org/) to help setup the development environment. The commands are listed below in 3 categories: +- `dev environment` +- `utility` +- `testing` + +Each command can be executed with `invoke `. Each command also has its own help `invoke --help` + +### Local dev environment + +``` + build Build all docker images. + clean Remove the project specific image. + docs Build and serve docs locally. + rebuild Clean the Docker image and then rebuild without using cache. +``` + +### Utility + +``` + cli Enter the image to perform troubleshooting or dev work. + clean Remove stopped containers that source for image `circuit-maintenance-parser:` + generate-release-notes Generate Release Notes using Towncrier. +``` + +### Testing + +``` + autoformat (a) Run code autoformatting. + pylint Run pylint for the specified name and Python version. + ruff Run ruff to perform code formatting and/or linting. + pytest Run pytest for the specified name and Python version. + tests Run all tests for the specified name and Python version. + yamllint Run yamllint to validate formatting adheres to NTC defined YAML standards. +``` + +## Local Development Environment + +### Requirements + +- Install `poetry` +- Install dependencies and library locally: `poetry install` +- Run CI tests locally: `invoke tests` + +> Note: you can run the tasks without Docker by setting the environment variable `INVOKE_PARSER_LOCAL=True`. This will run the tasks directly on your local machine instead of inside a Docker container. \ No newline at end of file diff --git a/docs/dev/extending.md b/docs/dev/extending.md new file mode 100644 index 00000000..3360ab44 --- /dev/null +++ b/docs/dev/extending.md @@ -0,0 +1,140 @@ +# Extending the Library + +Extending the library is welcome, however it is best to open an issue first, to ensure that a PR would be accepted and makes sense in terms of features and design. + +## Adding a New Provider + +Adding a new `Provider` is straightforward. Here's an example adding support for an imaginary provider, ABCDE, that uses HTML notifications. + +### Step 1: Create the Parser + +Create a new file `circuit_maintenance_parser/parsers/abcde.py` with your custom parser(s). This file will contain all the custom parsers needed for the provider and will import the base classes for each parser type from `circuit_maintenance_parser.parser`: + +```python +from typing import Dict +import bs4 # type: ignore +from bs4.element import ResultSet # type: ignore +from circuit_maintenance_parser.parser import Html + +class HtmlParserABCDE1(Html): + def parse_html(self, soup: ResultSet) -> Dict: + data = {} + self._parse_bs(soup.find_all("b"), data) + self._parse_tables(soup.find_all("table"), data) + return [data] + + def _parse_bs(self, btags: ResultSet, data: Dict): + ... + + def _parse_tables(self, tables: ResultSet, data: Dict): + ... +``` + +### Step 2: Update Parser Tests + +Update `tests/unit/test_parsers.py` with the new parsers, providing some data to test and validate the extracted data. + +### Step 3: Create the Provider + +Define a new `Provider` by creating a new class in `circuit_maintenance_parser/provider.py`. This class inherits from `GenericProvider` and only needs to define two attributes: + +- `_processors`: a list of `Processor` instances that use several data `Parsers`. You can reuse generic `Processors` or create custom ones. If creating a custom one, place it in the `processors` folder. + - The `Provider` also supports the definition of a `_include_filter` and a `_exclude_filter` to limit the notifications that are actually processed, avoiding false positive errors for notifications that are not relevant. +- `_default_organizer`: a default helper to fill the `organizer` attribute in the `Maintenance` if the information is not part of the original notification. + +```python +class ABCDE(GenericProvider): + _processors: List[GenericProcessor] = [ + CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserABCDE1]), + ] + _default_organizer = "noc@abcde.com" +``` + +### Step 4: Update End-to-End Tests + +Update `tests/unit/test_e2e.py` with the new provider, providing some data to test and validate the final `Maintenances` created. + +### Step 5: Expose the Provider + +Update the map `SUPPORTED_PROVIDERS` in `circuit_maintenance_parser/__init__.py` to officially expose the `Provider`: + +```python +from .provider import ( + GenericProvider, + ABCDE, + ... +) + +SUPPORTED_PROVIDERS = ( + GenericProvider, + ABCDE, + ... +) +``` + +### Step 6: Run Unit Tests + +You can run some tests to verify that your new unit tests do not cause issues with existing tests, and in general they work as expected. You can do this by running `pytest --log-cli-level=DEBUG --capture=tee-sys`. You can narrow down the tests that you want to execute with the `-k` flag. If successful, your results should look similar to the following: + +``` +-> % pytest --log-cli-level=DEBUG --capture=tee-sys -k test_parsers +...omitted debug logs... +====================================================== 99 passed, 174 deselected, 17 warnings in 10.35s ====================================================== +``` + +### Step 7: Run CI Tests Locally + +Run some final CI tests locally to ensure that there is no linting/formatting issues with your changes. You should look to get a code score of 10/10. See the example below: `invoke tests` + +``` +-> % poetry run invoke tests +DOCKER - Running command: ruff format --check . container: circuit_maintenance_parser:latest +52 files already formatted +DOCKER - Running command: ruff check --output-format concise . container: circuit_maintenance_parser:latest +All checks passed! +DOCKER - Running command: find . -name "*.py" | grep -vE "tests/unit" | xargs pylint container: circuit_maintenance_parser:latest + +------------------------------------ +Your code has been rated at 10.00/10 +``` + +### Test Data + +Add the necessary data samples in `tests/unit/data/abcde/`. + +You can anonymize your IPv4 and IPv6 addresses using `invoke anonymize-ips`. Keep in mind that only IPv4 addresses for documentation purposes (RFC5737: "192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24") are preserved, in case you need to check these IPs in your test output (unlikely). + +## Debugging the Library Locally + +1. `poetry install` updates the library and its dependencies locally. +2. `circuit-maintenance-parser` is now built with your recent local changes. + +If you were to add loggers or debuggers to one of the classes: + +```python +class HtmlParserZayo1(Html): + def parse_bs(self, btags: ResultSet, data: dict): + """Parse B tag.""" + raise Exception('Debugging exception') +``` + +After running `poetry install`: + +``` +-> % circuit-maintenance-parser --data-file ~/Downloads/zayo.eml --data-type email --provider-type zayo +Provider processing failed: Failed creating Maintenance notification for Zayo. +Details: +- Processor CombinedProcessor from Zayo failed due to: Debugging exception +``` + +> Note: `invoke build` will result in an error due to no Dockerfile. This is expected as the library runs simple pytest testing without a container. + +``` +-> % invoke build +Building image circuit-maintenance-parser:2.2.2-py3.8 +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 2B done +#1 DONE 0.0s +WARNING: failed to get git remote url: fatal: No remote configured to list refs from. +ERROR: failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to read dockerfile: open /var/lib/docker/tmp/buildkit-mount1243547759/Dockerfile: no such file or directory +``` diff --git a/docs/dev/release_checklist.md b/docs/dev/release_checklist.md new file mode 100644 index 00000000..9acd0c3c --- /dev/null +++ b/docs/dev/release_checklist.md @@ -0,0 +1,192 @@ +# Release Checklist + +This document is intended for library maintainers and outlines the steps to perform when releasing a new version of the library. + +!!! important + Before starting, make sure your **local** `develop`, `main` are all up to date with upstream! + + ``` + git fetch + git switch develop && git pull + ``` + +Choose your own adventure: + +- Patch release from `develop`? Jump [here](#all-releases-from-develop). +- Minor release? Continue with [Minor Version Bumps](#minor-version-bumps) and then [All Releases from `develop`](#all-releases-from-develop). + +## Minor Version Bumps + +### Update Requirements + +Every minor version release should refresh `poetry.lock`, so that it lists the most recent stable release of each package. To do this: + +0. Run `poetry update --dry-run` to have Poetry automatically tell you what package updates are available and the versions it would upgrade to. This requires an existing environment created from the lock file (i.e. via `poetry install`). +1. Review each requirement's release notes for any breaking or otherwise noteworthy changes. +2. Run `poetry update ` to update the package versions in `poetry.lock` as appropriate. +3. If a required package requires updating to a new release not covered in the version constraints for a package as defined in `pyproject.toml`, (e.g. `Django ~3.1.7` would never install `Django >=4.0.0`), update it manually in `pyproject.toml`. +4. Run `poetry install` to install the refreshed versions of all required packages. +5. Run all tests (`poetry run invoke tests`) and check that the UI and API function as expected. + +### Update Documentation + +If there are any changes to the compatibility matrix (such as a bump in the minimum supported Nautobot version), update it accordingly. + +Commit any resulting changes from the following sections to the documentation before proceeding with the release. + +!!! tip + Fire up the documentation server in your development environment with `poetry run mkdocs serve`! This allows you to view the documentation site locally (the link is in the output of the command) and automatically rebuilds it as you make changes. + +### Verify the Installation and Upgrade Steps + +Follow the [installation instructions](../admin/install.md) to perform a new production installation of the library. If possible, also test the [upgrade process](../admin/upgrade.md) from the previous released version. + +The goal of this step is to walk through the entire install process *as documented* to make sure nothing there needs to be changed or updated, to catch any errors or omissions in the documentation, and to ensure that it is current with each release. + +--- + +## All Releases from `develop` + +### Verify CI Build Status + +Ensure that continuous integration testing on the `develop` branch is completing successfully. + +### Bump the Version + +Update the package version using `poetry version` if necessary. This command shows the current version of the project or bumps the version of the project and writes the new version back to `pyproject.toml` if a valid bump rule is provided. + +The new version must be a valid semver string or a valid bump rule: `patch`, `minor`, `major`, `prepatch`, `preminor`, `premajor`, `prerelease`. Always try to use a bump rule when you can. + +Display the current version with no arguments: + +```no-highlight +> poetry version +circuit-maintenance-parser 1.0.0-beta.2 +``` + +Bump pre-release versions using `prerelease`: + +```no-highlight +> poetry version prerelease +Bumping version from 1.0.0-beta.2 to 1.0.0-beta.3 +``` + +For major versions, use `major`: + +```no-highlight +> poetry version major +Bumping version from 1.0.0-beta.2 to 1.0.0 +``` + +For patch versions, use `minor`: + +```no-highlight +> poetry version minor +Bumping version from 1.0.0 to 1.1.0 +``` + +And lastly, for patch versions, you guessed it, use `patch`: + +```no-highlight +> poetry version patch +Bumping version from 1.1.0 to 1.1.1 +``` + +Please see the [official Poetry documentation on `version`](https://python-poetry.org/docs/cli/#version) for more information. + +### Update the Changelog + +!!! important + The changelog must adhere to the [Keep a Changelog](https://keepachangelog.com/) style guide. + +This guide uses `1.4.2` as the new version in its examples, so change it to match the version you bumped to in the previous step! Every. single. time. you. copy/paste commands :) + +First, create a release branch off of `develop` (`git switch -c release-1.4.2 develop`). + +> You will need to have the project's poetry environment built at this stage, as the towncrier command runs **locally only**. If you don't have it, run `poetry install` first. +Generate release notes with `invoke generate-release-notes --version 1.4.2` and answer `yes` to the prompt `Is it okay if I remove those files? [Y/n]:`. This will update the release notes in `docs/admin/release_notes/version_X.Y.md`, stage that file in git, and `git rm` all the fragments that have now been incorporated into the release notes. + +There are two possibilities: + +1. If you're releasing a new major or minor version, rename the `version_X.Y.md` file accordingly (e.g. rename to `docs/admin/release_notes/version_1.4.md`). Update the `Release Overview` and add this new page to the table of contents within `mkdocs.yml`. +2. If you're releasing a patch version, copy your version's section from the `version_X.Y.md` file into the already existing `docs/admin/release_notes/version_1.4.md` file. Delete the `version_X.Y.md` file. + +Stage all the changes (`git add`) and check the diffs to verify all of the changes are correct (`git diff --cached`). + +Commit `git commit -m "Release v1.4.2"` and `git push` the staged changes. + +### Submit Release Pull Request + +Submit a pull request titled `Release v1.4.2` to merge your release branch into `main`. Copy the documented release notes into the pull request's body. + +!!! important + Do not squash merge this branch into `main`. Make sure to select `Create a merge commit` when merging in GitHub. + +Once CI has completed on the PR, merge it. + +### Create a New Release in GitHub + +Draft a [new release](https://github.com/networktocode/circuit-maintenance-parser/releases/new) with the following parameters. + +* **Tag:** Input current version (e.g. `v1.4.2`) and select `Create new tag: v1.4.2 on publish` +* **Target:** `main` +* **Title:** Version and date (e.g. `v1.4.2 - 2024-04-02`) + +Click "Generate Release Notes" and edit the auto-generated content as follows: + +- Change the entries generated by GitHub to only the usernames of the contributors. e.g. `* Updated dockerfile by @ntc_user in https://github.com/networktocode/circuit-maintenance-parser/pull/123` -> `* @ntc_user`. + - This should give you the list for the new `Contributors` section. + - Make sure there are no duplicated entries. +- Replace the content of the `What's Changed` section with the description of changes from the release PR (what towncrier generated). +- If it exists, leave the `New Contributors` list as it is. + +The release notes should look as follows: + +```markdown +## What's Changed + +**Towncrier generated Changed/Fixed/Housekeeping etc. sections here** + +## Contributors + +* @alice +* @bob + +## New Contributors + +* @bob + +**Full Changelog**: https://github.com/networktocode/circuit-maintenance-parser/compare/v1.4.1...v1.4.2 +``` + +Publish the release! + +### Create a PR from `main` back to `develop` + +First, sync your `main` branch with upstream changes: `git switch main && git pull`. + +Create a new branch from `main` called `release-1.4.2-to-develop` and use `poetry version prepatch` to bump the development version to the next release. + +For example, if you just released `v1.4.2`: + +```no-highlight +> git switch -c release-1.4.2-to-develop main +Switched to a new branch 'release-1.4.2-to-develop' +> poetry version prepatch +Bumping version from 1.4.2 to 1.4.3a1 +> git add pyproject.toml && git commit -m "Bump version" +> git push +``` + +!!! important + Do not squash merge this branch into `develop`. Make sure to select `Create a merge commit` when merging in GitHub. + +Open a new PR from `release-1.4.2-to-develop` against `develop`, wait for CI to pass, and merge it. + +### Final checks + +At this stage, the CI should be running or finished for the `v1.4.2` tag and a package successfully published to PyPI and added into the GitHub Release. Double check that's the case. + +Documentation should also have been built for the tag on ReadTheDocs and if you're reading this page online, refresh it and look for the new version in the little version fly-out menu down at the bottom right of the page. + +All done! diff --git a/docs/generate_code_reference_pages.py b/docs/generate_code_reference_pages.py new file mode 100644 index 00000000..0fed3899 --- /dev/null +++ b/docs/generate_code_reference_pages.py @@ -0,0 +1,20 @@ +"""Generate code reference pages.""" + +from pathlib import Path + +import mkdocs_gen_files + +for file_path in Path("circuit-maintenance-parser").rglob("*.py"): + module_path = file_path.with_suffix("") + doc_path = file_path.with_suffix(".md") + full_doc_path = Path("code-reference", doc_path) + + parts = list(module_path.parts) + if parts[-1] == "__init__": + parts = parts[:-1] + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + identifier = ".".join(parts) + print(f"::: {identifier}", file=fd) + + mkdocs_gen_files.set_edit_path(full_doc_path, file_path) diff --git a/docs/images/networktocode_logo.svg b/docs/images/networktocode_logo.svg new file mode 100644 index 00000000..348e5241 --- /dev/null +++ b/docs/images/networktocode_logo.svg @@ -0,0 +1,150 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..f32fd72b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,6 @@ +--- +hide: + - navigation +--- + +--8<-- "README.md" diff --git a/docs/user/faq.md b/docs/user/faq.md new file mode 100644 index 00000000..62d37b69 --- /dev/null +++ b/docs/user/faq.md @@ -0,0 +1,21 @@ +# Frequently Asked Questions + +## Q: My provider is not supported. What can I do? + +You can extend the library by adding a new provider. See the [Extending the Library](../dev/extending.md) guide for instructions. Pull requests for new providers are always welcome! + +## Q: The parser for my provider is returning incorrect data. What should I do? + +Please [open an issue](https://github.com/networktocode/circuit-maintenance-parser/issues/new) with the details. If possible, include a sanitized sample of the notification data that is being parsed incorrectly. + +## Q: Can I use the LLM-powered parser without a specific provider? + +Yes, the LLM-powered parsers are automatically appended after all existing processors for each defined Provider when the appropriate environment variables are set. You can use them with any provider, including the `GenericProvider`. + +## Q: How do I check if a maintenance was parsed by an LLM? + +Each `Maintenance` object has a `metadata` attribute. Check `maintenance.metadata.generated_by_llm` to determine if LLM parsing was used. + +## Q: Where can I get help? + +Feel free to swing by the [Network to Code Slack](https://networktocode.slack.com/) (channel `#networktocode`). Sign up [here](http://slack.networktocode.com/) if you don't have an account. diff --git a/docs/user/lib_getting_started.md b/docs/user/lib_getting_started.md new file mode 100644 index 00000000..d7d6b1cf --- /dev/null +++ b/docs/user/lib_getting_started.md @@ -0,0 +1,70 @@ +# Getting Started with the Library + +This document provides a step-by-step tutorial on how to get the library going and how to use it. + +## Install the Library + +To install the library, please follow the instructions detailed in the [Installation Guide](../admin/install.md). + +## First steps with the Library + +### Parse an iCalendar Notification + +The simplest use case is parsing a standard iCalendar (BCOP) notification: + +```python +from circuit_maintenance_parser import init_provider, NotificationData + +# Initialize a generic provider (supports standard iCalendar format) +provider = init_provider() + +# Create notification data from raw iCalendar content +raw_data = b"""BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Maint Note//https://github.com/maint-notification// +BEGIN:VEVENT +SUMMARY:Maint Note Example +DTSTART;VALUE=DATE-TIME:20151010T080000Z +DTEND;VALUE=DATE-TIME:20151010T100000Z +DTSTAMP;VALUE=DATE-TIME:20151010T001000Z +UID:42 +SEQUENCE:1 +X-MAINTNOTE-PROVIDER:example.com +X-MAINTNOTE-ACCOUNT:137.035999173 +X-MAINTNOTE-MAINTENANCE-ID:WorkOrder-31415 +X-MAINTNOTE-IMPACT:OUTAGE +X-MAINTNOTE-OBJECT-ID;X-MAINTNOTE-OBJECT-IMPACT=OUTAGE:circuit-1 +X-MAINTNOTE-STATUS:TENTATIVE +ORGANIZER;CN="Example NOC":mailto:noone@example.com +END:VEVENT +END:VCALENDAR +""" + +data = NotificationData.init_from_raw("ical", raw_data) +maintenances = provider.get_maintenances(data) + +print(maintenances[0].to_json()) +``` + +### Parse a Provider-Specific Notification + +For providers that don't use the standard iCalendar format, initialize a provider-specific instance: + +```python +from circuit_maintenance_parser import init_provider + +# Initialize a provider-specific parser (e.g., Zayo) +zayo_provider = init_provider("zayo") +``` + +### Use the CLI + +The library also provides a command-line interface: + +```bash +circuit-maintenance-parser --data-file notification.eml --data-type email --provider-type zayo +``` + +## What are the next steps? + +You can check out the [Use Cases](./lib_use_cases.md) section for more examples. diff --git a/docs/user/lib_overview.md b/docs/user/lib_overview.md new file mode 100644 index 00000000..02f7fb6e --- /dev/null +++ b/docs/user/lib_overview.md @@ -0,0 +1,23 @@ +# Library Overview + +This document provides an overview of the library including critical information and important considerations. + +## Description + +`circuit-maintenance-parser` is a Python library that parses circuit maintenance notifications from Network Service Providers (NSPs), converting heterogeneous formats to a well-defined structured format. + +Every network depends on external circuits provided by NSPs who interconnect them to the Internet, to office branches or to external service providers such as Public Clouds. These services occasionally require operation windows to upgrade or fix related issues, usually in the form of **circuit maintenance periods**. NSPs generally notify customers of these upcoming events so that customers can take actions to minimize impact. + +The challenge is that almost every NSP defines its own maintenance notification format, even though the relevant information is mostly the same. This library parses notification formats from several providers and returns a standardized object struct, making it easier to process them. + +The output format follows the [BCOP](https://github.com/jda/maintnote-std/blob/master/standard.md) defined during a NANOG meeting that promotes the usage of the iCalendar format. + +## Audience (User Personas) - Who should use this Library? + +- **Network Engineers** who need to automate the processing of circuit maintenance notifications from multiple providers. +- **Network Operations Center (NOC) teams** who want to standardize how maintenance windows are tracked across different NSPs. +- **Automation developers** building workflows that need to programmatically consume and act on circuit maintenance notifications. + +## Authors and Maintainers + +This library is maintained by [Network to Code](https://www.networktocode.com/). For a full list of contributors, see the [GitHub contributors page](https://github.com/networktocode/circuit-maintenance-parser/graphs/contributors). diff --git a/docs/user/lib_use_cases.md b/docs/user/lib_use_cases.md new file mode 100644 index 00000000..477c00fd --- /dev/null +++ b/docs/user/lib_use_cases.md @@ -0,0 +1,81 @@ +# Using the Library + +This document describes common use-cases and scenarios for this library. + +## General Usage + +The library follows a three-step pattern: + +1. **Initialize a Provider** - Select the appropriate provider for your NSP. +2. **Create NotificationData** - Wrap your notification content (iCal, HTML, email, etc.). +3. **Parse Maintenances** - Call `provider.get_maintenances(data)` to get structured results. + +## Use-cases and common workflows + +### Parsing Email Notifications + +Many providers send maintenance notifications via email. You can parse an entire email message: + +```python +from circuit_maintenance_parser import init_provider, NotificationData + +provider = init_provider("zayo") + +with open("maintenance_email.eml", "rb") as f: + raw_email = f.read() + +data = NotificationData.init_from_emailmessage(raw_email) +maintenances = provider.get_maintenances(data) +``` + +### Parsing HTML Notifications + +For providers that send HTML-formatted notifications: + +```python +from circuit_maintenance_parser import init_provider, NotificationData + +provider = init_provider("lumen") + +with open("notification.html", "rb") as f: + html_content = f.read() + +data = NotificationData.init_from_raw("html", html_content) +maintenances = provider.get_maintenances(data) +``` + +### Using LLM-powered Parsing + +When specific parsers are insufficient, LLM-powered parsing can provide best-effort results. Set the required environment variables: + +```bash +export PARSER_OPENAI_API_KEY="your-api-key" +``` + +Install the OpenAI extra: + +```bash +pip install circuit-maintenance-parser[openai] +``` + +The LLM parsers are automatically appended after all existing processors. You can check the `metadata` attribute to see if LLM was used: + +```python +for maintenance in maintenances: + if maintenance.metadata.generated_by_llm: + print("Warning: This maintenance was parsed by LLM") +``` + +### Checking Parse Metadata + +Every maintenance includes metadata about how it was parsed: + +```python +maintenance = maintenances[0] +print(maintenance.metadata) +# provider='genericprovider' processor="SimpleProcessor" parsers=["ICal"], generated_by_llm=False +``` + +## Supported Providers + +For a complete list of supported providers, see the [README](https://github.com/networktocode/circuit-maintenance-parser#supported-providers). diff --git a/example.invoke.yml b/example.invoke.yml new file mode 100644 index 00000000..02d8c420 --- /dev/null +++ b/example.invoke.yml @@ -0,0 +1,7 @@ +--- +circuit-maintenance-parser: + python_ver: "3.10" + local: false + # image_name: "circuit-maintenance-parser" + # image_ver: "latest" + # pwd: "." diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..289aa26e --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,142 @@ +--- +dev_addr: "127.0.0.1:8001" +edit_uri: "edit/main/circuit-maintenance-parser/docs" +site_dir: "circuit-maintenance-parser/static/circuit-maintenance-parser/docs" +site_name: "Circuit-Maintenance-Parser Documentation" +site_url: "https://circuit-maintenance-parser.readthedocs.io/en/latest/" +repo_url: "https://github.com/networktocode/circuit-maintenance-parser" +copyright: "Copyright © The Authors" +theme: + name: "material" + navigation_depth: 4 + custom_dir: "docs/assets/overrides" + hljs_languages: + - "python" + - "yaml" + features: + - "content.action.edit" + - "content.action.view" + - "content.code.copy" + - "navigation.footer" + - "navigation.indexes" + - "navigation.tabs" + - "navigation.tabs.sticky" + - "navigation.tracking" + - "search.highlight" + - "search.share" + - "search.suggest" + favicon: "assets/favicon.ico" + logo: "assets/networktocode_logo.svg" + palette: + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: "default" + primary: "black" + toggle: + icon: "material/weather-sunny" + name: "Switch to dark mode" + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: "slate" + primary: "black" + toggle: + icon: "material/weather-night" + name: "Switch to light mode" +extra_css: + - "assets/extra.css" + +extra: + generator: false + ntc_sponsor: true + social: + - icon: "fontawesome/solid/rss" + link: "https://blog.networktocode.com/" + name: "Network to Code Blog" + - icon: "fontawesome/brands/youtube" + link: "https://www.youtube.com/channel/UCwBh-dDdoqzxXKyvTw3BuTw" + name: "Network to Code Videos" + - icon: "fontawesome/brands/slack" + link: "https://www.networktocode.com/community/" + name: "Network to Code Community" + - icon: "fontawesome/brands/github" + link: "https://github.com/networktocode/" + name: "GitHub Organization" + - icon: "fontawesome/brands/twitter" + link: "https://twitter.com/networktocode" + name: "Network to Code Twitter" +markdown_extensions: + - "markdown_version_annotations": + admonition_tag: "???" + - "admonition" + - "toc": + permalink: true + - "attr_list" + - "md_in_html" + - "markdown_data_tables": + base_path: "docs" + - "pymdownx.details" + # Need pymdownx.emoji for Grid icon search + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - "pymdownx.highlight": + anchor_linenums: true + - "pymdownx.inlinehilite" + - "pymdownx.snippets" + - "pymdownx.superfences": + custom_fences: + - name: "mermaid" + class: "mermaid" + format: !!python/name:pymdownx.superfences.fence_code_format + - "pymdownx.tabbed": + "alternate_style": true + - "pymdownx.tilde" + +plugins: + - "search" + - "gen-files": + scripts: + - "docs/generate_code_reference_pages.py" + - "glightbox": + manual: true # See https://blueswen.github.io/mkdocs-glightbox/flexibility/enable-by-image-or-page/ + - "section-index" + - "mkdocstrings": + default_handler: "python" + handlers: + python: + paths: ["."] + options: + heading_level: 1 + show_root_heading: true + show_root_members_full_path: true + show_source: false + +validation: + absolute_links: "warn" + anchors: "warn" + omitted_files: "warn" + unrecognized_links: "warn" + +nav: + - Overview: "index.md" + - User Guide: + - Library Overview: "user/lib_overview.md" + - Getting Started: "user/lib_getting_started.md" + - Using the Library: "user/lib_use_cases.md" + - Frequently Asked Questions: "user/faq.md" + - Administrator Guide: + - Install and Configure: "admin/install.md" + - Upgrade: "admin/upgrade.md" + - Uninstall: "admin/uninstall.md" + - Release Notes: + - "admin/release_notes/index.md" + - v2.10: "admin/release_notes/version_2.10.md" + - v1.0: "admin/release_notes/version_1.0.md" + - Full Changelog: "release_notes.md" + - Developer Guide: + - Extending the Library: "dev/extending.md" + - Contributing to the Library: "dev/contributing.md" + - Development Environment: "dev/dev_environment.md" + - Release Checklist: "dev/release_checklist.md" + - Architecture Design Records: "dev/arch_decision.md" diff --git a/poetry.lock b/poetry.lock index d6de2750..f33b94cb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -48,6 +48,21 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4", markers = "python_version < \"3.11\""} +[[package]] +name = "babel" +version = "2.18.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35"}, + {file = "babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d"}, +] + +[package.extras] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] + [[package]] name = "backoff" version = "2.2.1" @@ -60,6 +75,26 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] +[[package]] +name = "backrefs" +version = "5.9" +description = "A wrapper around re and regex that adds additional back references." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "backrefs-5.9-py310-none-any.whl", hash = "sha256:db8e8ba0e9de81fcd635f440deab5ae5f2591b54ac1ebe0550a2ca063488cd9f"}, + {file = "backrefs-5.9-py311-none-any.whl", hash = "sha256:6907635edebbe9b2dc3de3a2befff44d74f30a4562adbb8b36f21252ea19c5cf"}, + {file = "backrefs-5.9-py312-none-any.whl", hash = "sha256:7fdf9771f63e6028d7fee7e0c497c81abda597ea45d6b8f89e8ad76994f5befa"}, + {file = "backrefs-5.9-py313-none-any.whl", hash = "sha256:cc37b19fa219e93ff825ed1fed8879e47b4d89aa7a1884860e2db64ccd7c676b"}, + {file = "backrefs-5.9-py314-none-any.whl", hash = "sha256:df5e169836cc8acb5e440ebae9aad4bf9d15e226d3bad049cf3f6a5c20cc8dc9"}, + {file = "backrefs-5.9-py39-none-any.whl", hash = "sha256:f48ee18f6252b8f5777a22a00a09a85de0ca931658f1dd96d4406a34f3748c60"}, + {file = "backrefs-5.9.tar.gz", hash = "sha256:808548cb708d66b82ee231f962cb36faaf4f2baab032f2fbb783e9c2fdddaa59"}, +] + +[package.extras] +extras = ["regex"] + [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -116,7 +151,7 @@ version = "2026.1.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c"}, {file = "certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120"}, @@ -221,7 +256,7 @@ version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, @@ -344,7 +379,7 @@ version = "8.3.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, @@ -359,7 +394,7 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -593,6 +628,39 @@ dev-test = ["coverage", "pytest (>=3.10)", "pytest-asyncio (>=0.17)", "sphinx (< requests = ["requests (>=2.16.2)", "urllib3 (>=1.24.2)"] timezone = ["pytz"] +[[package]] +name = "ghp-import" +version = "2.1.0" +description = "Copy your docs directly to the gh-pages branch." +optional = false +python-versions = "*" +groups = ["docs"] +files = [ + {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, + {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, +] + +[package.dependencies] +python-dateutil = ">=2.8.1" + +[package.extras] +dev = ["flake8", "markdown", "twine", "wheel"] + +[[package]] +name = "griffe" +version = "1.1.1" +description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "griffe-1.1.1-py3-none-any.whl", hash = "sha256:0c469411e8d671a545725f5c0851a746da8bd99d354a79fdc4abd45219252efb"}, + {file = "griffe-1.1.1.tar.gz", hash = "sha256:faeb78764c0b2bd010719d6e015d07709b0f260258b5d4dd6c88343d9702aa30"}, +] + +[package.dependencies] +colorama = ">=0.4" + [[package]] name = "h11" version = "0.16.0" @@ -664,6 +732,18 @@ all = ["cartopy", "contextily", "geodatasets", "geopandas", "geoviews", "h3[test numpy = ["numpy"] test = ["numpy", "pytest", "pytest-cov", "ruff"] +[[package]] +name = "hjson" +version = "3.1.0" +description = "Hjson, a user interface for JSON." +optional = false +python-versions = "*" +groups = ["docs"] +files = [ + {file = "hjson-3.1.0-py3-none-any.whl", hash = "sha256:65713cdcf13214fb554eb8b4ef803419733f4f5e551047c9b711098ab7186b89"}, + {file = "hjson-3.1.0.tar.gz", hash = "sha256:55af475a27cf83a7969c808399d7bccdec8fb836a07ddbd574587593b9cdcf75"}, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -735,7 +815,7 @@ version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, @@ -791,7 +871,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -1072,13 +1152,61 @@ html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] +[[package]] +name = "markdown" +version = "3.10.2" +description = "Python implementation of John Gruber's Markdown." +optional = false +python-versions = ">=3.10" +groups = ["docs"] +files = [ + {file = "markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36"}, + {file = "markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950"}, +] + +[package.extras] +docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python] (>=0.28.3)"] +testing = ["coverage", "pyyaml"] + +[[package]] +name = "markdown-data-tables" +version = "1.0.0" +description = "Embed data files such as YAML as tables in a Markdown document" +optional = false +python-versions = ">=3.8,<4.0" +groups = ["docs"] +files = [ + {file = "markdown_data_tables-1.0.0-py3-none-any.whl", hash = "sha256:a59c6743685691ced4341bdb01024b7a863a1adaa3a2ef92fa068a7e90227d9a"}, + {file = "markdown_data_tables-1.0.0.tar.gz", hash = "sha256:ac1b07c58bb66e9f060ba81cdd63070ec94deb21f0147e519c77c8475ba696ea"}, +] + +[package.dependencies] +markdown = ">=3.3.7,<4.0.0" +pyyaml = ">=6.0,<7.0" +tabulate = ">=0.9.0,<0.10.0" + +[[package]] +name = "markdown-version-annotations" +version = "1.0.1" +description = "Markdown plugin to add custom admonitions for documenting version differences" +optional = false +python-versions = "<4.0,>=3.7" +groups = ["docs"] +files = [ + {file = "markdown_version_annotations-1.0.1-py3-none-any.whl", hash = "sha256:6df0b2ac08bab906c8baa425f59fc0fe342fbe8b3917c144fb75914266b33200"}, + {file = "markdown_version_annotations-1.0.1.tar.gz", hash = "sha256:620aade507ef175ccfb2059db152a34c6a1d2add28c2be16ea4de38d742e6132"}, +] + +[package.dependencies] +markdown = ">=3.3.7,<4.0.0" + [[package]] name = "markupsafe" version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -1183,6 +1311,252 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +description = "A deep merge function for 🐍." +optional = false +python-versions = ">=3.6" +groups = ["docs"] +files = [ + {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, + {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +description = "Project documentation with Markdown." +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, + {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", markers = "platform_system == \"Windows\""} +ghp-import = ">=1.0" +jinja2 = ">=2.11.1" +markdown = ">=3.3.6" +markupsafe = ">=2.0.1" +mergedeep = ">=1.3.4" +mkdocs-get-deps = ">=0.2.0" +packaging = ">=20.5" +pathspec = ">=0.11.1" +pyyaml = ">=5.1" +pyyaml-env-tag = ">=0.1" +watchdog = ">=2.0" + +[package.extras] +i18n = ["babel (>=2.9.0)"] +min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4) ; platform_system == \"Windows\"", "ghp-import (==1.0)", "importlib-metadata (==4.4) ; python_version < \"3.10\"", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] + +[[package]] +name = "mkdocs-autorefs" +version = "1.4.4" +description = "Automatically link across pages in MkDocs." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089"}, + {file = "mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197"}, +] + +[package.dependencies] +Markdown = ">=3.3" +markupsafe = ">=2.0.1" +mkdocs = ">=1.1" + +[[package]] +name = "mkdocs-gen-files" +version = "0.5.0" +description = "MkDocs plugin to programmatically generate documentation pages during the build" +optional = false +python-versions = ">=3.7" +groups = ["docs"] +files = [ + {file = "mkdocs_gen_files-0.5.0-py3-none-any.whl", hash = "sha256:7ac060096f3f40bd19039e7277dd3050be9a453c8ac578645844d4d91d7978ea"}, + {file = "mkdocs_gen_files-0.5.0.tar.gz", hash = "sha256:4c7cf256b5d67062a788f6b1d035e157fc1a9498c2399be9af5257d4ff4d19bc"}, +] + +[package.dependencies] +mkdocs = ">=1.0.3" + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.2" +description = "An extra command for MkDocs that infers required PyPI packages from `plugins` in mkdocs.yml" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650"}, + {file = "mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1"}, +] + +[package.dependencies] +mergedeep = ">=1.3.4" +platformdirs = ">=2.2.0" +pyyaml = ">=5.1" + +[[package]] +name = "mkdocs-glightbox" +version = "0.4.0" +description = "MkDocs plugin supports image lightbox with GLightbox." +optional = false +python-versions = "*" +groups = ["docs"] +files = [ + {file = "mkdocs-glightbox-0.4.0.tar.gz", hash = "sha256:392b34207bf95991071a16d5f8916d1d2f2cd5d5bb59ae2997485ccd778c70d9"}, + {file = "mkdocs_glightbox-0.4.0-py3-none-any.whl", hash = "sha256:e0107beee75d3eb7380ac06ea2d6eac94c999eaa49f8c3cbab0e7be2ac006ccf"}, +] + +[[package]] +name = "mkdocs-macros-plugin" +version = "1.3.7" +description = "Unleash the power of MkDocs with macros and variables" +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "mkdocs_macros_plugin-1.3.7-py3-none-any.whl", hash = "sha256:02432033a5b77fb247d6ec7924e72fc4ceec264165b1644ab8d0dc159c22ce59"}, + {file = "mkdocs_macros_plugin-1.3.7.tar.gz", hash = "sha256:17c7fd1a49b94defcdb502fd453d17a1e730f8836523379d21292eb2be4cb523"}, +] + +[package.dependencies] +hjson = "*" +jinja2 = "*" +mkdocs = ">=0.17" +packaging = "*" +pathspec = "*" +python-dateutil = "*" +pyyaml = "*" +super-collections = "*" +termcolor = "*" + +[package.extras] +test = ["mkdocs-d2-plugin", "mkdocs-include-markdown-plugin", "mkdocs-macros-test", "mkdocs-material (>=6.2)", "mkdocs-test"] + +[[package]] +name = "mkdocs-material" +version = "9.6.15" +description = "Documentation that simply works" +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "mkdocs_material-9.6.15-py3-none-any.whl", hash = "sha256:ac969c94d4fe5eb7c924b6d2f43d7db41159ea91553d18a9afc4780c34f2717a"}, + {file = "mkdocs_material-9.6.15.tar.gz", hash = "sha256:64adf8fa8dba1a17905b6aee1894a5aafd966d4aeb44a11088519b0f5ca4f1b5"}, +] + +[package.dependencies] +babel = ">=2.10,<3.0" +backrefs = ">=5.7.post1,<6.0" +colorama = ">=0.4,<1.0" +jinja2 = ">=3.1,<4.0" +markdown = ">=3.2,<4.0" +mkdocs = ">=1.6,<2.0" +mkdocs-material-extensions = ">=1.3,<2.0" +paginate = ">=0.5,<1.0" +pygments = ">=2.16,<3.0" +pymdown-extensions = ">=10.2,<11.0" +requests = ">=2.26,<3.0" + +[package.extras] +git = ["mkdocs-git-committers-plugin-2 (>=1.1,<3)", "mkdocs-git-revision-date-localized-plugin (>=1.2.4,<2.0)"] +imaging = ["cairosvg (>=2.6,<3.0)", "pillow (>=10.2,<11.0)"] +recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2.0)", "mkdocs-rss-plugin (>=1.6,<2.0)"] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +description = "Extension pack for Python Markdown and MkDocs Material." +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, + {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, +] + +[[package]] +name = "mkdocs-redirects" +version = "1.2.2" +description = "A MkDocs plugin for dynamic page redirects to prevent broken links" +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "mkdocs_redirects-1.2.2-py3-none-any.whl", hash = "sha256:7dbfa5647b79a3589da4401403d69494bd1f4ad03b9c15136720367e1f340ed5"}, + {file = "mkdocs_redirects-1.2.2.tar.gz", hash = "sha256:3094981b42ffab29313c2c1b8ac3969861109f58b2dd58c45fc81cd44bfa0095"}, +] + +[package.dependencies] +mkdocs = ">=1.1.1" + +[[package]] +name = "mkdocs-section-index" +version = "0.3.10" +description = "MkDocs plugin to allow clickable sections that lead to an index page" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "mkdocs_section_index-0.3.10-py3-none-any.whl", hash = "sha256:bc27c0d0dc497c0ebaee1fc72839362aed77be7318b5ec0c30628f65918e4776"}, + {file = "mkdocs_section_index-0.3.10.tar.gz", hash = "sha256:a82afbda633c82c5568f0e3b008176b9b365bf4bd8b6f919d6eff09ee146b9f8"}, +] + +[package.dependencies] +mkdocs = ">=1.2" + +[[package]] +name = "mkdocstrings" +version = "0.27.0" +description = "Automatic documentation from sources, for MkDocs." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "mkdocstrings-0.27.0-py3-none-any.whl", hash = "sha256:6ceaa7ea830770959b55a16203ac63da24badd71325b96af950e59fd37366332"}, + {file = "mkdocstrings-0.27.0.tar.gz", hash = "sha256:16adca6d6b0a1f9e0c07ff0b02ced8e16f228a9d65a37c063ec4c14d7b76a657"}, +] + +[package.dependencies] +click = ">=7.0" +Jinja2 = ">=2.11.1" +Markdown = ">=3.6" +MarkupSafe = ">=1.1" +mkdocs = ">=1.4" +mkdocs-autorefs = ">=1.2" +platformdirs = ">=2.2" +pymdown-extensions = ">=6.3" + +[package.extras] +crystal = ["mkdocstrings-crystal (>=0.3.4)"] +python = ["mkdocstrings-python (>=0.5.2)"] +python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] + +[[package]] +name = "mkdocstrings-python" +version = "1.13.0" +description = "A Python handler for mkdocstrings." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "mkdocstrings_python-1.13.0-py3-none-any.whl", hash = "sha256:b88bbb207bab4086434743849f8e796788b373bd32e7bfefbf8560ac45d88f97"}, + {file = "mkdocstrings_python-1.13.0.tar.gz", hash = "sha256:2dbd5757e8375b9720e81db16f52f1856bf59905428fd7ef88005d1370e2f64c"}, +] + +[package.dependencies] +griffe = ">=0.49" +mkdocs-autorefs = ">=1.2" +mkdocstrings = ">=0.26" + [[package]] name = "netconan" version = "0.12.3" @@ -1321,12 +1695,28 @@ version = "26.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] +[[package]] +name = "paginate" +version = "0.5.7" +description = "Divides large result sets into pages for easier browsing" +optional = false +python-versions = "*" +groups = ["docs"] +files = [ + {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, + {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, +] + +[package.extras] +dev = ["pytest", "tox"] +lint = ["black"] + [[package]] name = "pandas" version = "2.3.3" @@ -1468,7 +1858,7 @@ version = "1.0.3" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "pathspec-1.0.3-py3-none-any.whl", hash = "sha256:e80767021c1cc524aa3fb14bedda9c34406591343cc42797b386ce7b9354fb6c"}, {file = "pathspec-1.0.3.tar.gz", hash = "sha256:bac5cf97ae2c2876e2d25ebb15078eb04d76e4b98921ee31c6f85ade8b59444d"}, @@ -1486,7 +1876,7 @@ version = "4.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31"}, {file = "platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda"}, @@ -1687,7 +2077,7 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -1726,6 +2116,25 @@ tomlkit = ">=0.10.1" spelling = ["pyenchant (>=3.2,<4.0)"] testutils = ["gitpython (>3)"] +[[package]] +name = "pymdown-extensions" +version = "10.21" +description = "Extension pack for Python Markdown." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "pymdown_extensions-10.21-py3-none-any.whl", hash = "sha256:91b879f9f864d49794c2d9534372b10150e6141096c3908a455e45ca72ad9d3f"}, + {file = "pymdown_extensions-10.21.tar.gz", hash = "sha256:39f4a020f40773f6b2ff31d2cd2546c2c04d0a6498c31d9c688d2be07e1767d5"}, +] + +[package.dependencies] +markdown = ">=3.6" +pyyaml = "*" + +[package.extras] +extra = ["pygments (>=2.19.1)"] + [[package]] name = "pytest" version = "9.0.2" @@ -1756,7 +2165,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] +groups = ["main", "docs"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -1783,7 +2192,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -1860,13 +2269,28 @@ files = [ {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +description = "A custom YAML tag for referencing environment variables in YAML files." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04"}, + {file = "pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff"}, +] + +[package.dependencies] +pyyaml = "*" + [[package]] name = "requests" version = "2.32.5" description = "Python HTTP for Humans." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -1935,7 +2359,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main", "dev"] +groups = ["main", "dev", "docs"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -1966,6 +2390,54 @@ files = [ {file = "soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349"}, ] +[[package]] +name = "super-collections" +version = "0.6.2" +description = "file: README.md" +optional = false +python-versions = ">=3.8" +groups = ["docs"] +files = [ + {file = "super_collections-0.6.2-py3-none-any.whl", hash = "sha256:291b74d26299e9051d69ad9d89e61b07b6646f86a57a2f5ab3063d206eee9c56"}, + {file = "super_collections-0.6.2.tar.gz", hash = "sha256:0c8d8abacd9fad2c7c1c715f036c29f5db213f8cac65f24d45ecba12b4da187a"}, +] + +[package.dependencies] +hjson = "*" + +[package.extras] +test = ["pytest (>=7.0)", "pyyaml", "rich"] + +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=3.7" +groups = ["docs"] +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "termcolor" +version = "3.3.0" +description = "ANSI color formatting for output in terminal" +optional = false +python-versions = ">=3.10" +groups = ["docs"] +files = [ + {file = "termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5"}, + {file = "termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5"}, +] + +[package.extras] +tests = ["pytest", "pytest-cov"] + [[package]] name = "timezonefinder" version = "8.2.0" @@ -2234,7 +2706,7 @@ version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["dev", "docs"] files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, @@ -2246,6 +2718,49 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] +[[package]] +name = "watchdog" +version = "6.0.0" +description = "Filesystem events monitoring" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e6f0e77c9417e7cd62af82529b10563db3423625c5fce018430b249bf977f9e8"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:90c8e78f3b94014f7aaae121e6b909674df5b46ec24d6bebc45c44c56729af2a"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7631a77ffb1f7d2eefa4445ebbee491c720a5661ddf6df3498ebecae5ed375c"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11"}, + {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a0e56874cfbc4b9b05c60c8a1926fedf56324bb08cfbc188969777940aef3aa"}, + {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6439e374fc012255b4ec786ae3c4bc838cd7309a540e5fe0952d03687d8804e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2"}, + {file = "watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a"}, + {file = "watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680"}, + {file = "watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f"}, + {file = "watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + [[package]] name = "yamllint" version = "1.38.0" @@ -2271,5 +2786,5 @@ xlsx = ["openpyxl", "pandas"] [metadata] lock-version = "2.1" -python-versions = "^3.10" -content-hash = "5de4b112413d841214125ea53cdff500aa40b87272468f8dc72cdedf2210c6a2" +python-versions = ">=3.10,<3.14" +content-hash = "6bba954a14e852b48528814ed21319d92588272e02cf3769394bf50379acdb55" diff --git a/pyproject.toml b/pyproject.toml index 88cda0e8..b9470cfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,13 +2,16 @@ name = "circuit-maintenance-parser" version = "2.10.0" description = "Python library to parse Circuit Maintenance notifications and return a structured data back" -authors = ["Network to Code "] +authors = ["Network to Code, LLC "] license = "Apache-2.0" -homepage = "https://github.com/networktocode/circuit-maintenance-parser" +homepage = "https://circuit-maintenance-parser.readthedocs.io/" repository = "https://github.com/networktocode/circuit-maintenance-parser" +documentation = "https://circuit-maintenance-parser.readthedocs.io/" readme = "README.md" keywords = ["parser", "circuit", "maintenance"] classifiers = [ + "Intended Audience :: Developers", + "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -16,12 +19,12 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] include = [ - "README.md", "LICENSE", + "README.md", ] [tool.poetry.dependencies] -python = "^3.10" +python = ">=3.10,<3.14" click = ">=7.1, <9.0" pydantic = ">=1.10.4,<3" icalendar = "^5.0.0" @@ -63,6 +66,31 @@ types-chardet = "^5.0.4" pandas-stubs = "^2.3.2" coverage = "^7.6.12" +[tool.poetry.group.docs.dependencies] +# Rendering docs to HTML +mkdocs = "1.6.1" +# Embedding YAML files into Markdown documents as tables +markdown-data-tables = "1.0.0" +# Render custom markdown for version added/changed/remove notes +markdown-version-annotations = "1.0.1" +# Automatically generate some files as part of mkdocs build +mkdocs-gen-files = "0.5.0" +# Image lightboxing in mkdocs +mkdocs-glightbox = "0.4.0" +# Use Jinja2 templating in docs - see settings.md +mkdocs-macros-plugin = "1.3.7" +# Material for mkdocs theme +mkdocs-material = "9.6.15" +# Handle docs redirections +mkdocs-redirects = "1.2.2" +# Automatically handle index pages for docs sections +mkdocs-section-index = "0.3.10" +# Automatic documentation from sources, for MkDocs +mkdocstrings = "0.27.0" +# Python-specific extension to mkdocstrings +mkdocstrings-python = "1.13.0" +griffe = "1.1.1" + [tool.poetry.scripts] circuit-maintenance-parser = "circuit_maintenance_parser.cli:main" @@ -131,7 +159,7 @@ notes = """, """ [build-system] -requires = ["poetry_core>=1.0.0"] +requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" [tool.pytest.ini_options] @@ -144,10 +172,15 @@ addopts = "-vv --doctest-modules -p no:warnings --ignore-glob='*mock*'" [tool.towncrier] package = "circuit-maintenance-parser" directory = "changes" -filename = "docs/release_notes.md" +filename = "docs/admin/release_notes/version_X.Y.md" template = "towncrier_template.j2" start_string = "" -issue_format = "[#{issue}](https://github.com/networktocode/circuit_maintenance_parser/issues/{issue})" +issue_format = "[#{issue}](https://github.com/networktocode/circuit-maintenance-parser/issues/{issue})" + +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking Changes" +showcontent = true [[tool.towncrier.type]] directory = "security" diff --git a/tasks.py b/tasks.py index c90ca315..87f38ba6 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,8 @@ """Tasks for use with Invoke.""" import os +import re +from pathlib import Path from invoke import Collection, Exit from invoke import task as invoke_task @@ -9,8 +11,7 @@ def is_truthy(arg): """Convert "truthy" strings into Booleans. - Examples - -------- + Examples: >>> is_truthy('yes') True Args: @@ -29,7 +30,7 @@ def is_truthy(arg): # Use pyinvoke configuration for default values, see http://docs.pyinvoke.org/en/stable/concepts/configuration.html -# Variables may be overwritten in invoke.yml or by the environment variables INVOKE_PYNAUTOBOT_xxx +# Variables may be overwritten in invoke.yml or by the environment variables INVOKE_CIRCUIT-MAINTENANCE-PARSER_xxx namespace = Collection("circuit_maintenance_parser") namespace.configure( { @@ -39,7 +40,7 @@ def is_truthy(arg): "local": is_truthy(os.getenv("INVOKE_PARSER_LOCAL", "false")), "image_name": "circuit_maintenance_parser", "image_ver": os.getenv("INVOKE_PARSER_IMAGE_VER", "latest"), - "pwd": ".", + "pwd": Path(__file__).parent, } } ) @@ -65,13 +66,14 @@ def task_wrapper(function=None): return task_wrapper -def run_command(context, exec_cmd, port=None): +def run_command(context, exec_cmd, port=None, rm=True): """Wrapper to run the invoke task commands. Args: context ([invoke.task]): Invoke task object. exec_cmd ([str]): Command to run. port (int): Used to serve local docs. + rm (bool): Whether to remove the container after running the command. Returns: result (obj): Contains Invoke result from running task. @@ -85,18 +87,21 @@ def run_command(context, exec_cmd, port=None): ) if port: result = context.run( - f"docker run -it -p {port} -v {context.circuit_maintenance_parser.pwd}:/local {context.circuit_maintenance_parser.image_name}:{context.circuit_maintenance_parser.image_ver} sh -c '{exec_cmd}'", + f"docker run -it {'--rm' if rm else ''} -p {port} -v {context.circuit_maintenance_parser.pwd}:/local {context.circuit_maintenance_parser.image_name}:{context.circuit_maintenance_parser.image_ver} sh -c '{exec_cmd}'", pty=True, ) else: result = context.run( - f"docker run -it -v {context.circuit_maintenance_parser.pwd}:/local {context.circuit_maintenance_parser.image_name}:{context.circuit_maintenance_parser.image_ver} sh -c '{exec_cmd}'", + f"docker run -it {'--rm' if rm else ''} -v {context.circuit_maintenance_parser.pwd}:/local {context.circuit_maintenance_parser.image_name}:{context.circuit_maintenance_parser.image_ver} sh -c '{exec_cmd}'", pty=True, ) return result +# ------------------------------------------------------------------------------ +# BUILD +# ------------------------------------------------------------------------------ @task( help={ "cache": "Whether to use Docker's cache when building images (default enabled)", @@ -123,6 +128,26 @@ def build(context, cache=True, force_rm=False, hide=False): ) +@task +def generate_packages(context): + """Generate all Python packages inside docker and copy the file locally under dist/.""" + command = "poetry build" + run_command(context, command) + + +@task( + help={ + "check": ( + "If enabled, check for outdated dependencies in the poetry.lock file, " + "instead of generating a new one. (default: disabled)" + ) + } +) +def lock(context, check=False): + """Generate poetry.lock inside the library container.""" + run_command(context, f"poetry {'check' if check else 'lock --no-update'}") + + @task def clean(context): """Remove the project specific image.""" @@ -145,9 +170,34 @@ def rebuild(context): @task -def pytest(context, args=""): +def coverage(context): + """Run the coverage report against pytest.""" + exec_cmd = "coverage run --source=circuit_maintenance_parser -m pytest" + run_command(context, exec_cmd) + run_command(context, "coverage report") + run_command(context, "coverage html") + + +@task( + help={ + "pattern": "Only run tests which match the given substring. Can be used multiple times.", + "label": "Module path to run (e.g., tests/unit/test_foo.py). Can be used multiple times.", + }, + iterable=["pattern", "label"], +) +def pytest(context, pattern=None, label=None): """Run pytest test cases.""" - exec_cmd = f"pytest {args}" + exec_cmd = "pytest -vv --doctest-modules circuit_maintenance_parser/ && coverage run --source=circuit_maintenance_parser -m pytest && coverage report" + run_command(context, exec_cmd) + + doc_test_cmd = "pytest -vv --doctest-modules circuit_maintenance_parser/" + pytest_cmd = "coverage run --source=circuit_maintenance_parser -m pytest" + if pattern: + pytest_cmd += "".join([f" -k {_pattern}" for _pattern in pattern]) + if label: + pytest_cmd += "".join([f" {_label}" for _label in label]) + coverage_cmd = "coverage report" + exec_cmd = " && ".join([doc_test_cmd, pytest_cmd, coverage_cmd]) run_command(context, exec_cmd) @@ -229,25 +279,58 @@ def cli(context): context.run(f"{dev}", pty=True) -@task -def tests(context): +@task( + help={ + "lint-only": "Only run linters; unit tests will be excluded. (default: False)", + } +) +def tests(context, lint_only=False): """Run all tests for the specified name and Python version. Args: context (obj): Used to run specific commands + lint_only (bool): If True, only run linters and skip unit tests. """ + # If we are not running locally, start the docker containers so we don't have to for each test + # Sorted loosely from fastest to slowest + print("Running ruff...") ruff(context) - pylint(context) + print("Running yamllint...") yamllint(context) - pytest(context) - + print("Running poetry check...") + lock(context, check=True) + print("Running pylint...") + pylint(context) + print("Running mkdocs...") + build_and_check_docs(context) + if not lint_only: + print("Running unit tests...") + pytest(context) print("All tests have passed!") +@task +def build_and_check_docs(context): + """Build documentation and test the configuration.""" + command = "mkdocs build --no-directory-urls --strict" + run_command(context, command) + + # Check for the existence of a release notes file for the current version if it's not a prerelease. + version = context.run("poetry version --short", hide=True) + match = re.match(r"^(\d+)\.(\d+)\.\d+$", version.stdout.strip()) + if match: + major = match.group(1) + minor = match.group(2) + release_notes_file = Path(__file__).parent / "docs" / "admin" / "release_notes" / f"version_{major}.{minor}.md" + if not release_notes_file.exists(): + print(f"Release notes file `version_{major}.{minor}.md` does not exist.") + raise Exit(code=1) + + @task def docs(context): """Build and serve docs locally for development.""" - exec_cmd = "mkdocs serve -v --dev-addr=0.0.0.0:8001" + exec_cmd = "mkdocs serve -v" run_command(context, exec_cmd, port="8001:8001") diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..c66cd71b --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests package.""" diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 5fdf1a10..ea3f8b92 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1 +1 @@ -"""Init for tests.""" +"""Unit tests package.""" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3417f960..667ed817 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,7 +12,10 @@ def maintenance_data(): return { "account": "12345000", "maintenance_id": "VNOC-1-99999999999", - "circuits": [{"circuit_id": "123", "impact": "NO-IMPACT"}, {"circuit_id": "456"}], + "circuits": [ + {"circuit_id": "123", "impact": "NO-IMPACT"}, + {"circuit_id": "456"}, + ], "organizer": "myemail@example.com", "provider": "A random NSP", "sequence": 1, @@ -23,7 +26,9 @@ def maintenance_data(): "summary": "This is a maintenance notification", "uid": "VNOC-1-99999999999", "_metadata": Metadata( - provider="some provider", processor="some processor", parsers=["some parser 1", "some parser 2"] + provider="some provider", + processor="some processor", + parsers=["some parser 1", "some parser 2"], ), } diff --git a/tests/unit/test_basics.py b/tests/unit/test_basics.py new file mode 100644 index 00000000..b7264224 --- /dev/null +++ b/tests/unit/test_basics.py @@ -0,0 +1,29 @@ +"""Basic tests that do not require Circuit-Maintenance-Parser.""" + +import os +import re +import unittest + +import toml + + +class TestDocsReleaseNotes(unittest.TestCase): + """Test that mkdocs has the release notes for the current version.""" + + def test_version_file_found(self): + """Verify that if the current version has no letters, which would see in alpha or beta has an associated release note file.""" + parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + poetry_path = os.path.join(parent_path, "pyproject.toml") + project_version = toml.load(poetry_path)["tool"]["poetry"]["version"] + + docs_path = os.path.join(parent_path, "docs") + release_notes_files = [file for file in os.listdir(f"{docs_path}/admin/release_notes/") if file.endswith(".md")] + version_pattern = re.compile(r"^(\d+)\.(\d+)\.\d+$") + + match = version_pattern.match(project_version) + # If there is no match, then it is likely an alpha or beta version and we can skip this test. + if match: + major, minor = match.groups() + version_str = f"version_{major}.{minor}.md" + if version_str not in release_notes_files: + self.fail(f"Release note file for version {version_str} not found in release notes folder.") diff --git a/towncrier_template.j2 b/towncrier_template.j2 index 33d996d8..a4b46c8f 100644 --- a/towncrier_template.j2 +++ b/towncrier_template.j2 @@ -9,7 +9,7 @@ This document describes all new features and changes in the release. The format - Changes to compatibility with Nautobot and/or other apps, libraries etc. {% if render_title %} -## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/networktocode/circuit_maintenance_parser/releases/tag/v{{ versiondata.version}}) +## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/networktocode/circuit-maintenance-parser/releases/tag/v{{ versiondata.version}}) {% endif %} {% for section, _ in sections.items() %}