Merge pull request #41 from SpecterOps/feature/ziprelease

martinsohn · web-flow · commit 954b3522753e · 2026-01-02T19:38:27.000+01:00
Added ability to export query bundle to zip + create release/download
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,21 +1,16 @@
-name: Query syntax validation
+name: Generate query bundle and push release
 on:
-  pull_request:
-    branches: [ 'main' ]
-
   workflow_dispatch:
-
-permissions:
-  contents: write
+  push:
+    branches:
+      - main
 
 jobs:
   test:
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.head_ref }}
     - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
@@ -38,42 +33,46 @@ jobs:
         name: test-report
         path: test-report.md
 
-  build:
+
+  release:
     runs-on: ubuntu-latest
     needs: test
     steps:
       - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
           python-version: "3.10"
+        
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-  
-      - name: Convert queries into single json
+    
+      - name: Convert queries into single zip file
         run: |
-          python utilities/python/convert.py ./queries ./Queries.json
+          python utilities/python/convert.py ./queries ./Queries.zip --file-format zip
 
-      - name: Configure Git
+      - name: Convert queries into single json file
         run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
-      - name: Commit if changed
+          python utilities/python/convert.py ./queries ./Queries.json
+      
+      - name: Set metadata
+        id: release_meta
         run: |
-          git add ./Queries.json
-          if git diff --staged --quiet; then
-            echo "No changes to commit"
-          else
-            git commit -m "Update combined queries"
-            git push
-          fi
+          release_date="$(date -u +%Y-%m-%d)"
+          echo "release_date=${release_date}" >> "$GITHUB_OUTPUT"
+          echo "release_tag=queries-${release_date}" >> "$GITHUB_OUTPUT"
+          echo "release_name=Queries ${release_date}" >> "$GITHUB_OUTPUT"
+    
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.release_meta.outputs.release_tag }}
+          name: ${{ steps.release_meta.outputs.release_name }}
+          files: |
+            Queries.zip
+            Queries.json
+          body: | 
+            This release contains all queries exported as JSON and bundled in a single file. The compressed .zip file can be uploaded to BloodHound to bulk-import all queries.
+          draft: true
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,37 @@
+name: Query syntax validation
+on:
+  pull_request:
+    branches: [ 'main' ]
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Test queries with pytest
+      run: |
+        pytest tests/test_cypher_syntax.py
+
+    - name: Add test report to summary
+      run: cat test-report.md >> $GITHUB_STEP_SUMMARY
+
+    - name: Upload test report 
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-report
+        path: test-report.md
diff --git a/queries/All Azure VMs with a tied Managed Identity.yml b/queries/All Azure VMs with a tied Managed Identity.yml
@@ -9,5 +9,5 @@ query: |-
   MATCH p=(:AZVM)-[:AZManagedIdentity]->(n) 
   RETURN p
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/All direct Controllers of MS Graph.yml b/queries/All direct Controllers of MS Graph.yml
@@ -10,5 +10,5 @@ query: |-
   WHERE g.displayname = "MICROSOFT GRAPH"
   RETURN p
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/All privileged Azure Service Principals.yml b/queries/All privileged Azure Service Principals.yml
@@ -10,5 +10,5 @@ query: |-
   WHERE r.displayname =~ '(?i)Global Administrator|User Administrator|Cloud Application Administrator|Authentication Policy Administrator|Exchange Administrator|Helpdesk Administrator|PRIVILEGED AUTHENTICATION ADMINISTRATOR|Domain Name Administrator|Hybrid Identity Administrator|External Identity Provider Administrator|Privileged Role Administrator|Partner Tier2 Support|Application Administrator|Directory Synchronization Accounts' 
   RETURN p
 revision: 1
-resources: -
+resources:
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/Disabled Tier Zero High Value principals - AD.yml b/queries/Disabled Tier Zero High Value principals - AD.yml
@@ -1,4 +1,4 @@
-name: Disabled Tier Zero / High Value principals
+name: Disabled Tier Zero / High Value principals (AD)
 guid: d65a801f-d3ef-4b7e-8030-99ebfd6dad12
 prebuilt: true
 platforms: Active Directory
diff --git a/queries/Disabled Tier Zero High Value principals - AZ.yml b/queries/Disabled Tier Zero High Value principals - AZ.yml
@@ -1,4 +1,4 @@
-name: Disabled Tier Zero / High Value principals
+name: Disabled Tier Zero / High Value principals (AZ)
 guid: 860d5c2d-84fe-4c85-80de-e0a9badbd0e7
 prebuilt: true
 platforms: Azure
diff --git a/queries/Locations of Owned objects - AD.yml b/queries/Locations of Owned objects - AD.yml
@@ -1,4 +1,4 @@
-name: Locations of Owned objects
+name: Locations of Owned objects (AD)
 guid: c88bfab4-3da0-4b36-b71d-7b324ebd2243
 prebuilt: false
 platforms: Active Directory
diff --git a/queries/Locations of Owned objects - AZ.yml b/queries/Locations of Owned objects - AZ.yml
@@ -1,4 +1,4 @@
-name: Locations of Owned objects
+name: Locations of Owned objects (AZ)
 guid: 350b8b8a-ea4c-44f3-874b-c9316de6c41b
 prebuilt: false
 platforms: Azure
diff --git a/queries/Owners of Azure Applications.yml b/queries/Owners of Azure Applications.yml
@@ -9,5 +9,5 @@ query: |-
   MATCH p = (n)-[r:AZOwns]->(g:AZApp) 
   RETURN p
 revision: 1
-resources: -
+resources:
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/Owners of Azure Subscriptions.yml b/queries/Owners of Azure Subscriptions.yml
@@ -11,5 +11,5 @@ query: |-
   RETURN p
   LIMIT 1000
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/Shortest Paths from Azure Users to Azure Keyvaults.yml b/queries/Shortest Paths from Azure Users to Azure Keyvaults.yml
@@ -9,5 +9,5 @@ query: |-
   MATCH p = shortestPath((n:AZUser)-[:AZ_ATTACK_PATHS*..]->(g:AZKeyVault))
   RETURN p
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/Shortest Paths from Azure Users to Azure VMs.yml b/queries/Shortest Paths from Azure Users to Azure VMs.yml
@@ -9,5 +9,5 @@ query: |-
   MATCH p = shortestPath((m:AZUser)-[:AZ_ATTACK_PATHS*..]->(n:AZVM)) 
   RETURN p
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/Shortest Paths from Owned Azure Users to Azure Keyvaults.yml b/queries/Shortest Paths from Owned Azure Users to Azure Keyvaults.yml
@@ -10,5 +10,5 @@ query: |-
   WHERE m.system_tags CONTAINS 'owned'
   RETURN p
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/queries/Shortest Paths from Owned Azure Users to Azure VMs.yml b/queries/Shortest Paths from Owned Azure Users to Azure VMs.yml
@@ -10,5 +10,5 @@ query: |-
   WHERE m.system_tags CONTAINS 'owned'
   RETURN p
 revision: 1
-resources: -
+resources: 
 acknowledgements: Daniel Scheidt, @theluemmel
diff --git a/utilities/python/convert.py b/utilities/python/convert.py
@@ -1,16 +1,62 @@
-from typing_extensions import Annotated
+from typing_extensions import Annotated, Optional
 from pathlib import Path
 from schema import CypherQuery
+from io import TextIOWrapper
 import json
-import glob
 import typer
 import yaml
+import zipfile
+
+
+ALLOWED_FORMATS = ["json", "zip"]
 
 app = typer.Typer()
 
 
+def validate_format(value: str):
+    if value.lower() not in ALLOWED_FORMATS:
+        raise typer.BadParameter(f"Only {', '.join(ALLOWED_FORMATS)} is allowed")
+    return value
+
+
+class QueryBundle:
+    def __init__(self, queries: list[CypherQuery]):
+        self.queries = queries
+
+    @staticmethod
+    def load_query(cypher_query: Path) -> CypherQuery:
+        with open(cypher_query, "r") as yaml_file:
+            yaml_obj = yaml.safe_load(yaml_file)
+            return CypherQuery(**yaml_obj)
+
+    @classmethod
+    def from_path(cls, input_dir: Path) -> "QueryBundle":
+        cypher_queries = list(input_dir.rglob("*.yml"))
+        queries = [
+            QueryBundle.load_query(cypher_query) for cypher_query in cypher_queries
+        ]
+        return cls(queries)
+
+    def to_json(self, output_file: TextIOWrapper) -> None:
+        all_objects = [query.model_dump() for query in self.queries]
+        output_file.write(json.dumps(all_objects, indent=2))
+
+    def to_zip(self, output_file: TextIOWrapper) -> None:
+        with zipfile.ZipFile(
+            file=output_file.name,
+            mode="w",
+            compression=zipfile.ZIP_DEFLATED,
+            compresslevel=9,
+        ) as archive:
+            for query in self.queries:
+                archive.writestr(
+                    zinfo_or_arcname=f"{query.name}.json",
+                    data=query.model_dump_json().encode(),
+                )
+
+
 @app.command()
-def to_json(
+def convert(
     input_dir: Annotated[
         Path,
         typer.Argument(
@@ -21,19 +67,25 @@ def to_json(
             resolve_path=True,
         ),
     ],
-    output_file: Annotated[typer.FileTextWrite, typer.Argument()]
+    output_file: Annotated[typer.FileTextWrite, typer.Argument()],
+    file_format: Annotated[
+        Optional[str],
+        typer.Option(help="Format for export (json/zip)", callback=validate_format),
+    ] = "json",
 ):
-    cypher_queries = glob.glob(f"{input_dir}/**/*.yml", recursive=True)
-    typer.echo(f"Converting Queries {len(cypher_queries)} to combined JSON")
-    all_objects = []
-    for cypher_query in cypher_queries:
-        with open(cypher_query, "r") as yaml_file:
-            yaml_obj = yaml.safe_load(yaml_file)
-            query = CypherQuery(**yaml_obj)
-            all_objects.append(query.model_dump())
 
-    output_file.write(json.dumps(all_objects, indent=2))
-    typer.echo(f"Finished converting Cypher queries to JSON to {output_file.name}")
+    typer.echo(f"Converting queries to {file_format} output")
+    cypher_queries = QueryBundle.from_path(input_dir=input_dir)
+
+    if file_format == "json":
+        cypher_queries.to_json(output_file)
+
+    else:
+        cypher_queries.to_zip(output_file)
+
+    typer.echo(
+        f"Finished converting {len(cypher_queries.queries)} Cypher queries ({file_format}) to {output_file.name}"
+    )
 
 
 if __name__ == "__main__":

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-name: Disabled Tier Zero / High Value principals`
	`1`	`+name: Disabled Tier Zero / High Value principals (AD)`
`2`	`2`	`guid: d65a801f-d3ef-4b7e-8030-99ebfd6dad12`
`3`	`3`	`prebuilt: true`
`4`	`4`	`platforms: Active Directory`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-name: Locations of Owned objects`
	`1`	`+name: Locations of Owned objects (AD)`
`2`	`2`	`guid: c88bfab4-3da0-4b36-b71d-7b324ebd2243`
`3`	`3`	`prebuilt: false`
`4`	`4`	`platforms: Active Directory`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-name: Locations of Owned objects`
	`1`	`+name: Locations of Owned objects (AZ)`
`2`	`2`	`guid: 350b8b8a-ea4c-44f3-874b-c9316de6c41b`
`3`	`3`	`prebuilt: false`
`4`	`4`	`platforms: Azure`