Skip to content

Commit 4a51bd4

Browse files
zephyrzillazephyrzilla
committed
feat: build workflow from dict, add lib versioning
1 parent cf40f9a commit 4a51bd4

9 files changed

Lines changed: 224 additions & 33 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ name: Run CI
33
on:
44
push:
55
branches: [ main ]
6+
tags: [ "v*" ]
67
pull_request:
78
branches: [ main ]
89
types: [ opened, synchronize, reopened, ready_for_review ]
@@ -75,6 +76,37 @@ jobs:
7576
- name: Setup dev environment
7677
run: make setup-dev
7778

79+
# ---- VERSION CHECK ----
80+
- name: Validate version format
81+
run: |
82+
VERSION=$(python3 -c "import re; m=re.search(r'__version__\s*=\s*\"(\d+\.\d+\.\d+(?:\.(?:post|dev)\d+|(?:a|b|rc)\d+)?)\"', open('sygra/__init__.py').read()); print(m.group(1) if m else 'INVALID')")
83+
if [ "$VERSION" = "INVALID" ]; then
84+
echo "[ERROR] __version__ in sygra/__init__.py is not valid PEP 440 (e.g. X.Y.Z, X.Y.Z.postN)"
85+
exit 1
86+
fi
87+
echo "[SUCCESS] Version: $VERSION"
88+
89+
- name: Validate tag matches __version__ (tagged builds only)
90+
if: startsWith(github.ref, 'refs/tags/v')
91+
run: |
92+
TAG_VERSION="${GITHUB_REF#refs/tags/v}"
93+
CODE_VERSION=$(python3 -c "import re; print(re.search(r'__version__\s*=\s*\"(.+?)\"', open('sygra/__init__.py').read()).group(1))")
94+
if [ "$TAG_VERSION" != "$CODE_VERSION" ]; then
95+
echo "[WARNING] Tag version ($TAG_VERSION) != __version__ ($CODE_VERSION) in sygra/__init__.py"
96+
echo ""
97+
echo " If you used 'make bump-version' + 'git tag' (CLI flow):"
98+
echo " Run: make bump-version V=$TAG_VERSION"
99+
echo " Then re-tag and push."
100+
echo ""
101+
echo " If you created a Release via GitHub UI:"
102+
echo " This is expected — publish.yml patches the version at build time."
103+
echo " The publish will succeed; this CI check is informational only."
104+
echo ""
105+
echo " To avoid this warning, run 'make bump-version V=X.Y.Z' before tagging."
106+
exit 1
107+
fi
108+
echo "[SUCCESS] Tag v$TAG_VERSION matches __version__"
109+
78110
# ---- FORMAT ----
79111
- name: Run formatter
80112
run: make check-format

‎.github/workflows/publish.yml‎

Lines changed: 103 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,13 @@ name: Publish to PyPI
33
on:
44
push:
55
tags:
6-
- "v*" # Triggers when a new GitHub Tag is published eg: v1.2.3
6+
- "v*" # Triggers on tag push (e.g. git push origin --tags)
7+
release:
8+
types: [ published ] # Triggers on GitHub UI Release creation
9+
10+
concurrency:
11+
group: publish-${{ github.ref }}
12+
cancel-in-progress: false
713

814
jobs:
915
publish:
@@ -27,37 +33,111 @@ jobs:
2733
- name: Install dependencies
2834
run: make setup-dev
2935

30-
- name: Install versioning deps
36+
# ---- VERSION EXTRACTION (handles push-tag + release events, with/without v prefix) ----
37+
- name: Extract and validate version from tag
3138
run: |
32-
python -m pip install --upgrade pip
33-
pip install tomlkit
39+
# Get tag name — works for both push (refs/tags/v1.2.3) and release events
40+
if [ "${{ github.event_name }}" = "release" ]; then
41+
TAG="${{ github.event.release.tag_name }}"
42+
else
43+
TAG="${GITHUB_REF#refs/tags/}"
44+
fi
45+
46+
# Strip optional 'v' prefix: v2.1.0 → 2.1.0, 2.1.0 → 2.1.0
47+
VERSION="${TAG#v}"
48+
49+
# Validate PEP 440 format
50+
python3 -c "
51+
import re, sys
52+
v = '$VERSION'
53+
if not re.fullmatch(r'\d+\.\d+\.\d+([.](post|dev)\d+|(a|b|rc)\d+)?', v):
54+
print(f'❌ Tag \'{TAG}\' does not contain a valid PEP 440 version (extracted: \'{v}\')')
55+
print(' Expected formats: X.Y.Z, X.Y.Z.postN, X.Y.Z.devN, X.Y.ZaN, X.Y.ZbN, X.Y.ZrcN')
56+
sys.exit(1)
57+
"
58+
59+
# Export for all subsequent steps
60+
echo "VERSION=$VERSION" >> $GITHUB_ENV
61+
echo "✅ Extracted version: $VERSION (from tag: $TAG, event: ${{ github.event_name }})"
3462
35-
- name: Set version from GitHub tag
63+
# ---- PRE-FLIGHT: Check version is not already burnt on PyPI ----
64+
- name: Check version is available on PyPI
3665
run: |
37-
# Extract tag like "v1.2.3" → "1.2.3"
38-
export VERSION="${GITHUB_REF#refs/tags/v}"
39-
echo "Setting [project].version to $VERSION"
40-
python - << 'PY'
41-
from pathlib import Path
42-
from tomlkit import parse, dumps
43-
import os
44-
version = os.environ["VERSION"]
45-
p = Path('pyproject.toml')
46-
doc = parse(p.read_text(encoding='utf-8'))
47-
# Update PEP 621 version
48-
if 'project' in doc:
49-
doc['project']['version'] = version
50-
p.write_text(dumps(doc), encoding='utf-8')
51-
PY
66+
python3 -c "
67+
import urllib.request, urllib.error, sys
68+
try:
69+
urllib.request.urlopen('https://pypi.org/pypi/sygra/${{ env.VERSION }}/json')
70+
print('❌ Version ${{ env.VERSION }} already exists on PyPI — this version is burnt.')
71+
print(' Options:')
72+
print(' • Use a .postN suffix: v${{ env.VERSION }}.post1')
73+
print(' • Bump to the next version: make bump-version V=X.Y.Z')
74+
sys.exit(1)
75+
except urllib.error.HTTPError as e:
76+
if e.code == 404:
77+
print('✅ Version ${{ env.VERSION }} is available on PyPI')
78+
sys.exit(0)
79+
print(f'⚠️ PyPI check returned HTTP {e.code} — proceeding anyway')
80+
except Exception as e:
81+
print(f'⚠️ PyPI check failed ({e}) — proceeding anyway')
82+
"
83+
84+
# ---- PATCH VERSION ----
85+
- name: Set version in source files
86+
run: |
87+
echo "Setting __version__ to $VERSION"
88+
89+
# Patch sygra/__init__.py (hatchling reads version from here)
90+
python3 -c "
91+
import re, pathlib
92+
p = pathlib.Path('sygra/__init__.py')
93+
p.write_text(re.sub(r'^__version__ = \".*\"', '__version__ = \"$VERSION\"', p.read_text(), count=1, flags=re.MULTILINE))
94+
"
5295
96+
# Patch [tool.poetry] version so Poetry stays consistent
97+
python3 -c "
98+
import re, pathlib
99+
p = pathlib.Path('pyproject.toml')
100+
p.write_text(re.sub(r'(\[tool\.poetry\]\nversion\s*=\s*)\"[^\"]*\"', r'\g<1>\"$VERSION\"', p.read_text(), count=1))
101+
"
102+
103+
# Verify
104+
grep '__version__' sygra/__init__.py
105+
echo "✅ Version set to $VERSION"
106+
107+
- name: Validate version consistency
108+
run: |
109+
BUILT_VERSION=$(python3 -c "import re; m=re.search(r'__version__\s*=\s*\"(.+?)\"', open('sygra/__init__.py').read()); print(m.group(1))")
110+
if [ "$VERSION" != "$BUILT_VERSION" ]; then
111+
echo "❌ Version mismatch: tag=$VERSION, __init__.py=$BUILT_VERSION"
112+
exit 1
113+
fi
114+
echo "✅ Version validated: $VERSION"
115+
116+
# ---- BUILD ----
53117
- name: Build package
54118
run: make build
55119

120+
- name: Validate built artifacts
121+
run: |
122+
ls dist/
123+
if ! ls dist/sygra-${VERSION}-*.whl 1>/dev/null 2>&1; then
124+
echo "❌ No wheel found for version $VERSION in dist/"
125+
ls dist/
126+
exit 1
127+
fi
128+
if ! ls dist/sygra-${VERSION}.tar.gz 1>/dev/null 2>&1; then
129+
echo "❌ No sdist found for version $VERSION in dist/"
130+
ls dist/
131+
exit 1
132+
fi
133+
echo "✅ Built artifacts verified for version $VERSION"
134+
135+
# ---- PUBLISH ----
56136
- name: Publish to PyPI
57137
env:
58138
TWINE_USERNAME: __token__
59139
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
60140
run: |
61-
python -m pip install --upgrade pip
62-
pip install twine
63-
python -m twine upload --repository pypi dist/* --verbose
141+
python3 -m pip install --upgrade pip
142+
python3 -m pip install twine
143+
python3 -m twine upload --repository pypi dist/* --verbose

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,13 @@ repos:
6161

6262
- repo: local
6363
hooks:
64+
- id: version-check
65+
name: Validate __version__ is valid PEP 440
66+
entry: python3 -c "import re, sys; content=open('sygra/__init__.py').read(); m=re.search(r'__version__\s*=\s*\"(\d+\.\d+\.\d+(?:\.(?:post|dev)\d+|(?:a|b|rc)\d+)?)\"', content); sys.exit(0) if m else (print('[ERROR] Invalid __version__ in sygra/__init__.py. Must be PEP 440 (e.g. X.Y.Z, X.Y.Z.postN)') or sys.exit(1))"
67+
language: system
68+
files: ^sygra/__init__\.py$
69+
pass_filenames: false
70+
6471
- id: pytest
6572
name: Run tests with pytest
6673
entry: uv run pytest -q tests

‎Makefile‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ studio-build: ## Build the Studio frontend (only if not already built)
5555
echo "📦 Building Studio frontend..."; \
5656
cd $(STUDIO_FRONTEND_DIR) && npm install && npm run build; \
5757
else \
58-
echo "✅ Studio frontend already built. Use 'make studio-rebuild' to force rebuild."; \
58+
echo "[SUCCESS] Studio frontend already built. Use 'make studio-rebuild' to force rebuild."; \
5959
fi
6060

6161
.PHONY: studio-rebuild
@@ -112,6 +112,29 @@ docs-serve: ## Serve documentation locally
112112
# BUILDING & PUBLISHING
113113
########################################################################################################################
114114

115+
.PHONY: version
116+
version: ## Show current version
117+
@python3 -c "import re; m=re.search(r'__version__\s*=\s*\"(.+?)\"', open('sygra/__init__.py').read()); print(m.group(1))"
118+
119+
.PHONY: bump-version
120+
bump-version: ## Bump version: make bump-version V=2.1.0 (or V=2.1.0.post1)
121+
@if [ -z "$(V)" ]; then \
122+
echo "[ERROR] Usage: make bump-version V=X.Y.Z[.postN]"; \
123+
exit 1; \
124+
fi
125+
@if ! echo "$(V)" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.(post|dev)[0-9]+|(a|b|rc)[0-9]+)?$$'; then \
126+
echo "[ERROR] Invalid version: $(V). Must be PEP 440 (e.g. X.Y.Z, X.Y.Z.postN)"; \
127+
exit 1; \
128+
fi
129+
@python3 -c "import re, pathlib; p=pathlib.Path('sygra/__init__.py'); p.write_text(re.sub(r'^__version__ = \".*\"', '__version__ = \"$(V)\"', p.read_text(), count=1, flags=re.MULTILINE))"
130+
@python3 -c "import re, pathlib; p=pathlib.Path('pyproject.toml'); p.write_text(re.sub(r'(\[tool\.poetry\]\nversion\s*=\s*)\"[^\"]*\"', r'\1\"$(V)\"', p.read_text(), count=1))"
131+
@echo "[SUCCESS] Version bumped to $(V) in sygra/__init__.py and pyproject.toml"
132+
@echo " Next steps:"
133+
@echo " 1. git add sygra/__init__.py pyproject.toml"
134+
@echo " 2. git commit -m 'Bump version to $(V)'"
135+
@echo " 3. git tag v$(V)"
136+
@echo " 4. git push origin main --tags"
137+
115138
.PHONY: build
116139
build: ## Build package
117140
$(UV) run $(PYTHON) -m build

‎pyproject.toml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
44

55
[project]
66
name = "sygra"
7-
version = "1.0.0"
7+
dynamic = ["version"]
88
description = "Graph-oriented Synthetic data generation Pipeline library"
99
readme = "README.md"
1010
requires-python = ">=3.9,<3.12,!=3.9.7"
@@ -90,6 +90,9 @@ Releases = "https://github.com/ServiceNow/SyGra/releases"
9090
Issues = "https://github.com/ServiceNow/SyGra/issues"
9191
Discussions = "https://github.com/ServiceNow/SyGra/discussions"
9292

93+
[tool.hatch.version]
94+
path = "sygra/__init__.py"
95+
9396
[tool.hatch.build.targets.wheel]
9497
packages = ["sygra", "studio"]
9598
include = [
@@ -156,5 +159,8 @@ module = ["tests.*"]
156159
disallow_untyped_defs = false
157160
check_untyped_defs = false
158161

162+
[tool.poetry]
163+
version = "2.0.0.post1"
164+
159165
[tool.poetry.group.dev.dependencies]
160166
uvicorn = "^0.38.0"

‎sygra/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@
122122
DATA_UTILS_AVAILABLE = False
123123

124124

125-
__version__ = "1.0.0"
125+
__version__ = "2.0.0.post1"
126126
__author__ = "SyGra Team"
127127
__description__ = "Graph-oriented Synthetic data generation Pipeline library"
128128

‎sygra/configuration/loader.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
1+
from __future__ import annotations
2+
13
import os
24
from pathlib import Path
3-
from typing import Any, Union
5+
from typing import TYPE_CHECKING, Any, Union
46

57
import yaml
68

9+
if TYPE_CHECKING:
10+
from sygra.workflow import Workflow
11+
712
try:
813
from sygra.core.dataset.dataset_config import DataSourceConfig, OutputConfig # noqa: F401
914
from sygra.core.graph.graph_config import GraphConfig # noqa: F401
1015
from sygra.utils import utils
11-
from sygra.workflow import AutoNestedDict
16+
from sygra.workflow import AutoNestedDict # noqa: F401
1217

1318
UTILS_AVAILABLE = True
1419
except ImportError:
@@ -42,12 +47,12 @@ def load(self, config_path: Union[str, Path, dict[str, Any]]) -> dict[str, Any]:
4247

4348
return config
4449

45-
def load_and_create(self, config_path: Union[str, Path, dict[str, Any]]):
50+
def load_and_create(self, config_path: Union[str, Path, dict[str, Any]]) -> Workflow:
4651
"""Load config and create appropriate Workflow or Graph object."""
4752
config = self.load(config_path)
4853

4954
# Import here to avoid circular imports
50-
from ..workflow import Workflow
55+
from ..workflow import AutoNestedDict, Workflow
5156

5257
workflow = Workflow()
5358
workflow._config = AutoNestedDict.convert_dict(config)
@@ -60,8 +65,16 @@ def load_and_create(self, config_path: Union[str, Path, dict[str, Any]]):
6065

6166
if isinstance(config_path, (str, Path)):
6267
workflow.name = Path(config_path).parent.name
68+
workflow._is_existing_task = True
6369
else:
6470
workflow.name = config.get("task_name", "loaded_workflow")
71+
# Mark as existing task if config has nodes defined
72+
if config.get("graph_config", {}).get("nodes"):
73+
workflow._is_existing_task = True
74+
75+
# Track node count from loaded config
76+
if "graph_config" in config and "nodes" in config["graph_config"]:
77+
workflow._node_counter = len(config["graph_config"]["nodes"])
6578

6679
return workflow
6780

‎sygra/workflow/__init__.py‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,38 @@ def __init__(self, name: Optional[str] = None):
192192

193193
self._load_existing_config_if_present()
194194

195+
@classmethod
196+
def from_config(cls, config: Union[str, Path, dict[str, Any]]) -> "Workflow":
197+
"""
198+
Create a Workflow from a configuration dictionary or YAML file path.
199+
200+
Args:
201+
config: A dictionary containing the full workflow configuration,
202+
or a path to a YAML configuration file.
203+
204+
Returns:
205+
Workflow: A configured Workflow instance ready for execution.
206+
207+
Examples:
208+
# From dictionary
209+
>>> config = {
210+
... "data_config": {"source": {"type": "disk", "file_path": "data.json", "file_format": "json"}},
211+
... "graph_config": {
212+
... "nodes": {"llm_1": {"node_type": "llm", "model": {"name": "gpt-4o"}, "prompt": [{"user": "Hello {text}"}]}},
213+
... "edges": [{"from": "START", "to": "llm_1"}, {"from": "llm_1", "to": "END"}]
214+
... }
215+
... }
216+
>>> workflow = Workflow.from_config(config)
217+
>>> workflow.run(num_records=1)
218+
219+
# From YAML file
220+
>>> workflow = Workflow.from_config("tasks/examples/text_to_speech/graph_config.yaml")
221+
"""
222+
from sygra.configuration import ConfigLoader
223+
224+
loader = ConfigLoader()
225+
return loader.load_and_create(config)
226+
195227
def _load_existing_config_if_present(self):
196228
"""Load existing task configuration if this appears to be a task path."""
197229
if self.name and (os.path.exists(self.name) or "/" in self.name or "\\" in self.name):
@@ -863,8 +895,7 @@ def _execute_existing_task(
863895
if kwargs.get("quality_only", False):
864896
executor = JudgeQualityTaskExecutor(args, kwargs.get("quality_config"))
865897
else:
866-
executor = DefaultTaskExecutor(args)
867-
BaseTaskExecutor.__init__(executor, args, modified_config)
898+
executor = DefaultTaskExecutor(args, modified_config)
868899

869900
result = executor.execute()
870901
logger.info(f"Successfully executed task: {task_name}")

‎uv.lock‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)