feat(pydevlake): migrate Python plugin framework to Pydantic v2 / SQLModel 0.0.38

backend/python/README.md

@@ -25,7 +25,7 @@ can be found in [here](./pydevlake).
 ## Create the plugin project
 
 
-Make sure you have [Poetry](https://python-poetry.org/docs/#installation) installed.
+Make sure you have [Poetry](https://python-poetry.org/docs/#installation) installed (recommended pinned version: `2.4.1`, for example `pipx install "poetry==2.4.1"`).
 Move to `python/plugins` and execute `poetry new myplugin`.
 This will generate a new directory for your plugin.
 

backend/python/plugins/azuredevops/azuredevops/models.py

@@ -30,26 +30,26 @@
 
 class AzureDevOpsConnection(Connection):
     token: SecretStr
-    organization: Optional[str]
+    organization: Optional[str] = None
 
 
 class GitRepositoryConfig(ScopeConfig):
-    refdiff: Optional[RefDiffOptions]
-    deployment_pattern: Optional[re.Pattern]
-    production_pattern: Optional[re.Pattern]
+    refdiff: Optional[RefDiffOptions] = None
+    deployment_pattern: Optional[re.Pattern] = None
+    production_pattern: Optional[re.Pattern] = None
     # Optional pattern with capture group to extract environment name from job/stage names
     # Example: r'(?:deploy|predeploy)[_-](.+?)(?:[_-](?:helm|terraform))?$' extracts 'xxxx-prod' from 'deploy_xxxx-prod_helm'
-    environment_pattern: Optional[re.Pattern]
+    environment_pattern: Optional[re.Pattern] = None
 
 
 class GitRepository(ToolScope, table=True):
     url: str
-    remote_url: Optional[str]
-    default_branch: Optional[str]
+    remote_url: Optional[str] = None
+    default_branch: Optional[str] = None
     project_id: str
     org_id: str
     parent_repository_url: Optional[str] = Field(source='/parentRepository/url')
-    provider: Optional[str]
+    provider: Optional[str] = None
     updated_date: datetime.datetime = Field(source='/project/lastUpdateTime')
 
     def is_external(self):
@@ -63,20 +63,20 @@ class PRStatus(Enum):
         Completed = "completed"
 
     pull_request_id: int = Field(primary_key=True)
-    description: Optional[str]
+    description: Optional[str] = None
     status: PRStatus
     created_by_id: str = Field(source='/createdBy/id')
     created_by_name: str = Field(source='/createdBy/displayName')
     creation_date: datetime.datetime
-    closed_date: Optional[datetime.datetime]
+    closed_date: Optional[datetime.datetime] = None
     source_commit_sha: str = Field(source='/lastMergeSourceCommit/commitId')
     target_commit_sha: str = Field(source='/lastMergeTargetCommit/commitId')
     merge_commit_sha: Optional[str] = Field(source='/lastMergeCommit/commitId')
-    url: Optional[str]
+    url: Optional[str] = None
     type: Optional[str] = Field(source='/labels/0/name')  # TODO: Add regex to scope config
-    title: Optional[str]
-    target_ref_name: Optional[str]
-    source_ref_name: Optional[str]
+    title: Optional[str] = None
+    target_ref_name: Optional[str] = None
+    source_ref_name: Optional[str] = None
     fork_repo_id: Optional[str] = Field(source='/forkSource/repository/id')
 
 
@@ -112,10 +112,10 @@ def __str__(self) -> str:
     id: int = Field(primary_key=True)
     name: str = Field(source='/definition/name')
     queue_time: Optional[datetime.datetime] = Field(source='/queueTime')
-    start_time: Optional[datetime.datetime]
-    finish_time: Optional[datetime.datetime]
+    start_time: Optional[datetime.datetime] = None
+    finish_time: Optional[datetime.datetime] = None
     status: BuildStatus
-    result: Optional[BuildResult]
+    result: Optional[BuildResult] = None
     source_branch: str
     source_version: str
     display_title: Optional[str] = Field(source='/triggerInfo/ci.message')
@@ -145,10 +145,10 @@ def __str__(self) -> str:
     id: str = Field(primary_key=True)
     build_id: str = Field(primary_key=True)
     name: str
-    start_time: Optional[datetime.datetime]
-    finish_time: Optional[datetime.datetime]
+    start_time: Optional[datetime.datetime] = None
+    finish_time: Optional[datetime.datetime] = None
     state: JobState
-    result: Optional[JobResult]
-    identifier: Optional[str]
-    type: Optional[str]
+    result: Optional[JobResult] = None
+    identifier: Optional[str] = None
+    type: Optional[str] = None
     parent_id: Optional[str] = Field(source='/parentId')

backend/python/plugins/azuredevops/pyproject.toml

@@ -26,7 +26,7 @@ pydevlake = { path = "../../pydevlake", develop = true }
 
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.2.2"
+pytest = "^9.1"
 
 [build-system]
 requires = ["poetry-core"]

backend/python/pydevlake/pydevlake/__init__.py

@@ -27,14 +27,26 @@ def Field(*args, primary_key: bool = False, auto_increment: Optional[bool] = Non
     """
     A wrapper around sqlmodel.Field that adds a source parameter.
     """
-    schema_extra = kwargs.get('schema_extra', {})
+    schema_extra = kwargs.pop('schema_extra', {})
     if source is not None:
         schema_extra['source'] = source
     if primary_key:
         schema_extra['primaryKey'] = True
     if auto_increment is not None:
         schema_extra['autoIncrement'] = auto_increment
-    return _Field(*args, **kwargs, primary_key=primary_key, schema_extra=schema_extra)
+    # Only pass primary_key if True; SQLModel 0.0.38 rejects it alongside sa_column
+    if primary_key:
+        field = _Field(*args, **kwargs, primary_key=True, schema_extra=schema_extra)
+    elif schema_extra:
+        field = _Field(*args, **kwargs, schema_extra=schema_extra)
+    else:
+        field = _Field(*args, **kwargs)
+    # SQLModel 0.0.38 doesn't propagate schema_extra to FieldInfo.json_schema_extra,
+    # so we set it manually for consumers like autoextract and model_json_schema.
+    if schema_extra:
+        field.json_schema_extra = schema_extra
+    return field
+
 
 
 from .model import ToolModel, ToolScope, DomainScope, Connection, ScopeConfig, DomainType, domain_id

backend/python/pydevlake/pydevlake/domain_layer/code.py

@@ -33,11 +33,11 @@ class PullRequest(DomainModel, table=True):
     url: str
     author_name: str
     author_id: str
-    parent_pr_id: Optional[str]
+    parent_pr_id: Optional[str] = None
     pull_request_key: int
     created_date: datetime
-    merged_date: Optional[datetime]
-    closed_date: Optional[datetime]
+    merged_date: Optional[datetime] = None
+    closed_date: Optional[datetime] = None
     type: str
     component: str
     merge_commit_sha: str
@@ -78,8 +78,8 @@ class Commit(NoPKModel, table=True):
     __tablename__ = 'commits'
     sha: str = Field(primary_key=True)
     additions: str
-    deletions: Optional[str]
-    dev_eq: Optional[str]
+    deletions: Optional[str] = None
+    dev_eq: Optional[str] = None
     message: str
     author_name: str
     author_email: int
@@ -144,12 +144,12 @@ class Repo(DomainScope, table=True):
     __tablename__ = "repos"
     name: str
     url: str
-    description: Optional[str]
-    owner_id: Optional[str]
-    language: Optional[str]
-    forked_from: Optional[str]
-    created_date: Optional[datetime]
-    updated_date: Optional[datetime]
+    description: Optional[str] = None
+    owner_id: Optional[str] = None
+    language: Optional[str] = None
+    forked_from: Optional[str] = None
+    created_date: Optional[datetime] = None
+    updated_date: Optional[datetime] = None
     deleted: bool
 
 

backend/python/pydevlake/pydevlake/domain_layer/devops.py

@@ -51,26 +51,26 @@ class CICDPipeline(DomainModel, table=True):
     __tablename__ = 'cicd_pipelines'
 
     name: str
-    cicd_scope_id: Optional[str]
+    cicd_scope_id: Optional[str] = None
 
-    status: Optional[CICDStatus]
-    result: Optional[CICDResult]
-    original_status: Optional[str]
-    original_result: Optional[str]
+    status: Optional[CICDStatus] = None
+    result: Optional[CICDResult] = None
+    original_status: Optional[str] = None
+    original_result: Optional[str] = None
 
-    created_date: Optional[datetime]
-    started_date: Optional[datetime]
-    queued_date: Optional[datetime]
-    finished_date: Optional[datetime]
+    created_date: Optional[datetime] = None
+    started_date: Optional[datetime] = None
+    queued_date: Optional[datetime] = None
+    finished_date: Optional[datetime] = None
 
-    duration_sec: Optional[float]
-    queued_duration_sec: Optional[float]
+    duration_sec: Optional[float] = None
+    queued_duration_sec: Optional[float] = None
 
-    type: Optional[CICDType]
-    environment: Optional[CICDEnvironment]
+    type: Optional[CICDType] = None
+    environment: Optional[CICDEnvironment] = None
 
-    display_title: Optional[str]
-    url: Optional[str]
+    display_title: Optional[str] = None
+    url: Optional[str] = None
 
 
 class CiCDPipelineCommit(NoPKModel, table=True):
@@ -80,17 +80,17 @@ class CiCDPipelineCommit(NoPKModel, table=True):
     branch: str
     repo_id: str
     repo_url: str
-    display_title: Optional[str]
-    url: Optional[str]
+    display_title: Optional[str] = None
+    url: Optional[str] = None
 
 
 class CicdScope(DomainScope):
     __tablename__ = 'cicd_scopes'
     name: str
-    description: Optional[str]
-    url: Optional[str]
-    createdDate: Optional[datetime]
-    updatedDate: Optional[datetime]
+    description: Optional[str] = None
+    url: Optional[str] = None
+    createdDate: Optional[datetime] = None
+    updatedDate: Optional[datetime] = None
 
 
 class CICDTask(DomainModel, table=True):
@@ -100,18 +100,18 @@ class CICDTask(DomainModel, table=True):
     pipeline_id: str
     cicd_scope_id: str
 
-    result: Optional[CICDResult]
-    status: Optional[CICDStatus]
-    original_status: Optional[str]
-    original_result: Optional[str]
+    result: Optional[CICDResult] = None
+    status: Optional[CICDStatus] = None
+    original_status: Optional[str] = None
+    original_result: Optional[str] = None
 
-    type: Optional[CICDType]
-    environment: Optional[CICDEnvironment]
+    type: Optional[CICDType] = None
+    environment: Optional[CICDEnvironment] = None
 
-    created_date: Optional[datetime]
-    queued_date: Optional[datetime]
-    started_date: Optional[datetime]
-    finished_date: Optional[datetime]
+    created_date: Optional[datetime] = None
+    queued_date: Optional[datetime] = None
+    started_date: Optional[datetime] = None
+    finished_date: Optional[datetime] = None
 
     duration_sec: float
-    queued_duration_sec: Optional[float]
+    queued_duration_sec: Optional[float] = None

backend/python/pydevlake/pydevlake/extractor.py

@@ -14,13 +14,26 @@
 # limitations under the License.
 
 
-from typing import Type
+from typing import Type, get_args, get_origin, Union
+from types import NoneType
 
+from pydantic_core import PydanticUndefined
 from jsonpointer import resolve_pointer, JsonPointerException
 
 from pydevlake import ToolModel
 
 
+def _is_optional(field_info) -> bool:
+    """Check if a field's annotation allows None (is Optional)."""
+    annotation = field_info.annotation
+    if annotation is None:
+        return True
+    origin = get_origin(annotation)
+    if origin is Union:
+        return NoneType in get_args(annotation)
+    return False
+
+
 def autoextract(json: dict, model_cls: Type[ToolModel]) -> ToolModel:
     """
     Automatically extract a tool model from a json object.
@@ -42,18 +55,33 @@ class DummyModel(ToolModel):
         model = autoextract(json, DummyModel)
     """
     attributes = {}
-    for field in model_cls.__fields__.values():
-        pointer = field.field_info.extra.get('source')
+    for field_name, field_info in model_cls.model_fields.items():
+        # In SQLModel 0.0.38, schema_extra is stored in field_info.json_schema_extra
+        extra = field_info.json_schema_extra or {}
+        if isinstance(extra, dict):
+            pointer = extra.get('source')
+        else:
+            pointer = None
 
         if pointer:
-            if field.required:
+            # A field is considered optional if its annotation allows None
+            is_optional = _is_optional(field_info)
+            if not is_optional:
                 try:
                     value = resolve_pointer(json, pointer)
                 except JsonPointerException:
-                    raise ValueError(f"Missing required value for field {field.name} at {pointer}")
+                    raise ValueError(f"Missing required value for field {field_name} at {pointer}")
             else:
-                value = resolve_pointer(json, pointer, field.default)
+                default = field_info.default if field_info.default is not PydanticUndefined else None
+                try:
+                    value = resolve_pointer(json, pointer, default)
+                except JsonPointerException:
+                    value = default
         else:
-            value = json.get(field.name) or json.get(field.alias)
-        attributes[field.name] = value
-    return model_cls(**attributes)
+            alias = field_info.alias
+            value = json.get(field_name) or json.get(alias) if alias else json.get(field_name)
+        attributes[field_name] = value
+    # Use model_validate so values are coerced (e.g. str -> datetime/Enum).
+    # SQLModel table=True models skip validation on __init__, which would leave
+    # raw strings uncoerced under Pydantic v2, so validate explicitly here.
+    return model_cls.model_validate(attributes)

backend/python/pydevlake/pydevlake/ipc.py

@@ -39,7 +39,7 @@ def open_send_channel() -> TextIO:
     def send_output(send_ch: TextIO, obj: object):
         if not isinstance(obj, Message):
             raise Exception(f"Not a message: {obj}")
-        send_ch.write(obj.json(exclude_none=True, by_alias=True))
+        send_ch.write(obj.model_dump_json(exclude_none=True, by_alias=True))
         send_ch.write('\n')
         send_ch.flush()