Skip to content

Commit 6083062

Browse files
kanterovkanterov
authored
[Python] Add generated code for pipelines (#2639)
## Changes Add generated code for pipelines support introduced in #2574
1 parent 7ffdd1b commit 6083062

26 files changed

Lines changed: 2157 additions & 13 deletions

‎experimental/python/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ fmt:
66

77
docs:
88
# Python 3.12+ is needed for get_overloads
9-
uv run --python 3.12 sphinx-build docs docs/_output --fail-on-warning --show-traceback --nitpicky --fresh-env --keep-going
9+
uv run --python 3.12 sphinx-build docs docs/_output --show-traceback --nitpicky --fresh-env --keep-going
1010

1111
lint:
1212
# check if lock matches the project metadata

‎experimental/python/databricks/bundles/pipelines/__init__.py‎

Lines changed: 305 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,309 @@
1-
from databricks.bundles.pipelines._models.pipeline import Pipeline, PipelineParam
2-
31
__all__ = [
2+
"Adlsgen2Info",
3+
"Adlsgen2InfoDict",
4+
"Adlsgen2InfoParam",
5+
"AwsAttributes",
6+
"AwsAttributesDict",
7+
"AwsAttributesParam",
8+
"AwsAvailability",
9+
"AwsAvailabilityParam",
10+
"AzureAttributes",
11+
"AzureAttributesDict",
12+
"AzureAttributesParam",
13+
"AzureAvailability",
14+
"AzureAvailabilityParam",
15+
"ClusterLogConf",
16+
"ClusterLogConfDict",
17+
"ClusterLogConfParam",
18+
"DayOfWeek",
19+
"DayOfWeekParam",
20+
"DbfsStorageInfo",
21+
"DbfsStorageInfoDict",
22+
"DbfsStorageInfoParam",
23+
"EbsVolumeType",
24+
"EbsVolumeTypeParam",
25+
"EventLogSpec",
26+
"EventLogSpecDict",
27+
"EventLogSpecParam",
28+
"FileLibrary",
29+
"FileLibraryDict",
30+
"FileLibraryParam",
31+
"Filters",
32+
"FiltersDict",
33+
"FiltersParam",
34+
"GcpAttributes",
35+
"GcpAttributesDict",
36+
"GcpAttributesParam",
37+
"GcpAvailability",
38+
"GcpAvailabilityParam",
39+
"GcsStorageInfo",
40+
"GcsStorageInfoDict",
41+
"GcsStorageInfoParam",
42+
"IngestionConfig",
43+
"IngestionConfigDict",
44+
"IngestionConfigParam",
45+
"IngestionGatewayPipelineDefinition",
46+
"IngestionGatewayPipelineDefinitionDict",
47+
"IngestionGatewayPipelineDefinitionParam",
48+
"IngestionPipelineDefinition",
49+
"IngestionPipelineDefinitionDict",
50+
"IngestionPipelineDefinitionParam",
51+
"InitScriptInfo",
52+
"InitScriptInfoDict",
53+
"InitScriptInfoParam",
54+
"LocalFileInfo",
55+
"LocalFileInfoDict",
56+
"LocalFileInfoParam",
57+
"LogAnalyticsInfo",
58+
"LogAnalyticsInfoDict",
59+
"LogAnalyticsInfoParam",
60+
"MavenLibrary",
61+
"MavenLibraryDict",
62+
"MavenLibraryParam",
63+
"NotebookLibrary",
64+
"NotebookLibraryDict",
65+
"NotebookLibraryParam",
66+
"Notifications",
67+
"NotificationsDict",
68+
"NotificationsParam",
469
"Pipeline",
70+
"PipelineCluster",
71+
"PipelineClusterAutoscale",
72+
"PipelineClusterAutoscaleDict",
73+
"PipelineClusterAutoscaleMode",
74+
"PipelineClusterAutoscaleModeParam",
75+
"PipelineClusterAutoscaleParam",
76+
"PipelineClusterDict",
77+
"PipelineClusterParam",
78+
"PipelineDict",
79+
"PipelineLibrary",
80+
"PipelineLibraryDict",
81+
"PipelineLibraryParam",
582
"PipelineParam",
83+
"PipelinePermission",
84+
"PipelinePermissionDict",
85+
"PipelinePermissionLevel",
86+
"PipelinePermissionLevelParam",
87+
"PipelinePermissionParam",
88+
"ReportSpec",
89+
"ReportSpecDict",
90+
"ReportSpecParam",
91+
"RestartWindow",
92+
"RestartWindowDict",
93+
"RestartWindowParam",
94+
"RunAs",
95+
"RunAsDict",
96+
"RunAsParam",
97+
"S3StorageInfo",
98+
"S3StorageInfoDict",
99+
"S3StorageInfoParam",
100+
"SchemaSpec",
101+
"SchemaSpecDict",
102+
"SchemaSpecParam",
103+
"TableSpec",
104+
"TableSpecDict",
105+
"TableSpecParam",
106+
"TableSpecificConfig",
107+
"TableSpecificConfigDict",
108+
"TableSpecificConfigParam",
109+
"TableSpecificConfigScdType",
110+
"TableSpecificConfigScdTypeParam",
111+
"VolumesStorageInfo",
112+
"VolumesStorageInfoDict",
113+
"VolumesStorageInfoParam",
114+
"WorkspaceStorageInfo",
115+
"WorkspaceStorageInfoDict",
116+
"WorkspaceStorageInfoParam",
6117
]
118+
119+
120+
from databricks.bundles.compute._models.adlsgen2_info import (
121+
Adlsgen2Info,
122+
Adlsgen2InfoDict,
123+
Adlsgen2InfoParam,
124+
)
125+
from databricks.bundles.compute._models.aws_attributes import (
126+
AwsAttributes,
127+
AwsAttributesDict,
128+
AwsAttributesParam,
129+
)
130+
from databricks.bundles.compute._models.aws_availability import (
131+
AwsAvailability,
132+
AwsAvailabilityParam,
133+
)
134+
from databricks.bundles.compute._models.azure_attributes import (
135+
AzureAttributes,
136+
AzureAttributesDict,
137+
AzureAttributesParam,
138+
)
139+
from databricks.bundles.compute._models.azure_availability import (
140+
AzureAvailability,
141+
AzureAvailabilityParam,
142+
)
143+
from databricks.bundles.compute._models.cluster_log_conf import (
144+
ClusterLogConf,
145+
ClusterLogConfDict,
146+
ClusterLogConfParam,
147+
)
148+
from databricks.bundles.compute._models.dbfs_storage_info import (
149+
DbfsStorageInfo,
150+
DbfsStorageInfoDict,
151+
DbfsStorageInfoParam,
152+
)
153+
from databricks.bundles.compute._models.ebs_volume_type import (
154+
EbsVolumeType,
155+
EbsVolumeTypeParam,
156+
)
157+
from databricks.bundles.compute._models.gcp_attributes import (
158+
GcpAttributes,
159+
GcpAttributesDict,
160+
GcpAttributesParam,
161+
)
162+
from databricks.bundles.compute._models.gcp_availability import (
163+
GcpAvailability,
164+
GcpAvailabilityParam,
165+
)
166+
from databricks.bundles.compute._models.gcs_storage_info import (
167+
GcsStorageInfo,
168+
GcsStorageInfoDict,
169+
GcsStorageInfoParam,
170+
)
171+
from databricks.bundles.compute._models.init_script_info import (
172+
InitScriptInfo,
173+
InitScriptInfoDict,
174+
InitScriptInfoParam,
175+
)
176+
from databricks.bundles.compute._models.local_file_info import (
177+
LocalFileInfo,
178+
LocalFileInfoDict,
179+
LocalFileInfoParam,
180+
)
181+
from databricks.bundles.compute._models.log_analytics_info import (
182+
LogAnalyticsInfo,
183+
LogAnalyticsInfoDict,
184+
LogAnalyticsInfoParam,
185+
)
186+
from databricks.bundles.compute._models.maven_library import (
187+
MavenLibrary,
188+
MavenLibraryDict,
189+
MavenLibraryParam,
190+
)
191+
from databricks.bundles.compute._models.s3_storage_info import (
192+
S3StorageInfo,
193+
S3StorageInfoDict,
194+
S3StorageInfoParam,
195+
)
196+
from databricks.bundles.compute._models.volumes_storage_info import (
197+
VolumesStorageInfo,
198+
VolumesStorageInfoDict,
199+
VolumesStorageInfoParam,
200+
)
201+
from databricks.bundles.compute._models.workspace_storage_info import (
202+
WorkspaceStorageInfo,
203+
WorkspaceStorageInfoDict,
204+
WorkspaceStorageInfoParam,
205+
)
206+
from databricks.bundles.pipelines._models.day_of_week import DayOfWeek, DayOfWeekParam
207+
from databricks.bundles.pipelines._models.event_log_spec import (
208+
EventLogSpec,
209+
EventLogSpecDict,
210+
EventLogSpecParam,
211+
)
212+
from databricks.bundles.pipelines._models.file_library import (
213+
FileLibrary,
214+
FileLibraryDict,
215+
FileLibraryParam,
216+
)
217+
from databricks.bundles.pipelines._models.filters import (
218+
Filters,
219+
FiltersDict,
220+
FiltersParam,
221+
)
222+
from databricks.bundles.pipelines._models.ingestion_config import (
223+
IngestionConfig,
224+
IngestionConfigDict,
225+
IngestionConfigParam,
226+
)
227+
from databricks.bundles.pipelines._models.ingestion_gateway_pipeline_definition import (
228+
IngestionGatewayPipelineDefinition,
229+
IngestionGatewayPipelineDefinitionDict,
230+
IngestionGatewayPipelineDefinitionParam,
231+
)
232+
from databricks.bundles.pipelines._models.ingestion_pipeline_definition import (
233+
IngestionPipelineDefinition,
234+
IngestionPipelineDefinitionDict,
235+
IngestionPipelineDefinitionParam,
236+
)
237+
from databricks.bundles.pipelines._models.notebook_library import (
238+
NotebookLibrary,
239+
NotebookLibraryDict,
240+
NotebookLibraryParam,
241+
)
242+
from databricks.bundles.pipelines._models.notifications import (
243+
Notifications,
244+
NotificationsDict,
245+
NotificationsParam,
246+
)
247+
from databricks.bundles.pipelines._models.pipeline import (
248+
Pipeline,
249+
PipelineDict,
250+
PipelineParam,
251+
)
252+
from databricks.bundles.pipelines._models.pipeline_cluster import (
253+
PipelineCluster,
254+
PipelineClusterDict,
255+
PipelineClusterParam,
256+
)
257+
from databricks.bundles.pipelines._models.pipeline_cluster_autoscale import (
258+
PipelineClusterAutoscale,
259+
PipelineClusterAutoscaleDict,
260+
PipelineClusterAutoscaleParam,
261+
)
262+
from databricks.bundles.pipelines._models.pipeline_cluster_autoscale_mode import (
263+
PipelineClusterAutoscaleMode,
264+
PipelineClusterAutoscaleModeParam,
265+
)
266+
from databricks.bundles.pipelines._models.pipeline_library import (
267+
PipelineLibrary,
268+
PipelineLibraryDict,
269+
PipelineLibraryParam,
270+
)
271+
from databricks.bundles.pipelines._models.pipeline_permission import (
272+
PipelinePermission,
273+
PipelinePermissionDict,
274+
PipelinePermissionParam,
275+
)
276+
from databricks.bundles.pipelines._models.pipeline_permission_level import (
277+
PipelinePermissionLevel,
278+
PipelinePermissionLevelParam,
279+
)
280+
from databricks.bundles.pipelines._models.report_spec import (
281+
ReportSpec,
282+
ReportSpecDict,
283+
ReportSpecParam,
284+
)
285+
from databricks.bundles.pipelines._models.restart_window import (
286+
RestartWindow,
287+
RestartWindowDict,
288+
RestartWindowParam,
289+
)
290+
from databricks.bundles.pipelines._models.run_as import RunAs, RunAsDict, RunAsParam
291+
from databricks.bundles.pipelines._models.schema_spec import (
292+
SchemaSpec,
293+
SchemaSpecDict,
294+
SchemaSpecParam,
295+
)
296+
from databricks.bundles.pipelines._models.table_spec import (
297+
TableSpec,
298+
TableSpecDict,
299+
TableSpecParam,
300+
)
301+
from databricks.bundles.pipelines._models.table_specific_config import (
302+
TableSpecificConfig,
303+
TableSpecificConfigDict,
304+
TableSpecificConfigParam,
305+
)
306+
from databricks.bundles.pipelines._models.table_specific_config_scd_type import (
307+
TableSpecificConfigScdType,
308+
TableSpecificConfigScdTypeParam,
309+
)

‎experimental/python/databricks/bundles/pipelines/_models/day_of_week.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
from enum import Enum
2+
from typing import Literal
3+
4+
5+
class DayOfWeek(Enum):
6+
"""
7+
:meta private: [EXPERIMENTAL]
8+
9+
Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).
10+
If not specified all days of the week will be used.
11+
"""
12+
13+
MONDAY = "MONDAY"
14+
TUESDAY = "TUESDAY"
15+
WEDNESDAY = "WEDNESDAY"
16+
THURSDAY = "THURSDAY"
17+
FRIDAY = "FRIDAY"
18+
SATURDAY = "SATURDAY"
19+
SUNDAY = "SUNDAY"
20+
21+
22+
DayOfWeekParam = (
23+
Literal[
24+
"MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", "SUNDAY"
25+
]
26+
| DayOfWeek
27+
)

0 commit comments

Comments
 (0)