Skip to content

Feature/overide sniffed csv settings #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexbourret merged 21 commits into from
May 18, 2026
+325 −39
Merged

Feature/overide sniffed csv settings #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
af9ff5d
add option overide for csv decoding
alexbourret Mar 11, 2026
34f3125
beta.3
alexbourret Mar 11, 2026
9fe53a8
removing mtls from recipe and connector
alexbourret Mar 12, 2026
3bf56ea
Moving mtls to secure and normal presets
alexbourret Mar 12, 2026
1875efa
adding warnings
alexbourret Mar 13, 2026
bd62302
beta.4
alexbourret Mar 13, 2026
4cf3490
update tags and category
alexbourret Mar 16, 2026
c5a0c20
removing beta tag
alexbourret Mar 24, 2026
38af12d
storing key+certificats as passwords
alexbourret Apr 2, 2026
6252b88
using temp file for using key+certificats stored in presets
alexbourret Apr 2, 2026
734dd6c
adding mtls keys to data that should not be displayed
alexbourret Apr 2, 2026
987c597
beta.5
alexbourret Apr 2, 2026
581a267
Clarify the UI
alexbourret Apr 16, 2026
1cfed44
Add normalization for RSA keys
alexbourret Apr 16, 2026
59597ce
fix to accept RSA keys
alexbourret Apr 16, 2026
b807d85
removing comments
alexbourret Apr 16, 2026
185915b
fix get_endpoint_parameters
alexbourret May 13, 2026
bba5751
UI: moving above the csv parameters names
alexbourret May 13, 2026
a39cb71
fix log
alexbourret May 18, 2026
ed6537d
Merge pull request #68 from dataiku/feature/accept-mtls-keys-instead-…
alexbourret May 18, 2026
e0f799a
Merge pull request #67 from dataiku/feature/move-mtls-settings-to-pre…
alexbourret May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 62 additions & 8 deletions custom-recipes/api-connect/recipe.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,25 +292,79 @@
"defaultValue": false
},
{
"name": "use_mtls",
"label": "Use mTLS",
"name": "force_csv_parameters",
"label": "Force CSV parameters",
"description": "",
"type": "BOOLEAN",
"defaultValue": false
},
{
"name": "mtls_certificate_path",
"label": "Path to certificate",
"name": "csv_delimiter",
"label": "Delimiter",
"description": "",
"type": "STRING",
"visibilityCondition": "model.use_mtls==true"
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "mtls_key_path",
"label": "Path to key",
"name": "csv_doublequote",
"label": "Double quote",
"description": "",
"type": "SELECT",
"selectChoices":[
{"value": null, "label": "Auto detect"},
{"value": "double_quote", "label": "Double quote"},
{"value": "not_double_quote", "label": "No double quote"}
],
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_escapechar",
"label": "Escape char",
"description": "",
"type": "STRING",
"visibilityCondition": "model.use_mtls==true"
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_lineterminator",
"label": "Line terminator",
"description": "",
"type": "STRING",
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_quotechar",
"label": "Quote char",
"description": "",
"type": "STRING",
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_quoting",
"label": "Quote",
"description": "",
"type": "SELECT",
"selectChoices":[
{"value": null, "label": "Auto detect"},
{"value": 0, "label": "Minimal"},
{"value": 1, "label": "All"},
{"value": 2, "label": "Non numeric"},
{"value": 3, "label": "None"},
{"value": 4, "label": "Strings"},
{"value": 5, "label": "Not null"}
],
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_skipinitialspace",
"label": "Skip initial space",
"description": "",
"type": "SELECT",
"selectChoices":[
{"value": null, "label": "Auto detect"},
{"value": "skip", "label": "Skip"},
{"value": "not_skip", "label": "Do not skip"}
],
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "redirect_auth_header",
Expand Down
21 changes: 21 additions & 0 deletions parameter-sets/credential/parameter-set.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,27 @@
"label": "User key/values",
"description": "User defined keys/values that can be used later in url, query string...",
"type": "KEY_VALUE_LIST"
},
{
"name": "use_mtls",
"label": "Use mTLS",
"description": "",
"type": "BOOLEAN",
"defaultValue": false
},
{
"name": "mtls_certificate_path",
"label": "Path to certificate",
"description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----",
"type": "PASSWORD",
"visibilityCondition": "model.use_mtls==true"
},
{
"name": "mtls_key_path",
"label": "Path to key",
"description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----",
"type": "PASSWORD",
"visibilityCondition": "model.use_mtls==true"
}
]
}
27 changes: 27 additions & 0 deletions parameter-sets/secure-basic/parameter-set.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,33 @@
"label": "NTLM"
}
]
},
{
"name": "use_mtls",
"label": "Use mTLS",
"description": "",
"type": "BOOLEAN",
"defaultValue": false
},
{
"type": "SEPARATOR",
"label": "Warning",
"description": "Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.",
"visibilityCondition": "model.use_mtls==true"
},
{
"name": "mtls_certificate_path",
"label": "Path to certificate",
"description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----",
"type": "PASSWORD",
"visibilityCondition": "model.use_mtls==true"
},
{
"name": "mtls_key_path",
"label": "Path to key",
"description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----",
"type": "PASSWORD",
"visibilityCondition": "model.use_mtls==true"
}
]
}
27 changes: 27 additions & 0 deletions parameter-sets/secure-oauth/parameter-set.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,33 @@
"label": "Domain",
"description": "",
"type": "STRING"
},
{
"name": "use_mtls",
"label": "Use mTLS",
"description": "",
"type": "BOOLEAN",
"defaultValue": false
},
{
"type": "SEPARATOR",
"label": "Warning",
"description": "1 - Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.\n2 - Because the OAuth flow is not controled by the plugin, mTLS cannot be used for the retrieving the access token itself.",
"visibilityCondition": "model.use_mtls==true"
},
{
"name": "mtls_certificate_path",
"label": "Path to certificate",
"description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----",
"type": "PASSWORD",
"visibilityCondition": "model.use_mtls==true"
},
{
"name": "mtls_key_path",
"label": "Path to key",
"description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----",
"type": "PASSWORD",
"visibilityCondition": "model.use_mtls==true"
}
]
}
3 changes: 1 addition & 2 deletions plugin.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
"description": "Retrieve data from any REST API",
"author": "Dataiku (Alex Bourret)",
"icon": "icon-rocket",
"category": "Connect",
"tags": ["API", "Recipe", "Dataset"],
"tags": ["Connector"],
"url": "https://www.dataiku.com/product/plugins/api-connect/",
"licenseInfo": "Apache Software License",
"recipesCategory": "visual"
Expand Down
70 changes: 62 additions & 8 deletions python-connectors/api-connect_dataset/connector.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,25 +239,79 @@
"defaultValue": false
},
{
"name": "use_mtls",
"name": "force_csv_parameters",
"label": " ",
"description": "Use mTLS",
"description": "Force CSV parameters",
"type": "BOOLEAN",
"defaultValue": false
},
{
"name": "mtls_certificate_path",
"label": "Path to certificate",
"name": "csv_delimiter",
"label": "Delimiter",
"description": "",
"type": "STRING",
"visibilityCondition": "model.use_mtls==true"
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "mtls_key_path",
"label": "Path to key",
"name": "csv_doublequote",
"label": "Double quote",
"description": "",
"type": "SELECT",
"selectChoices":[
{"value": null, "label": "Auto detect"},
{"value": "double_quote", "label": "Double quote"},
{"value": "not_double_quote", "label": "No double quote"}
],
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_escapechar",
"label": "Escape char",
"description": "",
"type": "STRING",
"visibilityCondition": "model.use_mtls==true"
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_lineterminator",
"label": "Line terminator",
"description": "",
"type": "STRING",
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_quotechar",
"label": "Quote char",
"description": "",
"type": "STRING",
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_quoting",
"label": "Quote",
"description": "",
"type": "SELECT",
"selectChoices":[
{"value": null, "label": "Auto detect"},
{"value": 0, "label": "Minimal"},
{"value": 1, "label": "All"},
{"value": 2, "label": "Non numeric"},
{"value": 3, "label": "None"},
{"value": 4, "label": "Strings"},
{"value": 5, "label": "Not null"}
],
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "csv_skipinitialspace",
"label": "Skip initial space",
"description": "",
"type": "SELECT",
"selectChoices":[
{"value": null, "label": "Auto detect"},
{"value": "skip", "label": "Skip"},
{"value": "not_skip", "label": "Do not skip"}
],
"visibilityCondition": "model.force_csv_parameters==true"
},
{
"name": "redirect_auth_header",
Expand Down
3 changes: 2 additions & 1 deletion python-connectors/api-connect_dataset/connector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ def __init__(self, config, plugin_config):
self.raw_output = endpoint_parameters.get("raw_output", None)
self.maximum_number_rows = config.get("maximum_number_rows", -1)
self.display_metadata = config.get("display_metadata", False)
self.csv_configuration = config

def get_read_schema(self):
# In this example, we don't specify a schema here, so DSS will infer the schema
Expand Down Expand Up @@ -60,7 +61,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
record_count += 1
yield self.format_output(data, metadata)
else:
csv_data = decode_csv_data(data)
csv_data = decode_csv_data(data, self.csv_configuration)
if csv_data:
record_count += len(csv_data)
for row in csv_data:
Expand Down
4 changes: 2 additions & 2 deletions python-lib/dku_constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
class DKUConstants(object):
API_RESPONSE_KEY = "api_response"
FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token", "mtls_key_path", "mtls_certificate_path"]
FORM_DATA_BODY_FORMAT = "FORM_DATA"
PLUGIN_VERSION = "1.2.7-beta.2"
PLUGIN_VERSION = "1.2.7-beta.5"
RAW_BODY_FORMAT = "RAW"
REPONSE_ERROR_KEY = "dku_error"
Loading