diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed57b3ea56..9a1082f85c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,7 +35,9 @@ Added StackStorm role mappings. This means that the same role can now be granted via multiple RBAC mapping files. #3763 - +* Add new Jinja filters ``from_json_string``, ``from_yaml_string``, and ``jsonpath_query``. + #3763 + Fixed ~~~~~ diff --git a/contrib/examples/actions/workflows/tests/mistral-test-func-from-json-string.yaml b/contrib/examples/actions/workflows/tests/mistral-test-func-from-json-string.yaml new file mode 100644 index 0000000000..860b6fc077 --- /dev/null +++ b/contrib/examples/actions/workflows/tests/mistral-test-func-from-json-string.yaml @@ -0,0 +1,16 @@ +version: '2.0' + +examples.mistral-test-func-from-json-string: + description: A workflow for testing from_json_string custom filter in mistral + type: direct + input: + - input_str + output: + result_jinja: <% $.result_jinja %> + result_yaql: <% $.result_yaql %> + tasks: + task1: + action: std.noop + publish: + result_jinja: "{{ from_json_string(_.input_str) }}" + result_yaql: '<% from_json_string($.input_str) %>' diff --git a/contrib/examples/actions/workflows/tests/mistral-test-func-from-yaml-string.yaml b/contrib/examples/actions/workflows/tests/mistral-test-func-from-yaml-string.yaml new file mode 100644 index 0000000000..af45f2e233 --- /dev/null +++ b/contrib/examples/actions/workflows/tests/mistral-test-func-from-yaml-string.yaml @@ -0,0 +1,16 @@ +version: '2.0' + +examples.mistral-test-func-from-yaml-string: + description: A workflow for testing from_yaml_string custom filter in mistral + type: direct + input: + - input_str + output: + result_jinja: <% $.result_jinja %> + result_yaql: <% $.result_yaql %> + tasks: + task1: + action: std.noop + publish: + result_jinja: "{{ from_yaml_string(_.input_str) }}" + result_yaql: '<% from_yaml_string($.input_str) %>' diff --git a/contrib/examples/actions/workflows/tests/mistral-test-func-jsonpath-query.yaml b/contrib/examples/actions/workflows/tests/mistral-test-func-jsonpath-query.yaml new file mode 100644 index 0000000000..9d2002664d --- /dev/null +++ b/contrib/examples/actions/workflows/tests/mistral-test-func-jsonpath-query.yaml @@ -0,0 +1,18 @@ +version: '2.0' + +examples.mistral-test-func-jsonpath-query: + description: A workflow for testing jsonpath_query custom filter in mistral + type: direct + input: + - input_obj + - input_query + output: + result_jinja: <% $.result_jinja %> + result_yaql: <% $.result_yaql %> + tasks: + + task2: + action: std.noop + publish: + result_jinja: '{{ jsonpath_query(_.input_obj, _.input_query) }}' + result_yaql: '<% jsonpath_query($.input_obj, $.input_query) %>' diff --git a/st2common/st2common/jinja/filters/data.py b/st2common/st2common/jinja/filters/data.py index b19d61244e..a65ab9609f 100644 --- a/st2common/st2common/jinja/filters/data.py +++ b/st2common/st2common/jinja/filters/data.py @@ -17,11 +17,21 @@ import yaml __all__ = [ + 'from_json_string', + 'from_yaml_string', 'to_json_string', 'to_yaml_string', ] +def from_json_string(value): + return json.loads(value) + + +def from_yaml_string(value): + return yaml.safe_load(value) + + def to_json_string(value, indent=4, sort_keys=False, separators=(',', ':')): return json.dumps(value, indent=indent, separators=separators, sort_keys=sort_keys) diff --git a/st2common/st2common/jinja/filters/jsonpath_query.py b/st2common/st2common/jinja/filters/jsonpath_query.py new file mode 100644 index 0000000000..61b9345c17 --- /dev/null +++ b/st2common/st2common/jinja/filters/jsonpath_query.py @@ -0,0 +1,35 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import jsonpath_rw + +__all__ = [ + 'jsonpath_query', +] + + +def jsonpath_query(value, query): + """Extracts data from an object `value` using a JSONPath `query`. + :link: https://github.com/kennknowles/python-jsonpath-rw + :param value: a object (dict, array, etc) to query + :param query: a JSONPath query expression (string) + :returns: the result of the query executed on the value + :rtype: dict, array, int, string, bool + """ + expr = jsonpath_rw.parse(query) + matches = [match.value for match in expr.find(value)] + if not matches: + return None + return matches diff --git a/st2common/st2common/util/jinja.py b/st2common/st2common/util/jinja.py index 8b9f679494..ffebbd319e 100644 --- a/st2common/st2common/util/jinja.py +++ b/st2common/st2common/util/jinja.py @@ -60,12 +60,15 @@ def get_filters(): from st2common.jinja.filters import time from st2common.jinja.filters import version from st2common.jinja.filters import json_escape + from st2common.jinja.filters import jsonpath_query # IMPORTANT NOTE - these filters were recently duplicated in st2mistral so that # they are also available in Mistral workflows. Please ensure any additions you # make here are also made there so that feature parity is maintained. return { 'decrypt_kv': crypto.decrypt_kv, + 'from_json_string': data.from_json_string, + 'from_yaml_string': data.from_yaml_string, 'to_json_string': data.to_json_string, 'to_yaml_string': data.to_yaml_string, @@ -89,7 +92,8 @@ def get_filters(): 'version_strip_patch': version.version_strip_patch, 'use_none': use_none, - 'json_escape': json_escape.json_escape + 'json_escape': json_escape.json_escape, + 'jsonpath_query': jsonpath_query.jsonpath_query } diff --git a/st2common/tests/unit/test_jinja_render_data_filters.py b/st2common/tests/unit/test_jinja_render_data_filters.py index 2fe349e5ae..87b253bf68 100644 --- a/st2common/tests/unit/test_jinja_render_data_filters.py +++ b/st2common/tests/unit/test_jinja_render_data_filters.py @@ -22,6 +22,32 @@ class JinjaUtilsDataFilterTestCase(unittest2.TestCase): + def test_filter_from_json_string(self): + env = jinja_utils.get_jinja_environment() + expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} + obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}' + + template = '{{k1 | from_json_string}}' + + obj_str = env.from_string(template).render({'k1': obj_json_str}) + obj = eval(obj_str) + self.assertDictEqual(obj, expected_obj) + + def test_filter_from_yaml_string(self): + env = jinja_utils.get_jinja_environment() + expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} + obj_yaml_str = ("---\n" + "a: b\n" + "c:\n" + " d: e\n" + " f: 1\n" + " g: true\n") + + template = '{{k1 | from_yaml_string}}' + obj_str = env.from_string(template).render({'k1': obj_yaml_str}) + obj = eval(obj_str) + self.assertDictEqual(obj, expected_obj) + def test_filter_to_json_string(self): env = jinja_utils.get_jinja_environment() obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} diff --git a/st2common/tests/unit/test_jinja_render_jsonpath_query_filters.py b/st2common/tests/unit/test_jinja_render_jsonpath_query_filters.py new file mode 100644 index 0000000000..d75ed5e84f --- /dev/null +++ b/st2common/tests/unit/test_jinja_render_jsonpath_query_filters.py @@ -0,0 +1,68 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest2 + +from st2common.util import jinja as jinja_utils + + +class JinjaUtilsJsonpathQueryTestCase(unittest2.TestCase): + + def test_jsonpath_query_static(self): + env = jinja_utils.get_jinja_environment() + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + + template = '{{ obj | jsonpath_query("people[*].first") }}' + actual_str = env.from_string(template).render({'obj': obj}) + actual = eval(actual_str) + expected = ['James', 'Jacob', 'Jayden'] + self.assertEqual(actual, expected) + + def test_jsonpath_query_dynamic(self): + env = jinja_utils.get_jinja_environment() + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + query = "people[*].last" + + template = '{{ obj | jsonpath_query(query) }}' + actual_str = env.from_string(template).render({'obj': obj, + 'query': query}) + actual = eval(actual_str) + expected = ['d', 'e', 'f'] + self.assertEqual(actual, expected) + + def test_jsonpath_query_no_results(self): + env = jinja_utils.get_jinja_environment() + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + query = "query_returns_no_results" + + template = '{{ obj | jsonpath_query(query) }}' + actual_str = env.from_string(template).render({'obj': obj, + 'query': query}) + actual = eval(actual_str) + expected = None + self.assertEqual(actual, expected) diff --git a/st2tests/integration/mistral/test_filters.py b/st2tests/integration/mistral/test_filters.py index 3d8694d90d..4c06258c20 100644 --- a/st2tests/integration/mistral/test_filters.py +++ b/st2tests/integration/mistral/test_filters.py @@ -25,6 +25,46 @@ ] +class FromJsonStringFiltersTest(base.TestWorkflowExecution): + + def test_from_json_string(self): + + execution = self._execute_workflow( + 'examples.mistral-test-func-from-json-string', + parameters={ + "input_str": '{"a": "b"}' + } + ) + execution = self._wait_for_completion(execution) + self._assert_success(execution, num_tasks=1) + jinja_dict = execution.result['result_jinja'] + yaql_dict = execution.result['result_yaql'] + self.assertTrue(isinstance(jinja_dict, dict)) + self.assertEqual(jinja_dict["a"], "b") + self.assertTrue(isinstance(yaql_dict, dict)) + self.assertEqual(yaql_dict["a"], "b") + + +class FromYamlStringFiltersTest(base.TestWorkflowExecution): + + def test_from_yaml_string(self): + + execution = self._execute_workflow( + 'examples.mistral-test-func-from-yaml-string', + parameters={ + "input_str": 'a: b' + } + ) + execution = self._wait_for_completion(execution) + self._assert_success(execution, num_tasks=1) + jinja_dict = execution.result['result_jinja'] + yaql_dict = execution.result['result_yaql'] + self.assertTrue(isinstance(jinja_dict, dict)) + self.assertEqual(jinja_dict["a"], "b") + self.assertTrue(isinstance(yaql_dict, dict)) + self.assertEqual(yaql_dict["a"], "b") + + class JsonEscapeFiltersTest(base.TestWorkflowExecution): def test_json_escape(self): @@ -44,6 +84,32 @@ def test_json_escape(self): self.assertEqual(yaql_dict["title"], breaking_str) +class JsonpathQueryFiltersTest(base.TestWorkflowExecution): + + def test_jsonpath_query(self): + + execution = self._execute_workflow( + 'examples.mistral-test-func-jsonpath-query', + parameters={ + "input_obj": {'people': [{'first': 'James', 'last': 'Smith'}, + {'first': 'Jacob', 'last': 'Alberts'}, + {'first': 'Jayden', 'last': 'Davis'}, + {'missing': 'different'}]}, + "input_query": "people[*].last" + } + ) + expected_result = ['Smith', 'Alberts', 'Davis'] + + execution = self._wait_for_completion(execution) + self._assert_success(execution, num_tasks=1) + jinja_result = execution.result['result_jinja'] + yaql_result = execution.result['result_yaql'] + self.assertTrue(isinstance(jinja_result, list)) + self.assertEqual(jinja_result, expected_result) + self.assertTrue(isinstance(yaql_result, list)) + self.assertEqual(yaql_result, expected_result) + + class RegexMatchFiltersTest(base.TestWorkflowExecution): def test_regex_match(self):