From ceeb3f3bff0b178b2bc73c185f2d995a16a35b3b Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Wed, 27 Sep 2017 19:33:57 -0400 Subject: [PATCH 1/4] Ported new jinja filters to st2mistral --- requirements.txt | 1 + setup.cfg | 3 ++ st2mistral/functions/data.py | 9 ++++++ st2mistral/functions/jmespath_query.py | 31 ++++++++++++++++++ st2mistral/tests/unit/functions/test_data.py | 34 ++++++++++++++++++++ st2mistral/tests/unit/test_function_base.py | 4 +++ 6 files changed, 82 insertions(+) create mode 100644 st2mistral/functions/jmespath_query.py diff --git a/requirements.txt b/requirements.txt index b702a74..ea18269 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +jmespath<1.0.0,>=0.9.3 retrying<1.4,>=1.3 semver==2.7.2 diff --git a/setup.cfg b/setup.cfg index f37b521..a5e2ccf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,6 +10,9 @@ mistral.actions = st2.action = st2mistral.actions.stackstorm:St2Action mistral.expression.functions = + from_json_string = st2mistral.functions.data:from_json_string + from_yaml_string = st2mistral.functions.data:from_yaml_string + jmespath_query = st2mistral.functions.jmespath_query:jmespath_query json_escape = st2mistral.functions.json_escape:json_escape regex_match = st2mistral.functions.regex:regex_match regex_replace = st2mistral.functions.regex:regex_replace diff --git a/st2mistral/functions/data.py b/st2mistral/functions/data.py index ffca7ea..aaffb29 100644 --- a/st2mistral/functions/data.py +++ b/st2mistral/functions/data.py @@ -17,11 +17,20 @@ import yaml __all__ = [ + 'from_json_string', + 'from_yaml_string', 'to_complex', 'to_json_string', 'to_yaml_string' ] +def from_json_string(value): + return json.loads(value) + + +def from_yaml_string(value): + return yaml.safe_load(value) + def to_complex(context, value): return json.dumps(value) diff --git a/st2mistral/functions/jmespath_query.py b/st2mistral/functions/jmespath_query.py new file mode 100644 index 0000000..22aba24 --- /dev/null +++ b/st2mistral/functions/jmespath_query.py @@ -0,0 +1,31 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import jmespath + +__all__ = [ + 'jmespath_query', +] + + +def jmespath_query(value, query): + """Extracts data from an object `value` using a jmespath `query`. + :link: http://jmespath.org + :param value: a object (dict, array, etc) to query + :param query: a jmsepath query expression (string) + :returns: the result of the query executed on the value + :rtype: dict, array, int, string, bool + """ + return jmespath.search(query, value) diff --git a/st2mistral/tests/unit/functions/test_data.py b/st2mistral/tests/unit/functions/test_data.py index 14bdcb0..ad16960 100644 --- a/st2mistral/tests/unit/functions/test_data.py +++ b/st2mistral/tests/unit/functions/test_data.py @@ -24,6 +24,22 @@ class JinjaDataTestCase(base.JinjaFunctionTestCase): + def test_function_from_json_string(self): + obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} + obj_json_str = json.dumps(obj) + template = '{{ from_json_string(_.k1) }}' + obj_str = self.eval_expression(template, {"k1": obj_json_str}) + actual_obj = eval(obj_str) + self.assertDictEqual(obj, actual_obj) + + def test_function_from_yaml_string(self): + obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} + obj_yaml_str = yaml.safe_dump(obj) + template = '{{ from_yaml_string(_.k1) }}' + obj_str = self.eval_expression(template, {"k1": obj_yaml_str}) + actual_obj = eval(obj_str) + self.assertDictEqual(obj, actual_obj) + def test_function_to_complex(self): obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} template = '{{ to_complex(_.k1) }}' @@ -48,6 +64,24 @@ def test_function_to_yaml_string(self): class YAQLDataTestCase(base.YaqlFunctionTestCase): + def test_function_from_json_string(self): + obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} + obj_json_str = json.dumps(obj) + result = YAQL_ENGINE('from_json_string($.k1)').evaluate( + context=self.get_yaql_context({'k1': obj_json_str}) + ) + actual_obj = eval(result) + self.assertDictEqual(obj, actual_obj) + + def test_function_from_yaml_string(self): + obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} + obj_yaml_str = yaml.safe_dump(obj) + result = YAQL_ENGINE('from_yaml_string($.k1)').evaluate( + context=self.get_yaql_context({'k1': obj_yaml_str}) + ) + actual_obj = eval(result) + self.assertDictEqual(obj, actual_obj) + def test_to_complex(self): obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} result = YAQL_ENGINE('to_complex($.k1)').evaluate( diff --git a/st2mistral/tests/unit/test_function_base.py b/st2mistral/tests/unit/test_function_base.py index 9bc5eed..5844635 100644 --- a/st2mistral/tests/unit/test_function_base.py +++ b/st2mistral/tests/unit/test_function_base.py @@ -22,6 +22,7 @@ def get_functions(): from st2mistral.functions import data + from st2mistral.functions import jmespath_query from st2mistral.functions import json_escape from st2mistral.functions import regex from st2mistral.functions import time @@ -29,6 +30,9 @@ def get_functions(): from st2mistral.functions import version return { + 'from_json_string': data.from_json_string, + 'from_yaml_string': data.from_yaml_string, + 'jmespath_query': jmespath_query.jmespath_query, 'json_escape': json_escape.json_escape, 'regex_match': regex.regex_match, 'regex_replace': regex.regex_replace, From df4ecb0c6568f1e0678c5e5ccc59bba1f55230fb Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Wed, 27 Sep 2017 20:46:39 -0400 Subject: [PATCH 2/4] Added tests for jmespath_query. Fixed a few bugs found during unit testing the other new Jinja filters --- st2mistral/functions/data.py | 5 +- st2mistral/functions/jmespath_query.py | 6 +- st2mistral/tests/unit/functions/test_data.py | 14 ++-- .../unit/functions/test_jmespath_query.py | 81 +++++++++++++++++++ 4 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 st2mistral/tests/unit/functions/test_jmespath_query.py diff --git a/st2mistral/functions/data.py b/st2mistral/functions/data.py index aaffb29..f391b59 100644 --- a/st2mistral/functions/data.py +++ b/st2mistral/functions/data.py @@ -24,11 +24,12 @@ 'to_yaml_string' ] -def from_json_string(value): + +def from_json_string(context, value): return json.loads(value) -def from_yaml_string(value): +def from_yaml_string(context, value): return yaml.safe_load(value) diff --git a/st2mistral/functions/jmespath_query.py b/st2mistral/functions/jmespath_query.py index 22aba24..c65e69d 100644 --- a/st2mistral/functions/jmespath_query.py +++ b/st2mistral/functions/jmespath_query.py @@ -20,12 +20,14 @@ ] -def jmespath_query(value, query): +def jmespath_query(context, value, query): """Extracts data from an object `value` using a jmespath `query`. - :link: http://jmespath.org + + http://jmespath.org :param value: a object (dict, array, etc) to query :param query: a jmsepath query expression (string) :returns: the result of the query executed on the value :rtype: dict, array, int, string, bool """ + return jmespath.search(query, value) diff --git a/st2mistral/tests/unit/functions/test_data.py b/st2mistral/tests/unit/functions/test_data.py index ad16960..5f42112 100644 --- a/st2mistral/tests/unit/functions/test_data.py +++ b/st2mistral/tests/unit/functions/test_data.py @@ -28,16 +28,16 @@ def test_function_from_json_string(self): obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} obj_json_str = json.dumps(obj) template = '{{ from_json_string(_.k1) }}' - obj_str = self.eval_expression(template, {"k1": obj_json_str}) - actual_obj = eval(obj_str) + result = self.eval_expression(template, {"k1": obj_json_str}) + actual_obj = eval(result) self.assertDictEqual(obj, actual_obj) def test_function_from_yaml_string(self): obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} obj_yaml_str = yaml.safe_dump(obj) template = '{{ from_yaml_string(_.k1) }}' - obj_str = self.eval_expression(template, {"k1": obj_yaml_str}) - actual_obj = eval(obj_str) + result = self.eval_expression(template, {"k1": obj_yaml_str}) + actual_obj = eval(result) self.assertDictEqual(obj, actual_obj) def test_function_to_complex(self): @@ -70,8 +70,7 @@ def test_function_from_json_string(self): result = YAQL_ENGINE('from_json_string($.k1)').evaluate( context=self.get_yaql_context({'k1': obj_json_str}) ) - actual_obj = eval(result) - self.assertDictEqual(obj, actual_obj) + self.assertDictEqual(obj, result) def test_function_from_yaml_string(self): obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} @@ -79,8 +78,7 @@ def test_function_from_yaml_string(self): result = YAQL_ENGINE('from_yaml_string($.k1)').evaluate( context=self.get_yaql_context({'k1': obj_yaml_str}) ) - actual_obj = eval(result) - self.assertDictEqual(obj, actual_obj) + self.assertDictEqual(obj, result) def test_to_complex(self): obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} diff --git a/st2mistral/tests/unit/functions/test_jmespath_query.py b/st2mistral/tests/unit/functions/test_jmespath_query.py new file mode 100644 index 0000000..8b59084 --- /dev/null +++ b/st2mistral/tests/unit/functions/test_jmespath_query.py @@ -0,0 +1,81 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import jmespath + +from st2mistral.tests.unit import test_function_base as base + +from yaql.language import factory +YAQL_ENGINE = factory.YaqlFactory().create() + + +class JinjaDataTestCase(base.JinjaFunctionTestCase): + + def test_function_jmespath_query_static(self): + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + + template = '{{ jmespath_query(_.obj, "people[*].first") }}' + result = self.eval_expression(template, {"obj": obj}) + actual = eval(result) + expected = ['James', 'Jacob', 'Jayden'] + self.assertEqual(actual, expected) + + def test_function_jmespath_query_dynamic(self): + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + query = "people[*].last" + + template = '{{ jmespath_query(_.obj, _.query) }}' + result = self.eval_expression(template, {"obj": obj, + 'query': query}) + actual = eval(result) + expected = ['d', 'e', 'f'] + self.assertEqual(actual, expected) + + +class YAQLDataTestCase(base.YaqlFunctionTestCase): + + def test_function_jmespath_query_static(self): + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + result = YAQL_ENGINE('jmespath_query($.obj, "people[*].first")').evaluate( + context=self.get_yaql_context({'obj': obj}) + ) + expected = ['James', 'Jacob', 'Jayden'] + self.assertEqual(result, expected) + + def test_function_jmespath_query_dynamic(self): + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + query = "people[*].last" + result = YAQL_ENGINE('jmespath_query($.obj, $.query)').evaluate( + context=self.get_yaql_context({'obj': obj, + 'query': query}) + ) + expected = ['d', 'e', 'f'] + self.assertEqual(result, expected) From 9f6ead96388f83fc7fe19da3589487dcfb087f89 Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Thu, 28 Sep 2017 19:59:21 -0400 Subject: [PATCH 3/4] Migrated from JMESPath to JSONPath to be consistent with st2 repo --- requirements.txt | 2 +- setup.cfg | 2 +- .../{jmespath_query.py => jsonpath_query.py} | 16 ++++--- ...espath_query.py => test_jsonpath_query.py} | 47 +++++++++++++++---- st2mistral/tests/unit/test_function_base.py | 4 +- 5 files changed, 52 insertions(+), 19 deletions(-) rename st2mistral/functions/{jmespath_query.py => jsonpath_query.py} (74%) rename st2mistral/tests/unit/functions/{test_jmespath_query.py => test_jsonpath_query.py} (62%) diff --git a/requirements.txt b/requirements.txt index ea18269..3ebf090 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -jmespath<1.0.0,>=0.9.3 +jsonpath-rw==1.4.0 retrying<1.4,>=1.3 semver==2.7.2 diff --git a/setup.cfg b/setup.cfg index a5e2ccf..db0692e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,8 +12,8 @@ mistral.actions = mistral.expression.functions = from_json_string = st2mistral.functions.data:from_json_string from_yaml_string = st2mistral.functions.data:from_yaml_string - jmespath_query = st2mistral.functions.jmespath_query:jmespath_query json_escape = st2mistral.functions.json_escape:json_escape + jsonpath_query = st2mistral.functions.jsonpath_query:jsonpath_query regex_match = st2mistral.functions.regex:regex_match regex_replace = st2mistral.functions.regex:regex_replace regex_search = st2mistral.functions.regex:regex_search diff --git a/st2mistral/functions/jmespath_query.py b/st2mistral/functions/jsonpath_query.py similarity index 74% rename from st2mistral/functions/jmespath_query.py rename to st2mistral/functions/jsonpath_query.py index c65e69d..3265f2c 100644 --- a/st2mistral/functions/jmespath_query.py +++ b/st2mistral/functions/jsonpath_query.py @@ -13,21 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import jmespath +import jsonpath __all__ = [ - 'jmespath_query', + 'jsonpath_query', ] -def jmespath_query(context, value, query): - """Extracts data from an object `value` using a jmespath `query`. +def jsonpath_query(context, value, query): + """Extracts data from an object `value` using a JSONPath `query`. - http://jmespath.org + :link: https://github.com/kennknowles/python-jsonpath-rw :param value: a object (dict, array, etc) to query :param query: a jmsepath query expression (string) :returns: the result of the query executed on the value :rtype: dict, array, int, string, bool """ - return jmespath.search(query, value) + expr = jsonpath_rw.parse(query) + matches = [match.value for match in expr.find(value)] + if not matches: + return None + return matches diff --git a/st2mistral/tests/unit/functions/test_jmespath_query.py b/st2mistral/tests/unit/functions/test_jsonpath_query.py similarity index 62% rename from st2mistral/tests/unit/functions/test_jmespath_query.py rename to st2mistral/tests/unit/functions/test_jsonpath_query.py index 8b59084..19379fc 100644 --- a/st2mistral/tests/unit/functions/test_jmespath_query.py +++ b/st2mistral/tests/unit/functions/test_jsonpath_query.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import jmespath +import jsonpath-rw from st2mistral.tests.unit import test_function_base as base @@ -23,20 +23,20 @@ class JinjaDataTestCase(base.JinjaFunctionTestCase): - def test_function_jmespath_query_static(self): + def test_function_jsonpath_query_static(self): obj = {'people': [{'first': 'James', 'last': 'd'}, {'first': 'Jacob', 'last': 'e'}, {'first': 'Jayden', 'last': 'f'}, {'missing': 'different'}], 'foo': {'bar': 'baz'}} - template = '{{ jmespath_query(_.obj, "people[*].first") }}' + template = '{{ jsonpath_query(_.obj, "people[*].first") }}' result = self.eval_expression(template, {"obj": obj}) actual = eval(result) expected = ['James', 'Jacob', 'Jayden'] self.assertEqual(actual, expected) - def test_function_jmespath_query_dynamic(self): + def test_function_jsonpath_query_dynamic(self): obj = {'people': [{'first': 'James', 'last': 'd'}, {'first': 'Jacob', 'last': 'e'}, {'first': 'Jayden', 'last': 'f'}, @@ -44,38 +44,67 @@ def test_function_jmespath_query_dynamic(self): 'foo': {'bar': 'baz'}} query = "people[*].last" - template = '{{ jmespath_query(_.obj, _.query) }}' + template = '{{ jsonpath_query(_.obj, _.query) }}' result = self.eval_expression(template, {"obj": obj, 'query': query}) actual = eval(result) expected = ['d', 'e', 'f'] self.assertEqual(actual, expected) + def test_function_jsonpath_query_no_results(self): + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + query = "query_returns_no_results" + + template = '{{ jsonpath_query(_.obj, _.query) }}' + result = self.eval_expression(template, {"obj": obj, + 'query': query}) + actual = eval(result) + expected = None + self.assertEqual(actual, expected) + class YAQLDataTestCase(base.YaqlFunctionTestCase): - def test_function_jmespath_query_static(self): + def test_function_jsonpath_query_static(self): obj = {'people': [{'first': 'James', 'last': 'd'}, {'first': 'Jacob', 'last': 'e'}, {'first': 'Jayden', 'last': 'f'}, {'missing': 'different'}], 'foo': {'bar': 'baz'}} - result = YAQL_ENGINE('jmespath_query($.obj, "people[*].first")').evaluate( + result = YAQL_ENGINE('jsonpath_query($.obj, "people[*].first")').evaluate( context=self.get_yaql_context({'obj': obj}) ) expected = ['James', 'Jacob', 'Jayden'] self.assertEqual(result, expected) - def test_function_jmespath_query_dynamic(self): + def test_function_jsonpath_query_dynamic(self): obj = {'people': [{'first': 'James', 'last': 'd'}, {'first': 'Jacob', 'last': 'e'}, {'first': 'Jayden', 'last': 'f'}, {'missing': 'different'}], 'foo': {'bar': 'baz'}} query = "people[*].last" - result = YAQL_ENGINE('jmespath_query($.obj, $.query)').evaluate( + result = YAQL_ENGINE('jsonpath_query($.obj, $.query)').evaluate( context=self.get_yaql_context({'obj': obj, 'query': query}) ) expected = ['d', 'e', 'f'] self.assertEqual(result, expected) + + def test_function_jsonpath_query_no_results(self): + obj = {'people': [{'first': 'James', 'last': 'd'}, + {'first': 'Jacob', 'last': 'e'}, + {'first': 'Jayden', 'last': 'f'}, + {'missing': 'different'}], + 'foo': {'bar': 'baz'}} + query = "query_returns_no_results" + result = YAQL_ENGINE('jsonpath_query($.obj, $.query)').evaluate( + context=self.get_yaql_context({'obj': obj, + 'query': query}) + ) + expected = None + self.assertEqual(result, expected) diff --git a/st2mistral/tests/unit/test_function_base.py b/st2mistral/tests/unit/test_function_base.py index 5844635..9bdabbc 100644 --- a/st2mistral/tests/unit/test_function_base.py +++ b/st2mistral/tests/unit/test_function_base.py @@ -22,8 +22,8 @@ def get_functions(): from st2mistral.functions import data - from st2mistral.functions import jmespath_query from st2mistral.functions import json_escape + from st2mistral.functions import jsonpath_query from st2mistral.functions import regex from st2mistral.functions import time from st2mistral.functions import use_none @@ -32,8 +32,8 @@ def get_functions(): return { 'from_json_string': data.from_json_string, 'from_yaml_string': data.from_yaml_string, - 'jmespath_query': jmespath_query.jmespath_query, 'json_escape': json_escape.json_escape, + 'jsonpath_query': jsonpath_query.jsonpath_query, 'regex_match': regex.regex_match, 'regex_replace': regex.regex_replace, 'regex_search': regex.regex_search, From a1852abe77b0c4021cc5c7e72f1044e40e31c9ef Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Thu, 28 Sep 2017 20:02:54 -0400 Subject: [PATCH 4/4] Fixed jsonpath import --- st2mistral/functions/jsonpath_query.py | 2 +- st2mistral/tests/unit/functions/test_jsonpath_query.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/st2mistral/functions/jsonpath_query.py b/st2mistral/functions/jsonpath_query.py index 3265f2c..70c86f4 100644 --- a/st2mistral/functions/jsonpath_query.py +++ b/st2mistral/functions/jsonpath_query.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import jsonpath +import jsonpath_rw __all__ = [ 'jsonpath_query', diff --git a/st2mistral/tests/unit/functions/test_jsonpath_query.py b/st2mistral/tests/unit/functions/test_jsonpath_query.py index 19379fc..af1083d 100644 --- a/st2mistral/tests/unit/functions/test_jsonpath_query.py +++ b/st2mistral/tests/unit/functions/test_jsonpath_query.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import jsonpath-rw +import jsonpath_rw from st2mistral.tests.unit import test_function_base as base