From 4e6b445f2dbe8a79d220c697abff946e00b2e57b Mon Sep 17 00:00:00 2001 From: Victor Zhestkov Date: Mon, 2 Oct 2023 13:26:20 +0200 Subject: [PATCH] Improve salt.utils.json.find_json (bsc#1213293) * Improve salt.utils.json.find_json * Move tests of find_json to pytest --- salt/utils/json.py | 39 +++++++- tests/pytests/unit/utils/test_json.py | 122 ++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 tests/pytests/unit/utils/test_json.py diff --git a/salt/utils/json.py b/salt/utils/json.py index 33cdbf401d..0845b64694 100644 --- a/salt/utils/json.py +++ b/salt/utils/json.py @@ -32,18 +32,51 @@ def find_json(raw): """ ret = {} lines = __split(raw) + lengths = list(map(len, lines)) + starts = [] + ends = [] + + # Search for possible starts end ends of the json fragments for ind, _ in enumerate(lines): + line = lines[ind].lstrip() + if line == "{" or line == "[": + starts.append((ind, line)) + if line == "}" or line == "]": + ends.append((ind, line)) + + # List all the possible pairs of starts and ends, + # and fill the length of each block to sort by size after + starts_ends = [] + for start, start_br in starts: + for end, end_br in reversed(ends): + if end > start and ( + (start_br == "{" and end_br == "}") + or (start_br == "[" and end_br == "]") + ): + starts_ends.append((start, end, sum(lengths[start : end + 1]))) + + # Iterate through all the possible pairs starting from the largest + starts_ends.sort(key=lambda x: (x[2], x[1] - x[0], x[0]), reverse=True) + for start, end, _ in starts_ends: + working = "\n".join(lines[start : end + 1]) try: - working = "\n".join(lines[ind:]) - except UnicodeDecodeError: - working = "\n".join(salt.utils.data.decode(lines[ind:])) + ret = json.loads(working) + except ValueError: + continue + if ret: + return ret + # Fall back to old implementation for backward compatibility + # excpecting json after the text + for ind, _ in enumerate(lines): + working = "\n".join(lines[ind:]) try: ret = json.loads(working) except ValueError: continue if ret: return ret + if not ret: # Not json, raise an error raise ValueError diff --git a/tests/pytests/unit/utils/test_json.py b/tests/pytests/unit/utils/test_json.py new file mode 100644 index 0000000000..72b1023003 --- /dev/null +++ b/tests/pytests/unit/utils/test_json.py @@ -0,0 +1,122 @@ +""" +Tests for salt.utils.json +""" + +import textwrap + +import pytest + +import salt.utils.json + + +def test_find_json(): + some_junk_text = textwrap.dedent( + """ + Just some junk text + with multiline + """ + ) + some_warning_message = textwrap.dedent( + """ + [WARNING] Test warning message + """ + ) + test_small_json = textwrap.dedent( + """ + { + "local": true + } + """ + ) + test_sample_json = """ + { + "glossary": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } + } + """ + expected_ret = { + "glossary": { + "GlossDiv": { + "GlossList": { + "GlossEntry": { + "GlossDef": { + "GlossSeeAlso": ["GML", "XML"], + "para": ( + "A meta-markup language, used to create markup" + " languages such as DocBook." + ), + }, + "GlossSee": "markup", + "Acronym": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "SortAs": "SGML", + "Abbrev": "ISO 8879:1986", + "ID": "SGML", + } + }, + "title": "S", + }, + "title": "example glossary", + } + } + + # First test the valid JSON + ret = salt.utils.json.find_json(test_sample_json) + assert ret == expected_ret + + # Now pre-pend some garbage and re-test + garbage_prepend_json = f"{some_junk_text}{test_sample_json}" + ret = salt.utils.json.find_json(garbage_prepend_json) + assert ret == expected_ret + + # Now post-pend some garbage and re-test + garbage_postpend_json = f"{test_sample_json}{some_junk_text}" + ret = salt.utils.json.find_json(garbage_postpend_json) + assert ret == expected_ret + + # Now pre-pend some warning and re-test + warning_prepend_json = f"{some_warning_message}{test_sample_json}" + ret = salt.utils.json.find_json(warning_prepend_json) + assert ret == expected_ret + + # Now post-pend some warning and re-test + warning_postpend_json = f"{test_sample_json}{some_warning_message}" + ret = salt.utils.json.find_json(warning_postpend_json) + assert ret == expected_ret + + # Now put around some garbage and re-test + garbage_around_json = f"{some_junk_text}{test_sample_json}{some_junk_text}" + ret = salt.utils.json.find_json(garbage_around_json) + assert ret == expected_ret + + # Now pre-pend small json and re-test + small_json_pre_json = f"{test_small_json}{test_sample_json}" + ret = salt.utils.json.find_json(small_json_pre_json) + assert ret == expected_ret + + # Now post-pend small json and re-test + small_json_post_json = f"{test_sample_json}{test_small_json}" + ret = salt.utils.json.find_json(small_json_post_json) + assert ret == expected_ret + + # Test to see if a ValueError is raised if no JSON is passed in + with pytest.raises(ValueError): + ret = salt.utils.json.find_json(some_junk_text) -- 2.42.0