From 0e1aa32b8a28e9c2545e8b063843a47d7ffe1864 Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Wed, 24 Dec 2025 20:01:23 +0000 Subject: [PATCH 1/7] just for backup --- aci-preupgrade-validation-script.py | 88 ++++++++- .../fabricNode.json | 93 +++++++++ .../fabricNode_no_apic.json | 48 +++++ .../fabricNode_old.json | 62 ++++++ .../infraWiNode_apic1.json | 62 ++++++ ...est_infinite_snapshot_file_access_check.py | 176 ++++++++++++++++++ 6 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 tests/checks/infinite_snapshot_file_access_check/fabricNode.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe0477..1d9b2fe 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -65,7 +65,7 @@ tz = time.strftime('%z') ts = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') -BUNDLE_NAME = 'preupgrade_validator_%s%s.tgz' % (ts, tz) +BUNDLE_NAME = 'preupgrade_validator_sudharson1_%s%s.tgz' % (ts, tz) DIR = 'preupgrade_validator_logs/' JSON_DIR = os.path.join(DIR, 'json_results/') META_FILE = os.path.join(DIR, 'meta.json') @@ -6007,6 +6007,91 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title='infinite snapshot file access check') +def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, password, **kwargs): + result = PASS + headers = ['apic_id', 'apic_name', 'snapshot_files'] + data = [] + recommended_action = 'Contact Cisco TAC for Support before upgrade' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#infinite-snapshot-file-access-check' + if cversion.older_than('6.0(3d)'): + apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] + log.debug("APIC nodes found: %s", apics) + if not apics: + return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) + # `fabricNode` in pre-4.0 does not have `address` + if not apics[0]["fabricNode"]["attributes"].get("address"): + apic1 = [apic for apic in apics if apic["fabricNode"]["attributes"]["id"] == "1"][0] + log.debug("Re-fetching infraWiNode.json from APIC-1 using : %s", apic1) + apic1_dn = apic1["fabricNode"]["attributes"]["dn"] + apics = icurl("class", "{}/infraWiNode.json".format(apic1_dn)) + log.debug("Re-fetched infraWiNode.json from APIC-1: %s", apics) + has_error = False + for apic in apics: + if apic.get("fabricNode"): + apic_id = apic["fabricNode"]["attributes"]["id"] + apic_name = apic["fabricNode"]["attributes"]["name"] + apic_addr = apic["fabricNode"]["attributes"]["address"] + else: + apic_id = apic["infraWiNode"]["attributes"]["id"] + apic_name = apic["infraWiNode"]["attributes"]["nodeName"] + apic_addr = apic["infraWiNode"]["attributes"]["addr"] + try: + c = Connection(apic_addr) + c.username = username + c.password = password + c.log = LOG_FILE + c.connect() + except Exception as e: + data.append([apic_id, apic_name]) + has_error = True + continue + try: + """ c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') """ + c.cmd('tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404') + access_logs = c.output.splitlines() + + requests = [] # [(timestamp, filename), ...] + + for line in access_logs: + # Extract timestamp: [22/Dec/2025:04:05:02 +0000] + timestamp_match = re.search(r'\[(\d{1,2}/\w{3}/\d{4}):(\d{2}:\d{2}:\d{2})', line) + # Extract filename: ce2_NDI_EXPORT_POLICY-2025-12-22T10-04-36.tar.gz + filename_match = re.search(r'GET /snapshots/([^\s]+)', line) + + if timestamp_match and filename_match: + timestamp_str = f"{timestamp_match.group(1)}:{timestamp_match.group(2)}" + filename = filename_match.group(1) + try: + timestamp = datetime.strptime(timestamp_str, "%d/%b/%Y:%H:%M:%S") + requests.append((timestamp, filename)) + except: + continue + + requests.sort() + + # Checking if any 10 consecutive requests are within 2 minutes + if len(requests) >= 10: + for i in range(len(requests) - 9): + time_diff = (requests[i + 9][0] - requests[i][0]).total_seconds() + if time_diff <= 120: + window_files = [filename for _, filename in requests[i:i+10]] + for filename in window_files: + data.append([apic_id, apic_name, filename]) + break + + except Exception as e: + data.append([apic_id, apic_name, str(e)]) + has_error = True + continue + if has_error: + result = ERROR + elif data: + result = FAIL_UF + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6253,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + infinite_snapshot_file_access_check, ] ssh_checks = [ diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode.json b/tests/checks/infinite_snapshot_file_access_check/fabricNode.json new file mode 100644 index 0000000..962a4ad --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/fabricNode.json @@ -0,0 +1,93 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.2", + "dn": "topology/pod-1/node-2", + "fabricSt": "commissioned", + "id": "2", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic2", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.3", + "dn": "topology/pod-2/node-3", + "fabricSt": "commissioned", + "id": "3", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic3", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json b/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json new file mode 100644 index 0000000..254f40d --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json @@ -0,0 +1,48 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json b/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json new file mode 100644 index 0000000..f71fb9f --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json @@ -0,0 +1,62 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-2", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "2", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic2", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-2/node-3", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "3", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic3", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "", + "role": "leaf", + "adSt": "on", + "name": "leaf1", + "model": "N9K-C9396PX" + } + } + } +] diff --git a/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json b/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json new file mode 100644 index 0000000..b6626d0 --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json @@ -0,0 +1,62 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.2", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-2", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "2", + "mbSn": "FCH1235ABCD", + "name": "", + "nodeName": "apic2", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.3", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-3", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "3", + "mbSn": "FCH1236ABCD", + "name": "", + "nodeName": "apic3", + "operSt": "available", + "podId": "1", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py b/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py new file mode 100644 index 0000000..1c01ed8 --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py @@ -0,0 +1,176 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "infinite_snapshot_file_access_check" +infraWiNode = "topology/pod-1/node-1/infraWiNode.json" + +apic_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] + +grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404' + +# Sample log output with 10+ requests within 2 minutes (issue detected) +grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:25 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-30-15.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-25.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:45 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-30-35.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:55 +0000]"GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-45.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:05 +0000]"GET /snapshots/ce2_config_export-2025-12-24T14-30-55.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:25 +0000]"GET /snapshots/ce2_fabric_backup-2025-12-24T14-31-15.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-31-25.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:45 +0000]"GET /snapshots/ce2_scheduler_backup-2025-12-24T14-31-35.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:55 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-31-45.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:32:05 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-55.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) +grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", + [ + # Version not affected (6.0(3d) or newer) + ( + {}, + False, + [], + "6.0(3d)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, but no issues found + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Connection failure + ( + {}, + True, + [], + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Exception during grep command execution + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Pass (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_no_apic.json"), + script.ERROR, + ) + ], +) +def test_infinite_snapshot_file_access_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): + result = run_check( + cversion=script.AciVersion(cversion), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result \ No newline at end of file From 5b417656b7523b1cb9ae1eb4a63e5e1ab30ac5b2 Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Tue, 6 Jan 2026 12:38:46 +0000 Subject: [PATCH 2/7] changed after shutdown holiday --- aci-preupgrade-validation-script.py | 32 +- docs/docs/validations.md | 16 +- ...est_infinite_snapshot_file_access_check.py | 176 ---- .../fabricNode.json | 0 .../fabricNode_no_apic.json | 0 .../fabricNode_old.json | 0 .../fabricNode_old_single_apic.json | 32 + .../fabricNode_single_apic.json | 32 + .../infraWiNode_apic1.json | 0 .../infraWiNode_single_apic.json | 22 + .../test_snapshot_files_check.py | 882 ++++++++++++++++++ 11 files changed, 999 insertions(+), 193 deletions(-) delete mode 100644 tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/fabricNode.json (100%) rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/fabricNode_no_apic.json (100%) rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/fabricNode_old.json (100%) create mode 100644 tests/checks/snapshot_files_check/fabricNode_old_single_apic.json create mode 100644 tests/checks/snapshot_files_check/fabricNode_single_apic.json rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/infraWiNode_apic1.json (100%) create mode 100644 tests/checks/snapshot_files_check/infraWiNode_single_apic.json create mode 100644 tests/checks/snapshot_files_check/test_snapshot_files_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 1d9b2fe..b6e41e3 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -65,7 +65,7 @@ tz = time.strftime('%z') ts = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') -BUNDLE_NAME = 'preupgrade_validator_sudharson1_%s%s.tgz' % (ts, tz) +BUNDLE_NAME = 'preupgrade_validator_%s%s.tgz' % (ts, tz) DIR = 'preupgrade_validator_logs/' JSON_DIR = os.path.join(DIR, 'json_results/') META_FILE = os.path.join(DIR, 'meta.json') @@ -6008,25 +6008,22 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): doc_url=doc_url) -@check_wrapper(check_title='infinite snapshot file access check') -def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, password, **kwargs): +@check_wrapper(check_title='Snapshot files check') +def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): result = PASS headers = ['apic_id', 'apic_name', 'snapshot_files'] data = [] recommended_action = 'Contact Cisco TAC for Support before upgrade' - doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#infinite-snapshot-file-access-check' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#Snapshot-files-check' if cversion.older_than('6.0(3d)'): apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] - log.debug("APIC nodes found: %s", apics) if not apics: return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) # `fabricNode` in pre-4.0 does not have `address` if not apics[0]["fabricNode"]["attributes"].get("address"): apic1 = [apic for apic in apics if apic["fabricNode"]["attributes"]["id"] == "1"][0] - log.debug("Re-fetching infraWiNode.json from APIC-1 using : %s", apic1) apic1_dn = apic1["fabricNode"]["attributes"]["dn"] apics = icurl("class", "{}/infraWiNode.json".format(apic1_dn)) - log.debug("Re-fetched infraWiNode.json from APIC-1: %s", apics) has_error = False for apic in apics: if apic.get("fabricNode"): @@ -6048,16 +6045,17 @@ def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, passwo has_error = True continue try: - """ c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') """ - c.cmd('tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404') + c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') access_logs = c.output.splitlines() - - requests = [] # [(timestamp, filename), ...] + if len(access_logs)<15 and any("No such file or directory" in line for line in access_logs): + data.append([apic_id, apic_name, '/var/log/dme/log/access.log not found']) + has_error = True + continue + + requests = [] for line in access_logs: - # Extract timestamp: [22/Dec/2025:04:05:02 +0000] timestamp_match = re.search(r'\[(\d{1,2}/\w{3}/\d{4}):(\d{2}:\d{2}:\d{2})', line) - # Extract filename: ce2_NDI_EXPORT_POLICY-2025-12-22T10-04-36.tar.gz filename_match = re.search(r'GET /snapshots/([^\s]+)', line) if timestamp_match and filename_match: @@ -6071,11 +6069,11 @@ def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, passwo requests.sort() - # Checking if any 10 consecutive requests are within 2 minutes + # Checking if any 10 consecutive requests are within 1 minute if len(requests) >= 10: for i in range(len(requests) - 9): time_diff = (requests[i + 9][0] - requests[i][0]).total_seconds() - if time_diff <= 120: + if time_diff <= 60: window_files = [filename for _, filename in requests[i:i+10]] for filename in window_files: data.append([apic_id, apic_name, filename]) @@ -6085,10 +6083,12 @@ def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, passwo data.append([apic_id, apic_name, str(e)]) has_error = True continue + if has_error: result = ERROR elif data: result = FAIL_UF + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) @@ -6253,7 +6253,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, - infinite_snapshot_file_access_check, + snapshot_files_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e..144371e 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[Snapshot files check][d29] | CSCwe07002 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,7 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash - +[d29]: #Snapshot-files-check ## General Check Details @@ -2614,6 +2615,18 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### Snapshot files check + +RCA: +Issue occured in 3 node apic cluster, AE process on one apic is busy. Reason is it's trying to fetch the snapshot files which was taken earlier is missing on all apics. + +IMPACT: +Access logs will be flooded with the GET calls. APIC Upgrade/downgrade will fail with message Installer Exited - Pre-upgrade callbacks were not completed. + +Suggestion: +Restart AE on each APIC one at a time. For Reference [CSCwe07002][62]. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2676,3 +2689,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression +[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe07002 \ No newline at end of file diff --git a/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py b/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py deleted file mode 100644 index 1c01ed8..0000000 --- a/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py +++ /dev/null @@ -1,176 +0,0 @@ -import os -import pytest -import logging -import importlib -from helpers.utils import read_data - -script = importlib.import_module("aci-preupgrade-validation-script") - -log = logging.getLogger(__name__) -dir = os.path.dirname(os.path.abspath(__file__)) - -test_function = "infinite_snapshot_file_access_check" -infraWiNode = "topology/pod-1/node-1/infraWiNode.json" - -apic_ips = [ - node["fabricNode"]["attributes"]["address"] - for node in read_data(dir, "fabricNode.json") - if node["fabricNode"]["attributes"]["role"] == "controller" -] - -grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404' - -# Sample log output with 10+ requests within 2 minutes (issue detected) -grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:25 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-30-15.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-25.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:45 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-30-35.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:55 +0000]"GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-45.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:05 +0000]"GET /snapshots/ce2_config_export-2025-12-24T14-30-55.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-05.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:25 +0000]"GET /snapshots/ce2_fabric_backup-2025-12-24T14-31-15.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-31-25.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:45 +0000]"GET /snapshots/ce2_scheduler_backup-2025-12-24T14-31-35.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:55 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-31-45.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:32:05 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-55.tar.gz HTTP/1.1" 404 151 "-" "-" -""" - -# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) -grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" -""" - -@pytest.mark.parametrize( - "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", - [ - # Version not affected (6.0(3d) or newer) - ( - {}, - False, - [], - "6.0(3d)", - read_data(dir, "fabricNode.json"), - script.PASS, - ), - # Version affected, but no issues found - ( - {}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ] - for apic_ip in apic_ips - }, - "5.2(1h)", - read_data(dir, "fabricNode.json"), - script.PASS, - ), - # Version affected, issue detected (10+ requests in 2 minutes) - ( - {}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - }, - "6.0(2a)", - read_data(dir, "fabricNode.json"), - script.FAIL_UF, - ), - # Connection failure - ( - {}, - True, - [], - "5.2(1h)", - read_data(dir, "fabricNode.json"), - script.ERROR, - ), - # Exception during grep command execution - ( - {}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "", - "exception": Exception("Simulated exception at grep command"), - } - ] - for apic_ip in apic_ips - }, - "6.0(2a)", - read_data(dir, "fabricNode.json"), - script.ERROR, - ), - # Pass (pre-4.0 with infraWiNode) - ( - {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - }, - ] - for apic_ip in apic_ips - }, - "3.2(6o)", - read_data(dir, "fabricNode_old.json"), - script.PASS, - ), - ( - {}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - }, - ] - for apic_ip in apic_ips - }, - "3.2(6o)", - read_data(dir, "fabricNode_no_apic.json"), - script.ERROR, - ) - ], -) -def test_infinite_snapshot_file_access_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): - result = run_check( - cversion=script.AciVersion(cversion), - username="fake_username", - password="fake_password", - fabric_nodes=fabric_nodes, - ) - assert result.result == expected_result \ No newline at end of file diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode.json b/tests/checks/snapshot_files_check/fabricNode.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/fabricNode.json rename to tests/checks/snapshot_files_check/fabricNode.json diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json b/tests/checks/snapshot_files_check/fabricNode_no_apic.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json rename to tests/checks/snapshot_files_check/fabricNode_no_apic.json diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json b/tests/checks/snapshot_files_check/fabricNode_old.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json rename to tests/checks/snapshot_files_check/fabricNode_old.json diff --git a/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json new file mode 100644 index 0000000..5e5f451 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "A", + "role": "leaf", + "adSt": "on", + "name": "leaf101", + "model": "N9K-C93180YC-EX" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/fabricNode_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_single_apic.json new file mode 100644 index 0000000..cf35918 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-EX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + } +] diff --git a/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json b/tests/checks/snapshot_files_check/infraWiNode_apic1.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json rename to tests/checks/snapshot_files_check/infraWiNode_apic1.json diff --git a/tests/checks/snapshot_files_check/infraWiNode_single_apic.json b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json new file mode 100644 index 0000000..fe17cdf --- /dev/null +++ b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json @@ -0,0 +1,22 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py new file mode 100644 index 0000000..c76c8c5 --- /dev/null +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -0,0 +1,882 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "snapshot_files_check" +infraWiNode = "topology/pod-1/node-1/infraWiNode.json" + +apic_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] +apic_single_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode_single_apic.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] + +grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile1.txt | grep "GET /snapshots" | grep 404' + +# Sample log output with 10+ requests within 2 minutes (issue detected) +grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-12.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-30-18.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-24.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_config_export-2025-12-24T14-30-30.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-36.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_fabric_backup-2025-12-24T14-30-42.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-48.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_scheduler_backup-2025-12-24T14-30-54.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:00 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-31-00.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:06 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-06.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) +grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +grep_output_no_file = """\ +grep: /data/techsupport/snapshotfile1.txt: No such file or directory +fabric-apic# +""" + +# Edge case: Empty log output (no 404 requests found) +grep_output_empty = "" + +# Edge case: Exactly 10 requests within 60 seconds (boundary - should trigger) +grep_output_boundary_fail = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_snapshot10.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Edge case: 9 requests within 60 seconds (should pass - below threshold) +grep_output_boundary_pass = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:08 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:16 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:32 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:40 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:56 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:04 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", + [ + # Version not affected (6.0(3d) or newer) + ( + {}, + False, + [], + "6.0(3d)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, but no issues found + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Connection failure + ( + {}, + True, + [], + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Exception during grep command execution + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # error (no such file or directory for cmd execution) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Pass (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # fail (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # connection failure(pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # exception during grep command execution (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # error (pre-4.0 with infraWiNode and no such file or directory for cmd execution) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case: Single APIC with issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.FAIL_UF, + ), + # Edge case: Single APIC without issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.PASS, + ), + # Edge case: Single APIC with connection failure + ( + {}, + True, + [], + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Single APIC with file not found + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with first APIC having issue, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with last APIC having issue, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with middle APIC having connection issue + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Connection timeout"), + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with one file not found, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Empty log output (no 404 requests) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: 9 requests within 60 seconds (boundary - should pass) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Multi-APIC with all having issues + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with mixed issues (issue + file not found + clean) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Single APIC without issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.PASS, + ), + # Edge case (pre-4.0): Single APIC with connection failure + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with file not found + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with first APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with last APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with middle APIC having connection issue + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Connection timeout"), + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with one file not found, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Empty log output (no 404 requests) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): 9 requests within 60 seconds + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Multi-APIC with all having issues + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with mixed issues (issue + file not found + clean) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + ], +) +def test_snapshot_files_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): + result = run_check( + cversion=script.AciVersion(cversion), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result \ No newline at end of file From ae55164d1861dcaf7ea51c3afbf836276daba2af Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Tue, 6 Jan 2026 12:48:02 +0000 Subject: [PATCH 3/7] latest changes --- aci-preupgrade-validation-script.py | 4 ++-- .../snapshot_files_check/test_snapshot_files_check.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index b6e41e3..b7378bb 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6041,13 +6041,13 @@ def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): c.log = LOG_FILE c.connect() except Exception as e: - data.append([apic_id, apic_name]) + data.append([apic_id, apic_name, str(e)]) has_error = True continue try: c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') access_logs = c.output.splitlines() - if len(access_logs)<15 and any("No such file or directory" in line for line in access_logs): + if len(access_logs) < 15 and any("No such file or directory" in line for line in access_logs): data.append([apic_id, apic_name, '/var/log/dme/log/access.log not found']) has_error = True continue diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py index c76c8c5..b9f05f8 100644 --- a/tests/checks/snapshot_files_check/test_snapshot_files_check.py +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -23,7 +23,7 @@ if node["fabricNode"]["attributes"]["role"] == "controller" ] -grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile1.txt | grep "GET /snapshots" | grep 404' +grep_cmd = 'tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404' # Sample log output with 10+ requests within 2 minutes (issue detected) grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" @@ -46,14 +46,14 @@ """ grep_output_no_file = """\ -grep: /data/techsupport/snapshotfile1.txt: No such file or directory +grep: /var/log/dme/log/access.log: No such file or directory fabric-apic# """ # Edge case: Empty log output (no 404 requests found) grep_output_empty = "" -# Edge case: Exactly 10 requests within 60 seconds (boundary - should trigger) +# Edge case: Exactly 10 requests within 60 seconds grep_output_boundary_fail = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" @@ -66,7 +66,7 @@ 10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_snapshot10.tar.gz HTTP/1.1" 404 151 "-" "-" """ -# Edge case: 9 requests within 60 seconds (should pass - below threshold) +# Edge case: 9 requests within 60 seconds grep_output_boundary_pass = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:08 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:16 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" From be81efce72142284db317b5b82f29cdd66fb0843 Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Thu, 22 Jan 2026 08:52:53 +0000 Subject: [PATCH 4/7] added tversion check and removed unnecessary pytest methods --- aci-preupgrade-validation-script.py | 14 +- .../test_snapshot_files_check.py | 222 ++++-------------- 2 files changed, 49 insertions(+), 187 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 4bc769f..bc589cd 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6027,13 +6027,14 @@ def apic_downgrade_compat_warning_check(cversion, tversion, **kwargs): @check_wrapper(check_title='Snapshot files check') -def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): +def snapshot_files_check(fabric_nodes, cversion, tversion, username, password, **kwargs): result = PASS headers = ['apic_id', 'apic_name', 'snapshot_files'] data = [] recommended_action = 'Contact Cisco TAC for Support before upgrade' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#Snapshot-files-check' - if cversion.older_than('6.0(3d)'): + + if cversion.older_than('6.0(3d)') or tversion.older_than('6.0(3d)'): apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] if not apics: return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) @@ -6095,17 +6096,18 @@ def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): window_files = [filename for _, filename in requests[i:i+10]] for filename in window_files: data.append([apic_id, apic_name, filename]) + result = FAIL_UF break except Exception as e: data.append([apic_id, apic_name, str(e)]) has_error = True continue - - if has_error: + + if has_error and result == PASS: result = ERROR - elif data: - result = FAIL_UF + else: + result = NA return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py index b9f05f8..c6b695a 100644 --- a/tests/checks/snapshot_files_check/test_snapshot_files_check.py +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -79,7 +79,7 @@ """ @pytest.mark.parametrize( - "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", + "icurl_outputs, conn_failure, conn_cmds, cversion, tversion, fabric_nodes, expected_result", [ # Version not affected (6.0(3d) or newer) ( @@ -87,8 +87,9 @@ False, [], "6.0(3d)", + "6.0(3e)", read_data(dir, "fabricNode.json"), - script.PASS, + script.NA, ), # Version affected, but no issues found ( @@ -105,10 +106,11 @@ for apic_ip in apic_ips }, "5.2(1h)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.PASS, ), - # Version affected, issue detected (10+ requests in 2 minutes) + # Version affected, issue detected (10+ requests in 1 minutes) ( {}, False, @@ -136,10 +138,11 @@ ], }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), - # Version affected, issue detected (10+ requests in 2 minutes) + # Version affected, issue detected (10+ requests in 1 minutes) ( {}, False, @@ -167,6 +170,7 @@ ], }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -176,6 +180,7 @@ True, [], "5.2(1h)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -194,6 +199,7 @@ for apic_ip in apic_ips }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -212,6 +218,7 @@ for apic_ip in apic_ips }, "5.2(1h)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -230,6 +237,7 @@ for apic_ip in apic_ips }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.PASS, ), @@ -247,7 +255,8 @@ ] for apic_ip in apic_ips }, - "3.2(6o)", + "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), @@ -257,6 +266,7 @@ True, [], "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -275,6 +285,7 @@ for apic_ip in apic_ips }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -293,6 +304,7 @@ for apic_ip in apic_ips }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -310,6 +322,7 @@ ] }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.FAIL_UF, ), @@ -327,6 +340,7 @@ ] }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.PASS, ), @@ -336,6 +350,7 @@ True, [], "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.ERROR, ), @@ -353,102 +368,10 @@ ] }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.ERROR, ), - # Edge case: Multi-APIC with first APIC having issue, others clean - ( - {}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - }, - "5.2(8g)", - read_data(dir, "fabricNode.json"), - script.FAIL_UF, - ), - # Edge case: Multi-APIC with last APIC having issue, others clean - ( - {}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_issue]), - "exception": None, - } - ], - }, - "5.2(8g)", - read_data(dir, "fabricNode.json"), - script.FAIL_UF, - ), - # Edge case: Multi-APIC with middle APIC having connection issue - ( - {}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "", - "exception": Exception("Connection timeout"), - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - }, - "6.0(2a)", - read_data(dir, "fabricNode.json"), - script.ERROR, - ), # Edge case: Multi-APIC with one file not found, others clean ( {}, @@ -477,6 +400,7 @@ ], }, "5.2(8g)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -495,6 +419,7 @@ for apic_ip in apic_ips }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.PASS, ), @@ -513,6 +438,7 @@ for apic_ip in apic_ips }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -531,6 +457,7 @@ for apic_ip in apic_ips }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.PASS, ), @@ -549,6 +476,7 @@ for apic_ip in apic_ips }, "5.2(8g)", + "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -580,8 +508,9 @@ ], }, "6.0(2a)", + "6.0(3d)", read_data(dir, "fabricNode.json"), - script.ERROR, + script.FAIL_UF, ), # Edge case (pre-4.0): Single APIC with issue ( @@ -597,6 +526,7 @@ ] }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.FAIL_UF, ), @@ -614,6 +544,7 @@ ] }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.PASS, ), @@ -623,6 +554,7 @@ True, [], "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.ERROR, ), @@ -640,6 +572,7 @@ ] }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.ERROR, ), @@ -671,71 +604,10 @@ ], }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), - # Edge case (pre-4.0): Multi-APIC with last APIC having issue, others clean - ( - {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_issue]), - "exception": None, - } - ], - }, - "3.2(6o)", - read_data(dir, "fabricNode_old.json"), - script.FAIL_UF, - ), - # Edge case (pre-4.0): Multi-APIC with middle APIC having connection issue - ( - {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "", - "exception": Exception("Connection timeout"), - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - }, - "3.2(6o)", - read_data(dir, "fabricNode_old.json"), - script.ERROR, - ), # Edge case (pre-4.0): Multi-APIC with one file not found, others clean ( {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, @@ -764,6 +636,7 @@ ], }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -782,6 +655,7 @@ for apic_ip in apic_ips }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.PASS, ), @@ -800,6 +674,7 @@ for apic_ip in apic_ips }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), @@ -818,27 +693,10 @@ for apic_ip in apic_ips }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.PASS, ), - # Edge case (pre-4.0): Multi-APIC with all having issues - ( - {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_issue]), - "exception": None, - } - ] - for apic_ip in apic_ips - }, - "3.2(6o)", - read_data(dir, "fabricNode_old.json"), - script.FAIL_UF, - ), # Edge case (pre-4.0): Multi-APIC with mixed issues (issue + file not found + clean) ( {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, @@ -867,14 +725,16 @@ ], }, "3.2(6o)", + "6.0(3d)", read_data(dir, "fabricNode_old.json"), - script.ERROR, + script.FAIL_UF, ), ], ) -def test_snapshot_files_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): +def test_snapshot_files_check(run_check, mock_icurl, mock_conn, cversion, tversion, fabric_nodes, expected_result): result = run_check( cversion=script.AciVersion(cversion), + tversion=script.AciVersion(tversion), username="fake_username", password="fake_password", fabric_nodes=fabric_nodes, From ab601d60ad5f77770daec89a818039f6bfd5e4cc Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Thu, 22 Jan 2026 09:37:38 +0000 Subject: [PATCH 5/7] adding check name to ssh_checks instead of api_checks --- aci-preupgrade-validation-script.py | 2 +- .../checks/snapshot_files_check/test_snapshot_files_check.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index bc589cd..aecdab6 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6274,7 +6274,6 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, - snapshot_files_check, ] ssh_checks = [ @@ -6287,6 +6286,7 @@ class CheckManager: # Bugs observer_db_size_check, + snapshot_files_check, ] cli_checks = [ # General diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py index c6b695a..b5ce533 100644 --- a/tests/checks/snapshot_files_check/test_snapshot_files_check.py +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -25,7 +25,7 @@ grep_cmd = 'tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404' -# Sample log output with 10+ requests within 2 minutes (issue detected) +# Sample log output with 10+ requests within 1 minute (issue detected) grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-12.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-30-18.tar.gz HTTP/1.1" 404 151 "-" "-" @@ -39,7 +39,7 @@ 10.0.0.3 (-) - - [24/Dec/2025:14:31:06 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-06.tar.gz HTTP/1.1" 404 151 "-" "-" """ -# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) +# Sample log output with less than 10 requests or spread over more than 1 minute (no issue) grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" From 6a61ffef1a5817af8569576d82d1e1b7ce2b03be Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Tue, 3 Feb 2026 05:30:57 +0000 Subject: [PATCH 6/7] Removing Cverion check --- aci-preupgrade-validation-script.py | 4 +- .../test_snapshot_files_check.py | 37 +------------------ 2 files changed, 4 insertions(+), 37 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index aecdab6..dc15851 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6027,14 +6027,14 @@ def apic_downgrade_compat_warning_check(cversion, tversion, **kwargs): @check_wrapper(check_title='Snapshot files check') -def snapshot_files_check(fabric_nodes, cversion, tversion, username, password, **kwargs): +def snapshot_files_check(fabric_nodes, tversion, username, password, **kwargs): result = PASS headers = ['apic_id', 'apic_name', 'snapshot_files'] data = [] recommended_action = 'Contact Cisco TAC for Support before upgrade' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#Snapshot-files-check' - if cversion.older_than('6.0(3d)') or tversion.older_than('6.0(3d)'): + if tversion.older_than('6.0(3d)'): apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] if not apics: return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py index b5ce533..bf9cba5 100644 --- a/tests/checks/snapshot_files_check/test_snapshot_files_check.py +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -79,14 +79,13 @@ """ @pytest.mark.parametrize( - "icurl_outputs, conn_failure, conn_cmds, cversion, tversion, fabric_nodes, expected_result", + "icurl_outputs, conn_failure, conn_cmds, tversion, fabric_nodes, expected_result", [ # Version not affected (6.0(3d) or newer) ( {}, False, [], - "6.0(3d)", "6.0(3e)", read_data(dir, "fabricNode.json"), script.NA, @@ -106,7 +105,6 @@ for apic_ip in apic_ips }, "5.2(1h)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.PASS, ), @@ -138,7 +136,6 @@ ], }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -170,7 +167,6 @@ ], }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -180,7 +176,6 @@ True, [], "5.2(1h)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -199,7 +194,6 @@ for apic_ip in apic_ips }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -218,7 +212,6 @@ for apic_ip in apic_ips }, "5.2(1h)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -237,7 +230,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.PASS, ), @@ -256,7 +248,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), @@ -266,7 +257,6 @@ True, [], "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -285,7 +275,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -304,7 +293,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -322,7 +310,6 @@ ] }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.FAIL_UF, ), @@ -340,7 +327,6 @@ ] }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.PASS, ), @@ -350,7 +336,6 @@ True, [], "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.ERROR, ), @@ -368,7 +353,6 @@ ] }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode_single_apic.json"), script.ERROR, ), @@ -400,7 +384,6 @@ ], }, "5.2(8g)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.ERROR, ), @@ -419,7 +402,6 @@ for apic_ip in apic_ips }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.PASS, ), @@ -438,7 +420,6 @@ for apic_ip in apic_ips }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -457,7 +438,6 @@ for apic_ip in apic_ips }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.PASS, ), @@ -476,7 +456,6 @@ for apic_ip in apic_ips }, "5.2(8g)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -508,7 +487,6 @@ ], }, "6.0(2a)", - "6.0(3d)", read_data(dir, "fabricNode.json"), script.FAIL_UF, ), @@ -526,7 +504,6 @@ ] }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.FAIL_UF, ), @@ -544,7 +521,6 @@ ] }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.PASS, ), @@ -554,7 +530,6 @@ True, [], "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.ERROR, ), @@ -572,7 +547,6 @@ ] }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old_single_apic.json"), script.ERROR, ), @@ -604,7 +578,6 @@ ], }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), @@ -636,7 +609,6 @@ ], }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.ERROR, ), @@ -655,7 +627,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.PASS, ), @@ -674,7 +645,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), @@ -693,7 +663,6 @@ for apic_ip in apic_ips }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.PASS, ), @@ -725,15 +694,13 @@ ], }, "3.2(6o)", - "6.0(3d)", read_data(dir, "fabricNode_old.json"), script.FAIL_UF, ), ], ) -def test_snapshot_files_check(run_check, mock_icurl, mock_conn, cversion, tversion, fabric_nodes, expected_result): +def test_snapshot_files_check(run_check, mock_icurl, mock_conn, tversion, fabric_nodes, expected_result): result = run_check( - cversion=script.AciVersion(cversion), tversion=script.AciVersion(tversion), username="fake_username", password="fake_password", From 5efb105a42e37d6057861c986404311093c5b4e3 Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Tue, 3 Feb 2026 11:14:01 +0000 Subject: [PATCH 7/7] modified validations.md file for the check --- docs/docs/validations.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 6daecab..7bd0c79 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2651,14 +2651,15 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC ### Snapshot files check + RCA: -Issue occured in 3 node apic cluster, AE process on one apic is busy. Reason is it's trying to fetch the snapshot files which was taken earlier is missing on all apics. +Due to defect [CSCwe07002][62], APIC upgrades or downgrades may fail when the AE (Application Engine) process becomes stuck in an infinite retry loop attempting to download snapshot files that no longer exist but still have configuration references. IMPACT: -Access logs will be flooded with the GET calls. APIC Upgrade/downgrade will fail with message Installer Exited - Pre-upgrade callbacks were not completed. +This causes the AE process to remain busy, and the upgrade will fail with the error `Installer Exited - Pre-upgrade callbacks were not completed`. When this occurs, the APIC access logs will be flooded with failed GET requests for the missing snapshot file. Suggestion: -Restart AE on each APIC one at a time. For Reference [CSCwe07002][62]. +Prior to the fix, there was no limit on retry attempts. The fix in [CSCwe07002][62] limits snapshot file synchronization retries to 5 attempts. Cisco TAC must be contacted to restart the AE service as a workaround. [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script