diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 4b83f4c..dc15851 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6024,6 +6024,92 @@ def apic_downgrade_compat_warning_check(cversion, tversion, **kwargs): data.append([cversion, tversion, "Downgrading APIC from 6.2(1)+ to pre-6.2(1) will not be supported."]) return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + +@check_wrapper(check_title='Snapshot files check') +def snapshot_files_check(fabric_nodes, tversion, username, password, **kwargs): + result = PASS + headers = ['apic_id', 'apic_name', 'snapshot_files'] + data = [] + recommended_action = 'Contact Cisco TAC for Support before upgrade' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#Snapshot-files-check' + + if tversion.older_than('6.0(3d)'): + apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] + if not apics: + return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) + # `fabricNode` in pre-4.0 does not have `address` + if not apics[0]["fabricNode"]["attributes"].get("address"): + apic1 = [apic for apic in apics if apic["fabricNode"]["attributes"]["id"] == "1"][0] + apic1_dn = apic1["fabricNode"]["attributes"]["dn"] + apics = icurl("class", "{}/infraWiNode.json".format(apic1_dn)) + has_error = False + for apic in apics: + if apic.get("fabricNode"): + apic_id = apic["fabricNode"]["attributes"]["id"] + apic_name = apic["fabricNode"]["attributes"]["name"] + apic_addr = apic["fabricNode"]["attributes"]["address"] + else: + apic_id = apic["infraWiNode"]["attributes"]["id"] + apic_name = apic["infraWiNode"]["attributes"]["nodeName"] + apic_addr = apic["infraWiNode"]["attributes"]["addr"] + try: + c = Connection(apic_addr) + c.username = username + c.password = password + c.log = LOG_FILE + c.connect() + except Exception as e: + data.append([apic_id, apic_name, str(e)]) + has_error = True + continue + try: + c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') + access_logs = c.output.splitlines() + if len(access_logs) < 15 and any("No such file or directory" in line for line in access_logs): + data.append([apic_id, apic_name, '/var/log/dme/log/access.log not found']) + has_error = True + continue + + requests = [] + + for line in access_logs: + timestamp_match = re.search(r'\[(\d{1,2}/\w{3}/\d{4}):(\d{2}:\d{2}:\d{2})', line) + filename_match = re.search(r'GET /snapshots/([^\s]+)', line) + + if timestamp_match and filename_match: + timestamp_str = f"{timestamp_match.group(1)}:{timestamp_match.group(2)}" + filename = filename_match.group(1) + try: + timestamp = datetime.strptime(timestamp_str, "%d/%b/%Y:%H:%M:%S") + requests.append((timestamp, filename)) + except: + continue + + requests.sort() + + # Checking if any 10 consecutive requests are within 1 minute + if len(requests) >= 10: + for i in range(len(requests) - 9): + time_diff = (requests[i + 9][0] - requests[i][0]).total_seconds() + if time_diff <= 60: + window_files = [filename for _, filename in requests[i:i+10]] + for filename in window_files: + data.append([apic_id, apic_name, filename]) + result = FAIL_UF + break + + except Exception as e: + data.append([apic_id, apic_name, str(e)]) + has_error = True + continue + + if has_error and result == PASS: + result = ERROR + else: + result = NA + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) # ---- Script Execution ---- @@ -6200,6 +6286,7 @@ class CheckManager: # Bugs observer_db_size_check, + snapshot_files_check, ] cli_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 68ca1c0..7bd0c79 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -193,6 +193,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[Snapshot files check][d29] | CSCwe07002 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -222,7 +223,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash - +[d29]: #Snapshot-files-check ## General Check Details @@ -2648,6 +2649,19 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### Snapshot files check + + +RCA: +Due to defect [CSCwe07002][62], APIC upgrades or downgrades may fail when the AE (Application Engine) process becomes stuck in an infinite retry loop attempting to download snapshot files that no longer exist but still have configuration references. + +IMPACT: +This causes the AE process to remain busy, and the upgrade will fail with the error `Installer Exited - Pre-upgrade callbacks were not completed`. When this occurs, the APIC access logs will be flooded with failed GET requests for the missing snapshot file. + +Suggestion: +Prior to the fix, there was no limit on retry attempts. The fix in [CSCwe07002][62] limits snapshot file synchronization retries to 5 attempts. Cisco TAC must be contacted to restart the AE service as a workaround. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2710,3 +2724,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression +[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe07002 \ No newline at end of file diff --git a/tests/checks/snapshot_files_check/fabricNode.json b/tests/checks/snapshot_files_check/fabricNode.json new file mode 100644 index 0000000..962a4ad --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode.json @@ -0,0 +1,93 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.2", + "dn": "topology/pod-1/node-2", + "fabricSt": "commissioned", + "id": "2", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic2", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.3", + "dn": "topology/pod-2/node-3", + "fabricSt": "commissioned", + "id": "3", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic3", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/snapshot_files_check/fabricNode_no_apic.json b/tests/checks/snapshot_files_check/fabricNode_no_apic.json new file mode 100644 index 0000000..254f40d --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_no_apic.json @@ -0,0 +1,48 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/snapshot_files_check/fabricNode_old.json b/tests/checks/snapshot_files_check/fabricNode_old.json new file mode 100644 index 0000000..f71fb9f --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_old.json @@ -0,0 +1,62 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-2", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "2", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic2", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-2/node-3", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "3", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic3", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "", + "role": "leaf", + "adSt": "on", + "name": "leaf1", + "model": "N9K-C9396PX" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json new file mode 100644 index 0000000..5e5f451 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "A", + "role": "leaf", + "adSt": "on", + "name": "leaf101", + "model": "N9K-C93180YC-EX" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/fabricNode_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_single_apic.json new file mode 100644 index 0000000..cf35918 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-EX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/infraWiNode_apic1.json b/tests/checks/snapshot_files_check/infraWiNode_apic1.json new file mode 100644 index 0000000..b6626d0 --- /dev/null +++ b/tests/checks/snapshot_files_check/infraWiNode_apic1.json @@ -0,0 +1,62 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.2", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-2", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "2", + "mbSn": "FCH1235ABCD", + "name": "", + "nodeName": "apic2", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.3", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-3", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "3", + "mbSn": "FCH1236ABCD", + "name": "", + "nodeName": "apic3", + "operSt": "available", + "podId": "1", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/infraWiNode_single_apic.json b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json new file mode 100644 index 0000000..fe17cdf --- /dev/null +++ b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json @@ -0,0 +1,22 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py new file mode 100644 index 0000000..bf9cba5 --- /dev/null +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -0,0 +1,709 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "snapshot_files_check" +infraWiNode = "topology/pod-1/node-1/infraWiNode.json" + +apic_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] +apic_single_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode_single_apic.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] + +grep_cmd = 'tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404' + +# Sample log output with 10+ requests within 1 minute (issue detected) +grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-12.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-30-18.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-24.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_config_export-2025-12-24T14-30-30.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-36.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_fabric_backup-2025-12-24T14-30-42.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-48.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_scheduler_backup-2025-12-24T14-30-54.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:00 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-31-00.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:06 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-06.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Sample log output with less than 10 requests or spread over more than 1 minute (no issue) +grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +grep_output_no_file = """\ +grep: /var/log/dme/log/access.log: No such file or directory +fabric-apic# +""" + +# Edge case: Empty log output (no 404 requests found) +grep_output_empty = "" + +# Edge case: Exactly 10 requests within 60 seconds +grep_output_boundary_fail = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_snapshot10.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Edge case: 9 requests within 60 seconds +grep_output_boundary_pass = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:08 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:16 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:32 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:40 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:56 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:04 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, tversion, fabric_nodes, expected_result", + [ + # Version not affected (6.0(3d) or newer) + ( + {}, + False, + [], + "6.0(3e)", + read_data(dir, "fabricNode.json"), + script.NA, + ), + # Version affected, but no issues found + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, issue detected (10+ requests in 1 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Version affected, issue detected (10+ requests in 1 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Connection failure + ( + {}, + True, + [], + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Exception during grep command execution + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # error (no such file or directory for cmd execution) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Pass (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # fail (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # connection failure(pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # exception during grep command execution (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # error (pre-4.0 with infraWiNode and no such file or directory for cmd execution) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case: Single APIC with issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.FAIL_UF, + ), + # Edge case: Single APIC without issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.PASS, + ), + # Edge case: Single APIC with connection failure + ( + {}, + True, + [], + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Single APIC with file not found + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with one file not found, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Empty log output (no 404 requests) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: 9 requests within 60 seconds (boundary - should pass) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Multi-APIC with all having issues + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with mixed issues (issue + file not found + clean) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Single APIC with issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Single APIC without issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.PASS, + ), + # Edge case (pre-4.0): Single APIC with connection failure + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with file not found + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with first APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with one file not found, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Empty log output (no 404 requests) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): 9 requests within 60 seconds + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Multi-APIC with mixed issues (issue + file not found + clean) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + ], +) +def test_snapshot_files_check(run_check, mock_icurl, mock_conn, tversion, fabric_nodes, expected_result): + result = run_check( + tversion=script.AciVersion(tversion), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result \ No newline at end of file