diff --git a/nodescraper/plugins/inband/network/collector_args.py b/nodescraper/plugins/inband/network/collector_args.py index 61ca7f23..1c0cfc89 100644 --- a/nodescraper/plugins/inband/network/collector_args.py +++ b/nodescraper/plugins/inband/network/collector_args.py @@ -40,3 +40,7 @@ class NetworkCollectorArgs(CollectorArgs): default=None, description="Tool to use for network connectivity probe: ping, wget, or curl.", ) + exclusion_regex: Optional[list[str]] = Field( + default=None, + description="Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it).", + ) diff --git a/nodescraper/plugins/inband/network/ethtool_vendor.py b/nodescraper/plugins/inband/network/ethtool_vendor.py index 47e91be9..d43791b4 100644 --- a/nodescraper/plugins/inband/network/ethtool_vendor.py +++ b/nodescraper/plugins/inband/network/ethtool_vendor.py @@ -111,9 +111,15 @@ class PollaraEthtoolStatistics(BaseModel): "frames_rx_dropped", "frames_rx_less_than_64b", "frames_tx_bad", + "frames_tx_less_than_64b", + "frames_tx_truncated", + ] + + warning_fields: ClassVar[list[str]] = [ + "frames_rx_pause", + "frames_rx_pripause", "frames_tx_pause", "frames_tx_pripause", - "frames_tx_less_than_64b", "frames_tx_pri_0", "frames_tx_pri_1", "frames_tx_pri_2", @@ -130,12 +136,6 @@ class PollaraEthtoolStatistics(BaseModel): "tx_pripause_5_1us_count", "tx_pripause_6_1us_count", "tx_pripause_7_1us_count", - "frames_tx_truncated", - ] - - warning_fields: ClassVar[list[str]] = [ - "frames_rx_pause", - "frames_rx_pripause", "frames_rx_pri_0", "frames_rx_pri_1", "frames_rx_pri_2", @@ -274,28 +274,14 @@ class Thor2EthtoolStatistics(BaseModel): "rx_runt_frames", "rx_stat_discard", "rx_stat_err", - "tx_pause_frames", - "tx_pfc_frames", "tx_jabber_frames", "tx_fcs_err_frames", "tx_err", "tx_fifo_underruns", - "tx_pfc_ena_frames_pri0", - "tx_pfc_ena_frames_pri1", - "tx_pfc_ena_frames_pri2", - "tx_pfc_ena_frames_pri3", - "tx_pfc_ena_frames_pri4", - "tx_pfc_ena_frames_pri5", - "tx_pfc_ena_frames_pri6", - "tx_pfc_ena_frames_pri7", "tx_total_collisions", "tx_stat_discard", "tx_stat_error", "link_down_events", - "continuous_pause_events", - "resume_pause_events", - "continuous_roce_pause_events", - "resume_roce_pause_events", "rx_pcs_symbol_err", "rx_discard_bytes_cos0", "rx_discard_packets_cos0", @@ -315,14 +301,6 @@ class Thor2EthtoolStatistics(BaseModel): "rx_discard_packets_cos7", "rx_fec_uncorrectable_blocks", "rx_filter_miss", - "pfc_pri0_tx_transitions", - "pfc_pri1_tx_transitions", - "pfc_pri2_tx_transitions", - "pfc_pri3_tx_transitions", - "pfc_pri4_tx_transitions", - "pfc_pri5_tx_transitions", - "pfc_pri6_tx_transitions", - "pfc_pri7_tx_transitions", "hw_db_recov_dbs_dropped", "hw_db_recov_oo_drop_count", "lpbk_tx_discards", @@ -352,6 +330,28 @@ class Thor2EthtoolStatistics(BaseModel): "pfc_pri5_rx_transitions", "pfc_pri6_rx_transitions", "pfc_pri7_rx_transitions", + "tx_pause_frames", + "tx_pfc_frames", + "tx_pfc_ena_frames_pri0", + "tx_pfc_ena_frames_pri1", + "tx_pfc_ena_frames_pri2", + "tx_pfc_ena_frames_pri3", + "tx_pfc_ena_frames_pri4", + "tx_pfc_ena_frames_pri5", + "tx_pfc_ena_frames_pri6", + "tx_pfc_ena_frames_pri7", + "continuous_pause_events", + "resume_pause_events", + "continuous_roce_pause_events", + "resume_roce_pause_events", + "pfc_pri0_tx_transitions", + "pfc_pri1_tx_transitions", + "pfc_pri2_tx_transitions", + "pfc_pri3_tx_transitions", + "pfc_pri4_tx_transitions", + "pfc_pri5_tx_transitions", + "pfc_pri6_tx_transitions", + "pfc_pri7_tx_transitions", ] @@ -513,7 +513,6 @@ class Cx7EthtoolStatistics(BaseModel): "rx_oversize_pkts_phy", "rx_symbol_err_phy", "rx_unsupported_op_phy", - "tx_pause_ctrl_phy", "rx_discards_phy", "tx_discards_phy", "tx_errors_phy", @@ -536,24 +535,6 @@ class Cx7EthtoolStatistics(BaseModel): "rx_prio5_discards", "rx_prio6_discards", "rx_prio7_discards", - "tx_global_pause", - "tx_prio0_pause", - "tx_prio1_pause", - "tx_prio2_pause", - "tx_prio3_pause", - "tx_prio4_pause", - "tx_prio5_pause", - "tx_prio6_pause", - "tx_prio7_pause", - "tx_global_pause_duration", - "tx_prio0_pause_duration", - "tx_prio1_pause_duration", - "tx_prio2_pause_duration", - "tx_prio3_pause_duration", - "tx_prio4_pause_duration", - "tx_prio5_pause_duration", - "tx_prio6_pause_duration", - "tx_prio7_pause_duration", "tx_pause_storm_warning_events", "tx_pause_storm_error_events", "module_unplug", @@ -619,6 +600,25 @@ class Cx7EthtoolStatistics(BaseModel): "rx_prio5_pause_duration", "rx_prio6_pause_duration", "rx_prio7_pause_duration", + "tx_pause_ctrl_phy", + "tx_global_pause", + "tx_prio0_pause", + "tx_prio1_pause", + "tx_prio2_pause", + "tx_prio3_pause", + "tx_prio4_pause", + "tx_prio5_pause", + "tx_prio6_pause", + "tx_prio7_pause", + "tx_global_pause_duration", + "tx_prio0_pause_duration", + "tx_prio1_pause_duration", + "tx_prio2_pause_duration", + "tx_prio3_pause_duration", + "tx_prio4_pause_duration", + "tx_prio5_pause_duration", + "tx_prio6_pause_duration", + "tx_prio7_pause_duration", ] diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 27280c37..2967df59 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -23,7 +23,6 @@ # SOFTWARE. # ############################################################################### -import re from typing import Optional from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer @@ -35,40 +34,26 @@ class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]): - """Check network statistics for errors (PFC and other network error counters).""" + """Check network statistics for errors.""" DATA_MODEL = NetworkDataModel - # Regex patterns for error fields checked from network statistics - ERROR_REGEX: list[ErrorRegex] = [ - ErrorRegex( - regex=re.compile(r"^tx_pfc_frames$"), - message="tx_pfc_frames is non-zero", - event_category=EventCategory.NETWORK, - ), - ErrorRegex( - regex=re.compile(r"^tx_pfc_ena_frames_pri\d+$"), - message="tx_pfc_ena_frames_pri* is non-zero", - event_category=EventCategory.NETWORK, - ), - ErrorRegex( - regex=re.compile(r"^pfc_pri\d+_tx_transitions$"), - message="pfc_pri*_tx_transitions is non-zero", - event_category=EventCategory.NETWORK, - ), - ] + # No built-in regex patterns: RDMA-scoped counter classification lives in the vendor + # ethtool models (ethtool_vendor.py). This list is only extended by user-supplied + # NetworkAnalyzerArgs.error_regex patterns. + ERROR_REGEX: list[ErrorRegex] = [] def analyze_data( self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None ) -> TaskResult: - """Analyze ethtool -S statistics: regex-based (per interface) and vendor-based (RDMA-scoped). + """Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex. Args: data: Network data model with ethtool_info and/or rdma_ethtool_statistics. args: Optional analyzer arguments with custom error regex support. Returns: - TaskResult with OK, WARNING (no data or vendor warning counters only), or ERROR. + TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ if not data.ethtool_info and not data.rdma_ethtool_statistics: self.result.message = "No network devices found" @@ -81,8 +66,9 @@ def analyze_data( final_error_regex = self._convert_and_extend_error_regex(args.error_regex, self.ERROR_REGEX) regex_error = False + regex_warning = False for interface_name, ethtool_info in data.ethtool_info.items(): - errors_on_interface: list[tuple[str, int]] = [] + matches_on_interface: list[tuple[str, int, EventPriority]] = [] for stat_name, stat_value in ethtool_info.statistics.items(): for error_regex_obj in final_error_regex: if error_regex_obj.regex.match(stat_name): @@ -92,26 +78,38 @@ def analyze_data( break if value > 0: - errors_on_interface.append((stat_name, value)) + matches_on_interface.append( + (stat_name, value, error_regex_obj.event_priority) + ) break - if errors_on_interface: - regex_error = True - error_names = [e[0] for e in errors_on_interface] - errors_data = {field: value for field, value in errors_on_interface} + if matches_on_interface: + has_error = any( + priority == EventPriority.ERROR for _, _, priority in matches_on_interface + ) + if has_error: + regex_error = True + else: + regex_warning = True + priority = EventPriority.ERROR if has_error else EventPriority.WARNING + severity = "error" if has_error else "warning" + match_names = [match[0] for match in matches_on_interface] + matches_data = {name: value for name, value, _ in matches_on_interface} self._log_event( category=EventCategory.NETWORK, - description=f"Network error detected on {interface_name}: [{', '.join(error_names)}]", + description=f"Network {severity} detected on {interface_name}: [{', '.join(match_names)}]", data={ "interface": interface_name, - "errors": errors_data, + "errors": matches_data, }, - priority=EventPriority.ERROR, + priority=priority, console_log=True, ) vendor_error = False vendor_warning = False + vendor_error_fields: set[str] = set() + vendor_warning_fields: set[str] = set() for stat in data.rdma_ethtool_statistics: if stat.vendor_statistics is None: continue @@ -127,13 +125,22 @@ def analyze_data( priority = EventPriority.WARNING if is_warning_tier else EventPriority.ERROR if is_warning_tier: vendor_warning = True + vendor_warning_fields.add(field_name) else: vendor_error = True + vendor_error_fields.add(field_name) + # Use a single grouped description per severity so the run summary + # collapses every occurrence into one "Ethtool warning detected" / + # "Ethtool error detected" entry instead of one line per field. The + # specific field is still preserved in the event data below and in + # the consolidated console line emitted after the loop. desc = ( - f"Ethtool warning detected: {field_name}" - if is_warning_tier - else f"Ethtool error detected: {field_name}" + "Ethtool warning detected" if is_warning_tier else "Ethtool error detected" ) + # Per-field events are still recorded for the run summary and event + # log, but console logging is suppressed here to avoid repeating the + # same message once per device. A single consolidated line is emitted + # after the loop instead. self._log_event( category=EventCategory.NETWORK, description=desc, @@ -144,16 +151,24 @@ def analyze_data( "error_count": error_value, }, priority=priority, - console_log=True, + console_log=False, ) + if vendor_error: + self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields))) + if vendor_warning: + self.logger.warning( + "Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields)) + ) + if regex_error or vendor_error: self.result.message = "Network errors detected in statistics" self.result.status = ExecutionStatus.ERROR - elif vendor_warning: - self.result.message = "Network vendor ethtool warning counters non-zero" + elif regex_warning or vendor_warning: + self.result.message = "Network warning counters non-zero in statistics" self.result.status = ExecutionStatus.WARNING else: self.result.message = "No network errors detected in statistics" self.result.status = ExecutionStatus.OK + return self.result diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 7e5e4a39..b960e574 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -30,7 +30,6 @@ from pydantic import ValidationError from nodescraper.base import InBandDataCollector -from nodescraper.connection.inband import TextFileArtifact from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult from nodescraper.utils import get_exception_traceback @@ -483,36 +482,34 @@ def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, st stats_dict[key.strip()] = value.strip() return stats_dict - def _collect_ethtool_info(self, interfaces: List[NetworkInterface]) -> Dict[str, EthtoolInfo]: + def _collect_ethtool_info( + self, + interfaces: List[NetworkInterface], + exclusions: Optional[List[re.Pattern]] = None, + ) -> Tuple[Dict[str, EthtoolInfo], set[str]]: """Collect ethtool information for all network interfaces. Args: interfaces: List of NetworkInterface objects to collect ethtool info for + exclusions: Compiled regex patterns; interfaces whose name matches any + pattern are skipped (ethtool is not run against them). Returns: - Dictionary mapping interface name to EthtoolInfo + Tuple of (dictionary mapping interface name to EthtoolInfo, set of + interface names skipped due to an exclusion regex) """ ethtool_data = {} + skipped: set[str] = set() for iface in interfaces: + if exclusions and any(pattern.search(iface.name) for pattern in exclusions): + skipped.add(iface.name) + continue cmd = self.CMD_ETHTOOL_TEMPLATE.format(interface=iface.name) res_ethtool = self._run_sut_cmd(cmd, sudo=True) if res_ethtool.exit_code == 0: ethtool_info = self._parse_ethtool(iface.name, res_ethtool.stdout) - # Collect ethtool -S (statistics) for error/health analysis - cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=iface.name) - res_ethtool_s = self._run_sut_cmd(cmd_s, sudo=True) - if res_ethtool_s.exit_code == 0 and res_ethtool_s.stdout: - ethtool_info.statistics = self._parse_ethtool_statistics( - res_ethtool_s.stdout, iface.name - ) - self.result.artifacts.append( - TextFileArtifact( - filename=f"{iface.name}.log", - contents=res_ethtool_s.stdout, - ) - ) ethtool_data[iface.name] = ethtool_info self._log_event( category=EventCategory.NETWORK, @@ -527,7 +524,7 @@ def _collect_ethtool_info(self, interfaces: List[NetworkInterface]) -> Dict[str, priority=EventPriority.WARNING, ) - return ethtool_data + return ethtool_data, skipped def _collect_rdma_link_json(self) -> Optional[list[dict]]: """Parse JSON from `rdma link -j`. Returns None on failure, [] when no links.""" @@ -570,7 +567,7 @@ def _collect_rdma_scoped_ethtool_statistic( self, netdev: str, ifname: str ) -> Optional[EthtoolStatistics]: """Run `ethtool -S` for netdev and attach vendor-parsed stats (prefix from RDMA ifname).""" - cmd_s = f"ethtool -S {netdev}" + cmd_s = self.CMD_ETHTOOL_S_TEMPLATE.format(interface=netdev) res = self._run_sut_cmd(cmd_s, sudo=True) if res.exit_code != 0: self._log_event( @@ -586,13 +583,6 @@ def _collect_rdma_scoped_ethtool_statistic( ) return None - if res.stdout: - self.result.artifacts.append( - TextFileArtifact( - filename=f"rdma-ethtool-{netdev}.log", - contents=res.stdout, - ) - ) stats_dict = self._parse_ethtool_statistics(res.stdout, netdev) vendor_stats: Optional[VendorEthtoolStatisticsModel] = None @@ -634,14 +624,26 @@ def _collect_rdma_scoped_ethtool_statistic( vendor_statistics=vendor_stats, ) - def _collect_rdma_scoped_ethtool(self) -> tuple[List[str], List[EthtoolStatistics]]: - """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity).""" + def _collect_rdma_scoped_ethtool( + self, exclusions: Optional[List[re.Pattern]] = None + ) -> tuple[List[str], List[EthtoolStatistics], set[str]]: + """Collect ethtool -S for netdevs listed on RDMA links (error-scraper EthtoolCollector parity). + + Args: + exclusions: Compiled regex patterns; netdevs whose name matches any pattern + are skipped (ethtool -S is not run against them). + + Returns: + Tuple of (netdev list, statistics list, set of netdev names skipped due to + an exclusion regex) + """ netdev_list: List[str] = [] statistics_list: List[EthtoolStatistics] = [] + skipped: set[str] = set() link_data = self._collect_rdma_link_json() if link_data is None: - return netdev_list, statistics_list + return netdev_list, statistics_list, skipped for link in link_data: if not isinstance(link, dict): @@ -657,6 +659,9 @@ def _collect_rdma_scoped_ethtool(self) -> tuple[List[str], List[EthtoolStatistic ifname = link.get("ifname") or "" if netdev: + if exclusions and any(pattern.search(netdev) for pattern in exclusions): + skipped.add(netdev) + continue netdev_list.append(netdev) stat = self._collect_rdma_scoped_ethtool_statistic(netdev, ifname) if stat is not None: @@ -672,7 +677,7 @@ def _collect_rdma_scoped_ethtool(self) -> tuple[List[str], List[EthtoolStatistic priority=EventPriority.INFO, ) - return netdev_list, statistics_list + return netdev_list, statistics_list, skipped def _collect_lldp_info(self) -> None: """Collect LLDP information using lldpcli and lldpctl commands.""" @@ -771,10 +776,16 @@ def collect_data( routes = [] rules = [] neighbors = [] - ethtool_data = {} + ethtool_data: Dict[str, EthtoolInfo] = {} network_accessible: Optional[bool] = None rdma_ethtool_netdevs: List[str] = [] rdma_ethtool_statistics: List[EthtoolStatistics] = [] + skipped_devices: set[str] = set() + + # Compile device-exclusion regex patterns (netdevs matching any pattern are skipped) + compiled_exclusions = [ + re.compile(pattern) for pattern in ((args.exclusion_regex if args else None) or []) + ] # Check network connectivity if URL is provided if args and args.url: @@ -812,7 +823,10 @@ def collect_data( # Collect ethtool information for interfaces if interfaces: - ethtool_data = self._collect_ethtool_info(interfaces) + ethtool_data, ethtool_skipped = self._collect_ethtool_info( + interfaces, compiled_exclusions + ) + skipped_devices |= ethtool_skipped self._log_event( category=EventCategory.NETWORK, description=f"Collected ethtool info for {len(ethtool_data)} interfaces", @@ -820,7 +834,12 @@ def collect_data( ) if self.system_info.os_family == OSFamily.LINUX: - rdma_ethtool_netdevs, rdma_ethtool_statistics = self._collect_rdma_scoped_ethtool() + ( + rdma_ethtool_netdevs, + rdma_ethtool_statistics, + rdma_skipped, + ) = self._collect_rdma_scoped_ethtool(compiled_exclusions) + skipped_devices |= rdma_skipped # Collect routing table res_route = self._run_sut_cmd(self.CMD_ROUTE) @@ -877,6 +896,8 @@ def collect_data( self._collect_lldp_info() self.result.message = "Network data collected successfully" + if skipped_devices: + self.result.message += f" ({len(skipped_devices)} skipped)" network_data = NetworkDataModel( interfaces=interfaces, diff --git a/nodescraper/plugins/inband/rdma/analyzer_args.py b/nodescraper/plugins/inband/rdma/analyzer_args.py new file mode 100644 index 00000000..0507ed15 --- /dev/null +++ b/nodescraper/plugins/inband/rdma/analyzer_args.py @@ -0,0 +1,39 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class RdmaAnalyzerArgs(AnalyzerArgs): + """Arguments for the RDMA analyzer.""" + + exclusion_regex: Optional[list[str]] = Field( + default=None, + description="Regex patterns matched against an interface netdev; matching interfaces are skipped.", + ) diff --git a/nodescraper/plugins/inband/rdma/rdma_analyzer.py b/nodescraper/plugins/inband/rdma/rdma_analyzer.py index 00f6977f..ba48c20e 100644 --- a/nodescraper/plugins/inband/rdma/rdma_analyzer.py +++ b/nodescraper/plugins/inband/rdma/rdma_analyzer.py @@ -23,21 +23,25 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.interfaces import DataAnalyzer from nodescraper.models import TaskResult +from .analyzer_args import RdmaAnalyzerArgs from .rdmadata import RdmaDataModel -class RdmaAnalyzer(DataAnalyzer[RdmaDataModel, None]): +class RdmaAnalyzer(DataAnalyzer[RdmaDataModel, RdmaAnalyzerArgs]): """Check RDMA statistics for errors (RoCE and other RDMA error counters).""" DATA_MODEL = RdmaDataModel - def analyze_data(self, data: RdmaDataModel, args: Optional[None] = None) -> TaskResult: + def analyze_data( + self, data: RdmaDataModel, args: Optional[RdmaAnalyzerArgs] = None + ) -> TaskResult: """Analyze RDMA statistics for non-zero error counters. Error and critical counter names come from each vendor's statistics model @@ -55,42 +59,76 @@ def analyze_data(self, data: RdmaDataModel, args: Optional[None] = None) -> Task self.result.status = ExecutionStatus.WARNING return self.result - error_state = False + if not args: + args = RdmaAnalyzerArgs() + + compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] + + error_detected = False + critical_detected = False + error_fields_seen: set[str] = set() + critical_fields_seen: set[str] = set() + skipped_count = 0 for stat in data.statistic_list: + # Skip this interface if its netdev matches any exclusion regex + if stat.netdev and any(pattern.search(stat.netdev) for pattern in compiled_exclusions): + skipped_count += 1 + continue + if stat.vendor_statistics is None: continue error_fields = stat.vendor_statistics.error_fields - critical_fields = stat.vendor_statistics.critial_error_fields + critical_fields = stat.vendor_statistics.critical_error_fields + detected_errors: dict[str, int] = {} + has_critical = False for error_field in error_fields + critical_fields: error_value = getattr(stat.vendor_statistics, error_field, None) - if error_value is not None and error_value > 0: - priority = ( - EventPriority.CRITICAL - if error_field in critical_fields - else EventPriority.ERROR - ) - self._log_event( - category=EventCategory.NETWORK, - description=f"RDMA error detected: {error_field}", - data={ - "interface": stat.ifname, - "port": stat.port, - "error_field": error_field, - "error_count": error_value, - }, - priority=priority, - console_log=True, - ) - error_state = True - - if error_state: + detected_errors[error_field] = error_value + if error_field in critical_fields: + has_critical = True + critical_detected = True + critical_fields_seen.add(error_field) + else: + error_detected = True + error_fields_seen.add(error_field) + + if not detected_errors: + continue + + priority = EventPriority.CRITICAL if has_critical else EventPriority.ERROR + desc = "RDMA critical error detected" if has_critical else "RDMA error detected" + self._log_event( + category=EventCategory.NETWORK, + description=desc, + data={ + "netdev": stat.netdev, + "interface": stat.ifname, + "port": stat.port, + "errors": detected_errors, + }, + priority=priority, + console_log=False, + ) + + if critical_detected: + self.logger.critical( + "RDMA critical error detected: %s", ", ".join(sorted(critical_fields_seen)) + ) + if error_detected: + self.logger.error("RDMA error detected: %s", ", ".join(sorted(error_fields_seen))) + + if error_detected or critical_detected: self.result.message = "RDMA errors detected in statistics" self.result.status = ExecutionStatus.ERROR else: self.result.message = "No RDMA errors detected in statistics" self.result.status = ExecutionStatus.OK + + if skipped_count: + self.result.message += f" ({skipped_count} skipped)" + return self.result diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 67f4073b..740f5998 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -289,9 +289,9 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: data={"exception": get_exception_traceback(e)}, priority=EventPriority.WARNING, ) - return links + return None - def collect_data(self, args: None = None) -> tuple[TaskResult, Optional[RdmaDataModel]]: + def collect_data(self, args=None) -> tuple[TaskResult, Optional[RdmaDataModel]]: """Collect RDMA statistics, link data, and device/link text output. Returns: @@ -301,6 +301,14 @@ def collect_data(self, args: None = None) -> tuple[TaskResult, Optional[RdmaData links = self._get_rdma_link() statistics = self._get_rdma_statistics() + # Cross-reference netdev from link data onto statistics (the + # 'rdma statistic' output does not include the netdev name). + if statistics and links: + netdev_map = {link.ifname: link.netdev for link in links if link.ifname is not None} + for stat in statistics: + if stat.ifname in netdev_map: + stat.netdev = netdev_map[stat.ifname] + dev_list: list[RdmaDevice] = [] res_rdma_dev = self._run_sut_cmd(self.CMD_RDMA_DEV) if res_rdma_dev.exit_code == 0: diff --git a/nodescraper/plugins/inband/rdma/rdma_plugin.py b/nodescraper/plugins/inband/rdma/rdma_plugin.py index fac85862..4bf5e8cc 100644 --- a/nodescraper/plugins/inband/rdma/rdma_plugin.py +++ b/nodescraper/plugins/inband/rdma/rdma_plugin.py @@ -25,14 +25,16 @@ ############################################################################### from nodescraper.base import InBandDataPlugin +from .analyzer_args import RdmaAnalyzerArgs from .rdma_analyzer import RdmaAnalyzer from .rdma_collector import RdmaCollector from .rdmadata import RdmaDataModel -class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, None]): +class RdmaPlugin(InBandDataPlugin[RdmaDataModel, None, RdmaAnalyzerArgs]): """Plugin for collection and analysis of RDMA statistics and link data.""" DATA_MODEL = RdmaDataModel COLLECTOR = RdmaCollector ANALYZER = RdmaAnalyzer + ANALYZER_ARGS = RdmaAnalyzerArgs diff --git a/nodescraper/plugins/inband/rdma/rdmadata.py b/nodescraper/plugins/inband/rdma/rdmadata.py index 965cb5b5..2859d325 100644 --- a/nodescraper/plugins/inband/rdma/rdmadata.py +++ b/nodescraper/plugins/inband/rdma/rdmadata.py @@ -131,7 +131,7 @@ class PollaraRdmaStatistics(BaseModel): "rx_rdma_mtu_discard_pkts", ] - critial_error_fields: ClassVar[list[str]] = [] + critical_error_fields: ClassVar[list[str]] = [] class Thor2RdmaStatistics(BaseModel): @@ -291,7 +291,7 @@ class Thor2RdmaStatistics(BaseModel): "rx_icrc_encapsulated", ] - critial_error_fields: ClassVar[list[str]] = [ + critical_error_fields: ClassVar[list[str]] = [ "unrecoverable_err", "res_tx_pci_err", "res_rx_pci_err", @@ -354,7 +354,7 @@ class Cx7RdmaStatistics(BaseModel): "rx_icrc_encapsulated", ] - critial_error_fields: ClassVar[list[str]] = [] + critical_error_fields: ClassVar[list[str]] = [] RdmaVendorStatistics = Union[PollaraRdmaStatistics, Thor2RdmaStatistics, Cx7RdmaStatistics] @@ -370,6 +370,7 @@ class Cx7RdmaStatistics(BaseModel): class RdmaStatistics(BaseModel): # Interface information ifname: Optional[str] = None + netdev: Optional[str] = None port: Optional[int] = None vendor_statistics: Optional[RdmaVendorStatistics] = None diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 6b7aeff3..832125e5 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -37,6 +37,16 @@ NetworkDataModel, ) +# Built-in regex defaults were removed; the vendor ethtool models own RDMA counter +# classification. These custom patterns exercise the user-supplied regex path. +CUSTOM_REGEX = [ + { + "regex": r"^custom_err_\d+$", + "message": "custom err counter non-zero", + "event_category": "NETWORK", + } +] + @pytest.fixture def network_analyzer(system_info): @@ -91,60 +101,70 @@ def test_no_errors_detected(network_analyzer, clean_network_model): assert len(result.events) == 0 -def test_single_error_detected(network_analyzer, clean_ethtool_info): - """Test with data containing a single error.""" - clean_ethtool_info.statistics["tx_pfc_frames"] = "5" - model = NetworkDataModel(ethtool_info={"eth0": clean_ethtool_info}) - result = network_analyzer.analyze_data(model) +def test_single_custom_match_detected(network_analyzer): + """A single non-zero counter matched by a user-supplied custom regex reports as ERROR.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_err_0": "5"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR - assert "Network errors detected in statistics" in result.message + assert "errors detected" in result.message assert len(result.events) == 1 - assert result.events[0].description == "Network error detected on eth0: [tx_pfc_frames]" + assert result.events[0].description == "Network error detected on eth0: [custom_err_0]" assert result.events[0].priority == EventPriority.ERROR - assert result.events[0].data["errors"] == {"tx_pfc_frames": 5} + assert result.events[0].data["errors"] == {"custom_err_0": 5} assert result.events[0].data["interface"] == "eth0" -def test_multiple_errors_same_interface(network_analyzer, clean_ethtool_info): - """Test with data containing multiple errors on the same interface.""" - clean_ethtool_info.statistics["tx_pfc_frames"] = "10" - clean_ethtool_info.statistics["tx_pfc_ena_frames_pri0"] = "3" - clean_ethtool_info.statistics["pfc_pri2_tx_transitions"] = "7" - model = NetworkDataModel(ethtool_info={"eth0": clean_ethtool_info}) - result = network_analyzer.analyze_data(model) +def test_multiple_matches_same_interface(network_analyzer): + """Multiple non-zero counters on one interface produce a single grouped event.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_err_0": "10", "custom_err_1": "3", "custom_err_2": "7"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR - assert "Network errors detected in statistics" in result.message + assert "errors detected" in result.message assert len(result.events) == 1 # one event per interface assert result.events[0].priority == EventPriority.ERROR - # Check all 3 errors are present + # Check all 3 counters are present assert len(result.events[0].data["errors"]) == 3 - assert result.events[0].data["errors"]["tx_pfc_frames"] == 10 - assert result.events[0].data["errors"]["tx_pfc_ena_frames_pri0"] == 3 - assert result.events[0].data["errors"]["pfc_pri2_tx_transitions"] == 7 + assert result.events[0].data["errors"]["custom_err_0"] == 10 + assert result.events[0].data["errors"]["custom_err_1"] == 3 + assert result.events[0].data["errors"]["custom_err_2"] == 7 -def test_multiple_interfaces_with_errors(network_analyzer): - """Test with errors across multiple interfaces.""" +def test_multiple_interfaces_with_matches(network_analyzer): + """Test with custom-regex matches across multiple interfaces.""" eth0 = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "15", - "tx_pfc_ena_frames_pri1": "0", + "custom_err_0": "15", + "custom_err_1": "0", }, ) eth1 = EthtoolInfo( interface="eth1", raw_output="dummy", statistics={ - "pfc_pri3_tx_transitions": "8", + "custom_err_3": "8", }, ) eth2 = EthtoolInfo( interface="eth2", raw_output="dummy", statistics={ - "tx_pfc_ena_frames_pri7": "100", + "custom_err_7": "100", }, ) model = NetworkDataModel( @@ -154,7 +174,9 @@ def test_multiple_interfaces_with_errors(network_analyzer): "eth2": eth2, } ) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 3 interfaces = {event.data["interface"] for event in result.events} @@ -174,14 +196,14 @@ def test_rdma_ethtool_vendor_error_only(network_analyzer): stat = EthtoolStatistics( netdev="eth0", rdma_ifname="bnxt0", - vendor_statistics=Thor2EthtoolStatistics(tx_pfc_frames=4), + vendor_statistics=Thor2EthtoolStatistics(rx_fcs_err_frames=4), ) model = NetworkDataModel(ethtool_info={}, rdma_ethtool_statistics=[stat]) result = network_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "Network errors detected" in result.message assert len(result.events) == 1 - assert result.events[0].data["error_field"] == "tx_pfc_frames" + assert result.events[0].data["error_field"] == "rx_fcs_err_frames" assert result.events[0].data["error_count"] == 4 assert result.events[0].priority == EventPriority.ERROR @@ -202,6 +224,15 @@ def test_rdma_ethtool_vendor_warning_only(network_analyzer): assert result.events[0].priority == EventPriority.WARNING +def test_thor2_tx_pause_counters_are_warning_tier(): + """Sync with error-scraper: TX pause/PFC counters are warning-tier, not error-tier.""" + assert "tx_pfc_frames" in Thor2EthtoolStatistics.warning_fields + assert "tx_pfc_frames" not in Thor2EthtoolStatistics.error_fields + assert "tx_pause_frames" in Thor2EthtoolStatistics.warning_fields + # RX-side hard error counters remain error-tier + assert "rx_fcs_err_frames" in Thor2EthtoolStatistics.error_fields + + def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): """RDMA ethtool row without parsed vendor statistics is ignored by vendor path.""" stat = EthtoolStatistics(netdev="eth0", rdma_ifname="unknown0", vendor_statistics=None) @@ -211,27 +242,27 @@ def test_rdma_ethtool_no_vendor_model_ok(network_analyzer): assert len(result.events) == 0 -def test_regex_patterns_priority_numbers(network_analyzer): - """Test that regex patterns match various priority numbers (0-7 and beyond).""" +def test_custom_regex_matches_multi_digit_suffixes(network_analyzer): + """Test that a user-supplied \\d+ pattern matches single- and double-digit suffixes.""" ethtool = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_ena_frames_pri0": "1", - "tx_pfc_ena_frames_pri3": "2", - "tx_pfc_ena_frames_pri7": "3", - "tx_pfc_ena_frames_pri10": "4", # Test double-digit - "pfc_pri0_tx_transitions": "5", - "pfc_pri5_tx_transitions": "6", - "pfc_pri15_tx_transitions": "7", # Test double-digit + "custom_err_0": "1", + "custom_err_3": "2", + "custom_err_7": "3", + "custom_err_10": "4", # Test double-digit + "custom_err_15": "5", # Test double-digit }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 - # All 7 errors should be detected - assert len(result.events[0].data["errors"]) == 7 + # All 5 counters should be detected + assert len(result.events[0].data["errors"]) == 5 def test_non_numeric_values_ignored(network_analyzer): @@ -240,18 +271,20 @@ def test_non_numeric_values_ignored(network_analyzer): interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "N/A", # Non-numeric - "tx_pfc_ena_frames_pri0": "invalid", # Non-numeric - "pfc_pri1_tx_transitions": "5", # Valid error + "custom_err_0": "N/A", # Non-numeric + "custom_err_1": "invalid", # Non-numeric + "custom_err_2": "5", # Valid }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 - # Only the valid numeric error should be reported + # Only the valid numeric counter should be reported assert len(result.events[0].data["errors"]) == 1 - assert result.events[0].data["errors"]["pfc_pri1_tx_transitions"] == 5 + assert result.events[0].data["errors"]["custom_err_2"] == 5 def test_zero_values_not_reported(network_analyzer): @@ -260,13 +293,15 @@ def test_zero_values_not_reported(network_analyzer): interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "0", - "tx_pfc_ena_frames_pri0": "0", - "pfc_pri1_tx_transitions": "0", + "custom_err_0": "0", + "custom_err_1": "0", + "custom_err_2": "0", }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 @@ -277,43 +312,45 @@ def test_non_matching_fields_ignored(network_analyzer): interface="eth0", raw_output="dummy", statistics={ - "rx_bytes": "999999999", # High value but not an error field - "tx_bytes": "888888888", # High value but not an error field - "some_random_counter": "12345", # Not an error field - "tx_pfc_frames": "5", # This SHOULD be detected + "rx_bytes": "999999999", # High value but not a matched field + "tx_bytes": "888888888", # High value but not a matched field + "some_random_counter": "12345", # Not a matched field + "custom_err_0": "5", # This SHOULD be detected }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR assert len(result.events) == 1 - # Only tx_pfc_frames should be reported + # Only custom_err_0 should be reported assert len(result.events[0].data["errors"]) == 1 - assert "tx_pfc_frames" in result.events[0].data["errors"] + assert "custom_err_0" in result.events[0].data["errors"] def test_mixed_interfaces_with_and_without_errors(network_analyzer): - """Test with some interfaces having errors and others clean.""" + """Test with some interfaces having matches and others clean.""" eth0_error = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ - "tx_pfc_frames": "10", + "custom_err_0": "10", }, ) eth1_clean = EthtoolInfo( interface="eth1", raw_output="dummy", statistics={ - "tx_pfc_frames": "0", - "tx_pfc_ena_frames_pri0": "0", + "custom_err_0": "0", + "custom_err_1": "0", }, ) eth2_error = EthtoolInfo( interface="eth2", raw_output="dummy", statistics={ - "pfc_pri5_tx_transitions": "20", + "custom_err_5": "20", }, ) model = NetworkDataModel( @@ -323,7 +360,9 @@ def test_mixed_interfaces_with_and_without_errors(network_analyzer): "eth2": eth2_error, } ) - result = network_analyzer.analyze_data(model) + result = network_analyzer.analyze_data( + model, args=NetworkAnalyzerArgs(error_regex=CUSTOM_REGEX) + ) assert result.status == ExecutionStatus.ERROR # Only 2 events (eth0 and eth2), eth1 should not generate an event assert len(result.events) == 2 @@ -332,13 +371,13 @@ def test_mixed_interfaces_with_and_without_errors(network_analyzer): def test_custom_error_regex_detected(network_analyzer): - """Test that custom regex in analyzer args is applied in addition to defaults.""" + """Test that user-supplied custom regex patterns are applied (no built-in defaults).""" ethtool = EthtoolInfo( interface="eth0", raw_output="dummy", statistics={ "custom_tx_drops": "9", # Matched via custom regex only - "tx_pfc_frames": "0", + "tx_pfc_frames": "0", # No built-in pattern; ignored unless RDMA vendor-scoped }, ) model = NetworkDataModel(ethtool_info={"eth0": ethtool}) @@ -358,3 +397,32 @@ def test_custom_error_regex_detected(network_analyzer): assert len(result.events) == 1 assert result.events[0].data["interface"] == "eth0" assert result.events[0].data["errors"] == {"custom_tx_drops": 9} + + +def test_custom_warning_priority_regex(network_analyzer): + """A user-supplied WARNING-priority custom pattern yields WARNING status via the regex path.""" + ethtool = EthtoolInfo( + interface="eth0", + raw_output="dummy", + statistics={"custom_warn": "3"}, + ) + model = NetworkDataModel(ethtool_info={"eth0": ethtool}) + args = NetworkAnalyzerArgs( + error_regex=[ + { + "regex": r"^custom_warn$", + "message": "custom warn counter", + "event_category": "NETWORK", + "event_priority": "WARNING", + } + ] + ) + + result = network_analyzer.analyze_data(model, args=args) + + assert result.status == ExecutionStatus.WARNING + assert "warning counters" in result.message + assert len(result.events) == 1 + assert result.events[0].priority == EventPriority.WARNING + assert result.events[0].description == "Network warning detected on eth0: [custom_warn]" + assert result.events[0].data["errors"] == {"custom_warn": 3} diff --git a/test/unit/plugin/test_rdma_analyzer.py b/test/unit/plugin/test_rdma_analyzer.py index ada67c04..1a3804a5 100644 --- a/test/unit/plugin/test_rdma_analyzer.py +++ b/test/unit/plugin/test_rdma_analyzer.py @@ -30,6 +30,7 @@ import pytest from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.plugins.inband.rdma.analyzer_args import RdmaAnalyzerArgs from nodescraper.plugins.inband.rdma.rdma_analyzer import RdmaAnalyzer from nodescraper.plugins.inband.rdma.rdmadata import ( VENDOR_PREFIX_MAP, @@ -103,9 +104,9 @@ def test_single_error_detected(rdma_analyzer, example_stat_dicts): assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message assert len(result.events) == 1 - assert result.events[0].description == "RDMA error detected: req_rx_pkt_seq_err" + assert result.events[0].description == "RDMA error detected" assert result.events[0].priority == EventPriority.ERROR - assert result.events[0].data["error_count"] == 5 + assert result.events[0].data["errors"]["req_rx_pkt_seq_err"] == 5 assert result.events[0].data["interface"] == "ionic_0" @@ -118,9 +119,17 @@ def test_multiple_errors_detected(rdma_analyzer, example_stat_dicts): result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message - assert len(result.events) == 3 + # Errors on the same interface are aggregated into a single event: + # ionic_0 (2 counters) + mlx5_0 (1 counter) -> 2 events + assert len(result.events) == 2 for event in result.events: assert event.priority == EventPriority.ERROR + events_by_iface = {event.data["interface"]: event for event in result.events} + assert events_by_iface["ionic_0"].data["errors"] == { + "req_rx_rmt_acc_err": 10, + "req_tx_loc_oper_err": 3, + } + assert events_by_iface["mlx5_0"].data["errors"] == {"packet_seq_err": 7} def test_critical_error_detected(rdma_analyzer): @@ -138,9 +147,10 @@ def test_critical_error_detected(rdma_analyzer): result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR assert "RDMA errors detected in statistics" in result.message - assert len(result.events) == 2 - critical_events = [e for e in result.events if e.priority == EventPriority.CRITICAL] - assert len(critical_events) == 2 + # One event per interface; escalated to CRITICAL because critical counters are set. + assert len(result.events) == 1 + assert result.events[0].priority == EventPriority.CRITICAL + assert result.events[0].data["errors"] == {"unrecoverable_err": 1, "res_tx_pci_err": 2} def test_empty_statistics(rdma_analyzer): @@ -184,7 +194,7 @@ def test_all_error_types(rdma_analyzer): model = RdmaDataModel(statistic_list=stats) result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 3 + assert len(result.events) == 2 interfaces = {event.data["interface"] for event in result.events} assert interfaces == {"ionic_test", "mlx5_test"} @@ -310,3 +320,60 @@ def test_rdma_link_multiple_interfaces(rdma_analyzer, clean_stats): result = rdma_analyzer.analyze_data(model) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 + + +def test_netdev_used_in_event(rdma_analyzer): + stats = [ + RdmaStatistics( + ifname="ionic_0", + netdev="benic8p1", + port=1, + vendor_statistics=PollaraRdmaStatistics(req_rx_pkt_seq_err=4), + ) + ] + model = RdmaDataModel(statistic_list=stats) + result = rdma_analyzer.analyze_data(model) + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert result.events[0].data["netdev"] == "benic8p1" + assert result.events[0].description == "RDMA error detected" + + +def test_exclusion_regex_skips_interface(rdma_analyzer): + stats = [ + RdmaStatistics( + ifname="ionic_0", + netdev="benic8p1", + port=1, + vendor_statistics=PollaraRdmaStatistics(req_rx_pkt_seq_err=4), + ), + RdmaStatistics( + ifname="mlx5_0", + netdev="benic9p1", + port=1, + vendor_statistics=Cx7RdmaStatistics(packet_seq_err=2), + ), + ] + model = RdmaDataModel(statistic_list=stats) + result = rdma_analyzer.analyze_data(model, RdmaAnalyzerArgs(exclusion_regex=["benic8"])) + assert result.status == ExecutionStatus.ERROR + # ionic_0 (netdev benic8p1) is excluded; only the mlx5_0 error is reported. + assert len(result.events) == 1 + assert result.events[0].data["interface"] == "mlx5_0" + assert "1 skipped" in result.message + + +def test_exclusion_regex_all_skipped(rdma_analyzer): + stats = [ + RdmaStatistics( + ifname="ionic_0", + netdev="benic8p1", + port=1, + vendor_statistics=PollaraRdmaStatistics(req_rx_pkt_seq_err=4), + ) + ] + model = RdmaDataModel(statistic_list=stats) + result = rdma_analyzer.analyze_data(model, RdmaAnalyzerArgs(exclusion_regex=["benic8p1"])) + assert result.status == ExecutionStatus.OK + assert len(result.events) == 0 + assert "1 skipped" in result.message diff --git a/test/unit/plugin/test_rdma_collector.py b/test/unit/plugin/test_rdma_collector.py index eb687c54..38d162ca 100644 --- a/test/unit/plugin/test_rdma_collector.py +++ b/test/unit/plugin/test_rdma_collector.py @@ -78,6 +78,8 @@ def test_collect_success(collector, conn_mock, rdma_link_output, rdma_statistic_ # Full link fixture has 4 ionic links assert len(data.link_list) == 4 assert data.link_list[0].ifname == "ionic_0" + # netdev is cross-referenced from link data onto statistics + assert data.statistic_list[0].netdev == "benic8p1" def test_collect_both_commands_fail(collector, conn_mock):