Skip to content

RDMA and Ethool enhancement #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sunnyhe2 merged 5 commits into from
Jul 7, 2026
Merged

RDMA and Ethool enhancement #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
136d54d
sync with errorscraper
sunnyhe2 Jul 6, 2026
d0b3cf6
Merge branch 'development' into sunny_rdmaethool_enhance
sunnyhe2 Jul 6, 2026
b6c59de
fix exclusion regex
sunnyhe2 Jul 6, 2026
e9c1c9f
removed empty collector arg
sunnyhe2 Jul 7, 2026
88e42f1
Merge branch 'development' into sunny_rdmaethool_enhance
sunnyhe2 Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions nodescraper/plugins/inband/network/collector_args.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,7 @@ class NetworkCollectorArgs(CollectorArgs):
default=None,
description="Tool to use for network connectivity probe: ping, wget, or curl.",
)
exclusion_regex: Optional[list[str]] = Field(
default=None,
description="Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it).",
)
96 changes: 48 additions & 48 deletions nodescraper/plugins/inband/network/ethtool_vendor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,15 @@ class PollaraEthtoolStatistics(BaseModel):
"frames_rx_dropped",
"frames_rx_less_than_64b",
"frames_tx_bad",
"frames_tx_less_than_64b",
"frames_tx_truncated",
]

warning_fields: ClassVar[list[str]] = [
"frames_rx_pause",
"frames_rx_pripause",
"frames_tx_pause",
"frames_tx_pripause",
"frames_tx_less_than_64b",
"frames_tx_pri_0",
"frames_tx_pri_1",
"frames_tx_pri_2",
Expand All @@ -130,12 +136,6 @@ class PollaraEthtoolStatistics(BaseModel):
"tx_pripause_5_1us_count",
"tx_pripause_6_1us_count",
"tx_pripause_7_1us_count",
"frames_tx_truncated",
]

warning_fields: ClassVar[list[str]] = [
"frames_rx_pause",
"frames_rx_pripause",
"frames_rx_pri_0",
"frames_rx_pri_1",
"frames_rx_pri_2",
Expand Down Expand Up @@ -274,28 +274,14 @@ class Thor2EthtoolStatistics(BaseModel):
"rx_runt_frames",
"rx_stat_discard",
"rx_stat_err",
"tx_pause_frames",
"tx_pfc_frames",
"tx_jabber_frames",
"tx_fcs_err_frames",
"tx_err",
"tx_fifo_underruns",
"tx_pfc_ena_frames_pri0",
"tx_pfc_ena_frames_pri1",
"tx_pfc_ena_frames_pri2",
"tx_pfc_ena_frames_pri3",
"tx_pfc_ena_frames_pri4",
"tx_pfc_ena_frames_pri5",
"tx_pfc_ena_frames_pri6",
"tx_pfc_ena_frames_pri7",
"tx_total_collisions",
"tx_stat_discard",
"tx_stat_error",
"link_down_events",
"continuous_pause_events",
"resume_pause_events",
"continuous_roce_pause_events",
"resume_roce_pause_events",
"rx_pcs_symbol_err",
"rx_discard_bytes_cos0",
"rx_discard_packets_cos0",
Expand All @@ -315,14 +301,6 @@ class Thor2EthtoolStatistics(BaseModel):
"rx_discard_packets_cos7",
"rx_fec_uncorrectable_blocks",
"rx_filter_miss",
"pfc_pri0_tx_transitions",
"pfc_pri1_tx_transitions",
"pfc_pri2_tx_transitions",
"pfc_pri3_tx_transitions",
"pfc_pri4_tx_transitions",
"pfc_pri5_tx_transitions",
"pfc_pri6_tx_transitions",
"pfc_pri7_tx_transitions",
"hw_db_recov_dbs_dropped",
"hw_db_recov_oo_drop_count",
"lpbk_tx_discards",
Expand Down Expand Up @@ -352,6 +330,28 @@ class Thor2EthtoolStatistics(BaseModel):
"pfc_pri5_rx_transitions",
"pfc_pri6_rx_transitions",
"pfc_pri7_rx_transitions",
"tx_pause_frames",
"tx_pfc_frames",
"tx_pfc_ena_frames_pri0",
"tx_pfc_ena_frames_pri1",
"tx_pfc_ena_frames_pri2",
"tx_pfc_ena_frames_pri3",
"tx_pfc_ena_frames_pri4",
"tx_pfc_ena_frames_pri5",
"tx_pfc_ena_frames_pri6",
"tx_pfc_ena_frames_pri7",
"continuous_pause_events",
"resume_pause_events",
"continuous_roce_pause_events",
"resume_roce_pause_events",
"pfc_pri0_tx_transitions",
"pfc_pri1_tx_transitions",
"pfc_pri2_tx_transitions",
"pfc_pri3_tx_transitions",
"pfc_pri4_tx_transitions",
"pfc_pri5_tx_transitions",
"pfc_pri6_tx_transitions",
"pfc_pri7_tx_transitions",
]


Expand Down Expand Up @@ -513,7 +513,6 @@ class Cx7EthtoolStatistics(BaseModel):
"rx_oversize_pkts_phy",
"rx_symbol_err_phy",
"rx_unsupported_op_phy",
"tx_pause_ctrl_phy",
"rx_discards_phy",
"tx_discards_phy",
"tx_errors_phy",
Expand All @@ -536,24 +535,6 @@ class Cx7EthtoolStatistics(BaseModel):
"rx_prio5_discards",
"rx_prio6_discards",
"rx_prio7_discards",
"tx_global_pause",
"tx_prio0_pause",
"tx_prio1_pause",
"tx_prio2_pause",
"tx_prio3_pause",
"tx_prio4_pause",
"tx_prio5_pause",
"tx_prio6_pause",
"tx_prio7_pause",
"tx_global_pause_duration",
"tx_prio0_pause_duration",
"tx_prio1_pause_duration",
"tx_prio2_pause_duration",
"tx_prio3_pause_duration",
"tx_prio4_pause_duration",
"tx_prio5_pause_duration",
"tx_prio6_pause_duration",
"tx_prio7_pause_duration",
"tx_pause_storm_warning_events",
"tx_pause_storm_error_events",
"module_unplug",
Expand Down Expand Up @@ -619,6 +600,25 @@ class Cx7EthtoolStatistics(BaseModel):
"rx_prio5_pause_duration",
"rx_prio6_pause_duration",
"rx_prio7_pause_duration",
"tx_pause_ctrl_phy",
"tx_global_pause",
"tx_prio0_pause",
"tx_prio1_pause",
"tx_prio2_pause",
"tx_prio3_pause",
"tx_prio4_pause",
"tx_prio5_pause",
"tx_prio6_pause",
"tx_prio7_pause",
"tx_global_pause_duration",
"tx_prio0_pause_duration",
"tx_prio1_pause_duration",
"tx_prio2_pause_duration",
"tx_prio3_pause_duration",
"tx_prio4_pause_duration",
"tx_prio5_pause_duration",
"tx_prio6_pause_duration",
"tx_prio7_pause_duration",
]


Expand Down
89 changes: 52 additions & 37 deletions nodescraper/plugins/inband/network/network_analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
# SOFTWARE.
#
###############################################################################
import re
from typing import Optional

from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer
Expand All @@ -35,40 +34,26 @@


class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]):
"""Check network statistics for errors (PFC and other network error counters)."""
"""Check network statistics for errors."""

DATA_MODEL = NetworkDataModel

# Regex patterns for error fields checked from network statistics
ERROR_REGEX: list[ErrorRegex] = [
ErrorRegex(
regex=re.compile(r"^tx_pfc_frames$"),
message="tx_pfc_frames is non-zero",
event_category=EventCategory.NETWORK,
),
ErrorRegex(
regex=re.compile(r"^tx_pfc_ena_frames_pri\d+$"),
message="tx_pfc_ena_frames_pri* is non-zero",
event_category=EventCategory.NETWORK,
),
ErrorRegex(
regex=re.compile(r"^pfc_pri\d+_tx_transitions$"),
message="pfc_pri*_tx_transitions is non-zero",
event_category=EventCategory.NETWORK,
),
]
# No built-in regex patterns: RDMA-scoped counter classification lives in the vendor
# ethtool models (ethtool_vendor.py). This list is only extended by user-supplied
# NetworkAnalyzerArgs.error_regex patterns.
ERROR_REGEX: list[ErrorRegex] = []

def analyze_data(
self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None
) -> TaskResult:
"""Analyze ethtool -S statistics: regex-based (per interface) and vendor-based (RDMA-scoped).
"""Analyze ethtool -S statistics via RDMA-scoped vendor models and any user-supplied regex.

Args:
data: Network data model with ethtool_info and/or rdma_ethtool_statistics.
args: Optional analyzer arguments with custom error regex support.

Returns:
TaskResult with OK, WARNING (no data or vendor warning counters only), or ERROR.
TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR.
"""
if not data.ethtool_info and not data.rdma_ethtool_statistics:
self.result.message = "No network devices found"
Expand All @@ -81,8 +66,9 @@ def analyze_data(
final_error_regex = self._convert_and_extend_error_regex(args.error_regex, self.ERROR_REGEX)

regex_error = False
regex_warning = False
for interface_name, ethtool_info in data.ethtool_info.items():
errors_on_interface: list[tuple[str, int]] = []
matches_on_interface: list[tuple[str, int, EventPriority]] = []
for stat_name, stat_value in ethtool_info.statistics.items():
for error_regex_obj in final_error_regex:
if error_regex_obj.regex.match(stat_name):
Expand All @@ -92,26 +78,38 @@ def analyze_data(
break

if value > 0:
errors_on_interface.append((stat_name, value))
matches_on_interface.append(
(stat_name, value, error_regex_obj.event_priority)
)
break

if errors_on_interface:
regex_error = True
error_names = [e[0] for e in errors_on_interface]
errors_data = {field: value for field, value in errors_on_interface}
if matches_on_interface:
has_error = any(
priority == EventPriority.ERROR for _, _, priority in matches_on_interface
)
if has_error:
regex_error = True
else:
regex_warning = True
priority = EventPriority.ERROR if has_error else EventPriority.WARNING
severity = "error" if has_error else "warning"
match_names = [match[0] for match in matches_on_interface]
matches_data = {name: value for name, value, _ in matches_on_interface}
self._log_event(
category=EventCategory.NETWORK,
description=f"Network error detected on {interface_name}: [{', '.join(error_names)}]",
description=f"Network {severity} detected on {interface_name}: [{', '.join(match_names)}]",
data={
"interface": interface_name,
"errors": errors_data,
"errors": matches_data,
},
priority=EventPriority.ERROR,
priority=priority,
console_log=True,
)

vendor_error = False
vendor_warning = False
vendor_error_fields: set[str] = set()
vendor_warning_fields: set[str] = set()
for stat in data.rdma_ethtool_statistics:
if stat.vendor_statistics is None:
continue
Expand All @@ -127,13 +125,22 @@ def analyze_data(
priority = EventPriority.WARNING if is_warning_tier else EventPriority.ERROR
if is_warning_tier:
vendor_warning = True
vendor_warning_fields.add(field_name)
else:
vendor_error = True
vendor_error_fields.add(field_name)
# Use a single grouped description per severity so the run summary
# collapses every occurrence into one "Ethtool warning detected" /
# "Ethtool error detected" entry instead of one line per field. The
# specific field is still preserved in the event data below and in
# the consolidated console line emitted after the loop.
desc = (
f"Ethtool warning detected: {field_name}"
if is_warning_tier
else f"Ethtool error detected: {field_name}"
"Ethtool warning detected" if is_warning_tier else "Ethtool error detected"
)
# Per-field events are still recorded for the run summary and event
# log, but console logging is suppressed here to avoid repeating the
# same message once per device. A single consolidated line is emitted
# after the loop instead.
self._log_event(
category=EventCategory.NETWORK,
description=desc,
Expand All @@ -144,16 +151,24 @@ def analyze_data(
"error_count": error_value,
},
priority=priority,
console_log=True,
console_log=False,
)

if vendor_error:
self.logger.error("Ethtool error detected: %s", ", ".join(sorted(vendor_error_fields)))
if vendor_warning:
self.logger.warning(
"Ethtool warning detected: %s", ", ".join(sorted(vendor_warning_fields))
)

if regex_error or vendor_error:
self.result.message = "Network errors detected in statistics"
self.result.status = ExecutionStatus.ERROR
elif vendor_warning:
self.result.message = "Network vendor ethtool warning counters non-zero"
elif regex_warning or vendor_warning:
self.result.message = "Network warning counters non-zero in statistics"
self.result.status = ExecutionStatus.WARNING
else:
self.result.message = "No network errors detected in statistics"
self.result.status = ExecutionStatus.OK

return self.result
Loading