From acd06031921fe30589cf6d00700876aafa548775 Mon Sep 17 00:00:00 2001 From: nikfot Date: Sat, 16 May 2026 09:32:17 +0300 Subject: [PATCH 1/2] Support standard vmstat output without -a flag Auto-detect vmstat column headers from the log file instead of hardcoding the column order. This adds support for standard vmstat output (buff/cache columns) alongside the existing vmstat -a format (inact/active columns). - Parser reads the header line and detects available columns - Timeseries.commit_raw_data() accepts a dynamic headers list - Memory plotting adapts to show buff/cache or inact/active - Add buff/cache to CLI --column choices - Add example_standard.log fixture for testing --- examples/example_standard.log | 12 +++ src/vmstat_visualizer/cli/visualizer.py | 5 +- src/vmstat_visualizer/parser/parser.py | 115 +++++++++++++++------ src/vmstat_visualizer/parser/timeseries.py | 94 +++++++++-------- 4 files changed, 143 insertions(+), 83 deletions(-) create mode 100644 examples/example_standard.log diff --git a/examples/example_standard.log b/examples/example_standard.log new file mode 100644 index 0000000..31a2706 --- /dev/null +++ b/examples/example_standard.log @@ -0,0 +1,12 @@ +procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- + r b swpd free buff cache si so bi bo in cs us sy id wa st + 3 0 0 523 64 995 0 0 4 1 71 111 0 0 99 0 0 2025-08-01 10:06:32 + 1 0 0 523 64 995 0 0 0 0 1815 3452 4 1 96 0 0 2025-08-01 10:06:33 + 1 0 0 523 64 995 0 0 0 16 2530 4711 4 3 93 0 0 2025-08-01 10:06:34 + 1 0 0 523 64 995 0 0 0 0 2061 3776 4 2 94 0 0 2025-08-01 10:06:35 + 1 0 0 523 64 995 0 0 0 0 2181 4079 4 3 94 0 0 2025-08-01 10:06:36 + 2 0 0 523 64 995 0 0 0 16 1906 3617 4 1 95 0 0 2025-08-01 10:06:37 + 1 0 0 523 64 995 0 0 0 0 2108 3965 2 2 95 0 0 2025-08-01 10:06:38 + 1 0 0 523 64 995 0 0 0 0 2670 4944 5 3 92 0 0 2025-08-01 10:06:39 + 1 0 0 523 64 995 0 0 0 176 2093 3917 4 3 94 0 0 2025-08-01 10:06:40 + 1 0 0 523 64 995 0 0 0 0 2359 4400 7 2 92 0 0 2025-08-01 10:06:41 diff --git a/src/vmstat_visualizer/cli/visualizer.py b/src/vmstat_visualizer/cli/visualizer.py index 5beb14b..9832bb9 100644 --- a/src/vmstat_visualizer/cli/visualizer.py +++ b/src/vmstat_visualizer/cli/visualizer.py @@ -64,11 +64,12 @@ def visualize( "free", "si", "so", "us", "sy", "id", "wa", "st", - "inact", "active"], case_sensitive=False), + "inact", "active", + "buff", "cache"], case_sensitive=False), multiple=True, help="""Specific column(s) to visualize. Can be specified multiple times. For each metric, the relevant columns are:\n - cpu: us, sy, id, wa, st\n - - memory: swpd, free, inact, active\n + - memory: swpd, free, inact, active (vmstat -a) or buff, cache (standard vmstat)\n - system_load: r, b, si, so\n Example: -c us -c sy -c id """ diff --git a/src/vmstat_visualizer/parser/parser.py b/src/vmstat_visualizer/parser/parser.py index 37868bb..bb2f637 100644 --- a/src/vmstat_visualizer/parser/parser.py +++ b/src/vmstat_visualizer/parser/parser.py @@ -7,10 +7,15 @@ import time import vmstat_visualizer.parser.timeseries as ts import matplotlib.pyplot as plt -from vmstat_visualizer.checks.check import check_vmstat_columns from matplotlib.dates import DateFormatter +KNOWN_VMSTAT_COLUMNS = { + 'r', 'b', 'swpd', 'free', 'inact', 'active', 'buff', 'cache', + 'si', 'so', 'bi', 'bo', 'in', 'cs', 'us', 'sy', 'id', 'wa', 'st', 'gu', +} + + class Parser: """ Parser class for reading and processing data from a specified file. @@ -32,36 +37,45 @@ def __init__(self, filename): self.timeseries = [] self.figure_size = (10, 4) self.enabled_columns = [] + self.detected_headers = None + self.vmstat_format = None + + @staticmethod + def _detect_headers(line): + """Detect vmstat column headers from a header line. + + Returns a list of column names if the line is a header, else None. + """ + parts = line.strip().split() + known_hits = sum(1 for p in parts if p in KNOWN_VMSTAT_COLUMNS) + if known_hits >= 5: + return parts + return None def parse(self): - has_st, has_gu = check_vmstat_columns() - if has_st and has_gu: - print("System supports 'st' and 'gu' columns.") - else: - print(f"Missing columns: st={not has_st}, gu={not has_gu}") + time_regex = re.compile(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') with open(self.filename, 'r') as file: data = file.readlines() for line in data: - timeseries_entry = ts.Timeseries() - # Regex to match a timestamp like '2025-07-31 23:52:52' - time_regex = re.compile(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') - if ("procs" in line or "memory" in line or "cpu" in line) or ("free" in line or "inact" in line or "active" in line): - continue if not line.strip(): - continue + continue + + detected = self._detect_headers(line) + if detected is not None: + self.detected_headers = detected + if 'buff' in detected and 'cache' in detected: + self.vmstat_format = 'standard' + elif 'inact' in detected and 'active' in detected: + self.vmstat_format = 'active' + continue + + if "procs" in line or "memory" in line or "---" in line: + continue + parts = line.strip().split() if len(parts) < 3: continue - # Skip lines where any part is not a digit - # if any(not part.isdigit() for part in parts): - # continue - # isDataLine = True - # for part in parts: - # if not part.replace('.', '', 1).replace('-', '', 1).isdigit(): - # isDataLine = False - # break - # if not isDataLine: - # continue + data_points = [] if len(parts) >= 2 and time_regex.match(" ".join(parts[-2:])): time_column = " ".join(parts[-2:]) @@ -69,8 +83,12 @@ def parse(self): else: time_column = " ".join(parts[:2]) data_points = [time_column] + [item.strip() for item in parts[2:]] + + headers_with_time = list(self.detected_headers) + ['time'] if self.detected_headers else None + + timeseries_entry = ts.Timeseries() timeseries_entry.add_data_point(data_points) - timeseries_entry.commit_raw_data() + timeseries_entry.commit_raw_data(headers=headers_with_time) self.timeseries.append(timeseries_entry) def plot(self, output_file_prefix='vmstat', output_format='png'): @@ -90,10 +108,13 @@ def plot(self, output_file_prefix='vmstat', output_format='png'): # 2. Memory Usage self.plot_metric('memory', inactive_memory_kb=plot_metrics.inactive_memory_kb, active_memory_kb=plot_metrics.active_memory_kb, + buff_memory_kb=plot_metrics.buff_memory_kb, + cache_memory_kb=plot_metrics.cache_memory_kb, swapped_memory_kb=plot_metrics.swapped_memory_kb, free_memory_kb=plot_metrics.free_memory_kb, t=plot_metrics.t, tstart=tstart, output_file_prefix=output_file_prefix, - output_format=output_format, now_unix=now_unix) + output_format=output_format, now_unix=now_unix, + has_standard_memory=plot_metrics.has_standard_memory) # 3. CPU Usage self.plot_metric('cpu', user_cpu_percent=plot_metrics.user_cpu_percent, system_cpu_percent=plot_metrics.system_cpu_percent, @@ -223,7 +244,10 @@ def plot_metric(self, metric, **kwargs): self._plot_memory( kwargs.get('inactive_memory_kb'), kwargs.get('active_memory_kb'), kwargs.get('swapped_memory_kb'), kwargs.get('free_memory_kb'), kwargs.get('t'), - kwargs.get('tstart'), kwargs.get('output_file_prefix'), kwargs.get('output_format'), kwargs.get('now_unix') + kwargs.get('tstart'), kwargs.get('output_file_prefix'), kwargs.get('output_format'), kwargs.get('now_unix'), + buff_memory_kb=kwargs.get('buff_memory_kb'), + cache_memory_kb=kwargs.get('cache_memory_kb'), + has_standard_memory=kwargs.get('has_standard_memory', False), ) elif metric == 'system_load': self._plot_system_load( @@ -300,23 +324,34 @@ def _plot_system_load(self, run_queue, blocked_processes, t, tstart, output_file plt.close() def _plot_memory(self, inactive_memory_kb, active_memory_kb, swapped_memory_kb, free_memory_kb, - t, tstart, output_file_prefix, output_format, now_unix, comparison=False, filenames=None): + t, tstart, output_file_prefix, output_format, now_unix, comparison=False, filenames=None, + buff_memory_kb=None, cache_memory_kb=None, has_standard_memory=False): import matplotlib.pyplot as plt plt.figure(figsize=self.figure_size) if not comparison: - # Convert memory values to integers for plotting - inactive_memory_kb_int = [int(x) for x in inactive_memory_kb] - active_memory_kb_int = [int(x) for x in active_memory_kb] + all_memory = [] swapped_memory_kb_int = [int(x) for x in swapped_memory_kb] free_memory_kb_int = [int(x) for x in free_memory_kb] - plt.plot(t, inactive_memory_kb_int, label='Inactive Memory (KB)') - plt.plot(t, active_memory_kb_int, label='Active Memory (KB)') plt.plot(t, swapped_memory_kb_int, label='Swapped Memory (KB)') plt.plot(t, free_memory_kb_int, label='Free Memory (KB)') + all_memory += swapped_memory_kb_int + free_memory_kb_int + + if has_standard_memory and buff_memory_kb and any(v is not None for v in buff_memory_kb): + buff_int = [int(x) for x in buff_memory_kb] + cache_int = [int(x) for x in cache_memory_kb] + plt.plot(t, buff_int, label='Buffers (KB)') + plt.plot(t, cache_int, label='Cache (KB)') + all_memory += buff_int + cache_int + else: + inactive_memory_kb_int = [int(x) for x in inactive_memory_kb] + active_memory_kb_int = [int(x) for x in active_memory_kb] + plt.plot(t, inactive_memory_kb_int, label='Inactive Memory (KB)') + plt.plot(t, active_memory_kb_int, label='Active Memory (KB)') + all_memory += inactive_memory_kb_int + active_memory_kb_int + plt.title('Memory Usage') plt.xlabel('Seconds') - all_memory = inactive_memory_kb_int + active_memory_kb_int + swapped_memory_kb_int + free_memory_kb_int else: # Get filenames for labels file1_label = filenames[0] if filenames else 'File 1' @@ -569,6 +604,8 @@ def __init__(self, timeseries, relative_time=False): self.free_memory_kb = [] self.inactive_memory_kb = [] self.active_memory_kb = [] + self.buff_memory_kb = [] + self.cache_memory_kb = [] self.swapped_memory_kb = [] self.user_cpu_percent = [] self.system_cpu_percent = [] @@ -585,13 +622,15 @@ def __init__(self, timeseries, relative_time=False): t_dt = datetime.datetime.strptime(ts_entry.time, '%Y-%m-%d %H:%M:%S') if not relative_time: self.t.append(t_dt.strftime('%M:%S')) - else: + else: self.t.append(next(self._counter)) self.run_queue.append(ts_entry.run_queue) self.blocked_processes.append(ts_entry.blocked_processes) self.free_memory_kb.append(ts_entry.free_memory_kb) self.inactive_memory_kb.append(ts_entry.inactive_memory_kb) self.active_memory_kb.append(ts_entry.active_memory_kb) + self.buff_memory_kb.append(ts_entry.buff_memory_kb) + self.cache_memory_kb.append(ts_entry.cache_memory_kb) self.swapped_memory_kb.append(ts_entry.swapped_memory_kb) self.user_cpu_percent.append(ts_entry.user_cpu_percent) self.system_cpu_percent.append(ts_entry.system_cpu_percent) @@ -604,6 +643,16 @@ def __init__(self, timeseries, relative_time=False): self.blocks_in.append(ts_entry.blocks_in) self.blocks_out.append(ts_entry.blocks_out) + @property + def has_standard_memory(self): + """True when buff/cache columns are present (standard vmstat).""" + return any(v is not None for v in self.buff_memory_kb) + + @property + def has_active_memory(self): + """True when inact/active columns are present (vmstat -a).""" + return any(v is not None for v in self.inactive_memory_kb) + @staticmethod def _counter_gen(): n = 0 diff --git a/src/vmstat_visualizer/parser/timeseries.py b/src/vmstat_visualizer/parser/timeseries.py index 1b4a610..bca426e 100644 --- a/src/vmstat_visualizer/parser/timeseries.py +++ b/src/vmstat_visualizer/parser/timeseries.py @@ -4,20 +4,35 @@ successive measurements made over a time interval. """ + +COLUMN_TO_ATTR = { + 'r': 'run_queue', + 'b': 'blocked_processes', + 'swpd': 'swapped_memory_kb', + 'free': 'free_memory_kb', + 'inact': 'inactive_memory_kb', + 'active': 'active_memory_kb', + 'buff': 'buff_memory_kb', + 'cache': 'cache_memory_kb', + 'si': 'swap_in_kb', + 'so': 'swap_out_kb', + 'bi': 'blocks_in', + 'bo': 'blocks_out', + 'in': 'interrupts', + 'cs': 'context_switches', + 'us': 'user_cpu_percent', + 'sy': 'system_cpu_percent', + 'id': 'idle_cpu_percent', + 'wa': 'wait_cpu_percent', + 'st': 'steal_cpu_percent', + 'gu': 'guest_cpu_percent', + 'time': 'time', +} + + class Timeseries: - """ - Timeseries class for handling time series data. + """Single vmstat sample with named attributes for each metric column.""" - Attributes: - data (list): A list to store the time series data points. - Methods: - __init__(): - Initializes the Timeseries with an empty data list. - add_data_point(point): - Adds a data point to the time series. - get_data(): - Returns the list of data points in the time series. - """ def __init__(self): self.time = None self.run_queue = None @@ -26,6 +41,8 @@ def __init__(self): self.free_memory_kb = None self.inactive_memory_kb = None self.active_memory_kb = None + self.buff_memory_kb = None + self.cache_memory_kb = None self.swap_in_kb = None self.swap_out_kb = None self.blocks_in = None @@ -50,6 +67,8 @@ def __repr__(self): f"free_memory_kb={self.free_memory_kb}, " f"inactive_memory_kb={self.inactive_memory_kb}, " f"active_memory_kb={self.active_memory_kb}, " + f"buff_memory_kb={self.buff_memory_kb}, " + f"cache_memory_kb={self.cache_memory_kb}, " f"swap_in_kb={self.swap_in_kb}, " f"swap_out_kb={self.swap_out_kb}, " f"blocks_in={self.blocks_in}, " @@ -70,40 +89,19 @@ def add_data_point(self, point): def get_data(self): """Returns the list of data points in the time series.""" - return self.data + return self.raw_data - def commit_raw_data(self): - """ - Commits the raw_data points to the actual attributes of - the Timeseries instance. Maps raw_data keys to attribute - names and appends values to corresponding lists. - Assumes raw_data is a list of dicts, each representing a data point. - """ - headers = [ - 'r', 'b', 'swpd', 'free', 'inact', 'active', 'si', 'so', - 'bi', 'bo', 'in', 'cs', 'us', 'sy', 'id', 'wa', 'st', 'gu', "time" - ] - key_map = { - 'r': 'run_queue', - 'b': 'blocked_processes', - 'swpd': 'swapped_memory_kb', - 'free': 'free_memory_kb', - 'inact': 'inactive_memory_kb', - 'active': 'active_memory_kb', - 'si': 'swap_in_kb', - 'so': 'swap_out_kb', - 'bi': 'blocks_in', - 'bo': 'blocks_out', - 'in': 'interrupts', - 'cs': 'context_switches', - 'us': 'user_cpu_percent', - 'sy': 'system_cpu_percent', - 'id': 'idle_cpu_percent', - 'wa': 'wait_cpu_percent', - 'st': 'steal_cpu_percent', - 'gu': 'guest_cpu_percent', - "time": "time" - } - for idx, row in enumerate(self.raw_data): - attr = key_map[headers[idx]] - setattr(self, attr, row) + def commit_raw_data(self, headers=None): + """Map raw_data values to named attributes using the provided header list.""" + if headers is None: + headers = [ + 'r', 'b', 'swpd', 'free', 'inact', 'active', 'si', 'so', + 'bi', 'bo', 'in', 'cs', 'us', 'sy', 'id', 'wa', 'st', 'gu', 'time', + ] + for idx, value in enumerate(self.raw_data): + if idx >= len(headers): + break + col = headers[idx] + attr = COLUMN_TO_ATTR.get(col) + if attr: + setattr(self, attr, value) From f5c76c66910cfb647e334b0f033601ab3fad70cb Mon Sep 17 00:00:00 2001 From: nikfot Date: Sat, 16 May 2026 10:16:53 +0300 Subject: [PATCH 2/2] Add tests for auto-detect vmstat column changes --- tests/conftest.py | 7 + tests/test_cli.py | 126 ++++++++++-------- tests/test_parser.py | 247 ++++++++++++++++++++++++----------- tests/test_plot.py | 50 +++---- tests/test_timeseries.py | 273 ++++++++++++++++++++++++++++++--------- 5 files changed, 484 insertions(+), 219 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7e35b09..972525b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,3 +34,10 @@ def example_log(): return os.path.join( os.path.dirname(__file__), "..", "examples", "example.log" ) + + +@pytest.fixture +def example_standard_log(): + return os.path.join( + os.path.dirname(__file__), "..", "examples", "example_standard.log" + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 073361e..9436e6b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,8 @@ import os import glob import pytest -from unittest.mock import patch from click.testing import CliRunner + from vmstat_visualizer.cli.__main__ import cli import vmstat_visualizer.cli.visualizer as viz @@ -17,98 +17,118 @@ def _register_commands(): class TestCLIHelp: def test_main_help(self): runner = CliRunner() - result = runner.invoke(cli, ['--help']) + result = runner.invoke(cli, ["--help"]) assert result.exit_code == 0 - assert 'visualize' in result.output - assert 'compare' in result.output + assert "visualize" in result.output + assert "compare" in result.output def test_visualize_help(self): runner = CliRunner() - result = runner.invoke(cli, ['visualize', '--help']) + result = runner.invoke(cli, ["visualize", "--help"]) assert result.exit_code == 0 - assert '--output-prefix' in result.output - assert '--output-extension' in result.output + assert "--output-prefix" in result.output + assert "--output-extension" in result.output def test_compare_help(self): runner = CliRunner() - result = runner.invoke(cli, ['compare', '--help']) + result = runner.invoke(cli, ["compare", "--help"]) assert result.exit_code == 0 - assert '--metric' in result.output - assert '--column' in result.output + assert "--metric" in result.output + assert "--column" in result.output class TestVisualizeCommand: - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_visualize_active_log(self, mock_check, active_log, tmp_path): + def test_visualize_active_log(self, active_log, tmp_path): runner = CliRunner() prefix = str(tmp_path / "viz") - result = runner.invoke(cli, [ - 'visualize', active_log, '-o', prefix, '-e', 'png' - ]) + result = runner.invoke(cli, ["visualize", active_log, "-o", prefix, "-e", "png"]) assert result.exit_code == 0 - assert '5 time series entries' in result.output + assert "5 time series entries" in result.output pngs = glob.glob(str(tmp_path / "viz_*.png")) assert len(pngs) == 5 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_visualize_example_log(self, mock_check, example_log, tmp_path): + def test_visualize_example_log(self, example_log, tmp_path): runner = CliRunner() prefix = str(tmp_path / "ex") - result = runner.invoke(cli, [ - 'visualize', example_log, '-o', prefix - ]) + result = runner.invoke(cli, ["visualize", example_log, "-o", prefix]) assert result.exit_code == 0 - assert '30 time series entries' in result.output + assert "30 time series entries" in result.output def test_visualize_missing_file(self, tmp_path): runner = CliRunner() - result = runner.invoke(cli, [ - 'visualize', str(tmp_path / 'nonexistent.log') - ]) + result = runner.invoke(cli, ["visualize", str(tmp_path / "nonexistent.log")]) assert result.exit_code != 0 class TestCompareCommand: - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_compare_cpu(self, mock_check, active_log, tmp_path): + def test_compare_cpu(self, active_log, tmp_path): runner = CliRunner() prefix = str(tmp_path / "cmp") - result = runner.invoke(cli, [ - 'compare', active_log, active_log, - '-m', 'cpu', '-o', prefix - ]) + result = runner.invoke( + cli, + ["compare", active_log, active_log, "-m", "cpu", "-o", prefix], + ) assert result.exit_code == 0 - assert 'Comparing' in result.output + assert "Comparing" in result.output - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_compare_all(self, mock_check, active_log, tmp_path): + def test_compare_all(self, active_log, tmp_path): runner = CliRunner() prefix = str(tmp_path / "all") - result = runner.invoke(cli, [ - 'compare', active_log, active_log, - '-m', 'all', '-o', prefix - ]) + result = runner.invoke( + cli, + ["compare", active_log, active_log, "-m", "all", "-o", prefix], + ) assert result.exit_code == 0 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_compare_with_column_filter(self, mock_check, active_log, tmp_path): + def test_compare_with_column_filter(self, active_log, tmp_path): runner = CliRunner() prefix = str(tmp_path / "filt") - result = runner.invoke(cli, [ - 'compare', active_log, active_log, - '-m', 'cpu', '-c', 'us', '-c', 'sy', '-o', prefix - ]) + result = runner.invoke( + cli, + [ + "compare", + active_log, + active_log, + "-m", + "cpu", + "-c", + "us", + "-c", + "sy", + "-o", + prefix, + ], + ) + assert result.exit_code == 0 + assert "Filtering columns: us, sy" in result.output + + def test_compare_memory_with_buff_and_cache_columns(self, standard_log, tmp_path): + runner = CliRunner() + prefix = str(tmp_path / "bc") + result = runner.invoke( + cli, + [ + "compare", + standard_log, + standard_log, + "-m", + "memory", + "-c", + "buff", + "-c", + "cache", + "-o", + prefix, + ], + ) assert result.exit_code == 0 - assert 'Filtering columns: us, sy' in result.output + assert "buff" in result.output and "cache" in result.output + pngs = glob.glob(str(tmp_path / "bc_memory_comparison*.png")) + assert len(pngs) >= 1 def test_compare_invalid_metric(self, active_log): runner = CliRunner() - result = runner.invoke(cli, [ - 'compare', active_log, active_log, '-m', 'bogus' - ]) + result = runner.invoke( + cli, ["compare", active_log, active_log, "-m", "bogus"] + ) assert result.exit_code != 0 diff --git a/tests/test_parser.py b/tests/test_parser.py index 92a1aba..491d833 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,150 +1,209 @@ import pytest -from unittest.mock import patch from vmstat_visualizer.parser.parser import Parser -def _mock_check_vmstat_columns(): - """Mock check_vmstat_columns to avoid running vmstat subprocess.""" - return True, True - - class TestParserInit: def test_initial_state(self): p = Parser("dummy.log") assert p.filename == "dummy.log" assert p.timeseries == [] assert p.enabled_columns == [] + assert p.detected_headers is None + assert p.vmstat_format is None + + +class TestDetectHeaders: + """_detect_headers: header lines yield column tokens; prose/data lines yield None.""" + + def test_returns_active_header_tokens(self): + line = ( + " r b swpd free inact active si so bi bo " + "in cs us sy id wa st gu\n" + ) + got = Parser._detect_headers(line) + assert got is not None + assert "inact" in got + assert "active" in got + assert "buff" not in got + assert "cache" not in got + + def test_returns_standard_header_tokens_with_buff_cache(self): + line = ( + " r b swpd free buff cache si so bi bo " + "in cs us sy id wa st gu\n" + ) + got = Parser._detect_headers(line) + assert got is not None + assert "buff" in got + assert "cache" in got + + def test_rejects_procs_banner_line(self): + line = "procs -----------memory---------- ---swap-- -----io---- -system-- -------cpu-------\n" + assert Parser._detect_headers(line) is None + + def test_rejects_data_line(self): + line = " 3 0 0 523 128 1059 0 0 4 1 71 111 0 0 99 0 0 0 2025-08-01 10:06:32\n" + assert Parser._detect_headers(line) is None + + def test_rejects_line_with_below_threshold_known_tokens(self): + line = "r b swpd free\n" + assert Parser._detect_headers(line) is None class TestParseExampleLog: - """Parse the shipped example.log fixture (headerless, vmstat -a with gu).""" + """Parse the shipped example.log (no header banner; defaults to vmstat -a mapping).""" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_entry_count(self, mock_check, example_log): + def test_entry_count(self, example_log): p = Parser(example_log) p.parse() assert len(p.timeseries) == 30 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_first_entry_values(self, mock_check, example_log): + def test_first_entry_values(self, example_log): p = Parser(example_log) p.parse() first = p.timeseries[0] - assert first.run_queue == '3' - assert first.blocked_processes == '0' - assert first.swapped_memory_kb == '0' - assert first.free_memory_kb == '523' - assert first.inactive_memory_kb == '128' - assert first.active_memory_kb == '1059' - assert first.swap_in_kb == '0' - assert first.swap_out_kb == '0' - assert first.blocks_in == '4' - assert first.blocks_out == '1' - assert first.user_cpu_percent == '0' - assert first.system_cpu_percent == '0' - assert first.idle_cpu_percent == '99' - assert first.wait_cpu_percent == '0' - assert first.steal_cpu_percent == '0' - assert first.guest_cpu_percent == '0' - assert first.time == '2025-08-01 10:06:32' - - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_last_entry_time(self, mock_check, example_log): + assert first.run_queue == "3" + assert first.blocked_processes == "0" + assert first.swapped_memory_kb == "0" + assert first.free_memory_kb == "523" + assert first.inactive_memory_kb == "128" + assert first.active_memory_kb == "1059" + assert first.swap_in_kb == "0" + assert first.swap_out_kb == "0" + assert first.blocks_in == "4" + assert first.blocks_out == "1" + assert first.user_cpu_percent == "0" + assert first.system_cpu_percent == "0" + assert first.idle_cpu_percent == "99" + assert first.wait_cpu_percent == "0" + assert first.steal_cpu_percent == "0" + assert first.guest_cpu_percent == "0" + assert first.time == "2025-08-01 10:06:32" + assert first.buff_memory_kb is None + assert first.cache_memory_kb is None + + def test_last_entry_time(self, example_log): p = Parser(example_log) p.parse() - assert p.timeseries[-1].time == '2025-08-01 10:07:01' + assert p.timeseries[-1].time == "2025-08-01 10:07:01" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_timestamps_sequential(self, mock_check, example_log): + def test_timestamps_sequential(self, example_log): p = Parser(example_log) p.parse() times = [e.time for e in p.timeseries] assert times == sorted(times) + def test_headerless_defaults_no_detected_banner(self, example_log): + p = Parser(example_log) + p.parse() + assert p.detected_headers is None + assert p.vmstat_format is None + class TestParseActiveFixture: - """Parse the vmstat -a fixture file that has header rows.""" + """vmstat -a style file with banner + header row.""" + + def test_detected_headers_and_format(self, active_log): + p = Parser(active_log) + p.parse() + assert p.vmstat_format == "active" + assert p.detected_headers is not None + assert "inact" in p.detected_headers + assert "active" in p.detected_headers - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_entry_count(self, mock_check, active_log): + def test_entry_count(self, active_log): p = Parser(active_log) p.parse() assert len(p.timeseries) == 5 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_skips_header_and_procs_lines(self, mock_check, active_log): + def test_skips_header_and_procs_lines(self, active_log): p = Parser(active_log) p.parse() for ts_entry in p.timeseries: assert ts_entry.time is not None assert ts_entry.run_queue is not None + def test_active_memory_columns_populated_buff_cache_none(self, active_log): + p = Parser(active_log) + p.parse() + first = p.timeseries[0] + assert first.inactive_memory_kb == "128" + assert first.active_memory_kb == "1059" + assert first.buff_memory_kb is None + assert first.cache_memory_kb is None + + +class TestParseStandardFixture: + """Standard vmstat columns (buff/cache instead of inactive/active).""" + + def test_detected_headers_and_format(self, standard_log): + p = Parser(standard_log) + p.parse() + assert p.vmstat_format == "standard" + assert p.detected_headers is not None + assert "buff" in p.detected_headers + assert "cache" in p.detected_headers + + def test_entry_count(self, standard_log): + p = Parser(standard_log) + p.parse() + assert len(p.timeseries) == 5 + + def test_buff_cache_populated_inact_active_none(self, standard_log): + p = Parser(standard_log) + p.parse() + first = p.timeseries[0] + assert first.buff_memory_kb == "512" + assert first.cache_memory_kb == "8192" + assert first.inactive_memory_kb is None + assert first.active_memory_kb is None + class TestParseNoHeaderFixture: - """Parse a file with no header row (raw data only).""" + """File with raw data rows only.""" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_entry_count(self, mock_check, no_header_log): + def test_entry_count(self, no_header_log): p = Parser(no_header_log) p.parse() assert len(p.timeseries) == 3 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_first_entry(self, mock_check, no_header_log): + def test_first_entry(self, no_header_log): p = Parser(no_header_log) p.parse() first = p.timeseries[0] - assert first.run_queue == '3' - assert first.time == '2025-08-01 10:06:32' + assert first.run_queue == "3" + assert first.time == "2025-08-01 10:06:32" class TestParseEdgeCases: - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_empty_file(self, mock_check, empty_log): + def test_empty_file(self, empty_log): p = Parser(empty_log) p.parse() assert len(p.timeseries) == 0 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_header_only_file(self, mock_check, header_only_log): + def test_header_only_file(self, header_only_log): p = Parser(header_only_log) p.parse() assert len(p.timeseries) == 0 class TestPlotMetrics: - """Validate PlotMetrics correctly aggregates timeseries data.""" + """PlotMetrics aggregates and exposes memory-format hints.""" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_absolute_time(self, mock_check, active_log): + def test_absolute_time(self, active_log): p = Parser(active_log) p.parse() pm = p.PlotMetrics(p.timeseries) assert len(pm.t) == 5 - assert pm.t[0] == '06:32' + assert pm.t[0] == "06:32" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_relative_time(self, mock_check, active_log): + def test_relative_time(self, active_log): p = Parser(active_log) p.parse() pm = p.PlotMetrics(p.timeseries, relative_time=True) assert pm.t == [0, 1, 2, 3, 4] - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_metric_lists_length(self, mock_check, active_log): + def test_metric_lists_length(self, active_log): p = Parser(active_log) p.parse() pm = p.PlotMetrics(p.timeseries) @@ -153,12 +212,48 @@ def test_metric_lists_length(self, mock_check, active_log): assert len(pm.blocks_in) == 5 assert len(pm.swap_in_kb) == 5 assert len(pm.inactive_memory_kb) == 5 + assert len(pm.buff_memory_kb) == 5 + assert len(pm.cache_memory_kb) == 5 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_metric_values(self, mock_check, example_log): + def test_metric_values(self, example_log): p = Parser(example_log) p.parse() pm = p.PlotMetrics(p.timeseries) - assert pm.run_queue[0] == '3' - assert pm.idle_cpu_percent[0] == '99' + assert pm.run_queue[0] == "3" + assert pm.idle_cpu_percent[0] == "99" + + def test_has_active_memory_true_for_vmstat_active_fixture(self, active_log): + p = Parser(active_log) + p.parse() + pm = p.PlotMetrics(p.timeseries) + assert pm.has_active_memory is True + assert pm.has_standard_memory is False + + def test_has_standard_memory_true_for_standard_fixture(self, standard_log): + p = Parser(standard_log) + p.parse() + pm = p.PlotMetrics(p.timeseries) + assert pm.has_standard_memory is True + assert pm.has_active_memory is False + + def test_headerless_guest_log_uses_active_style_memory_columns(self, example_log): + p = Parser(example_log) + p.parse() + pm = p.PlotMetrics(p.timeseries) + assert pm.has_active_memory is True + assert pm.has_standard_memory is False + + +class TestParseExampleStandardLog: + """examples/example_standard.log: banner + buff/cache headers.""" + + def test_standard_format_and_buff_cache(self, example_standard_log): + p = Parser(example_standard_log) + p.parse() + assert p.vmstat_format == "standard" + first = p.timeseries[0] + assert first.buff_memory_kb == "64" + assert first.cache_memory_kb == "995" + assert first.inactive_memory_kb is None + assert first.active_memory_kb is None + diff --git a/tests/test_plot.py b/tests/test_plot.py index 48f1256..c3f9297 100644 --- a/tests/test_plot.py +++ b/tests/test_plot.py @@ -2,84 +2,74 @@ import glob import pytest import matplotlib -matplotlib.use('Agg') -from unittest.mock import patch + +matplotlib.use("Agg") from vmstat_visualizer.parser.parser import Parser class TestPlotOutput: """Integration: parse a fixture and verify plot files are created.""" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_visualize_creates_five_plots(self, mock_check, active_log, tmp_path): + def test_visualize_creates_five_plots(self, active_log, tmp_path): p = Parser(active_log) p.parse() prefix = str(tmp_path / "test_plot") - p.plot(output_file_prefix=prefix, output_format='png') + p.plot(output_file_prefix=prefix, output_format="png") pngs = glob.glob(str(tmp_path / "test_plot_*.png")) assert len(pngs) == 5 - expected_parts = ['system_load', 'memory', 'cpu', 'swap', 'io'] + expected_parts = ["system_load", "memory", "cpu", "swap", "io"] basenames = [os.path.basename(f) for f in pngs] for part in expected_parts: assert any(part in b for b in basenames), f"Missing plot for '{part}'" - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_svg_output_format(self, mock_check, active_log, tmp_path): + def test_svg_output_format(self, active_log, tmp_path): p = Parser(active_log) p.parse() prefix = str(tmp_path / "svg_plot") - p.plot(output_file_prefix=prefix, output_format='svg') + p.plot(output_file_prefix=prefix, output_format="svg") svgs = glob.glob(str(tmp_path / "svg_plot_*.svg")) assert len(svgs) == 5 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_plot_example_log(self, mock_check, example_log, tmp_path): + def test_plot_example_log(self, example_log, tmp_path): p = Parser(example_log) p.parse() prefix = str(tmp_path / "ex_plot") - p.plot(output_file_prefix=prefix, output_format='png') + p.plot(output_file_prefix=prefix, output_format="png") pngs = glob.glob(str(tmp_path / "ex_plot_*.png")) assert len(pngs) == 5 class TestComparisonPlot: - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_comparison_cpu(self, mock_check, active_log, tmp_path): + def test_comparison_cpu(self, active_log, tmp_path): p1 = Parser(active_log) p1.parse() p2 = Parser(active_log) p2.parse() prefix = str(tmp_path / "cmp") - p1.plot_comparison(p2, output_file_prefix=prefix, - output_format='png', metric='cpu') + p1.plot_comparison( + p2, output_file_prefix=prefix, output_format="png", metric="cpu" + ) pngs = glob.glob(str(tmp_path / "cmp_*.png")) assert len(pngs) >= 1 - assert any('cpu_comparison' in os.path.basename(f) for f in pngs) + assert any("cpu_comparison" in os.path.basename(f) for f in pngs) - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_comparison_all(self, mock_check, active_log, tmp_path): + def test_comparison_all(self, active_log, tmp_path): p1 = Parser(active_log) p1.parse() p2 = Parser(active_log) p2.parse() prefix = str(tmp_path / "cmp_all") - p1.plot_comparison(p2, output_file_prefix=prefix, - output_format='png', metric='all') + p1.plot_comparison( + p2, output_file_prefix=prefix, output_format="png", metric="all" + ) pngs = glob.glob(str(tmp_path / "cmp_all_*.png")) assert len(pngs) == 5 - @patch("vmstat_visualizer.parser.parser.check_vmstat_columns", - return_value=(True, True)) - def test_comparison_invalid_metric_raises(self, mock_check, active_log): + def test_comparison_invalid_metric_raises(self, active_log): p1 = Parser(active_log) p1.parse() p2 = Parser(active_log) p2.parse() with pytest.raises(ValueError, match="Unknown metric"): - p1.plot_comparison(p2, metric='nonexistent') + p1.plot_comparison(p2, metric="nonexistent") diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py index c697203..53443a8 100644 --- a/tests/test_timeseries.py +++ b/tests/test_timeseries.py @@ -1,20 +1,51 @@ import pytest -from vmstat_visualizer.parser.timeseries import Timeseries +from vmstat_visualizer.parser.timeseries import Timeseries, COLUMN_TO_ATTR +from vmstat_visualizer.parser.parser import KNOWN_VMSTAT_COLUMNS + + +class TestColumnToAttrMapping: + def test_key_count_matches_all_metric_columns_and_time(self): + assert len(COLUMN_TO_ATTR) == 21 + + def test_includes_buff_and_cache(self): + assert COLUMN_TO_ATTR["buff"] == "buff_memory_kb" + assert COLUMN_TO_ATTR["cache"] == "cache_memory_kb" + + def test_covers_known_vmstat_column_names_plus_time(self): + for name in KNOWN_VMSTAT_COLUMNS: + assert name in COLUMN_TO_ATTR + assert COLUMN_TO_ATTR.get("time") == "time" + + def test_values_are_unique_attribute_names(self): + attrs = list(COLUMN_TO_ATTR.values()) + assert len(attrs) == len(set(attrs)) class TestTimeseriesInit: """All metric attributes start as None; raw_data starts as empty dict.""" METRIC_ATTRS = [ - 'time', 'run_queue', 'blocked_processes', - 'swapped_memory_kb', 'free_memory_kb', - 'inactive_memory_kb', 'active_memory_kb', - 'swap_in_kb', 'swap_out_kb', - 'blocks_in', 'blocks_out', - 'interrupts', 'context_switches', - 'user_cpu_percent', 'system_cpu_percent', - 'idle_cpu_percent', 'wait_cpu_percent', - 'steal_cpu_percent', 'guest_cpu_percent', + "time", + "run_queue", + "blocked_processes", + "swapped_memory_kb", + "free_memory_kb", + "inactive_memory_kb", + "active_memory_kb", + "buff_memory_kb", + "cache_memory_kb", + "swap_in_kb", + "swap_out_kb", + "blocks_in", + "blocks_out", + "interrupts", + "context_switches", + "user_cpu_percent", + "system_cpu_percent", + "idle_cpu_percent", + "wait_cpu_percent", + "steal_cpu_percent", + "guest_cpu_percent", ] def test_all_metric_attributes_none(self): @@ -30,80 +61,202 @@ def test_raw_data_starts_empty(self): class TestAddDataPoint: def test_stores_list(self): ts = Timeseries() - data = ['3', '0', '0', '523', '128', '1059', '0', '0', - '4', '1', '71', '111', '0', '0', '99', '0', '0', '0', - '2025-08-01 10:06:32'] + data = [ + "3", + "0", + "0", + "523", + "128", + "1059", + "0", + "0", + "4", + "1", + "71", + "111", + "0", + "0", + "99", + "0", + "0", + "0", + "2025-08-01 10:06:32", + ] ts.add_data_point(data) assert ts.raw_data == data def test_replaces_previous_data(self): ts = Timeseries() - ts.add_data_point(['a', 'b']) - ts.add_data_point(['c', 'd']) - assert ts.raw_data == ['c', 'd'] + ts.add_data_point(["a", "b"]) + ts.add_data_point(["c", "d"]) + assert ts.raw_data == ["c", "d"] class TestCommitRawData: - """commit_raw_data maps positional values from raw_data to named attributes - using the hardcoded vmstat -a header order: - r b swpd free inact active si so bi bo in cs us sy id wa st gu time - """ + """commit_raw_data maps positional values using default vmstat -a order or explicit headers.""" - def _make_ts(self, data): + def _make_ts(self, data, headers=None): ts = Timeseries() ts.add_data_point(data) - ts.commit_raw_data() + ts.commit_raw_data(headers=headers) return ts - def test_maps_all_19_columns(self): - data = ['3', '0', '0', '523', '128', '1059', '0', '0', - '4', '1', '71', '111', '0', '0', '99', '0', '0', '0', - '2025-08-01 10:06:32'] + def test_maps_default_headers_vmstat_guest_order(self): + data = [ + "3", + "0", + "0", + "523", + "128", + "1059", + "0", + "0", + "4", + "1", + "71", + "111", + "0", + "0", + "99", + "0", + "0", + "0", + "2025-08-01 10:06:32", + ] ts = self._make_ts(data) - assert ts.run_queue == '3' - assert ts.blocked_processes == '0' - assert ts.swapped_memory_kb == '0' - assert ts.free_memory_kb == '523' - assert ts.inactive_memory_kb == '128' - assert ts.active_memory_kb == '1059' - assert ts.swap_in_kb == '0' - assert ts.swap_out_kb == '0' - assert ts.blocks_in == '4' - assert ts.blocks_out == '1' - assert ts.interrupts == '71' - assert ts.context_switches == '111' - assert ts.user_cpu_percent == '0' - assert ts.system_cpu_percent == '0' - assert ts.idle_cpu_percent == '99' - assert ts.wait_cpu_percent == '0' - assert ts.steal_cpu_percent == '0' - assert ts.guest_cpu_percent == '0' - assert ts.time == '2025-08-01 10:06:32' - - def test_cpu_values(self): - data = ['1', '0', '0', '500', '100', '2000', '0', '0', - '10', '5', '200', '300', '15', '2', '84', '0', '0', '0', - '2025-08-01 10:06:58'] + assert ts.run_queue == "3" + assert ts.blocked_processes == "0" + assert ts.swapped_memory_kb == "0" + assert ts.free_memory_kb == "523" + assert ts.inactive_memory_kb == "128" + assert ts.active_memory_kb == "1059" + assert ts.swap_in_kb == "0" + assert ts.swap_out_kb == "0" + assert ts.blocks_in == "4" + assert ts.blocks_out == "1" + assert ts.interrupts == "71" + assert ts.context_switches == "111" + assert ts.user_cpu_percent == "0" + assert ts.system_cpu_percent == "0" + assert ts.idle_cpu_percent == "99" + assert ts.wait_cpu_percent == "0" + assert ts.steal_cpu_percent == "0" + assert ts.guest_cpu_percent == "0" + assert ts.time == "2025-08-01 10:06:32" + assert ts.buff_memory_kb is None + assert ts.cache_memory_kb is None + + def test_explicit_standard_headers_map_buff_cache(self): + headers = [ + "r", + "b", + "swpd", + "free", + "buff", + "cache", + "si", + "so", + "bi", + "bo", + "in", + "cs", + "us", + "sy", + "id", + "wa", + "st", + "gu", + "time", + ] + data = [ + "2", + "0", + "0", + "15234", + "512", + "8192", + "0", + "0", + "12", + "8", + "200", + "450", + "5", + "2", + "92", + "1", + "0", + "0", + "2025-09-15 14:00:01", + ] + ts = self._make_ts(data, headers=headers) + assert ts.buff_memory_kb == "512" + assert ts.cache_memory_kb == "8192" + assert ts.inactive_memory_kb is None + assert ts.active_memory_kb is None + + def test_cpu_values_vmstat_guest_order(self): + data = [ + "1", + "0", + "0", + "500", + "100", + "2000", + "0", + "0", + "10", + "5", + "200", + "300", + "15", + "2", + "84", + "0", + "0", + "0", + "2025-08-01 10:06:58", + ] ts = self._make_ts(data) - assert ts.user_cpu_percent == '15' - assert ts.system_cpu_percent == '2' - assert ts.idle_cpu_percent == '84' + assert ts.user_cpu_percent == "15" + assert ts.system_cpu_percent == "2" + assert ts.idle_cpu_percent == "84" class TestRepr: def test_repr_contains_key_fields(self): ts = Timeseries() r = repr(ts) - assert 'Timeseries(' in r - assert 'run_queue=None' in r - assert 'time=None' in r - assert 'raw_data={}' in r + assert "Timeseries(" in r + assert "run_queue=None" in r + assert "time=None" in r + assert "raw_data={}" in r def test_repr_after_commit(self): ts = Timeseries() - ts.add_data_point(['3', '0', '0', '523', '128', '1059', '0', '0', - '4', '1', '71', '111', '0', '0', '99', '0', '0', '0', - '2025-08-01 10:06:32']) + ts.add_data_point( + [ + "3", + "0", + "0", + "523", + "128", + "1059", + "0", + "0", + "4", + "1", + "71", + "111", + "0", + "0", + "99", + "0", + "0", + "0", + "2025-08-01 10:06:32", + ] + ) ts.commit_raw_data() r = repr(ts) assert "run_queue=3" in r