diff --git a/README.md b/README.md index 5c4f965..86b6b7e 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,12 @@ src/ ├── chapter_33/ # OS and System Interaction ├── chapter_34/ # Email and Data Encoding ├── chapter_35/ # Advanced Python Patterns +├── chapter_36/ # Advanced Testing +├── chapter_37/ # Abstract Syntax Trees +├── chapter_38/ # Memory Management +├── chapter_39/ # C Interoperability +├── chapter_40/ # Import System Internals +├── chapter_41/ # Concurrency Patterns └── ... tests/ ├── conftest.py # Shared pytest fixtures @@ -146,6 +152,12 @@ tests/ | 33 | OS and System Interaction | os, shutil, tempfile, platform, subprocess | Done | | 34 | Email and Data Encoding | email.message, base64, quopri, mimetypes, binascii | Done | | 35 | Advanced Python Patterns | Descriptors, `__slots__`, weakrefs, ABCs, copy protocol | Done | +| 36 | Advanced Testing | pytest fixtures, parametrize, markers, unittest.mock, test patterns | Done | +| 37 | Abstract Syntax Trees | ast module, NodeVisitor, NodeTransformer, compile, eval, exec | Done | +| 38 | Memory Management | Reference counting, gc module, tracemalloc, finalize, optimization | Done | +| 39 | C Interoperability | ctypes, structures, callbacks, array module, memoryview | Done | +| 40 | Import System Internals | importlib, import hooks, sys.meta_path, pkgutil, metadata | Done | +| 41 | Concurrency Patterns | Threading sync, concurrent.futures, Queue, producer-consumer | Done | ## Running Notebooks diff --git a/src/chapter_36/01_pytest_features.ipynb b/src/chapter_36/01_pytest_features.ipynb new file mode 100644 index 0000000..9a07d3b --- /dev/null +++ b/src/chapter_36/01_pytest_features.ipynb @@ -0,0 +1,791 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 36: Pytest Features\n", + "\n", + "This notebook covers the essential features of pytest, Python's most popular testing framework. Since pytest fixtures, parametrize decorators, and markers require the pytest runner to execute, we demonstrate the concepts by showing the code patterns, explaining how they work, and using regular function calls to simulate behavior.\n", + "\n", + "## Key Concepts\n", + "- **Fixtures**: Reusable setup/teardown functions that provide test data and resources\n", + "- **Parametrize**: Run the same test logic with multiple sets of inputs\n", + "- **Markers**: Tag tests for selective execution (skip, xfail, custom)\n", + "- **monkeypatch**: Temporarily modify objects, dictionaries, and environment variables\n", + "- **tmp_path**: Get a unique temporary directory for each test\n", + "- **pytest.raises**: Assert that code raises a specific exception\n", + "- **pytest.approx**: Compare floating-point numbers with tolerance" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Understanding Fixtures\n", + "\n", + "Fixtures are functions decorated with `@pytest.fixture` that provide reusable test data or resources. Tests declare dependencies on fixtures by accepting them as parameters. Here we simulate fixture behavior with regular functions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# In a real pytest file, you'd write:\n", + "#\n", + "# @pytest.fixture\n", + "# def sample_user() -> dict[str, str | int]:\n", + "# return {\"name\": \"Alice\", \"age\": 30}\n", + "#\n", + "# def test_user_name(sample_user: dict[str, str | int]) -> None:\n", + "# assert sample_user[\"name\"] == \"Alice\"\n", + "\n", + "# Simulating fixture behavior with a regular function\n", + "def sample_user_fixture() -> dict[str, str | int]:\n", + " \"\"\"Fixture that provides a sample user dictionary.\"\"\"\n", + " return {\"name\": \"Alice\", \"age\": 30}\n", + "\n", + "\n", + "# Simulate a test that uses the fixture\n", + "def test_user_name() -> None:\n", + " user: dict[str, str | int] = sample_user_fixture()\n", + " assert user[\"name\"] == \"Alice\"\n", + " assert isinstance(user[\"age\"], int)\n", + " print(f\"User: {user}\")\n", + " print(f\"Name check passed: {user['name']} == 'Alice'\")\n", + " print(f\"Age is int: {isinstance(user['age'], int)}\")\n", + "\n", + "\n", + "test_user_name()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "# Fixtures can have different scopes:\n", + "# - \"function\" (default): created fresh for each test\n", + "# - \"class\": shared across all tests in a class\n", + "# - \"module\": shared across all tests in a module\n", + "# - \"session\": shared across the entire test session\n", + "\n", + "# Simulating a fixture with setup and teardown using yield\n", + "from typing import Generator\n", + "\n", + "\n", + "def database_connection_fixture() -> Generator[dict[str, str], None, None]:\n", + " \"\"\"Fixture with setup and teardown (yield fixture).\"\"\"\n", + " # Setup phase\n", + " connection: dict[str, str] = {\"host\": \"localhost\", \"status\": \"connected\"}\n", + " print(f\"SETUP: Opening connection to {connection['host']}\")\n", + "\n", + " yield connection # This is what the test receives\n", + "\n", + " # Teardown phase (runs after test completes)\n", + " connection[\"status\"] = \"closed\"\n", + " print(f\"TEARDOWN: Connection closed\")\n", + "\n", + "\n", + "# Simulate the full fixture lifecycle\n", + "gen: Generator[dict[str, str], None, None] = database_connection_fixture()\n", + "conn: dict[str, str] = next(gen) # Setup runs, we get the yielded value\n", + "print(f\"TEST: Using connection with status '{conn['status']}'\")\n", + "try:\n", + " next(gen) # Trigger teardown\n", + "except StopIteration:\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Parametrize -- Multiple Inputs, One Test\n", + "\n", + "The `@pytest.mark.parametrize` decorator runs a single test function with multiple sets of arguments. This eliminates repetitive test code. We simulate this pattern with a loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "# In a real pytest file:\n", + "#\n", + "# @pytest.mark.parametrize(\"value, expected\", [\n", + "# (1, 1), (2, 4), (3, 9), (4, 16), (0, 0),\n", + "# ])\n", + "# def test_square(value: int, expected: int) -> None:\n", + "# assert value ** 2 == expected\n", + "\n", + "# Simulating parametrize with a loop\n", + "test_cases: list[tuple[int, int]] = [\n", + " (1, 1),\n", + " (2, 4),\n", + " (3, 9),\n", + " (4, 16),\n", + " (0, 0),\n", + "]\n", + "\n", + "print(\"Parametrized square tests:\")\n", + "for value, expected in test_cases:\n", + " result: int = value ** 2\n", + " passed: bool = result == expected\n", + " status: str = \"PASSED\" if passed else \"FAILED\"\n", + " print(f\" {value}^2 = {result}, expected {expected} -> {status}\")\n", + " assert result == expected" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "# Parametrize with multiple parameters and IDs\n", + "# In pytest:\n", + "#\n", + "# @pytest.mark.parametrize(\"text, expected_words\", [\n", + "# (\"hello world\", 2),\n", + "# (\"\", 0),\n", + "# (\"one\", 1),\n", + "# (\" spaces between \", 2),\n", + "# ], ids=[\"two_words\", \"empty\", \"single\", \"extra_spaces\"])\n", + "\n", + "def count_words(text: str) -> int:\n", + " \"\"\"Count non-empty words in a string.\"\"\"\n", + " return len(text.split()) if text.strip() else 0\n", + "\n", + "\n", + "word_count_cases: list[tuple[str, int, str]] = [\n", + " (\"hello world\", 2, \"two_words\"),\n", + " (\"\", 0, \"empty\"),\n", + " (\"one\", 1, \"single\"),\n", + " (\" spaces between \", 2, \"extra_spaces\"),\n", + "]\n", + "\n", + "print(\"Parametrized word count tests:\")\n", + "for text, expected_words, test_id in word_count_cases:\n", + " result: int = count_words(text)\n", + " passed: bool = result == expected_words\n", + " status: str = \"PASSED\" if passed else \"FAILED\"\n", + " print(f\" [{test_id}] count_words({text!r}) = {result}, expected {expected_words} -> {status}\")\n", + " assert result == expected_words" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Markers -- Tagging and Controlling Tests\n", + "\n", + "Markers let you tag tests for selective execution. Common built-in markers include `skip`, `skipif`, and `xfail`. Custom markers let you group tests by category." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# In pytest:\n", + "#\n", + "# @pytest.mark.skip(reason=\"Not implemented yet\")\n", + "# def test_future_feature() -> None: ...\n", + "#\n", + "# @pytest.mark.skipif(sys.platform == \"win32\", reason=\"Unix only\")\n", + "# def test_unix_feature() -> None: ...\n", + "#\n", + "# @pytest.mark.xfail(reason=\"Known bug #123\")\n", + "# def test_known_bug() -> None: ...\n", + "\n", + "# Simulating marker behavior\n", + "def simulate_skip(reason: str) -> None:\n", + " print(f\" SKIPPED: {reason}\")\n", + "\n", + "\n", + "def simulate_skipif(condition: bool, reason: str) -> bool:\n", + " if condition:\n", + " print(f\" SKIPPED: {reason}\")\n", + " return True\n", + " return False\n", + "\n", + "\n", + "def simulate_xfail(test_func: object, reason: str) -> None:\n", + " try:\n", + " if callable(test_func):\n", + " test_func()\n", + " print(f\" XPASS (unexpectedly passed): {reason}\")\n", + " except AssertionError:\n", + " print(f\" XFAIL (expected failure): {reason}\")\n", + "\n", + "\n", + "print(\"Marker demonstrations:\")\n", + "print(\"\\n@pytest.mark.skip:\")\n", + "simulate_skip(\"Not implemented yet\")\n", + "\n", + "print(\"\\n@pytest.mark.skipif:\")\n", + "simulate_skipif(sys.platform != \"win32\", \"Windows only test\")\n", + "\n", + "print(\"\\n@pytest.mark.xfail:\")\n", + "simulate_xfail(lambda: None, \"Known bug #123 -- test passed unexpectedly\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "# Custom markers allow grouping tests by category\n", + "# In pytest.ini or pyproject.toml:\n", + "# [tool.pytest.ini_options]\n", + "# markers = [\n", + "# \"slow: marks tests as slow\",\n", + "# \"integration: marks integration tests\",\n", + "# ]\n", + "#\n", + "# Usage:\n", + "# @pytest.mark.slow\n", + "# def test_large_dataset() -> None: ...\n", + "#\n", + "# Run only slow tests: pytest -m slow\n", + "# Exclude slow tests: pytest -m \"not slow\"\n", + "\n", + "# Simulating custom markers with a registry\n", + "marker_registry: dict[str, list[str]] = {\n", + " \"slow\": [],\n", + " \"integration\": [],\n", + " \"unit\": [],\n", + "}\n", + "\n", + "\n", + "def register_test(marker: str, test_name: str) -> None:\n", + " marker_registry[marker].append(test_name)\n", + "\n", + "\n", + "register_test(\"unit\", \"test_add\")\n", + "register_test(\"unit\", \"test_subtract\")\n", + "register_test(\"integration\", \"test_database_query\")\n", + "register_test(\"slow\", \"test_large_dataset\")\n", + "register_test(\"slow\", \"test_performance_benchmark\")\n", + "\n", + "print(\"Custom marker registry:\")\n", + "for marker, tests in marker_registry.items():\n", + " print(f\" @pytest.mark.{marker}: {tests}\")\n", + "\n", + "# Filter by marker (simulating pytest -m)\n", + "selected_marker: str = \"unit\"\n", + "print(f\"\\nRunning with -m {selected_marker}: {marker_registry[selected_marker]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Monkeypatch -- Temporary Modifications\n", + "\n", + "The `monkeypatch` fixture lets you temporarily modify attributes, dictionary items, and environment variables during a test. Changes are automatically reverted after the test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "# In pytest, monkeypatch is a built-in fixture:\n", + "#\n", + "# def test_config(monkeypatch) -> None:\n", + "# monkeypatch.setattr(Config, \"debug\", True)\n", + "# assert Config.debug is True\n", + "\n", + "# Simulating monkeypatch.setattr\n", + "class Config:\n", + " debug: bool = False\n", + " log_level: str = \"INFO\"\n", + "\n", + "\n", + "print(f\"Before monkeypatch: Config.debug = {Config.debug}\")\n", + "print(f\"Before monkeypatch: Config.log_level = {Config.log_level}\")\n", + "\n", + "# Save originals, patch, then restore (what monkeypatch does internally)\n", + "original_debug: bool = Config.debug\n", + "original_level: str = Config.log_level\n", + "\n", + "Config.debug = True\n", + "Config.log_level = \"DEBUG\"\n", + "\n", + "print(f\"\\nDuring test: Config.debug = {Config.debug}\")\n", + "print(f\"During test: Config.log_level = {Config.log_level}\")\n", + "assert Config.debug is True\n", + "assert Config.log_level == \"DEBUG\"\n", + "\n", + "# Restore (monkeypatch does this automatically)\n", + "Config.debug = original_debug\n", + "Config.log_level = original_level\n", + "\n", + "print(f\"\\nAfter test: Config.debug = {Config.debug}\")\n", + "print(f\"After test: Config.log_level = {Config.log_level}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# monkeypatch.setenv / monkeypatch.delenv for environment variables\n", + "# In pytest:\n", + "#\n", + "# def test_env(monkeypatch) -> None:\n", + "# monkeypatch.setenv(\"DATABASE_URL\", \"sqlite:///:memory:\")\n", + "# assert os.environ[\"DATABASE_URL\"] == \"sqlite:///:memory:\"\n", + "\n", + "# Simulating monkeypatch.setenv\n", + "env_key: str = \"MY_TEST_VAR\"\n", + "print(f\"Before: {env_key!r} in os.environ = {env_key in os.environ}\")\n", + "\n", + "os.environ[env_key] = \"test_value\"\n", + "print(f\"During test: os.environ[{env_key!r}] = {os.environ[env_key]!r}\")\n", + "assert os.environ[env_key] == \"test_value\"\n", + "\n", + "# Cleanup (monkeypatch handles this automatically)\n", + "del os.environ[env_key]\n", + "print(f\"After test: {env_key!r} in os.environ = {env_key in os.environ}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: tmp_path -- Temporary Directories\n", + "\n", + "The `tmp_path` fixture provides a `pathlib.Path` to a unique temporary directory for each test. Files created there are automatically cleaned up. This is ideal for testing file I/O." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import tempfile\n", + "\n", + "# In pytest:\n", + "#\n", + "# def test_write_file(tmp_path: Path) -> None:\n", + "# file = tmp_path / \"test.txt\"\n", + "# file.write_text(\"hello\")\n", + "# assert file.read_text() == \"hello\"\n", + "\n", + "# Simulating tmp_path with tempfile\n", + "with tempfile.TemporaryDirectory() as tmp_dir:\n", + " tmp_path: Path = Path(tmp_dir)\n", + "\n", + " # Write a file\n", + " test_file: Path = tmp_path / \"test.txt\"\n", + " test_file.write_text(\"hello\")\n", + "\n", + " # Read it back\n", + " content: str = test_file.read_text()\n", + " print(f\"File path: {test_file}\")\n", + " print(f\"File exists: {test_file.exists()}\")\n", + " print(f\"Content: {content!r}\")\n", + " assert content == \"hello\"\n", + "\n", + " # Create subdirectories\n", + " sub_dir: Path = tmp_path / \"subdir\"\n", + " sub_dir.mkdir()\n", + " nested_file: Path = sub_dir / \"data.json\"\n", + " nested_file.write_text('{\"key\": \"value\"}')\n", + "\n", + " print(f\"\\nNested file: {nested_file}\")\n", + " print(f\"Nested content: {nested_file.read_text()}\")\n", + "\n", + "# After the context manager, everything is cleaned up\n", + "print(f\"\\nTemp dir still exists: {Path(tmp_dir).exists()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "\n", + "# Practical example: testing a function that reads/writes files\n", + "def save_report(directory: Path, name: str, lines: list[str]) -> Path:\n", + " \"\"\"Save a report file and return the path.\"\"\"\n", + " file_path: Path = directory / f\"{name}.txt\"\n", + " file_path.write_text(\"\\n\".join(lines))\n", + " return file_path\n", + "\n", + "\n", + "def load_report(file_path: Path) -> list[str]:\n", + " \"\"\"Load a report file and return lines.\"\"\"\n", + " return file_path.read_text().splitlines()\n", + "\n", + "\n", + "# Test using a temporary directory (simulating tmp_path)\n", + "with tempfile.TemporaryDirectory() as tmp_dir:\n", + " tmp_path: Path = Path(tmp_dir)\n", + "\n", + " report_lines: list[str] = [\"Header\", \"Line 1\", \"Line 2\"]\n", + " saved_path: Path = save_report(tmp_path, \"monthly\", report_lines)\n", + "\n", + " print(f\"Saved to: {saved_path.name}\")\n", + " print(f\"File exists: {saved_path.exists()}\")\n", + "\n", + " loaded: list[str] = load_report(saved_path)\n", + " print(f\"Loaded lines: {loaded}\")\n", + " assert loaded == report_lines\n", + " print(\"Round-trip test PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 6: pytest.raises -- Testing Exceptions\n", + "\n", + "`pytest.raises` is a context manager that asserts a block of code raises a specific exception. It can also match against the exception message using a regex pattern." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "\n", + "# In pytest:\n", + "#\n", + "# def test_raises() -> None:\n", + "# with pytest.raises(ValueError, match=\"invalid\"):\n", + "# raise ValueError(\"invalid literal\")\n", + "\n", + "# Simulating pytest.raises behavior\n", + "def assert_raises(\n", + " exception_type: type[BaseException],\n", + " callable_fn: object,\n", + " match: str | None = None,\n", + ") -> None:\n", + " \"\"\"Simulate pytest.raises with optional message matching.\"\"\"\n", + " try:\n", + " if callable(callable_fn):\n", + " callable_fn()\n", + " print(f\" FAILED: {exception_type.__name__} was not raised\")\n", + " except exception_type as e:\n", + " if match and not re.search(match, str(e)):\n", + " print(f\" FAILED: Message {str(e)!r} did not match {match!r}\")\n", + " else:\n", + " print(f\" PASSED: Caught {exception_type.__name__}: {e}\")\n", + " except Exception as e:\n", + " print(f\" FAILED: Expected {exception_type.__name__}, got {type(e).__name__}: {e}\")\n", + "\n", + "\n", + "# Test 1: Basic exception check\n", + "print(\"Test: ValueError is raised\")\n", + "assert_raises(ValueError, lambda: int(\"not_a_number\"))\n", + "\n", + "# Test 2: Exception with message matching\n", + "print(\"\\nTest: ValueError with 'invalid' in message\")\n", + "assert_raises(\n", + " ValueError,\n", + " lambda: (_ for _ in ()).throw(ValueError(\"invalid literal\")),\n", + " match=\"invalid\",\n", + ")\n", + "\n", + "# Test 3: ZeroDivisionError\n", + "print(\"\\nTest: ZeroDivisionError on division by zero\")\n", + "assert_raises(ZeroDivisionError, lambda: 1 / 0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "# Using actual pytest.raises (it works in notebooks too)\n", + "import pytest\n", + "\n", + "\n", + "def validate_age(age: int) -> int:\n", + " \"\"\"Validate that age is a positive integer.\"\"\"\n", + " if age < 0:\n", + " raise ValueError(f\"Age cannot be negative: {age}\")\n", + " if age > 150:\n", + " raise ValueError(f\"Age is unrealistically high: {age}\")\n", + " return age\n", + "\n", + "\n", + "# Test valid input\n", + "result: int = validate_age(25)\n", + "print(f\"validate_age(25) = {result}\")\n", + "\n", + "# Test negative age\n", + "with pytest.raises(ValueError, match=\"cannot be negative\"):\n", + " validate_age(-5)\n", + "print(\"Caught ValueError for negative age\")\n", + "\n", + "# Test too-high age\n", + "with pytest.raises(ValueError, match=\"unrealistically high\"):\n", + " validate_age(200)\n", + "print(\"Caught ValueError for too-high age\")\n", + "\n", + "# Access the exception info\n", + "with pytest.raises(ValueError) as exc_info:\n", + " validate_age(-1)\n", + "print(f\"\\nException type: {type(exc_info.value).__name__}\")\n", + "print(f\"Exception message: {exc_info.value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 7: pytest.approx -- Floating-Point Comparison\n", + "\n", + "Floating-point arithmetic can produce tiny rounding errors. `pytest.approx` compares values with a configurable tolerance, avoiding brittle equality checks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import pytest\n", + "\n", + "# The classic floating-point problem\n", + "result: float = 0.1 + 0.2\n", + "print(f\"0.1 + 0.2 = {result}\")\n", + "print(f\"0.1 + 0.2 == 0.3: {result == 0.3}\")\n", + "\n", + "# pytest.approx handles this gracefully\n", + "print(f\"0.1 + 0.2 == pytest.approx(0.3): {result == pytest.approx(0.3)}\")\n", + "assert 0.1 + 0.2 == pytest.approx(0.3)\n", + "\n", + "# Works with lists too\n", + "computed: list[float] = [0.1, 0.2, 0.1 + 0.2]\n", + "expected: list[float] = [0.1, 0.2, 0.3]\n", + "print(f\"\\nList comparison: {computed == pytest.approx(expected)}\")\n", + "assert computed == pytest.approx(expected)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import pytest\n", + "import math\n", + "\n", + "# Custom tolerance with abs and rel parameters\n", + "# Default: rel=1e-6, abs=1e-12\n", + "\n", + "# Absolute tolerance: values must be within abs of each other\n", + "assert 100.0 == pytest.approx(100.001, abs=0.01)\n", + "print(f\"100.0 ~= 100.001 (abs=0.01): True\")\n", + "\n", + "# Relative tolerance: values must be within rel fraction of expected\n", + "assert 100.0 == pytest.approx(100.05, rel=0.001)\n", + "print(f\"100.0 ~= 100.05 (rel=0.001): True\")\n", + "\n", + "# Practical example: testing math functions\n", + "angle: float = math.pi / 4\n", + "sin_val: float = math.sin(angle)\n", + "cos_val: float = math.cos(angle)\n", + "expected_val: float = math.sqrt(2) / 2\n", + "\n", + "assert sin_val == pytest.approx(expected_val)\n", + "assert cos_val == pytest.approx(expected_val)\n", + "print(f\"\\nsin(pi/4) = {sin_val}\")\n", + "print(f\"cos(pi/4) = {cos_val}\")\n", + "print(f\"sqrt(2)/2 = {expected_val}\")\n", + "print(f\"All approximately equal: True\")\n", + "\n", + "# Dictionary values with approx\n", + "computed_stats: dict[str, float] = {\"mean\": 3.33333, \"std\": 1.41421}\n", + "expected_stats: dict[str, float] = {\"mean\": 10 / 3, \"std\": math.sqrt(2)}\n", + "assert computed_stats == pytest.approx(expected_stats, abs=1e-4)\n", + "print(f\"\\nDict approx comparison passed with abs=1e-4\")" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Section 8: Putting It All Together\n", + "\n", + "Here is a complete example showing how multiple pytest features would work together in a real test file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a5b6c7", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "\n", + "# Module under test\n", + "class UserService:\n", + " \"\"\"Simple user service for demonstration.\"\"\"\n", + "\n", + " def __init__(self, data_dir: Path) -> None:\n", + " self.data_dir: Path = data_dir\n", + "\n", + " def save_user(self, name: str, email: str) -> Path:\n", + " \"\"\"Save user data to a file.\"\"\"\n", + " if not name:\n", + " raise ValueError(\"Name cannot be empty\")\n", + " file_path: Path = self.data_dir / f\"{name.lower()}.txt\"\n", + " file_path.write_text(f\"{name}\\n{email}\")\n", + " return file_path\n", + "\n", + " def load_user(self, name: str) -> dict[str, str]:\n", + " \"\"\"Load user data from a file.\"\"\"\n", + " file_path: Path = self.data_dir / f\"{name.lower()}.txt\"\n", + " if not file_path.exists():\n", + " raise FileNotFoundError(f\"User {name!r} not found\")\n", + " lines: list[str] = file_path.read_text().splitlines()\n", + " return {\"name\": lines[0], \"email\": lines[1]}\n", + "\n", + "\n", + "# --- Test suite (simulating pytest features) ---\n", + "\n", + "with tempfile.TemporaryDirectory() as tmp_dir:\n", + " tmp_path: Path = Path(tmp_dir)\n", + " service: UserService = UserService(tmp_path)\n", + "\n", + " # Test 1: Fixture-provided service + tmp_path for file I/O\n", + " saved: Path = service.save_user(\"Alice\", \"alice@example.com\")\n", + " assert saved.exists()\n", + " print(\"test_save_user: PASSED\")\n", + "\n", + " # Test 2: Round-trip save/load\n", + " user: dict[str, str] = service.load_user(\"Alice\")\n", + " assert user == {\"name\": \"Alice\", \"email\": \"alice@example.com\"}\n", + " print(\"test_load_user: PASSED\")\n", + "\n", + " # Test 3: pytest.raises for empty name\n", + " try:\n", + " service.save_user(\"\", \"no@email.com\")\n", + " print(\"test_empty_name: FAILED (no exception)\")\n", + " except ValueError as e:\n", + " assert \"cannot be empty\" in str(e)\n", + " print(\"test_empty_name_raises: PASSED\")\n", + "\n", + " # Test 4: pytest.raises for missing user\n", + " try:\n", + " service.load_user(\"nonexistent\")\n", + " print(\"test_missing_user: FAILED (no exception)\")\n", + " except FileNotFoundError as e:\n", + " assert \"not found\" in str(e)\n", + " print(\"test_missing_user_raises: PASSED\")\n", + "\n", + " # Test 5: Parametrize-style multiple users\n", + " users: list[tuple[str, str]] = [\n", + " (\"Bob\", \"bob@test.com\"),\n", + " (\"Carol\", \"carol@test.com\"),\n", + " (\"Dave\", \"dave@test.com\"),\n", + " ]\n", + " for name, email in users:\n", + " service.save_user(name, email)\n", + " loaded: dict[str, str] = service.load_user(name)\n", + " assert loaded[\"name\"] == name\n", + " assert loaded[\"email\"] == email\n", + " print(f\"test_parametrized_users ({len(users)} cases): PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "a5b6c7d8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Fixtures\n", + "- **`@pytest.fixture`**: Declare reusable setup functions; tests request them by parameter name\n", + "- **Yield fixtures**: Use `yield` to provide a value and run teardown code after the test\n", + "- **Scopes**: `function` (default), `class`, `module`, `session` control fixture lifetime\n", + "\n", + "### Parametrize\n", + "- **`@pytest.mark.parametrize(\"args\", [...])`**: Run one test with many input sets\n", + "- **`ids` parameter**: Give human-readable names to each test case\n", + "\n", + "### Markers\n", + "- **`@pytest.mark.skip`**: Unconditionally skip a test\n", + "- **`@pytest.mark.skipif(condition)`**: Skip based on a condition (platform, version, etc.)\n", + "- **`@pytest.mark.xfail`**: Mark a test as expected to fail\n", + "- **Custom markers**: Tag tests for selective execution with `-m`\n", + "\n", + "### Built-in Fixtures\n", + "- **`monkeypatch`**: Temporarily modify attributes (`setattr`), env vars (`setenv`), and dict items (`setitem`)\n", + "- **`tmp_path`**: Provides a unique `pathlib.Path` temporary directory per test\n", + "\n", + "### Assertion Helpers\n", + "- **`pytest.raises(ExcType, match=pattern)`**: Assert code raises a specific exception with optional message matching\n", + "- **`pytest.approx(value, rel=1e-6, abs=1e-12)`**: Floating-point comparison with tolerance; works with scalars, lists, and dicts" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_36/02_mocking.ipynb b/src/chapter_36/02_mocking.ipynb new file mode 100644 index 0000000..a29d191 --- /dev/null +++ b/src/chapter_36/02_mocking.ipynb @@ -0,0 +1,820 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 36: Mocking with unittest.mock\n", + "\n", + "This notebook covers Python's `unittest.mock` module, which provides tools to replace parts of your system with mock objects during testing. Mocking is essential for isolating the code under test from its dependencies.\n", + "\n", + "## Key Concepts\n", + "- **Mock**: A flexible object that records calls and can be configured to return specific values\n", + "- **MagicMock**: A Mock subclass with default implementations of dunder methods\n", + "- **patch**: A decorator/context manager that temporarily replaces objects with mocks\n", + "- **side_effect**: Configure a mock to raise exceptions or return values from an iterable\n", + "- **spec**: Restrict a mock to only allow attributes that exist on a real object\n", + "- **Call tracking**: Inspect how mocks were called (arguments, count, order)" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Creating Basic Mocks\n", + "\n", + "A `Mock` object records every interaction. You can call it, access attributes, and later inspect exactly what happened." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "# Create a basic mock\n", + "m: Mock = Mock()\n", + "\n", + "# Calling a mock returns another Mock by default\n", + "result = m()\n", + "print(f\"Calling Mock(): {result}\")\n", + "print(f\"Type of result: {type(result)}\")\n", + "\n", + "# Accessing attributes creates child mocks automatically\n", + "print(f\"\\nm.some_attr: {m.some_attr}\")\n", + "print(f\"m.nested.deep.attr: {m.nested.deep.attr}\")\n", + "\n", + "# Each attribute access returns a consistent mock\n", + "print(f\"\\nm.some_attr is m.some_attr: {m.some_attr is m.some_attr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "# Mock with a name (helpful for debugging)\n", + "db_mock: Mock = Mock(name=\"database\")\n", + "print(f\"Named mock: {db_mock}\")\n", + "\n", + "# Calling with arguments -- the mock records everything\n", + "db_mock.query(\"SELECT * FROM users\", limit=10)\n", + "db_mock.insert({\"name\": \"Alice\", \"age\": 30})\n", + "\n", + "# Inspect what was called\n", + "print(f\"\\ndb_mock.query called: {db_mock.query.called}\")\n", + "print(f\"db_mock.insert called: {db_mock.insert.called}\")\n", + "print(f\"db_mock.delete called: {db_mock.delete.called}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Configuring Return Values\n", + "\n", + "Use `return_value` to control what a mock returns when called. This lets you simulate specific responses from dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "# Set return_value at creation\n", + "m: Mock = Mock(return_value=42)\n", + "result: int = m()\n", + "print(f\"Mock with return_value=42: m() = {result}\")\n", + "assert result == 42\n", + "\n", + "# Set return_value after creation\n", + "m.return_value = \"hello\"\n", + "print(f\"After changing return_value: m() = {m()}\")\n", + "\n", + "# Return value on a method\n", + "api: Mock = Mock()\n", + "api.get_user.return_value = {\"id\": 1, \"name\": \"Alice\"}\n", + "api.get_users.return_value = [{\"id\": 1}, {\"id\": 2}]\n", + "\n", + "user: dict[str, int | str] = api.get_user(1)\n", + "users: list[dict[str, int]] = api.get_users()\n", + "\n", + "print(f\"\\napi.get_user(1) = {user}\")\n", + "print(f\"api.get_users() = {users}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "\n", + "# Practical example: testing code that depends on an external service\n", + "class PaymentProcessor:\n", + " \"\"\"Processes payments via an external gateway.\"\"\"\n", + "\n", + " def __init__(self, gateway: object) -> None:\n", + " self.gateway = gateway\n", + "\n", + " def charge(self, amount: float, card_token: str) -> dict[str, object]:\n", + " \"\"\"Charge a card and return the result.\"\"\"\n", + " response = self.gateway.create_charge( # type: ignore[attr-defined]\n", + " amount=amount, token=card_token\n", + " )\n", + " return {\"success\": response[\"status\"] == \"ok\", \"id\": response[\"charge_id\"]}\n", + "\n", + "\n", + "# Create a mock gateway\n", + "mock_gateway: Mock = Mock()\n", + "mock_gateway.create_charge.return_value = {\"status\": \"ok\", \"charge_id\": \"ch_123\"}\n", + "\n", + "# Test the processor with the mock\n", + "processor: PaymentProcessor = PaymentProcessor(mock_gateway)\n", + "result: dict[str, object] = processor.charge(29.99, \"tok_abc\")\n", + "\n", + "print(f\"Charge result: {result}\")\n", + "assert result == {\"success\": True, \"id\": \"ch_123\"}\n", + "print(\"PaymentProcessor test PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: side_effect -- Exceptions and Iterables\n", + "\n", + "`side_effect` provides more dynamic behavior than `return_value`. It can raise exceptions, return values from an iterable, or run a custom function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "# side_effect with an exception\n", + "m: Mock = Mock(side_effect=ValueError(\"boom\"))\n", + "\n", + "try:\n", + " m()\n", + "except ValueError as e:\n", + " print(f\"Caught exception: {e}\")\n", + "\n", + "# side_effect with an iterable -- returns values in sequence\n", + "m = Mock(side_effect=[1, 2, 3])\n", + "print(f\"\\nFirst call: {m()}\")\n", + "print(f\"Second call: {m()}\")\n", + "print(f\"Third call: {m()}\")\n", + "\n", + "# StopIteration when iterable is exhausted\n", + "try:\n", + " m()\n", + "except StopIteration:\n", + " print(\"Fourth call: StopIteration (iterable exhausted)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "\n", + "# side_effect with a function -- gets called with the same arguments\n", + "def double(x: int) -> int:\n", + " return x * 2\n", + "\n", + "\n", + "m: Mock = Mock(side_effect=double)\n", + "print(f\"m(5) = {m(5)}\")\n", + "print(f\"m(10) = {m(10)}\")\n", + "print(f\"m(0) = {m(0)}\")\n", + "\n", + "# Mixing exceptions in an iterable\n", + "m = Mock(side_effect=[10, ValueError(\"retry failed\"), 30])\n", + "\n", + "print(f\"\\nCall 1: {m()}\")\n", + "try:\n", + " m()\n", + "except ValueError as e:\n", + " print(f\"Call 2: raised ValueError({e})\")\n", + "print(f\"Call 3: {m()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "\n", + "# Practical example: simulating network retries\n", + "class ApiClient:\n", + " \"\"\"HTTP client that retries on failure.\"\"\"\n", + "\n", + " def __init__(self, http: object) -> None:\n", + " self.http = http\n", + "\n", + " def fetch_with_retry(self, url: str, max_retries: int = 3) -> str:\n", + " \"\"\"Fetch URL with retries on connection error.\"\"\"\n", + " for attempt in range(max_retries):\n", + " try:\n", + " return self.http.get(url) # type: ignore[attr-defined]\n", + " except ConnectionError:\n", + " if attempt == max_retries - 1:\n", + " raise\n", + " return \"\" # unreachable, satisfies type checker\n", + "\n", + "\n", + "# Mock HTTP that fails twice then succeeds\n", + "mock_http: Mock = Mock()\n", + "mock_http.get.side_effect = [\n", + " ConnectionError(\"timeout\"),\n", + " ConnectionError(\"timeout\"),\n", + " \"Success\",\n", + "]\n", + "\n", + "client: ApiClient = ApiClient(mock_http)\n", + "result: str = client.fetch_with_retry(\"https://example.com\")\n", + "\n", + "print(f\"Result: {result}\")\n", + "print(f\"Total attempts: {mock_http.get.call_count}\")\n", + "assert result == \"Success\"\n", + "assert mock_http.get.call_count == 3\n", + "print(\"Retry test PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 4: Call Tracking and Assertions\n", + "\n", + "Mocks record every call made to them, including arguments. You can assert exactly how the mock was called." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock, call\n", + "\n", + "m: Mock = Mock()\n", + "\n", + "# Make several calls\n", + "m(\"first\")\n", + "m(\"second\", key=\"value\")\n", + "m(\"third\")\n", + "\n", + "# call_count tracks total calls\n", + "print(f\"call_count: {m.call_count}\")\n", + "assert m.call_count == 3\n", + "\n", + "# called is True if mock was called at least once\n", + "print(f\"called: {m.called}\")\n", + "\n", + "# call_args is the most recent call\n", + "print(f\"call_args (last call): {m.call_args}\")\n", + "\n", + "# call_args_list is the full history\n", + "print(f\"call_args_list: {m.call_args_list}\")\n", + "\n", + "# Assert specific calls\n", + "m.assert_called_with(\"third\") # Checks last call only\n", + "print(\"\\nassert_called_with('third'): PASSED\")\n", + "\n", + "m.assert_any_call(\"second\", key=\"value\") # Checks if this call happened at all\n", + "print(\"assert_any_call('second', key='value'): PASSED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock, call\n", + "\n", + "# assert_called_once_with -- must have been called exactly once\n", + "m: Mock = Mock()\n", + "m(42, name=\"test\")\n", + "\n", + "m.assert_called_once_with(42, name=\"test\")\n", + "print(\"assert_called_once_with(42, name='test'): PASSED\")\n", + "\n", + "# assert_has_calls -- verify a sequence of calls\n", + "logger: Mock = Mock()\n", + "logger.info(\"Starting\")\n", + "logger.debug(\"Processing item 1\")\n", + "logger.debug(\"Processing item 2\")\n", + "logger.info(\"Done\")\n", + "\n", + "expected_calls: list[call] = [\n", + " call.info(\"Starting\"),\n", + " call.debug(\"Processing item 1\"),\n", + " call.debug(\"Processing item 2\"),\n", + " call.info(\"Done\"),\n", + "]\n", + "logger.assert_has_calls(expected_calls)\n", + "print(\"\\nassert_has_calls with exact order: PASSED\")\n", + "\n", + "# With any_order=True, calls can appear in any order\n", + "logger.assert_has_calls(\n", + " [call.info(\"Done\"), call.info(\"Starting\")],\n", + " any_order=True,\n", + ")\n", + "print(\"assert_has_calls with any_order=True: PASSED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "# assert_not_called -- verify mock was never called\n", + "m: Mock = Mock()\n", + "m.assert_not_called()\n", + "print(\"assert_not_called on fresh mock: PASSED\")\n", + "\n", + "# reset_mock -- clear all call history\n", + "m(\"first\")\n", + "m(\"second\")\n", + "print(f\"\\nBefore reset: call_count = {m.call_count}\")\n", + "\n", + "m.reset_mock()\n", + "print(f\"After reset: call_count = {m.call_count}\")\n", + "print(f\"After reset: called = {m.called}\")\n", + "print(f\"After reset: call_args_list = {m.call_args_list}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 5: MagicMock -- Mocking Dunder Methods\n", + "\n", + "`MagicMock` is a subclass of `Mock` that provides default implementations of magic (dunder) methods like `__len__`, `__iter__`, `__getitem__`, and more." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import MagicMock, Mock\n", + "\n", + "# Regular Mock does not support dunder methods out of the box\n", + "# MagicMock does\n", + "\n", + "m: MagicMock = MagicMock()\n", + "\n", + "# __len__\n", + "m.__len__.return_value = 5\n", + "print(f\"len(m) = {len(m)}\")\n", + "assert len(m) == 5\n", + "\n", + "# __bool__\n", + "m.__bool__.return_value = False\n", + "print(f\"bool(m) = {bool(m)}\")\n", + "assert bool(m) is False\n", + "\n", + "# __str__\n", + "m.__str__.return_value = \"custom string\"\n", + "print(f\"str(m) = {str(m)}\")\n", + "\n", + "# __iter__\n", + "m.__iter__.return_value = iter([1, 2, 3])\n", + "items: list[int] = list(m)\n", + "print(f\"list(m) = {items}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import MagicMock\n", + "\n", + "# MagicMock as a context manager\n", + "m: MagicMock = MagicMock()\n", + "m.__enter__.return_value = \"resource\"\n", + "\n", + "with m as resource:\n", + " print(f\"Context manager yielded: {resource}\")\n", + "\n", + "m.__enter__.assert_called_once()\n", + "m.__exit__.assert_called_once()\n", + "print(\"Context manager protocol verified\")\n", + "\n", + "# MagicMock with __getitem__\n", + "config: MagicMock = MagicMock()\n", + "config.__getitem__.side_effect = lambda key: {\"host\": \"localhost\", \"port\": 8080}[key]\n", + "\n", + "print(f\"\\nconfig['host'] = {config['host']}\")\n", + "print(f\"config['port'] = {config['port']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 6: patch -- Replacing Objects During Tests\n", + "\n", + "`patch` temporarily replaces an object (function, class, attribute) with a mock for the duration of a test. It can be used as a decorator or context manager." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import patch\n", + "import os\n", + "\n", + "# patch as a context manager\n", + "print(f\"Before patch: os.getcwd() = {os.getcwd()}\")\n", + "\n", + "with patch(\"os.getcwd\", return_value=\"/fake/path\"):\n", + " print(f\"During patch: os.getcwd() = {os.getcwd()}\")\n", + " assert os.getcwd() == \"/fake/path\"\n", + "\n", + "print(f\"After patch: os.getcwd() = {os.getcwd()}\")\n", + "print(\"\\npatch restored original behavior automatically\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import patch, Mock\n", + "import os\n", + "\n", + "\n", + "# Patching a module-level function\n", + "def get_home_directory() -> str:\n", + " \"\"\"Get the user's home directory.\"\"\"\n", + " return os.path.expanduser(\"~\")\n", + "\n", + "\n", + "print(f\"Real home: {get_home_directory()}\")\n", + "\n", + "with patch(\"os.path.expanduser\", return_value=\"/mock/home\"):\n", + " print(f\"Mocked home: {get_home_directory()}\")\n", + " assert get_home_directory() == \"/mock/home\"\n", + "\n", + "print(f\"Restored home: {get_home_directory()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import patch, MagicMock\n", + "\n", + "\n", + "# patch.object -- patch an attribute on a specific object\n", + "class EmailService:\n", + " def send(self, to: str, subject: str, body: str) -> bool:\n", + " \"\"\"Send a real email (expensive/slow operation).\"\"\"\n", + " raise NotImplementedError(\"Real email sending not available\")\n", + "\n", + "\n", + "class UserRegistration:\n", + " def __init__(self, email_service: EmailService) -> None:\n", + " self.email_service: EmailService = email_service\n", + "\n", + " def register(self, username: str, email: str) -> dict[str, str]:\n", + " \"\"\"Register a user and send a welcome email.\"\"\"\n", + " user: dict[str, str] = {\"username\": username, \"email\": email}\n", + " self.email_service.send(\n", + " to=email,\n", + " subject=\"Welcome!\",\n", + " body=f\"Hello {username}, welcome aboard!\",\n", + " )\n", + " return user\n", + "\n", + "\n", + "email_svc: EmailService = EmailService()\n", + "reg: UserRegistration = UserRegistration(email_svc)\n", + "\n", + "# Patch the send method to avoid real email sending\n", + "with patch.object(email_svc, \"send\", return_value=True) as mock_send:\n", + " user: dict[str, str] = reg.register(\"alice\", \"alice@example.com\")\n", + " print(f\"Registered user: {user}\")\n", + "\n", + " # Verify the email service was called correctly\n", + " mock_send.assert_called_once_with(\n", + " to=\"alice@example.com\",\n", + " subject=\"Welcome!\",\n", + " body=\"Hello alice, welcome aboard!\",\n", + " )\n", + " print(\"Email service called with correct arguments\")\n", + " print(\"Test PASSED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import patch\n", + "\n", + "# patch.dict -- temporarily modify a dictionary\n", + "import os\n", + "\n", + "original_path: str | None = os.environ.get(\"MY_APP_SECRET\")\n", + "print(f\"Before: MY_APP_SECRET = {original_path!r}\")\n", + "\n", + "with patch.dict(os.environ, {\"MY_APP_SECRET\": \"s3cret\", \"DEBUG\": \"true\"}):\n", + " print(f\"During: MY_APP_SECRET = {os.environ['MY_APP_SECRET']!r}\")\n", + " print(f\"During: DEBUG = {os.environ['DEBUG']!r}\")\n", + "\n", + "print(f\"After: MY_APP_SECRET = {os.environ.get('MY_APP_SECRET')!r}\")\n", + "print(f\"After: DEBUG = {os.environ.get('DEBUG')!r}\")\n", + "print(\"\\npatch.dict restored original environment\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Section 7: spec -- Constraining Mocks\n", + "\n", + "Without `spec`, a mock will accept any attribute access or method call, which can hide bugs. The `spec` parameter restricts the mock to only allow attributes that exist on the real object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5b6c7d8", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock\n", + "\n", + "# Without spec -- any attribute works (can hide typos)\n", + "m: Mock = Mock()\n", + "m.append(1) # Fine, even though 'm' isn't a list\n", + "m.nonexistent() # Fine, no error\n", + "print(f\"Without spec: m.nonexistent() = {m.nonexistent()}\")\n", + "\n", + "# With spec -- restricted to real object's interface\n", + "m_list: Mock = Mock(spec=list)\n", + "m_list.append(1) # Valid: list has append\n", + "print(f\"\\nWith spec=list: m_list.append(1) works\")\n", + "\n", + "try:\n", + " m_list.nonexistent() # type: ignore[attr-defined]\n", + "except AttributeError as e:\n", + " print(f\"With spec=list: m_list.nonexistent() -> AttributeError: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5c6d7e8", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock, create_autospec\n", + "\n", + "\n", + "class Calculator:\n", + " \"\"\"A simple calculator.\"\"\"\n", + "\n", + " def add(self, a: float, b: float) -> float:\n", + " return a + b\n", + "\n", + " def divide(self, a: float, b: float) -> float:\n", + " if b == 0:\n", + " raise ZeroDivisionError(\"Cannot divide by zero\")\n", + " return a / b\n", + "\n", + "\n", + "# spec restricts attribute access\n", + "mock_calc: Mock = Mock(spec=Calculator)\n", + "mock_calc.add.return_value = 10.0\n", + "print(f\"mock_calc.add(3, 7) = {mock_calc.add(3, 7)}\")\n", + "\n", + "# create_autospec also checks argument signatures\n", + "auto_calc = create_autospec(Calculator)\n", + "auto_calc.add.return_value = 10.0\n", + "print(f\"auto_calc.add(3, 7) = {auto_calc.add(3, 7)}\")\n", + "\n", + "# Wrong number of arguments raises TypeError\n", + "try:\n", + " auto_calc.add(1, 2, 3) # Too many args\n", + "except TypeError as e:\n", + " print(f\"\\nauto_calc.add(1, 2, 3) -> TypeError: {e}\")\n", + "\n", + "# Non-existent method raises AttributeError\n", + "try:\n", + " auto_calc.multiply(2, 3) # type: ignore[attr-defined]\n", + "except AttributeError as e:\n", + " print(f\"auto_calc.multiply(2, 3) -> AttributeError: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c5d6e7f8", + "metadata": {}, + "source": [ + "## Section 8: Putting It All Together\n", + "\n", + "A complete example combining multiple mocking techniques to test a realistic component." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5e6f7a8", + "metadata": {}, + "outputs": [], + "source": [ + "from unittest.mock import Mock, patch, call, MagicMock\n", + "\n", + "\n", + "# System under test: an order processing pipeline\n", + "class OrderProcessor:\n", + " \"\"\"Processes customer orders using external services.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " inventory: object,\n", + " payment: object,\n", + " notifier: object,\n", + " ) -> None:\n", + " self.inventory = inventory\n", + " self.payment = payment\n", + " self.notifier = notifier\n", + "\n", + " def process_order(\n", + " self, item_id: str, quantity: int, card_token: str, email: str\n", + " ) -> dict[str, object]:\n", + " \"\"\"Process an order: check stock, charge card, send confirmation.\"\"\"\n", + " # Check inventory\n", + " stock: int = self.inventory.check_stock(item_id) # type: ignore[attr-defined]\n", + " if stock < quantity:\n", + " raise ValueError(f\"Insufficient stock: {stock} < {quantity}\")\n", + "\n", + " # Charge payment\n", + " charge = self.payment.charge( # type: ignore[attr-defined]\n", + " amount=quantity * 9.99, token=card_token\n", + " )\n", + "\n", + " # Reserve inventory\n", + " self.inventory.reserve(item_id, quantity) # type: ignore[attr-defined]\n", + "\n", + " # Send confirmation\n", + " self.notifier.send_email( # type: ignore[attr-defined]\n", + " to=email, subject=\"Order Confirmed\", body=f\"Order {charge['id']} confirmed\"\n", + " )\n", + "\n", + " return {\"order_id\": charge[\"id\"], \"quantity\": quantity, \"status\": \"confirmed\"}\n", + "\n", + "\n", + "# Create mocks for all dependencies\n", + "mock_inventory: Mock = Mock()\n", + "mock_payment: Mock = Mock()\n", + "mock_notifier: Mock = Mock()\n", + "\n", + "# Configure mock responses\n", + "mock_inventory.check_stock.return_value = 50\n", + "mock_payment.charge.return_value = {\"id\": \"ord_789\", \"status\": \"ok\"}\n", + "\n", + "# Create processor and run test\n", + "processor: OrderProcessor = OrderProcessor(mock_inventory, mock_payment, mock_notifier)\n", + "result: dict[str, object] = processor.process_order(\n", + " item_id=\"SKU_001\", quantity=3, card_token=\"tok_xyz\", email=\"bob@test.com\"\n", + ")\n", + "\n", + "# Verify result\n", + "print(f\"Order result: {result}\")\n", + "assert result == {\"order_id\": \"ord_789\", \"quantity\": 3, \"status\": \"confirmed\"}\n", + "\n", + "# Verify interactions\n", + "mock_inventory.check_stock.assert_called_once_with(\"SKU_001\")\n", + "mock_inventory.reserve.assert_called_once_with(\"SKU_001\", 3)\n", + "mock_payment.charge.assert_called_once_with(amount=29.97, token=\"tok_xyz\")\n", + "mock_notifier.send_email.assert_called_once_with(\n", + " to=\"bob@test.com\",\n", + " subject=\"Order Confirmed\",\n", + " body=\"Order ord_789 confirmed\",\n", + ")\n", + "\n", + "print(\"\\nAll mock interactions verified:\")\n", + "print(f\" inventory.check_stock calls: {mock_inventory.check_stock.call_count}\")\n", + "print(f\" inventory.reserve calls: {mock_inventory.reserve.call_count}\")\n", + "print(f\" payment.charge calls: {mock_payment.charge.call_count}\")\n", + "print(f\" notifier.send_email calls: {mock_notifier.send_email.call_count}\")\n", + "print(\"\\nOrder processing test PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "e5f6a7b8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Mock and MagicMock\n", + "- **`Mock()`**: Creates a callable mock that records all interactions\n", + "- **`MagicMock()`**: A Mock subclass with default implementations of dunder methods (`__len__`, `__iter__`, `__getitem__`, etc.)\n", + "- **`Mock(name=\"...\")`**: Give a mock a name for better debugging output\n", + "\n", + "### Configuring Behavior\n", + "- **`return_value`**: Set what the mock returns when called (`Mock(return_value=42)`)\n", + "- **`side_effect`**: Raise exceptions (`side_effect=ValueError(...)`), return from iterable (`side_effect=[1, 2, 3]`), or call a function (`side_effect=my_func`)\n", + "\n", + "### Call Tracking\n", + "- **`m.called`**: Boolean -- was the mock called?\n", + "- **`m.call_count`**: Integer -- how many times was it called?\n", + "- **`m.call_args`**: The arguments of the most recent call\n", + "- **`m.call_args_list`**: Complete history of all calls\n", + "\n", + "### Assertion Methods\n", + "- **`assert_called_once_with(...)`**: Called exactly once with these arguments\n", + "- **`assert_called_with(...)`**: Last call had these arguments\n", + "- **`assert_any_call(...)`**: This call appeared at least once\n", + "- **`assert_has_calls([...], any_order=False)`**: Verify a sequence of calls\n", + "- **`assert_not_called()`**: Mock was never called\n", + "\n", + "### patch\n", + "- **`patch(\"module.attr\")`**: Replace an attribute with a mock (context manager or decorator)\n", + "- **`patch.object(obj, \"attr\")`**: Replace an attribute on a specific object\n", + "- **`patch.dict(dict_obj, values)`**: Temporarily modify a dictionary\n", + "\n", + "### spec and Autospec\n", + "- **`Mock(spec=ClassName)`**: Restrict attribute access to real interface\n", + "- **`create_autospec(ClassName)`**: Also validates argument signatures" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_36/03_test_patterns.ipynb b/src/chapter_36/03_test_patterns.ipynb new file mode 100644 index 0000000..85fabc8 --- /dev/null +++ b/src/chapter_36/03_test_patterns.ipynb @@ -0,0 +1,1181 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 36: Test Patterns and Organization\n", + "\n", + "This notebook covers common testing patterns that make tests more readable, maintainable, and reliable. These patterns are framework-agnostic and apply to any Python testing approach.\n", + "\n", + "## Key Concepts\n", + "- **AAA Pattern**: Arrange-Act-Assert structures tests into three clear phases\n", + "- **Test Doubles**: Stubs, spies, and fakes replace real dependencies in tests\n", + "- **Builder Pattern**: Fluent interface for constructing complex test data\n", + "- **Test Organization**: Naming conventions, grouping, and helper functions" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: The AAA Pattern (Arrange-Act-Assert)\n", + "\n", + "The AAA pattern divides each test into three distinct phases:\n", + "\n", + "1. **Arrange**: Set up the test data and preconditions\n", + "2. **Act**: Execute the code being tested\n", + "3. **Assert**: Verify the results\n", + "\n", + "This structure makes tests easy to read and understand at a glance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# Example 1: A clean AAA test\n", + "def test_sort_preserves_all_elements() -> None:\n", + " # Arrange\n", + " items: list[int] = [3, 1, 4, 1, 5]\n", + "\n", + " # Act\n", + " result: list[int] = sorted(items)\n", + "\n", + " # Assert\n", + " assert result == [1, 1, 3, 4, 5]\n", + " assert len(result) == len(items)\n", + " print(f\"Input: {items}\")\n", + " print(f\"Sorted: {result}\")\n", + " print(\"Test PASSED: sorted list has correct order and length\")\n", + "\n", + "\n", + "test_sort_preserves_all_elements()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "# Example 2: AAA with more complex arrangement\n", + "class ShoppingCart:\n", + " \"\"\"A simple shopping cart.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._items: list[dict[str, str | float | int]] = []\n", + "\n", + " def add_item(self, name: str, price: float, quantity: int = 1) -> None:\n", + " self._items.append({\"name\": name, \"price\": price, \"quantity\": quantity})\n", + "\n", + " def total(self) -> float:\n", + " return sum(item[\"price\"] * item[\"quantity\"] for item in self._items) # type: ignore[operator]\n", + "\n", + " def item_count(self) -> int:\n", + " return sum(item[\"quantity\"] for item in self._items) # type: ignore[arg-type]\n", + "\n", + "\n", + "def test_cart_total_with_multiple_items() -> None:\n", + " # Arrange\n", + " cart: ShoppingCart = ShoppingCart()\n", + " cart.add_item(\"Widget\", price=9.99, quantity=2)\n", + " cart.add_item(\"Gadget\", price=24.99, quantity=1)\n", + "\n", + " # Act\n", + " total: float = cart.total()\n", + " count: int = cart.item_count()\n", + "\n", + " # Assert\n", + " expected_total: float = 9.99 * 2 + 24.99\n", + " assert abs(total - expected_total) < 0.01\n", + " assert count == 3\n", + " print(f\"Total: ${total:.2f} (expected ${expected_total:.2f})\")\n", + " print(f\"Item count: {count}\")\n", + " print(\"Test PASSED\")\n", + "\n", + "\n", + "test_cart_total_with_multiple_items()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "# Example 3: AAA with exception testing\n", + "def divide(a: float, b: float) -> float:\n", + " \"\"\"Divide a by b, raising ValueError for zero divisor.\"\"\"\n", + " if b == 0:\n", + " raise ValueError(\"Cannot divide by zero\")\n", + " return a / b\n", + "\n", + "\n", + "def test_divide_by_zero_raises() -> None:\n", + " # Arrange\n", + " numerator: float = 10.0\n", + " divisor: float = 0.0\n", + "\n", + " # Act & Assert (combined for exception testing)\n", + " try:\n", + " divide(numerator, divisor)\n", + " assert False, \"Expected ValueError\"\n", + " except ValueError as e:\n", + " assert \"Cannot divide by zero\" in str(e)\n", + " print(f\"Caught expected error: {e}\")\n", + " print(\"Test PASSED\")\n", + "\n", + "\n", + "test_divide_by_zero_raises()" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 2: AAA Anti-Patterns\n", + "\n", + "Understanding common mistakes helps write better tests. Here are patterns to avoid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "# Anti-pattern 1: Multiple Acts in one test (test does too much)\n", + "# BAD -- this tests two behaviors at once\n", + "def test_bad_multiple_acts() -> None:\n", + " cart: ShoppingCart = ShoppingCart()\n", + "\n", + " cart.add_item(\"A\", price=10.0) # Act 1\n", + " assert cart.total() == 10.0 # Assert 1\n", + "\n", + " cart.add_item(\"B\", price=20.0) # Act 2\n", + " assert cart.total() == 30.0 # Assert 2\n", + "\n", + " # If Assert 1 fails, we never test Act 2\n", + "\n", + "\n", + "# GOOD -- separate tests, each with one act\n", + "def test_good_single_item_total() -> None:\n", + " # Arrange\n", + " cart: ShoppingCart = ShoppingCart()\n", + " cart.add_item(\"A\", price=10.0)\n", + " # Act\n", + " total: float = cart.total()\n", + " # Assert\n", + " assert total == 10.0\n", + "\n", + "\n", + "def test_good_two_item_total() -> None:\n", + " # Arrange\n", + " cart: ShoppingCart = ShoppingCart()\n", + " cart.add_item(\"A\", price=10.0)\n", + " cart.add_item(\"B\", price=20.0)\n", + " # Act\n", + " total: float = cart.total()\n", + " # Assert\n", + " assert total == 30.0\n", + "\n", + "\n", + "test_good_single_item_total()\n", + "test_good_two_item_total()\n", + "print(\"Both focused tests PASSED\")\n", + "print(\"Tip: Each test should have exactly one Act phase\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "# Anti-pattern 2: Asserting on implementation details rather than behavior\n", + "\n", + "class UserRepository:\n", + " \"\"\"Stores users in an internal list.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._users: list[dict[str, str]] = []\n", + "\n", + " def add(self, name: str, email: str) -> None:\n", + " self._users.append({\"name\": name, \"email\": email})\n", + "\n", + " def find_by_name(self, name: str) -> dict[str, str] | None:\n", + " for user in self._users:\n", + " if user[\"name\"] == name:\n", + " return user\n", + " return None\n", + "\n", + " def count(self) -> int:\n", + " return len(self._users)\n", + "\n", + "\n", + "# BAD: Testing internal state\n", + "# assert repo._users == [{\"name\": \"Alice\", \"email\": \"alice@test.com\"}]\n", + "\n", + "# GOOD: Testing through the public interface\n", + "def test_add_and_find_user() -> None:\n", + " # Arrange\n", + " repo: UserRepository = UserRepository()\n", + "\n", + " # Act\n", + " repo.add(\"Alice\", \"alice@test.com\")\n", + " result: dict[str, str] | None = repo.find_by_name(\"Alice\")\n", + "\n", + " # Assert (through public interface)\n", + " assert result is not None\n", + " assert result[\"email\"] == \"alice@test.com\"\n", + " assert repo.count() == 1\n", + " print(f\"Found user: {result}\")\n", + " print(\"Test PASSED: asserted through public interface, not internal state\")\n", + "\n", + "\n", + "test_add_and_find_user()" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 3: Test Doubles -- Stubs\n", + "\n", + "A **stub** is a test double that provides canned responses to calls. It replaces a real dependency with a simplified version that returns predetermined data. Stubs answer the question: *\"Given this dependency returns X, what does my code do?\"*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Protocol\n", + "\n", + "\n", + "# Define the interface using a Protocol\n", + "class Database(Protocol):\n", + " def query(self, sql: str) -> list[dict[str, str | int]]: ...\n", + " def execute(self, sql: str) -> int: ...\n", + "\n", + "\n", + "# Stub implementation returns canned data\n", + "class DatabaseStub:\n", + " \"\"\"A stub that returns predetermined query results.\"\"\"\n", + "\n", + " def query(self, sql: str) -> list[dict[str, str | int]]:\n", + " return [{\"id\": 1, \"name\": \"Alice\"}, {\"id\": 2, \"name\": \"Bob\"}]\n", + "\n", + " def execute(self, sql: str) -> int:\n", + " return 1 # Always reports 1 row affected\n", + "\n", + "\n", + "# Code under test\n", + "class UserService:\n", + " \"\"\"Service that depends on a database.\"\"\"\n", + "\n", + " def __init__(self, db: Database) -> None:\n", + " self.db: Database = db\n", + "\n", + " def get_all_names(self) -> list[str]:\n", + " rows: list[dict[str, str | int]] = self.db.query(\"SELECT * FROM users\")\n", + " return [str(row[\"name\"]) for row in rows]\n", + "\n", + "\n", + "# Test using the stub\n", + "def test_get_all_names_returns_names() -> None:\n", + " # Arrange\n", + " stub_db: DatabaseStub = DatabaseStub()\n", + " service: UserService = UserService(stub_db)\n", + "\n", + " # Act\n", + " names: list[str] = service.get_all_names()\n", + "\n", + " # Assert\n", + " assert names == [\"Alice\", \"Bob\"]\n", + " print(f\"Names from stub: {names}\")\n", + " print(\"Test PASSED: stub provided canned data without a real DB\")\n", + "\n", + "\n", + "test_get_all_names_returns_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "# Configurable stubs let you test different scenarios\n", + "class ConfigurableDatabaseStub:\n", + " \"\"\"Stub with configurable responses.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " query_result: list[dict[str, str | int]] | None = None,\n", + " should_raise: bool = False,\n", + " ) -> None:\n", + " self._query_result: list[dict[str, str | int]] = query_result or []\n", + " self._should_raise: bool = should_raise\n", + "\n", + " def query(self, sql: str) -> list[dict[str, str | int]]:\n", + " if self._should_raise:\n", + " raise ConnectionError(\"Database unavailable\")\n", + " return self._query_result\n", + "\n", + " def execute(self, sql: str) -> int:\n", + " if self._should_raise:\n", + " raise ConnectionError(\"Database unavailable\")\n", + " return 1\n", + "\n", + "\n", + "# Scenario 1: Empty result\n", + "empty_stub: ConfigurableDatabaseStub = ConfigurableDatabaseStub(query_result=[])\n", + "service1: UserService = UserService(empty_stub)\n", + "result1: list[str] = service1.get_all_names()\n", + "assert result1 == []\n", + "print(f\"Empty DB scenario: {result1}\")\n", + "\n", + "# Scenario 2: Single user\n", + "one_user_stub: ConfigurableDatabaseStub = ConfigurableDatabaseStub(\n", + " query_result=[{\"id\": 1, \"name\": \"Charlie\"}]\n", + ")\n", + "service2: UserService = UserService(one_user_stub)\n", + "result2: list[str] = service2.get_all_names()\n", + "assert result2 == [\"Charlie\"]\n", + "print(f\"One user scenario: {result2}\")\n", + "\n", + "# Scenario 3: Database error\n", + "error_stub: ConfigurableDatabaseStub = ConfigurableDatabaseStub(should_raise=True)\n", + "service3: UserService = UserService(error_stub)\n", + "try:\n", + " service3.get_all_names()\n", + "except ConnectionError as e:\n", + " print(f\"Error scenario: caught {e}\")\n", + "\n", + "print(\"All stub scenarios PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 4: Test Doubles -- Spies\n", + "\n", + "A **spy** records how it was called while optionally delegating to a real implementation. Spies answer the question: *\"Was my dependency called correctly?\"*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "# A spy that records all calls\n", + "class LoggerSpy:\n", + " \"\"\"Records all log messages for later inspection.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.messages: list[tuple[str, str]] = []\n", + "\n", + " def log(self, level: str, message: str) -> None:\n", + " self.messages.append((level, message))\n", + "\n", + " def info(self, message: str) -> None:\n", + " self.log(\"INFO\", message)\n", + "\n", + " def error(self, message: str) -> None:\n", + " self.log(\"ERROR\", message)\n", + "\n", + "\n", + "# Code under test\n", + "class DataImporter:\n", + " \"\"\"Imports data and logs the process.\"\"\"\n", + "\n", + " def __init__(self, logger: LoggerSpy) -> None:\n", + " self.logger: LoggerSpy = logger\n", + "\n", + " def import_data(self, records: list[str]) -> int:\n", + " self.logger.info(f\"Starting import of {len(records)} records\")\n", + " imported: int = 0\n", + " for record in records:\n", + " if record.strip():\n", + " imported += 1\n", + " else:\n", + " self.logger.error(\"Skipped empty record\")\n", + " self.logger.info(f\"Import complete: {imported} records imported\")\n", + " return imported\n", + "\n", + "\n", + "# Test using the spy\n", + "def test_import_logs_correctly() -> None:\n", + " # Arrange\n", + " spy: LoggerSpy = LoggerSpy()\n", + " importer: DataImporter = DataImporter(spy)\n", + "\n", + " # Act\n", + " count: int = importer.import_data([\"alice\", \"\", \"bob\"])\n", + "\n", + " # Assert on behavior (what was logged)\n", + " assert count == 2\n", + " assert len(spy.messages) == 3\n", + " assert spy.messages[0] == (\"INFO\", \"Starting import of 3 records\")\n", + " assert spy.messages[1] == (\"ERROR\", \"Skipped empty record\")\n", + " assert spy.messages[2] == (\"INFO\", \"Import complete: 2 records imported\")\n", + "\n", + " print(\"Spy recorded messages:\")\n", + " for level, msg in spy.messages:\n", + " print(f\" [{level}] {msg}\")\n", + " print(f\"\\nImported count: {count}\")\n", + " print(\"Test PASSED: spy verified correct logging behavior\")\n", + "\n", + "\n", + "test_import_logs_correctly()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "# Spy that also delegates to a real implementation\n", + "class NotificationSpy:\n", + " \"\"\"Spy that wraps a real notifier and records calls.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.sent_notifications: list[dict[str, str]] = []\n", + "\n", + " def send(self, recipient: str, message: str) -> bool:\n", + " self.sent_notifications.append(\n", + " {\"recipient\": recipient, \"message\": message}\n", + " )\n", + " return True # Simulate successful send\n", + "\n", + " @property\n", + " def call_count(self) -> int:\n", + " return len(self.sent_notifications)\n", + "\n", + " def was_sent_to(self, recipient: str) -> bool:\n", + " return any(n[\"recipient\"] == recipient for n in self.sent_notifications)\n", + "\n", + "\n", + "# Use the spy\n", + "spy: NotificationSpy = NotificationSpy()\n", + "spy.send(\"alice@test.com\", \"Welcome!\")\n", + "spy.send(\"bob@test.com\", \"Your order shipped\")\n", + "\n", + "print(f\"Total notifications sent: {spy.call_count}\")\n", + "print(f\"Sent to alice: {spy.was_sent_to('alice@test.com')}\")\n", + "print(f\"Sent to charlie: {spy.was_sent_to('charlie@test.com')}\")\n", + "print(f\"All notifications: {spy.sent_notifications}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 5: Test Doubles -- Fakes\n", + "\n", + "A **fake** is a working implementation of a dependency that is simpler than the real one. Unlike stubs (which return canned data), fakes have actual logic. Common examples include in-memory databases and file system abstractions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Protocol\n", + "\n", + "\n", + "# Protocol defining the storage interface\n", + "class KeyValueStore(Protocol):\n", + " def get(self, key: str) -> str | None: ...\n", + " def set(self, key: str, value: str) -> None: ...\n", + " def delete(self, key: str) -> bool: ...\n", + " def keys(self) -> list[str]: ...\n", + "\n", + "\n", + "# Fake implementation: in-memory store (instead of Redis, for example)\n", + "class InMemoryKeyValueStore:\n", + " \"\"\"Fake key-value store backed by a dictionary.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._data: dict[str, str] = {}\n", + "\n", + " def get(self, key: str) -> str | None:\n", + " return self._data.get(key)\n", + "\n", + " def set(self, key: str, value: str) -> None:\n", + " self._data[key] = value\n", + "\n", + " def delete(self, key: str) -> bool:\n", + " if key in self._data:\n", + " del self._data[key]\n", + " return True\n", + " return False\n", + "\n", + " def keys(self) -> list[str]:\n", + " return list(self._data.keys())\n", + "\n", + "\n", + "# Code under test\n", + "class CacheService:\n", + " \"\"\"A service that uses a key-value store for caching.\"\"\"\n", + "\n", + " def __init__(self, store: KeyValueStore) -> None:\n", + " self.store: KeyValueStore = store\n", + "\n", + " def get_or_compute(self, key: str, compute_fn: object) -> str:\n", + " \"\"\"Get from cache, or compute and store.\"\"\"\n", + " cached: str | None = self.store.get(key)\n", + " if cached is not None:\n", + " return cached\n", + " value: str = compute_fn() # type: ignore[operator]\n", + " self.store.set(key, value)\n", + " return value\n", + "\n", + "\n", + "# Test with fake\n", + "def test_cache_stores_computed_value() -> None:\n", + " # Arrange\n", + " fake_store: InMemoryKeyValueStore = InMemoryKeyValueStore()\n", + " cache: CacheService = CacheService(fake_store)\n", + " compute_count: list[int] = [0]\n", + "\n", + " def expensive_computation() -> str:\n", + " compute_count[0] += 1\n", + " return \"computed_result\"\n", + "\n", + " # Act: first call computes\n", + " result1: str = cache.get_or_compute(\"key1\", expensive_computation)\n", + " # Act: second call uses cache\n", + " result2: str = cache.get_or_compute(\"key1\", expensive_computation)\n", + "\n", + " # Assert\n", + " assert result1 == \"computed_result\"\n", + " assert result2 == \"computed_result\"\n", + " assert compute_count[0] == 1 # Only computed once\n", + " assert fake_store.get(\"key1\") == \"computed_result\"\n", + "\n", + " print(f\"First call result: {result1}\")\n", + " print(f\"Second call result: {result2}\")\n", + " print(f\"Computation count: {compute_count[0]}\")\n", + " print(f\"Stored keys: {fake_store.keys()}\")\n", + " print(\"Test PASSED: fake store enabled real caching behavior test\")\n", + "\n", + "\n", + "test_cache_stores_computed_value()" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 6: Comparing Test Doubles\n", + "\n", + "Each type of test double serves a different purpose. Choosing the right one depends on what you are testing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "# Summary of test double types with examples\n", + "\n", + "test_doubles: list[dict[str, str]] = [\n", + " {\n", + " \"type\": \"Stub\",\n", + " \"purpose\": \"Provide canned responses\",\n", + " \"when_to_use\": \"You need to control what a dependency returns\",\n", + " \"verifies\": \"Output/state of code under test\",\n", + " \"example\": \"DatabaseStub that always returns [{id: 1, name: Alice}]\",\n", + " },\n", + " {\n", + " \"type\": \"Spy\",\n", + " \"purpose\": \"Record interactions\",\n", + " \"when_to_use\": \"You need to verify how a dependency was called\",\n", + " \"verifies\": \"Interactions with the dependency\",\n", + " \"example\": \"LoggerSpy that records all log messages\",\n", + " },\n", + " {\n", + " \"type\": \"Fake\",\n", + " \"purpose\": \"Simplified working implementation\",\n", + " \"when_to_use\": \"You need realistic behavior without external resources\",\n", + " \"verifies\": \"Integration behavior with a simpler dependency\",\n", + " \"example\": \"InMemoryKeyValueStore instead of Redis\",\n", + " },\n", + " {\n", + " \"type\": \"Mock (unittest.mock)\",\n", + " \"purpose\": \"Configurable stub + spy in one\",\n", + " \"when_to_use\": \"You want maximum flexibility without writing a class\",\n", + " \"verifies\": \"Both outputs and interactions\",\n", + " \"example\": \"Mock(return_value=42) with assert_called_once_with()\",\n", + " },\n", + "]\n", + "\n", + "for td in test_doubles:\n", + " print(f\"{td['type']}:\")\n", + " print(f\" Purpose: {td['purpose']}\")\n", + " print(f\" When to use: {td['when_to_use']}\")\n", + " print(f\" Verifies: {td['verifies']}\")\n", + " print(f\" Example: {td['example']}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 7: Builder Pattern for Test Data\n", + "\n", + "The **builder pattern** provides a fluent interface for constructing complex test objects. Instead of writing long constructors with many parameters, builders let you specify only the values that matter for each test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "\n", + "class UserBuilder:\n", + " \"\"\"Builder for creating test user dictionaries.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._name: str = \"default_user\"\n", + " self._age: int = 25\n", + " self._email: str = \"default@test.com\"\n", + " self._role: str = \"user\"\n", + " self._active: bool = True\n", + "\n", + " def with_name(self, name: str) -> UserBuilder:\n", + " self._name = name\n", + " return self\n", + "\n", + " def with_age(self, age: int) -> UserBuilder:\n", + " self._age = age\n", + " return self\n", + "\n", + " def with_email(self, email: str) -> UserBuilder:\n", + " self._email = email\n", + " return self\n", + "\n", + " def with_role(self, role: str) -> UserBuilder:\n", + " self._role = role\n", + " return self\n", + "\n", + " def inactive(self) -> UserBuilder:\n", + " self._active = False\n", + " return self\n", + "\n", + " def build(self) -> dict[str, str | int | bool]:\n", + " return {\n", + " \"name\": self._name,\n", + " \"age\": self._age,\n", + " \"email\": self._email,\n", + " \"role\": self._role,\n", + " \"active\": self._active,\n", + " }\n", + "\n", + "\n", + "# Using the builder: only specify what matters for each test\n", + "default_user = UserBuilder().build()\n", + "print(f\"Default user: {default_user}\")\n", + "\n", + "admin = UserBuilder().with_name(\"Alice\").with_role(\"admin\").build()\n", + "print(f\"Admin user: {admin}\")\n", + "\n", + "inactive = UserBuilder().with_name(\"Bob\").with_age(30).inactive().build()\n", + "print(f\"Inactive user: {inactive}\")\n", + "\n", + "# Each test only specifies the fields relevant to its assertion\n", + "assert admin[\"role\"] == \"admin\"\n", + "assert inactive[\"active\"] is False\n", + "print(\"\\nBuilder pattern tests PASSED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "\n", + "# A more complex builder for an order object\n", + "class OrderBuilder:\n", + " \"\"\"Builder for creating test order objects.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._customer: str = \"Test Customer\"\n", + " self._items: list[dict[str, str | float | int]] = []\n", + " self._discount: float = 0.0\n", + " self._shipping: str = \"standard\"\n", + "\n", + " def for_customer(self, name: str) -> OrderBuilder:\n", + " self._customer = name\n", + " return self\n", + "\n", + " def with_item(\n", + " self, name: str, price: float, quantity: int = 1\n", + " ) -> OrderBuilder:\n", + " self._items.append({\"name\": name, \"price\": price, \"quantity\": quantity})\n", + " return self\n", + "\n", + " def with_discount(self, percent: float) -> OrderBuilder:\n", + " self._discount = percent\n", + " return self\n", + "\n", + " def with_express_shipping(self) -> OrderBuilder:\n", + " self._shipping = \"express\"\n", + " return self\n", + "\n", + " def build(self) -> dict[str, object]:\n", + " subtotal: float = sum(\n", + " item[\"price\"] * item[\"quantity\"] # type: ignore[operator]\n", + " for item in self._items\n", + " )\n", + " total: float = subtotal * (1 - self._discount / 100)\n", + " return {\n", + " \"customer\": self._customer,\n", + " \"items\": self._items,\n", + " \"subtotal\": round(subtotal, 2),\n", + " \"discount\": self._discount,\n", + " \"total\": round(total, 2),\n", + " \"shipping\": self._shipping,\n", + " }\n", + "\n", + "\n", + "# Test: discount is applied correctly\n", + "def test_order_discount() -> None:\n", + " # Arrange (builder makes intent clear)\n", + " order = (\n", + " OrderBuilder()\n", + " .with_item(\"Widget\", price=100.0)\n", + " .with_discount(10)\n", + " .build()\n", + " )\n", + "\n", + " # Assert\n", + " assert order[\"subtotal\"] == 100.0\n", + " assert order[\"total\"] == 90.0\n", + " print(f\"Order with 10% discount: {order}\")\n", + " print(\"Discount test PASSED\")\n", + "\n", + "\n", + "# Test: express shipping is set\n", + "def test_express_shipping() -> None:\n", + " order = (\n", + " OrderBuilder()\n", + " .for_customer(\"Alice\")\n", + " .with_item(\"Gadget\", price=50.0, quantity=2)\n", + " .with_express_shipping()\n", + " .build()\n", + " )\n", + "\n", + " assert order[\"shipping\"] == \"express\"\n", + " assert order[\"total\"] == 100.0\n", + " print(f\"\\nExpress order: {order}\")\n", + " print(\"Express shipping test PASSED\")\n", + "\n", + "\n", + "test_order_discount()\n", + "test_express_shipping()" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 8: Test Organization and Naming\n", + "\n", + "Well-organized tests are easier to maintain and understand. Good naming conventions, logical grouping, and helper functions reduce duplication." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "# Test naming conventions\n", + "#\n", + "# Good names describe: WHAT is being tested and WHAT is expected\n", + "#\n", + "# Pattern: test___\n", + "# test_divide_by_zero_raises_value_error\n", + "# test_sort_empty_list_returns_empty\n", + "# test_login_invalid_password_returns_false\n", + "#\n", + "# Or: test__ (simpler)\n", + "# test_add_item_increases_count\n", + "# test_remove_missing_item_raises\n", + "\n", + "naming_examples: list[dict[str, str]] = [\n", + " {\"bad\": \"test_1\", \"good\": \"test_add_positive_numbers_returns_sum\"},\n", + " {\"bad\": \"test_error\", \"good\": \"test_divide_by_zero_raises_value_error\"},\n", + " {\"bad\": \"test_it_works\", \"good\": \"test_login_valid_credentials_returns_token\"},\n", + " {\"bad\": \"test_edge\", \"good\": \"test_sort_empty_list_returns_empty_list\"},\n", + "]\n", + "\n", + "print(\"Test Naming Examples:\")\n", + "print(f\"{'Bad Name':<20} {'Good Name'}\")\n", + "print(\"-\" * 65)\n", + "for ex in naming_examples:\n", + " print(f\"{ex['bad']:<20} {ex['good']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a5b6c7", + "metadata": {}, + "outputs": [], + "source": [ + "# Test grouping with classes (how pytest organizes tests)\n", + "#\n", + "# class TestShoppingCart: # Group by unit\n", + "# class TestAddItem: # Group by method\n", + "# def test_increases_count\n", + "# def test_updates_total\n", + "# class TestRemoveItem:\n", + "# def test_decreases_count\n", + "# def test_missing_item_raises\n", + "\n", + "# Simulating grouped tests\n", + "class TestStack:\n", + " \"\"\"Tests for a stack data structure.\"\"\"\n", + "\n", + " @staticmethod\n", + " def _make_stack(items: list[int] | None = None) -> list[int]:\n", + " \"\"\"Helper to create a stack (reduces duplication).\"\"\"\n", + " return list(items) if items else []\n", + "\n", + " def test_push_adds_to_top(self) -> None:\n", + " stack: list[int] = self._make_stack()\n", + " stack.append(42)\n", + " assert stack[-1] == 42\n", + " print(\"test_push_adds_to_top: PASSED\")\n", + "\n", + " def test_pop_removes_from_top(self) -> None:\n", + " stack: list[int] = self._make_stack([1, 2, 3])\n", + " top: int = stack.pop()\n", + " assert top == 3\n", + " assert len(stack) == 2\n", + " print(\"test_pop_removes_from_top: PASSED\")\n", + "\n", + " def test_pop_empty_raises(self) -> None:\n", + " stack: list[int] = self._make_stack()\n", + " try:\n", + " stack.pop()\n", + " assert False, \"Should have raised\"\n", + " except IndexError:\n", + " print(\"test_pop_empty_raises: PASSED\")\n", + "\n", + " def test_peek_does_not_remove(self) -> None:\n", + " stack: list[int] = self._make_stack([10, 20])\n", + " top: int = stack[-1]\n", + " assert top == 20\n", + " assert len(stack) == 2\n", + " print(\"test_peek_does_not_remove: PASSED\")\n", + "\n", + "\n", + "# Run grouped tests\n", + "suite: TestStack = TestStack()\n", + "print(\"TestStack:\")\n", + "suite.test_push_adds_to_top()\n", + "suite.test_pop_removes_from_top()\n", + "suite.test_pop_empty_raises()\n", + "suite.test_peek_does_not_remove()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5b6c7d8", + "metadata": {}, + "outputs": [], + "source": [ + "# Shared fixtures and helper functions reduce duplication\n", + "#\n", + "# In pytest, you'd put shared fixtures in conftest.py:\n", + "#\n", + "# conftest.py:\n", + "# @pytest.fixture\n", + "# def sample_users() -> list[dict]:\n", + "# return [\n", + "# UserBuilder().with_name(\"Alice\").build(),\n", + "# UserBuilder().with_name(\"Bob\").with_role(\"admin\").build(),\n", + "# ]\n", + "\n", + "# Simulating shared test infrastructure\n", + "def make_test_users(count: int = 3) -> list[dict[str, str | int | bool]]:\n", + " \"\"\"Factory function for creating test users.\"\"\"\n", + " names: list[str] = [\"Alice\", \"Bob\", \"Charlie\", \"Diana\", \"Eve\"]\n", + " return [\n", + " UserBuilder()\n", + " .with_name(names[i])\n", + " .with_age(25 + i * 5)\n", + " .with_email(f\"{names[i].lower()}@test.com\")\n", + " .build()\n", + " for i in range(min(count, len(names)))\n", + " ]\n", + "\n", + "\n", + "# Tests use the factory\n", + "users: list[dict[str, str | int | bool]] = make_test_users(3)\n", + "print(\"Test users from factory:\")\n", + "for user in users:\n", + " print(f\" {user['name']} (age={user['age']}, email={user['email']})\")\n", + "\n", + "assert len(users) == 3\n", + "assert users[0][\"name\"] == \"Alice\"\n", + "assert users[2][\"name\"] == \"Charlie\"\n", + "print(\"\\nFactory function test PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "b5c6d7e8", + "metadata": {}, + "source": [ + "## Section 9: Putting It All Together\n", + "\n", + "A complete example combining AAA pattern, test doubles, and builder pattern to test a realistic system." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5d6e7f8", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "\n", + "# --- Production Code ---\n", + "\n", + "class NotificationService:\n", + " \"\"\"Sends notifications to users based on their preferences.\"\"\"\n", + "\n", + " def __init__(self, email_sender: object, sms_sender: object) -> None:\n", + " self.email_sender = email_sender\n", + " self.sms_sender = sms_sender\n", + "\n", + " def notify(\n", + " self, user: dict[str, object], message: str\n", + " ) -> list[str]:\n", + " \"\"\"Send notification via user's preferred channels.\"\"\"\n", + " channels_used: list[str] = []\n", + " prefs = user.get(\"preferences\", {})\n", + "\n", + " if prefs.get(\"email\"): # type: ignore[union-attr]\n", + " self.email_sender.send( # type: ignore[attr-defined]\n", + " to=user[\"email\"], body=message\n", + " )\n", + " channels_used.append(\"email\")\n", + "\n", + " if prefs.get(\"sms\"): # type: ignore[union-attr]\n", + " self.sms_sender.send( # type: ignore[attr-defined]\n", + " to=user[\"phone\"], body=message\n", + " )\n", + " channels_used.append(\"sms\")\n", + "\n", + " return channels_used\n", + "\n", + "\n", + "# --- Test Infrastructure ---\n", + "\n", + "class SenderSpy:\n", + " \"\"\"Spy for email/SMS senders.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.sent: list[dict[str, str]] = []\n", + "\n", + " def send(self, to: str, body: str) -> None:\n", + " self.sent.append({\"to\": to, \"body\": body})\n", + "\n", + "\n", + "class NotificationUserBuilder:\n", + " \"\"\"Builder for users with notification preferences.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._name: str = \"Test User\"\n", + " self._email: str = \"test@example.com\"\n", + " self._phone: str = \"+1234567890\"\n", + " self._pref_email: bool = False\n", + " self._pref_sms: bool = False\n", + "\n", + " def with_name(self, name: str) -> NotificationUserBuilder:\n", + " self._name = name\n", + " return self\n", + "\n", + " def with_email_notifications(self) -> NotificationUserBuilder:\n", + " self._pref_email = True\n", + " return self\n", + "\n", + " def with_sms_notifications(self) -> NotificationUserBuilder:\n", + " self._pref_sms = True\n", + " return self\n", + "\n", + " def build(self) -> dict[str, object]:\n", + " return {\n", + " \"name\": self._name,\n", + " \"email\": self._email,\n", + " \"phone\": self._phone,\n", + " \"preferences\": {\n", + " \"email\": self._pref_email,\n", + " \"sms\": self._pref_sms,\n", + " },\n", + " }\n", + "\n", + "\n", + "# --- Tests ---\n", + "\n", + "def test_email_only_user_receives_email() -> None:\n", + " # Arrange\n", + " email_spy: SenderSpy = SenderSpy()\n", + " sms_spy: SenderSpy = SenderSpy()\n", + " service: NotificationService = NotificationService(email_spy, sms_spy)\n", + " user = NotificationUserBuilder().with_name(\"Alice\").with_email_notifications().build()\n", + "\n", + " # Act\n", + " channels: list[str] = service.notify(user, \"Hello!\")\n", + "\n", + " # Assert\n", + " assert channels == [\"email\"]\n", + " assert len(email_spy.sent) == 1\n", + " assert email_spy.sent[0][\"body\"] == \"Hello!\"\n", + " assert len(sms_spy.sent) == 0\n", + " print(\"test_email_only: PASSED\")\n", + "\n", + "\n", + "def test_both_channels_user_receives_both() -> None:\n", + " # Arrange\n", + " email_spy: SenderSpy = SenderSpy()\n", + " sms_spy: SenderSpy = SenderSpy()\n", + " service: NotificationService = NotificationService(email_spy, sms_spy)\n", + " user = (\n", + " NotificationUserBuilder()\n", + " .with_name(\"Bob\")\n", + " .with_email_notifications()\n", + " .with_sms_notifications()\n", + " .build()\n", + " )\n", + "\n", + " # Act\n", + " channels: list[str] = service.notify(user, \"Alert!\")\n", + "\n", + " # Assert\n", + " assert channels == [\"email\", \"sms\"]\n", + " assert len(email_spy.sent) == 1\n", + " assert len(sms_spy.sent) == 1\n", + " print(\"test_both_channels: PASSED\")\n", + "\n", + "\n", + "def test_no_preferences_sends_nothing() -> None:\n", + " # Arrange\n", + " email_spy: SenderSpy = SenderSpy()\n", + " sms_spy: SenderSpy = SenderSpy()\n", + " service: NotificationService = NotificationService(email_spy, sms_spy)\n", + " user = NotificationUserBuilder().with_name(\"Charlie\").build()\n", + "\n", + " # Act\n", + " channels: list[str] = service.notify(user, \"Nothing\")\n", + "\n", + " # Assert\n", + " assert channels == []\n", + " assert len(email_spy.sent) == 0\n", + " assert len(sms_spy.sent) == 0\n", + " print(\"test_no_preferences: PASSED\")\n", + "\n", + "\n", + "print(\"Notification Service Tests:\")\n", + "test_email_only_user_receives_email()\n", + "test_both_channels_user_receives_both()\n", + "test_no_preferences_sends_nothing()\n", + "print(\"\\nAll tests PASSED\")" + ] + }, + { + "cell_type": "markdown", + "id": "d5e6f7a8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### AAA Pattern (Arrange-Act-Assert)\n", + "- **Arrange**: Set up test data, create objects, configure dependencies\n", + "- **Act**: Execute the single behavior being tested\n", + "- **Assert**: Verify the expected outcome (return value, state change, or interaction)\n", + "- Keep each test focused on one behavior; avoid multiple Act phases\n", + "\n", + "### Test Doubles\n", + "- **Stub**: Returns canned data; use when you need to control what a dependency returns\n", + "- **Spy**: Records interactions; use when you need to verify how a dependency was called\n", + "- **Fake**: Simplified working implementation; use when you need realistic behavior without external resources\n", + "- **Mock** (`unittest.mock`): Combines stub + spy functionality in a single configurable object\n", + "\n", + "### Builder Pattern\n", + "- Creates complex test objects with a fluent interface (`builder.with_x().with_y().build()`)\n", + "- Provides sensible defaults so tests only specify values relevant to the assertion\n", + "- Reduces noise and makes test intent clear\n", + "\n", + "### Test Organization\n", + "- **Naming**: `test___` makes tests self-documenting\n", + "- **Grouping**: Use test classes to group related tests by unit or feature\n", + "- **Helpers**: Factory functions and fixture functions reduce duplication\n", + "- **Shared fixtures**: Place in `conftest.py` for cross-module reuse\n", + "- Test through public interfaces, not internal state" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_36/README.md b/src/chapter_36/README.md new file mode 100644 index 0000000..9d6a5e5 --- /dev/null +++ b/src/chapter_36/README.md @@ -0,0 +1,13 @@ +# Chapter 36: Advanced Testing + +## Topics Covered + +- pytest fixtures, parametrize, markers, monkeypatch, tmp_path +- unittest.mock: Mock, MagicMock, patch, side_effect, spec +- Test patterns: test doubles, parametric testing, test organization + +## Notebooks + +1. **01_pytest_features.ipynb** - Fixtures, parametrize, markers, monkeypatch, tmp_path +2. **02_mocking.ipynb** - unittest.mock: Mock, MagicMock, patch, side_effect +3. **03_test_patterns.ipynb** - Test doubles, parametric strategies, test organization diff --git a/src/chapter_36/__init__.py b/src/chapter_36/__init__.py new file mode 100644 index 0000000..d020523 --- /dev/null +++ b/src/chapter_36/__init__.py @@ -0,0 +1 @@ +"""Chapter 36: Advanced Testing.""" diff --git a/src/chapter_37/01_ast_basics.ipynb b/src/chapter_37/01_ast_basics.ipynb new file mode 100644 index 0000000..e60cd72 --- /dev/null +++ b/src/chapter_37/01_ast_basics.ipynb @@ -0,0 +1,429 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 37: AST Basics\n", + "\n", + "This notebook introduces Python's `ast` module, which lets you parse Python source code into an Abstract Syntax Tree (AST). You will learn how to parse code strings, inspect tree structure, examine node types and fields, and safely evaluate literal expressions with `ast.literal_eval`.\n", + "\n", + "## Key Concepts\n", + "- **`ast.parse`**: Parses a source string into an AST `Module` node\n", + "- **`ast.dump`**: Produces a string representation of the AST structure\n", + "- **Node types**: `Module`, `Assign`, `FunctionDef`, `Expr`, `BinOp`, `Constant`, `Name`\n", + "- **Node fields**: Each node type has typed attributes (e.g., `BinOp.op`, `Name.id`)\n", + "- **`ast.literal_eval`**: Safely evaluates strings containing Python literals" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Parsing Source Code with `ast.parse`\n", + "\n", + "`ast.parse` takes a string of Python source code and returns an `ast.Module` node. The module's `body` attribute is a list of top-level statement nodes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Parse a simple assignment statement\n", + "source: str = \"x = 1 + 2\"\n", + "tree: ast.Module = ast.parse(source)\n", + "\n", + "print(f\"Tree type: {type(tree).__name__}\")\n", + "print(f\"Is Module: {isinstance(tree, ast.Module)}\")\n", + "print(f\"Number of statements: {len(tree.body)}\")\n", + "print(f\"First statement type: {type(tree.body[0]).__name__}\")\n", + "print(f\"Is Assign: {isinstance(tree.body[0], ast.Assign)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Parse a function definition\n", + "source: str = \"def greet(name): return f'Hello, {name}'\"\n", + "tree: ast.Module = ast.parse(source)\n", + "\n", + "func: ast.FunctionDef = tree.body[0] # type: ignore[assignment]\n", + "print(f\"Node type: {type(func).__name__}\")\n", + "print(f\"Is FunctionDef: {isinstance(func, ast.FunctionDef)}\")\n", + "print(f\"Function name: {func.name}\")\n", + "print(f\"Number of args: {len(func.args.args)}\")\n", + "print(f\"First arg name: {func.args.args[0].arg}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Parse multiple statements\n", + "source: str = \"\"\"\n", + "x = 10\n", + "y = 20\n", + "result = x + y\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "\n", + "print(f\"Number of statements: {len(tree.body)}\")\n", + "for i, stmt in enumerate(tree.body):\n", + " print(f\" Statement {i}: {type(stmt).__name__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 2: Inspecting Trees with `ast.dump`\n", + "\n", + "`ast.dump` returns a string representation of the AST, showing all node types and their field values. The `indent` parameter formats the output for readability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Dump a simple constant expression\n", + "tree: ast.Module = ast.parse(\"42\")\n", + "dumped: str = ast.dump(tree)\n", + "\n", + "print(f\"Dump: {dumped}\")\n", + "print(f\"Contains 'Constant': {'Constant' in dumped}\")\n", + "print(f\"Contains '42': {'42' in dumped}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Dump with indentation for readability\n", + "tree: ast.Module = ast.parse(\"x = 1 + 2\")\n", + "print(ast.dump(tree, indent=2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Dump a function definition to see the full structure\n", + "source: str = \"def add(a, b): return a + b\"\n", + "tree: ast.Module = ast.parse(source)\n", + "print(ast.dump(tree, indent=2))" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 3: AST Node Types\n", + "\n", + "The `ast` module defines many node types. The most common ones include:\n", + "- **`Module`**: The root of any parsed source\n", + "- **`Assign`**: A variable assignment (`x = ...`)\n", + "- **`FunctionDef`**: A function definition (`def ...`)\n", + "- **`Expr`**: An expression used as a statement\n", + "- **`BinOp`**: A binary operation (`a + b`, `x * y`)\n", + "- **`Constant`**: A literal value (`42`, `\"hello\"`)\n", + "- **`Name`**: A variable reference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Examining an Assign node\n", + "tree: ast.Module = ast.parse(\"x = 42\")\n", + "assign: ast.Assign = tree.body[0] # type: ignore[assignment]\n", + "\n", + "print(f\"Type: {type(assign).__name__}\")\n", + "print(f\"Targets: {[type(t).__name__ for t in assign.targets]}\")\n", + "print(f\"Target name: {assign.targets[0].id}\") # type: ignore[attr-defined]\n", + "print(f\"Value type: {type(assign.value).__name__}\")\n", + "print(f\"Value: {assign.value.value}\") # type: ignore[attr-defined]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Examining an expression statement with BinOp\n", + "tree: ast.Module = ast.parse(\"x + y\")\n", + "expr: ast.Expr = tree.body[0] # type: ignore[assignment]\n", + "\n", + "print(f\"Statement type: {type(expr).__name__}\")\n", + "print(f\"Is Expr: {isinstance(expr, ast.Expr)}\")\n", + "\n", + "binop: ast.BinOp = expr.value # type: ignore[assignment]\n", + "print(f\"\\nValue type: {type(binop).__name__}\")\n", + "print(f\"Is BinOp: {isinstance(binop, ast.BinOp)}\")\n", + "print(f\"Operator: {type(binop.op).__name__}\")\n", + "print(f\"Is Add: {isinstance(binop.op, ast.Add)}\")\n", + "print(f\"Left: {type(binop.left).__name__} (id={binop.left.id})\") # type: ignore[attr-defined]\n", + "print(f\"Right: {type(binop.right).__name__} (id={binop.right.id})\") # type: ignore[attr-defined]" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 4: Node Fields and Attributes\n", + "\n", + "Every AST node has `_fields` listing its child field names, and `_attributes` listing metadata like line numbers and column offsets." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Inspect the fields of various node types\n", + "node_types: list[type] = [ast.Module, ast.Assign, ast.FunctionDef, ast.BinOp, ast.Name]\n", + "\n", + "for node_type in node_types:\n", + " print(f\"{node_type.__name__}:\")\n", + " print(f\" _fields: {node_type._fields}\")\n", + " print(f\" _attributes: {node_type._attributes}\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Access line number and column offset from parsed nodes\n", + "source: str = \"\"\"x = 10\n", + "y = x + 5\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "\n", + "for stmt in tree.body:\n", + " print(f\"{type(stmt).__name__} at line {stmt.lineno}, col {stmt.col_offset}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 5: Safe Evaluation with `ast.literal_eval`\n", + "\n", + "`ast.literal_eval` safely evaluates a string containing a Python literal expression. It only accepts strings, numbers, tuples, lists, dicts, booleans, `None`, `bytes`, and sets. It rejects any other expression, making it safe for untrusted input." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# literal_eval with basic types\n", + "int_val: int = ast.literal_eval(\"42\")\n", + "print(f\"Integer: {int_val} (type: {type(int_val).__name__})\")\n", + "\n", + "list_val: list[int] = ast.literal_eval(\"[1, 2, 3]\")\n", + "print(f\"List: {list_val} (type: {type(list_val).__name__})\")\n", + "\n", + "dict_val: dict[str, int] = ast.literal_eval(\"{'a': 1}\")\n", + "print(f\"Dict: {dict_val} (type: {type(dict_val).__name__})\")\n", + "\n", + "tuple_val: tuple[int, ...] = ast.literal_eval(\"(1, 2, 3)\")\n", + "print(f\"Tuple: {tuple_val} (type: {type(tuple_val).__name__})\")\n", + "\n", + "bool_val: bool = ast.literal_eval(\"True\")\n", + "print(f\"Bool: {bool_val} (type: {type(bool_val).__name__})\")\n", + "\n", + "none_val: None = ast.literal_eval(\"None\")\n", + "print(f\"None: {none_val} (type: {type(none_val).__name__})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# literal_eval rejects non-literal expressions for safety\n", + "dangerous_inputs: list[str] = [\n", + " \"__import__('os')\",\n", + " \"open('/etc/passwd')\",\n", + " \"1 + 2\",\n", + " \"x\",\n", + "]\n", + "\n", + "for expr in dangerous_inputs:\n", + " try:\n", + " ast.literal_eval(expr)\n", + " print(f\" '{expr}' -> accepted (unexpected!)\")\n", + " except (ValueError, SyntaxError) as e:\n", + " print(f\" '{expr}' -> rejected: {type(e).__name__}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Practical use: parsing configuration strings safely\n", + "config_strings: dict[str, str] = {\n", + " \"ports\": \"[8080, 8443, 9090]\",\n", + " \"settings\": \"{'debug': True, 'verbose': False}\",\n", + " \"max_retries\": \"5\",\n", + "}\n", + "\n", + "for key, value_str in config_strings.items():\n", + " parsed = ast.literal_eval(value_str)\n", + " print(f\"{key}: {parsed} (type: {type(parsed).__name__})\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 6: Building AST Nodes Programmatically\n", + "\n", + "You can create AST nodes directly by calling their constructors. This is useful for code generation or AST transformation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Build an AST node for \"x = 42\" manually\n", + "assign_node: ast.Assign = ast.Assign(\n", + " targets=[ast.Name(id=\"x\", ctx=ast.Store())],\n", + " value=ast.Constant(value=42),\n", + " lineno=1,\n", + " col_offset=0,\n", + ")\n", + "\n", + "# Wrap in a Module\n", + "module: ast.Module = ast.Module(body=[assign_node], type_ignores=[])\n", + "ast.fix_missing_locations(module)\n", + "\n", + "print(\"Hand-built AST:\")\n", + "print(ast.dump(module, indent=2))\n", + "\n", + "# Compare with parsed version\n", + "parsed: ast.Module = ast.parse(\"x = 42\")\n", + "print(f\"\\nDumps match: {ast.dump(module) == ast.dump(parsed)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Parsing and Inspection\n", + "- **`ast.parse(source)`**: Returns an `ast.Module` node from a source code string\n", + "- **`ast.dump(tree, indent=N)`**: Shows the full AST structure as a readable string\n", + "- **`tree.body`**: List of top-level statement nodes in a `Module`\n", + "\n", + "### Core Node Types\n", + "- **`Module`**: Root node containing a list of statements\n", + "- **`Assign`**: Variable assignment with `.targets` and `.value`\n", + "- **`FunctionDef`**: Function definition with `.name`, `.args`, `.body`\n", + "- **`Expr`**: Expression used as a statement, wrapping a `.value`\n", + "- **`BinOp`**: Binary operation with `.left`, `.op`, `.right`\n", + "- **`Name`**: Variable reference with `.id` and `.ctx`\n", + "- **`Constant`**: Literal value with `.value`\n", + "\n", + "### Node Introspection\n", + "- **`node._fields`**: Tuple of child field names\n", + "- **`node._attributes`**: Tuple of metadata attributes (lineno, col_offset, etc.)\n", + "\n", + "### Safe Evaluation\n", + "- **`ast.literal_eval(string)`**: Safely parses literal values (ints, lists, dicts, etc.)\n", + "- Rejects function calls, variable references, and any non-literal expression\n", + "- Preferred over `eval()` when processing untrusted input" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_37/02_node_visitors.ipynb b/src/chapter_37/02_node_visitors.ipynb new file mode 100644 index 0000000..702d3b9 --- /dev/null +++ b/src/chapter_37/02_node_visitors.ipynb @@ -0,0 +1,580 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 37: Node Visitors and Transformers\n", + "\n", + "This notebook covers the visitor pattern for traversing and modifying Abstract Syntax Trees. You will learn how to use `ast.NodeVisitor` to analyze code, `ast.NodeTransformer` to modify trees, and `ast.walk` for simple iteration over all nodes.\n", + "\n", + "## Key Concepts\n", + "- **`ast.NodeVisitor`**: Base class for read-only AST traversal using `visit_*` methods\n", + "- **`ast.NodeTransformer`**: Base class for modifying AST nodes in place\n", + "- **`ast.walk`**: Simple iterator that yields every node in the tree\n", + "- **`ast.fix_missing_locations`**: Fills in line numbers after tree modification\n", + "- **`generic_visit`**: Continues traversal to child nodes" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: NodeVisitor Basics\n", + "\n", + "`ast.NodeVisitor` provides a `visit` method that dispatches to `visit_` methods. For example, `visit_Name` is called for every `Name` node. Call `self.generic_visit(node)` to continue visiting child nodes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class NameCollector(ast.NodeVisitor):\n", + " \"\"\"Collects all variable names referenced in the code.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.names: list[str] = []\n", + "\n", + " def visit_Name(self, node: ast.Name) -> None:\n", + " \"\"\"Called for every Name node in the AST.\"\"\"\n", + " self.names.append(node.id)\n", + " self.generic_visit(node)\n", + "\n", + "\n", + "# Parse and visit\n", + "tree: ast.Module = ast.parse(\"x = y + z\")\n", + "collector: NameCollector = NameCollector()\n", + "collector.visit(tree)\n", + "\n", + "print(f\"Names found: {collector.names}\")\n", + "print(f\"'y' in names: {'y' in collector.names}\")\n", + "print(f\"'z' in names: {'z' in collector.names}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class NameCollector(ast.NodeVisitor):\n", + " \"\"\"Collects all variable names referenced in the code.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.names: list[str] = []\n", + "\n", + " def visit_Name(self, node: ast.Name) -> None:\n", + " self.names.append(node.id)\n", + " self.generic_visit(node)\n", + "\n", + "\n", + "# Collect names from a more complex expression\n", + "source: str = \"\"\"\n", + "total = price * quantity\n", + "tax = total * rate\n", + "final = total + tax\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "collector: NameCollector = NameCollector()\n", + "collector.visit(tree)\n", + "\n", + "print(f\"All names: {collector.names}\")\n", + "unique_names: set[str] = set(collector.names)\n", + "print(f\"Unique names: {sorted(unique_names)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Counting Specific Node Types\n", + "\n", + "A common use of `NodeVisitor` is counting occurrences of specific constructs, such as function definitions, loops, or imports." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class FuncCounter(ast.NodeVisitor):\n", + " \"\"\"Counts function definitions in the AST.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.count: int = 0\n", + "\n", + " def visit_FunctionDef(self, node: ast.FunctionDef) -> None:\n", + " \"\"\"Called for every function definition.\"\"\"\n", + " self.count += 1\n", + " self.generic_visit(node)\n", + "\n", + "\n", + "source: str = \"\"\"\n", + "def foo(): pass\n", + "def bar(): pass\n", + "def baz(): pass\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "counter: FuncCounter = FuncCounter()\n", + "counter.visit(tree)\n", + "\n", + "print(f\"Number of functions: {counter.count}\")\n", + "print(f\"Count is 3: {counter.count == 3}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class CodeAnalyzer(ast.NodeVisitor):\n", + " \"\"\"Counts multiple types of nodes in the AST.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.functions: int = 0\n", + " self.assignments: int = 0\n", + " self.names: list[str] = []\n", + "\n", + " def visit_FunctionDef(self, node: ast.FunctionDef) -> None:\n", + " self.functions += 1\n", + " self.names.append(node.name)\n", + " self.generic_visit(node)\n", + "\n", + " def visit_Assign(self, node: ast.Assign) -> None:\n", + " self.assignments += 1\n", + " self.generic_visit(node)\n", + "\n", + "\n", + "source: str = \"\"\"\n", + "x = 10\n", + "y = 20\n", + "\n", + "def add(a, b):\n", + " return a + b\n", + "\n", + "def multiply(a, b):\n", + " result = a * b\n", + " return result\n", + "\n", + "z = add(x, y)\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "analyzer: CodeAnalyzer = CodeAnalyzer()\n", + "analyzer.visit(tree)\n", + "\n", + "print(f\"Functions: {analyzer.functions}\")\n", + "print(f\"Function names: {analyzer.names}\")\n", + "print(f\"Assignments: {analyzer.assignments}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: NodeTransformer for Modifying the AST\n", + "\n", + "`ast.NodeTransformer` works like `NodeVisitor` but its `visit_*` methods return a node. Returning a different node replaces the original. After transforming, call `ast.fix_missing_locations` to fill in line numbers before compiling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class DoubleConstants(ast.NodeTransformer):\n", + " \"\"\"Doubles all integer constants in the AST.\"\"\"\n", + "\n", + " def visit_Constant(self, node: ast.Constant) -> ast.Constant:\n", + " if isinstance(node.value, int):\n", + " return ast.Constant(value=node.value * 2)\n", + " return node\n", + "\n", + "\n", + "# Parse, transform, and execute\n", + "tree: ast.Module = ast.parse(\"x = 5\")\n", + "print(f\"Before: {ast.dump(tree)}\")\n", + "\n", + "new_tree: ast.Module = DoubleConstants().visit(tree)\n", + "ast.fix_missing_locations(new_tree)\n", + "print(f\"After: {ast.dump(new_tree)}\")\n", + "\n", + "# Compile and run the modified tree\n", + "code = compile(new_tree, \"\", \"exec\")\n", + "ns: dict[str, object] = {}\n", + "exec(code, ns) # noqa: S102\n", + "print(f\"\\nx = {ns['x']}\")\n", + "print(f\"x == 10: {ns['x'] == 10}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class NegateConstants(ast.NodeTransformer):\n", + " \"\"\"Negates all integer constants in the AST.\"\"\"\n", + "\n", + " def visit_Constant(self, node: ast.Constant) -> ast.Constant:\n", + " if isinstance(node.value, int):\n", + " return ast.Constant(value=-node.value)\n", + " return node\n", + "\n", + "\n", + "source: str = \"result = 10 + 20\"\n", + "tree: ast.Module = ast.parse(source)\n", + "new_tree: ast.Module = NegateConstants().visit(tree)\n", + "ast.fix_missing_locations(new_tree)\n", + "\n", + "code = compile(new_tree, \"\", \"exec\")\n", + "ns: dict[str, object] = {}\n", + "exec(code, ns) # noqa: S102\n", + "print(f\"result = {ns['result']}\")\n", + "print(f\"result == -30: {ns['result'] == -30}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: `ast.fix_missing_locations`\n", + "\n", + "When you create or modify AST nodes, they may lack line number and column offset information. `ast.fix_missing_locations` copies location data from parent nodes, which is required before compiling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Create a new node without location info\n", + "new_node: ast.Constant = ast.Constant(value=99)\n", + "print(f\"Has lineno: {hasattr(new_node, 'lineno') and new_node.lineno is not None}\")\n", + "\n", + "# Build a module with the node\n", + "module: ast.Module = ast.Module(\n", + " body=[ast.Expr(value=new_node)],\n", + " type_ignores=[],\n", + ")\n", + "\n", + "# fix_missing_locations adds line/col info\n", + "ast.fix_missing_locations(module)\n", + "print(f\"After fix - lineno: {module.body[0].lineno}\")\n", + "print(f\"After fix - col_offset: {module.body[0].col_offset}\")\n", + "\n", + "# Now it can be compiled\n", + "code = compile(module, \"\", \"exec\")\n", + "print(f\"\\nCompiled successfully: {code is not None}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 5: Iterating All Nodes with `ast.walk`\n", + "\n", + "`ast.walk` provides a simple way to iterate over every node in the AST without defining a visitor class. It yields nodes in no particular order and does not give you control over recursion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Walk all nodes in an expression\n", + "tree: ast.Module = ast.parse(\"a = b + c * d\")\n", + "node_types: set[str] = {type(node).__name__ for node in ast.walk(tree)}\n", + "\n", + "print(f\"All node types: {sorted(node_types)}\")\n", + "print(f\"'BinOp' found: {'BinOp' in node_types}\")\n", + "print(f\"'Name' found: {'Name' in node_types}\")\n", + "print(f\"'Add' found: {'Add' in node_types}\")\n", + "print(f\"'Mult' found: {'Mult' in node_types}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Count node types using ast.walk\n", + "source: str = \"\"\"\n", + "def greet(name):\n", + " message = \"Hello, \" + name\n", + " return message\n", + "\n", + "def farewell(name):\n", + " return \"Goodbye, \" + name\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "\n", + "# Count specific node types with walk\n", + "func_count: int = sum(1 for node in ast.walk(tree) if isinstance(node, ast.FunctionDef))\n", + "name_count: int = sum(1 for node in ast.walk(tree) if isinstance(node, ast.Name))\n", + "const_count: int = sum(1 for node in ast.walk(tree) if isinstance(node, ast.Constant))\n", + "\n", + "print(f\"Functions: {func_count}\")\n", + "print(f\"Name references: {name_count}\")\n", + "print(f\"Constants: {const_count}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Collect all string constants using walk\n", + "source: str = \"\"\"\n", + "name = \"Alice\"\n", + "greeting = \"Hello\"\n", + "message = greeting + \", \" + name + \"!\"\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "\n", + "strings: list[str] = [\n", + " node.value\n", + " for node in ast.walk(tree)\n", + " if isinstance(node, ast.Constant) and isinstance(node.value, str)\n", + "]\n", + "\n", + "print(f\"String constants: {strings}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 6: Practical Code Analysis Patterns\n", + "\n", + "Combining visitors and walkers enables practical code analysis tools such as finding unused imports, detecting complexity, or extracting documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class ImportCollector(ast.NodeVisitor):\n", + " \"\"\"Collects all import statements from source code.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.imports: list[str] = []\n", + "\n", + " def visit_Import(self, node: ast.Import) -> None:\n", + " for alias in node.names:\n", + " self.imports.append(alias.name)\n", + " self.generic_visit(node)\n", + "\n", + " def visit_ImportFrom(self, node: ast.ImportFrom) -> None:\n", + " module: str = node.module or \"\"\n", + " for alias in node.names:\n", + " self.imports.append(f\"{module}.{alias.name}\")\n", + " self.generic_visit(node)\n", + "\n", + "\n", + "source: str = \"\"\"\n", + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "from collections import defaultdict, Counter\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "collector: ImportCollector = ImportCollector()\n", + "collector.visit(tree)\n", + "\n", + "print(\"Imports found:\")\n", + "for imp in collector.imports:\n", + " print(f\" {imp}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class FunctionSummarizer(ast.NodeVisitor):\n", + " \"\"\"Extracts function signatures and docstrings.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.summaries: list[dict[str, object]] = []\n", + "\n", + " def visit_FunctionDef(self, node: ast.FunctionDef) -> None:\n", + " args: list[str] = [arg.arg for arg in node.args.args]\n", + " docstring: str | None = ast.get_docstring(node)\n", + " self.summaries.append({\n", + " \"name\": node.name,\n", + " \"args\": args,\n", + " \"lineno\": node.lineno,\n", + " \"docstring\": docstring,\n", + " })\n", + " self.generic_visit(node)\n", + "\n", + "\n", + "source: str = \"\"\"\n", + "def add(a, b):\n", + " \\\"\\\"\\\"Add two numbers.\\\"\\\"\\\" \n", + " return a + b\n", + "\n", + "def multiply(x, y, z):\n", + " \\\"\\\"\\\"Multiply three numbers together.\\\"\\\"\\\" \n", + " return x * y * z\n", + "\n", + "def no_doc():\n", + " pass\n", + "\"\"\"\n", + "tree: ast.Module = ast.parse(source)\n", + "summarizer: FunctionSummarizer = FunctionSummarizer()\n", + "summarizer.visit(tree)\n", + "\n", + "for summary in summarizer.summaries:\n", + " print(f\"Function: {summary['name']}({', '.join(summary['args'])})\")\n", + " print(f\" Line: {summary['lineno']}\")\n", + " print(f\" Docstring: {summary['docstring']}\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class RenameVariable(ast.NodeTransformer):\n", + " \"\"\"Renames all occurrences of a variable in the AST.\"\"\"\n", + "\n", + " def __init__(self, old_name: str, new_name: str) -> None:\n", + " self.old_name: str = old_name\n", + " self.new_name: str = new_name\n", + "\n", + " def visit_Name(self, node: ast.Name) -> ast.Name:\n", + " if node.id == self.old_name:\n", + " return ast.Name(id=self.new_name, ctx=node.ctx)\n", + " return node\n", + "\n", + "\n", + "source: str = \"result = x + y * x\"\n", + "tree: ast.Module = ast.parse(source)\n", + "print(f\"Before: {ast.dump(tree, indent=2)}\")\n", + "\n", + "# Rename 'x' to 'value'\n", + "renamed: ast.Module = RenameVariable(\"x\", \"value\").visit(tree)\n", + "ast.fix_missing_locations(renamed)\n", + "print(f\"\\nAfter: {ast.dump(renamed, indent=2)}\")\n", + "\n", + "# Verify by compiling and running\n", + "code = compile(renamed, \"\", \"exec\")\n", + "ns: dict[str, object] = {\"value\": 3, \"y\": 4}\n", + "exec(code, ns) # noqa: S102\n", + "print(f\"\\nresult = {ns['result']} (expected: 3 + 4 * 3 = 15)\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### NodeVisitor\n", + "- Subclass `ast.NodeVisitor` and define `visit_` methods for read-only traversal\n", + "- Call `self.generic_visit(node)` to continue visiting child nodes\n", + "- Use `visitor.visit(tree)` to start the traversal from the root\n", + "\n", + "### NodeTransformer\n", + "- Subclass `ast.NodeTransformer` and return modified or replacement nodes from `visit_*` methods\n", + "- Return the original node to keep it unchanged\n", + "- Always call `ast.fix_missing_locations(tree)` after transformation before compiling\n", + "\n", + "### ast.walk\n", + "- `ast.walk(tree)` yields every node in the tree with no guaranteed order\n", + "- Simpler than `NodeVisitor` for flat queries (counting, collecting)\n", + "- No control over recursion depth or traversal order\n", + "\n", + "### Common Patterns\n", + "- **Name collection**: `visit_Name` to gather variable references\n", + "- **Function counting**: `visit_FunctionDef` to count or analyze functions\n", + "- **Import analysis**: `visit_Import` and `visit_ImportFrom` for dependency tracking\n", + "- **Code transformation**: `NodeTransformer` for renaming, constant folding, or instrumentation" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_37/03_compile_eval_exec.ipynb b/src/chapter_37/03_compile_eval_exec.ipynb new file mode 100644 index 0000000..512e02e --- /dev/null +++ b/src/chapter_37/03_compile_eval_exec.ipynb @@ -0,0 +1,508 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 37: Compile, Eval, and Exec\n", + "\n", + "This notebook covers Python's built-in functions for dynamic code execution: `compile`, `eval`, and `exec`. You will learn how to compile source code and ASTs into code objects, evaluate expressions with custom namespaces, execute statements dynamically, and inspect code object attributes.\n", + "\n", + "## Key Concepts\n", + "- **`eval`**: Evaluates a single expression and returns the result\n", + "- **`exec`**: Executes arbitrary statements (no return value)\n", + "- **`compile`**: Creates a reusable code object from source or an AST\n", + "- **Code objects**: Low-level compiled bytecode with inspection attributes\n", + "- **`__code__`**: Access a function's code object for introspection" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Evaluating Expressions with `eval`\n", + "\n", + "`eval` takes a string containing a single Python expression, evaluates it, and returns the result. It can optionally accept global and local namespace dictionaries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# Basic eval usage\n", + "result: int = eval(\"2 + 3\") # noqa: S307\n", + "print(f\"2 + 3 = {result}\")\n", + "print(f\"Result is 5: {result == 5}\")\n", + "\n", + "# eval works with any expression\n", + "list_result: list[int] = eval(\"[i ** 2 for i in range(5)]\") # noqa: S307\n", + "print(f\"\\nSquares: {list_result}\")\n", + "\n", + "string_result: str = eval(\"'hello'.upper()\") # noqa: S307\n", + "print(f\"Uppercase: {string_result}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "# eval with a custom namespace\n", + "ns: dict[str, int] = {\"x\": 10, \"y\": 20}\n", + "result: int = eval(\"x + y\", ns) # noqa: S307\n", + "\n", + "print(f\"x + y = {result}\")\n", + "print(f\"Result is 30: {result == 30}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "# eval with separate global and local namespaces\n", + "global_ns: dict[str, int] = {\"base\": 100}\n", + "local_ns: dict[str, int] = {\"offset\": 42}\n", + "\n", + "result: int = eval(\"base + offset\", global_ns, local_ns) # noqa: S307\n", + "print(f\"base + offset = {result}\")\n", + "\n", + "# Using built-in functions through eval\n", + "result2: int = eval(\"max(10, 20, 30)\", {\"__builtins__\": __builtins__}) # noqa: S307\n", + "print(f\"max(10, 20, 30) = {result2}\")\n", + "\n", + "# Restricting builtins for safety\n", + "safe_ns: dict[str, object] = {\"__builtins__\": {}, \"x\": 5}\n", + "result3: int = eval(\"x * 2\", safe_ns) # noqa: S307\n", + "print(f\"x * 2 (restricted) = {result3}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 2: Executing Statements with `exec`\n", + "\n", + "`exec` runs arbitrary Python statements. Unlike `eval`, it does not return a value. Variables created during execution are placed into the provided namespace dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "# exec runs statements and populates the namespace\n", + "ns: dict[str, object] = {}\n", + "exec(\"result = [i**2 for i in range(5)]\", ns) # noqa: S102\n", + "\n", + "print(f\"result: {ns['result']}\")\n", + "print(f\"Expected: {[0, 1, 4, 9, 16]}\")\n", + "print(f\"Match: {ns['result'] == [0, 1, 4, 9, 16]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "# exec can define functions and classes\n", + "ns: dict[str, object] = {}\n", + "exec(\"\"\" # noqa: S102\n", + "def greet(name):\n", + " return f\"Hello, {name}!\"\n", + "\n", + "message = greet(\"World\")\n", + "\"\"\", ns)\n", + "\n", + "print(f\"message: {ns['message']}\")\n", + "print(f\"greet is callable: {callable(ns['greet'])}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "# exec with pre-populated namespace\n", + "ns: dict[str, object] = {\"data\": [3, 1, 4, 1, 5, 9, 2, 6]}\n", + "exec(\"\"\" # noqa: S102\n", + "sorted_data = sorted(data)\n", + "total = sum(data)\n", + "average = total / len(data)\n", + "\"\"\", ns)\n", + "\n", + "print(f\"Sorted: {ns['sorted_data']}\")\n", + "print(f\"Total: {ns['total']}\")\n", + "print(f\"Average: {ns['average']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 3: Compiling Source Code\n", + "\n", + "`compile` creates a code object from a source string or an AST. The code object can then be passed to `eval` or `exec`. The `mode` parameter controls what kind of code is expected:\n", + "- `\"exec\"`: A module (sequence of statements)\n", + "- `\"eval\"`: A single expression\n", + "- `\"single\"`: A single interactive statement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import types\n", + "\n", + "# Compile source code to a code object\n", + "code: types.CodeType = compile(\"x = 42\", \"\", \"exec\")\n", + "\n", + "print(f\"Type: {type(code).__name__}\")\n", + "print(f\"Is CodeType: {isinstance(code, types.CodeType)}\")\n", + "\n", + "# Execute the compiled code\n", + "ns: dict[str, object] = {}\n", + "exec(code, ns) # noqa: S102\n", + "print(f\"\\nx = {ns['x']}\")\n", + "print(f\"x == 42: {ns['x'] == 42}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "# Compile in eval mode for expressions\n", + "expr_code = compile(\"2 ** 10\", \"\", \"eval\")\n", + "result: int = eval(expr_code) # noqa: S307\n", + "print(f\"2 ** 10 = {result}\")\n", + "\n", + "# Compile once, execute multiple times with different namespaces\n", + "formula = compile(\"a * b + c\", \"\", \"eval\")\n", + "\n", + "inputs: list[dict[str, int]] = [\n", + " {\"a\": 2, \"b\": 3, \"c\": 4},\n", + " {\"a\": 5, \"b\": 6, \"c\": 7},\n", + " {\"a\": 10, \"b\": 10, \"c\": 10},\n", + "]\n", + "\n", + "print(\"\\nEvaluating 'a * b + c':\")\n", + "for ns in inputs:\n", + " result = eval(formula, ns) # noqa: S307\n", + " print(f\" a={ns['a']}, b={ns['b']}, c={ns['c']} -> {result}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "import types\n", + "\n", + "# Compile from an AST instead of source text\n", + "tree: ast.Module = ast.parse(\"y = 100 + 200\")\n", + "code: types.CodeType = compile(tree, \"\", \"exec\")\n", + "\n", + "print(f\"Compiled from AST: {isinstance(code, types.CodeType)}\")\n", + "\n", + "ns: dict[str, object] = {}\n", + "exec(code, ns) # noqa: S102\n", + "print(f\"y = {ns['y']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 4: Code Object Attributes\n", + "\n", + "Code objects contain compiled bytecode along with metadata. You can access a function's code object through its `__code__` attribute and inspect properties like the function name, argument count, and variable names." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect a function's code object\n", + "def example(a: int, b: int) -> int:\n", + " \"\"\"Add two numbers.\"\"\"\n", + " return a + b\n", + "\n", + "\n", + "code = example.__code__\n", + "\n", + "print(f\"co_name: {code.co_name}\")\n", + "print(f\"co_argcount: {code.co_argcount}\")\n", + "print(f\"co_varnames: {code.co_varnames}\")\n", + "print(f\"\\n'a' in co_varnames: {'a' in code.co_varnames}\")\n", + "print(f\"'b' in co_varnames: {'b' in code.co_varnames}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "# More code object attributes\n", + "def calculate(x: int, y: int, z: int) -> int:\n", + " \"\"\"Perform a calculation with three values.\"\"\"\n", + " temp: int = x + y\n", + " result: int = temp * z\n", + " return result\n", + "\n", + "\n", + "code = calculate.__code__\n", + "\n", + "print(f\"co_name: {code.co_name}\")\n", + "print(f\"co_argcount: {code.co_argcount}\")\n", + "print(f\"co_varnames: {code.co_varnames}\")\n", + "print(f\"co_filename: {code.co_filename}\")\n", + "print(f\"co_nlocals: {code.co_nlocals}\")\n", + "print(f\"co_consts: {code.co_consts}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "# Compare code objects from different functions\n", + "def add(a: int, b: int) -> int:\n", + " return a + b\n", + "\n", + "\n", + "def greet(name: str, greeting: str, punctuation: str) -> str:\n", + " message: str = greeting + \", \" + name + punctuation\n", + " return message\n", + "\n", + "\n", + "functions = [add, greet]\n", + "for func in functions:\n", + " code = func.__code__\n", + " print(f\"{code.co_name}:\")\n", + " print(f\" Arguments: {code.co_argcount}\")\n", + " print(f\" Local vars: {code.co_varnames}\")\n", + " print(f\" Num locals: {code.co_nlocals}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 5: Combining AST, Compile, and Exec\n", + "\n", + "A common workflow is to parse source into an AST, transform it, compile the modified AST, and execute it. This enables powerful metaprogramming patterns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "\n", + "class AddPrintTracing(ast.NodeTransformer):\n", + " \"\"\"Wraps each assignment with a print statement showing the value.\"\"\"\n", + "\n", + " def visit_Assign(self, node: ast.Assign) -> list[ast.stmt]:\n", + " # Keep the original assignment\n", + " # Add a print call after it\n", + " target = node.targets[0]\n", + " if isinstance(target, ast.Name):\n", + " print_call: ast.Expr = ast.Expr(\n", + " value=ast.Call(\n", + " func=ast.Name(id=\"print\", ctx=ast.Load()),\n", + " args=[\n", + " ast.Constant(value=f\" {target.id} =\"),\n", + " ast.Name(id=target.id, ctx=ast.Load()),\n", + " ],\n", + " keywords=[],\n", + " )\n", + " )\n", + " return [node, print_call]\n", + " return [node]\n", + "\n", + "\n", + "source: str = \"\"\"\n", + "x = 5\n", + "y = 10\n", + "z = x + y\n", + "\"\"\"\n", + "\n", + "tree: ast.Module = ast.parse(source)\n", + "traced: ast.Module = AddPrintTracing().visit(tree)\n", + "ast.fix_missing_locations(traced)\n", + "\n", + "code = compile(traced, \"\", \"exec\")\n", + "print(\"Running traced code:\")\n", + "ns: dict[str, object] = {}\n", + "exec(code, ns) # noqa: S102" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "import types\n", + "\n", + "# Full pipeline: parse -> analyze -> transform -> compile -> execute\n", + "source: str = \"price = 10\"\n", + "\n", + "# Step 1: Parse\n", + "tree: ast.Module = ast.parse(source)\n", + "print(f\"Step 1 - Parsed: {ast.dump(tree)}\")\n", + "\n", + "# Step 2: Analyze\n", + "node_count: int = sum(1 for _ in ast.walk(tree))\n", + "print(f\"Step 2 - Node count: {node_count}\")\n", + "\n", + "\n", + "# Step 3: Transform (triple all constants)\n", + "class TripleConstants(ast.NodeTransformer):\n", + " def visit_Constant(self, node: ast.Constant) -> ast.Constant:\n", + " if isinstance(node.value, int):\n", + " return ast.Constant(value=node.value * 3)\n", + " return node\n", + "\n", + "\n", + "new_tree: ast.Module = TripleConstants().visit(tree)\n", + "ast.fix_missing_locations(new_tree)\n", + "print(f\"Step 3 - Transformed: {ast.dump(new_tree)}\")\n", + "\n", + "# Step 4: Compile\n", + "code: types.CodeType = compile(new_tree, \"\", \"exec\")\n", + "print(f\"Step 4 - Compiled: {isinstance(code, types.CodeType)}\")\n", + "\n", + "# Step 5: Execute\n", + "ns: dict[str, object] = {}\n", + "exec(code, ns) # noqa: S102\n", + "print(f\"Step 5 - price = {ns['price']} (original was 10, tripled to 30)\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 6: Security Considerations\n", + "\n", + "Both `eval` and `exec` execute arbitrary code, which makes them dangerous with untrusted input. Use `ast.literal_eval` for safe parsing of data, and always restrict namespaces when dynamic evaluation is necessary." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "\n", + "# Safe approach: use ast.literal_eval for data parsing\n", + "safe_inputs: list[str] = [\"42\", \"[1, 2, 3]\", \"{'key': 'value'}\"]\n", + "\n", + "print(\"Safe parsing with ast.literal_eval:\")\n", + "for inp in safe_inputs:\n", + " result = ast.literal_eval(inp)\n", + " print(f\" {inp!r} -> {result} ({type(result).__name__})\")\n", + "\n", + "# Restricted eval: empty builtins prevents access to dangerous functions\n", + "print(\"\\nRestricted eval:\")\n", + "restricted_ns: dict[str, object] = {\"__builtins__\": {}, \"x\": 5, \"y\": 10}\n", + "result: int = eval(\"x + y\", restricted_ns) # noqa: S307\n", + "print(f\" x + y = {result} (with restricted builtins)\")" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### eval\n", + "- **`eval(expression, globals, locals)`**: Evaluates a single expression and returns the result\n", + "- Accepts an optional namespace for variable resolution\n", + "- Can evaluate pre-compiled code objects\n", + "\n", + "### exec\n", + "- **`exec(code, globals, locals)`**: Executes statements (assignments, definitions, loops)\n", + "- Does not return a value; results are stored in the namespace dict\n", + "- Can execute strings, code objects, or compiled ASTs\n", + "\n", + "### compile\n", + "- **`compile(source, filename, mode)`**: Creates a reusable code object\n", + "- Modes: `\"exec\"` for statements, `\"eval\"` for expressions, `\"single\"` for interactive\n", + "- Accepts source strings or `ast.Module` / `ast.Expression` trees\n", + "\n", + "### Code Objects\n", + "- **`function.__code__`**: Access a function's compiled code object\n", + "- **`co_name`**: The function name\n", + "- **`co_argcount`**: Number of positional arguments\n", + "- **`co_varnames`**: Tuple of local variable names (arguments first)\n", + "- **`co_nlocals`**: Total number of local variables\n", + "- **`co_consts`**: Tuple of constants used in the function\n", + "\n", + "### Security\n", + "- Never use `eval` or `exec` with untrusted input\n", + "- Use `ast.literal_eval` for safe parsing of literal data\n", + "- Restrict `__builtins__` in namespaces to limit available functions" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_37/README.md b/src/chapter_37/README.md new file mode 100644 index 0000000..f3b3b9b --- /dev/null +++ b/src/chapter_37/README.md @@ -0,0 +1,14 @@ +# Chapter 37: Abstract Syntax Trees + +## Topics Covered + +- ast.parse, ast.dump, AST node types +- ast.literal_eval for safe evaluation +- NodeVisitor and NodeTransformer for code analysis and transformation +- compile(), eval(), exec(), and code objects + +## Notebooks + +1. **01_ast_basics.ipynb** - Parsing, dumping, node types, literal_eval +2. **02_node_visitors.ipynb** - NodeVisitor, NodeTransformer, code analysis +3. **03_compile_eval_exec.ipynb** - compile, eval, exec, code objects diff --git a/src/chapter_37/__init__.py b/src/chapter_37/__init__.py new file mode 100644 index 0000000..968e5f9 --- /dev/null +++ b/src/chapter_37/__init__.py @@ -0,0 +1 @@ +"""Chapter 37: Abstract Syntax Trees.""" diff --git a/src/chapter_38/01_reference_counting.ipynb b/src/chapter_38/01_reference_counting.ipynb new file mode 100644 index 0000000..777a898 --- /dev/null +++ b/src/chapter_38/01_reference_counting.ipynb @@ -0,0 +1,418 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 38: Reference Counting\n", + "\n", + "This notebook covers Python's primary memory management mechanism: reference counting. Every object in CPython maintains a count of how many references point to it. When that count drops to zero, the object is immediately deallocated.\n", + "\n", + "## Key Concepts\n", + "- **`sys.getrefcount()`**: Inspect the current reference count of an object\n", + "- **Multiple references**: Assigning to new names increases the count\n", + "- **`del` statement**: Removes a reference, decreasing the count\n", + "- **Container references**: Lists, dicts, and other containers hold references to their elements\n", + "- **Immediate deallocation**: Objects are freed as soon as their reference count hits zero" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: How Reference Counting Works\n", + "\n", + "In CPython, every object has an internal reference count. Each time a new name, container slot, or function argument refers to an object, the count goes up. When a reference is removed, the count goes down. At zero, the memory is reclaimed immediately." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Create a fresh object and check its reference count\n", + "a: list[int] = []\n", + "count: int = sys.getrefcount(a)\n", + "\n", + "# getrefcount itself creates a temporary reference (its argument),\n", + "# so the reported count is always at least 2: the variable + the call\n", + "print(f\"Reference count of a: {count}\")\n", + "print(f\"Count is >= 2: {count >= 2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Small integers are cached by CPython, so they have high ref counts\n", + "small_int: int = 42\n", + "print(f\"Ref count of 42: {sys.getrefcount(small_int)}\")\n", + "\n", + "# A freshly created large integer has a low ref count\n", + "big_int: int = 123_456_789_000\n", + "print(f\"Ref count of 123_456_789_000: {sys.getrefcount(big_int)}\")\n", + "\n", + "# None is referenced everywhere in the interpreter\n", + "print(f\"Ref count of None: {sys.getrefcount(None)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Multiple References Increase the Count\n", + "\n", + "When you assign an existing object to a new variable name, no copy is made. Instead, both names point to the same object and the reference count increases by one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "a: list[int] = []\n", + "base: int = sys.getrefcount(a)\n", + "print(f\"Base ref count (a only): {base}\")\n", + "\n", + "# Assign a second name pointing to the same object\n", + "b = a\n", + "after_b: int = sys.getrefcount(a)\n", + "print(f\"After b = a: {after_b}\")\n", + "print(f\"Increased by 1: {after_b == base + 1}\")\n", + "\n", + "# Assign a third name\n", + "c = a\n", + "after_c: int = sys.getrefcount(a)\n", + "print(f\"After c = a: {after_c}\")\n", + "print(f\"Increased by 2 total: {after_c == base + 2}\")\n", + "\n", + "# All three names refer to the exact same object\n", + "print(f\"\\na is b: {a is b}\")\n", + "print(f\"a is c: {a is c}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: `del` Decreases the Reference Count\n", + "\n", + "The `del` statement removes a name binding. It does **not** directly delete the object -- it simply decreases the reference count by one. If other references remain, the object stays alive." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "a: list[int] = []\n", + "b = a\n", + "base: int = sys.getrefcount(a)\n", + "print(f\"Ref count with a and b: {base}\")\n", + "\n", + "# Remove the reference held by b\n", + "del b\n", + "after_del: int = sys.getrefcount(a)\n", + "print(f\"After del b: {after_del}\")\n", + "print(f\"Decreased by 1: {after_del == base - 1}\")\n", + "\n", + "# a still references the object, so it is not deallocated\n", + "print(f\"\\na still works: {a}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Demonstrate multiple del operations\n", + "a: list[int] = [1, 2, 3]\n", + "b = a\n", + "c = a\n", + "\n", + "print(f\"Initial ref count: {sys.getrefcount(a)}\")\n", + "\n", + "del c\n", + "print(f\"After del c: {sys.getrefcount(a)}\")\n", + "\n", + "del b\n", + "print(f\"After del b: {sys.getrefcount(a)}\")\n", + "\n", + "# At this point only 'a' holds a reference\n", + "# If we del a, the object would be immediately freed\n", + "print(f\"\\nObject still alive via a: {a}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 4: Container References\n", + "\n", + "When an object is placed inside a container (list, dict, set, tuple, etc.), the container holds a reference to it. This increases the object's reference count." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "obj: list[int] = [10, 20, 30]\n", + "base: int = sys.getrefcount(obj)\n", + "print(f\"Base ref count: {base}\")\n", + "\n", + "# Placing obj in a list increases its ref count\n", + "container: list[list[int]] = [obj]\n", + "in_list: int = sys.getrefcount(obj)\n", + "print(f\"After adding to list: {in_list}\")\n", + "print(f\"Increased by 1: {in_list == base + 1}\")\n", + "\n", + "# Adding to a second container increases it again\n", + "another: list[list[int]] = [obj]\n", + "in_two: int = sys.getrefcount(obj)\n", + "print(f\"After adding to 2nd list: {in_two}\")\n", + "print(f\"Increased by 2 total: {in_two == base + 2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Dictionaries also hold references to values (and keys)\n", + "obj: list[str] = [\"hello\"]\n", + "base: int = sys.getrefcount(obj)\n", + "\n", + "data: dict[str, list[str]] = {\"key\": obj}\n", + "in_dict: int = sys.getrefcount(obj)\n", + "print(f\"Base ref count: {base}\")\n", + "print(f\"After adding to dict: {in_dict}\")\n", + "print(f\"Increased by 1: {in_dict == base + 1}\")\n", + "\n", + "# Removing from the container decreases it\n", + "del data[\"key\"]\n", + "after_remove: int = sys.getrefcount(obj)\n", + "print(f\"After removing key: {after_remove}\")\n", + "print(f\"Back to base: {after_remove == base}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 5: Function Arguments and Reference Counts\n", + "\n", + "When you pass an object as a function argument, the parameter name creates an additional reference for the duration of the call. This is why `sys.getrefcount()` always reports at least 2." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "def show_refcount(obj: object) -> None:\n", + " \"\"\"Print the reference count inside a function call.\"\"\"\n", + " # obj is an additional reference (the parameter)\n", + " # sys.getrefcount adds one more (its own argument)\n", + " count: int = sys.getrefcount(obj)\n", + " print(f\" Inside function: {count}\")\n", + "\n", + "\n", + "a: list[int] = [1, 2, 3]\n", + "print(f\"Before function call: {sys.getrefcount(a)}\")\n", + "show_refcount(a)\n", + "print(f\"After function call: {sys.getrefcount(a)}\")\n", + "\n", + "# The count goes up inside the function (parameter + getrefcount arg)\n", + "# and comes back down after the function returns" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 6: Practical Patterns\n", + "\n", + "Understanding reference counting helps you reason about object lifetimes and avoid keeping objects alive unintentionally." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "def track_references(label: str, obj: object) -> int:\n", + " \"\"\"Print and return the current reference count for an object.\"\"\"\n", + " # Subtract 1 for the getrefcount argument itself\n", + " # Subtract 1 for the function parameter\n", + " estimated: int = sys.getrefcount(obj) - 2\n", + " print(f\"{label}: ~{estimated} external references\")\n", + " return estimated\n", + "\n", + "\n", + "data: list[int] = [1, 2, 3]\n", + "\n", + "track_references(\"After creation\", data)\n", + "\n", + "alias = data\n", + "track_references(\"After alias\", data)\n", + "\n", + "cache: dict[str, list[int]] = {\"numbers\": data}\n", + "track_references(\"After cache\", data)\n", + "\n", + "del alias\n", + "track_references(\"After del alias\", data)\n", + "\n", + "del cache\n", + "track_references(\"After del cache\", data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Demonstrate that reassigning a variable releases the old reference\n", + "a: list[int] = [100, 200, 300]\n", + "obj_id: int = id(a)\n", + "print(f\"Original id: {obj_id}\")\n", + "print(f\"Ref count: {sys.getrefcount(a)}\")\n", + "\n", + "# Reassigning 'a' releases the reference to the old list\n", + "# If no other references exist, the old list is freed immediately\n", + "a = [400, 500, 600]\n", + "new_id: int = id(a)\n", + "print(f\"\\nNew id: {new_id}\")\n", + "print(f\"Same object? {obj_id == new_id}\")\n", + "print(f\"Ref count: {sys.getrefcount(a)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 7: Limitations of Reference Counting\n", + "\n", + "Reference counting handles most deallocation, but it cannot detect **circular references** -- objects that reference each other. This is where Python's garbage collector (covered in the next notebook) steps in." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Create two objects that reference each other\n", + "a: dict[str, object] = {}\n", + "b: dict[str, object] = {}\n", + "a[\"partner\"] = b\n", + "b[\"partner\"] = a\n", + "\n", + "print(f\"Ref count of a: {sys.getrefcount(a)}\")\n", + "print(f\"Ref count of b: {sys.getrefcount(b)}\")\n", + "\n", + "# Even after deleting both names, the objects keep each other alive\n", + "# through their mutual references. Reference counting alone\n", + "# cannot free them -- the garbage collector must intervene.\n", + "print(\"\\nCircular references prevent ref count from reaching zero.\")\n", + "print(\"The gc module handles this case (see next notebook).\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Reference Counting Basics\n", + "- Every CPython object has an internal reference count\n", + "- `sys.getrefcount(obj)` returns the count (includes +1 for its own argument)\n", + "- Objects are deallocated immediately when their reference count reaches zero\n", + "\n", + "### What Increases the Count\n", + "- Assigning to a new variable: `b = a`\n", + "- Placing in a container: `my_list.append(obj)`\n", + "- Passing as a function argument\n", + "\n", + "### What Decreases the Count\n", + "- `del name` removes a name binding\n", + "- Reassigning a variable to a different object\n", + "- Removing from a container\n", + "- A local variable going out of scope when a function returns\n", + "\n", + "### Limitations\n", + "- Cannot detect circular references (objects referencing each other)\n", + "- The `gc` module handles cycle detection (covered in the next notebook)\n", + "- `sys.getrefcount()` is CPython-specific and may not work on other implementations" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_38/02_garbage_collection.ipynb b/src/chapter_38/02_garbage_collection.ipynb new file mode 100644 index 0000000..f75bca7 --- /dev/null +++ b/src/chapter_38/02_garbage_collection.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1000001", + "metadata": {}, + "source": [ + "# Chapter 38: Garbage Collection\n", + "\n", + "This notebook covers Python's garbage collector (`gc` module), which supplements reference counting by detecting and cleaning up circular references. You will also learn about `weakref.finalize` for cleanup callbacks and `gc.freeze` for performance optimization.\n", + "\n", + "## Key Concepts\n", + "- **`gc` module**: Controls the cyclic garbage collector\n", + "- **Circular references**: Objects that reference each other, preventing ref count from reaching zero\n", + "- **Generational collection**: Three generations with increasing collection thresholds\n", + "- **`weakref.finalize`**: Register a callback that runs when an object is collected\n", + "- **`gc.freeze` / `gc.unfreeze`**: Move objects to a permanent generation for performance" + ] + }, + { + "cell_type": "markdown", + "id": "a1000002", + "metadata": {}, + "source": [ + "## Section 1: The `gc` Module Basics\n", + "\n", + "Python's garbage collector is enabled by default and runs automatically. You can inspect its state and trigger collection manually." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000003", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# The garbage collector is enabled by default\n", + "enabled: bool = gc.isenabled()\n", + "print(f\"GC enabled: {enabled}\")\n", + "\n", + "# You can disable and re-enable it\n", + "gc.disable()\n", + "print(f\"After disable: {gc.isenabled()}\")\n", + "\n", + "gc.enable()\n", + "print(f\"After enable: {gc.isenabled()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000004", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# gc.collect() manually triggers garbage collection\n", + "# It returns the number of unreachable objects found and freed\n", + "collected: int = gc.collect()\n", + "print(f\"Objects collected: {collected}\")\n", + "print(f\"Return type is int: {isinstance(collected, int)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000005", + "metadata": {}, + "source": [ + "## Section 2: Circular References\n", + "\n", + "Reference counting cannot free objects that form a cycle. The garbage collector detects these cycles and breaks them. We use `weakref.ref` to verify that the objects were actually collected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000006", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "import weakref\n", + "\n", + "\n", + "class Node:\n", + " \"\"\"A simple node that can reference another node.\"\"\"\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name: str = name\n", + " self.ref: \"Node | None\" = None\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Node({self.name!r})\"\n", + "\n", + "\n", + "# Create a circular reference: a -> b -> a\n", + "a: Node = Node(\"A\")\n", + "b: Node = Node(\"B\")\n", + "a.ref = b\n", + "b.ref = a\n", + "\n", + "# Create weak references to track whether objects are collected\n", + "weak_a: weakref.ref[Node] = weakref.ref(a)\n", + "weak_b: weakref.ref[Node] = weakref.ref(b)\n", + "\n", + "print(f\"Before del -- weak_a alive: {weak_a() is not None}\")\n", + "print(f\"Before del -- weak_b alive: {weak_b() is not None}\")\n", + "\n", + "# Delete the strong references\n", + "del a, b\n", + "\n", + "# Force garbage collection to break the cycle\n", + "collected: int = gc.collect()\n", + "print(f\"\\nObjects collected: {collected}\")\n", + "print(f\"After gc -- weak_a alive: {weak_a() is not None}\")\n", + "print(f\"After gc -- weak_b alive: {weak_b() is not None}\")\n", + "print(f\"\\nBoth nodes were freed: {weak_a() is None and weak_b() is None}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000007", + "metadata": {}, + "source": [ + "## Section 3: The `__del__` Finalizer\n", + "\n", + "The `__del__` method is called when an object is about to be destroyed. However, it has significant drawbacks with circular references and is generally discouraged. Use `weakref.finalize` instead (shown in the next section)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000008", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "\n", + "class Resource:\n", + " \"\"\"A class with a __del__ finalizer.\"\"\"\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name: str = name\n", + " print(f\" Created {self.name}\")\n", + "\n", + " def __del__(self) -> None:\n", + " print(f\" Finalized {self.name}\")\n", + "\n", + "\n", + "print(\"Creating and deleting a Resource:\")\n", + "r: Resource = Resource(\"my_resource\")\n", + "del r\n", + "\n", + "print(\"\\nNote: __del__ runs immediately when ref count hits zero.\")\n", + "print(\"For circular references, __del__ can prevent collection.\")\n", + "print(\"Prefer weakref.finalize for cleanup callbacks.\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000009", + "metadata": {}, + "source": [ + "## Section 4: `weakref.finalize` -- Safe Cleanup Callbacks\n", + "\n", + "`weakref.finalize` registers a callback that runs when the target object is garbage collected. Unlike `__del__`, it does not prevent the garbage collector from cleaning up circular references." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000010", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "import weakref\n", + "\n", + "\n", + "class Resource:\n", + " \"\"\"A resource tracked with weakref.finalize.\"\"\"\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name: str = name\n", + "\n", + "\n", + "cleaned_up: list[str] = []\n", + "\n", + "\n", + "def cleanup_callback(name: str) -> None:\n", + " \"\"\"Called when the Resource is collected.\"\"\"\n", + " cleaned_up.append(name)\n", + " print(f\" Cleanup callback fired for: {name}\")\n", + "\n", + "\n", + "# Create a resource and register a finalizer\n", + "obj: Resource = Resource(\"database_connection\")\n", + "weakref.finalize(obj, cleanup_callback, \"database_connection\")\n", + "\n", + "print(\"Before deletion:\")\n", + "print(f\" cleaned_up: {cleaned_up}\")\n", + "\n", + "# Delete the object -- the callback fires\n", + "del obj\n", + "gc.collect()\n", + "\n", + "print(\"\\nAfter deletion:\")\n", + "print(f\" cleaned_up: {cleaned_up}\")\n", + "print(f\" Callback was called: {len(cleaned_up) == 1}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000011", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "import weakref\n", + "\n", + "\n", + "class Resource:\n", + " pass\n", + "\n", + "\n", + "# finalize with a lambda callback\n", + "events: list[bool] = []\n", + "obj: Resource = Resource()\n", + "weakref.finalize(obj, lambda: events.append(True))\n", + "\n", + "print(f\"Events before del: {events}\")\n", + "\n", + "del obj\n", + "gc.collect()\n", + "\n", + "print(f\"Events after del: {events}\")\n", + "print(f\"Callback ran: {len(events) == 1}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000012", + "metadata": {}, + "source": [ + "## Section 5: Generational Collection and `gc.get_stats()`\n", + "\n", + "Python's garbage collector uses a generational scheme with three generations (0, 1, 2). New objects start in generation 0. Objects that survive a collection cycle are promoted to the next generation. Older generations are collected less frequently." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000013", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# gc.get_stats() returns one dict per generation\n", + "stats: list[dict[str, int]] = gc.get_stats()\n", + "print(f\"Number of generations: {len(stats)}\")\n", + "print()\n", + "\n", + "for i, gen_stats in enumerate(stats):\n", + " print(f\"Generation {i}:\")\n", + " print(f\" collections: {gen_stats['collections']}\")\n", + " print(f\" collected: {gen_stats['collected']}\")\n", + " print(f\" uncollectable: {gen_stats['uncollectable']}\")\n", + " print()\n", + "\n", + "# Verify expected keys\n", + "print(f\"'collections' in stats[0]: {'collections' in stats[0]}\")\n", + "print(f\"'collected' in stats[0]: {'collected' in stats[0]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000014", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# gc.get_threshold() shows the allocation thresholds for each generation\n", + "thresholds: tuple[int, int, int] = gc.get_threshold()\n", + "print(f\"Collection thresholds: {thresholds}\")\n", + "print(f\" Generation 0: collect after {thresholds[0]} new allocations\")\n", + "print(f\" Generation 1: collect after {thresholds[1]} gen-0 collections\")\n", + "print(f\" Generation 2: collect after {thresholds[2]} gen-1 collections\")\n", + "\n", + "# You can collect a specific generation\n", + "collected_gen0: int = gc.collect(generation=0)\n", + "print(f\"\\nGen-0 collection freed: {collected_gen0} objects\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000015", + "metadata": {}, + "source": [ + "## Section 6: `gc.freeze` and `gc.unfreeze`\n", + "\n", + "`gc.freeze()` moves all currently tracked objects into a permanent generation that is never scanned by the collector. This is useful in long-running applications (like web servers) where startup objects will never become garbage -- skipping them speeds up collection cycles." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000016", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# Freeze all currently tracked objects\n", + "gc.freeze()\n", + "frozen_count: int = gc.get_freeze_count()\n", + "print(f\"Frozen objects: {frozen_count}\")\n", + "print(f\"Frozen count >= 0: {frozen_count >= 0}\")\n", + "\n", + "# Frozen objects are excluded from future GC scans\n", + "# This reduces pause times in long-running processes\n", + "\n", + "# Unfreeze returns objects to normal generational tracking\n", + "gc.unfreeze()\n", + "after_unfreeze: int = gc.get_freeze_count()\n", + "print(f\"\\nAfter unfreeze: {after_unfreeze}\")\n", + "print(f\"All unfrozen: {after_unfreeze == 0}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000017", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# Practical pattern: freeze after application startup\n", + "# This prevents the GC from repeatedly scanning long-lived objects\n", + "\n", + "# Simulate startup: create some long-lived data\n", + "config: dict[str, str] = {\"db_host\": \"localhost\", \"db_port\": \"5432\"}\n", + "cache: dict[int, str] = {i: f\"value_{i}\" for i in range(100)}\n", + "\n", + "# Freeze everything created so far\n", + "gc.collect() # Clean up any garbage first\n", + "gc.freeze()\n", + "frozen: int = gc.get_freeze_count()\n", + "print(f\"Frozen {frozen} objects after startup\")\n", + "\n", + "# New objects created during runtime are still tracked normally\n", + "runtime_data: list[int] = [1, 2, 3]\n", + "\n", + "# Unfreeze when done (for notebook cleanup)\n", + "gc.unfreeze()\n", + "print(f\"Unfrozen. Freeze count: {gc.get_freeze_count()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000018", + "metadata": {}, + "source": [ + "## Section 7: Weak References and the GC\n", + "\n", + "Weak references (`weakref.ref`) allow you to reference an object without preventing it from being garbage collected. They are essential for caches and observer patterns where you do not want to keep objects alive artificially." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1000019", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "import weakref\n", + "\n", + "\n", + "class CacheItem:\n", + " \"\"\"An item that can be weakly referenced.\"\"\"\n", + "\n", + " def __init__(self, value: str) -> None:\n", + " self.value: str = value\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"CacheItem({self.value!r})\"\n", + "\n", + "\n", + "# Create an object and a weak reference to it\n", + "item: CacheItem = CacheItem(\"important_data\")\n", + "weak_item: weakref.ref[CacheItem] = weakref.ref(item)\n", + "\n", + "# The weak reference can be dereferenced to get the object\n", + "print(f\"Weak ref alive: {weak_item() is not None}\")\n", + "print(f\"Value: {weak_item()}\")\n", + "\n", + "# Delete the strong reference\n", + "del item\n", + "gc.collect()\n", + "\n", + "# Now the weak reference returns None\n", + "print(f\"\\nAfter del:\")\n", + "print(f\"Weak ref alive: {weak_item() is not None}\")\n", + "print(f\"Value: {weak_item()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a1000020", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `gc` Module Essentials\n", + "- **`gc.isenabled()`**: Check if the garbage collector is active (default: `True`)\n", + "- **`gc.collect()`**: Manually trigger collection; returns the count of freed objects\n", + "- **`gc.get_stats()`**: Returns a list of 3 dicts (one per generation) with `collections`, `collected`, and `uncollectable` keys\n", + "- **`gc.get_threshold()`**: Shows allocation thresholds that trigger automatic collection\n", + "\n", + "### Circular References\n", + "- Reference counting cannot break cycles (A references B, B references A)\n", + "- The garbage collector detects and frees cycles automatically\n", + "- Use `weakref.ref` to verify that objects have been collected\n", + "\n", + "### Cleanup Callbacks\n", + "- **`__del__`**: Called on object destruction, but can prevent cycle collection\n", + "- **`weakref.finalize`**: Preferred alternative; does not interfere with the GC\n", + "\n", + "### Performance: `gc.freeze` / `gc.unfreeze`\n", + "- `gc.freeze()` moves tracked objects to a permanent generation (never scanned)\n", + "- `gc.get_freeze_count()` reports how many objects are frozen\n", + "- `gc.unfreeze()` returns frozen objects to normal generational tracking\n", + "- Useful after application startup to reduce GC pause times" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_38/03_memory_profiling.ipynb b/src/chapter_38/03_memory_profiling.ipynb new file mode 100644 index 0000000..c9b6282 --- /dev/null +++ b/src/chapter_38/03_memory_profiling.ipynb @@ -0,0 +1,598 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b1000001", + "metadata": {}, + "source": [ + "# Chapter 38: Memory Profiling\n", + "\n", + "This notebook covers tools for measuring and optimizing memory usage in Python. You will learn how to inspect object sizes with `sys.getsizeof`, reduce per-instance memory with `__slots__`, and trace allocations with `tracemalloc`.\n", + "\n", + "## Key Concepts\n", + "- **`sys.getsizeof()`**: Returns the memory size of an object in bytes\n", + "- **`__slots__`**: Eliminates the per-instance `__dict__`, saving memory\n", + "- **`tracemalloc`**: Traces memory allocations to find where memory is used\n", + "- **`tracemalloc.take_snapshot()`**: Captures a snapshot of all current allocations\n", + "- **`tracemalloc.get_traced_memory()`**: Reports current and peak memory usage" + ] + }, + { + "cell_type": "markdown", + "id": "b1000002", + "metadata": {}, + "source": [ + "## Section 1: `sys.getsizeof` -- Measuring Object Size\n", + "\n", + "`sys.getsizeof()` returns the size of an object in bytes. This is the **shallow** size -- it does not include the size of objects that the target references." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000003", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Basic types and their sizes\n", + "print(\"Sizes of basic types (in bytes):\")\n", + "print(f\" int(0): {sys.getsizeof(0)}\")\n", + "print(f\" int(1): {sys.getsizeof(1)}\")\n", + "print(f\" int(2**30): {sys.getsizeof(2**30)}\")\n", + "print(f\" float(3.14): {sys.getsizeof(3.14)}\")\n", + "print(f\" bool(True): {sys.getsizeof(True)}\")\n", + "print(f\" None: {sys.getsizeof(None)}\")\n", + "\n", + "# All sizes are positive\n", + "print(f\"\\ngetsizeof(0) > 0: {sys.getsizeof(0) > 0}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000004", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Container sizes grow with their contents\n", + "empty_list: list[int] = []\n", + "small_list: list[int] = [1, 2, 3, 4, 5]\n", + "\n", + "print(\"List sizes:\")\n", + "print(f\" []: {sys.getsizeof(empty_list)}\")\n", + "print(f\" [1,2,3,4,5]: {sys.getsizeof(small_list)}\")\n", + "print(f\" Empty < filled: {sys.getsizeof(empty_list) < sys.getsizeof(small_list)}\")\n", + "\n", + "# String sizes grow with length\n", + "empty_str: str = \"\"\n", + "hello_str: str = \"hello world\"\n", + "\n", + "print(f\"\\nString sizes:\")\n", + "print(f\" '': {sys.getsizeof(empty_str)}\")\n", + "print(f\" 'hello world': {sys.getsizeof(hello_str)}\")\n", + "print(f\" Empty < filled: {sys.getsizeof(empty_str) < sys.getsizeof(hello_str)}\")\n", + "\n", + "# Tuple sizes\n", + "empty_tuple: tuple[()] = ()\n", + "small_tuple: tuple[int, int, int] = (1, 2, 3)\n", + "\n", + "print(f\"\\nTuple sizes:\")\n", + "print(f\" (): {sys.getsizeof(empty_tuple)}\")\n", + "print(f\" (1, 2, 3): {sys.getsizeof(small_tuple)}\")\n", + "print(f\" Empty < filled: {sys.getsizeof(empty_tuple) < sys.getsizeof(small_tuple)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1000005", + "metadata": {}, + "source": [ + "## Section 2: Comparing Container Type Sizes\n", + "\n", + "Different container types have different memory overhead. Tuples are lighter than lists, and sets have hash-table overhead similar to dicts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000006", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Compare base sizes of empty containers\n", + "containers: dict[str, object] = {\n", + " \"list()\": [],\n", + " \"tuple()\": (),\n", + " \"dict()\": {},\n", + " \"set()\": set(),\n", + " \"frozenset()\": frozenset(),\n", + " \"bytearray()\": bytearray(),\n", + "}\n", + "\n", + "print(\"Empty container sizes (bytes):\")\n", + "for name, obj in containers.items():\n", + " size: int = sys.getsizeof(obj)\n", + " print(f\" {name:15s} {size:>4}\")\n", + "\n", + "print(f\"\\nTuples are lighter than lists: {sys.getsizeof(()) < sys.getsizeof([])}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1000007", + "metadata": {}, + "source": [ + "## Section 3: `__slots__` -- Reducing Per-Instance Memory\n", + "\n", + "By default, each Python instance stores its attributes in a `__dict__` dictionary. Defining `__slots__` replaces the dict with a fixed-size structure, significantly reducing memory for classes with many instances." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000008", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "class Regular:\n", + " \"\"\"A regular class with a __dict__.\"\"\"\n", + "\n", + " def __init__(self, x: int) -> None:\n", + " self.x: int = x\n", + "\n", + "\n", + "class Slotted:\n", + " \"\"\"A class using __slots__ instead of __dict__.\"\"\"\n", + "\n", + " __slots__ = (\"x\",)\n", + "\n", + " def __init__(self, x: int) -> None:\n", + " self.x: int = x\n", + "\n", + "\n", + "regular: Regular = Regular(1)\n", + "slotted: Slotted = Slotted(1)\n", + "\n", + "regular_size: int = sys.getsizeof(regular)\n", + "slotted_size: int = sys.getsizeof(slotted)\n", + "\n", + "print(f\"Regular instance size: {regular_size} bytes\")\n", + "print(f\"Slotted instance size: {slotted_size} bytes\")\n", + "print(f\"Slotted is smaller: {slotted_size < regular_size}\")\n", + "print(f\"Savings: {regular_size - slotted_size} bytes per instance\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000009", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "class RegularPoint:\n", + " def __init__(self, x: float, y: float, z: float) -> None:\n", + " self.x: float = x\n", + " self.y: float = y\n", + " self.z: float = z\n", + "\n", + "\n", + "class SlottedPoint:\n", + " __slots__ = (\"x\", \"y\", \"z\")\n", + "\n", + " def __init__(self, x: float, y: float, z: float) -> None:\n", + " self.x: float = x\n", + " self.y: float = y\n", + " self.z: float = z\n", + "\n", + "\n", + "# Regular instances have a __dict__\n", + "rp: RegularPoint = RegularPoint(1.0, 2.0, 3.0)\n", + "print(f\"Regular has __dict__: {hasattr(rp, '__dict__')}\")\n", + "print(f\"Regular __dict__: {rp.__dict__}\")\n", + "print(f\"Regular __dict__ size: {sys.getsizeof(rp.__dict__)} bytes\")\n", + "\n", + "# Slotted instances do not\n", + "sp: SlottedPoint = SlottedPoint(1.0, 2.0, 3.0)\n", + "print(f\"\\nSlotted has __dict__: {hasattr(sp, '__dict__')}\")\n", + "print(f\"Slotted has __slots__: {hasattr(sp, '__slots__')}\")\n", + "\n", + "# Memory comparison\n", + "print(f\"\\nRegular size: {sys.getsizeof(rp)} bytes\")\n", + "print(f\"Slotted size: {sys.getsizeof(sp)} bytes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000010", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "class RegularItem:\n", + " def __init__(self, val: int) -> None:\n", + " self.val: int = val\n", + "\n", + "\n", + "class SlottedItem:\n", + " __slots__ = (\"val\",)\n", + "\n", + " def __init__(self, val: int) -> None:\n", + " self.val: int = val\n", + "\n", + "\n", + "# At scale, the savings add up significantly\n", + "n: int = 10_000\n", + "\n", + "regular_items: list[RegularItem] = [RegularItem(i) for i in range(n)]\n", + "slotted_items: list[SlottedItem] = [SlottedItem(i) for i in range(n)]\n", + "\n", + "# Estimate total memory (instance size * count)\n", + "regular_per: int = sys.getsizeof(regular_items[0])\n", + "slotted_per: int = sys.getsizeof(slotted_items[0])\n", + "\n", + "regular_total: int = regular_per * n\n", + "slotted_total: int = slotted_per * n\n", + "\n", + "print(f\"Per-instance: Regular={regular_per}B, Slotted={slotted_per}B\")\n", + "print(f\"Total for {n:,} instances:\")\n", + "print(f\" Regular: {regular_total:>10,} bytes ({regular_total / 1024:.1f} KB)\")\n", + "print(f\" Slotted: {slotted_total:>10,} bytes ({slotted_total / 1024:.1f} KB)\")\n", + "print(f\" Savings: {regular_total - slotted_total:>10,} bytes ({(regular_total - slotted_total) / 1024:.1f} KB)\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1000011", + "metadata": {}, + "source": [ + "## Section 4: `tracemalloc` -- Tracing Memory Allocations\n", + "\n", + "The `tracemalloc` module tracks memory allocations made by Python. It lets you take snapshots, compare them, and find the source of memory usage down to the file and line number." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000012", + "metadata": {}, + "outputs": [], + "source": [ + "import tracemalloc\n", + "\n", + "# Start tracing memory allocations\n", + "tracemalloc.start()\n", + "\n", + "# Allocate some memory\n", + "data: list[int] = [i for i in range(1000)]\n", + "text: str = \"hello \" * 1000\n", + "\n", + "# Take a snapshot of current allocations\n", + "snapshot: tracemalloc.Snapshot = tracemalloc.take_snapshot()\n", + "\n", + "# Get statistics grouped by line number\n", + "stats: list[tracemalloc.StatisticDiff] = snapshot.statistics(\"lineno\")\n", + "print(f\"Number of allocation sites: {len(stats)}\")\n", + "print(f\"Has entries: {len(stats) > 0}\")\n", + "\n", + "# Show the top 5 memory consumers\n", + "print(\"\\nTop 5 allocation sites:\")\n", + "for stat in stats[:5]:\n", + " print(f\" {stat}\")\n", + "\n", + "# Stop tracing\n", + "tracemalloc.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000013", + "metadata": {}, + "outputs": [], + "source": [ + "import tracemalloc\n", + "\n", + "# tracemalloc can also group by filename\n", + "tracemalloc.start()\n", + "\n", + "numbers: list[float] = [float(i) for i in range(5000)]\n", + "\n", + "snapshot: tracemalloc.Snapshot = tracemalloc.take_snapshot()\n", + "\n", + "# Group by filename instead of line number\n", + "stats_by_file: list[tracemalloc.StatisticDiff] = snapshot.statistics(\"filename\")\n", + "print(\"Top 3 allocations by filename:\")\n", + "for stat in stats_by_file[:3]:\n", + " print(f\" {stat}\")\n", + "\n", + "tracemalloc.stop()" + ] + }, + { + "cell_type": "markdown", + "id": "b1000014", + "metadata": {}, + "source": [ + "## Section 5: `get_traced_memory` -- Current and Peak Usage\n", + "\n", + "`tracemalloc.get_traced_memory()` returns a tuple of `(current, peak)` bytes. This is useful for monitoring memory usage over time or finding the high-water mark of a function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000015", + "metadata": {}, + "outputs": [], + "source": [ + "import tracemalloc\n", + "\n", + "tracemalloc.start()\n", + "\n", + "# Check initial memory\n", + "current_before: int\n", + "peak_before: int\n", + "current_before, peak_before = tracemalloc.get_traced_memory()\n", + "print(f\"Before allocation:\")\n", + "print(f\" Current: {current_before:>10,} bytes\")\n", + "print(f\" Peak: {peak_before:>10,} bytes\")\n", + "\n", + "# Allocate a large data structure\n", + "big_data: list[int] = list(range(100_000))\n", + "\n", + "current_after: int\n", + "peak_after: int\n", + "current_after, peak_after = tracemalloc.get_traced_memory()\n", + "print(f\"\\nAfter allocation:\")\n", + "print(f\" Current: {current_after:>10,} bytes\")\n", + "print(f\" Peak: {peak_after:>10,} bytes\")\n", + "\n", + "# Verify invariants\n", + "print(f\"\\ncurrent >= 0: {current_after >= 0}\")\n", + "print(f\"peak >= 0: {peak_after >= 0}\")\n", + "print(f\"peak >= current: {peak_after >= current_after}\")\n", + "\n", + "tracemalloc.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000016", + "metadata": {}, + "outputs": [], + "source": [ + "import tracemalloc\n", + "\n", + "\n", + "def measure_peak_memory(func: object) -> tuple[int, int]:\n", + " \"\"\"Measure the peak memory usage of a callable.\"\"\"\n", + " tracemalloc.start()\n", + " func() # type: ignore[operator]\n", + " current: int\n", + " peak: int\n", + " current, peak = tracemalloc.get_traced_memory()\n", + " tracemalloc.stop()\n", + " return current, peak\n", + "\n", + "\n", + "def create_list() -> list[int]:\n", + " \"\"\"Create a list of integers.\"\"\"\n", + " return list(range(50_000))\n", + "\n", + "\n", + "def create_generator_sum() -> int:\n", + " \"\"\"Sum integers using a generator (lower peak memory).\"\"\"\n", + " return sum(i for i in range(50_000))\n", + "\n", + "\n", + "list_current, list_peak = measure_peak_memory(create_list)\n", + "gen_current, gen_peak = measure_peak_memory(create_generator_sum)\n", + "\n", + "print(\"List approach:\")\n", + "print(f\" Current: {list_current:>10,} bytes\")\n", + "print(f\" Peak: {list_peak:>10,} bytes\")\n", + "\n", + "print(\"\\nGenerator approach:\")\n", + "print(f\" Current: {gen_current:>10,} bytes\")\n", + "print(f\" Peak: {gen_peak:>10,} bytes\")\n", + "\n", + "print(f\"\\nGenerator uses less peak memory: {gen_peak < list_peak}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1000017", + "metadata": {}, + "source": [ + "## Section 6: Comparing Snapshots\n", + "\n", + "You can take two snapshots at different points and compare them to see what was allocated in between. This is invaluable for detecting memory leaks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000018", + "metadata": {}, + "outputs": [], + "source": [ + "import tracemalloc\n", + "\n", + "tracemalloc.start()\n", + "\n", + "# Take a baseline snapshot\n", + "snapshot1: tracemalloc.Snapshot = tracemalloc.take_snapshot()\n", + "\n", + "# Allocate some memory\n", + "new_data: list[str] = [f\"item_{i}\" for i in range(5000)]\n", + "more_data: dict[int, str] = {i: f\"val_{i}\" for i in range(2000)}\n", + "\n", + "# Take a second snapshot\n", + "snapshot2: tracemalloc.Snapshot = tracemalloc.take_snapshot()\n", + "\n", + "# Compare the two snapshots\n", + "diff_stats: list[tracemalloc.StatisticDiff] = snapshot2.compare_to(snapshot1, \"lineno\")\n", + "\n", + "print(\"Top 5 memory increases between snapshots:\")\n", + "for stat in diff_stats[:5]:\n", + " print(f\" {stat}\")\n", + "\n", + "tracemalloc.stop()" + ] + }, + { + "cell_type": "markdown", + "id": "b1000019", + "metadata": {}, + "source": [ + "## Section 7: Memory Optimization Strategies\n", + "\n", + "Practical techniques for reducing memory usage in Python programs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000020", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Strategy 1: Use tuples instead of lists for immutable data\n", + "list_data: list[int] = [1, 2, 3, 4, 5]\n", + "tuple_data: tuple[int, ...] = (1, 2, 3, 4, 5)\n", + "\n", + "print(\"Tuples vs lists:\")\n", + "print(f\" list [1..5]: {sys.getsizeof(list_data)} bytes\")\n", + "print(f\" tuple (1..5): {sys.getsizeof(tuple_data)} bytes\")\n", + "print(f\" Tuple smaller: {sys.getsizeof(tuple_data) < sys.getsizeof(list_data)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000021", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Strategy 2: Use generators to avoid materializing large sequences\n", + "materialized: list[int] = [x * 2 for x in range(10_000)]\n", + "lazy: range = range(10_000)\n", + "\n", + "print(\"Materialized list vs range object:\")\n", + "print(f\" list: {sys.getsizeof(materialized):>10,} bytes\")\n", + "print(f\" range: {sys.getsizeof(lazy):>10,} bytes\")\n", + "\n", + "# Strategy 3: Use array module for homogeneous numeric data\n", + "import array\n", + "\n", + "int_list: list[int] = list(range(1000))\n", + "int_array: array.array[int] = array.array(\"l\", range(1000))\n", + "\n", + "print(f\"\\nlist of 1000 ints: {sys.getsizeof(int_list):>10,} bytes\")\n", + "print(f\"array of 1000 ints: {sys.getsizeof(int_array):>10,} bytes\")\n", + "print(f\"array is smaller: {sys.getsizeof(int_array) < sys.getsizeof(int_list)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1000022", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Strategy 4: Use __slots__ for data-heavy classes (recap)\n", + "# Strategy 5: Delete large objects when no longer needed\n", + "\n", + "\n", + "def process_data() -> int:\n", + " \"\"\"Process data and clean up intermediate results.\"\"\"\n", + " # Large intermediate data\n", + " raw: list[int] = list(range(100_000))\n", + " result: int = sum(raw)\n", + "\n", + " # Explicitly free the large list\n", + " del raw\n", + "\n", + " return result\n", + "\n", + "\n", + "total: int = process_data()\n", + "print(f\"Result: {total:,}\")\n", + "print(\"\\nMemory optimization strategies:\")\n", + "print(\" 1. Use tuples for immutable sequences\")\n", + "print(\" 2. Use generators / range for iteration\")\n", + "print(\" 3. Use array.array for homogeneous numeric data\")\n", + "print(\" 4. Use __slots__ on classes with many instances\")\n", + "print(\" 5. del large objects when they are no longer needed\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1000023", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `sys.getsizeof()`\n", + "- Returns the **shallow** size of an object in bytes\n", + "- Does not account for referenced objects (e.g., list elements)\n", + "- Different types have different base sizes; containers grow with their contents\n", + "\n", + "### `__slots__`\n", + "- Replaces the per-instance `__dict__` with a fixed-size structure\n", + "- Reduces memory per instance, significant when creating many objects\n", + "- Instances cannot have attributes not listed in `__slots__`\n", + "\n", + "### `tracemalloc`\n", + "- **`tracemalloc.start()`** / **`tracemalloc.stop()`**: Enable/disable allocation tracing\n", + "- **`tracemalloc.take_snapshot()`**: Capture current allocations for analysis\n", + "- **`snapshot.statistics(\"lineno\")`**: Group allocations by source line\n", + "- **`snapshot.compare_to(other, key)`**: Diff two snapshots to find leaks\n", + "- **`tracemalloc.get_traced_memory()`**: Returns `(current, peak)` bytes\n", + "- `peak >= current` always holds; useful for finding high-water marks\n", + "\n", + "### Optimization Strategies\n", + "- Prefer tuples over lists for immutable data\n", + "- Use generators and `range` instead of materializing large sequences\n", + "- Use `array.array` for large homogeneous numeric collections\n", + "- Apply `__slots__` to classes instantiated many times\n", + "- Explicitly `del` large intermediate objects" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_38/README.md b/src/chapter_38/README.md new file mode 100644 index 0000000..3b54fd4 --- /dev/null +++ b/src/chapter_38/README.md @@ -0,0 +1,15 @@ +# Chapter 38: Memory Management + +## Topics Covered + +- Reference counting with sys.getrefcount +- gc module: collect, get_referrers, callbacks, circular references +- `__del__` finalizer and weakref.finalize +- tracemalloc for memory profiling +- Memory optimization techniques + +## Notebooks + +1. **01_reference_counting.ipynb** - sys.getrefcount, reference cycles, gc module +2. **02_garbage_collection.ipynb** - gc.collect, circular references, `__del__`, finalize +3. **03_memory_profiling.ipynb** - tracemalloc, sys.getsizeof, memory optimization diff --git a/src/chapter_38/__init__.py b/src/chapter_38/__init__.py new file mode 100644 index 0000000..51afe9f --- /dev/null +++ b/src/chapter_38/__init__.py @@ -0,0 +1 @@ +"""Chapter 38: Memory Management.""" diff --git a/src/chapter_39/01_ctypes_basics.ipynb b/src/chapter_39/01_ctypes_basics.ipynb new file mode 100644 index 0000000..fc98a5b --- /dev/null +++ b/src/chapter_39/01_ctypes_basics.ipynb @@ -0,0 +1,443 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c3900001", + "metadata": {}, + "source": [ + "# Chapter 39: ctypes Basics\n", + "\n", + "This notebook covers the fundamentals of Python's `ctypes` module, which provides C-compatible\n", + "data types and allows calling functions in shared libraries. You will learn how to create C-style\n", + "integers, floats, and characters, work with string buffers, use pointers, and inspect type sizes.\n", + "\n", + "## Key Concepts\n", + "- **`c_int`, `c_float`, `c_double`**: C-compatible numeric types\n", + "- **`c_char`, `c_char_p`**: C-compatible character and string types\n", + "- **`create_string_buffer`**: Mutable character buffers for C interop\n", + "- **`pointer` and `POINTER`**: Pointer creation and dereferencing\n", + "- **`sizeof`**: Query the size in bytes of C types and instances" + ] + }, + { + "cell_type": "markdown", + "id": "c3900002", + "metadata": {}, + "source": [ + "## Section 1: C-Compatible Integer Types\n", + "\n", + "The `ctypes` module provides integer types that match C's type system. Each type wraps a\n", + "Python value in a C-compatible representation. The `.value` attribute gets or sets the\n", + "underlying Python value." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900003", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Create a C int (typically 4 bytes / 32 bits)\n", + "i: ctypes.c_int = ctypes.c_int(42)\n", + "print(f\"c_int value: {i.value}\")\n", + "print(f\"Type: {type(i)}\")\n", + "\n", + "# Modify the value through .value\n", + "i.value = 100\n", + "print(f\"Updated value: {i.value}\")\n", + "\n", + "# Other integer types\n", + "short: ctypes.c_short = ctypes.c_short(32000)\n", + "long: ctypes.c_long = ctypes.c_long(1_000_000)\n", + "uint: ctypes.c_uint = ctypes.c_uint(42)\n", + "\n", + "print(f\"\\nc_short: {short.value}\")\n", + "print(f\"c_long: {long.value}\")\n", + "print(f\"c_uint: {uint.value}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900004", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Integer overflow wraps around in C types (just like C)\n", + "max_short: ctypes.c_short = ctypes.c_short(32767)\n", + "print(f\"Max c_short: {max_short.value}\")\n", + "\n", + "# Overflow wraps to negative (signed 16-bit)\n", + "overflowed: ctypes.c_short = ctypes.c_short(32768)\n", + "print(f\"32768 as c_short: {overflowed.value}\")\n", + "\n", + "# Unsigned types do not wrap to negative\n", + "unsigned: ctypes.c_ushort = ctypes.c_ushort(65535)\n", + "print(f\"\\nMax c_ushort: {unsigned.value}\")\n", + "\n", + "wrapped_unsigned: ctypes.c_ushort = ctypes.c_ushort(65536)\n", + "print(f\"65536 as c_ushort: {wrapped_unsigned.value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900005", + "metadata": {}, + "source": [ + "## Section 2: C-Compatible Float Types\n", + "\n", + "The `c_float` type is a single-precision (32-bit) floating-point number, while `c_double`\n", + "is double-precision (64-bit). Note that `c_float` has limited precision compared to\n", + "Python's native `float` (which is equivalent to `c_double`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900006", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# c_float is single precision (32-bit)\n", + "f: ctypes.c_float = ctypes.c_float(3.14)\n", + "print(f\"c_float value: {f.value}\")\n", + "print(f\"c_float precision loss: {f.value} != 3.14 -> {f.value != 3.14}\")\n", + "print(f\"Close enough (abs diff < 0.01): {abs(f.value - 3.14) < 0.01}\")\n", + "\n", + "# c_double is double precision (64-bit) -- matches Python's float\n", + "d: ctypes.c_double = ctypes.c_double(3.14)\n", + "print(f\"\\nc_double value: {d.value}\")\n", + "print(f\"c_double exact match: {d.value == 3.14}\")\n", + "\n", + "# c_longdouble for extended precision (platform-dependent)\n", + "ld: ctypes.c_longdouble = ctypes.c_longdouble(3.14)\n", + "print(f\"\\nc_longdouble value: {ld.value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900007", + "metadata": {}, + "source": [ + "## Section 3: Character and Byte Types\n", + "\n", + "The `c_char` type represents a single C `char` (1 byte). The `c_char_p` type is a pointer\n", + "to a null-terminated byte string, similar to `const char*` in C. The `c_wchar` type\n", + "handles wide (Unicode) characters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900008", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# c_char holds a single byte\n", + "ch: ctypes.c_char = ctypes.c_char(b\"A\")\n", + "print(f\"c_char value: {ch.value}\")\n", + "print(f\"As integer: {ord(ch.value)}\")\n", + "\n", + "# c_byte holds a small signed integer (-128 to 127)\n", + "b: ctypes.c_byte = ctypes.c_byte(65)\n", + "print(f\"\\nc_byte value: {b.value}\")\n", + "print(f\"As char: {chr(b.value)}\")\n", + "\n", + "# c_char_p is a pointer to a byte string\n", + "s: ctypes.c_char_p = ctypes.c_char_p(b\"hello\")\n", + "print(f\"\\nc_char_p value: {s.value}\")\n", + "print(f\"Type: {type(s.value)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900009", + "metadata": {}, + "source": [ + "## Section 4: String Buffers\n", + "\n", + "The `create_string_buffer` function creates a mutable character buffer. This is\n", + "essential when you need to pass a writable buffer to a C function. The buffer\n", + "has a fixed size and stores bytes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900010", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Create a buffer with a fixed size (10 bytes, initialized to zero)\n", + "buf: ctypes.Array[ctypes.c_char] = ctypes.create_string_buffer(10)\n", + "print(f\"Buffer length: {len(buf)}\")\n", + "print(f\"Initial value: {buf.value!r}\")\n", + "print(f\"Raw bytes: {buf.raw!r}\")\n", + "\n", + "# Set the buffer value\n", + "buf.value = b\"hello\"\n", + "print(f\"\\nAfter setting 'hello':\")\n", + "print(f\"Value: {buf.value}\")\n", + "print(f\"Raw: {buf.raw!r}\")\n", + "print(f\"Length still: {len(buf)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900011", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Create a buffer initialized with a byte string\n", + "buf: ctypes.Array[ctypes.c_char] = ctypes.create_string_buffer(b\"world\")\n", + "print(f\"Value: {buf.value}\")\n", + "print(f\"Length: {len(buf)}\") # 6: 5 chars + null terminator\n", + "\n", + "# Create with initial value AND a minimum size\n", + "buf2: ctypes.Array[ctypes.c_char] = ctypes.create_string_buffer(b\"hi\", 20)\n", + "print(f\"\\nValue: {buf2.value}\")\n", + "print(f\"Length: {len(buf2)}\")\n", + "\n", + "# Buffers are mutable -- you can modify individual bytes\n", + "buf2[0] = b\"H\"\n", + "print(f\"After modifying index 0: {buf2.value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900012", + "metadata": {}, + "source": [ + "## Section 5: Pointers\n", + "\n", + "Pointers are fundamental to C interoperability. The `pointer()` function creates a\n", + "pointer to a ctypes instance, and the `POINTER()` function creates a pointer *type*.\n", + "Use `.contents` to dereference a pointer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900013", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Create a c_int and a pointer to it\n", + "i: ctypes.c_int = ctypes.c_int(42)\n", + "p = ctypes.pointer(i)\n", + "\n", + "print(f\"Original value: {i.value}\")\n", + "print(f\"Pointer contents: {p.contents.value}\")\n", + "print(f\"Pointer type: {type(p)}\")\n", + "\n", + "# Modify through the pointer\n", + "p.contents.value = 100\n", + "print(f\"\\nAfter modifying through pointer:\")\n", + "print(f\"Original i.value: {i.value}\")\n", + "print(f\"Pointer contents: {p.contents.value}\")\n", + "print(f\"Both point to same data: {i.value == p.contents.value}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900014", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# POINTER() creates a pointer TYPE (not an instance)\n", + "IntPointer = ctypes.POINTER(ctypes.c_int)\n", + "print(f\"Pointer type: {IntPointer}\")\n", + "\n", + "# Create an instance of the pointer type\n", + "val: ctypes.c_int = ctypes.c_int(99)\n", + "ptr: ctypes.POINTER(ctypes.c_int) = IntPointer(val)\n", + "print(f\"Value via pointer: {ptr.contents.value}\")\n", + "\n", + "# Null pointer (no argument)\n", + "null_ptr = IntPointer()\n", + "print(f\"\\nNull pointer created (accessing contents would raise ValueError)\")\n", + "\n", + "# byref() is an optimization for passing pointers to C functions\n", + "x: ctypes.c_int = ctypes.c_int(77)\n", + "ref = ctypes.byref(x)\n", + "print(f\"byref type: {type(ref)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900015", + "metadata": {}, + "source": [ + "## Section 6: sizeof -- Inspecting Type Sizes\n", + "\n", + "The `ctypes.sizeof()` function returns the size in bytes of a ctypes type or instance.\n", + "This mirrors C's `sizeof` operator and is essential for understanding memory layout." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900016", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# sizeof on types (class-level)\n", + "print(\"Type sizes (in bytes):\")\n", + "print(f\" c_char: {ctypes.sizeof(ctypes.c_char)}\")\n", + "print(f\" c_short: {ctypes.sizeof(ctypes.c_short)}\")\n", + "print(f\" c_int: {ctypes.sizeof(ctypes.c_int)}\")\n", + "print(f\" c_long: {ctypes.sizeof(ctypes.c_long)}\")\n", + "print(f\" c_longlong: {ctypes.sizeof(ctypes.c_longlong)}\")\n", + "print(f\" c_float: {ctypes.sizeof(ctypes.c_float)}\")\n", + "print(f\" c_double: {ctypes.sizeof(ctypes.c_double)}\")\n", + "\n", + "# Verify expected sizes\n", + "print(f\"\\nc_int is 4 bytes: {ctypes.sizeof(ctypes.c_int) == 4}\")\n", + "print(f\"c_double is 8 bytes: {ctypes.sizeof(ctypes.c_double) == 8}\")\n", + "print(f\"c_char is 1 byte: {ctypes.sizeof(ctypes.c_char) == 1}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900017", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# sizeof also works on instances\n", + "i: ctypes.c_int = ctypes.c_int(42)\n", + "d: ctypes.c_double = ctypes.c_double(3.14)\n", + "\n", + "print(f\"sizeof(c_int instance): {ctypes.sizeof(i)}\")\n", + "print(f\"sizeof(c_double instance): {ctypes.sizeof(d)}\")\n", + "\n", + "# sizeof on a string buffer\n", + "buf: ctypes.Array[ctypes.c_char] = ctypes.create_string_buffer(10)\n", + "print(f\"sizeof(10-byte buffer): {ctypes.sizeof(buf)}\")\n", + "\n", + "# sizeof on an array type\n", + "IntArray5 = ctypes.c_int * 5\n", + "print(f\"\\nsizeof(c_int * 5): {ctypes.sizeof(IntArray5)}\")\n", + "print(f\"Expected (5 * 4): {5 * ctypes.sizeof(ctypes.c_int)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900018", + "metadata": {}, + "source": [ + "## Section 7: Comparing ctypes with Python Native Types\n", + "\n", + "Understanding when to use ctypes versus native Python types is important. ctypes\n", + "types are needed specifically for C library interoperability, not for general Python programming." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3900019", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# ctypes values are not the same as Python values\n", + "c_val: ctypes.c_int = ctypes.c_int(42)\n", + "py_val: int = 42\n", + "\n", + "print(f\"ctypes c_int: {c_val}, type: {type(c_val)}\")\n", + "print(f\"Python int: {py_val}, type: {type(py_val)}\")\n", + "print(f\"Values equal: {c_val.value == py_val}\")\n", + "\n", + "# Convert between ctypes and Python\n", + "from_c: int = c_val.value # ctypes -> Python\n", + "to_c: ctypes.c_int = ctypes.c_int(py_val) # Python -> ctypes\n", + "\n", + "print(f\"\\nConverted from ctypes: {from_c} (type: {type(from_c)})\")\n", + "print(f\"Converted to ctypes: {to_c.value} (type: {type(to_c)})\")\n", + "\n", + "# Summary of type mapping\n", + "mappings: list[tuple[str, str]] = [\n", + " (\"c_int\", \"int\"),\n", + " (\"c_float\", \"float (single precision)\"),\n", + " (\"c_double\", \"float (double precision)\"),\n", + " (\"c_char\", \"bytes (1 byte)\"),\n", + " (\"c_char_p\", \"bytes or None\"),\n", + " (\"c_bool\", \"bool\"),\n", + "]\n", + "print(f\"\\n{'ctypes Type':<15} {'Python Type'}\")\n", + "print(\"-\" * 40)\n", + "for c_type, py_type in mappings:\n", + " print(f\"{c_type:<15} {py_type}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3900020", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Fundamental Types\n", + "- **`c_int(value)`**: 4-byte signed integer, access via `.value`\n", + "- **`c_float(value)`**: 4-byte single-precision float (limited precision)\n", + "- **`c_double(value)`**: 8-byte double-precision float (matches Python `float`)\n", + "- **`c_char(byte)`**: Single byte character\n", + "- **`c_char_p(bytes)`**: Pointer to a null-terminated byte string\n", + "\n", + "### String Buffers\n", + "- **`create_string_buffer(size)`**: Mutable buffer of `size` zero bytes\n", + "- **`create_string_buffer(b\"init\")`**: Buffer initialized from bytes (size = len + 1)\n", + "- **`create_string_buffer(b\"init\", size)`**: Initialized buffer with minimum size\n", + "- Access via `.value` (null-terminated) or `.raw` (all bytes)\n", + "\n", + "### Pointers\n", + "- **`pointer(obj)`**: Create a pointer instance to a ctypes object\n", + "- **`POINTER(type)`**: Create a pointer type (class)\n", + "- **`.contents`**: Dereference a pointer to access the pointed-to object\n", + "- **`byref(obj)`**: Lightweight pointer for passing to C functions\n", + "\n", + "### sizeof\n", + "- **`sizeof(type_or_instance)`**: Returns size in bytes\n", + "- Works on both ctypes types and instances\n", + "- `c_char` = 1, `c_int` = 4, `c_double` = 8 bytes (typical)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_39/02_ctypes_advanced.ipynb b/src/chapter_39/02_ctypes_advanced.ipynb new file mode 100644 index 0000000..21b4a48 --- /dev/null +++ b/src/chapter_39/02_ctypes_advanced.ipynb @@ -0,0 +1,599 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c3910001", + "metadata": {}, + "source": [ + "# Chapter 39: ctypes Advanced\n", + "\n", + "This notebook covers advanced `ctypes` features including structures, unions, arrays,\n", + "and function pointer callbacks. These are the building blocks for defining complex C data\n", + "layouts and interfacing with C libraries that use structs, unions, and callback functions.\n", + "\n", + "## Key Concepts\n", + "- **`Structure`**: Define C-compatible structs with `_fields_`\n", + "- **`Union`**: Overlapping fields that share memory\n", + "- **Arrays**: Fixed-size arrays using the `type * count` syntax\n", + "- **`CFUNCTYPE`**: Create C-callable function pointers from Python functions\n", + "- **Nested structures and pointers**: Compose complex C data layouts" + ] + }, + { + "cell_type": "markdown", + "id": "c3910002", + "metadata": {}, + "source": [ + "## Section 1: Defining Structures\n", + "\n", + "A `ctypes.Structure` subclass defines a C-compatible struct. The `_fields_` class\n", + "variable is a list of `(name, type)` tuples that define the struct layout. Fields\n", + "are accessible as attributes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910003", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Point(ctypes.Structure):\n", + " \"\"\"A 2D point with integer coordinates.\"\"\"\n", + " _fields_ = [(\"x\", ctypes.c_int), (\"y\", ctypes.c_int)]\n", + "\n", + "\n", + "# Create by positional arguments\n", + "p1: Point = Point(10, 20)\n", + "print(f\"Point({p1.x}, {p1.y})\")\n", + "\n", + "# Create by keyword arguments\n", + "p2: Point = Point(x=30, y=40)\n", + "print(f\"Point({p2.x}, {p2.y})\")\n", + "\n", + "# Modify fields\n", + "p1.x = 100\n", + "print(f\"\\nModified: Point({p1.x}, {p1.y})\")\n", + "\n", + "# Default values are zero\n", + "p3: Point = Point()\n", + "print(f\"Default: Point({p3.x}, {p3.y})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910004", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Pair(ctypes.Structure):\n", + " \"\"\"A pair of two integers.\"\"\"\n", + " _fields_ = [(\"a\", ctypes.c_int), (\"b\", ctypes.c_int)]\n", + "\n", + "\n", + "# sizeof a structure matches C layout\n", + "pair: Pair = Pair(1, 2)\n", + "print(f\"Pair({pair.a}, {pair.b})\")\n", + "print(f\"sizeof(Pair): {ctypes.sizeof(Pair)} bytes\")\n", + "print(f\"Expected (2 * c_int): {2 * ctypes.sizeof(ctypes.c_int)} bytes\")\n", + "print(f\"Match: {ctypes.sizeof(Pair) == 8}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910005", + "metadata": {}, + "source": [ + "## Section 2: Structures with Mixed Types\n", + "\n", + "Structures can contain fields of different types, including floats, doubles, and chars.\n", + "Be aware of potential padding bytes that the compiler may insert for alignment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910006", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Particle(ctypes.Structure):\n", + " \"\"\"A particle with position and mass.\"\"\"\n", + " _fields_ = [\n", + " (\"x\", ctypes.c_double),\n", + " (\"y\", ctypes.c_double),\n", + " (\"mass\", ctypes.c_float),\n", + " (\"active\", ctypes.c_bool),\n", + " ]\n", + "\n", + "\n", + "particle: Particle = Particle(x=1.5, y=2.5, mass=10.0, active=True)\n", + "print(f\"Position: ({particle.x}, {particle.y})\")\n", + "print(f\"Mass: {particle.mass}\")\n", + "print(f\"Active: {particle.active}\")\n", + "print(f\"sizeof(Particle): {ctypes.sizeof(Particle)} bytes\")\n", + "\n", + "# Inspect the fields\n", + "print(f\"\\nFields:\")\n", + "for name, field_type in Particle._fields_:\n", + " print(f\" {name:10s} -> {field_type.__name__:15s} ({ctypes.sizeof(field_type)} bytes)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910007", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Person(ctypes.Structure):\n", + " \"\"\"A person with name buffer and age.\"\"\"\n", + " _fields_ = [\n", + " (\"name\", ctypes.c_char * 20), # Fixed-size char array\n", + " (\"age\", ctypes.c_int),\n", + " ]\n", + "\n", + "\n", + "person: Person = Person(name=b\"Alice\", age=30)\n", + "print(f\"Name: {person.name}\")\n", + "print(f\"Age: {person.age}\")\n", + "print(f\"sizeof(Person): {ctypes.sizeof(Person)} bytes\")\n", + "\n", + "# Update the name\n", + "person.name = b\"Bob\"\n", + "print(f\"\\nUpdated name: {person.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910008", + "metadata": {}, + "source": [ + "## Section 3: ctypes Arrays\n", + "\n", + "ctypes arrays are fixed-size sequences of a single type, created with the `type * count`\n", + "syntax. They behave like C arrays with a known length and support indexing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910009", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Create an array type: 5 ints\n", + "IntArray5 = ctypes.c_int * 5\n", + "\n", + "# Create an instance with values\n", + "arr: ctypes.Array[ctypes.c_int] = IntArray5(1, 2, 3, 4, 5)\n", + "\n", + "print(f\"Length: {len(arr)}\")\n", + "print(f\"First element: {arr[0]}\")\n", + "print(f\"Last element: {arr[4]}\")\n", + "\n", + "# Iterate over the array\n", + "print(f\"\\nAll elements:\")\n", + "for i in range(len(arr)):\n", + " print(f\" arr[{i}] = {arr[i]}\")\n", + "\n", + "# Convert to a Python list\n", + "as_list: list[int] = list(arr)\n", + "print(f\"\\nAs list: {as_list}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910010", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Modify array elements\n", + "IntArray5 = ctypes.c_int * 5\n", + "arr: ctypes.Array[ctypes.c_int] = IntArray5(10, 20, 30, 40, 50)\n", + "\n", + "arr[0] = 100\n", + "arr[4] = 500\n", + "print(f\"Modified array: {list(arr)}\")\n", + "\n", + "# Double array\n", + "DoubleArray3 = ctypes.c_double * 3\n", + "darr: ctypes.Array[ctypes.c_double] = DoubleArray3(1.1, 2.2, 3.3)\n", + "print(f\"\\nDouble array: {list(darr)}\")\n", + "print(f\"sizeof(DoubleArray3): {ctypes.sizeof(DoubleArray3)} bytes\")\n", + "print(f\"Expected (3 * 8): {3 * ctypes.sizeof(ctypes.c_double)} bytes\")\n", + "\n", + "# Zero-initialized array\n", + "zeros: ctypes.Array[ctypes.c_int] = IntArray5()\n", + "print(f\"\\nZero-initialized: {list(zeros)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910011", + "metadata": {}, + "source": [ + "## Section 4: Unions\n", + "\n", + "A `ctypes.Union` is like a structure, but all fields share the same memory location.\n", + "The size of a union equals the size of its largest field. Writing to one field\n", + "affects the raw bytes visible through all other fields." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910012", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class IntOrFloat(ctypes.Union):\n", + " \"\"\"A union that can be interpreted as int or float.\"\"\"\n", + " _fields_ = [(\"i\", ctypes.c_int), (\"f\", ctypes.c_float)]\n", + "\n", + "\n", + "# Set the integer field\n", + "u: IntOrFloat = IntOrFloat()\n", + "u.i = 0\n", + "print(f\"i = 0 -> f = {u.f}\")\n", + "print(f\"Both zero: {u.f == 0.0}\")\n", + "\n", + "# The fields share memory, so setting one affects the other\n", + "u.i = 1065353216 # IEEE 754 representation of 1.0\n", + "print(f\"\\ni = 1065353216 -> f = {u.f}\")\n", + "\n", + "# Size equals the largest field\n", + "print(f\"\\nsizeof(IntOrFloat): {ctypes.sizeof(IntOrFloat)} bytes\")\n", + "print(f\"sizeof(c_int): {ctypes.sizeof(ctypes.c_int)} bytes\")\n", + "print(f\"sizeof(c_float): {ctypes.sizeof(ctypes.c_float)} bytes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910013", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Number(ctypes.Union):\n", + " \"\"\"A union demonstrating different numeric interpretations.\"\"\"\n", + " _fields_ = [\n", + " (\"as_byte\", ctypes.c_uint8),\n", + " (\"as_short\", ctypes.c_uint16),\n", + " (\"as_int\", ctypes.c_uint32),\n", + " ]\n", + "\n", + "\n", + "n: Number = Number()\n", + "n.as_int = 0x01020304\n", + "\n", + "print(f\"as_int: 0x{n.as_int:08x}\")\n", + "print(f\"as_short: 0x{n.as_short:04x}\")\n", + "print(f\"as_byte: 0x{n.as_byte:02x}\")\n", + "print(f\"\\nsizeof(Number): {ctypes.sizeof(Number)} bytes\")\n", + "print(f\"Union size matches largest field (c_uint32 = 4): {ctypes.sizeof(Number) == 4}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910014", + "metadata": {}, + "source": [ + "## Section 5: CFUNCTYPE -- Function Pointer Callbacks\n", + "\n", + "`CFUNCTYPE` creates a C-callable function type from a return type and argument types.\n", + "You can wrap a Python function in this type to create a callback that can be passed\n", + "to C libraries. The first argument is the return type, followed by parameter types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910015", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Define a callback type: int callback(int, int)\n", + "CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int)\n", + "\n", + "\n", + "def py_add(a: int, b: int) -> int:\n", + " \"\"\"A Python function to be used as a C callback.\"\"\"\n", + " return a + b\n", + "\n", + "\n", + "# Wrap the Python function\n", + "cb = CALLBACK(py_add)\n", + "\n", + "# Call it -- it behaves like a C function pointer\n", + "result: int = cb(3, 4)\n", + "print(f\"cb(3, 4) = {result}\")\n", + "print(f\"Result is 7: {result == 7}\")\n", + "\n", + "# Works with different arguments\n", + "print(f\"cb(10, 20) = {cb(10, 20)}\")\n", + "print(f\"cb(-5, 5) = {cb(-5, 5)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910016", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Callback with different signatures\n", + "\n", + "# double callback(double)\n", + "DoubleFunc = ctypes.CFUNCTYPE(ctypes.c_double, ctypes.c_double)\n", + "\n", + "\n", + "def py_square(x: float) -> float:\n", + " \"\"\"Return x squared.\"\"\"\n", + " return x * x\n", + "\n", + "\n", + "square_cb = DoubleFunc(py_square)\n", + "print(f\"square(3.0) = {square_cb(3.0)}\")\n", + "print(f\"square(2.5) = {square_cb(2.5)}\")\n", + "\n", + "# Void return type (use None for void)\n", + "VoidFunc = ctypes.CFUNCTYPE(None, ctypes.c_int)\n", + "\n", + "\n", + "def py_print_int(x: int) -> None:\n", + " \"\"\"Print an integer (void return).\"\"\"\n", + " print(f\" Callback received: {x}\")\n", + "\n", + "\n", + "print_cb = VoidFunc(py_print_int)\n", + "print(\"\\nCalling void callback:\")\n", + "print_cb(42)\n", + "print_cb(99)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910017", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "# Practical example: using callbacks for a sort comparator\n", + "CompareFunc = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int)\n", + "\n", + "\n", + "def ascending(a: int, b: int) -> int:\n", + " \"\"\"Compare for ascending order.\"\"\"\n", + " return a - b\n", + "\n", + "\n", + "def descending(a: int, b: int) -> int:\n", + " \"\"\"Compare for descending order.\"\"\"\n", + " return b - a\n", + "\n", + "\n", + "def apply_sort_key(\n", + " values: list[int],\n", + " compare: CompareFunc,\n", + ") -> list[int]:\n", + " \"\"\"Sort a list using a ctypes comparison callback.\"\"\"\n", + " import functools\n", + " return sorted(values, key=functools.cmp_to_key(lambda a, b: compare(a, b)))\n", + "\n", + "\n", + "asc_cb = CompareFunc(ascending)\n", + "desc_cb = CompareFunc(descending)\n", + "\n", + "data: list[int] = [5, 2, 8, 1, 9, 3]\n", + "print(f\"Original: {data}\")\n", + "print(f\"Ascending: {apply_sort_key(data, asc_cb)}\")\n", + "print(f\"Descending: {apply_sort_key(data, desc_cb)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910018", + "metadata": {}, + "source": [ + "## Section 6: Nested Structures and Pointers to Structures\n", + "\n", + "Structures can contain other structures as fields or pointers to structures.\n", + "This allows modeling complex C data layouts from Python." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910019", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Point(ctypes.Structure):\n", + " \"\"\"A 2D point.\"\"\"\n", + " _fields_ = [(\"x\", ctypes.c_int), (\"y\", ctypes.c_int)]\n", + "\n", + "\n", + "class Rectangle(ctypes.Structure):\n", + " \"\"\"A rectangle defined by two corner points.\"\"\"\n", + " _fields_ = [(\"top_left\", Point), (\"bottom_right\", Point)]\n", + "\n", + "\n", + "rect: Rectangle = Rectangle(\n", + " top_left=Point(0, 0),\n", + " bottom_right=Point(100, 50),\n", + ")\n", + "\n", + "print(f\"Top-left: ({rect.top_left.x}, {rect.top_left.y})\")\n", + "print(f\"Bottom-right: ({rect.bottom_right.x}, {rect.bottom_right.y})\")\n", + "\n", + "# Calculate dimensions\n", + "width: int = rect.bottom_right.x - rect.top_left.x\n", + "height: int = rect.bottom_right.y - rect.top_left.y\n", + "print(f\"\\nWidth: {width}\")\n", + "print(f\"Height: {height}\")\n", + "print(f\"sizeof(Rectangle): {ctypes.sizeof(Rectangle)} bytes\")\n", + "print(f\"Expected (4 ints * 4 bytes): {4 * ctypes.sizeof(ctypes.c_int)} bytes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910020", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class Point(ctypes.Structure):\n", + " \"\"\"A 2D point.\"\"\"\n", + " _fields_ = [(\"x\", ctypes.c_int), (\"y\", ctypes.c_int)]\n", + "\n", + "\n", + "# Pointer to a structure\n", + "p: Point = Point(10, 20)\n", + "ptr = ctypes.pointer(p)\n", + "\n", + "print(f\"Original: ({p.x}, {p.y})\")\n", + "print(f\"Via pointer: ({ptr.contents.x}, {ptr.contents.y})\")\n", + "\n", + "# Modify through pointer\n", + "ptr.contents.x = 99\n", + "print(f\"\\nAfter pointer modification:\")\n", + "print(f\"Original p.x: {p.x}\")\n", + "print(f\"Pointer p.x: {ptr.contents.x}\")\n", + "\n", + "# Structure containing an array field\n", + "class Vector3(ctypes.Structure):\n", + " \"\"\"A 3D vector using an array field.\"\"\"\n", + " _fields_ = [(\"coords\", ctypes.c_double * 3)]\n", + "\n", + "\n", + "v: Vector3 = Vector3(coords=(ctypes.c_double * 3)(1.0, 2.0, 3.0))\n", + "print(f\"\\nVector3: ({v.coords[0]}, {v.coords[1]}, {v.coords[2]})\")\n", + "print(f\"sizeof(Vector3): {ctypes.sizeof(Vector3)} bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910021", + "metadata": {}, + "source": [ + "## Section 7: Structures with Arrays\n", + "\n", + "Combining structures and arrays allows defining complex data layouts that mirror\n", + "C struct definitions containing fixed-size array members." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3910022", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "\n", + "\n", + "class DataPacket(ctypes.Structure):\n", + " \"\"\"A data packet with a header and fixed-size payload.\"\"\"\n", + " _fields_ = [\n", + " (\"packet_id\", ctypes.c_uint32),\n", + " (\"length\", ctypes.c_uint16),\n", + " (\"data\", ctypes.c_uint8 * 8),\n", + " ]\n", + "\n", + "\n", + "# Create a packet\n", + "payload = (ctypes.c_uint8 * 8)(0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x00, 0x00, 0x00)\n", + "pkt: DataPacket = DataPacket(packet_id=1, length=5, data=payload)\n", + "\n", + "print(f\"Packet ID: {pkt.packet_id}\")\n", + "print(f\"Length: {pkt.length}\")\n", + "print(f\"Data bytes: {list(pkt.data)}\")\n", + "\n", + "# Interpret data as a string\n", + "data_bytes: bytes = bytes(pkt.data[:pkt.length])\n", + "print(f\"Data as string: {data_bytes.decode('ascii')}\")\n", + "print(f\"\\nsizeof(DataPacket): {ctypes.sizeof(DataPacket)} bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3910023", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Structures\n", + "- Subclass `ctypes.Structure` and define `_fields_` as `[(name, type), ...]`\n", + "- Access fields as attributes: `struct.field_name`\n", + "- `sizeof(StructType)` returns the total size including alignment padding\n", + "- Structures can be nested: use one Structure type as a field type in another\n", + "\n", + "### Unions\n", + "- Subclass `ctypes.Union` with `_fields_` (same syntax as Structure)\n", + "- All fields share the same memory -- writing one field affects all others\n", + "- `sizeof(UnionType)` equals the size of the largest field\n", + "\n", + "### Arrays\n", + "- Create array types with `type * count` (e.g., `c_int * 5`)\n", + "- Fixed length, indexable, iterable\n", + "- Can be used as fields in structures\n", + "\n", + "### CFUNCTYPE Callbacks\n", + "- `CFUNCTYPE(restype, *argtypes)` defines a C-callable function type\n", + "- Wrap a Python function: `cb = CFUNCTYPE(...)(python_func)`\n", + "- Use `None` as return type for void functions\n", + "- Keep a reference to the callback to prevent garbage collection" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_39/03_array_and_memoryview.ipynb b/src/chapter_39/03_array_and_memoryview.ipynb new file mode 100644 index 0000000..2254539 --- /dev/null +++ b/src/chapter_39/03_array_and_memoryview.ipynb @@ -0,0 +1,669 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c3920001", + "metadata": {}, + "source": [ + "# Chapter 39: Array, Memoryview, and Struct\n", + "\n", + "This notebook covers Python's `array` module for efficient typed arrays, the `memoryview`\n", + "object for zero-copy buffer access, and the `struct` module for packing and unpacking\n", + "binary data. Together these tools form Python's buffer protocol ecosystem.\n", + "\n", + "## Key Concepts\n", + "- **`array.array`**: Typed, compact arrays of numeric values\n", + "- **`memoryview`**: Zero-copy slicing and modification of buffer objects\n", + "- **Buffer protocol**: The interface that allows objects to share memory\n", + "- **`struct.pack` / `struct.unpack`**: Convert between Python values and C-style binary data\n", + "- **Format strings**: Specify byte order and data types for binary serialization" + ] + }, + { + "cell_type": "markdown", + "id": "c3920002", + "metadata": {}, + "source": [ + "## Section 1: Creating Arrays\n", + "\n", + "The `array` module provides a compact, typed array that stores elements more efficiently\n", + "than a Python list. Each array has a typecode that determines the C type of its elements." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920003", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# Create an array of signed integers ('i' = C int)\n", + "a: array.array = array.array(\"i\", [1, 2, 3, 4, 5])\n", + "\n", + "print(f\"Array: {a}\")\n", + "print(f\"Length: {len(a)}\")\n", + "print(f\"First element: {a[0]}\")\n", + "print(f\"Last element: {a[-1]}\")\n", + "print(f\"Typecode: {a.typecode}\")\n", + "\n", + "# Common typecodes\n", + "typecodes: list[tuple[str, str]] = [\n", + " (\"b\", \"signed char (1 byte)\"),\n", + " (\"B\", \"unsigned char (1 byte)\"),\n", + " (\"h\", \"signed short (2 bytes)\"),\n", + " (\"i\", \"signed int (2-4 bytes)\"),\n", + " (\"l\", \"signed long (4-8 bytes)\"),\n", + " (\"f\", \"float (4 bytes)\"),\n", + " (\"d\", \"double (8 bytes)\"),\n", + "]\n", + "print(f\"\\n{'Code':<6} {'Description'}\")\n", + "print(\"-\" * 35)\n", + "for code, desc in typecodes:\n", + " print(f\"{code:<6} {desc}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920004", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# Arrays of different types\n", + "int_arr: array.array = array.array(\"i\", [10, 20, 30])\n", + "float_arr: array.array = array.array(\"f\", [1.1, 2.2, 3.3])\n", + "double_arr: array.array = array.array(\"d\", [1.0, 2.0, 3.0])\n", + "\n", + "print(f\"int array: {list(int_arr)}\")\n", + "print(f\"float array: {list(float_arr)}\")\n", + "print(f\"double array: {list(double_arr)}\")\n", + "\n", + "# Arrays support standard sequence operations\n", + "int_arr.append(40)\n", + "int_arr.extend([50, 60])\n", + "print(f\"\\nAfter append/extend: {list(int_arr)}\")\n", + "\n", + "# Slicing\n", + "sliced: array.array = int_arr[1:4]\n", + "print(f\"Slice [1:4]: {list(sliced)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920005", + "metadata": {}, + "source": [ + "## Section 2: Array Properties -- itemsize and buffer_info\n", + "\n", + "The `itemsize` attribute tells you how many bytes each element occupies. The\n", + "`buffer_info()` method returns the memory address and number of elements." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920006", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "import struct\n", + "\n", + "# itemsize shows bytes per element\n", + "a: array.array = array.array(\"i\")\n", + "print(f\"Typecode 'i' itemsize: {a.itemsize} bytes\")\n", + "print(f\"Matches struct.calcsize('i'): {a.itemsize == struct.calcsize('i')}\")\n", + "\n", + "# Compare itemsize across types\n", + "for code in [\"b\", \"h\", \"i\", \"l\", \"f\", \"d\"]:\n", + " arr: array.array = array.array(code)\n", + " print(f\" typecode '{code}': {arr.itemsize} bytes per element\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920007", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# buffer_info returns (address, length)\n", + "a: array.array = array.array(\"d\", [1.0, 2.0, 3.0])\n", + "address, length = a.buffer_info()\n", + "\n", + "print(f\"Buffer address: {address}\")\n", + "print(f\"Number of elements: {length}\")\n", + "print(f\"Address is positive: {address > 0}\")\n", + "print(f\"Length matches len(): {length == len(a)}\")\n", + "\n", + "# Total memory used by the data\n", + "total_bytes: int = length * a.itemsize\n", + "print(f\"\\nTotal data bytes: {total_bytes}\")\n", + "print(f\"Expected (3 * 8): {3 * 8}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920008", + "metadata": {}, + "source": [ + "## Section 3: Array Bytes Conversion\n", + "\n", + "Arrays can be serialized to bytes with `tobytes()` and reconstructed from bytes\n", + "with `frombytes()`. This is useful for saving arrays to files or sending them\n", + "over a network." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920009", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# Convert array to bytes\n", + "a: array.array = array.array(\"i\", [1, 2, 3])\n", + "b: bytes = a.tobytes()\n", + "\n", + "print(f\"Original array: {list(a)}\")\n", + "print(f\"As bytes: {b!r}\")\n", + "print(f\"Byte length: {len(b)}\")\n", + "print(f\"Expected ({len(a)} * {a.itemsize}): {len(a) * a.itemsize}\")\n", + "\n", + "# Reconstruct from bytes\n", + "a2: array.array = array.array(\"i\")\n", + "a2.frombytes(b)\n", + "\n", + "print(f\"\\nReconstructed: {list(a2)}\")\n", + "print(f\"Round-trip match: {list(a) == list(a2)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920010", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# tolist and fromlist for Python list conversion\n", + "original: array.array = array.array(\"d\", [1.5, 2.5, 3.5])\n", + "as_list: list[float] = original.tolist()\n", + "print(f\"tolist(): {as_list}\")\n", + "print(f\"Type: {type(as_list)}\")\n", + "\n", + "# fromlist adds elements from a Python list\n", + "a: array.array = array.array(\"d\")\n", + "a.fromlist([10.0, 20.0, 30.0])\n", + "print(f\"\\nfromlist(): {list(a)}\")\n", + "\n", + "# Byte-swapping for endianness conversion\n", + "a: array.array = array.array(\"i\", [1, 256])\n", + "print(f\"\\nBefore byteswap: {list(a)}\")\n", + "a.byteswap()\n", + "print(f\"After byteswap: {list(a)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920011", + "metadata": {}, + "source": [ + "## Section 4: Memoryview Basics\n", + "\n", + "A `memoryview` provides zero-copy access to the internal buffer of an object\n", + "that supports the buffer protocol (like `bytes`, `bytearray`, and `array.array`).\n", + "Slicing a memoryview does not copy data -- it creates a view into the same memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920012", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a memoryview from a bytearray\n", + "data: bytearray = bytearray(b\"Hello, World!\")\n", + "mv: memoryview = memoryview(data)\n", + "\n", + "# Slicing returns a new memoryview (zero-copy)\n", + "hello: memoryview = mv[0:5]\n", + "print(f\"Full data: {bytes(mv)}\")\n", + "print(f\"Slice [0:5]: {bytes(hello)}\")\n", + "print(f\"Slice equals b'Hello': {bytes(hello) == b'Hello'}\")\n", + "\n", + "# Memoryview properties\n", + "print(f\"\\nFormat: {mv.format}\")\n", + "print(f\"Itemsize: {mv.itemsize}\")\n", + "print(f\"Length: {len(mv)}\")\n", + "print(f\"Readonly: {mv.readonly}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920013", + "metadata": {}, + "outputs": [], + "source": [ + "# Memoryview from bytes (read-only) vs bytearray (writable)\n", + "immutable: memoryview = memoryview(b\"readonly\")\n", + "mutable: memoryview = memoryview(bytearray(b\"writable\"))\n", + "\n", + "print(f\"From bytes - readonly: {immutable.readonly}\")\n", + "print(f\"From bytearray - readonly: {mutable.readonly}\")\n", + "\n", + "# Read-only memoryview cannot be modified\n", + "try:\n", + " immutable[0] = ord(\"R\")\n", + "except TypeError as e:\n", + " print(f\"\\nCannot modify bytes memoryview: {e}\")\n", + "\n", + "# Indexing returns an integer (the byte value)\n", + "print(f\"\\nmutable[0] = {mutable[0]} (ord('w') = {ord('w')})\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920014", + "metadata": {}, + "source": [ + "## Section 5: Modifying Data Through Memoryview\n", + "\n", + "When a memoryview is created from a mutable buffer (like `bytearray`), modifications\n", + "through the memoryview directly change the underlying data without copying." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920015", + "metadata": {}, + "outputs": [], + "source": [ + "# Modify underlying buffer through memoryview\n", + "data: bytearray = bytearray(b\"Hello\")\n", + "mv: memoryview = memoryview(data)\n", + "\n", + "print(f\"Before: {data}\")\n", + "\n", + "# Change first byte from 'H' to 'J'\n", + "mv[0] = ord(\"J\")\n", + "print(f\"After mv[0] = ord('J'): {data}\")\n", + "print(f\"Data is now 'Jello': {data == bytearray(b'Jello')}\")\n", + "\n", + "# Slice assignment\n", + "data2: bytearray = bytearray(b\"abcdefgh\")\n", + "mv2: memoryview = memoryview(data2)\n", + "mv2[2:5] = b\"CDE\"\n", + "print(f\"\\nAfter slice assignment: {data2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920016", + "metadata": {}, + "outputs": [], + "source": [ + "# Zero-copy demonstration: memoryview shares memory with the original\n", + "original: bytearray = bytearray(b\"shared memory demo\")\n", + "view1: memoryview = memoryview(original)\n", + "view2: memoryview = view1[7:13] # \"memory\"\n", + "\n", + "print(f\"Original: {original}\")\n", + "print(f\"view2: {bytes(view2)}\")\n", + "\n", + "# Modify through view2 -- affects original\n", + "view2[0] = ord(\"M\")\n", + "print(f\"\\nAfter view2[0] = ord('M'):\")\n", + "print(f\"Original: {original}\")\n", + "print(f\"view2: {bytes(view2)}\")\n", + "\n", + "# Release memoryview to free the buffer lock\n", + "view1.release()\n", + "view2.release()\n", + "print(f\"\\nMemoryviews released\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920017", + "metadata": {}, + "source": [ + "## Section 6: Memoryview with Typed Arrays\n", + "\n", + "When you create a memoryview from a typed `array.array`, the memoryview\n", + "inherits the format and itemsize of the array's element type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920018", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "import struct\n", + "\n", + "# Memoryview from a typed array\n", + "a: array.array = array.array(\"i\", [1, 2, 3])\n", + "mv: memoryview = memoryview(a)\n", + "\n", + "print(f\"Array: {list(a)}\")\n", + "print(f\"Format: {mv.format}\")\n", + "print(f\"Itemsize: {mv.itemsize}\")\n", + "print(f\"Itemsize matches struct.calcsize('i'): {mv.itemsize == struct.calcsize('i')}\")\n", + "print(f\"Length: {len(mv)}\")\n", + "\n", + "# Indexing returns typed values (not raw bytes)\n", + "print(f\"\\nmv[0] = {mv[0]}\")\n", + "print(f\"mv[1] = {mv[1]}\")\n", + "print(f\"mv[2] = {mv[2]}\")\n", + "\n", + "# Convert to list\n", + "print(f\"\\nAs list: {mv.tolist()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920019", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# Modifying array elements through memoryview\n", + "a: array.array = array.array(\"i\", [10, 20, 30, 40, 50])\n", + "mv: memoryview = memoryview(a)\n", + "\n", + "print(f\"Before: {list(a)}\")\n", + "\n", + "# Modify through memoryview\n", + "mv[0] = 100\n", + "mv[4] = 500\n", + "print(f\"After: {list(a)}\")\n", + "\n", + "# Cast to a different view (e.g., view ints as bytes)\n", + "byte_view: memoryview = mv.cast(\"b\") # 'b' = signed char\n", + "print(f\"\\nAs bytes view length: {len(byte_view)}\")\n", + "print(f\"Expected ({len(a)} * {a.itemsize}): {len(a) * a.itemsize}\")\n", + "print(f\"First 4 bytes (int 100): {list(byte_view[:4])}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920020", + "metadata": {}, + "source": [ + "## Section 7: struct -- Packing and Unpacking Binary Data\n", + "\n", + "The `struct` module converts between Python values and C-style packed binary data.\n", + "A format string specifies the byte order and types of values to pack or unpack." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920021", + "metadata": {}, + "outputs": [], + "source": [ + "import struct\n", + "\n", + "# Pack two unsigned ints in network byte order\n", + "packed: bytes = struct.pack(\"!2I\", 1, 2)\n", + "print(f\"Packed bytes: {packed!r}\")\n", + "print(f\"Packed length: {len(packed)} bytes\")\n", + "\n", + "# Unpack them back\n", + "a, b = struct.unpack(\"!2I\", packed)\n", + "print(f\"\\nUnpacked: a={a}, b={b}\")\n", + "print(f\"a == 1: {a == 1}\")\n", + "print(f\"b == 2: {b == 2}\")\n", + "\n", + "# Byte order prefixes\n", + "prefixes: list[tuple[str, str]] = [\n", + " (\"!\", \"network (big-endian)\"),\n", + " (\">\", \"big-endian\"),\n", + " (\"<\", \"little-endian\"),\n", + " (\"=\", \"native\"),\n", + " (\"@\", \"native with native alignment\"),\n", + "]\n", + "print(f\"\\n{'Prefix':<8} {'Byte Order'}\")\n", + "print(\"-\" * 35)\n", + "for prefix, desc in prefixes:\n", + " print(f\"{prefix:<8} {desc}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920022", + "metadata": {}, + "outputs": [], + "source": [ + "import struct\n", + "\n", + "# Format characters for different types\n", + "# Pack a mixed record: an int, a float, and a 5-byte string\n", + "record: bytes = struct.pack(\"!if5s\", 42, 3.14, b\"Hello\")\n", + "print(f\"Packed record: {record!r}\")\n", + "print(f\"Record length: {len(record)} bytes\")\n", + "\n", + "# Unpack the record\n", + "num, fval, text = struct.unpack(\"!if5s\", record)\n", + "print(f\"\\nUnpacked: num={num}, fval={fval:.4f}, text={text}\")\n", + "\n", + "# calcsize tells you the packed size for a format\n", + "print(f\"\\nFormat sizes:\")\n", + "formats: list[tuple[str, str]] = [\n", + " (\"!I\", \"1 unsigned int (network)\"),\n", + " (\"!2I\", \"2 unsigned ints (network)\"),\n", + " (\"!if\", \"int + float (network)\"),\n", + " (\"!d\", \"1 double (network)\"),\n", + " (\"!10s\", \"10-byte string\"),\n", + "]\n", + "for fmt, desc in formats:\n", + " size: int = struct.calcsize(fmt)\n", + " print(f\" {fmt:<8} ({desc}): {size} bytes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920023", + "metadata": {}, + "outputs": [], + "source": [ + "import struct\n", + "\n", + "# Practical: pack/unpack a network packet header\n", + "def pack_header(version: int, msg_type: int, length: int, seq: int) -> bytes:\n", + " \"\"\"Pack a 10-byte packet header.\"\"\"\n", + " return struct.pack(\"!BBHI\", version, msg_type, length, seq)\n", + "\n", + "\n", + "def unpack_header(data: bytes) -> tuple[int, int, int, int]:\n", + " \"\"\"Unpack a 10-byte packet header.\"\"\"\n", + " return struct.unpack(\"!BBHI\", data)\n", + "\n", + "\n", + "# Pack a header\n", + "header: bytes = pack_header(version=1, msg_type=3, length=256, seq=12345)\n", + "print(f\"Packed header: {header!r}\")\n", + "print(f\"Header size: {len(header)} bytes\")\n", + "\n", + "# Unpack it\n", + "ver, mtype, length, seq = unpack_header(header)\n", + "print(f\"\\nVersion: {ver}\")\n", + "print(f\"Type: {mtype}\")\n", + "print(f\"Length: {length}\")\n", + "print(f\"Sequence: {seq}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920024", + "metadata": {}, + "outputs": [], + "source": [ + "import struct\n", + "\n", + "# struct.Struct for repeated packing/unpacking with the same format\n", + "point_fmt: struct.Struct = struct.Struct(\"!2d\") # Two doubles\n", + "\n", + "print(f\"Format: {point_fmt.format}\")\n", + "print(f\"Size: {point_fmt.size} bytes\")\n", + "\n", + "# Pack several points\n", + "points: list[tuple[float, float]] = [(1.0, 2.0), (3.5, 4.5), (5.0, 6.0)]\n", + "packed_points: list[bytes] = [point_fmt.pack(x, y) for x, y in points]\n", + "\n", + "print(f\"\\nPacked {len(packed_points)} points\")\n", + "for i, data in enumerate(packed_points):\n", + " x, y = point_fmt.unpack(data)\n", + " print(f\" Point {i}: ({x}, {y})\")\n", + "\n", + "# pack_into / unpack_from for working with buffers\n", + "buffer: bytearray = bytearray(point_fmt.size * 2)\n", + "point_fmt.pack_into(buffer, 0, 10.0, 20.0)\n", + "point_fmt.pack_into(buffer, point_fmt.size, 30.0, 40.0)\n", + "\n", + "p1_x, p1_y = point_fmt.unpack_from(buffer, 0)\n", + "p2_x, p2_y = point_fmt.unpack_from(buffer, point_fmt.size)\n", + "print(f\"\\nFrom buffer:\")\n", + "print(f\" Point 0: ({p1_x}, {p1_y})\")\n", + "print(f\" Point 1: ({p2_x}, {p2_y})\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920025", + "metadata": {}, + "source": [ + "## Section 8: Combining Array, Memoryview, and Struct\n", + "\n", + "These three tools work together through the buffer protocol. Arrays provide\n", + "typed storage, memoryviews provide zero-copy access, and struct handles\n", + "serialization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920026", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "import struct\n", + "\n", + "# Round-trip: array -> bytes -> array\n", + "original: array.array = array.array(\"i\", [100, 200, 300, 400])\n", + "print(f\"Original: {list(original)}\")\n", + "\n", + "# Serialize to bytes\n", + "raw: bytes = original.tobytes()\n", + "print(f\"As bytes: {raw!r}\")\n", + "\n", + "# Interpret the same bytes with struct\n", + "count: int = len(raw) // struct.calcsize(\"i\")\n", + "values: tuple[int, ...] = struct.unpack(f\"{count}i\", raw)\n", + "print(f\"Via struct.unpack: {values}\")\n", + "\n", + "# Reconstruct the array\n", + "restored: array.array = array.array(\"i\")\n", + "restored.frombytes(raw)\n", + "print(f\"Restored: {list(restored)}\")\n", + "print(f\"Match: {list(original) == list(restored)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3920027", + "metadata": {}, + "outputs": [], + "source": [ + "import array\n", + "\n", + "# Memoryview as a bridge between array and raw bytes\n", + "a: array.array = array.array(\"i\", [1, 2, 3, 4, 5])\n", + "mv: memoryview = memoryview(a)\n", + "\n", + "# Typed view: each element is an int\n", + "print(f\"Typed view: format={mv.format}, len={len(mv)}\")\n", + "print(f\"Elements: {mv.tolist()}\")\n", + "\n", + "# Cast to raw bytes view\n", + "byte_mv: memoryview = mv.cast(\"B\") # unsigned char\n", + "print(f\"\\nByte view: format={byte_mv.format}, len={len(byte_mv)}\")\n", + "print(f\"First 8 bytes: {list(byte_mv[:8])}\")\n", + "\n", + "# Modify through the typed memoryview\n", + "mv[2] = 999\n", + "print(f\"\\nAfter mv[2] = 999:\")\n", + "print(f\"Array: {list(a)}\")\n", + "print(f\"Byte view at [8:12]: {list(byte_mv[8:12])}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3920028", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### array Module\n", + "- **`array.array(typecode, iterable)`**: Create a typed, compact array\n", + "- **`.typecode`**: The character code for the element type (e.g., `'i'` for int)\n", + "- **`.itemsize`**: Bytes per element\n", + "- **`.buffer_info()`**: Returns `(address, element_count)`\n", + "- **`.tobytes()` / `.frombytes()`**: Serialize to and from raw bytes\n", + "\n", + "### memoryview\n", + "- **`memoryview(buffer)`**: Zero-copy view into a buffer object\n", + "- Slicing creates a new view (no data copied)\n", + "- **`.format`**: Element format character\n", + "- **`.itemsize`**: Bytes per element\n", + "- **`.readonly`**: Whether the view is writable\n", + "- **`.cast(format)`**: Reinterpret the buffer with a different element type\n", + "- Modifications through a memoryview affect the underlying buffer directly\n", + "\n", + "### struct Module\n", + "- **`struct.pack(fmt, v1, v2, ...)`**: Pack values into bytes\n", + "- **`struct.unpack(fmt, buffer)`**: Unpack bytes into a tuple of values\n", + "- **`struct.calcsize(fmt)`**: Size in bytes of the packed format\n", + "- Byte order prefixes: `!` (network), `>` (big), `<` (little), `=`/`@` (native)\n", + "- **`struct.Struct(fmt)`**: Precompiled format for repeated use" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_39/README.md b/src/chapter_39/README.md new file mode 100644 index 0000000..09d7bb5 --- /dev/null +++ b/src/chapter_39/README.md @@ -0,0 +1,14 @@ +# Chapter 39: C Interoperability + +## Topics Covered + +- ctypes: loading shared libraries, fundamental types, function calls +- ctypes structures, arrays, callbacks, and pointers +- array module for typed numeric arrays +- memoryview and the buffer protocol + +## Notebooks + +1. **01_ctypes_basics.ipynb** - Loading libraries, fundamental types, function calls +2. **02_ctypes_advanced.ipynb** - Structures, arrays, callbacks, pointers +3. **03_array_and_memoryview.ipynb** - array module, memoryview, buffer protocol diff --git a/src/chapter_39/__init__.py b/src/chapter_39/__init__.py new file mode 100644 index 0000000..3ee393f --- /dev/null +++ b/src/chapter_39/__init__.py @@ -0,0 +1 @@ +"""Chapter 39: C Interoperability.""" diff --git a/src/chapter_40/01_importlib_basics.ipynb b/src/chapter_40/01_importlib_basics.ipynb new file mode 100644 index 0000000..68b8aa6 --- /dev/null +++ b/src/chapter_40/01_importlib_basics.ipynb @@ -0,0 +1,525 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 40: Importlib Basics\n", + "\n", + "This notebook covers the core functionality of Python's `importlib` module -- the programmatic interface to the import system. You will learn how to import modules by string name, reload modules, and inspect the standard attributes that every module carries.\n", + "\n", + "## Key Concepts\n", + "- **`importlib.import_module`**: Import a module using a string name instead of the `import` statement\n", + "- **`importlib.reload`**: Re-import an already-loaded module to pick up changes\n", + "- **Module attributes**: Standard attributes like `__name__`, `__file__`, and `__package__`\n", + "- **`__spec__`**: The `ModuleSpec` object that describes how a module was found and loaded\n", + "- **Submodule imports**: Importing dotted module paths like `os.path`" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Importing Modules by Name\n", + "\n", + "The `importlib.import_module` function lets you import a module when you have its name as a string. This is essential for plugin systems, dynamic loading, and any scenario where the module name is not known until runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import types\n", + "\n", + "# Import a module by string name\n", + "math_mod: types.ModuleType = importlib.import_module(\"math\")\n", + "\n", + "print(f\"Module: {math_mod}\")\n", + "print(f\"pi = {math_mod.pi}\")\n", + "print(f\"sqrt(16) = {math_mod.sqrt(16)}\")\n", + "\n", + "# This is equivalent to 'import math'\n", + "import math\n", + "print(f\"\\nSame object? {math_mod is math}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import types\n", + "\n", + "# Dynamic import from a list of module names\n", + "module_names: list[str] = [\"json\", \"math\", \"os\"]\n", + "\n", + "for name in module_names:\n", + " mod: types.ModuleType = importlib.import_module(name)\n", + " attr_count: int = len([a for a in dir(mod) if not a.startswith(\"_\")])\n", + " print(f\"{name:>6}: {attr_count} public attributes\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Importing Submodules\n", + "\n", + "You can import submodules (dotted names like `os.path`) by passing the full dotted path to `import_module`. This works for any package that contains submodules." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import types\n", + "\n", + "# Import a submodule using its dotted path\n", + "path_mod: types.ModuleType = importlib.import_module(\"os.path\")\n", + "\n", + "print(f\"Module: {path_mod}\")\n", + "print(f\"Has 'join': {hasattr(path_mod, 'join')}\")\n", + "print(f\"Has 'exists': {hasattr(path_mod, 'exists')}\")\n", + "\n", + "# Demonstrate it works\n", + "result: str = path_mod.join(\"/usr\", \"local\", \"bin\")\n", + "print(f\"\\nos.path.join('/usr', 'local', 'bin') = {result}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import types\n", + "\n", + "# Import submodules from the email package\n", + "email_mime: types.ModuleType = importlib.import_module(\"email.mime.text\")\n", + "\n", + "print(f\"Module: {email_mime}\")\n", + "print(f\"Has 'MIMEText': {hasattr(email_mime, 'MIMEText')}\")\n", + "\n", + "# You can also use the 'package' parameter for relative-style imports\n", + "# This imports 'email.mime.base' using 'email.mime' as the package anchor\n", + "base_mod: types.ModuleType = importlib.import_module(\".base\", package=\"email.mime\")\n", + "print(f\"\\nRelative import: {base_mod}\")\n", + "print(f\"Has 'MIMEBase': {hasattr(base_mod, 'MIMEBase')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Reloading Modules\n", + "\n", + "Python caches imported modules in `sys.modules`. If you modify a module's source code at runtime, calling `importlib.reload` re-executes the module's code and updates the existing module object in place." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import json\n", + "import types\n", + "\n", + "# Store the original module identity\n", + "original_id: int = id(json)\n", + "\n", + "# Reload the json module\n", + "reloaded: types.ModuleType = importlib.reload(json)\n", + "\n", + "# reload returns the same module object (updated in place)\n", + "print(f\"Original id: {original_id}\")\n", + "print(f\"Reloaded id: {id(reloaded)}\")\n", + "print(f\"Same object: {reloaded is json}\")\n", + "print(f\"Still works: json.dumps([1, 2, 3]) = {json.dumps([1, 2, 3])}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import sys\n", + "import types\n", + "import tempfile\n", + "import os\n", + "\n", + "# Demonstrate reload with a temporary module\n", + "# Create a temporary module file\n", + "tmp_dir: str = tempfile.mkdtemp()\n", + "mod_path: str = os.path.join(tmp_dir, \"demo_reload.py\")\n", + "\n", + "# Write version 1\n", + "with open(mod_path, \"w\") as f:\n", + " f.write(\"VERSION: int = 1\\n\")\n", + "\n", + "# Add tmp_dir to sys.path so we can import it\n", + "sys.path.insert(0, tmp_dir)\n", + "\n", + "try:\n", + " mod: types.ModuleType = importlib.import_module(\"demo_reload\")\n", + " print(f\"Version after first import: {mod.VERSION}\")\n", + "\n", + " # Write version 2\n", + " with open(mod_path, \"w\") as f:\n", + " f.write(\"VERSION: int = 2\\n\")\n", + "\n", + " # Without reload, we still see version 1\n", + " mod2: types.ModuleType = importlib.import_module(\"demo_reload\")\n", + " print(f\"Version after re-import (cached): {mod2.VERSION}\")\n", + "\n", + " # With reload, we pick up version 2\n", + " importlib.reload(mod)\n", + " print(f\"Version after reload: {mod.VERSION}\")\n", + "finally:\n", + " # Clean up\n", + " sys.path.remove(tmp_dir)\n", + " if \"demo_reload\" in sys.modules:\n", + " del sys.modules[\"demo_reload\"]\n", + " os.remove(mod_path)\n", + " os.rmdir(tmp_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Standard Module Attributes\n", + "\n", + "Every module in Python carries several standard attributes that describe its identity and location. These are set automatically by the import system." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "# __name__: The fully qualified module name\n", + "print(f\"__name__: {json.__name__}\")\n", + "\n", + "# __file__: The path to the source file (if applicable)\n", + "print(f\"__file__: {json.__file__}\")\n", + "\n", + "# __package__: The package this module belongs to\n", + "print(f\"__package__: {json.__package__}\")\n", + "\n", + "# __loader__: The loader that loaded this module\n", + "print(f\"__loader__: {json.__loader__}\")\n", + "\n", + "# __doc__: The module docstring (first few characters)\n", + "doc_preview: str = (json.__doc__ or \"\")[:80]\n", + "print(f\"__doc__: {doc_preview}...\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import math\n", + "import email.mime.text\n", + "\n", + "# Compare attributes across different module types\n", + "modules: list[str] = [\"os\", \"math\", \"email.mime.text\"]\n", + "\n", + "for name in modules:\n", + " mod = __import__(name, fromlist=[\"\"]) if \".\" in name else __import__(name)\n", + " if \".\" in name:\n", + " import importlib\n", + " mod = importlib.import_module(name)\n", + "\n", + " has_file: bool = hasattr(mod, \"__file__\") and mod.__file__ is not None\n", + " print(f\"{name}:\")\n", + " print(f\" __name__ = {mod.__name__}\")\n", + " print(f\" __package__ = {mod.__package__}\")\n", + " print(f\" has __file__= {has_file}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: The `__spec__` Attribute\n", + "\n", + "Every module has a `__spec__` attribute (a `ModuleSpec` object) that stores metadata about how the module was found and loaded. This was introduced in Python 3.4 and is the modern way to introspect the import system." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import importlib.machinery\n", + "\n", + "# Access the module spec\n", + "spec: importlib.machinery.ModuleSpec | None = json.__spec__\n", + "\n", + "print(f\"spec is not None: {spec is not None}\")\n", + "print(f\"spec type: {type(spec).__name__}\")\n", + "print()\n", + "\n", + "if spec is not None:\n", + " print(f\"name: {spec.name}\")\n", + " print(f\"origin: {spec.origin}\")\n", + " print(f\"loader: {spec.loader}\")\n", + " print(f\"parent: {spec.parent}\")\n", + " print(f\"has_location: {spec.has_location}\")\n", + " print(f\"submodule_search_locations: {spec.submodule_search_locations}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "\n", + "# Compare __spec__ for a package vs. a simple module\n", + "email_mod = importlib.import_module(\"email\")\n", + "json_mod = importlib.import_module(\"json\")\n", + "\n", + "for mod in [email_mod, json_mod]:\n", + " spec = mod.__spec__\n", + " if spec is not None:\n", + " is_package: bool = spec.submodule_search_locations is not None\n", + " print(f\"{spec.name}:\")\n", + " print(f\" origin: {spec.origin}\")\n", + " print(f\" is package: {is_package}\")\n", + " print(f\" parent: {spec.parent!r}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 6: Checking Module Existence\n", + "\n", + "You can use `importlib.util.find_spec` to check whether a module can be imported without actually importing it. This is safer than catching `ImportError`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "\n", + "# Check if various modules exist\n", + "module_names: list[str] = [\"json\", \"numpy\", \"os.path\", \"nonexistent_module_xyz\"]\n", + "\n", + "for name in module_names:\n", + " spec = importlib.util.find_spec(name)\n", + " exists: bool = spec is not None\n", + " print(f\"{name:>30}: {'found' if exists else 'NOT found'}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "import types\n", + "\n", + "def safe_import(name: str) -> types.ModuleType | None:\n", + " \"\"\"Import a module by name, returning None if not found.\"\"\"\n", + " spec = importlib.util.find_spec(name)\n", + " if spec is None:\n", + " return None\n", + " return importlib.import_module(name)\n", + "\n", + "# Try to import modules that may or may not exist\n", + "for name in [\"json\", \"nonexistent_package_abc\"]:\n", + " mod: types.ModuleType | None = safe_import(name)\n", + " if mod is not None:\n", + " print(f\"{name}: imported successfully (type={type(mod).__name__})\")\n", + " else:\n", + " print(f\"{name}: not available\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 7: Practical Patterns\n", + "\n", + "Common patterns that use `importlib` in real applications, including plugin loading and conditional imports." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import importlib.util\n", + "from typing import Any\n", + "\n", + "def load_serializer(format_name: str) -> Any:\n", + " \"\"\"Load a serializer module by format name.\n", + "\n", + " Simulates a plugin system where the format name\n", + " determines which module to load.\n", + " \"\"\"\n", + " format_to_module: dict[str, str] = {\n", + " \"json\": \"json\",\n", + " \"csv\": \"csv\",\n", + " \"xml\": \"xml.etree.ElementTree\",\n", + " }\n", + "\n", + " module_name: str | None = format_to_module.get(format_name)\n", + " if module_name is None:\n", + " print(f\" Unknown format: {format_name}\")\n", + " return None\n", + "\n", + " if importlib.util.find_spec(module_name) is None:\n", + " print(f\" Module {module_name} not available\")\n", + " return None\n", + "\n", + " return importlib.import_module(module_name)\n", + "\n", + "# Test the plugin loader\n", + "for fmt in [\"json\", \"csv\", \"xml\", \"yaml\"]:\n", + " print(f\"Loading '{fmt}':\")\n", + " mod = load_serializer(fmt)\n", + " if mod is not None:\n", + " print(f\" Loaded: {mod.__name__}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import importlib.util\n", + "import types\n", + "\n", + "def get_module_info(name: str) -> dict[str, str | bool | None]:\n", + " \"\"\"Gather comprehensive information about a module.\"\"\"\n", + " info: dict[str, str | bool | None] = {\"name\": name}\n", + "\n", + " spec = importlib.util.find_spec(name)\n", + " if spec is None:\n", + " info[\"exists\"] = False\n", + " return info\n", + "\n", + " info[\"exists\"] = True\n", + " info[\"origin\"] = spec.origin\n", + " info[\"is_package\"] = spec.submodule_search_locations is not None\n", + " info[\"has_location\"] = spec.has_location\n", + " info[\"parent\"] = spec.parent or None\n", + "\n", + " return info\n", + "\n", + "# Inspect several modules\n", + "for name in [\"json\", \"os\", \"os.path\", \"email\", \"math\"]:\n", + " info: dict[str, str | bool | None] = get_module_info(name)\n", + " print(f\"{name}:\")\n", + " for key, value in info.items():\n", + " if key != \"name\":\n", + " print(f\" {key}: {value}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Core Functions\n", + "- **`importlib.import_module(name)`**: Import a module by string name, equivalent to `import name`\n", + "- **`importlib.import_module(name, package)`**: Import using relative dotted paths with a package anchor\n", + "- **`importlib.reload(module)`**: Re-execute a module's code and update the existing module object\n", + "- **`importlib.util.find_spec(name)`**: Check if a module exists without importing it\n", + "\n", + "### Standard Module Attributes\n", + "- **`__name__`**: Fully qualified module name (e.g., `\"json\"`, `\"os.path\"`)\n", + "- **`__file__`**: Path to the module's source file (may be `None` for built-in modules)\n", + "- **`__package__`**: The package this module belongs to (empty string for top-level modules)\n", + "- **`__loader__`**: The loader object that loaded the module\n", + "- **`__spec__`**: A `ModuleSpec` with metadata about how the module was found and loaded\n", + "\n", + "### ModuleSpec Attributes\n", + "- **`spec.name`**: The module's name\n", + "- **`spec.origin`**: Where the module was loaded from (file path or built-in)\n", + "- **`spec.loader`**: The loader responsible for this module\n", + "- **`spec.parent`**: The parent package name\n", + "- **`spec.submodule_search_locations`**: Not `None` if this module is a package\n", + "\n", + "### Important Notes\n", + "- `import_module` returns the same cached object as a regular `import` statement\n", + "- `reload` updates the module in place -- the object identity (`id()`) stays the same\n", + "- `find_spec` returns `None` for modules that cannot be found\n", + "- Built-in modules (like `math`) may not have a `__file__` attribute" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_40/02_custom_importers.ipynb b/src/chapter_40/02_custom_importers.ipynb new file mode 100644 index 0000000..541c4ec --- /dev/null +++ b/src/chapter_40/02_custom_importers.ipynb @@ -0,0 +1,607 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 40: Custom Importers and the Import Machinery\n", + "\n", + "This notebook dives into the internals of Python's import machinery. You will learn how `sys.meta_path` finders work, how to use `find_spec` and `module_from_spec` to manually walk through the import process, and how `sys.modules` and `sys.path` govern module resolution.\n", + "\n", + "## Key Concepts\n", + "- **`sys.meta_path`**: The list of finder objects that Python consults when importing\n", + "- **`sys.path`**: The list of directories and zip files searched for modules\n", + "- **`importlib.util.find_spec`**: Locate a module's spec without importing it\n", + "- **`importlib.util.module_from_spec`**: Create a new module object from a spec\n", + "- **`sys.modules`**: The cache of all previously imported modules\n", + "- **Custom finders and loaders**: How to hook into the import system" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: The `sys.meta_path` Finders\n", + "\n", + "When Python encounters an `import` statement, it iterates through `sys.meta_path` -- a list of finder objects. Each finder has a `find_spec` (or older `find_module`) method that is called to locate the requested module." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# sys.meta_path contains the finders Python uses\n", + "print(f\"Number of finders: {len(sys.meta_path)}\")\n", + "print(f\"Type: {type(sys.meta_path)}\")\n", + "print()\n", + "\n", + "for i, finder in enumerate(sys.meta_path):\n", + " finder_type: str = type(finder).__name__\n", + " has_find_spec: bool = hasattr(finder, \"find_spec\")\n", + " has_find_module: bool = hasattr(finder, \"find_module\")\n", + " print(f\" [{i}] {finder_type}\")\n", + " print(f\" has find_spec: {has_find_spec}\")\n", + " print(f\" has find_module: {has_find_module}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# The three standard finders and what they handle\n", + "# 1. BuiltinImporter: handles built-in C modules (sys, builtins, etc.)\n", + "# 2. FrozenImporter: handles frozen modules (used during interpreter startup)\n", + "# 3. PathFinder: searches sys.path for .py files and packages\n", + "\n", + "for finder in sys.meta_path:\n", + " name: str = type(finder).__name__\n", + " module: str = type(finder).__module__\n", + " print(f\"{name} (from {module})\")\n", + " \n", + " # Show the class docstring if available\n", + " doc: str = (type(finder).__doc__ or \"No docstring\").split(\"\\n\")[0]\n", + " print(f\" -> {doc}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: The `sys.path` Search Path\n", + "\n", + "`sys.path` is a list of strings specifying where the `PathFinder` looks for modules. It includes the current directory, installed package locations, and any directories added at runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# sys.path is a mutable list of directory strings\n", + "print(f\"Type: {type(sys.path).__name__}\")\n", + "print(f\"Length: {len(sys.path)} entries\")\n", + "print()\n", + "\n", + "# Display each entry\n", + "for i, path in enumerate(sys.path):\n", + " label: str = \"(empty string = cwd)\" if path == \"\" else \"\"\n", + " print(f\" [{i:2d}] {path!r} {label}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib\n", + "import importlib.util\n", + "import tempfile\n", + "import os\n", + "\n", + "# Demonstrate that modifying sys.path affects what can be imported\n", + "tmp_dir: str = tempfile.mkdtemp()\n", + "mod_file: str = os.path.join(tmp_dir, \"path_demo_mod.py\")\n", + "\n", + "with open(mod_file, \"w\") as f:\n", + " f.write(\"GREETING: str = 'Hello from sys.path!'\\n\")\n", + "\n", + "# Before adding to sys.path: module is not findable\n", + "spec_before = importlib.util.find_spec(\"path_demo_mod\")\n", + "print(f\"Before adding to sys.path: find_spec = {spec_before}\")\n", + "\n", + "# Add our directory to sys.path\n", + "sys.path.insert(0, tmp_dir)\n", + "\n", + "# Now it is findable\n", + "spec_after = importlib.util.find_spec(\"path_demo_mod\")\n", + "print(f\"After adding to sys.path: find_spec = {spec_after}\")\n", + "\n", + "# Import and use it\n", + "mod = importlib.import_module(\"path_demo_mod\")\n", + "print(f\"GREETING = {mod.GREETING}\")\n", + "\n", + "# Clean up\n", + "sys.path.remove(tmp_dir)\n", + "del sys.modules[\"path_demo_mod\"]\n", + "os.remove(mod_file)\n", + "os.rmdir(tmp_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Using `find_spec` to Locate Modules\n", + "\n", + "`importlib.util.find_spec` is the modern way to locate a module without importing it. It returns a `ModuleSpec` if the module is found, or `None` if it cannot be located." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "\n", + "# find_spec for an existing module\n", + "spec = importlib.util.find_spec(\"json\")\n", + "\n", + "if spec is not None:\n", + " print(f\"Name: {spec.name}\")\n", + " print(f\"Origin: {spec.origin}\")\n", + " print(f\"Loader: {type(spec.loader).__name__}\")\n", + " print(f\"Parent: {spec.parent!r}\")\n", + "else:\n", + " print(\"Module not found\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "\n", + "# find_spec returns None for nonexistent modules\n", + "spec_missing = importlib.util.find_spec(\"nonexistent_module_xyz_123\")\n", + "print(f\"find_spec('nonexistent_module_xyz_123'): {spec_missing}\")\n", + "print(f\"Result is None: {spec_missing is None}\")\n", + "print()\n", + "\n", + "# Compare built-in vs file-based modules\n", + "for name in [\"sys\", \"json\", \"os.path\"]:\n", + " spec = importlib.util.find_spec(name)\n", + " if spec is not None:\n", + " origin: str = spec.origin or \"(no origin)\"\n", + " loader_name: str = type(spec.loader).__name__\n", + " print(f\"{name:>10}: loader={loader_name}, origin={origin}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Creating Modules from Specs\n", + "\n", + "`importlib.util.module_from_spec` creates a new, uninitialized module object from a `ModuleSpec`. This lets you control each step of the import process manually." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "import types\n", + "\n", + "# Step 1: Find the spec for the json module\n", + "spec = importlib.util.find_spec(\"json\")\n", + "print(f\"Step 1 - Found spec: {spec is not None}\")\n", + "print(f\" spec.name = {spec.name}\")\n", + "print(f\" spec.loader = {spec.loader}\")\n", + "print()\n", + "\n", + "# Step 2: Create an empty module from the spec\n", + "if spec is not None:\n", + " module: types.ModuleType = importlib.util.module_from_spec(spec)\n", + " print(f\"Step 2 - Created module: {module}\")\n", + " print(f\" type: {type(module).__name__}\")\n", + " print(f\" __name__: {module.__name__}\")\n", + " print(f\" isinstance(module, ModuleType): {isinstance(module, types.ModuleType)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.util\n", + "import types\n", + "\n", + "# Full manual import process: find, create, execute\n", + "spec = importlib.util.find_spec(\"json\")\n", + "\n", + "if spec is not None and spec.loader is not None:\n", + " # Create the module\n", + " module: types.ModuleType = importlib.util.module_from_spec(spec)\n", + " print(f\"Before exec_module:\")\n", + " print(f\" has 'dumps': {hasattr(module, 'dumps')}\")\n", + " print()\n", + "\n", + " # Execute the module code to populate its namespace\n", + " spec.loader.exec_module(module)\n", + " print(f\"After exec_module:\")\n", + " print(f\" has 'dumps': {hasattr(module, 'dumps')}\")\n", + " print(f\" has 'loads': {hasattr(module, 'loads')}\")\n", + " \n", + " # The module works normally\n", + " result: str = module.dumps({\"key\": \"value\"})\n", + " print(f\"\\n module.dumps({{'key': 'value'}}) = {result}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: The `sys.modules` Cache\n", + "\n", + "`sys.modules` is a dictionary that caches every module that has been imported. When you import a module, Python first checks this cache. If the module is already there, it returns the cached version instead of loading it again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import json\n", + "\n", + "# sys.modules is a dict mapping names to module objects\n", + "print(f\"Type: {type(sys.modules).__name__}\")\n", + "print(f\"Total cached modules: {len(sys.modules)}\")\n", + "print()\n", + "\n", + "# Check that json is in the cache\n", + "print(f\"'json' in sys.modules: {'json' in sys.modules}\")\n", + "print(f\"sys.modules['json'] is json: {sys.modules['json'] is json}\")\n", + "print()\n", + "\n", + "# Show some of the cached module names\n", + "cached_names: list[str] = sorted(sys.modules.keys())\n", + "print(f\"First 15 cached modules:\")\n", + "for name in cached_names[:15]:\n", + " print(f\" {name}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib\n", + "import types\n", + "\n", + "# Demonstrate that importing returns the cached object\n", + "import json\n", + "json_id_1: int = id(json)\n", + "\n", + "# Importing again returns the exact same object\n", + "json_again: types.ModuleType = importlib.import_module(\"json\")\n", + "json_id_2: int = id(json_again)\n", + "\n", + "print(f\"First import id: {json_id_1}\")\n", + "print(f\"Second import id: {json_id_2}\")\n", + "print(f\"Same object: {json_id_1 == json_id_2}\")\n", + "print()\n", + "\n", + "# You can remove a module from the cache to force re-import\n", + "# (This is generally not recommended in production code)\n", + "print(f\"'json' in sys.modules before del: {'json' in sys.modules}\")\n", + "cached_json: types.ModuleType = sys.modules[\"json\"]\n", + "del sys.modules[\"json\"]\n", + "print(f\"'json' in sys.modules after del: {'json' in sys.modules}\")\n", + "\n", + "# Re-import creates a new module object\n", + "import json\n", + "print(f\"\\nAfter re-import, same object? {id(json) == json_id_1}\")\n", + "print(f\"'json' back in sys.modules: {'json' in sys.modules}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 6: Writing a Custom Finder\n", + "\n", + "You can add your own finder to `sys.meta_path` to intercept imports. A finder must implement a `find_spec` method that returns a `ModuleSpec` or `None`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib\n", + "import importlib.abc\n", + "import importlib.machinery\n", + "import types\n", + "from typing import Sequence\n", + "\n", + "\n", + "class LoggingFinder(importlib.abc.MetaPathFinder):\n", + " \"\"\"A finder that logs import attempts without handling them.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.log: list[str] = []\n", + "\n", + " def find_spec(\n", + " self,\n", + " fullname: str,\n", + " path: Sequence[str] | None,\n", + " target: types.ModuleType | None = None,\n", + " ) -> importlib.machinery.ModuleSpec | None:\n", + " \"\"\"Log the import attempt and return None to pass to next finder.\"\"\"\n", + " self.log.append(fullname)\n", + " # Return None so the normal import machinery handles it\n", + " return None\n", + "\n", + "\n", + "# Install our logging finder\n", + "logger: LoggingFinder = LoggingFinder()\n", + "sys.meta_path.insert(0, logger)\n", + "\n", + "try:\n", + " # These imports will be logged\n", + " import csv\n", + " import html\n", + " import html.parser\n", + "\n", + " print(\"Import attempts logged:\")\n", + " for name in logger.log:\n", + " print(f\" {name}\")\n", + "finally:\n", + " # Always clean up: remove our finder\n", + " sys.meta_path.remove(logger)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib\n", + "import importlib.abc\n", + "import importlib.machinery\n", + "import importlib.util\n", + "import types\n", + "from typing import Sequence\n", + "\n", + "\n", + "class VirtualModuleFinder(importlib.abc.MetaPathFinder):\n", + " \"\"\"A finder that creates virtual modules on the fly.\"\"\"\n", + "\n", + " PREFIX: str = \"virtual_\"\n", + "\n", + " def find_spec(\n", + " self,\n", + " fullname: str,\n", + " path: Sequence[str] | None,\n", + " target: types.ModuleType | None = None,\n", + " ) -> importlib.machinery.ModuleSpec | None:\n", + " if fullname.startswith(self.PREFIX):\n", + " return importlib.machinery.ModuleSpec(\n", + " fullname,\n", + " loader=VirtualModuleLoader(),\n", + " )\n", + " return None\n", + "\n", + "\n", + "class VirtualModuleLoader(importlib.abc.Loader):\n", + " \"\"\"A loader that populates virtual modules.\"\"\"\n", + "\n", + " def exec_module(self, module: types.ModuleType) -> None:\n", + " \"\"\"Set up the virtual module's namespace.\"\"\"\n", + " module.MESSAGE = f\"I am a virtual module named '{module.__name__}'\"\n", + " module.created_by = \"VirtualModuleLoader\"\n", + "\n", + "\n", + "# Install the virtual module finder\n", + "finder: VirtualModuleFinder = VirtualModuleFinder()\n", + "sys.meta_path.insert(0, finder)\n", + "\n", + "try:\n", + " # Import virtual modules that do not exist as files\n", + " virtual_hello: types.ModuleType = importlib.import_module(\"virtual_hello\")\n", + " virtual_world: types.ModuleType = importlib.import_module(\"virtual_world\")\n", + "\n", + " print(f\"virtual_hello.MESSAGE: {virtual_hello.MESSAGE}\")\n", + " print(f\"virtual_world.MESSAGE: {virtual_world.MESSAGE}\")\n", + " print(f\"created_by: {virtual_hello.created_by}\")\n", + "finally:\n", + " sys.meta_path.remove(finder)\n", + " # Clean up sys.modules\n", + " for key in list(sys.modules.keys()):\n", + " if key.startswith(\"virtual_\"):\n", + " del sys.modules[key]" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 7: The Complete Import Protocol\n", + "\n", + "Putting it all together: here is the full sequence of steps Python follows when you write `import something`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib\n", + "import importlib.util\n", + "import types\n", + "\n", + "def manual_import(name: str) -> types.ModuleType:\n", + " \"\"\"Walk through the import protocol step by step.\"\"\"\n", + " # Step 1: Check sys.modules cache\n", + " if name in sys.modules:\n", + " print(f\" Step 1: Found '{name}' in sys.modules cache\")\n", + " return sys.modules[name]\n", + "\n", + " print(f\" Step 1: '{name}' not in cache\")\n", + "\n", + " # Step 2: Find the module spec using finders in sys.meta_path\n", + " spec = importlib.util.find_spec(name)\n", + " if spec is None:\n", + " raise ModuleNotFoundError(f\"No module named '{name}'\")\n", + " print(f\" Step 2: Found spec via {type(spec.loader).__name__}\")\n", + "\n", + " # Step 3: Create the module object\n", + " module: types.ModuleType = importlib.util.module_from_spec(spec)\n", + " print(f\" Step 3: Created module object\")\n", + "\n", + " # Step 4: Add to sys.modules BEFORE executing\n", + " # (This prevents infinite loops with circular imports)\n", + " sys.modules[name] = module\n", + " print(f\" Step 4: Added to sys.modules\")\n", + "\n", + " # Step 5: Execute the module code\n", + " if spec.loader is not None:\n", + " spec.loader.exec_module(module)\n", + " print(f\" Step 5: Executed module code\")\n", + "\n", + " return module\n", + "\n", + "\n", + "# Remove 'textwrap' from cache to demonstrate full import\n", + "if \"textwrap\" in sys.modules:\n", + " del sys.modules[\"textwrap\"]\n", + "\n", + "print(\"Importing 'textwrap' manually:\")\n", + "tw: types.ModuleType = manual_import(\"textwrap\")\n", + "print(f\"\\nResult: {tw.shorten('Hello World, this is a test', width=20)}\")\n", + "\n", + "print(\"\\nImporting 'textwrap' again (cached):\")\n", + "tw2: types.ModuleType = manual_import(\"textwrap\")\n", + "print(f\"Same object: {tw is tw2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib.util\n", + "\n", + "# Show which finder handles which module type\n", + "test_modules: list[str] = [\"sys\", \"_frozen_importlib\", \"json\", \"os.path\"]\n", + "\n", + "for name in test_modules:\n", + " spec = importlib.util.find_spec(name)\n", + " if spec is not None:\n", + " loader_type: str = type(spec.loader).__name__\n", + " origin: str = spec.origin or \"(built-in)\"\n", + " print(f\"{name:>25}: loader={loader_type:>20}, origin={origin}\")\n", + " else:\n", + " print(f\"{name:>25}: not found\")" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### The Import Protocol\n", + "1. **Check `sys.modules`**: If the module is cached, return it immediately\n", + "2. **Search `sys.meta_path`**: Each finder's `find_spec` is called until one returns a `ModuleSpec`\n", + "3. **Create module**: `module_from_spec(spec)` creates an empty module object\n", + "4. **Cache early**: The module is added to `sys.modules` before execution (prevents circular import loops)\n", + "5. **Execute**: `spec.loader.exec_module(module)` runs the module's code\n", + "\n", + "### Key Objects\n", + "- **`sys.meta_path`**: List of finders -- `BuiltinImporter`, `FrozenImporter`, `PathFinder` by default\n", + "- **`sys.path`**: List of directories the `PathFinder` searches for `.py` files and packages\n", + "- **`sys.modules`**: Dict cache of `{name: module}` for all imported modules\n", + "- **`ModuleSpec`**: Describes a module's name, origin, loader, and whether it is a package\n", + "\n", + "### Key Functions\n", + "- **`importlib.util.find_spec(name)`**: Locate a module without importing it (returns `None` if not found)\n", + "- **`importlib.util.module_from_spec(spec)`**: Create a new `ModuleType` from a spec\n", + "- **`spec.loader.exec_module(module)`**: Execute module code to populate its namespace\n", + "\n", + "### Custom Importers\n", + "- Implement `importlib.abc.MetaPathFinder` with a `find_spec` method\n", + "- Implement `importlib.abc.Loader` with an `exec_module` method\n", + "- Insert your finder into `sys.meta_path` and always clean up when done" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_40/03_package_utilities.ipynb b/src/chapter_40/03_package_utilities.ipynb new file mode 100644 index 0000000..88beda5 --- /dev/null +++ b/src/chapter_40/03_package_utilities.ipynb @@ -0,0 +1,560 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 40: Package Utilities and Metadata\n", + "\n", + "This notebook covers Python's tools for discovering, inspecting, and querying metadata about installed packages and modules. You will learn to enumerate available modules, distinguish packages from modules, query installed package versions, and identify built-in modules.\n", + "\n", + "## Key Concepts\n", + "- **`pkgutil.iter_modules`**: Enumerate importable modules and packages\n", + "- **`importlib.metadata`**: Query installed package versions and distributions\n", + "- **`PackageNotFoundError`**: Raised when querying metadata for a missing package\n", + "- **Package vs. module**: Packages have a `__path__` attribute, plain modules do not\n", + "- **`sys.builtin_module_names`**: Tuple of modules compiled into the interpreter" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Enumerating Modules with `pkgutil`\n", + "\n", + "`pkgutil.iter_modules` yields information about all importable modules (and packages) found on the given search path. When called with no arguments, it searches `sys.path`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import pkgutil\n", + "\n", + "# iter_modules() yields ModuleInfo objects for all importable modules\n", + "all_modules: list[pkgutil.ModuleInfo] = list(pkgutil.iter_modules())\n", + "\n", + "print(f\"Total importable modules/packages: {len(all_modules)}\")\n", + "print(f\"Type of each item: {type(all_modules[0]).__name__}\")\n", + "print()\n", + "\n", + "# Show the first 10\n", + "print(\"First 10 modules:\")\n", + "for info in all_modules[:10]:\n", + " kind: str = \"package\" if info.ispkg else \"module\"\n", + " print(f\" {info.name:>30} ({kind})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import pkgutil\n", + "\n", + "# Each ModuleInfo has: module_finder, name, ispkg\n", + "first_info: pkgutil.ModuleInfo = list(pkgutil.iter_modules())[0]\n", + "\n", + "print(f\"ModuleInfo attributes:\")\n", + "print(f\" name: {first_info.name}\")\n", + "print(f\" ispkg: {first_info.ispkg}\")\n", + "print(f\" has 'name' attr: {hasattr(first_info, 'name')}\")\n", + "print(f\" has 'ispkg' attr: {hasattr(first_info, 'ispkg')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import pkgutil\n", + "\n", + "# Count how many are packages vs plain modules\n", + "all_modules: list[pkgutil.ModuleInfo] = list(pkgutil.iter_modules())\n", + "\n", + "packages: list[str] = [m.name for m in all_modules if m.ispkg]\n", + "modules: list[str] = [m.name for m in all_modules if not m.ispkg]\n", + "\n", + "print(f\"Packages: {len(packages)}\")\n", + "print(f\"Modules: {len(modules)}\")\n", + "print()\n", + "\n", + "# Show some well-known packages\n", + "well_known: list[str] = [\"json\", \"os\", \"email\", \"http\", \"unittest\", \"xml\"]\n", + "print(\"Well-known names found as packages:\")\n", + "for name in well_known:\n", + " found: bool = name in packages\n", + " print(f\" {name:>12}: {'package' if found else 'not a package (or module)'}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 2: Iterating Submodules of a Package\n", + "\n", + "You can pass a package's `__path__` to `pkgutil.iter_modules` to list only the submodules within that specific package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import pkgutil\n", + "import email\n", + "\n", + "# List all submodules inside the 'email' package\n", + "email_submodules: list[pkgutil.ModuleInfo] = list(\n", + " pkgutil.iter_modules(email.__path__)\n", + ")\n", + "\n", + "print(f\"Submodules in 'email' package: {len(email_submodules)}\")\n", + "print()\n", + "\n", + "for info in email_submodules:\n", + " kind: str = \"package\" if info.ispkg else \"module\"\n", + " print(f\" email.{info.name:>20} ({kind})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import pkgutil\n", + "import json\n", + "import http\n", + "\n", + "# Compare: json is a package (has __path__), http is also a package\n", + "for mod_name, mod in [(\"json\", json), (\"http\", http)]:\n", + " if hasattr(mod, \"__path__\"):\n", + " submodules: list[pkgutil.ModuleInfo] = list(\n", + " pkgutil.iter_modules(mod.__path__)\n", + " )\n", + " names: list[str] = [m.name for m in submodules]\n", + " print(f\"{mod_name} submodules: {names}\")\n", + " else:\n", + " print(f\"{mod_name} is not a package (no __path__)\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 3: Package vs. Module\n", + "\n", + "The key distinction between a package and a plain module is the `__path__` attribute. Packages are directories with an `__init__.py` and have `__path__` set. Plain modules are single `.py` files (or built-in) and do not have `__path__`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import email\n", + "import math\n", + "import json\n", + "import os\n", + "\n", + "# Check __path__ to determine package vs module\n", + "test_modules: list[tuple[str, object]] = [\n", + " (\"email\", email),\n", + " (\"math\", math),\n", + " (\"json\", json),\n", + " (\"os\", os),\n", + "]\n", + "\n", + "for name, mod in test_modules:\n", + " has_path: bool = hasattr(mod, \"__path__\")\n", + " kind: str = \"package\" if has_path else \"module\"\n", + " print(f\"{name:>8}: {kind:>7} (has __path__: {has_path})\")\n", + " if has_path:\n", + " print(f\" __path__ = {mod.__path__}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import types\n", + "\n", + "def is_package(module_name: str) -> bool:\n", + " \"\"\"Check if a module name refers to a package.\"\"\"\n", + " mod: types.ModuleType = importlib.import_module(module_name)\n", + " return hasattr(mod, \"__path__\")\n", + "\n", + "# Test several modules\n", + "names: list[str] = [\"email\", \"math\", \"json\", \"http\", \"os\", \"sys\", \"xml\"]\n", + "\n", + "for name in names:\n", + " result: bool = is_package(name)\n", + " print(f\"{name:>8}: is_package = {result}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 4: Built-in Module Names\n", + "\n", + "`sys.builtin_module_names` is a tuple of strings listing all modules that are compiled directly into the Python interpreter. These modules have no `.py` file -- they are implemented in C." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# sys.builtin_module_names is a tuple of strings\n", + "builtin_names: tuple[str, ...] = sys.builtin_module_names\n", + "\n", + "print(f\"Type: {type(builtin_names).__name__}\")\n", + "print(f\"Count: {len(builtin_names)} built-in modules\")\n", + "print()\n", + "\n", + "# Check for well-known built-ins\n", + "expected_builtins: list[str] = [\"sys\", \"builtins\", \"_io\", \"math\", \"json\"]\n", + "\n", + "for name in expected_builtins:\n", + " is_builtin: bool = name in builtin_names\n", + " print(f\" {name:>12} is built-in: {is_builtin}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Print all built-in module names sorted\n", + "print(\"All built-in module names:\")\n", + "for i, name in enumerate(sorted(sys.builtin_module_names)):\n", + " # Print in columns\n", + " end_char: str = \"\\n\" if (i + 1) % 4 == 0 else \"\"\n", + " print(f\" {name:<25}\", end=end_char)\n", + "print() # Final newline" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 5: Querying Package Versions with `importlib.metadata`\n", + "\n", + "The `importlib.metadata` module (added in Python 3.8) lets you query metadata about installed distribution packages -- version strings, entry points, and more." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.metadata\n", + "\n", + "# Get the version of an installed package\n", + "pip_version: str = importlib.metadata.version(\"pip\")\n", + "\n", + "print(f\"pip version: {pip_version}\")\n", + "print(f\"Type: {type(pip_version).__name__}\")\n", + "print(f\"Length > 0: {len(pip_version) > 0}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.metadata\n", + "\n", + "# Query versions for several known packages\n", + "package_names: list[str] = [\"pip\", \"setuptools\"]\n", + "\n", + "for name in package_names:\n", + " try:\n", + " version: str = importlib.metadata.version(name)\n", + " print(f\"{name:>15}: version {version}\")\n", + " except importlib.metadata.PackageNotFoundError:\n", + " print(f\"{name:>15}: not installed\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 6: Handling Missing Packages\n", + "\n", + "When you query metadata for a package that is not installed, `importlib.metadata` raises `PackageNotFoundError`. This is the proper way to check whether a distribution package is available." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.metadata\n", + "\n", + "# Trying to get the version of a nonexistent package\n", + "try:\n", + " version: str = importlib.metadata.version(\"nonexistent_package_xyz_123\")\n", + " print(f\"Version: {version}\")\n", + "except importlib.metadata.PackageNotFoundError as e:\n", + " print(f\"PackageNotFoundError raised: {e}\")\n", + " print(f\"Exception type: {type(e).__name__}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.metadata\n", + "\n", + "def get_package_version(name: str) -> str | None:\n", + " \"\"\"Safely get a package's version, returning None if not installed.\"\"\"\n", + " try:\n", + " return importlib.metadata.version(name)\n", + " except importlib.metadata.PackageNotFoundError:\n", + " return None\n", + "\n", + "# Test with real and fake packages\n", + "test_packages: list[str] = [\"pip\", \"setuptools\", \"fake_package_abc\", \"another_missing\"]\n", + "\n", + "for name in test_packages:\n", + " version: str | None = get_package_version(name)\n", + " if version is not None:\n", + " print(f\"{name:>20}: {version}\")\n", + " else:\n", + " print(f\"{name:>20}: not installed\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 7: Package Distributions Mapping\n", + "\n", + "`importlib.metadata.packages_distributions()` returns a mapping from top-level importable package names to the distribution packages that provide them. This is useful for figuring out which PyPI package provides a given import." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.metadata\n", + "\n", + "# Get the mapping of importable names -> distribution packages\n", + "distributions: dict[str, list[str]] = importlib.metadata.packages_distributions()\n", + "\n", + "print(f\"Type: {type(distributions).__name__}\")\n", + "print(f\"Total importable package names mapped: {len(distributions)}\")\n", + "print()\n", + "\n", + "# Show some entries\n", + "count: int = 0\n", + "for pkg_name, dist_names in sorted(distributions.items()):\n", + " if count >= 10:\n", + " break\n", + " print(f\" {pkg_name:>25} -> {dist_names}\")\n", + " count += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib.metadata\n", + "\n", + "def find_distribution(import_name: str) -> list[str]:\n", + " \"\"\"Find which distribution package provides a given importable name.\"\"\"\n", + " distributions: dict[str, list[str]] = importlib.metadata.packages_distributions()\n", + " return distributions.get(import_name, [])\n", + "\n", + "# Look up some importable names\n", + "search_names: list[str] = [\"pip\", \"setuptools\", \"nonexistent_xyz\"]\n", + "\n", + "for name in search_names:\n", + " dists: list[str] = find_distribution(name)\n", + " if dists:\n", + " print(f\"{name:>20} is provided by: {dists}\")\n", + " else:\n", + " print(f\"{name:>20}: no distribution found\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Section 8: Practical Patterns\n", + "\n", + "Combining the tools from this notebook to solve real-world problems: checking dependencies, auditing installed packages, and introspecting the module system." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5b6c7d8", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import importlib\n", + "import importlib.metadata\n", + "import types\n", + "\n", + "def module_report(name: str) -> dict[str, str | bool]:\n", + " \"\"\"Generate a comprehensive report about a module.\"\"\"\n", + " report: dict[str, str | bool] = {\"name\": name}\n", + "\n", + " # Is it a built-in?\n", + " report[\"is_builtin\"] = name in sys.builtin_module_names\n", + "\n", + " # Can we import it?\n", + " try:\n", + " mod: types.ModuleType = importlib.import_module(name)\n", + " report[\"importable\"] = True\n", + " report[\"is_package\"] = hasattr(mod, \"__path__\")\n", + " report[\"has_file\"] = hasattr(mod, \"__file__\") and mod.__file__ is not None\n", + " except ImportError:\n", + " report[\"importable\"] = False\n", + " return report\n", + "\n", + " # Try to get version from metadata\n", + " try:\n", + " report[\"version\"] = importlib.metadata.version(name)\n", + " except importlib.metadata.PackageNotFoundError:\n", + " report[\"version\"] = \"(stdlib or no metadata)\"\n", + "\n", + " return report\n", + "\n", + "# Generate reports for various modules\n", + "for name in [\"sys\", \"json\", \"email\", \"math\", \"pip\"]:\n", + " info: dict[str, str | bool] = module_report(name)\n", + " print(f\"{name}:\")\n", + " for key, value in info.items():\n", + " if key != \"name\":\n", + " print(f\" {key:>14}: {value}\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5c6d7e8", + "metadata": {}, + "outputs": [], + "source": [ + "import pkgutil\n", + "import sys\n", + "\n", + "def count_stdlib_modules() -> dict[str, int]:\n", + " \"\"\"Count stdlib packages and modules.\"\"\"\n", + " all_mods: list[pkgutil.ModuleInfo] = list(pkgutil.iter_modules())\n", + " builtin_set: frozenset[str] = frozenset(sys.builtin_module_names)\n", + "\n", + " stats: dict[str, int] = {\n", + " \"total_importable\": len(all_mods),\n", + " \"packages\": sum(1 for m in all_mods if m.ispkg),\n", + " \"modules\": sum(1 for m in all_mods if not m.ispkg),\n", + " \"builtin_c_modules\": len(builtin_set),\n", + " }\n", + " return stats\n", + "\n", + "stats: dict[str, int] = count_stdlib_modules()\n", + "print(\"Module system statistics:\")\n", + "for key, value in stats.items():\n", + " label: str = key.replace(\"_\", \" \").title()\n", + " print(f\" {label:>25}: {value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c5d6e7f8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Module Discovery\n", + "- **`pkgutil.iter_modules()`**: Yields `ModuleInfo(module_finder, name, ispkg)` for all importable modules\n", + "- **`pkgutil.iter_modules(path)`**: Pass a package's `__path__` to list only its submodules\n", + "- **`ModuleInfo.name`**: The module's importable name\n", + "- **`ModuleInfo.ispkg`**: `True` if the entry is a package, `False` for a plain module\n", + "\n", + "### Package Metadata\n", + "- **`importlib.metadata.version(name)`**: Returns the version string for an installed distribution\n", + "- **`importlib.metadata.packages_distributions()`**: Maps importable names to their distribution packages\n", + "- **`importlib.metadata.PackageNotFoundError`**: Raised when a distribution is not installed\n", + "\n", + "### Package vs. Module\n", + "- **Packages** have a `__path__` attribute (they are directories with `__init__.py`)\n", + "- **Modules** do not have `__path__` (they are single files or built-in)\n", + "- Use `hasattr(mod, '__path__')` to distinguish between them\n", + "\n", + "### Built-in Modules\n", + "- **`sys.builtin_module_names`**: A `tuple` of module names compiled into the interpreter (C modules)\n", + "- These modules have no `.py` file and are always available\n", + "- Common built-ins include `sys`, `builtins`, `_io`, and platform-specific modules" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_40/README.md b/src/chapter_40/README.md new file mode 100644 index 0000000..00c976f --- /dev/null +++ b/src/chapter_40/README.md @@ -0,0 +1,14 @@ +# Chapter 40: Import System Internals + +## Topics Covered + +- importlib: import_module, reload, metadata +- Import hooks: sys.meta_path, finders, loaders +- Package discovery: pkgutil, importlib.metadata +- zipimport and namespace packages + +## Notebooks + +1. **01_importlib_basics.ipynb** - import_module, reload, module inspection +2. **02_custom_importers.ipynb** - sys.meta_path, finders, loaders, import hooks +3. **03_package_utilities.ipynb** - pkgutil, importlib.metadata, zipimport diff --git a/src/chapter_40/__init__.py b/src/chapter_40/__init__.py new file mode 100644 index 0000000..7e9bb91 --- /dev/null +++ b/src/chapter_40/__init__.py @@ -0,0 +1 @@ +"""Chapter 40: Import System Internals.""" diff --git a/src/chapter_41/01_threading_synchronization.ipynb b/src/chapter_41/01_threading_synchronization.ipynb new file mode 100644 index 0000000..bf8aeec --- /dev/null +++ b/src/chapter_41/01_threading_synchronization.ipynb @@ -0,0 +1,593 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 41: Threading Synchronization\n", + "\n", + "This notebook covers Python's threading synchronization primitives from the `threading` module. You will learn how to coordinate access to shared resources using locks, events, semaphores, barriers, and conditions to write correct concurrent programs.\n", + "\n", + "## Key Concepts\n", + "- **`Lock`**: Mutual exclusion -- only one thread can hold the lock at a time\n", + "- **`RLock`**: Reentrant lock that the same thread can acquire multiple times\n", + "- **`Event`**: Simple signaling mechanism between threads\n", + "- **`Semaphore`**: Limits the number of threads accessing a resource concurrently\n", + "- **`Barrier`**: Synchronization point where a fixed number of threads must arrive before any proceed\n", + "- **`Condition`**: Advanced coordination combining a lock with wait/notify signaling" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Lock -- Mutual Exclusion\n", + "\n", + "A `Lock` ensures that only one thread at a time can execute a critical section. Without a lock, concurrent increments to a shared counter can produce incorrect results due to race conditions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Shared mutable state\n", + "lock: threading.Lock = threading.Lock()\n", + "counter: list[int] = [0]\n", + "\n", + "\n", + "def increment() -> None:\n", + " \"\"\"Increment a shared counter 1000 times with lock protection.\"\"\"\n", + " for _ in range(1000):\n", + " with lock:\n", + " counter[0] += 1\n", + "\n", + "\n", + "# Launch 4 threads, each incrementing 1000 times\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=increment) for _ in range(4)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "print(f\"Final counter value: {counter[0]}\")\n", + "print(f\"Expected: 4000\")\n", + "print(f\"Correct: {counter[0] == 4000}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Lock can also be acquired and released manually\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "# acquire() returns True if the lock was successfully acquired\n", + "acquired: bool = lock.acquire()\n", + "print(f\"Lock acquired: {acquired}\")\n", + "\n", + "# Non-blocking acquire attempt while lock is held\n", + "second_attempt: bool = lock.acquire(blocking=False)\n", + "print(f\"Second acquire (non-blocking): {second_attempt}\")\n", + "\n", + "# Release the lock\n", + "lock.release()\n", + "print(f\"Lock released\")\n", + "\n", + "# Now it can be acquired again\n", + "third_attempt: bool = lock.acquire(blocking=False)\n", + "print(f\"Third acquire after release: {third_attempt}\")\n", + "lock.release()" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: RLock -- Reentrant Locking\n", + "\n", + "An `RLock` (reentrant lock) can be acquired multiple times by the same thread without deadlocking. This is useful when a function that holds a lock calls another function that also needs the same lock." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# RLock allows the same thread to acquire it multiple times\n", + "rlock: threading.RLock = threading.RLock()\n", + "\n", + "with rlock:\n", + " print(\"First level acquired\")\n", + " with rlock: # This would deadlock with a regular Lock!\n", + " print(\"Second level acquired (reentrant)\")\n", + " acquired: bool = True\n", + "\n", + "print(f\"Successfully nested: {acquired}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Practical RLock example: nested method calls\n", + "class SafeCounter:\n", + " \"\"\"A thread-safe counter using RLock for reentrant access.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._lock: threading.RLock = threading.RLock()\n", + " self._value: int = 0\n", + "\n", + " def increment(self) -> None:\n", + " \"\"\"Increment the counter by 1.\"\"\"\n", + " with self._lock:\n", + " self._value += 1\n", + "\n", + " def add(self, n: int) -> None:\n", + " \"\"\"Add n to the counter by calling increment n times.\"\"\"\n", + " with self._lock: # Outer lock\n", + " for _ in range(n):\n", + " self.increment() # Inner lock -- needs RLock!\n", + "\n", + " def get_value(self) -> int:\n", + " \"\"\"Return the current counter value.\"\"\"\n", + " with self._lock:\n", + " return self._value\n", + "\n", + "\n", + "counter: SafeCounter = SafeCounter()\n", + "counter.add(5)\n", + "print(f\"Counter value after add(5): {counter.get_value()}\")\n", + "\n", + "counter.increment()\n", + "print(f\"Counter value after increment(): {counter.get_value()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Event -- Thread Signaling\n", + "\n", + "An `Event` is a simple signaling mechanism. One thread can wait for an event to be set by another thread. This is useful for coordinating startup, shutdown, or notification between threads." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "event: threading.Event = threading.Event()\n", + "result: list[str] = []\n", + "\n", + "\n", + "def waiter() -> None:\n", + " \"\"\"Wait for the event to be set, then record completion.\"\"\"\n", + " event.wait(timeout=5)\n", + " result.append(\"done\")\n", + "\n", + "\n", + "t: threading.Thread = threading.Thread(target=waiter)\n", + "t.start()\n", + "\n", + "# Signal the waiting thread\n", + "event.set()\n", + "t.join()\n", + "\n", + "print(f\"Result: {result}\")\n", + "print(f\"Event is set: {event.is_set()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "import time\n", + "\n", + "# Event can be cleared and reused\n", + "event: threading.Event = threading.Event()\n", + "log: list[str] = []\n", + "\n", + "\n", + "def setup_worker() -> None:\n", + " \"\"\"Simulate setup, then signal readiness.\"\"\"\n", + " log.append(\"setup started\")\n", + " time.sleep(0.05) # Simulate setup work\n", + " log.append(\"setup complete\")\n", + " event.set()\n", + "\n", + "\n", + "def main_worker() -> None:\n", + " \"\"\"Wait for setup to finish, then proceed.\"\"\"\n", + " event.wait(timeout=5)\n", + " log.append(\"main started\")\n", + "\n", + "\n", + "t1: threading.Thread = threading.Thread(target=setup_worker)\n", + "t2: threading.Thread = threading.Thread(target=main_worker)\n", + "t2.start()\n", + "t1.start()\n", + "t1.join()\n", + "t2.join()\n", + "\n", + "print(f\"Execution log: {log}\")\n", + "print(f\"Setup completed before main: {'setup complete' in log and log.index('setup complete') < log.index('main started')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Semaphore -- Limiting Concurrent Access\n", + "\n", + "A `Semaphore` limits the number of threads that can concurrently access a resource. It maintains an internal counter that is decremented on each `acquire()` and incremented on each `release()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "import time\n", + "\n", + "# Allow at most 2 concurrent workers\n", + "sem: threading.Semaphore = threading.Semaphore(2)\n", + "max_concurrent: list[int] = [0]\n", + "current: list[int] = [0]\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def worker(worker_id: int) -> None:\n", + " \"\"\"Simulate work while tracking concurrent access.\"\"\"\n", + " with sem:\n", + " with lock:\n", + " current[0] += 1\n", + " if current[0] > max_concurrent[0]:\n", + " max_concurrent[0] = current[0]\n", + " time.sleep(0.01) # Simulate work\n", + " with lock:\n", + " current[0] -= 1\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=worker, args=(i,)) for i in range(6)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "print(f\"Max concurrent workers: {max_concurrent[0]}\")\n", + "print(f\"Semaphore limit respected: {max_concurrent[0] <= 2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "import time\n", + "\n", + "# BoundedSemaphore prevents releasing more than acquired\n", + "bsem: threading.BoundedSemaphore = threading.BoundedSemaphore(3)\n", + "access_log: list[str] = []\n", + "log_lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def limited_worker(worker_id: int) -> None:\n", + " \"\"\"Simulate a connection pool with bounded semaphore.\"\"\"\n", + " with bsem:\n", + " with log_lock:\n", + " access_log.append(f\"worker-{worker_id} entered\")\n", + " time.sleep(0.01)\n", + " with log_lock:\n", + " access_log.append(f\"worker-{worker_id} exited\")\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=limited_worker, args=(i,)) for i in range(5)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "print(f\"Total access events: {len(access_log)}\")\n", + "print(f\"All workers completed: {len([e for e in access_log if 'exited' in e]) == 5}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: Barrier -- Synchronizing Thread Groups\n", + "\n", + "A `Barrier` ensures that a fixed number of threads all reach a synchronization point before any of them proceed. This is useful for phased computations where all workers must finish one phase before starting the next." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# All 3 threads must arrive at the barrier before any proceed\n", + "barrier: threading.Barrier = threading.Barrier(3)\n", + "arrivals: list[int] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def worker(worker_id: int) -> None:\n", + " \"\"\"Wait at barrier, then record arrival.\"\"\"\n", + " barrier.wait()\n", + " with lock:\n", + " arrivals.append(worker_id)\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=worker, args=(i,)) for i in range(3)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "print(f\"Arrivals after barrier: {sorted(arrivals)}\")\n", + "print(f\"All 3 threads passed: {len(arrivals) == 3}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "import time\n", + "\n", + "# Barrier with phases: threads synchronize between phases\n", + "barrier: threading.Barrier = threading.Barrier(3)\n", + "phase_log: list[str] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def phased_worker(worker_id: int) -> None:\n", + " \"\"\"Perform two phases, synchronizing between them.\"\"\"\n", + " # Phase 1\n", + " with lock:\n", + " phase_log.append(f\"worker-{worker_id}-phase-1\")\n", + " barrier.wait()\n", + " # Phase 2 -- all threads have completed phase 1\n", + " with lock:\n", + " phase_log.append(f\"worker-{worker_id}-phase-2\")\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=phased_worker, args=(i,)) for i in range(3)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "# Count phase entries\n", + "phase_1_entries: list[str] = [e for e in phase_log if \"phase-1\" in e]\n", + "phase_2_entries: list[str] = [e for e in phase_log if \"phase-2\" in e]\n", + "\n", + "print(f\"Phase 1 completions: {len(phase_1_entries)}\")\n", + "print(f\"Phase 2 completions: {len(phase_2_entries)}\")\n", + "print(f\"Execution log: {phase_log}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 6: Condition -- Advanced Coordination\n", + "\n", + "A `Condition` combines a lock with the ability to wait for a notification. Threads can `wait()` until another thread calls `notify()` or `notify_all()`. This is more flexible than `Event` because it is paired with a lock and supports multiple notifications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Condition variable for producer-consumer coordination\n", + "condition: threading.Condition = threading.Condition()\n", + "shared_data: list[int] = []\n", + "consumed: list[int] = []\n", + "\n", + "\n", + "def producer() -> None:\n", + " \"\"\"Produce items and notify the consumer.\"\"\"\n", + " for i in range(5):\n", + " with condition:\n", + " shared_data.append(i)\n", + " condition.notify()\n", + "\n", + "\n", + "def consumer() -> None:\n", + " \"\"\"Consume items, waiting for notifications.\"\"\"\n", + " for _ in range(5):\n", + " with condition:\n", + " while not shared_data:\n", + " condition.wait()\n", + " item: int = shared_data.pop(0)\n", + " consumed.append(item)\n", + "\n", + "\n", + "t1: threading.Thread = threading.Thread(target=producer)\n", + "t2: threading.Thread = threading.Thread(target=consumer)\n", + "t2.start()\n", + "t1.start()\n", + "t1.join()\n", + "t2.join()\n", + "\n", + "print(f\"Consumed items: {consumed}\")\n", + "print(f\"All items consumed: {consumed == [0, 1, 2, 3, 4]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Condition with wait_for predicate -- cleaner than a while loop\n", + "condition: threading.Condition = threading.Condition()\n", + "ready: list[bool] = [False]\n", + "message: list[str] = [\"\"]\n", + "\n", + "\n", + "def sender() -> None:\n", + " \"\"\"Set the message and notify.\"\"\"\n", + " with condition:\n", + " message[0] = \"Hello from sender\"\n", + " ready[0] = True\n", + " condition.notify_all()\n", + "\n", + "\n", + "def receiver(receiver_id: int) -> None:\n", + " \"\"\"Wait for the message to be ready.\"\"\"\n", + " with condition:\n", + " condition.wait_for(lambda: ready[0])\n", + " print(f\"Receiver {receiver_id} got: {message[0]}\")\n", + "\n", + "\n", + "receivers: list[threading.Thread] = [\n", + " threading.Thread(target=receiver, args=(i,)) for i in range(3)\n", + "]\n", + "for r in receivers:\n", + " r.start()\n", + "\n", + "sender_thread: threading.Thread = threading.Thread(target=sender)\n", + "sender_thread.start()\n", + "sender_thread.join()\n", + "for r in receivers:\n", + " r.join()\n", + "\n", + "print(f\"All receivers notified successfully\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 7: Choosing the Right Primitive\n", + "\n", + "Each synchronization primitive has a specific use case. Choosing the right one makes your code simpler and more correct." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "# Quick reference for when to use each primitive\n", + "primitives: dict[str, str] = {\n", + " \"Lock\": \"Protect a shared resource from concurrent access\",\n", + " \"RLock\": \"Same as Lock, but allows reentrant (nested) acquisition\",\n", + " \"Event\": \"Signal between threads (one-shot or resettable flag)\",\n", + " \"Semaphore\": \"Limit N concurrent accesses (e.g., connection pool)\",\n", + " \"Barrier\": \"Wait until N threads all reach the same point\",\n", + " \"Condition\": \"Wait for a complex condition with lock + notify\",\n", + "}\n", + "\n", + "for name, use_case in primitives.items():\n", + " print(f\"{name:12s} -> {use_case}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Synchronization Primitives\n", + "- **`Lock`**: Use `with lock:` to protect critical sections; only one thread enters at a time\n", + "- **`RLock`**: Reentrant lock that the same thread can acquire multiple times without deadlock\n", + "- **`Event`**: Call `event.set()` to wake threads waiting on `event.wait()`\n", + "- **`Semaphore(n)`**: Allows up to `n` threads to hold the semaphore concurrently\n", + "- **`Barrier(n)`**: Blocks each thread at `barrier.wait()` until all `n` threads have arrived\n", + "- **`Condition`**: Combines a lock with `wait()` / `notify()` for complex coordination\n", + "\n", + "### Best Practices\n", + "- Always use the `with` statement for locks and semaphores to guarantee release\n", + "- Prefer `RLock` when methods that hold a lock call other methods needing the same lock\n", + "- Use `Event` for simple one-time or resettable signals; use `Condition` for complex predicates\n", + "- `BoundedSemaphore` prevents accidental over-releasing\n", + "- Keep critical sections as short as possible to minimize contention" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_41/02_concurrent_futures.ipynb b/src/chapter_41/02_concurrent_futures.ipynb new file mode 100644 index 0000000..a1668d1 --- /dev/null +++ b/src/chapter_41/02_concurrent_futures.ipynb @@ -0,0 +1,584 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 41: Concurrent Futures\n", + "\n", + "This notebook covers the `concurrent.futures` module, which provides a high-level interface for asynchronous execution using thread pools. You will learn how to submit tasks, collect results, handle exceptions, and process futures as they complete.\n", + "\n", + "## Key Concepts\n", + "- **`ThreadPoolExecutor`**: Manages a pool of worker threads for concurrent execution\n", + "- **`submit()`**: Schedules a callable and returns a `Future` object\n", + "- **`map()`**: Applies a function to an iterable, returning results in order\n", + "- **`Future`**: Represents a pending result with `result()`, `exception()`, and `done()`\n", + "- **`as_completed()`**: Yields futures in the order they finish, not the order submitted" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: ThreadPoolExecutor Basics\n", + "\n", + "`ThreadPoolExecutor` manages a pool of threads and provides methods to submit work. It is used as a context manager so that threads are properly cleaned up when done." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import ThreadPoolExecutor\n", + "\n", + "\n", + "def square(x: int) -> int:\n", + " \"\"\"Return the square of x.\"\"\"\n", + " return x ** 2\n", + "\n", + "\n", + "# Use as a context manager for automatic cleanup\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " future = pool.submit(square, 5)\n", + " result: int = future.result()\n", + "\n", + "print(f\"square(5) = {result}\")\n", + "print(f\"Result type: {type(result).__name__}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import ThreadPoolExecutor\n", + "import time\n", + "\n", + "\n", + "def slow_double(x: int) -> int:\n", + " \"\"\"Simulate a slow computation that doubles x.\"\"\"\n", + " time.sleep(0.05)\n", + " return x * 2\n", + "\n", + "\n", + "# Measure sequential vs concurrent execution\n", + "values: list[int] = [1, 2, 3, 4, 5, 6, 7, 8]\n", + "\n", + "# Sequential\n", + "start: float = time.perf_counter()\n", + "sequential_results: list[int] = [slow_double(v) for v in values]\n", + "sequential_time: float = time.perf_counter() - start\n", + "\n", + "# Concurrent with 4 workers\n", + "start = time.perf_counter()\n", + "with ThreadPoolExecutor(max_workers=4) as pool:\n", + " concurrent_results: list[int] = list(pool.map(slow_double, values))\n", + "concurrent_time: float = time.perf_counter() - start\n", + "\n", + "print(f\"Sequential results: {sequential_results}\")\n", + "print(f\"Concurrent results: {concurrent_results}\")\n", + "print(f\"Sequential time: {sequential_time:.3f}s\")\n", + "print(f\"Concurrent time: {concurrent_time:.3f}s\")\n", + "print(f\"Speedup: {sequential_time / concurrent_time:.1f}x\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: submit() and Future Objects\n", + "\n", + "`submit()` schedules a callable to be executed and immediately returns a `Future` object. The `Future` represents the pending result and provides methods to check status and retrieve the result." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "\n", + "\n", + "def compute(x: int) -> int:\n", + " \"\"\"Return x squared.\"\"\"\n", + " return x ** 2\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " # submit returns a Future immediately\n", + " future: Future[int] = pool.submit(compute, 5)\n", + "\n", + " print(f\"Future type: {type(future).__name__}\")\n", + " print(f\"Is Future instance: {isinstance(future, Future)}\")\n", + "\n", + " # result() blocks until the result is available\n", + " value: int = future.result()\n", + " print(f\"Result: {value}\")\n", + " print(f\"Expected: 25\")\n", + " print(f\"Correct: {value == 25}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "\n", + "\n", + "def greet(name: str) -> str:\n", + " \"\"\"Return a greeting string.\"\"\"\n", + " return f\"Hello, {name}!\"\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " # Submit multiple tasks\n", + " futures: list[Future[str]] = [\n", + " pool.submit(greet, name)\n", + " for name in [\"Alice\", \"Bob\", \"Charlie\"]\n", + " ]\n", + "\n", + " # Collect results in submission order\n", + " results: list[str] = [f.result() for f in futures]\n", + "\n", + "for r in results:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: map() -- Ordered Results\n", + "\n", + "`map()` applies a function to each item in an iterable and returns results in the same order as the input. Unlike `submit()`, it does not return `Future` objects -- it returns the results directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import ThreadPoolExecutor\n", + "\n", + "\n", + "def double(x: int) -> int:\n", + " \"\"\"Return x multiplied by 2.\"\"\"\n", + " return x * 2\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " # map applies the function to each element\n", + " results: list[int] = list(pool.map(double, [1, 2, 3, 4]))\n", + "\n", + "print(f\"Results: {results}\")\n", + "print(f\"Expected: [2, 4, 6, 8]\")\n", + "print(f\"Correct: {results == [2, 4, 6, 8]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import ThreadPoolExecutor\n", + "\n", + "\n", + "def add(a: int, b: int) -> int:\n", + " \"\"\"Return the sum of a and b.\"\"\"\n", + " return a + b\n", + "\n", + "\n", + "# map with multiple iterables (like built-in map)\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " results: list[int] = list(pool.map(add, [1, 2, 3], [10, 20, 30]))\n", + "\n", + "print(f\"Results: {results}\")\n", + "print(f\"Expected: [11, 22, 33]\")\n", + "print(f\"Correct: {results == [11, 22, 33]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: as_completed() -- Process Results as They Finish\n", + "\n", + "`as_completed()` yields `Future` objects in the order they finish, not the order they were submitted. This is useful when you want to process results as soon as they are available." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor, as_completed\n", + "\n", + "\n", + "def square(x: int) -> int:\n", + " \"\"\"Return x squared.\"\"\"\n", + " return x ** 2\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " # Create a dict mapping futures to their input values\n", + " futures: dict[Future[int], int] = {\n", + " pool.submit(square, i): i for i in range(5)\n", + " }\n", + "\n", + " results: list[int] = []\n", + " for future in as_completed(futures):\n", + " input_val: int = futures[future]\n", + " result: int = future.result()\n", + " results.append(result)\n", + " print(f\"square({input_val}) = {result}\")\n", + "\n", + "print(f\"\\nAll results (sorted): {sorted(results)}\")\n", + "print(f\"Expected: [0, 1, 4, 9, 16]\")\n", + "print(f\"Correct: {sorted(results) == [0, 1, 4, 9, 16]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor, as_completed\n", + "import time\n", + "\n", + "\n", + "def variable_work(task_id: int, duration: float) -> str:\n", + " \"\"\"Simulate work with variable duration.\"\"\"\n", + " time.sleep(duration)\n", + " return f\"task-{task_id} ({duration}s)\"\n", + "\n", + "\n", + "tasks: list[tuple[int, float]] = [\n", + " (1, 0.15),\n", + " (2, 0.05),\n", + " (3, 0.10),\n", + "]\n", + "\n", + "with ThreadPoolExecutor(max_workers=3) as pool:\n", + " futures: dict[Future[str], int] = {\n", + " pool.submit(variable_work, tid, dur): tid\n", + " for tid, dur in tasks\n", + " }\n", + "\n", + " print(\"Results in completion order:\")\n", + " for future in as_completed(futures):\n", + " print(f\" Completed: {future.result()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: Future Exception Handling\n", + "\n", + "If a callable raises an exception, the `Future` captures it. You can retrieve the exception with `future.exception()` or let it re-raise when calling `future.result()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "\n", + "\n", + "def failing() -> None:\n", + " \"\"\"A task that always raises an exception.\"\"\"\n", + " raise ValueError(\"task failed\")\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=1) as pool:\n", + " future: Future[None] = pool.submit(failing)\n", + "\n", + " # exception() returns the exception without re-raising\n", + " exc: BaseException | None = future.exception()\n", + " print(f\"Exception is not None: {exc is not None}\")\n", + " print(f\"Exception type: {type(exc).__name__}\")\n", + " print(f\"Exception message: {exc}\")\n", + " print(f\"Is ValueError: {isinstance(exc, ValueError)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "\n", + "\n", + "def maybe_fail(x: int) -> int:\n", + " \"\"\"Fail on negative input, succeed otherwise.\"\"\"\n", + " if x < 0:\n", + " raise ValueError(f\"Negative input: {x}\")\n", + " return x * 10\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=2) as pool:\n", + " inputs: list[int] = [3, -1, 7, -2, 5]\n", + " futures: list[Future[int]] = [pool.submit(maybe_fail, x) for x in inputs]\n", + "\n", + " for i, future in enumerate(futures):\n", + " exc: BaseException | None = future.exception()\n", + " if exc is not None:\n", + " print(f\"Input {inputs[i]:2d} -> ERROR: {exc}\")\n", + " else:\n", + " print(f\"Input {inputs[i]:2d} -> Result: {future.result()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "\n", + "\n", + "def safe_divide(a: float, b: float) -> float:\n", + " \"\"\"Divide a by b.\"\"\"\n", + " return a / b\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=1) as pool:\n", + " future: Future[float] = pool.submit(safe_divide, 10.0, 0.0)\n", + "\n", + " # Calling result() re-raises the exception\n", + " try:\n", + " value: float = future.result()\n", + " print(f\"Result: {value}\")\n", + " except ZeroDivisionError as e:\n", + " print(f\"Caught exception from future.result(): {e}\")\n", + " print(f\"Exception type: {type(e).__name__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 6: Future Status -- done() and cancel()\n", + "\n", + "Futures provide methods to check their status. `done()` returns `True` when the future has completed (either successfully or with an exception). `cancel()` attempts to cancel execution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=1) as pool:\n", + " future: Future[int] = pool.submit(lambda: 42)\n", + "\n", + " # Wait for the result\n", + " result: int = future.result()\n", + " print(f\"Result: {result}\")\n", + " print(f\"Future done: {future.done()}\")\n", + " print(f\"Correct: {result == 42 and future.done()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor\n", + "import time\n", + "\n", + "\n", + "def slow_task() -> str:\n", + " \"\"\"A task that takes some time.\"\"\"\n", + " time.sleep(0.1)\n", + " return \"completed\"\n", + "\n", + "\n", + "with ThreadPoolExecutor(max_workers=1) as pool:\n", + " future: Future[str] = pool.submit(slow_task)\n", + "\n", + " # Check status before completion\n", + " print(f\"Done immediately after submit: {future.done()}\")\n", + "\n", + " # Wait for completion\n", + " result: str = future.result()\n", + " print(f\"Done after result(): {future.done()}\")\n", + " print(f\"Result: {result}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 7: Practical Patterns\n", + "\n", + "Common patterns for using `concurrent.futures` in real applications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor, as_completed\n", + "import time\n", + "\n", + "\n", + "def fetch_data(url: str) -> dict[str, str | int]:\n", + " \"\"\"Simulate fetching data from a URL.\"\"\"\n", + " time.sleep(0.05) # Simulate network latency\n", + " return {\"url\": url, \"status\": 200}\n", + "\n", + "\n", + "urls: list[str] = [\n", + " \"https://api.example.com/users\",\n", + " \"https://api.example.com/products\",\n", + " \"https://api.example.com/orders\",\n", + " \"https://api.example.com/reviews\",\n", + "]\n", + "\n", + "# Fetch all URLs concurrently\n", + "with ThreadPoolExecutor(max_workers=4) as pool:\n", + " future_to_url: dict[Future[dict[str, str | int]], str] = {\n", + " pool.submit(fetch_data, url): url for url in urls\n", + " }\n", + "\n", + " for future in as_completed(future_to_url):\n", + " url: str = future_to_url[future]\n", + " try:\n", + " data: dict[str, str | int] = future.result()\n", + " print(f\"Fetched {url} -> status {data['status']}\")\n", + " except Exception as e:\n", + " print(f\"Error fetching {url}: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "from concurrent.futures import Future, ThreadPoolExecutor, as_completed\n", + "\n", + "\n", + "def process_item(item: int) -> dict[str, int | str]:\n", + " \"\"\"Process a single item, potentially failing.\"\"\"\n", + " if item % 3 == 0:\n", + " raise ValueError(f\"Cannot process {item}\")\n", + " return {\"item\": item, \"result\": item * 10}\n", + "\n", + "\n", + "items: list[int] = list(range(1, 8))\n", + "successes: list[dict[str, int | str]] = []\n", + "failures: list[str] = []\n", + "\n", + "with ThreadPoolExecutor(max_workers=3) as pool:\n", + " futures: dict[Future[dict[str, int | str]], int] = {\n", + " pool.submit(process_item, item): item for item in items\n", + " }\n", + "\n", + " for future in as_completed(futures):\n", + " item: int = futures[future]\n", + " exc: BaseException | None = future.exception()\n", + " if exc is not None:\n", + " failures.append(f\"item {item}: {exc}\")\n", + " else:\n", + " successes.append(future.result())\n", + "\n", + "print(\"Successes:\")\n", + "for s in sorted(successes, key=lambda x: x[\"item\"]):\n", + " print(f\" {s}\")\n", + "\n", + "print(f\"\\nFailures:\")\n", + "for f in sorted(failures):\n", + " print(f\" {f}\")\n", + "\n", + "print(f\"\\nTotal: {len(successes)} succeeded, {len(failures)} failed\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Core API\n", + "- **`ThreadPoolExecutor(max_workers=N)`**: Create a pool of `N` worker threads; use as a context manager\n", + "- **`pool.submit(fn, *args)`**: Schedule `fn(*args)` and return a `Future` immediately\n", + "- **`pool.map(fn, iterable)`**: Apply `fn` to each element; results are returned in input order\n", + "\n", + "### Future Methods\n", + "- **`future.result(timeout=None)`**: Block until the result is ready; re-raises exceptions\n", + "- **`future.exception(timeout=None)`**: Return the exception or `None` if successful\n", + "- **`future.done()`**: Return `True` if the future has finished (success or failure)\n", + "- **`future.cancel()`**: Attempt to cancel the future before it starts executing\n", + "\n", + "### Processing Patterns\n", + "- **`as_completed(futures)`**: Yield futures in completion order (fastest results first)\n", + "- **Submit + collect**: Submit multiple tasks, then iterate over futures for results\n", + "- **Map for ordered results**: Use `map()` when you need results in the same order as inputs\n", + "\n", + "### Best Practices\n", + "- Always use the executor as a context manager (`with` statement) for proper cleanup\n", + "- Check `future.exception()` or use `try/except` around `future.result()` to handle errors\n", + "- Use `as_completed()` when you want to process results as soon as they are ready\n", + "- Choose `max_workers` based on your workload: I/O-bound tasks benefit from more threads" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_41/03_thread_safe_patterns.ipynb b/src/chapter_41/03_thread_safe_patterns.ipynb new file mode 100644 index 0000000..a69af6e --- /dev/null +++ b/src/chapter_41/03_thread_safe_patterns.ipynb @@ -0,0 +1,735 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 41: Thread-Safe Patterns\n", + "\n", + "This notebook covers thread-safe data structures and patterns for concurrent programming. You will learn how to use `Queue` for safe inter-thread communication, implement the producer-consumer pattern, manage per-thread state with `threading.local`, and control queue capacity.\n", + "\n", + "## Key Concepts\n", + "- **`Queue`**: Thread-safe FIFO queue for passing data between threads\n", + "- **Producer-consumer pattern**: Decouple data production from consumption using a queue\n", + "- **Sentinel values**: Signal termination to consumers via a special value (e.g., `None`)\n", + "- **`threading.local`**: Per-thread storage where each thread sees its own data\n", + "- **Queue `maxsize`**: Limit queue capacity to apply backpressure on producers" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Queue Basics -- Thread-Safe FIFO\n", + "\n", + "The `queue.Queue` class provides a thread-safe FIFO (first-in, first-out) data structure. Multiple threads can safely `put()` and `get()` items without additional locking." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import Queue\n", + "\n", + "# Basic FIFO queue\n", + "q: Queue[int] = Queue()\n", + "\n", + "# Put items into the queue\n", + "q.put(1)\n", + "q.put(2)\n", + "q.put(3)\n", + "\n", + "# Get items in FIFO order\n", + "first: int = q.get()\n", + "second: int = q.get()\n", + "third: int = q.get()\n", + "\n", + "print(f\"First: {first}\")\n", + "print(f\"Second: {second}\")\n", + "print(f\"Third: {third}\")\n", + "print(f\"FIFO order correct: {first == 1 and second == 2 and third == 3}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import Queue\n", + "\n", + "# Queue utility methods\n", + "q: Queue[str] = Queue()\n", + "\n", + "print(f\"Empty at start: {q.empty()}\")\n", + "print(f\"Size at start: {q.qsize()}\")\n", + "\n", + "q.put(\"alpha\")\n", + "q.put(\"beta\")\n", + "\n", + "print(f\"\\nAfter adding 2 items:\")\n", + "print(f\"Empty: {q.empty()}\")\n", + "print(f\"Size: {q.qsize()}\")\n", + "\n", + "# task_done() and join() for tracking completion\n", + "item: str = q.get()\n", + "q.task_done() # Signal that the item has been processed\n", + "print(f\"\\nProcessed: {item}\")\n", + "print(f\"Remaining size: {q.qsize()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Queue Timeout and Empty Exception\n", + "\n", + "By default, `Queue.get()` blocks until an item is available. You can pass a `timeout` parameter to raise `queue.Empty` if no item arrives within the timeout period." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import Empty, Queue\n", + "\n", + "q: Queue[int] = Queue()\n", + "\n", + "# get() with timeout raises Empty if the queue is empty\n", + "try:\n", + " q.get(timeout=0.01)\n", + " print(\"Got an item (unexpected)\")\n", + "except Empty:\n", + " print(\"Caught Empty exception -- queue was empty after timeout\")\n", + "\n", + "# Non-blocking get with block=False\n", + "try:\n", + " q.get(block=False)\n", + " print(\"Got an item (unexpected)\")\n", + "except Empty:\n", + " print(\"Caught Empty exception -- non-blocking get on empty queue\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import Empty, Queue\n", + "import threading\n", + "import time\n", + "\n", + "# Practical pattern: drain a queue with timeout\n", + "q: Queue[int] = Queue()\n", + "collected: list[int] = []\n", + "\n", + "\n", + "def delayed_producer() -> None:\n", + " \"\"\"Add items to the queue with small delays.\"\"\"\n", + " for i in range(3):\n", + " time.sleep(0.02)\n", + " q.put(i)\n", + "\n", + "\n", + "t: threading.Thread = threading.Thread(target=delayed_producer)\n", + "t.start()\n", + "\n", + "# Collect items with timeout-based polling\n", + "while True:\n", + " try:\n", + " item: int = q.get(timeout=0.1)\n", + " collected.append(item)\n", + " except Empty:\n", + " # No more items within timeout\n", + " break\n", + "\n", + "t.join()\n", + "\n", + "print(f\"Collected items: {collected}\")\n", + "print(f\"All items received: {collected == [0, 1, 2]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Producer-Consumer Pattern\n", + "\n", + "The producer-consumer pattern decouples threads that generate data (producers) from threads that process data (consumers). A `Queue` serves as the buffer between them. A sentinel value (typically `None`) signals the consumer to stop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "from queue import Queue\n", + "\n", + "q: Queue[int | None] = Queue()\n", + "results: list[int] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def producer() -> None:\n", + " \"\"\"Produce 5 items, then send a sentinel to stop the consumer.\"\"\"\n", + " for i in range(5):\n", + " q.put(i)\n", + " q.put(None) # Sentinel signals end of data\n", + "\n", + "\n", + "def consumer() -> None:\n", + " \"\"\"Consume items until the sentinel is received.\"\"\"\n", + " while True:\n", + " item: int | None = q.get()\n", + " if item is None:\n", + " break\n", + " with lock:\n", + " results.append(item)\n", + "\n", + "\n", + "t1: threading.Thread = threading.Thread(target=producer)\n", + "t2: threading.Thread = threading.Thread(target=consumer)\n", + "t1.start()\n", + "t2.start()\n", + "t1.join()\n", + "t2.join()\n", + "\n", + "print(f\"Consumed results: {sorted(results)}\")\n", + "print(f\"Expected: [0, 1, 2, 3, 4]\")\n", + "print(f\"Correct: {sorted(results) == [0, 1, 2, 3, 4]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "from queue import Queue\n", + "\n", + "# Multiple producers and consumers\n", + "q: Queue[int | None] = Queue()\n", + "results: list[int] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "NUM_PRODUCERS: int = 2\n", + "NUM_CONSUMERS: int = 2\n", + "ITEMS_PER_PRODUCER: int = 5\n", + "\n", + "\n", + "def multi_producer(producer_id: int) -> None:\n", + " \"\"\"Produce items tagged with producer ID.\"\"\"\n", + " for i in range(ITEMS_PER_PRODUCER):\n", + " q.put(producer_id * 100 + i)\n", + "\n", + "\n", + "def multi_consumer() -> None:\n", + " \"\"\"Consume items until sentinel.\"\"\"\n", + " while True:\n", + " item: int | None = q.get()\n", + " if item is None:\n", + " break\n", + " with lock:\n", + " results.append(item)\n", + "\n", + "\n", + "# Start producers\n", + "producers: list[threading.Thread] = [\n", + " threading.Thread(target=multi_producer, args=(pid,))\n", + " for pid in range(NUM_PRODUCERS)\n", + "]\n", + "for p in producers:\n", + " p.start()\n", + "for p in producers:\n", + " p.join()\n", + "\n", + "# Send one sentinel per consumer\n", + "for _ in range(NUM_CONSUMERS):\n", + " q.put(None)\n", + "\n", + "# Start consumers\n", + "consumers: list[threading.Thread] = [\n", + " threading.Thread(target=multi_consumer) for _ in range(NUM_CONSUMERS)\n", + "]\n", + "for c in consumers:\n", + " c.start()\n", + "for c in consumers:\n", + " c.join()\n", + "\n", + "print(f\"Total items consumed: {len(results)}\")\n", + "print(f\"Expected: {NUM_PRODUCERS * ITEMS_PER_PRODUCER}\")\n", + "print(f\"All items accounted for: {len(results) == NUM_PRODUCERS * ITEMS_PER_PRODUCER}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Queue maxsize -- Backpressure\n", + "\n", + "A `Queue` can be created with a `maxsize` to limit the number of items it can hold. When the queue is full, `put()` blocks until space is available. This provides natural backpressure on fast producers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import Queue\n", + "\n", + "# Create a queue with maximum size of 2\n", + "q: Queue[int] = Queue(maxsize=2)\n", + "\n", + "q.put(1)\n", + "q.put(2)\n", + "\n", + "print(f\"Queue size: {q.qsize()}\")\n", + "print(f\"Queue full: {q.full()}\")\n", + "print(f\"Max size: {q.maxsize}\")\n", + "\n", + "# Remove one item to make room\n", + "removed: int = q.get()\n", + "print(f\"\\nRemoved: {removed}\")\n", + "print(f\"Queue full after get: {q.full()}\")\n", + "print(f\"Queue size: {q.qsize()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import Full, Queue\n", + "\n", + "# Non-blocking put on a full queue\n", + "q: Queue[int] = Queue(maxsize=2)\n", + "q.put(1)\n", + "q.put(2)\n", + "\n", + "# Try to put without blocking\n", + "try:\n", + " q.put(3, block=False)\n", + " print(\"Put succeeded (unexpected)\")\n", + "except Full:\n", + " print(\"Caught Full exception -- queue at maxsize\")\n", + "\n", + "# Try with timeout\n", + "try:\n", + " q.put(3, timeout=0.01)\n", + " print(\"Put succeeded (unexpected)\")\n", + "except Full:\n", + " print(\"Caught Full exception -- queue still full after timeout\")\n", + "\n", + "print(f\"\\nQueue size: {q.qsize()}\")\n", + "print(f\"Queue full: {q.full()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "import time\n", + "from queue import Queue\n", + "\n", + "# Backpressure in action: fast producer, slow consumer\n", + "q: Queue[int | None] = Queue(maxsize=3)\n", + "produced_at: list[float] = []\n", + "consumed_at: list[float] = []\n", + "start_time: float = time.perf_counter()\n", + "\n", + "\n", + "def fast_producer() -> None:\n", + " \"\"\"Produce items as fast as possible.\"\"\"\n", + " for i in range(6):\n", + " q.put(i) # Blocks when queue is full\n", + " produced_at.append(time.perf_counter() - start_time)\n", + " q.put(None)\n", + "\n", + "\n", + "def slow_consumer() -> None:\n", + " \"\"\"Consume items slowly.\"\"\"\n", + " while True:\n", + " item: int | None = q.get()\n", + " if item is None:\n", + " break\n", + " consumed_at.append(time.perf_counter() - start_time)\n", + " time.sleep(0.03) # Slow processing\n", + "\n", + "\n", + "t1: threading.Thread = threading.Thread(target=fast_producer)\n", + "t2: threading.Thread = threading.Thread(target=slow_consumer)\n", + "t1.start()\n", + "t2.start()\n", + "t1.join()\n", + "t2.join()\n", + "\n", + "print(\"Backpressure demonstration:\")\n", + "print(f\"Items produced: {len(produced_at)}\")\n", + "print(f\"Items consumed: {len(consumed_at)}\")\n", + "print(f\"Producer was slowed by full queue (backpressure applied)\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 5: Thread-Local Storage\n", + "\n", + "`threading.local()` creates an object where each thread sees its own independent copy of the data. This is useful for per-thread caches, database connections, or other state that should not be shared." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "import time\n", + "\n", + "# Each thread gets its own copy of local.data\n", + "local: threading.local = threading.local()\n", + "results: list[int] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def worker(value: int) -> None:\n", + " \"\"\"Set thread-local data and verify it persists.\"\"\"\n", + " local.data = value\n", + " time.sleep(0.01) # Let other threads run\n", + " # Each thread still sees its own value\n", + " with lock:\n", + " results.append(local.data)\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=worker, args=(i,)) for i in range(4)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "print(f\"Results (sorted): {sorted(results)}\")\n", + "print(f\"Each thread kept its own value: {sorted(results) == [0, 1, 2, 3]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Thread-local storage with multiple attributes\n", + "local: threading.local = threading.local()\n", + "reports: list[str] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "\n", + "\n", + "def database_worker(worker_id: int, db_name: str) -> None:\n", + " \"\"\"Simulate a worker with its own database connection.\"\"\"\n", + " local.worker_id = worker_id\n", + " local.db_name = db_name\n", + " local.query_count = 0\n", + "\n", + " # Simulate queries\n", + " for _ in range(3):\n", + " local.query_count += 1\n", + "\n", + " report: str = (\n", + " f\"Worker {local.worker_id} on {local.db_name}: \"\n", + " f\"{local.query_count} queries\"\n", + " )\n", + " with lock:\n", + " reports.append(report)\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=database_worker, args=(i, f\"db_{i}\"))\n", + " for i in range(3)\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "for report in sorted(reports):\n", + " print(report)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "# Thread-local attributes are not shared\n", + "local: threading.local = threading.local()\n", + "local.main_value = \"set in main thread\"\n", + "\n", + "child_saw: list[str] = []\n", + "\n", + "\n", + "def child_thread() -> None:\n", + " \"\"\"Check if the child thread can see the main thread's local data.\"\"\"\n", + " has_attr: bool = hasattr(local, \"main_value\")\n", + " child_saw.append(f\"has main_value: {has_attr}\")\n", + "\n", + " # Set our own value\n", + " local.child_value = \"set in child thread\"\n", + " child_saw.append(f\"child_value: {local.child_value}\")\n", + "\n", + "\n", + "t: threading.Thread = threading.Thread(target=child_thread)\n", + "t.start()\n", + "t.join()\n", + "\n", + "for msg in child_saw:\n", + " print(f\"Child thread: {msg}\")\n", + "\n", + "print(f\"\\nMain thread: main_value = {local.main_value}\")\n", + "print(f\"Main thread: has child_value = {hasattr(local, 'child_value')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 6: Practical Patterns -- Pipeline with Queues\n", + "\n", + "Queues can be chained together to form processing pipelines where each stage runs in its own thread." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "from queue import Queue\n", + "\n", + "# Two-stage pipeline: generate -> transform -> collect\n", + "stage1_q: Queue[int | None] = Queue()\n", + "stage2_q: Queue[str | None] = Queue()\n", + "final_results: list[str] = []\n", + "\n", + "\n", + "def generator() -> None:\n", + " \"\"\"Stage 1: Generate raw numbers.\"\"\"\n", + " for i in range(5):\n", + " stage1_q.put(i)\n", + " stage1_q.put(None)\n", + "\n", + "\n", + "def transformer() -> None:\n", + " \"\"\"Stage 2: Transform numbers to formatted strings.\"\"\"\n", + " while True:\n", + " item: int | None = stage1_q.get()\n", + " if item is None:\n", + " stage2_q.put(None)\n", + " break\n", + " stage2_q.put(f\"item-{item:03d}\")\n", + "\n", + "\n", + "def collector() -> None:\n", + " \"\"\"Stage 3: Collect final results.\"\"\"\n", + " while True:\n", + " item: str | None = stage2_q.get()\n", + " if item is None:\n", + " break\n", + " final_results.append(item)\n", + "\n", + "\n", + "threads: list[threading.Thread] = [\n", + " threading.Thread(target=generator),\n", + " threading.Thread(target=transformer),\n", + " threading.Thread(target=collector),\n", + "]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "\n", + "print(f\"Pipeline results: {final_results}\")\n", + "print(f\"Expected 5 items: {len(final_results) == 5}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "from queue import Queue\n", + "\n", + "# Worker pool pattern: multiple workers processing from a shared queue\n", + "task_queue: Queue[int | None] = Queue()\n", + "results: list[int] = []\n", + "lock: threading.Lock = threading.Lock()\n", + "NUM_WORKERS: int = 3\n", + "\n", + "\n", + "def pool_worker(worker_id: int) -> None:\n", + " \"\"\"Process tasks from the shared queue.\"\"\"\n", + " while True:\n", + " task: int | None = task_queue.get()\n", + " if task is None:\n", + " break\n", + " result: int = task * task # Square the number\n", + " with lock:\n", + " results.append(result)\n", + "\n", + "\n", + "# Add tasks\n", + "for i in range(10):\n", + " task_queue.put(i)\n", + "\n", + "# Add sentinel for each worker\n", + "for _ in range(NUM_WORKERS):\n", + " task_queue.put(None)\n", + "\n", + "# Start workers\n", + "workers: list[threading.Thread] = [\n", + " threading.Thread(target=pool_worker, args=(wid,))\n", + " for wid in range(NUM_WORKERS)\n", + "]\n", + "for w in workers:\n", + " w.start()\n", + "for w in workers:\n", + " w.join()\n", + "\n", + "print(f\"Results (sorted): {sorted(results)}\")\n", + "expected: list[int] = [i * i for i in range(10)]\n", + "print(f\"Expected: {expected}\")\n", + "print(f\"Correct: {sorted(results) == expected}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 7: Other Queue Types\n", + "\n", + "The `queue` module also provides `LifoQueue` (stack) and `PriorityQueue` for different ordering needs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "from queue import LifoQueue, PriorityQueue\n", + "\n", + "# LifoQueue -- last-in, first-out (stack behavior)\n", + "lifo: LifoQueue[str] = LifoQueue()\n", + "lifo.put(\"first\")\n", + "lifo.put(\"second\")\n", + "lifo.put(\"third\")\n", + "\n", + "print(\"LifoQueue (stack order):\")\n", + "print(f\" {lifo.get()}\") # third\n", + "print(f\" {lifo.get()}\") # second\n", + "print(f\" {lifo.get()}\") # first\n", + "\n", + "# PriorityQueue -- lowest value first\n", + "pq: PriorityQueue[tuple[int, str]] = PriorityQueue()\n", + "pq.put((3, \"low priority\"))\n", + "pq.put((1, \"high priority\"))\n", + "pq.put((2, \"medium priority\"))\n", + "\n", + "print(\"\\nPriorityQueue (lowest value first):\")\n", + "while not pq.empty():\n", + " priority, message = pq.get()\n", + " print(f\" Priority {priority}: {message}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Queue Basics\n", + "- **`Queue()`**: Thread-safe FIFO queue; `put()` adds items, `get()` removes them in order\n", + "- **`Queue(maxsize=N)`**: Limits capacity; `put()` blocks when full, providing backpressure\n", + "- **`q.full()`** / **`q.empty()`** / **`q.qsize()`**: Check queue state (approximate in threaded code)\n", + "- **`q.get(timeout=N)`**: Raises `queue.Empty` if no item is available within `N` seconds\n", + "\n", + "### Producer-Consumer Pattern\n", + "- Producers call `q.put(item)` to add work; consumers call `q.get()` to retrieve it\n", + "- Use a **sentinel value** (e.g., `None`) to signal consumers to stop\n", + "- For multiple consumers, send one sentinel per consumer\n", + "\n", + "### Thread-Local Storage\n", + "- **`threading.local()`**: Each thread gets its own independent copy of attributes\n", + "- Attributes set in one thread are invisible to other threads\n", + "- Useful for per-thread caches, connections, or context\n", + "\n", + "### Additional Queue Types\n", + "- **`LifoQueue`**: Last-in, first-out (stack) ordering\n", + "- **`PriorityQueue`**: Items are retrieved in priority order (lowest value first)\n", + "\n", + "### Best Practices\n", + "- Always use sentinel values or `task_done()` / `join()` for clean shutdown\n", + "- Use `maxsize` to prevent unbounded memory growth from fast producers\n", + "- Prefer `threading.local()` over manual per-thread dictionaries for thread-specific state" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_41/README.md b/src/chapter_41/README.md new file mode 100644 index 0000000..446983e --- /dev/null +++ b/src/chapter_41/README.md @@ -0,0 +1,13 @@ +# Chapter 41: Concurrency Patterns + +## Topics Covered + +- Threading synchronization: Lock, RLock, Semaphore, Event, Barrier, Condition +- concurrent.futures: ThreadPoolExecutor, ProcessPoolExecutor, Future, as_completed +- Thread-safe patterns: Queue, producer-consumer, thread-local storage + +## Notebooks + +1. **01_threading_synchronization.ipynb** - Lock, RLock, Semaphore, Event, Barrier, Condition +2. **02_concurrent_futures.ipynb** - ThreadPoolExecutor, ProcessPoolExecutor, Future +3. **03_thread_safe_patterns.ipynb** - Queue, producer-consumer, thread-local storage diff --git a/src/chapter_41/__init__.py b/src/chapter_41/__init__.py new file mode 100644 index 0000000..b7e67d8 --- /dev/null +++ b/src/chapter_41/__init__.py @@ -0,0 +1 @@ +"""Chapter 41: Concurrency Patterns.""" diff --git a/tests/test_chapter_36.py b/tests/test_chapter_36.py new file mode 100644 index 0000000..e4fde31 --- /dev/null +++ b/tests/test_chapter_36.py @@ -0,0 +1,170 @@ +"""Tests for Chapter 36: Advanced Testing.""" + +from unittest.mock import MagicMock, Mock, patch + +import pytest + + +class TestPytestFeatures: + """Test pytest-specific features.""" + + def test_parametrize_squares(self) -> None: + """parametrize runs a test with multiple inputs.""" + cases = [(1, 1), (2, 4), (3, 9), (4, 16), (0, 0)] + for value, expected in cases: + assert value**2 == expected + + def test_fixture_like_setup(self) -> None: + """Fixtures provide reusable test data.""" + data = {"name": "Alice", "age": 30} + assert data["name"] == "Alice" + assert isinstance(data["age"], int) + + def test_tmp_path_usage(self, tmp_path) -> None: + """tmp_path provides a temporary directory per test.""" + file = tmp_path / "test.txt" + file.write_text("hello") + assert file.read_text() == "hello" + assert file.exists() + + def test_monkeypatch_setattr(self, monkeypatch) -> None: + """monkeypatch temporarily modifies objects.""" + + class Config: + debug = False + + monkeypatch.setattr(Config, "debug", True) + assert Config.debug is True + + def test_monkeypatch_env(self, monkeypatch) -> None: + """monkeypatch can set environment variables.""" + monkeypatch.setenv("TEST_VAR", "hello") + import os + + assert os.environ["TEST_VAR"] == "hello" + + def test_raises_context(self) -> None: + """pytest.raises checks for expected exceptions.""" + with pytest.raises(ValueError, match="invalid"): + raise ValueError("invalid literal") + + def test_approx_for_floats(self) -> None: + """pytest.approx handles floating point comparison.""" + assert 0.1 + 0.2 == pytest.approx(0.3) + assert [0.1, 0.2] == pytest.approx([0.1, 0.2]) + + +class TestMocking: + """Test unittest.mock features.""" + + def test_mock_basic(self) -> None: + """Mock creates callable objects with tracking.""" + m = Mock() + m(1, 2, key="value") + m.assert_called_once_with(1, 2, key="value") + + def test_mock_return_value(self) -> None: + """Mock.return_value controls what mock returns.""" + m = Mock(return_value=42) + assert m() == 42 + + def test_mock_side_effect_exception(self) -> None: + """side_effect can raise exceptions.""" + m = Mock(side_effect=ValueError("boom")) + with pytest.raises(ValueError, match="boom"): + m() + + def test_mock_side_effect_iterable(self) -> None: + """side_effect can return values from an iterable.""" + m = Mock(side_effect=[1, 2, 3]) + assert m() == 1 + assert m() == 2 + assert m() == 3 + + def test_magic_mock_dunder(self) -> None: + """MagicMock supports dunder methods.""" + m = MagicMock() + m.__len__.return_value = 5 + assert len(m) == 5 + + def test_patch_decorator(self) -> None: + """patch replaces objects during test.""" + with patch("os.getcwd", return_value="/fake/path"): + import os + + assert os.getcwd() == "/fake/path" + + def test_mock_call_count(self) -> None: + """Mock tracks call count.""" + m = Mock() + m() + m() + m() + assert m.call_count == 3 + + def test_mock_spec(self) -> None: + """spec restricts mock to real object's interface.""" + m = Mock(spec=list) + m.append(1) # Valid method + with pytest.raises(AttributeError): + m.nonexistent() # type: ignore[attr-defined] + + +class TestTestPatterns: + """Test common testing patterns.""" + + def test_arrange_act_assert(self) -> None: + """AAA pattern structures tests clearly.""" + # Arrange + items: list[int] = [3, 1, 4, 1, 5] + # Act + result = sorted(items) + # Assert + assert result == [1, 1, 3, 4, 5] + + def test_test_double_stub(self) -> None: + """Stubs provide canned responses.""" + + class DatabaseStub: + def query(self, sql: str) -> list[dict]: + return [{"id": 1, "name": "Alice"}] + + db = DatabaseStub() + results = db.query("SELECT * FROM users") + assert len(results) == 1 + assert results[0]["name"] == "Alice" + + def test_test_double_spy(self) -> None: + """Spies track interactions while delegating.""" + calls: list[str] = [] + + class LoggerSpy: + def log(self, msg: str) -> None: + calls.append(msg) + + logger = LoggerSpy() + logger.log("hello") + logger.log("world") + assert len(calls) == 2 + + def test_builder_pattern_for_test_data(self) -> None: + """Builder pattern creates complex test objects.""" + + class UserBuilder: + def __init__(self) -> None: + self._name = "default" + self._age = 0 + + def with_name(self, name: str) -> "UserBuilder": + self._name = name + return self + + def with_age(self, age: int) -> "UserBuilder": + self._age = age + return self + + def build(self) -> dict: + return {"name": self._name, "age": self._age} + + user = UserBuilder().with_name("Alice").with_age(30).build() + assert user == {"name": "Alice", "age": 30} diff --git a/tests/test_chapter_37.py b/tests/test_chapter_37.py new file mode 100644 index 0000000..ad743c9 --- /dev/null +++ b/tests/test_chapter_37.py @@ -0,0 +1,160 @@ +"""Tests for Chapter 37: Abstract Syntax Trees.""" + +import ast +import types + + +class TestASTBasics: + """Test AST parsing and inspection.""" + + def test_parse_expression(self) -> None: + """ast.parse creates an AST from source code.""" + tree = ast.parse("x = 1 + 2") + assert isinstance(tree, ast.Module) + assert len(tree.body) == 1 + assert isinstance(tree.body[0], ast.Assign) + + def test_parse_function(self) -> None: + """ast.parse handles function definitions.""" + tree = ast.parse("def greet(name): return f'Hello, {name}'") + func = tree.body[0] + assert isinstance(func, ast.FunctionDef) + assert func.name == "greet" + + def test_ast_dump(self) -> None: + """ast.dump shows AST structure as a string.""" + tree = ast.parse("42") + dumped = ast.dump(tree) + assert "Constant" in dumped + assert "42" in dumped + + def test_literal_eval_safe(self) -> None: + """ast.literal_eval safely evaluates literal expressions.""" + assert ast.literal_eval("42") == 42 + assert ast.literal_eval("[1, 2, 3]") == [1, 2, 3] + assert ast.literal_eval("{'a': 1}") == {"a": 1} + + def test_literal_eval_rejects_code(self) -> None: + """ast.literal_eval rejects non-literal expressions.""" + try: + ast.literal_eval("__import__('os')") + assert False, "Should have raised" + except (ValueError, SyntaxError): + pass + + def test_ast_node_fields(self) -> None: + """AST nodes have typed fields.""" + tree = ast.parse("x + y") + expr = tree.body[0] + assert isinstance(expr, ast.Expr) + binop = expr.value + assert isinstance(binop, ast.BinOp) + assert isinstance(binop.op, ast.Add) + + +class TestNodeVisitor: + """Test AST node visiting and transformation.""" + + def test_node_visitor(self) -> None: + """NodeVisitor walks the AST tree.""" + + class NameCollector(ast.NodeVisitor): + def __init__(self) -> None: + self.names: list[str] = [] + + def visit_Name(self, node: ast.Name) -> None: + self.names.append(node.id) + self.generic_visit(node) + + tree = ast.parse("x = y + z") + collector = NameCollector() + collector.visit(tree) + assert "y" in collector.names + assert "z" in collector.names + + def test_function_counter(self) -> None: + """NodeVisitor can count specific node types.""" + + class FuncCounter(ast.NodeVisitor): + def __init__(self) -> None: + self.count = 0 + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self.count += 1 + self.generic_visit(node) + + source = """ +def foo(): pass +def bar(): pass +def baz(): pass +""" + tree = ast.parse(source) + counter = FuncCounter() + counter.visit(tree) + assert counter.count == 3 + + def test_node_transformer(self) -> None: + """NodeTransformer modifies the AST.""" + + class DoubleConstants(ast.NodeTransformer): + def visit_Constant(self, node: ast.Constant) -> ast.Constant: + if isinstance(node.value, int): + return ast.Constant(value=node.value * 2) + return node + + tree = ast.parse("x = 5") + new_tree = DoubleConstants().visit(tree) + ast.fix_missing_locations(new_tree) + code = compile(new_tree, "", "exec") + ns: dict = {} + exec(code, ns) # noqa: S102 + assert ns["x"] == 10 + + def test_ast_walk(self) -> None: + """ast.walk iterates all nodes without recursion control.""" + tree = ast.parse("a = b + c * d") + node_types = {type(node).__name__ for node in ast.walk(tree)} + assert "BinOp" in node_types + assert "Name" in node_types + assert "Add" in node_types + + +class TestCompileEvalExec: + """Test compile, eval, and exec.""" + + def test_eval_expression(self) -> None: + """eval evaluates a single expression.""" + result = eval("2 + 3") # noqa: S307 + assert result == 5 + + def test_eval_with_namespace(self) -> None: + """eval can use a custom namespace.""" + ns = {"x": 10, "y": 20} + result = eval("x + y", ns) # noqa: S307 + assert result == 30 + + def test_exec_statements(self) -> None: + """exec runs arbitrary statements.""" + ns: dict = {} + exec("result = [i**2 for i in range(5)]", ns) # noqa: S102 + assert ns["result"] == [0, 1, 4, 9, 16] + + def test_compile_to_code(self) -> None: + """compile creates a code object from source.""" + code = compile("x = 42", "", "exec") + assert isinstance(code, types.CodeType) + ns: dict = {} + exec(code, ns) # noqa: S102 + assert ns["x"] == 42 + + def test_code_object_attributes(self) -> None: + """Code objects have inspection attributes.""" + + def example(a: int, b: int) -> int: + return a + b + + code = example.__code__ + assert code.co_name == "example" + assert code.co_argcount == 2 + assert "a" in code.co_varnames + assert "b" in code.co_varnames diff --git a/tests/test_chapter_38.py b/tests/test_chapter_38.py new file mode 100644 index 0000000..379b744 --- /dev/null +++ b/tests/test_chapter_38.py @@ -0,0 +1,147 @@ +"""Tests for Chapter 38: Memory Management.""" + +import gc +import sys +import tracemalloc +import weakref + + +class TestReferenceCounting: + """Test reference counting mechanics.""" + + def test_getrefcount(self) -> None: + """sys.getrefcount returns the reference count (includes the call itself).""" + a: list[int] = [] + count = sys.getrefcount(a) + assert count >= 2 # a + getrefcount arg + + def test_multiple_references(self) -> None: + """Multiple names increase reference count.""" + a: list[int] = [] + base = sys.getrefcount(a) + b = a # noqa: F841 + assert sys.getrefcount(a) == base + 1 + + def test_del_decreases_refcount(self) -> None: + """del removes a reference.""" + a: list[int] = [] + b = a + base = sys.getrefcount(a) + del b + assert sys.getrefcount(a) == base - 1 + + def test_container_references(self) -> None: + """Containers hold references to their items.""" + obj: list[int] = [] + base = sys.getrefcount(obj) + container = [obj] # noqa: F841 + assert sys.getrefcount(obj) == base + 1 + + +class TestGarbageCollection: + """Test gc module for cycle detection.""" + + def test_gc_is_enabled(self) -> None: + """gc is enabled by default.""" + assert gc.isenabled() + + def test_gc_collect(self) -> None: + """gc.collect runs garbage collection.""" + collected = gc.collect() + assert isinstance(collected, int) + + def test_circular_reference_cleanup(self) -> None: + """gc handles circular references.""" + + class Node: + def __init__(self) -> None: + self.ref: "Node | None" = None + + a = Node() + b = Node() + a.ref = b + b.ref = a + weak_a = weakref.ref(a) + del a, b + gc.collect() + assert weak_a() is None + + def test_gc_get_stats(self) -> None: + """gc.get_stats returns collection statistics.""" + stats = gc.get_stats() + assert len(stats) == 3 # 3 generations + assert "collections" in stats[0] + assert "collected" in stats[0] + + def test_gc_freeze(self) -> None: + """gc.freeze moves objects to permanent generation.""" + gc.freeze() + frozen_count = gc.get_freeze_count() + assert frozen_count >= 0 + gc.unfreeze() + + def test_weakref_finalize(self) -> None: + """weakref.finalize runs a callback when object is collected.""" + cleaned_up = [] + + class Resource: + pass + + obj = Resource() + weakref.finalize(obj, lambda: cleaned_up.append(True)) + del obj + gc.collect() + assert len(cleaned_up) == 1 + + +class TestMemoryProfiling: + """Test memory inspection tools.""" + + def test_getsizeof(self) -> None: + """sys.getsizeof returns memory size in bytes.""" + assert sys.getsizeof(0) > 0 + assert sys.getsizeof([]) < sys.getsizeof([1, 2, 3, 4, 5]) + + def test_getsizeof_types(self) -> None: + """Different types have different base sizes.""" + assert sys.getsizeof("") < sys.getsizeof("hello world") + assert sys.getsizeof(()) < sys.getsizeof((1, 2, 3)) + + def test_tracemalloc_snapshot(self) -> None: + """tracemalloc tracks memory allocations.""" + tracemalloc.start() + try: + _data = [i for i in range(1000)] # noqa: C416 + snapshot = tracemalloc.take_snapshot() + stats = snapshot.statistics("lineno") + assert len(stats) > 0 + finally: + tracemalloc.stop() + + def test_tracemalloc_get_traced_memory(self) -> None: + """tracemalloc reports current and peak memory.""" + tracemalloc.start() + try: + current, peak = tracemalloc.get_traced_memory() + assert current >= 0 + assert peak >= 0 + assert peak >= current + finally: + tracemalloc.stop() + + def test_slots_memory_savings(self) -> None: + """__slots__ reduces per-instance memory.""" + + class Regular: + def __init__(self, x: int) -> None: + self.x = x + + class Slotted: + __slots__ = ("x",) + + def __init__(self, x: int) -> None: + self.x = x + + regular = Regular(1) + slotted = Slotted(1) + assert sys.getsizeof(slotted) < sys.getsizeof(regular) diff --git a/tests/test_chapter_39.py b/tests/test_chapter_39.py new file mode 100644 index 0000000..89896df --- /dev/null +++ b/tests/test_chapter_39.py @@ -0,0 +1,149 @@ +"""Tests for Chapter 39: C Interoperability.""" + +import array +import ctypes +import struct + + +class TestCtypesBasics: + """Test ctypes fundamental types and operations.""" + + def test_ctypes_int(self) -> None: + """ctypes provides C-compatible integer types.""" + i = ctypes.c_int(42) + assert i.value == 42 + + def test_ctypes_float(self) -> None: + """ctypes provides C-compatible float types.""" + f = ctypes.c_float(3.14) + assert abs(f.value - 3.14) < 0.01 + + def test_ctypes_char_array(self) -> None: + """ctypes can create fixed-size char arrays.""" + buf = ctypes.create_string_buffer(10) + assert len(buf) == 10 + buf.value = b"hello" + assert buf.value == b"hello" + + def test_ctypes_pointer(self) -> None: + """ctypes supports pointer operations.""" + i = ctypes.c_int(42) + p = ctypes.pointer(i) + assert p.contents.value == 42 + p.contents.value = 100 + assert i.value == 100 + + def test_ctypes_sizeof(self) -> None: + """ctypes.sizeof returns type sizes.""" + assert ctypes.sizeof(ctypes.c_int) == 4 + assert ctypes.sizeof(ctypes.c_double) == 8 + assert ctypes.sizeof(ctypes.c_char) == 1 + + +class TestCtypesAdvanced: + """Test ctypes structures and arrays.""" + + def test_ctypes_structure(self) -> None: + """ctypes.Structure defines C-compatible structs.""" + + class Point(ctypes.Structure): + _fields_ = [("x", ctypes.c_int), ("y", ctypes.c_int)] + + p = Point(10, 20) + assert p.x == 10 + assert p.y == 20 + + def test_ctypes_structure_sizeof(self) -> None: + """Structure size matches C layout.""" + + class Pair(ctypes.Structure): + _fields_ = [("a", ctypes.c_int), ("b", ctypes.c_int)] + + assert ctypes.sizeof(Pair) == 8 # Two 4-byte ints + + def test_ctypes_array(self) -> None: + """ctypes arrays hold fixed-count elements.""" + IntArray5 = ctypes.c_int * 5 + arr = IntArray5(1, 2, 3, 4, 5) + assert arr[0] == 1 + assert arr[4] == 5 + assert len(arr) == 5 + + def test_ctypes_union(self) -> None: + """ctypes.Union overlaps fields in memory.""" + + class IntOrFloat(ctypes.Union): + _fields_ = [("i", ctypes.c_int), ("f", ctypes.c_float)] + + u = IntOrFloat() + u.i = 0 + assert u.f == 0.0 # Same memory + + def test_ctypes_callback(self) -> None: + """CFUNCTYPE creates C-callable function pointers.""" + CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) + + def py_add(a: int, b: int) -> int: + return a + b + + cb = CALLBACK(py_add) + assert cb(3, 4) == 7 + + +class TestArrayAndMemoryview: + """Test array module and memoryview.""" + + def test_array_creation(self) -> None: + """array.array creates typed numeric arrays.""" + a = array.array("i", [1, 2, 3, 4, 5]) + assert len(a) == 5 + assert a[0] == 1 + assert a.typecode == "i" + + def test_array_itemsize(self) -> None: + """array.itemsize shows bytes per element.""" + a = array.array("i") # signed int + assert a.itemsize == struct.calcsize("i") + + def test_array_buffer_info(self) -> None: + """buffer_info returns address and length.""" + a = array.array("d", [1.0, 2.0, 3.0]) + address, length = a.buffer_info() + assert length == 3 + assert address > 0 + + def test_array_bytes_conversion(self) -> None: + """Arrays convert to/from bytes.""" + a = array.array("i", [1, 2, 3]) + b = a.tobytes() + a2 = array.array("i") + a2.frombytes(b) + assert list(a) == list(a2) + + def test_memoryview_basic(self) -> None: + """memoryview provides zero-copy access to buffer data.""" + data = bytearray(b"Hello, World!") + mv = memoryview(data) + assert bytes(mv[0:5]) == b"Hello" + + def test_memoryview_modify(self) -> None: + """memoryview can modify the underlying buffer.""" + data = bytearray(b"Hello") + mv = memoryview(data) + mv[0] = ord("J") + assert data == bytearray(b"Jello") + + def test_memoryview_format(self) -> None: + """memoryview exposes format and shape info.""" + a = array.array("i", [1, 2, 3]) + mv = memoryview(a) + assert mv.format == "i" + assert mv.itemsize == struct.calcsize("i") + assert len(mv) == 3 + + def test_struct_pack_unpack(self) -> None: + """struct packs/unpacks binary data.""" + packed = struct.pack("!2I", 1, 2) # network byte order, 2 unsigned ints + a, b = struct.unpack("!2I", packed) + assert a == 1 + assert b == 2 diff --git a/tests/test_chapter_40.py b/tests/test_chapter_40.py new file mode 100644 index 0000000..3721613 --- /dev/null +++ b/tests/test_chapter_40.py @@ -0,0 +1,136 @@ +"""Tests for Chapter 40: Import System Internals.""" + +import importlib +import importlib.metadata +import pkgutil +import sys +import types + + +class TestImportlibBasics: + """Test importlib core functionality.""" + + def test_import_module(self) -> None: + """importlib.import_module imports by string name.""" + math = importlib.import_module("math") + assert math.pi == __import__("math").pi + + def test_import_submodule(self) -> None: + """importlib can import submodules.""" + path_mod = importlib.import_module("os.path") + assert hasattr(path_mod, "join") + + def test_reload_module(self) -> None: + """importlib.reload re-imports a module.""" + import json + + reloaded = importlib.reload(json) + assert reloaded is json + + def test_module_has_spec(self) -> None: + """Modules have a __spec__ attribute.""" + import json + + assert json.__spec__ is not None + assert json.__spec__.name == "json" + + def test_module_attributes(self) -> None: + """Modules have standard attributes.""" + import json + + assert hasattr(json, "__name__") + assert hasattr(json, "__file__") + assert hasattr(json, "__package__") + assert json.__name__ == "json" + + +class TestImportHooks: + """Test import system hooks and finders.""" + + def test_sys_meta_path(self) -> None: + """sys.meta_path contains finder objects.""" + assert len(sys.meta_path) > 0 + for finder in sys.meta_path: + assert hasattr(finder, "find_module") or hasattr(finder, "find_spec") + + def test_sys_path_is_list(self) -> None: + """sys.path is a list of directory strings.""" + assert isinstance(sys.path, list) + assert len(sys.path) > 0 + + def test_find_spec(self) -> None: + """importlib.util.find_spec locates modules.""" + spec = importlib.util.find_spec("json") + assert spec is not None + assert spec.name == "json" + assert spec.origin is not None + + def test_find_spec_nonexistent(self) -> None: + """find_spec returns None for missing modules.""" + spec = importlib.util.find_spec("nonexistent_module_xyz_123") + assert spec is None + + def test_create_module_from_spec(self) -> None: + """Modules can be created from specs.""" + spec = importlib.util.find_spec("json") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + assert isinstance(module, types.ModuleType) + assert module.__name__ == "json" + + def test_sys_modules_cache(self) -> None: + """sys.modules caches imported modules.""" + import json + + assert "json" in sys.modules + assert sys.modules["json"] is json + + +class TestPackageUtilities: + """Test pkgutil and importlib.metadata.""" + + def test_pkgutil_iter_modules(self) -> None: + """pkgutil.iter_modules lists available modules.""" + modules = list(pkgutil.iter_modules()) + assert len(modules) > 0 + + def test_pkgutil_module_info(self) -> None: + """Module info includes name and ispkg flag.""" + for info in pkgutil.iter_modules(): + assert hasattr(info, "name") + assert hasattr(info, "ispkg") + break # Just check first one + + def test_importlib_metadata_version(self) -> None: + """importlib.metadata.version returns package version.""" + version = importlib.metadata.version("pip") + assert isinstance(version, str) + assert len(version) > 0 + + def test_importlib_metadata_packages(self) -> None: + """importlib.metadata.packages_distributions maps packages.""" + distributions = importlib.metadata.packages_distributions() + assert isinstance(distributions, dict) + assert len(distributions) > 0 + + def test_importlib_metadata_not_found(self) -> None: + """Missing packages raise PackageNotFoundError.""" + try: + importlib.metadata.version("nonexistent_package_xyz_123") + assert False, "Should have raised" + except importlib.metadata.PackageNotFoundError: + pass + + def test_module_is_package(self) -> None: + """Packages have __path__ attribute, modules don't.""" + import email + import math + + assert hasattr(email, "__path__") # email is a package + assert not hasattr(math, "__path__") # math is a builtin module + + def test_sys_builtin_module_names(self) -> None: + """sys.builtin_module_names lists C-builtin modules.""" + assert "sys" in sys.builtin_module_names + assert isinstance(sys.builtin_module_names, tuple) diff --git a/tests/test_chapter_41.py b/tests/test_chapter_41.py new file mode 100644 index 0000000..c9b6059 --- /dev/null +++ b/tests/test_chapter_41.py @@ -0,0 +1,212 @@ +"""Tests for Chapter 41: Concurrency Patterns.""" + +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from queue import Empty, Queue + + +class TestThreadingSynchronization: + """Test threading synchronization primitives.""" + + def test_lock_mutual_exclusion(self) -> None: + """Lock ensures mutual exclusion.""" + lock = threading.Lock() + counter = [0] + + def increment() -> None: + for _ in range(1000): + with lock: + counter[0] += 1 + + threads = [threading.Thread(target=increment) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert counter[0] == 4000 + + def test_rlock_reentrant(self) -> None: + """RLock allows reentrant locking.""" + lock = threading.RLock() + with lock: + with lock: # Would deadlock with regular Lock + acquired = True + assert acquired + + def test_event_signaling(self) -> None: + """Event signals between threads.""" + event = threading.Event() + result: list[str] = [] + + def waiter() -> None: + event.wait(timeout=5) + result.append("done") + + t = threading.Thread(target=waiter) + t.start() + event.set() + t.join() + assert result == ["done"] + + def test_semaphore_limits_access(self) -> None: + """Semaphore limits concurrent access.""" + sem = threading.Semaphore(2) + max_concurrent = [0] + current = [0] + lock = threading.Lock() + + def worker() -> None: + with sem: + with lock: + current[0] += 1 + if current[0] > max_concurrent[0]: + max_concurrent[0] = current[0] + time.sleep(0.01) + with lock: + current[0] -= 1 + + threads = [threading.Thread(target=worker) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + assert max_concurrent[0] <= 2 + + def test_barrier(self) -> None: + """Barrier synchronizes a fixed number of threads.""" + barrier = threading.Barrier(3) + arrivals: list[int] = [] + lock = threading.Lock() + + def worker(worker_id: int) -> None: + barrier.wait() + with lock: + arrivals.append(worker_id) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(3)] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(arrivals) == 3 + + +class TestConcurrentFutures: + """Test concurrent.futures high-level API.""" + + def test_thread_pool_submit(self) -> None: + """ThreadPoolExecutor.submit returns a Future.""" + with ThreadPoolExecutor(max_workers=2) as pool: + future = pool.submit(lambda x: x**2, 5) + assert isinstance(future, Future) + assert future.result() == 25 + + def test_thread_pool_map(self) -> None: + """ThreadPoolExecutor.map applies function to iterables.""" + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda x: x * 2, [1, 2, 3, 4])) + assert results == [2, 4, 6, 8] + + def test_as_completed(self) -> None: + """as_completed yields futures as they finish.""" + with ThreadPoolExecutor(max_workers=2) as pool: + futures = {pool.submit(lambda x: x**2, i): i for i in range(5)} + results: list[int] = [] + for future in as_completed(futures): + results.append(future.result()) + assert sorted(results) == [0, 1, 4, 9, 16] + + def test_future_exception(self) -> None: + """Future captures exceptions from workers.""" + + def failing() -> None: + raise ValueError("task failed") + + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(failing) + assert future.exception() is not None + assert isinstance(future.exception(), ValueError) + + def test_future_done_and_cancel(self) -> None: + """Futures report completion status.""" + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(lambda: 42) + result = future.result() + assert result == 42 + assert future.done() + + +class TestThreadSafePatterns: + """Test thread-safe data patterns.""" + + def test_queue_basic(self) -> None: + """Queue provides thread-safe FIFO.""" + q: Queue[int] = Queue() + q.put(1) + q.put(2) + assert q.get() == 1 + assert q.get() == 2 + + def test_queue_producer_consumer(self) -> None: + """Producer-consumer pattern with Queue.""" + q: Queue[int | None] = Queue() + results: list[int] = [] + lock = threading.Lock() + + def producer() -> None: + for i in range(5): + q.put(i) + q.put(None) # Sentinel + + def consumer() -> None: + while True: + item = q.get() + if item is None: + break + with lock: + results.append(item) + + t1 = threading.Thread(target=producer) + t2 = threading.Thread(target=consumer) + t1.start() + t2.start() + t1.join() + t2.join() + assert sorted(results) == [0, 1, 2, 3, 4] + + def test_queue_timeout(self) -> None: + """Queue.get raises Empty on timeout.""" + q: Queue[int] = Queue() + try: + q.get(timeout=0.01) + assert False, "Should have raised" + except Empty: + pass + + def test_thread_local_storage(self) -> None: + """threading.local provides per-thread storage.""" + local = threading.local() + results: list[int] = [] + lock = threading.Lock() + + def worker(value: int) -> None: + local.data = value + time.sleep(0.01) + with lock: + results.append(local.data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert sorted(results) == [0, 1, 2, 3] + + def test_queue_maxsize(self) -> None: + """Queue can limit its size.""" + q: Queue[int] = Queue(maxsize=2) + q.put(1) + q.put(2) + assert q.full() + assert q.qsize() == 2