diff --git a/README.md b/README.md index 958b4be..de3c1e7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Learning Python - 5th Edition (Full Notes) -Study notes and examples from **"Learning Python: Powerful Object-Oriented Programming, 5th Edition"** by Mark Lutz (O'Reilly) - the definitive guide to the Python language, covering fundamental to advanced concepts through interactive Jupyter notebooks. +Study notes and examples inspired by **"Learning Python: Powerful Object-Oriented Programming, 5th Edition"** by Mark Lutz (O'Reilly) — an independent companion resource covering core Python concepts through interactive Jupyter notebooks with original code examples. ## Requirements @@ -79,6 +79,11 @@ src/ ├── chapter_13/ # Regular Expressions and Text Processing ├── chapter_14/ # Data Structures and Collections ├── chapter_15/ # Design Patterns and Pythonic Code +├── chapter_16/ # Type Hints and Static Analysis +├── chapter_17/ # Networking and Protocols +├── chapter_18/ # Database Access +├── chapter_19/ # Packaging and Distribution +├── chapter_20/ # Python Internals and Performance └── ... tests/ ├── conftest.py # Shared pytest fixtures @@ -106,6 +111,11 @@ tests/ | 13 | Regular Expressions | re module, groups, lookahead, practical text processing | Done | | 14 | Data Structures and Collections | Counter, defaultdict, deque, dataclasses, enums | Done | | 15 | Design Patterns and Pythonic Code | SOLID, creational/structural/behavioral patterns | Done | +| 16 | Type Hints and Static Analysis | typing module, generics, Protocol, overload, mypy | Done | +| 17 | Networking and Protocols | Sockets, HTTP, urllib, asyncio networking | Done | +| 18 | Database Access | sqlite3, DB-API, transactions, row factories | Done | +| 19 | Packaging and Distribution | venv, pyproject.toml, wheels, entry points, PyPI | Done | +| 20 | Python Internals and Performance | Memory model, GIL, profiling, bytecode, optimization | Done | ## Running Notebooks diff --git a/src/chapter_16/01_annotation_fundamentals.ipynb b/src/chapter_16/01_annotation_fundamentals.ipynb new file mode 100644 index 0000000..fd79c63 --- /dev/null +++ b/src/chapter_16/01_annotation_fundamentals.ipynb @@ -0,0 +1,674 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 16: Type Annotation Fundamentals\n", + "\n", + "This notebook covers the foundations of Python's type annotation system. Type hints\n", + "make code more readable, enable static analysis with tools like mypy, and improve\n", + "IDE autocompletion - all without affecting runtime behavior.\n", + "\n", + "## Key Concepts\n", + "- Variable, function parameter, and return type annotations\n", + "- Inspecting annotations at runtime with `get_type_hints()`\n", + "- Built-in generic types: `list[int]`, `dict[str, int]`, `tuple`, `set`\n", + "- `Optional`, `Union`, and the `X | Y` union syntax (Python 3.10+)\n", + "- Class annotations and `__init__` type hints\n", + "- `None` return type for side-effect functions\n", + "- Type aliases with `TypeAlias`" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Variable and Function Annotations\n", + "\n", + "Python 3.0 introduced function annotations (PEP 3107) and Python 3.6 added variable\n", + "annotations (PEP 526). Annotations are **not enforced at runtime** - they are metadata\n", + "consumed by type checkers, IDEs, and documentation tools." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Variable annotations\n", + "name: str = \"Alice\"\n", + "age: int = 30\n", + "balance: float = 1042.50\n", + "is_active: bool = True\n", + "\n", + "# You can annotate without assigning (declaration only)\n", + "email: str # Not yet assigned - no runtime value\n", + "\n", + "\n", + "# Function annotations: parameters and return type\n", + "def greet(name: str, excited: bool = False) -> str:\n", + " \"\"\"Return a greeting for the given name.\"\"\"\n", + " base = f\"Hello, {name}!\"\n", + " return base.upper() if excited else base\n", + "\n", + "\n", + "print(greet(\"Alice\"))\n", + "print(greet(\"Bob\", excited=True))\n", + "\n", + "# Annotations don't prevent wrong types at runtime (no enforcement)\n", + "print(greet(42)) # Works at runtime, but mypy would flag this" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inspecting Annotations at Runtime\n", + "\n", + "Function annotations are stored in `__annotations__`. The `typing.get_type_hints()`\n", + "function is the preferred way to access them - it resolves forward references and\n", + "string annotations that `__annotations__` leaves as raw strings." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import get_type_hints\n", + "\n", + "\n", + "def calculate_bmi(weight_kg: float, height_m: float) -> float:\n", + " \"\"\"Calculate Body Mass Index.\"\"\"\n", + " return weight_kg / (height_m ** 2)\n", + "\n", + "\n", + "# Raw __annotations__ dict\n", + "print(\"__annotations__:\", calculate_bmi.__annotations__)\n", + "\n", + "# get_type_hints() is preferred - resolves forward references\n", + "hints = get_type_hints(calculate_bmi)\n", + "print(\"get_type_hints():\", hints)\n", + "\n", + "# Iterate over parameter types\n", + "for param, type_hint in hints.items():\n", + " print(f\" {param}: {type_hint.__name__ if hasattr(type_hint, '__name__') else type_hint}\")\n", + "\n", + "# Module-level variable annotations are in the module's __annotations__\n", + "print(\"\\nModule-level annotations (sample):\")\n", + "for var_name, var_type in list(__annotations__.items())[:4]:\n", + " print(f\" {var_name}: {var_type}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Built-in Generic Types\n", + "\n", + "Since Python 3.9 (PEP 585), you can use built-in collection types directly as\n", + "generics: `list[int]`, `dict[str, float]`, `tuple[str, ...]`, `set[int]`.\n", + "Before 3.9, you needed `typing.List`, `typing.Dict`, etc." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# list[T] - homogeneous sequence\n", + "scores: list[int] = [95, 87, 92, 78]\n", + "\n", + "# dict[K, V] - key-value mapping\n", + "user_ages: dict[str, int] = {\"Alice\": 30, \"Bob\": 25, \"Carol\": 28}\n", + "\n", + "# set[T] - unique elements\n", + "unique_tags: set[str] = {\"python\", \"typing\", \"tutorial\"}\n", + "\n", + "# tuple[T1, T2, ...] - fixed-length, heterogeneous\n", + "point: tuple[float, float] = (3.14, 2.72)\n", + "record: tuple[str, int, bool] = (\"Alice\", 30, True)\n", + "\n", + "# tuple[T, ...] - variable-length, homogeneous (the ellipsis is key)\n", + "readings: tuple[float, ...] = (98.6, 99.1, 97.8, 98.2)\n", + "\n", + "\n", + "def average(values: list[float]) -> float:\n", + " \"\"\"Calculate the average of a list of floats.\"\"\"\n", + " return sum(values) / len(values)\n", + "\n", + "\n", + "def merge_dicts(a: dict[str, int], b: dict[str, int]) -> dict[str, int]:\n", + " \"\"\"Merge two dictionaries, summing values for shared keys.\"\"\"\n", + " result = dict(a)\n", + " for key, value in b.items():\n", + " result[key] = result.get(key, 0) + value\n", + " return result\n", + "\n", + "\n", + "print(f\"Average: {average([1.0, 2.0, 3.0, 4.0])}\")\n", + "print(f\"Merged: {merge_dicts({'a': 1, 'b': 2}, {'b': 3, 'c': 4})}\")\n", + "\n", + "\n", + "# Nested generics work naturally\n", + "matrix: list[list[int]] = [\n", + " [1, 2, 3],\n", + " [4, 5, 6],\n", + " [7, 8, 9],\n", + "]\n", + "\n", + "user_data: dict[str, list[str]] = {\n", + " \"alice\": [\"admin\", \"editor\"],\n", + " \"bob\": [\"viewer\"],\n", + "}\n", + "\n", + "print(f\"Matrix: {matrix}\")\n", + "print(f\"User roles: {user_data}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optional and Union Types\n", + "\n", + "- `Union[X, Y]` means the value can be type `X` or type `Y`\n", + "- `Optional[X]` is shorthand for `Union[X, None]` - value can be `X` or `None`\n", + "- Python 3.10+ introduced `X | Y` syntax as a cleaner alternative to `Union`" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Optional, Union\n", + "\n", + "\n", + "# Union: accepts multiple types\n", + "def format_id(id_value: Union[int, str]) -> str:\n", + " \"\"\"Format an ID that might be numeric or string.\"\"\"\n", + " if isinstance(id_value, int):\n", + " return f\"ID-{id_value:06d}\"\n", + " return f\"ID-{id_value}\"\n", + "\n", + "\n", + "print(format_id(42))\n", + "print(format_id(\"ABC123\"))\n", + "\n", + "\n", + "# Optional: value or None\n", + "def find_user(user_id: int) -> Optional[str]:\n", + " \"\"\"Look up a user by ID; returns None if not found.\"\"\"\n", + " users = {1: \"Alice\", 2: \"Bob\", 3: \"Carol\"}\n", + " return users.get(user_id)\n", + "\n", + "\n", + "result = find_user(1)\n", + "print(f\"Found: {result}\")\n", + "\n", + "result = find_user(99)\n", + "print(f\"Not found: {result}\")\n", + "\n", + "\n", + "# Python 3.10+ union syntax: X | Y (cleaner and preferred)\n", + "def format_id_modern(id_value: int | str) -> str:\n", + " \"\"\"Same as format_id but using the modern | syntax.\"\"\"\n", + " if isinstance(id_value, int):\n", + " return f\"ID-{id_value:06d}\"\n", + " return f\"ID-{id_value}\"\n", + "\n", + "\n", + "def find_user_modern(user_id: int) -> str | None:\n", + " \"\"\"Same as find_user using modern syntax. str | None == Optional[str].\"\"\"\n", + " users = {1: \"Alice\", 2: \"Bob\", 3: \"Carol\"}\n", + " return users.get(user_id)\n", + "\n", + "\n", + "print(f\"\\nModern syntax: {format_id_modern(42)}\")\n", + "print(f\"Modern optional: {find_user_modern(2)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Class Annotations and `__init__` Type Hints\n", + "\n", + "Class-level annotations describe instance attributes. By convention, type hints\n", + "for instance attributes go in `__init__`, while class-level annotations describe\n", + "the expected shape of the class." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import get_type_hints\n", + "\n", + "\n", + "class Product:\n", + " \"\"\"A product with typed attributes.\"\"\"\n", + "\n", + " # Class-level annotations declare the attribute types\n", + " name: str\n", + " price: float\n", + " tags: list[str]\n", + " description: str | None\n", + "\n", + " def __init__(\n", + " self,\n", + " name: str,\n", + " price: float,\n", + " tags: list[str] | None = None,\n", + " description: str | None = None,\n", + " ) -> None:\n", + " self.name = name\n", + " self.price = price\n", + " self.tags = tags or []\n", + " self.description = description\n", + "\n", + " def display_price(self) -> str:\n", + " return f\"${self.price:.2f}\"\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Product({self.name!r}, {self.display_price()}, tags={self.tags})\"\n", + "\n", + "\n", + "laptop = Product(\"Laptop\", 999.99, [\"electronics\", \"computers\"])\n", + "book = Product(\"Python Cookbook\", 49.99, description=\"Recipes for mastering Python\")\n", + "\n", + "print(laptop)\n", + "print(book)\n", + "\n", + "# Inspect class annotations\n", + "print(\"\\nClass annotations:\")\n", + "for attr, hint in get_type_hints(Product).items():\n", + " print(f\" {attr}: {hint}\")\n", + "\n", + "# Inspect __init__ annotations\n", + "print(\"\\n__init__ annotations:\")\n", + "for param, hint in get_type_hints(Product.__init__).items():\n", + " if param != \"return\":\n", + " print(f\" {param}: {hint}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## None Return Type for Side-Effect Functions\n", + "\n", + "Functions that perform side effects (printing, writing files, mutating state) and\n", + "don't return a meaningful value should be annotated with `-> None`. This makes the\n", + "intent explicit and helps type checkers catch accidental use of the return value." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class Logger:\n", + " \"\"\"A simple logger demonstrating None return types.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._messages: list[str] = []\n", + "\n", + " def log(self, message: str) -> None:\n", + " \"\"\"Append a message. Returns nothing (side effect only).\"\"\"\n", + " self._messages.append(message)\n", + "\n", + " def clear(self) -> None:\n", + " \"\"\"Clear all messages. Returns nothing.\"\"\"\n", + " self._messages.clear()\n", + "\n", + " def get_messages(self) -> list[str]:\n", + " \"\"\"Return a copy of all messages.\"\"\"\n", + " return list(self._messages)\n", + "\n", + "\n", + "def print_header(title: str, width: int = 40) -> None:\n", + " \"\"\"Print a formatted header. Side-effect only.\"\"\"\n", + " print(\"=\" * width)\n", + " print(title.center(width))\n", + " print(\"=\" * width)\n", + "\n", + "\n", + "def update_dict_inplace(d: dict[str, int], key: str, value: int) -> None:\n", + " \"\"\"Mutate a dictionary in place. The -> None signals mutation.\"\"\"\n", + " d[key] = value\n", + "\n", + "\n", + "logger = Logger()\n", + "logger.log(\"Application started\")\n", + "logger.log(\"Processing data\")\n", + "print(f\"Messages: {logger.get_messages()}\")\n", + "\n", + "# mypy would flag this as an error:\n", + "# result: str = logger.log(\"test\") # error: incompatible types\n", + "\n", + "print_header(\"Type Hints Demo\")\n", + "\n", + "data: dict[str, int] = {\"a\": 1}\n", + "update_dict_inplace(data, \"b\", 2)\n", + "print(f\"Updated dict: {data}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Type Aliases with TypeAlias\n", + "\n", + "Complex type annotations can become unwieldy. **Type aliases** give meaningful names\n", + "to complex types, improving readability. Python 3.10 introduced the `TypeAlias`\n", + "annotation (PEP 613) to make aliases explicit, and Python 3.12 added the `type`\n", + "statement (PEP 695) for an even cleaner syntax." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypeAlias\n", + "\n", + "# Without aliases - hard to read\n", + "# def process(data: list[dict[str, list[tuple[int, float]]]]) -> dict[str, float]: ...\n", + "\n", + "# With TypeAlias - explicit and clear\n", + "Coordinate: TypeAlias = tuple[float, float]\n", + "Matrix: TypeAlias = list[list[float]]\n", + "Headers: TypeAlias = dict[str, str]\n", + "UserId: TypeAlias = int\n", + "JsonValue: TypeAlias = str | int | float | bool | None | list[\"JsonValue\"] | dict[str, \"JsonValue\"]\n", + "\n", + "\n", + "def distance(a: Coordinate, b: Coordinate) -> float:\n", + " \"\"\"Calculate Euclidean distance between two coordinates.\"\"\"\n", + " return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2) ** 0.5\n", + "\n", + "\n", + "def transpose(m: Matrix) -> Matrix:\n", + " \"\"\"Transpose a matrix.\"\"\"\n", + " return [list(row) for row in zip(*m)]\n", + "\n", + "\n", + "def get_user_name(user_id: UserId) -> str:\n", + " \"\"\"Look up user by their ID.\"\"\"\n", + " users: dict[UserId, str] = {1: \"Alice\", 2: \"Bob\"}\n", + " return users.get(user_id, \"Unknown\")\n", + "\n", + "\n", + "# Using the aliases\n", + "origin: Coordinate = (0.0, 0.0)\n", + "target: Coordinate = (3.0, 4.0)\n", + "print(f\"Distance: {distance(origin, target)}\")\n", + "\n", + "m: Matrix = [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]\n", + "print(f\"Original: {m}\")\n", + "print(f\"Transposed: {transpose(m)}\")\n", + "\n", + "print(f\"User 1: {get_user_name(1)}\")\n", + "print(f\"User 99: {get_user_name(99)}\")\n", + "\n", + "\n", + "# Python 3.12+ syntax (PEP 695) - even cleaner\n", + "# type Coordinate = tuple[float, float]\n", + "# type Matrix = list[list[float]]\n", + "# type JsonValue = str | int | float | bool | None | list[JsonValue] | dict[str, JsonValue]" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Callable Types\n", + "\n", + "Functions are first-class objects in Python. The `Callable` type from `collections.abc`\n", + "(or `typing`) describes the signature of a callable: `Callable[[ParamTypes], ReturnType]`." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from collections.abc import Callable\n", + "\n", + "\n", + "# A function that takes a transformation function as a parameter\n", + "def apply_to_all(\n", + " items: list[str],\n", + " transform: Callable[[str], str],\n", + ") -> list[str]:\n", + " \"\"\"Apply a transformation function to every item.\"\"\"\n", + " return [transform(item) for item in items]\n", + "\n", + "\n", + "# A function that returns a function (factory)\n", + "def make_multiplier(factor: float) -> Callable[[float], float]:\n", + " \"\"\"Create a multiplier function.\"\"\"\n", + " def multiplier(x: float) -> float:\n", + " return x * factor\n", + " return multiplier\n", + "\n", + "\n", + "# A callback type alias for clarity\n", + "ErrorHandler: TypeAlias = Callable[[Exception], None]\n", + "\n", + "\n", + "def safe_divide(\n", + " a: float,\n", + " b: float,\n", + " on_error: ErrorHandler | None = None,\n", + ") -> float | None:\n", + " \"\"\"Divide a by b, with optional error handler callback.\"\"\"\n", + " try:\n", + " return a / b\n", + " except ZeroDivisionError as e:\n", + " if on_error:\n", + " on_error(e)\n", + " return None\n", + "\n", + "\n", + "# Using apply_to_all\n", + "words = [\"hello\", \"world\", \"python\"]\n", + "print(f\"Upper: {apply_to_all(words, str.upper)}\")\n", + "print(f\"Title: {apply_to_all(words, str.title)}\")\n", + "\n", + "# Using make_multiplier\n", + "double = make_multiplier(2.0)\n", + "triple = make_multiplier(3.0)\n", + "print(f\"Double 5: {double(5.0)}\")\n", + "print(f\"Triple 5: {triple(5.0)}\")\n", + "\n", + "# Using safe_divide with error handler\n", + "def log_error(e: Exception) -> None:\n", + " print(f\" Error caught: {e}\")\n", + "\n", + "\n", + "print(f\"10 / 3 = {safe_divide(10, 3)}\")\n", + "print(f\"10 / 0 = {safe_divide(10, 0, on_error=log_error)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Putting It All Together\n", + "\n", + "Here is a small, fully-annotated module that combines variable annotations, function\n", + "signatures, class hints, generics, Optional, and type aliases into a cohesive example." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypeAlias\n", + "from collections.abc import Callable\n", + "\n", + "# Type aliases\n", + "Score: TypeAlias = float\n", + "StudentName: TypeAlias = str\n", + "GradeBook: TypeAlias = dict[StudentName, list[Score]]\n", + "GradingPolicy: TypeAlias = Callable[[list[Score]], str]\n", + "\n", + "\n", + "class Classroom:\n", + " \"\"\"A fully type-annotated classroom manager.\"\"\"\n", + "\n", + " name: str\n", + " _grades: GradeBook\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + " self._grades: GradeBook = {}\n", + "\n", + " def add_student(self, student: StudentName) -> None:\n", + " \"\"\"Register a student with an empty score list.\"\"\"\n", + " if student not in self._grades:\n", + " self._grades[student] = []\n", + "\n", + " def record_score(self, student: StudentName, score: Score) -> None:\n", + " \"\"\"Record a score for an existing student.\"\"\"\n", + " if student not in self._grades:\n", + " raise KeyError(f\"Student {student!r} not found\")\n", + " self._grades[student].append(score)\n", + "\n", + " def get_average(self, student: StudentName) -> Score | None:\n", + " \"\"\"Return the student's average score, or None if no scores.\"\"\"\n", + " scores = self._grades.get(student, [])\n", + " if not scores:\n", + " return None\n", + " return sum(scores) / len(scores)\n", + "\n", + " def apply_policy(\n", + " self,\n", + " student: StudentName,\n", + " policy: GradingPolicy,\n", + " ) -> str | None:\n", + " \"\"\"Apply a grading policy to a student's scores.\"\"\"\n", + " scores = self._grades.get(student)\n", + " if scores is None:\n", + " return None\n", + " return policy(scores)\n", + "\n", + " def summary(self) -> dict[StudentName, Score | None]:\n", + " \"\"\"Return a summary of all students and their averages.\"\"\"\n", + " return {name: self.get_average(name) for name in self._grades}\n", + "\n", + "\n", + "# A grading policy as a typed callable\n", + "def letter_grade(scores: list[Score]) -> str:\n", + " \"\"\"Convert a list of scores to a letter grade.\"\"\"\n", + " avg = sum(scores) / len(scores) if scores else 0\n", + " if avg >= 90:\n", + " return \"A\"\n", + " elif avg >= 80:\n", + " return \"B\"\n", + " elif avg >= 70:\n", + " return \"C\"\n", + " elif avg >= 60:\n", + " return \"D\"\n", + " return \"F\"\n", + "\n", + "\n", + "# Usage\n", + "room = Classroom(\"Python 101\")\n", + "room.add_student(\"Alice\")\n", + "room.add_student(\"Bob\")\n", + "\n", + "for score in [92.0, 88.5, 95.0]:\n", + " room.record_score(\"Alice\", score)\n", + "for score in [75.0, 82.0, 68.5]:\n", + " room.record_score(\"Bob\", score)\n", + "\n", + "print(f\"Classroom: {room.name}\")\n", + "print(f\"Averages: {room.summary()}\")\n", + "print(f\"Alice's grade: {room.apply_policy('Alice', letter_grade)}\")\n", + "print(f\"Bob's grade: {room.apply_policy('Bob', letter_grade)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Type Annotation Syntax\n", + "- **Variables**: `name: str = \"Alice\"` - annotate with `: Type` after the name\n", + "- **Function parameters**: `def greet(name: str)` - annotate each parameter\n", + "- **Return types**: `def greet(name: str) -> str` - use `->` after the parameter list\n", + "- **None return**: `def log(msg: str) -> None` - explicit for side-effect functions\n", + "\n", + "### Built-in Generics (Python 3.9+)\n", + "- `list[int]`, `dict[str, float]`, `set[str]`, `tuple[int, str]`, `tuple[int, ...]`\n", + "- Nested generics: `dict[str, list[int]]`, `list[tuple[str, float]]`\n", + "\n", + "### Union and Optional\n", + "- `Union[X, Y]` or `X | Y` (3.10+): value is one of multiple types\n", + "- `Optional[X]` or `X | None`: value can be `X` or `None`\n", + "- Prefer the `X | Y` syntax in modern Python\n", + "\n", + "### Type Aliases\n", + "- `TypeAlias` (3.10+): `Coordinate: TypeAlias = tuple[float, float]`\n", + "- `type` statement (3.12+): `type Coordinate = tuple[float, float]`\n", + "- Use aliases to name complex types for readability\n", + "\n", + "### Inspecting Annotations\n", + "- `typing.get_type_hints(obj)` is preferred over `obj.__annotations__`\n", + "- It resolves forward references and handles `from __future__ import annotations`" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_16/02_advanced_typing.ipynb b/src/chapter_16/02_advanced_typing.ipynb new file mode 100644 index 0000000..0d4172c --- /dev/null +++ b/src/chapter_16/02_advanced_typing.ipynb @@ -0,0 +1,897 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 16: Advanced Typing Constructs\n", + "\n", + "This notebook explores Python's advanced type system features for building\n", + "generic, flexible, and precisely-typed code. These tools go beyond basic annotations\n", + "to express complex relationships between types.\n", + "\n", + "## Key Concepts\n", + "- `TypeVar` for generic functions and classes\n", + "- `Generic[T]` for parameterized classes\n", + "- `Protocol` for structural subtyping (duck typing with type safety)\n", + "- `@runtime_checkable` for isinstance checks with Protocols\n", + "- `@overload` for documenting multiple call signatures\n", + "- `TypedDict` for typed dictionary shapes\n", + "- `Literal` for restricting to specific values\n", + "- `Final` and `ClassVar` for constants and class-level state" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TypeVar for Generic Functions\n", + "\n", + "A `TypeVar` is a placeholder for a type that gets filled in when the function is\n", + "called. It lets you write functions that preserve the relationship between input\n", + "and output types without committing to a specific type." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypeVar\n", + "\n", + "# T can be any type - it links input and output\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "def first(items: list[T]) -> T:\n", + " \"\"\"Return the first element. The return type matches the list element type.\"\"\"\n", + " if not items:\n", + " raise ValueError(\"Empty list\")\n", + " return items[0]\n", + "\n", + "\n", + "def identity(value: T) -> T:\n", + " \"\"\"Return the value unchanged. Output type == input type.\"\"\"\n", + " return value\n", + "\n", + "\n", + "# Type checker knows these return types:\n", + "x: int = first([1, 2, 3]) # T is int\n", + "y: str = first([\"a\", \"b\", \"c\"]) # T is str\n", + "z: float = identity(3.14) # T is float\n", + "\n", + "print(f\"first([1, 2, 3]) = {x}\")\n", + "print(f\"first(['a', 'b', 'c']) = {y}\")\n", + "print(f\"identity(3.14) = {z}\")\n", + "\n", + "\n", + "# Bounded TypeVar: restrict T to specific types or their subclasses\n", + "Number = TypeVar(\"Number\", int, float)\n", + "\n", + "\n", + "def add(a: Number, b: Number) -> Number:\n", + " \"\"\"Add two numbers. Only int or float allowed.\"\"\"\n", + " return a + b\n", + "\n", + "\n", + "print(f\"\\nadd(3, 4) = {add(3, 4)}\")\n", + "print(f\"add(1.5, 2.5) = {add(1.5, 2.5)}\")\n", + "# add(\"a\", \"b\") # mypy error: Value of type variable \"Number\" cannot be \"str\"\n", + "\n", + "\n", + "# Upper-bound TypeVar: T must be a subclass of the bound\n", + "Comparable = TypeVar(\"Comparable\", bound=float)\n", + "\n", + "\n", + "def clamp(value: Comparable, low: Comparable, high: Comparable) -> Comparable:\n", + " \"\"\"Clamp a value between low and high bounds.\"\"\"\n", + " if value < low:\n", + " return low\n", + " if value > high:\n", + " return high\n", + " return value\n", + "\n", + "\n", + "print(f\"clamp(15, 0, 10) = {clamp(15, 0, 10)}\")\n", + "print(f\"clamp(-5, 0, 10) = {clamp(-5, 0, 10)}\")\n", + "print(f\"clamp(3.7, 0.0, 10.0) = {clamp(3.7, 0.0, 10.0)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generic Classes with `Generic[T]`\n", + "\n", + "To create a class that is parameterized by one or more types, inherit from\n", + "`Generic[T]`. This makes the class a generic type that can be specialized\n", + "when used: `Stack[int]`, `Stack[str]`, etc." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypeVar, Generic\n", + "\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "class Stack(Generic[T]):\n", + " \"\"\"A type-safe stack. Stack[int] only holds ints, Stack[str] only strings.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._items: list[T] = []\n", + "\n", + " def push(self, item: T) -> None:\n", + " \"\"\"Push an item onto the stack.\"\"\"\n", + " self._items.append(item)\n", + "\n", + " def pop(self) -> T:\n", + " \"\"\"Remove and return the top item.\"\"\"\n", + " if not self._items:\n", + " raise IndexError(\"Pop from empty stack\")\n", + " return self._items.pop()\n", + "\n", + " def peek(self) -> T:\n", + " \"\"\"Return the top item without removing it.\"\"\"\n", + " if not self._items:\n", + " raise IndexError(\"Peek at empty stack\")\n", + " return self._items[-1]\n", + "\n", + " def is_empty(self) -> bool:\n", + " return len(self._items) == 0\n", + "\n", + " def __len__(self) -> int:\n", + " return len(self._items)\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Stack({self._items})\"\n", + "\n", + "\n", + "# Usage with specific types\n", + "int_stack: Stack[int] = Stack()\n", + "int_stack.push(1)\n", + "int_stack.push(2)\n", + "int_stack.push(3)\n", + "print(f\"Int stack: {int_stack}\")\n", + "print(f\"Pop: {int_stack.pop()}\")\n", + "print(f\"Peek: {int_stack.peek()}\")\n", + "\n", + "str_stack: Stack[str] = Stack()\n", + "str_stack.push(\"hello\")\n", + "str_stack.push(\"world\")\n", + "print(f\"\\nStr stack: {str_stack}\")\n", + "print(f\"Pop: {str_stack.pop()}\")\n", + "\n", + "\n", + "# Multi-type generic class\n", + "K = TypeVar(\"K\")\n", + "V = TypeVar(\"V\")\n", + "\n", + "\n", + "class Pair(Generic[K, V]):\n", + " \"\"\"An immutable pair of two values with different types.\"\"\"\n", + "\n", + " def __init__(self, key: K, value: V) -> None:\n", + " self._key = key\n", + " self._value = value\n", + "\n", + " @property\n", + " def key(self) -> K:\n", + " return self._key\n", + "\n", + " @property\n", + " def value(self) -> V:\n", + " return self._value\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Pair({self._key!r}, {self._value!r})\"\n", + "\n", + "\n", + "p1: Pair[str, int] = Pair(\"age\", 30)\n", + "p2: Pair[int, list[str]] = Pair(1, [\"a\", \"b\"])\n", + "\n", + "print(f\"\\n{p1} -> key={p1.key}, value={p1.value}\")\n", + "print(f\"{p2} -> key={p2.key}, value={p2.value}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Protocol for Structural Subtyping\n", + "\n", + "`Protocol` (PEP 544) formalizes duck typing. A class satisfies a Protocol if it has\n", + "the required methods and attributes - no inheritance needed. This is **structural\n", + "subtyping** (\"if it quacks like a duck\") vs **nominal subtyping** (\"if it inherits\n", + "from Duck\")." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Protocol\n", + "\n", + "\n", + "class Drawable(Protocol):\n", + " \"\"\"Any object that can draw itself. No inheritance required.\"\"\"\n", + "\n", + " def draw(self) -> str:\n", + " \"\"\"Return a string representation of the drawing.\"\"\"\n", + " ...\n", + "\n", + "\n", + "class Resizable(Protocol):\n", + " \"\"\"Any object that can be resized.\"\"\"\n", + "\n", + " def resize(self, factor: float) -> None: ...\n", + "\n", + "\n", + "# These classes DON'T inherit from Drawable or Resizable\n", + "class Circle:\n", + " def __init__(self, radius: float) -> None:\n", + " self.radius = radius\n", + "\n", + " def draw(self) -> str:\n", + " return f\"Circle(r={self.radius})\"\n", + "\n", + " def resize(self, factor: float) -> None:\n", + " self.radius *= factor\n", + "\n", + "\n", + "class Square:\n", + " def __init__(self, side: float) -> None:\n", + " self.side = side\n", + "\n", + " def draw(self) -> str:\n", + " return f\"Square(s={self.side})\"\n", + "\n", + " def resize(self, factor: float) -> None:\n", + " self.side *= factor\n", + "\n", + "\n", + "class TextLabel:\n", + " \"\"\"Only Drawable - not Resizable.\"\"\"\n", + "\n", + " def __init__(self, text: str) -> None:\n", + " self.text = text\n", + "\n", + " def draw(self) -> str:\n", + " return f\"Label({self.text!r})\"\n", + "\n", + "\n", + "# Functions accept the Protocol type\n", + "def render_all(shapes: list[Drawable]) -> None:\n", + " \"\"\"Render all drawable objects.\"\"\"\n", + " for shape in shapes:\n", + " print(f\" Drawing: {shape.draw()}\")\n", + "\n", + "\n", + "def scale_all(shapes: list[Resizable], factor: float) -> None:\n", + " \"\"\"Scale all resizable objects.\"\"\"\n", + " for shape in shapes:\n", + " shape.resize(factor)\n", + "\n", + "\n", + "# All three are Drawable\n", + "drawables: list[Drawable] = [Circle(5.0), Square(3.0), TextLabel(\"Hello\")]\n", + "print(\"All drawables:\")\n", + "render_all(drawables)\n", + "\n", + "# Only Circle and Square are Resizable\n", + "resizables: list[Resizable] = [Circle(5.0), Square(3.0)]\n", + "scale_all(resizables, 2.0)\n", + "print(\"\\nAfter scaling 2x:\")\n", + "render_all(resizables) # Circle and Square are also Drawable" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `@runtime_checkable` for isinstance Checks\n", + "\n", + "By default, Protocols are only checked statically by type checkers. Adding\n", + "`@runtime_checkable` enables `isinstance()` checks at runtime. Note that\n", + "runtime checks only verify method **existence**, not their **signatures**." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Protocol, runtime_checkable\n", + "\n", + "\n", + "@runtime_checkable\n", + "class Closeable(Protocol):\n", + " \"\"\"Any object that can be closed.\"\"\"\n", + " def close(self) -> None: ...\n", + "\n", + "\n", + "@runtime_checkable\n", + "class Sized(Protocol):\n", + " \"\"\"Any object with a length.\"\"\"\n", + " def __len__(self) -> int: ...\n", + "\n", + "\n", + "class DatabaseConnection:\n", + " \"\"\"Has a close() method - satisfies Closeable.\"\"\"\n", + " def __init__(self, url: str) -> None:\n", + " self.url = url\n", + " self._open = True\n", + "\n", + " def close(self) -> None:\n", + " self._open = False\n", + " print(f\" Closed connection to {self.url}\")\n", + "\n", + "\n", + "class FileWrapper:\n", + " \"\"\"Has both close() and __len__() - satisfies both protocols.\"\"\"\n", + " def __init__(self, content: str) -> None:\n", + " self.content = content\n", + "\n", + " def close(self) -> None:\n", + " self.content = \"\"\n", + " print(\" Closed file wrapper\")\n", + "\n", + " def __len__(self) -> int:\n", + " return len(self.content)\n", + "\n", + "\n", + "# isinstance checks with runtime_checkable Protocol\n", + "db = DatabaseConnection(\"postgres://localhost/mydb\")\n", + "fw = FileWrapper(\"Hello, World!\")\n", + "plain_list = [1, 2, 3]\n", + "\n", + "print(\"Closeable checks:\")\n", + "print(f\" DatabaseConnection: {isinstance(db, Closeable)}\")\n", + "print(f\" FileWrapper: {isinstance(fw, Closeable)}\")\n", + "print(f\" list: {isinstance(plain_list, Closeable)}\")\n", + "\n", + "print(\"\\nSized checks:\")\n", + "print(f\" FileWrapper: {isinstance(fw, Sized)}\")\n", + "print(f\" list: {isinstance(plain_list, Sized)}\")\n", + "print(f\" DatabaseConnection: {isinstance(db, Sized)}\")\n", + "\n", + "\n", + "# Practical use: close anything that is Closeable\n", + "def cleanup(resources: list[object]) -> None:\n", + " \"\"\"Close all resources that support it.\"\"\"\n", + " for resource in resources:\n", + " if isinstance(resource, Closeable):\n", + " resource.close()\n", + "\n", + "\n", + "print(\"\\nCleaning up:\")\n", + "cleanup([db, fw, plain_list, \"just a string\"])" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `@overload` for Multiple Signatures\n", + "\n", + "The `@overload` decorator documents that a function behaves differently depending\n", + "on the types of its arguments. The overloaded signatures are for the **type checker\n", + "only** - the actual implementation handles all cases." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import overload\n", + "\n", + "\n", + "# Overloaded signatures: type checker uses these\n", + "@overload\n", + "def process(value: int) -> str: ...\n", + "\n", + "\n", + "@overload\n", + "def process(value: str) -> int: ...\n", + "\n", + "\n", + "@overload\n", + "def process(value: list[int]) -> float: ...\n", + "\n", + "\n", + "# Actual implementation: must handle all overloaded cases\n", + "def process(value: int | str | list[int]) -> str | int | float:\n", + " \"\"\"Process a value differently based on its type.\n", + "\n", + " - int -> string representation\n", + " - str -> length as int\n", + " - list[int] -> average as float\n", + " \"\"\"\n", + " if isinstance(value, int):\n", + " return f\"Number: {value}\"\n", + " elif isinstance(value, str):\n", + " return len(value)\n", + " elif isinstance(value, list):\n", + " return sum(value) / len(value) if value else 0.0\n", + " raise TypeError(f\"Unsupported type: {type(value)}\")\n", + "\n", + "\n", + "# The type checker knows the exact return type for each input type:\n", + "result1 = process(42) # type checker knows this is str\n", + "result2 = process(\"hello\") # type checker knows this is int\n", + "result3 = process([1, 2, 3]) # type checker knows this is float\n", + "\n", + "print(f\"process(42) = {result1!r}\")\n", + "print(f\"process('hello') = {result2!r}\")\n", + "print(f\"process([1, 2, 3]) = {result3!r}\")\n", + "\n", + "\n", + "# Another common use: optional parameter changes return type\n", + "@overload\n", + "def get_config(key: str) -> str | None: ...\n", + "\n", + "\n", + "@overload\n", + "def get_config(key: str, default: str) -> str: ...\n", + "\n", + "\n", + "def get_config(key: str, default: str | None = None) -> str | None:\n", + " \"\"\"Get config value. Returns default if key not found.\"\"\"\n", + " config = {\"host\": \"localhost\", \"port\": \"8080\"}\n", + " result = config.get(key)\n", + " if result is None:\n", + " return default\n", + " return result\n", + "\n", + "\n", + "# With default: always returns str (never None)\n", + "host: str = get_config(\"host\", \"127.0.0.1\")\n", + "# Without default: might return None\n", + "missing: str | None = get_config(\"database\")\n", + "\n", + "print(f\"\\nget_config('host', '127.0.0.1') = {host!r}\")\n", + "print(f\"get_config('database') = {missing!r}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TypedDict for Typed Dictionary Shapes\n", + "\n", + "`TypedDict` (PEP 589) defines the expected shape of a dictionary - which keys\n", + "exist and what types their values have. This is especially useful for JSON data,\n", + "API responses, and configuration objects." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypedDict, NotRequired\n", + "\n", + "\n", + "class Address(TypedDict):\n", + " \"\"\"A typed dictionary for address data.\"\"\"\n", + " street: str\n", + " city: str\n", + " state: str\n", + " zip_code: str\n", + "\n", + "\n", + "class UserProfile(TypedDict):\n", + " \"\"\"A typed dictionary for user profile data.\"\"\"\n", + " name: str\n", + " email: str\n", + " age: int\n", + " address: Address # Nested TypedDict\n", + " phone: NotRequired[str] # Optional key (may be absent)\n", + "\n", + "\n", + "def format_address(addr: Address) -> str:\n", + " \"\"\"Format an address for display.\"\"\"\n", + " return f\"{addr['street']}, {addr['city']}, {addr['state']} {addr['zip_code']}\"\n", + "\n", + "\n", + "def greet_user(profile: UserProfile) -> str:\n", + " \"\"\"Generate a greeting from a user profile.\"\"\"\n", + " greeting = f\"Hello, {profile['name']} ({profile['email']})\"\n", + " if 'phone' in profile:\n", + " greeting += f\" | Phone: {profile['phone']}\"\n", + " return greeting\n", + "\n", + "\n", + "# TypedDict values are created as plain dicts\n", + "alice: UserProfile = {\n", + " \"name\": \"Alice\",\n", + " \"email\": \"alice@example.com\",\n", + " \"age\": 30,\n", + " \"address\": {\n", + " \"street\": \"123 Main St\",\n", + " \"city\": \"Springfield\",\n", + " \"state\": \"IL\",\n", + " \"zip_code\": \"62701\",\n", + " },\n", + " \"phone\": \"555-0123\",\n", + "}\n", + "\n", + "bob: UserProfile = {\n", + " \"name\": \"Bob\",\n", + " \"email\": \"bob@example.com\",\n", + " \"age\": 25,\n", + " \"address\": {\n", + " \"street\": \"456 Oak Ave\",\n", + " \"city\": \"Portland\",\n", + " \"state\": \"OR\",\n", + " \"zip_code\": \"97201\",\n", + " },\n", + " # No phone key - it's NotRequired\n", + "}\n", + "\n", + "print(greet_user(alice))\n", + "print(greet_user(bob))\n", + "print(f\"Alice's address: {format_address(alice['address'])}\")\n", + "\n", + "# TypedDict is still a regular dict at runtime\n", + "print(f\"\\ntype(alice) = {type(alice)}\")\n", + "print(f\"alice.keys() = {list(alice.keys())}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Literal for Restricting to Specific Values\n", + "\n", + "`Literal` (PEP 586) restricts a type to specific literal values. This is more\n", + "precise than `str` or `int` when only certain values are valid." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Literal\n", + "\n", + "# Direction is exactly one of these four strings\n", + "Direction = Literal[\"north\", \"south\", \"east\", \"west\"]\n", + "\n", + "# LogLevel is one of these specific strings\n", + "LogLevel = Literal[\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\"]\n", + "\n", + "# HttpMethod restricted to common verbs\n", + "HttpMethod = Literal[\"GET\", \"POST\", \"PUT\", \"DELETE\", \"PATCH\"]\n", + "\n", + "\n", + "def move(direction: Direction, steps: int = 1) -> str:\n", + " \"\"\"Move in a direction. Only four directions are valid.\"\"\"\n", + " return f\"Moving {steps} step(s) {direction}\"\n", + "\n", + "\n", + "def log(message: str, level: LogLevel = \"INFO\") -> str:\n", + " \"\"\"Log a message at a specific level.\"\"\"\n", + " return f\"[{level}] {message}\"\n", + "\n", + "\n", + "def api_request(url: str, method: HttpMethod = \"GET\") -> str:\n", + " \"\"\"Simulate an API request.\"\"\"\n", + " return f\"{method} {url} -> 200 OK\"\n", + "\n", + "\n", + "print(move(\"north\", 3))\n", + "print(move(\"east\"))\n", + "# move(\"up\") # mypy error: Argument 1 has incompatible type \"str\"\n", + "\n", + "print(f\"\\n{log('Server started')}\")\n", + "print(log(\"Disk space low\", \"WARNING\"))\n", + "print(log(\"Connection failed\", \"ERROR\"))\n", + "\n", + "print(f\"\\n{api_request('/api/users')}\")\n", + "print(api_request(\"/api/users\", \"POST\"))\n", + "\n", + "\n", + "# Literal with overload for type narrowing\n", + "from typing import overload\n", + "\n", + "\n", + "@overload\n", + "def fetch(url: str, format: Literal[\"json\"]) -> dict: ...\n", + "\n", + "\n", + "@overload\n", + "def fetch(url: str, format: Literal[\"text\"]) -> str: ...\n", + "\n", + "\n", + "def fetch(url: str, format: Literal[\"json\", \"text\"] = \"json\") -> dict | str:\n", + " \"\"\"Fetch data in either JSON or text format.\"\"\"\n", + " if format == \"json\":\n", + " return {\"url\": url, \"status\": \"ok\"}\n", + " return f\"Response from {url}\"\n", + "\n", + "\n", + "json_data = fetch(\"/api/data\", \"json\") # type checker knows: dict\n", + "text_data = fetch(\"/api/data\", \"text\") # type checker knows: str\n", + "print(f\"\\nJSON: {json_data}\")\n", + "print(f\"Text: {text_data}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Final and ClassVar\n", + "\n", + "- `Final` marks a variable or attribute that should **not be reassigned** after\n", + " initialization. The type checker enforces this.\n", + "- `ClassVar` marks an attribute as belonging to the **class itself**, not to instances.\n", + " It will not appear in `__init__` generated by dataclasses." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Final, ClassVar\n", + "from dataclasses import dataclass\n", + "\n", + "# Final: constants that should never change\n", + "MAX_RETRIES: Final[int] = 3\n", + "API_BASE_URL: Final[str] = \"https://api.example.com\"\n", + "DEFAULT_TIMEOUT: Final = 30.0 # Type is inferred as float\n", + "\n", + "print(f\"MAX_RETRIES = {MAX_RETRIES}\")\n", + "print(f\"API_BASE_URL = {API_BASE_URL}\")\n", + "print(f\"DEFAULT_TIMEOUT = {DEFAULT_TIMEOUT}\")\n", + "\n", + "# MAX_RETRIES = 5 # mypy error: Cannot assign to final name \"MAX_RETRIES\"\n", + "\n", + "\n", + "# ClassVar: class-level attributes (not per-instance)\n", + "@dataclass\n", + "class HttpClient:\n", + " \"\"\"An HTTP client with class-level defaults and instance config.\"\"\"\n", + "\n", + " # ClassVar: shared across all instances, not in __init__\n", + " default_timeout: ClassVar[float] = 30.0\n", + " max_retries: ClassVar[int] = 3\n", + " _instance_count: ClassVar[int] = 0\n", + "\n", + " # Instance attributes (appear in __init__)\n", + " base_url: str\n", + " timeout: float = 30.0\n", + "\n", + " def __post_init__(self) -> None:\n", + " HttpClient._instance_count += 1\n", + "\n", + " def request(self, path: str) -> str:\n", + " return f\"GET {self.base_url}{path} (timeout={self.timeout}s)\"\n", + "\n", + " @classmethod\n", + " def get_instance_count(cls) -> int:\n", + " return cls._instance_count\n", + "\n", + "\n", + "client1 = HttpClient(\"https://api.example.com\")\n", + "client2 = HttpClient(\"https://staging.example.com\", timeout=10.0)\n", + "\n", + "print(f\"\\n{client1.request('/users')}\")\n", + "print(client2.request(\"/health\"))\n", + "print(f\"Total instances: {HttpClient.get_instance_count()}\")\n", + "print(f\"Default timeout (class): {HttpClient.default_timeout}\")\n", + "print(f\"Max retries (class): {HttpClient.max_retries}\")\n", + "\n", + "\n", + "# Final methods and classes (mypy enforced)\n", + "class Config:\n", + " \"\"\"Configuration with final attributes.\"\"\"\n", + "\n", + " VERSION: Final[str] = \"1.0.0\"\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name: Final[str] = name # Instance-level Final\n", + "\n", + "\n", + "config = Config(\"MyApp\")\n", + "print(f\"\\nConfig: {config.name} v{Config.VERSION}\")\n", + "# config.name = \"Other\" # mypy error: Cannot assign to final attribute \"name\"" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Combining Advanced Types: A Practical Example\n", + "\n", + "Let's combine several advanced typing features into a small but realistic\n", + "event-processing system." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import (\n", + " TypeVar, Generic, Protocol, TypedDict,\n", + " Literal, Final, runtime_checkable,\n", + ")\n", + "from collections.abc import Callable\n", + "\n", + "\n", + "# Literal for event types\n", + "EventType = Literal[\"click\", \"keypress\", \"scroll\", \"resize\"]\n", + "\n", + "# TypedDict for event data shapes\n", + "class ClickEvent(TypedDict):\n", + " type: Literal[\"click\"]\n", + " x: int\n", + " y: int\n", + " button: Literal[\"left\", \"right\", \"middle\"]\n", + "\n", + "\n", + "class KeypressEvent(TypedDict):\n", + " type: Literal[\"keypress\"]\n", + " key: str\n", + " modifiers: list[str]\n", + "\n", + "\n", + "# Protocol for handlers\n", + "@runtime_checkable\n", + "class EventHandler(Protocol):\n", + " def handle(self, event: dict) -> None: ...\n", + "\n", + "\n", + "# Generic event bus\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "class EventBus(Generic[T]):\n", + " \"\"\"A type-safe event bus using generics.\"\"\"\n", + "\n", + " MAX_HANDLERS: Final[int] = 100\n", + "\n", + " def __init__(self) -> None:\n", + " self._handlers: dict[str, list[Callable[[T], None]]] = {}\n", + " self._event_count: int = 0\n", + "\n", + " def on(self, event_type: EventType, handler: Callable[[T], None]) -> None:\n", + " \"\"\"Register a handler for an event type.\"\"\"\n", + " if event_type not in self._handlers:\n", + " self._handlers[event_type] = []\n", + " self._handlers[event_type].append(handler)\n", + "\n", + " def emit(self, event_type: EventType, data: T) -> int:\n", + " \"\"\"Emit an event to all registered handlers. Returns handler count.\"\"\"\n", + " self._event_count += 1\n", + " handlers = self._handlers.get(event_type, [])\n", + " for handler in handlers:\n", + " handler(data)\n", + " return len(handlers)\n", + "\n", + " @property\n", + " def total_events(self) -> int:\n", + " return self._event_count\n", + "\n", + "\n", + "# Create a typed event bus\n", + "bus: EventBus[dict] = EventBus()\n", + "\n", + "\n", + "# Handlers\n", + "def on_click(event: dict) -> None:\n", + " print(f\" Click at ({event['x']}, {event['y']}) with {event['button']} button\")\n", + "\n", + "\n", + "def on_key(event: dict) -> None:\n", + " mods = \"+\".join(event[\"modifiers\"]) if event[\"modifiers\"] else \"none\"\n", + " print(f\" Key '{event['key']}' pressed (modifiers: {mods})\")\n", + "\n", + "\n", + "def log_event(event: dict) -> None:\n", + " print(f\" [LOG] Event: {event['type']}\")\n", + "\n", + "\n", + "# Register handlers\n", + "bus.on(\"click\", on_click)\n", + "bus.on(\"click\", log_event) # Multiple handlers per event\n", + "bus.on(\"keypress\", on_key)\n", + "bus.on(\"keypress\", log_event)\n", + "\n", + "# Emit events\n", + "click: ClickEvent = {\"type\": \"click\", \"x\": 100, \"y\": 200, \"button\": \"left\"}\n", + "keypress: KeypressEvent = {\"type\": \"keypress\", \"key\": \"s\", \"modifiers\": [\"ctrl\"]}\n", + "\n", + "print(\"Emitting click:\")\n", + "count = bus.emit(\"click\", click)\n", + "print(f\" ({count} handlers called)\")\n", + "\n", + "print(\"\\nEmitting keypress:\")\n", + "count = bus.emit(\"keypress\", keypress)\n", + "print(f\" ({count} handlers called)\")\n", + "\n", + "print(f\"\\nTotal events emitted: {bus.total_events}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Generic Programming\n", + "- `TypeVar(\"T\")`: placeholder type that links inputs to outputs in generic functions\n", + "- `TypeVar(\"T\", int, float)`: constrained to specific types\n", + "- `TypeVar(\"T\", bound=Base)`: upper-bounded, must be subclass of Base\n", + "- `Generic[T]`: base class for parameterized classes like `Stack[int]`\n", + "\n", + "### Structural Subtyping\n", + "- `Protocol`: defines an interface by method signatures, no inheritance required\n", + "- `@runtime_checkable`: enables `isinstance()` checks with Protocol classes\n", + "- Prefer Protocol over ABC when you want duck typing with type safety\n", + "\n", + "### Precise Type Descriptions\n", + "- `@overload`: document multiple call signatures for a single function\n", + "- `TypedDict`: define the exact key-value shape of a dictionary\n", + "- `Literal[\"a\", \"b\"]`: restrict to specific literal values\n", + "- `NotRequired[T]`: mark TypedDict keys as optional\n", + "\n", + "### Constants and Class-Level State\n", + "- `Final[T]`: value cannot be reassigned after initialization\n", + "- `ClassVar[T]`: attribute belongs to the class, not instances\n", + "- Both are enforced by type checkers, not at runtime" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_16/03_mypy_in_practice.ipynb b/src/chapter_16/03_mypy_in_practice.ipynb new file mode 100644 index 0000000..eee574e --- /dev/null +++ b/src/chapter_16/03_mypy_in_practice.ipynb @@ -0,0 +1,889 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 16: mypy in Practice\n", + "\n", + "This notebook covers practical usage of mypy - Python's most widely-used static\n", + "type checker. You will learn how to configure it, interpret its errors, add types\n", + "to existing code gradually, and use advanced features like type narrowing and casts.\n", + "\n", + "## Key Concepts\n", + "- mypy configuration in `pyproject.toml`\n", + "- Common mypy errors and how to fix them\n", + "- Gradual typing: adding types to untyped code\n", + "- Type narrowing with `isinstance`, `assert`, and type guards\n", + "- `cast()` for when you know better than the checker\n", + "- `reveal_type()` for debugging type inference\n", + "- Best practices: when to use `Any`, when to be strict" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## mypy Configuration in pyproject.toml\n", + "\n", + "mypy is configured in `pyproject.toml` under the `[tool.mypy]` section.\n", + "Good configuration catches real bugs while avoiding excessive noise.\n", + "Below is a recommended starting configuration and an explanation of each option." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# This cell shows a recommended pyproject.toml configuration.\n", + "# It is presented as a string since TOML files cannot be executed as Python.\n", + "\n", + "mypy_config = \"\"\"\n", + "[tool.mypy]\n", + "python_version = \"3.12\" # Target Python version for type checking\n", + "warn_return_any = true # Warn when a function returns Any\n", + "warn_unused_configs = true # Warn about unused [mypy-*] sections\n", + "warn_redundant_casts = true # Warn about unnecessary cast() calls\n", + "warn_unused_ignores = true # Warn about unnecessary # type: ignore\n", + "check_untyped_defs = true # Type-check inside untyped functions too\n", + "disallow_untyped_defs = false # Don't require all functions to have types (gradual)\n", + "disallow_any_generics = false # Allow bare list, dict (vs list[int], dict[str, int])\n", + "no_implicit_optional = true # Don't auto-convert None defaults to Optional\n", + "strict_equality = true # Flag nonsensical equality checks\n", + "\n", + "# Per-module overrides: stricter for your code, lenient for third-party\n", + "[[tool.mypy.overrides]]\n", + "module = \"my_project.*\" # Your project's modules\n", + "disallow_untyped_defs = true # Require all functions to have type hints\n", + "warn_return_any = true\n", + "\n", + "[[tool.mypy.overrides]]\n", + "module = \"tests.*\" # Test modules can be less strict\n", + "disallow_untyped_defs = false\n", + "\n", + "[[tool.mypy.overrides]]\n", + "module = \"third_party_lib.*\" # Ignore untyped third-party libraries\n", + "ignore_missing_imports = true\n", + "\"\"\"\n", + "\n", + "print(mypy_config)\n", + "\n", + "# Strictness progression (from loose to strict):\n", + "strictness_levels = [\n", + " (\"Level 1 - Minimal\", \"check_untyped_defs = true\"),\n", + " (\"Level 2 - Moderate\", \"+ warn_return_any, no_implicit_optional\"),\n", + " (\"Level 3 - Recommended\", \"+ disallow_untyped_defs (for your code)\"),\n", + " (\"Level 4 - Strict\", \"--strict flag (all checks enabled)\"),\n", + "]\n", + "\n", + "print(\"Recommended strictness progression:\")\n", + "for level, desc in strictness_levels:\n", + " print(f\" {level}: {desc}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Common mypy Errors and How to Fix Them\n", + "\n", + "mypy produces specific error codes that tell you exactly what went wrong.\n", + "Let's walk through the most common errors you will encounter and the\n", + "correct way to fix each one." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Optional\n", + "\n", + "# -------------------------------------------------------\n", + "# Error 1: Incompatible types in assignment [assignment]\n", + "# -------------------------------------------------------\n", + "# BAD: mypy error - Incompatible types (got \"str\", expected \"int\")\n", + "# x: int = \"hello\"\n", + "\n", + "# FIX: Use the correct type\n", + "x: int = 42\n", + "y: str = \"hello\"\n", + "print(f\"x={x}, y={y}\")\n", + "\n", + "# -------------------------------------------------------\n", + "# Error 2: Item \"None\" has no attribute [union-attr]\n", + "# -------------------------------------------------------\n", + "# BAD: mypy error - \"Optional[str]\" has no attribute \"upper\"\n", + "def get_name() -> str | None:\n", + " return \"Alice\"\n", + "\n", + "# name = get_name()\n", + "# print(name.upper()) # Error! name might be None\n", + "\n", + "# FIX: Check for None first (type narrowing)\n", + "name = get_name()\n", + "if name is not None:\n", + " print(f\"Upper: {name.upper()}\") # mypy knows name is str here\n", + "\n", + "# -------------------------------------------------------\n", + "# Error 3: Argument type incompatible [arg-type]\n", + "# -------------------------------------------------------\n", + "def double(n: int) -> int:\n", + " return n * 2\n", + "\n", + "# BAD: double(\"3\") # Error: expected int, got str\n", + "\n", + "# FIX: Pass the correct type\n", + "print(f\"double(3) = {double(3)}\")\n", + "\n", + "# -------------------------------------------------------\n", + "# Error 4: Missing return statement [return]\n", + "# -------------------------------------------------------\n", + "# BAD: not all paths return a value\n", + "# def categorize(n: int) -> str:\n", + "# if n > 0:\n", + "# return \"positive\"\n", + "# elif n < 0:\n", + "# return \"negative\"\n", + "# # Missing: what about n == 0?\n", + "\n", + "# FIX: Handle all cases\n", + "def categorize(n: int) -> str:\n", + " if n > 0:\n", + " return \"positive\"\n", + " elif n < 0:\n", + " return \"negative\"\n", + " return \"zero\" # All paths covered\n", + "\n", + "print(f\"categorize(5) = {categorize(5)}\")\n", + "print(f\"categorize(0) = {categorize(0)}\")\n", + "\n", + "# -------------------------------------------------------\n", + "# Error 5: Incompatible return type [return-value]\n", + "# -------------------------------------------------------\n", + "# BAD: returning wrong type\n", + "# def get_count() -> int:\n", + "# return \"three\" # Error: expected int, got str\n", + "\n", + "# FIX: Return the declared type\n", + "def get_count() -> int:\n", + " return 3\n", + "\n", + "print(f\"get_count() = {get_count()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gradual Typing: Adding Types to Untyped Code\n", + "\n", + "You do not need to annotate everything at once. mypy supports **gradual typing** -\n", + "start with the most important interfaces and expand outward. The `Any` type acts\n", + "as an escape hatch: it is compatible with every other type." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "# -------------------------------------------------------\n", + "# Step 1: Start with NO type hints (fully untyped)\n", + "# -------------------------------------------------------\n", + "def process_data_v1(data, config):\n", + " \"\"\"Version 1: No types. mypy skips checking this by default.\"\"\"\n", + " result = []\n", + " for item in data:\n", + " if config.get(\"uppercase\"):\n", + " result.append(item.upper())\n", + " else:\n", + " result.append(item)\n", + " return result\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# Step 2: Add types to public interfaces first\n", + "# -------------------------------------------------------\n", + "def process_data_v2(data: list[str], config: dict[str, Any]) -> list[str]:\n", + " \"\"\"Version 2: Public interface typed. Internal logic uses Any for config values.\"\"\"\n", + " result: list[str] = []\n", + " for item in data:\n", + " if config.get(\"uppercase\"):\n", + " result.append(item.upper())\n", + " else:\n", + " result.append(item)\n", + " return result\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# Step 3: Replace Any with precise types\n", + "# -------------------------------------------------------\n", + "from typing import TypedDict\n", + "\n", + "\n", + "class ProcessConfig(TypedDict, total=False):\n", + " \"\"\"Typed configuration for process_data.\"\"\"\n", + " uppercase: bool\n", + " prefix: str\n", + " max_length: int\n", + "\n", + "\n", + "def process_data_v3(data: list[str], config: ProcessConfig) -> list[str]:\n", + " \"\"\"Version 3: Fully typed, including configuration shape.\"\"\"\n", + " result: list[str] = []\n", + " for item in data:\n", + " processed = item\n", + " if config.get(\"uppercase\"):\n", + " processed = processed.upper()\n", + " prefix = config.get(\"prefix\", \"\")\n", + " if prefix:\n", + " processed = f\"{prefix}{processed}\"\n", + " max_len = config.get(\"max_length\")\n", + " if max_len is not None:\n", + " processed = processed[:max_len]\n", + " result.append(processed)\n", + " return result\n", + "\n", + "\n", + "# All three versions work at runtime\n", + "data = [\"hello\", \"world\", \"python\"]\n", + "\n", + "print(\"v1 (untyped):\", process_data_v1(data, {\"uppercase\": True}))\n", + "print(\"v2 (partial):\", process_data_v2(data, {\"uppercase\": True}))\n", + "\n", + "config: ProcessConfig = {\"uppercase\": True, \"prefix\": \">> \", \"max_length\": 10}\n", + "print(\"v3 (fully typed):\", process_data_v3(data, config))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Type Narrowing with isinstance, assert, and Type Guards\n", + "\n", + "**Type narrowing** is when the type checker recognizes that a variable's type\n", + "has been refined within a code block. mypy understands several patterns:\n", + "- `isinstance()` checks\n", + "- `is None` / `is not None` checks\n", + "- `assert` statements\n", + "- Custom `TypeGuard` functions (PEP 647)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypeGuard\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# isinstance narrowing\n", + "# -------------------------------------------------------\n", + "def describe(value: int | str | list[int]) -> str:\n", + " \"\"\"mypy narrows the type in each branch.\"\"\"\n", + " if isinstance(value, int):\n", + " # mypy knows: value is int\n", + " return f\"Integer: {value * 2}\"\n", + " elif isinstance(value, str):\n", + " # mypy knows: value is str\n", + " return f\"String: {value.upper()}\"\n", + " else:\n", + " # mypy knows: value is list[int]\n", + " return f\"List sum: {sum(value)}\"\n", + "\n", + "\n", + "print(describe(42))\n", + "print(describe(\"hello\"))\n", + "print(describe([1, 2, 3]))\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# None narrowing (is / is not)\n", + "# -------------------------------------------------------\n", + "def process_optional(value: str | None) -> str:\n", + " \"\"\"Narrow Optional to remove None.\"\"\"\n", + " if value is None:\n", + " return \"\"\n", + " # mypy knows: value is str (not None)\n", + " return value.strip().title()\n", + "\n", + "\n", + "print(f\"\\nprocess_optional(None) = {process_optional(None)!r}\")\n", + "print(f\"process_optional(' hello ') = {process_optional(' hello ')!r}\")\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# assert narrowing\n", + "# -------------------------------------------------------\n", + "def get_first_word(text: str | None) -> str:\n", + " \"\"\"Assert narrows None away.\"\"\"\n", + " assert text is not None, \"text must not be None\"\n", + " # mypy knows: text is str after the assert\n", + " return text.split()[0]\n", + "\n", + "\n", + "print(f\"get_first_word('hello world') = {get_first_word('hello world')!r}\")\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# TypeGuard for custom narrowing (PEP 647)\n", + "# -------------------------------------------------------\n", + "def is_string_list(values: list[object]) -> TypeGuard[list[str]]:\n", + " \"\"\"Return True if all elements are strings.\n", + "\n", + " When this returns True, mypy narrows list[object] to list[str].\n", + " \"\"\"\n", + " return all(isinstance(v, str) for v in values)\n", + "\n", + "\n", + "def process_items(items: list[object]) -> str:\n", + " \"\"\"Process items, with special handling for all-string lists.\"\"\"\n", + " if is_string_list(items):\n", + " # mypy knows: items is list[str]\n", + " return \", \".join(items) # .join() works because items is list[str]\n", + " return str(items)\n", + "\n", + "\n", + "print(f\"\\nprocess_items(['a', 'b', 'c']) = {process_items(['a', 'b', 'c'])!r}\")\n", + "print(f\"process_items([1, 2, 3]) = {process_items([1, 2, 3])!r}\")\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# Exhaustiveness checking with assert_never\n", + "# -------------------------------------------------------\n", + "from typing import Literal, assert_never\n", + "\n", + "Status = Literal[\"active\", \"inactive\", \"pending\"]\n", + "\n", + "\n", + "def handle_status(status: Status) -> str:\n", + " \"\"\"Handle all possible status values. mypy verifies exhaustiveness.\"\"\"\n", + " if status == \"active\":\n", + " return \"User is active\"\n", + " elif status == \"inactive\":\n", + " return \"User is inactive\"\n", + " elif status == \"pending\":\n", + " return \"User is pending approval\"\n", + " else:\n", + " # If we add a new Status value but forget to handle it,\n", + " # mypy will report an error here because status won't be Never\n", + " assert_never(status)\n", + "\n", + "\n", + "print(f\"\\nhandle_status('active') = {handle_status('active')!r}\")\n", + "print(f\"handle_status('pending') = {handle_status('pending')!r}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## cast() - When You Know Better Than the Checker\n", + "\n", + "`cast(T, value)` tells mypy to treat `value` as type `T`. It performs **no runtime\n", + "check** - it is purely a type checker directive. Use it sparingly: every `cast` is\n", + "a promise you are making that mypy cannot verify." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import cast, Any\n", + "import json\n", + "\n", + "\n", + "# Scenario 1: JSON parsing returns Any, but you know the shape\n", + "raw_json = '{\"name\": \"Alice\", \"age\": 30}'\n", + "parsed: Any = json.loads(raw_json) # json.loads returns Any\n", + "\n", + "# You KNOW it is a dict with string keys\n", + "user_data = cast(dict[str, Any], parsed)\n", + "print(f\"Name: {user_data['name']}\")\n", + "print(f\"Age: {user_data['age']}\")\n", + "\n", + "\n", + "# Scenario 2: Container with known element types\n", + "mixed_storage: dict[str, object] = {\n", + " \"count\": 42,\n", + " \"name\": \"Widget\",\n", + " \"prices\": [9.99, 19.99, 29.99],\n", + "}\n", + "\n", + "# You know \"count\" is an int, but mypy sees object\n", + "count = cast(int, mixed_storage[\"count\"])\n", + "doubled: int = count * 2 # Works because mypy thinks count is int\n", + "print(f\"\\nDoubled count: {doubled}\")\n", + "\n", + "# You know \"prices\" is a list of floats\n", + "prices = cast(list[float], mixed_storage[\"prices\"])\n", + "total: float = sum(prices)\n", + "print(f\"Total price: ${total:.2f}\")\n", + "\n", + "\n", + "# Scenario 3: After a validation check that mypy cannot follow\n", + "def parse_config(raw: dict[str, Any]) -> tuple[str, int]:\n", + " \"\"\"Parse and validate config, then cast.\"\"\"\n", + " # Validation logic\n", + " assert isinstance(raw[\"host\"], str), \"host must be a string\"\n", + " assert isinstance(raw[\"port\"], int), \"port must be an integer\"\n", + "\n", + " # After validation, we know the types are correct\n", + " host = cast(str, raw[\"host\"])\n", + " port = cast(int, raw[\"port\"])\n", + " return host, port\n", + "\n", + "\n", + "host, port = parse_config({\"host\": \"localhost\", \"port\": 8080})\n", + "print(f\"\\nConfig: {host}:{port}\")\n", + "\n", + "\n", + "# WARNING: cast() does NOT check at runtime - it can hide bugs!\n", + "bad_cast = cast(int, \"not an int\") # No error raised!\n", + "print(f\"\\nDangerous cast (no runtime error): {bad_cast!r}\")\n", + "print(f\"Type at runtime: {type(bad_cast).__name__}\") # Still str!" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## reveal_type() for Debugging Type Inference\n", + "\n", + "`reveal_type(expr)` is a special function recognized by mypy. When mypy analyzes\n", + "your code, it prints the inferred type of the expression. This is invaluable for\n", + "understanding what mypy \"thinks\" a variable's type is.\n", + "\n", + "**Note**: `reveal_type()` is a mypy directive, not a runtime function. In Python\n", + "3.11+, `typing.reveal_type()` was added as a real function that also works at runtime." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# In Python 3.11+, reveal_type is available from typing\n", + "from typing import reveal_type\n", + "\n", + "# Basic inference\n", + "x = 42\n", + "reveal_type(x) # mypy output: int | runtime: prints the type and value\n", + "\n", + "y = [1, 2, 3]\n", + "reveal_type(y) # mypy: list[int]\n", + "\n", + "z = {\"a\": 1, \"b\": 2}\n", + "reveal_type(z) # mypy: dict[str, int]\n", + "\n", + "\n", + "# Useful for understanding narrowing\n", + "def demo_narrowing(value: str | int | None) -> str:\n", + " # Before any checks:\n", + " # reveal_type(value) # mypy: str | int | None\n", + "\n", + " if value is None:\n", + " return \"nothing\"\n", + "\n", + " # After None check:\n", + " reveal_type(value) # mypy: str | int (None eliminated)\n", + "\n", + " if isinstance(value, str):\n", + " # After isinstance:\n", + " reveal_type(value) # mypy: str\n", + " return value.upper()\n", + "\n", + " # After both checks:\n", + " reveal_type(value) # mypy: int (only possibility left)\n", + " return str(value)\n", + "\n", + "\n", + "print(f\"\\ndemo_narrowing('hello') = {demo_narrowing('hello')}\")\n", + "print(f\"demo_narrowing(42) = {demo_narrowing(42)}\")\n", + "print(f\"demo_narrowing(None) = {demo_narrowing(None)}\")\n", + "\n", + "\n", + "# Reveal type in complex expressions\n", + "data: dict[str, list[int]] = {\"scores\": [90, 85, 92]}\n", + "scores = data.get(\"scores\", [])\n", + "reveal_type(scores) # mypy: list[int]\n", + "\n", + "first = scores[0] if scores else None\n", + "reveal_type(first) # mypy: int | None" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The `# type: ignore` Comment\n", + "\n", + "When you need to suppress a mypy error on a specific line, use `# type: ignore`.\n", + "Always include the error code to document **why** you are suppressing it and\n", + "to avoid accidentally hiding other errors." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# BAD: bare type: ignore hides ALL errors on the line\n", + "# result = some_untyped_function() # type: ignore\n", + "\n", + "# GOOD: specific error code documents the intent\n", + "# result = some_untyped_function() # type: ignore[no-untyped-call]\n", + "\n", + "# Common error codes you might suppress:\n", + "error_codes = {\n", + " \"assignment\": \"Incompatible types in assignment\",\n", + " \"arg-type\": \"Incompatible argument type\",\n", + " \"return-value\": \"Incompatible return value type\",\n", + " \"union-attr\": \"Attribute access on union type\",\n", + " \"no-untyped-call\": \"Calling an untyped function\",\n", + " \"no-untyped-def\": \"Function is missing type annotations\",\n", + " \"import-untyped\": \"Importing an untyped module\",\n", + " \"override\": \"Incompatible override of base class method\",\n", + " \"misc\": \"Miscellaneous error\",\n", + "}\n", + "\n", + "print(\"Common mypy error codes:\")\n", + "for code, description in error_codes.items():\n", + " print(f\" [{code}] {description}\")\n", + "\n", + "\n", + "# Practical example: known third-party library quirk\n", + "import json\n", + "\n", + "# json.loads returns Any, but we validate the shape ourselves\n", + "raw = json.loads('{\"value\": 42}')\n", + "\n", + "# Instead of cast, sometimes a targeted ignore is clearer:\n", + "value: int = raw[\"value\"] # type: ignore[assignment] # validated by schema\n", + "print(f\"\\nExtracted value: {value}\")\n", + "\n", + "# When to use # type: ignore vs cast():\n", + "# - cast(): You want to assert a type and keep type checking downstream\n", + "# - type: ignore: You want to silence a specific false positive\n", + "print(\"\\nRule of thumb:\")\n", + "print(\" cast() -> 'I know this type, continue checking from here'\")\n", + "print(\" type: ignore -> 'This line is a known false positive, skip it'\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Best Practices: When to Use Any, When to Be Strict\n", + "\n", + "Type annotations are a tool for catching bugs and communicating intent.\n", + "Being too strict wastes time; being too loose misses bugs. Here are\n", + "practical guidelines for finding the right balance." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "from collections.abc import Callable\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# GOOD uses of Any: genuine dynamism\n", + "# -------------------------------------------------------\n", + "\n", + "# 1. Plugin systems where you truly accept anything\n", + "def register_plugin(name: str, plugin: Any) -> None:\n", + " \"\"\"Register a plugin. Plugins can be any object with any interface.\"\"\"\n", + " print(f\"Registered plugin: {name} ({type(plugin).__name__})\")\n", + "\n", + "\n", + "# 2. Logging / debugging utilities\n", + "def debug_log(label: str, value: Any) -> None:\n", + " \"\"\"Log any value for debugging.\"\"\"\n", + " print(f\"[DEBUG] {label}: {value!r} (type={type(value).__name__})\")\n", + "\n", + "\n", + "# 3. Wrapping truly dynamic APIs (e.g., JSON, pickle)\n", + "def load_json(path: str) -> Any:\n", + " \"\"\"Load JSON data. Shape is unknown until runtime.\"\"\"\n", + " import json\n", + " # In real code, you'd validate the shape after loading\n", + " return json.loads('{\"key\": \"value\"}')\n", + "\n", + "\n", + "register_plugin(\"my_plugin\", {\"handler\": lambda: None})\n", + "debug_log(\"config\", {\"port\": 8080, \"debug\": True})\n", + "print()\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# BAD uses of Any: laziness or avoidance\n", + "# -------------------------------------------------------\n", + "\n", + "# BAD: Using Any because you didn't think about the type\n", + "# def process(data: Any) -> Any: # What goes in? What comes out? Who knows!\n", + "# return data.do_thing()\n", + "\n", + "# GOOD: Be specific about what you accept and return\n", + "from typing import Protocol\n", + "\n", + "\n", + "class Processable(Protocol):\n", + " def do_thing(self) -> str: ...\n", + "\n", + "\n", + "def process(data: Processable) -> str:\n", + " \"\"\"Now both input and output types are clear.\"\"\"\n", + " return data.do_thing()\n", + "\n", + "\n", + "# -------------------------------------------------------\n", + "# Priority order for adding types\n", + "# -------------------------------------------------------\n", + "priorities = [\n", + " (\"1. Public API functions\", \"These are consumed by other code - types serve as docs\"),\n", + " (\"2. Data models / classes\", \"__init__ parameters and key attributes\"),\n", + " (\"3. Return types\", \"Helps callers understand what they get back\"),\n", + " (\"4. Complex logic\", \"isinstance checks, unions, optional handling\"),\n", + " (\"5. Internal helpers\", \"Less critical but still useful for maintenance\"),\n", + " (\"6. Tests\", \"Lowest priority - types add less value in test code\"),\n", + "]\n", + "\n", + "print(\"Priority order for adding type annotations:\")\n", + "for priority, reason in priorities:\n", + " print(f\" {priority}\")\n", + " print(f\" {reason}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical Example: Typing a Real Module\n", + "\n", + "Let's apply everything we have learned to a small but realistic module\n", + "that processes user data - demonstrating gradual typing, narrowing,\n", + "TypedDict, and proper error handling." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypedDict, TypeGuard, Literal, Final, cast\n", + "from collections.abc import Callable\n", + "import json\n", + "\n", + "\n", + "# --- Type definitions ---\n", + "class RawUser(TypedDict):\n", + " \"\"\"Shape of user data as received from the API.\"\"\"\n", + " id: int\n", + " name: str\n", + " email: str\n", + " role: str\n", + " active: bool\n", + "\n", + "\n", + "Role = Literal[\"admin\", \"editor\", \"viewer\"]\n", + "VALID_ROLES: Final[set[str]] = {\"admin\", \"editor\", \"viewer\"}\n", + "\n", + "\n", + "class ValidatedUser(TypedDict):\n", + " \"\"\"Shape of user data after validation.\"\"\"\n", + " id: int\n", + " name: str\n", + " email: str\n", + " role: Role\n", + " active: bool\n", + "\n", + "\n", + "# --- Type guard ---\n", + "def is_valid_role(role: str) -> TypeGuard[Role]:\n", + " \"\"\"Check if a string is a valid Role literal.\"\"\"\n", + " return role in VALID_ROLES\n", + "\n", + "\n", + "# --- Validation ---\n", + "class ValidationError(Exception):\n", + " \"\"\"Raised when user data fails validation.\"\"\"\n", + " pass\n", + "\n", + "\n", + "def validate_user(raw: RawUser) -> ValidatedUser:\n", + " \"\"\"Validate and narrow a raw user to a validated user.\"\"\"\n", + " if not raw[\"name\"].strip():\n", + " raise ValidationError(f\"User {raw['id']}: name is empty\")\n", + "\n", + " if \"@\" not in raw[\"email\"]:\n", + " raise ValidationError(f\"User {raw['id']}: invalid email {raw['email']!r}\")\n", + "\n", + " role = raw[\"role\"]\n", + " if not is_valid_role(role):\n", + " raise ValidationError(f\"User {raw['id']}: invalid role {role!r}\")\n", + "\n", + " # After TypeGuard, mypy knows role is Role\n", + " return {\n", + " \"id\": raw[\"id\"],\n", + " \"name\": raw[\"name\"].strip(),\n", + " \"email\": raw[\"email\"].lower(),\n", + " \"role\": role,\n", + " \"active\": raw[\"active\"],\n", + " }\n", + "\n", + "\n", + "# --- Processing pipeline ---\n", + "Transformer = Callable[[ValidatedUser], ValidatedUser]\n", + "\n", + "\n", + "def apply_pipeline(\n", + " users: list[ValidatedUser],\n", + " *transforms: Transformer,\n", + ") -> list[ValidatedUser]:\n", + " \"\"\"Apply a sequence of transformations to validated users.\"\"\"\n", + " result = list(users)\n", + " for transform in transforms:\n", + " result = [transform(u) for u in result]\n", + " return result\n", + "\n", + "\n", + "def deactivate_viewers(user: ValidatedUser) -> ValidatedUser:\n", + " \"\"\"Deactivate all viewer accounts.\"\"\"\n", + " if user[\"role\"] == \"viewer\":\n", + " return {**user, \"active\": False}\n", + " return user\n", + "\n", + "\n", + "def normalize_email(user: ValidatedUser) -> ValidatedUser:\n", + " \"\"\"Ensure email is lowercase.\"\"\"\n", + " return {**user, \"email\": user[\"email\"].lower()}\n", + "\n", + "\n", + "# --- Main ---\n", + "raw_users: list[RawUser] = [\n", + " {\"id\": 1, \"name\": \"Alice\", \"email\": \"Alice@Example.COM\", \"role\": \"admin\", \"active\": True},\n", + " {\"id\": 2, \"name\": \"Bob\", \"email\": \"bob@test.com\", \"role\": \"viewer\", \"active\": True},\n", + " {\"id\": 3, \"name\": \"Carol\", \"email\": \"carol@test.com\", \"role\": \"editor\", \"active\": True},\n", + "]\n", + "\n", + "# Validate\n", + "validated: list[ValidatedUser] = []\n", + "for raw in raw_users:\n", + " try:\n", + " validated.append(validate_user(raw))\n", + " except ValidationError as e:\n", + " print(f\"Skipped: {e}\")\n", + "\n", + "# Transform\n", + "processed = apply_pipeline(validated, normalize_email, deactivate_viewers)\n", + "\n", + "print(\"Processed users:\")\n", + "for user in processed:\n", + " status = \"active\" if user[\"active\"] else \"inactive\"\n", + " print(f\" {user['name']} ({user['role']}, {status}): {user['email']}\")\n", + "\n", + "# Test validation error\n", + "try:\n", + " bad_user: RawUser = {\n", + " \"id\": 99, \"name\": \"Dan\", \"email\": \"invalid\",\n", + " \"role\": \"superadmin\", \"active\": True,\n", + " }\n", + " validate_user(bad_user)\n", + "except ValidationError as e:\n", + " print(f\"\\nValidation caught: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### mypy Configuration\n", + "- Configure in `pyproject.toml` under `[tool.mypy]`\n", + "- Start lenient, increase strictness gradually with per-module overrides\n", + "- Key settings: `check_untyped_defs`, `no_implicit_optional`, `warn_return_any`\n", + "\n", + "### Common Errors and Fixes\n", + "- `[assignment]`: wrong type assigned to variable - fix the type or the value\n", + "- `[union-attr]`: accessing attribute on `Optional` without None check - narrow first\n", + "- `[arg-type]`: wrong argument type - pass the correct type\n", + "- `[return]` / `[return-value]`: missing or wrong return - cover all code paths\n", + "\n", + "### Type Narrowing\n", + "- `isinstance(x, T)` narrows `x` to `T` in the True branch\n", + "- `x is not None` narrows `Optional[T]` to `T`\n", + "- `assert isinstance(x, T)` narrows from the assert onward\n", + "- `TypeGuard[T]` for custom narrowing functions\n", + "- `assert_never(x)` for exhaustiveness checking\n", + "\n", + "### Escape Hatches\n", + "- `cast(T, value)`: tell mypy a value is type T (no runtime check)\n", + "- `# type: ignore[code]`: suppress a specific error (document why)\n", + "- `Any`: opt out of type checking for truly dynamic code\n", + "- `reveal_type(x)`: ask mypy what type it inferred\n", + "\n", + "### Best Practices\n", + "- Type public APIs first, internal helpers last, tests are optional\n", + "- Prefer `Protocol` over `Any` when you need some structure\n", + "- Use `TypedDict` for JSON-like data shapes\n", + "- Use `TypeGuard` for custom validation that narrows types\n", + "- Always include error codes in `# type: ignore` comments" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_16/README.md b/src/chapter_16/README.md new file mode 100644 index 0000000..d7140d8 --- /dev/null +++ b/src/chapter_16/README.md @@ -0,0 +1,23 @@ +# Chapter 16: Type Hints and Static Analysis + +## Topics Covered +- Type annotation syntax and evolution (PEP 484, 526, 604) +- Built-in generics: `list[int]`, `dict[str, Any]`, `tuple[int, ...]` +- `typing` module: `TypeVar`, `Generic`, `ParamSpec`, `TypeAlias` +- Union types, Optional, and the `X | Y` syntax (3.10+) +- `Protocol` for structural subtyping +- `@overload` for multiple signatures +- `TypedDict`, `Literal`, `Final`, `ClassVar` +- `mypy` configuration and common patterns +- Runtime type checking vs static analysis + +## Notebooks +1. **01_annotation_fundamentals.ipynb** — Variables, functions, classes, generics +2. **02_advanced_typing.ipynb** — TypeVar, Protocol, overload, TypedDict, Literal +3. **03_mypy_in_practice.ipynb** — Configuration, common errors, gradual typing strategies + +## Key Takeaways +- Type hints are documentation that tools can verify +- Use generics for reusable, type-safe containers and functions +- Protocol enables duck typing with static verification +- Gradual typing lets you add hints incrementally to existing code diff --git a/src/chapter_16/__init__.py b/src/chapter_16/__init__.py new file mode 100644 index 0000000..759f2a6 --- /dev/null +++ b/src/chapter_16/__init__.py @@ -0,0 +1 @@ +"""Chapter 16: Type Hints and Static Analysis.""" diff --git a/src/chapter_17/01_socket_fundamentals.ipynb b/src/chapter_17/01_socket_fundamentals.ipynb new file mode 100644 index 0000000..5accbb1 --- /dev/null +++ b/src/chapter_17/01_socket_fundamentals.ipynb @@ -0,0 +1,773 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 17: Socket Fundamentals\n", + "\n", + "**Networking and Protocols**\n", + "\n", + "Sockets are the foundational building block for network communication in Python.\n", + "The `socket` module provides a low-level interface to the BSD socket API, enabling\n", + "programs to send and receive data over TCP, UDP, and other protocols. Understanding\n", + "sockets is essential even when using higher-level libraries, because they underpin\n", + "everything from HTTP clients to database drivers." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Socket Basics: Address Families and Socket Types\n", + "\n", + "A socket is created with two key parameters:\n", + "\n", + "- **Address family** (`AF_INET` for IPv4, `AF_INET6` for IPv6, `AF_UNIX` for local IPC)\n", + "- **Socket type** (`SOCK_STREAM` for TCP, `SOCK_DGRAM` for UDP)\n", + "\n", + "TCP (`SOCK_STREAM`) provides a reliable, ordered byte stream with connection setup.\n", + "UDP (`SOCK_DGRAM`) provides connectionless, unreliable datagrams -- faster but\n", + "with no delivery guarantees." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "\n", + "\n", + "# Inspect the key constants\n", + "print(\"=== Address Families ===\")\n", + "print(f\"AF_INET (IPv4): {socket.AF_INET}\")\n", + "print(f\"AF_INET6 (IPv6): {socket.AF_INET6}\")\n", + "print(f\"AF_UNIX (local): {socket.AF_UNIX}\")\n", + "\n", + "print(\"\\n=== Socket Types ===\")\n", + "print(f\"SOCK_STREAM (TCP): {socket.SOCK_STREAM}\")\n", + "print(f\"SOCK_DGRAM (UDP): {socket.SOCK_DGRAM}\")\n", + "\n", + "# Create a TCP socket and inspect its properties\n", + "tcp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", + "print(f\"\\nTCP socket: {tcp_sock}\")\n", + "print(f\" family: {tcp_sock.family}\")\n", + "print(f\" type: {tcp_sock.type}\")\n", + "print(f\" fileno: {tcp_sock.fileno()}\")\n", + "tcp_sock.close()\n", + "\n", + "# Create a UDP socket and inspect\n", + "udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", + "print(f\"\\nUDP socket: {udp_sock}\")\n", + "print(f\" family: {udp_sock.family}\")\n", + "print(f\" type: {udp_sock.type}\")\n", + "udp_sock.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Socket as a Context Manager\n", + "\n", + "Sockets implement the context manager protocol. Using `with` ensures the socket\n", + "is closed automatically, even if an exception occurs. This prevents file descriptor\n", + "leaks, which can exhaust operating system resources in long-running servers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "\n", + "\n", + "# Preferred pattern: socket as context manager\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", + " print(f\"Socket open, fileno={sock.fileno()}\")\n", + " # The socket is usable within this block\n", + "\n", + "# After the block, the socket is closed automatically\n", + "print(f\"Socket closed, fileno={sock.fileno()}\")\n", + "# fileno() returns -1 on a closed socket on most platforms\n", + "\n", + "\n", + "# Equivalent manual pattern (less safe):\n", + "sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", + "try:\n", + " print(f\"\\nManual socket open, fileno={sock2.fileno()}\")\n", + "finally:\n", + " sock2.close()\n", + " print(f\"Manual socket closed, fileno={sock2.fileno()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DNS Resolution with `getaddrinfo()`\n", + "\n", + "`socket.getaddrinfo()` resolves a hostname into one or more address tuples\n", + "that can be used to create and connect sockets. It is protocol-agnostic and\n", + "handles both IPv4 and IPv6. This is the recommended way to resolve addresses\n", + "rather than using `gethostbyname()`, which only returns IPv4." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "\n", + "\n", + "def resolve_address(\n", + " host: str,\n", + " port: int,\n", + " family: socket.AddressFamily = socket.AF_UNSPEC,\n", + " type_: socket.SocketKind = socket.SOCK_STREAM,\n", + ") -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]:\n", + " \"\"\"Resolve a host:port to a list of address info tuples.\"\"\"\n", + " results = socket.getaddrinfo(host, port, family, type_)\n", + " return results\n", + "\n", + "\n", + "# Resolve localhost for TCP\n", + "print(\"=== Resolving 'localhost' port 80 (TCP) ===\")\n", + "for info in resolve_address(\"localhost\", 80):\n", + " family, socktype, proto, canonname, sockaddr = info\n", + " print(f\" family={family.name}, type={socktype.name}, addr={sockaddr}\")\n", + "\n", + "# Resolve for UDP only\n", + "print(\"\\n=== Resolving 'localhost' port 53 (UDP) ===\")\n", + "for info in resolve_address(\"localhost\", 53, type_=socket.SOCK_DGRAM):\n", + " family, socktype, proto, canonname, sockaddr = info\n", + " print(f\" family={family.name}, type={socktype.name}, addr={sockaddr}\")\n", + "\n", + "# Resolve an external hostname (may return IPv4 and IPv6)\n", + "print(\"\\n=== Resolving 'example.com' port 443 (TCP) ===\")\n", + "try:\n", + " for info in resolve_address(\"example.com\", 443):\n", + " family, socktype, proto, canonname, sockaddr = info\n", + " print(f\" family={family.name}, type={socktype.name}, addr={sockaddr}\")\n", + "except socket.gaierror as e:\n", + " print(f\" DNS resolution failed: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Socket Options: `SO_REUSEADDR` and Timeouts\n", + "\n", + "Socket options control low-level behavior:\n", + "\n", + "- **`SO_REUSEADDR`**: Allows a server socket to bind to an address that is in\n", + " the `TIME_WAIT` state (e.g., after a recent restart). Without this, restarting\n", + " a server immediately may fail with \"Address already in use\".\n", + "- **Timeouts**: `settimeout()` controls how long blocking operations (connect,\n", + " recv, accept) wait before raising `socket.timeout`. A value of `None` means\n", + " blocking indefinitely; `0` means non-blocking." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "\n", + "\n", + "# Demonstrate SO_REUSEADDR\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:\n", + " # Check default value\n", + " reuse_before = server.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)\n", + " print(f\"SO_REUSEADDR before: {reuse_before}\")\n", + "\n", + " # Enable SO_REUSEADDR -- always do this for server sockets\n", + " server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", + " reuse_after = server.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)\n", + " print(f\"SO_REUSEADDR after: {reuse_after}\")\n", + "\n", + "# Demonstrate timeouts\n", + "print(\"\\n=== Socket Timeouts ===\")\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", + " # Default: blocking (None)\n", + " print(f\"Default timeout: {sock.gettimeout()}\")\n", + "\n", + " # Set a 2-second timeout\n", + " sock.settimeout(2.0)\n", + " print(f\"After settimeout(2.0): {sock.gettimeout()}\")\n", + "\n", + " # Non-blocking mode\n", + " sock.setblocking(False)\n", + " print(f\"After setblocking(False): timeout={sock.gettimeout()}\")\n", + "\n", + " # Back to blocking\n", + " sock.setblocking(True)\n", + " print(f\"After setblocking(True): timeout={sock.gettimeout()}\")\n", + "\n", + "# Demonstrate timeout exception\n", + "print(\"\\n=== Timeout on connect ===\")\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", + " sock.settimeout(0.001) # Very short timeout\n", + " try:\n", + " # Attempt connection to a non-routable address (will timeout)\n", + " sock.connect((\"192.0.2.1\", 80))\n", + " except socket.timeout:\n", + " print(\" Connection timed out (socket.timeout raised)\")\n", + " except OSError as e:\n", + " print(f\" OS error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TCP Client-Server Echo Example\n", + "\n", + "A classic TCP pattern: the **echo server** accepts connections and sends back\n", + "whatever data it receives. We use `threading` to run the server in the background\n", + "so client and server can coexist in a single notebook.\n", + "\n", + "TCP flow:\n", + "1. Server: `bind()` -> `listen()` -> `accept()` (blocks until client connects)\n", + "2. Client: `connect()` -> `sendall()` -> `recv()`\n", + "3. Server: `recv()` -> `sendall()` (echoes data back)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "import threading\n", + "\n", + "\n", + "HOST: str = \"127.0.0.1\"\n", + "PORT: int = 0 # Let the OS assign a free port\n", + "\n", + "\n", + "def echo_server(server_sock: socket.socket, ready: threading.Event) -> None:\n", + " \"\"\"Accept one connection and echo data back.\"\"\"\n", + " server_sock.listen(1)\n", + " ready.set() # Signal that the server is ready to accept\n", + "\n", + " conn, addr = server_sock.accept()\n", + " with conn:\n", + " print(f\"[Server] Connection from {addr}\")\n", + " while True:\n", + " data: bytes = conn.recv(1024)\n", + " if not data:\n", + " print(\"[Server] Client disconnected\")\n", + " break\n", + " print(f\"[Server] Received: {data!r}\")\n", + " conn.sendall(data) # Echo back\n", + "\n", + "\n", + "def echo_client(port: int, messages: list[str]) -> list[str]:\n", + " \"\"\"Connect to the echo server and send messages.\"\"\"\n", + " responses: list[str] = []\n", + " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", + " sock.connect((HOST, port))\n", + " print(f\"[Client] Connected to {HOST}:{port}\")\n", + "\n", + " for msg in messages:\n", + " sock.sendall(msg.encode(\"utf-8\"))\n", + " data = sock.recv(1024)\n", + " response = data.decode(\"utf-8\")\n", + " print(f\"[Client] Sent: {msg!r}, Received: {response!r}\")\n", + " responses.append(response)\n", + "\n", + " return responses\n", + "\n", + "\n", + "# Set up the server socket\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_sock:\n", + " server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", + " server_sock.bind((HOST, PORT))\n", + " actual_port: int = server_sock.getsockname()[1]\n", + " print(f\"Server bound to {HOST}:{actual_port}\")\n", + "\n", + " # Start server in a background thread\n", + " ready = threading.Event()\n", + " server_thread = threading.Thread(\n", + " target=echo_server, args=(server_sock, ready), daemon=True\n", + " )\n", + " server_thread.start()\n", + " ready.wait() # Wait until server is listening\n", + "\n", + " # Run the client\n", + " messages = [\"Hello, server!\", \"How are you?\", \"Goodbye!\"]\n", + " responses = echo_client(actual_port, messages)\n", + "\n", + " print(f\"\\nAll messages echoed correctly: {messages == responses}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Handling Partial Sends and Receives with `sendall()`\n", + "\n", + "TCP is a **stream protocol** -- it does not preserve message boundaries. A single\n", + "`send()` may not transmit all bytes at once, and a single `recv()` may return\n", + "partial data. Key rules:\n", + "\n", + "- **`sendall(data)`**: Repeatedly calls `send()` until all bytes are sent. Always\n", + " prefer this over raw `send()`.\n", + "- **`recv(bufsize)`**: Returns up to `bufsize` bytes. You must loop to receive a\n", + " complete message, using a framing protocol (length prefix, delimiter, etc.)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "import struct\n", + "import threading\n", + "\n", + "\n", + "def recv_exactly(sock: socket.socket, num_bytes: int) -> bytes:\n", + " \"\"\"Receive exactly num_bytes from the socket.\n", + "\n", + " Handles partial receives by looping until all bytes are collected.\n", + " Raises ConnectionError if the connection is closed prematurely.\n", + " \"\"\"\n", + " chunks: list[bytes] = []\n", + " bytes_received: int = 0\n", + " while bytes_received < num_bytes:\n", + " chunk = sock.recv(num_bytes - bytes_received)\n", + " if not chunk:\n", + " raise ConnectionError(\n", + " f\"Connection closed after {bytes_received}/{num_bytes} bytes\"\n", + " )\n", + " chunks.append(chunk)\n", + " bytes_received += len(chunk)\n", + " return b\"\".join(chunks)\n", + "\n", + "\n", + "def send_message(sock: socket.socket, message: bytes) -> None:\n", + " \"\"\"Send a length-prefixed message (4-byte big-endian length header).\"\"\"\n", + " header: bytes = struct.pack(\"!I\", len(message)) # 4-byte unsigned int\n", + " sock.sendall(header + message)\n", + "\n", + "\n", + "def recv_message(sock: socket.socket) -> bytes:\n", + " \"\"\"Receive a length-prefixed message.\"\"\"\n", + " header: bytes = recv_exactly(sock, 4)\n", + " (length,) = struct.unpack(\"!I\", header)\n", + " return recv_exactly(sock, length)\n", + "\n", + "\n", + "# Demonstrate length-prefixed messaging\n", + "def framing_server(server_sock: socket.socket, ready: threading.Event) -> None:\n", + " server_sock.listen(1)\n", + " ready.set()\n", + " conn, _ = server_sock.accept()\n", + " with conn:\n", + " # Receive and echo back 3 messages\n", + " for _ in range(3):\n", + " msg = recv_message(conn)\n", + " print(f\"[Server] Received {len(msg)} bytes: {msg!r}\")\n", + " send_message(conn, msg)\n", + "\n", + "\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv:\n", + " srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", + " srv.bind((\"127.0.0.1\", 0))\n", + " port = srv.getsockname()[1]\n", + "\n", + " ready = threading.Event()\n", + " t = threading.Thread(target=framing_server, args=(srv, ready), daemon=True)\n", + " t.start()\n", + " ready.wait()\n", + "\n", + " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client:\n", + " client.connect((\"127.0.0.1\", port))\n", + "\n", + " # Send messages of varying sizes\n", + " test_messages: list[bytes] = [\n", + " b\"Short\",\n", + " b\"A medium-length message for testing\",\n", + " b\"X\" * 500, # Larger message\n", + " ]\n", + "\n", + " for original in test_messages:\n", + " send_message(client, original)\n", + " echoed = recv_message(client)\n", + " assert echoed == original, f\"Mismatch: {original!r} != {echoed!r}\"\n", + " print(f\"[Client] Sent {len(original)} bytes, echoed correctly\")\n", + "\n", + " t.join(timeout=2)\n", + " print(\"\\nAll length-prefixed messages exchanged correctly!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## UDP Send/Receive Example\n", + "\n", + "UDP (`SOCK_DGRAM`) is connectionless -- each `sendto()` and `recvfrom()` is\n", + "an independent datagram. There is no `connect()`, `listen()`, or `accept()`.\n", + "Datagrams may arrive out of order, be duplicated, or be lost entirely.\n", + "\n", + "UDP is suitable for:\n", + "- DNS lookups\n", + "- Real-time audio/video streaming\n", + "- Game state updates\n", + "- Any scenario where low latency matters more than reliability" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "import threading\n", + "\n", + "\n", + "def udp_echo_server(server_sock: socket.socket, ready: threading.Event) -> None:\n", + " \"\"\"Simple UDP echo server that handles a fixed number of datagrams.\"\"\"\n", + " ready.set()\n", + " for _ in range(3): # Handle 3 datagrams then stop\n", + " data, client_addr = server_sock.recvfrom(1024)\n", + " print(f\"[UDP Server] Received {data!r} from {client_addr}\")\n", + " server_sock.sendto(data.upper(), client_addr) # Echo back uppercased\n", + "\n", + "\n", + "# Create UDP server socket\n", + "with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as server_sock:\n", + " server_sock.bind((\"127.0.0.1\", 0))\n", + " port = server_sock.getsockname()[1]\n", + " print(f\"UDP server listening on 127.0.0.1:{port}\")\n", + "\n", + " ready = threading.Event()\n", + " t = threading.Thread(\n", + " target=udp_echo_server, args=(server_sock, ready), daemon=True\n", + " )\n", + " t.start()\n", + " ready.wait()\n", + "\n", + " # UDP client\n", + " with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as client_sock:\n", + " client_sock.settimeout(2.0) # Timeout since UDP has no guarantees\n", + "\n", + " messages = [b\"hello udp\", b\"networking is fun\", b\"datagrams!\"]\n", + " for msg in messages:\n", + " client_sock.sendto(msg, (\"127.0.0.1\", port))\n", + " try:\n", + " data, server_addr = client_sock.recvfrom(1024)\n", + " print(f\"[UDP Client] Sent: {msg!r}, Got: {data!r}\")\n", + " except socket.timeout:\n", + " print(f\"[UDP Client] No response for {msg!r} (lost datagram)\")\n", + "\n", + " t.join(timeout=2)\n", + " print(\"\\nUDP echo complete!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-Client TCP Server with Threading\n", + "\n", + "A production-like TCP server handles multiple clients concurrently. Each accepted\n", + "connection is dispatched to a new thread. This pattern is the foundation for\n", + "many network services, though for high concurrency, `asyncio` (covered in\n", + "notebook 03) or `selectors` may be preferred." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "import threading\n", + "import time\n", + "\n", + "\n", + "def handle_client(\n", + " conn: socket.socket,\n", + " addr: tuple[str, int],\n", + " client_id: int,\n", + ") -> None:\n", + " \"\"\"Handle a single client connection in its own thread.\"\"\"\n", + " with conn:\n", + " print(f\"[Server] Client #{client_id} connected from {addr}\")\n", + " while True:\n", + " data = conn.recv(1024)\n", + " if not data:\n", + " break\n", + " response = f\"[echo #{client_id}] {data.decode()}\".encode()\n", + " conn.sendall(response)\n", + " print(f\"[Server] Client #{client_id} disconnected\")\n", + "\n", + "\n", + "def multi_client_server(\n", + " server_sock: socket.socket,\n", + " ready: threading.Event,\n", + " max_clients: int = 3,\n", + ") -> None:\n", + " \"\"\"Accept up to max_clients connections, each in a new thread.\"\"\"\n", + " server_sock.listen(5)\n", + " server_sock.settimeout(5.0) # Don't block forever\n", + " ready.set()\n", + "\n", + " handlers: list[threading.Thread] = []\n", + " for i in range(max_clients):\n", + " try:\n", + " conn, addr = server_sock.accept()\n", + " t = threading.Thread(\n", + " target=handle_client, args=(conn, addr, i + 1), daemon=True\n", + " )\n", + " t.start()\n", + " handlers.append(t)\n", + " except socket.timeout:\n", + " break\n", + "\n", + " for t in handlers:\n", + " t.join(timeout=5)\n", + "\n", + "\n", + "def run_client(port: int, client_name: str, messages: list[str]) -> None:\n", + " \"\"\"A simple client that sends messages and prints responses.\"\"\"\n", + " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n", + " sock.connect((\"127.0.0.1\", port))\n", + " for msg in messages:\n", + " sock.sendall(msg.encode())\n", + " response = sock.recv(1024).decode()\n", + " print(f\" [{client_name}] Sent: {msg!r}, Got: {response!r}\")\n", + " time.sleep(0.05) # Small delay between messages\n", + "\n", + "\n", + "# Launch server\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv:\n", + " srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", + " srv.bind((\"127.0.0.1\", 0))\n", + " port = srv.getsockname()[1]\n", + "\n", + " ready = threading.Event()\n", + " server_t = threading.Thread(\n", + " target=multi_client_server, args=(srv, ready, 3), daemon=True\n", + " )\n", + " server_t.start()\n", + " ready.wait()\n", + "\n", + " # Launch 3 clients concurrently\n", + " client_threads: list[threading.Thread] = []\n", + " for i in range(1, 4):\n", + " t = threading.Thread(\n", + " target=run_client,\n", + " args=(port, f\"Client-{i}\", [f\"msg-{j}\" for j in range(1, 3)]),\n", + " )\n", + " client_threads.append(t)\n", + " t.start()\n", + "\n", + " for t in client_threads:\n", + " t.join()\n", + "\n", + " server_t.join(timeout=3)\n", + " print(\"\\nMulti-client echo server demo complete!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TCP vs UDP Comparison\n", + "\n", + "| Feature | TCP (`SOCK_STREAM`) | UDP (`SOCK_DGRAM`) |\n", + "|---|---|---|\n", + "| **Connection** | Connection-oriented (3-way handshake) | Connectionless |\n", + "| **Reliability** | Guaranteed delivery, ordering, retransmission | No guarantees |\n", + "| **Data boundaries** | Byte stream (no message boundaries) | Preserves datagram boundaries |\n", + "| **Flow control** | Yes (TCP windowing) | None |\n", + "| **Overhead** | Higher (headers, handshake, ACKs) | Lower (minimal headers) |\n", + "| **Use cases** | HTTP, SSH, email, file transfer | DNS, streaming, gaming |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class ProtocolInfo:\n", + " name: str\n", + " socket_type: socket.SocketKind\n", + " connection_oriented: bool\n", + " reliable: bool\n", + " preserves_boundaries: bool\n", + " typical_uses: list[str]\n", + "\n", + "\n", + "protocols: list[ProtocolInfo] = [\n", + " ProtocolInfo(\n", + " name=\"TCP\",\n", + " socket_type=socket.SOCK_STREAM,\n", + " connection_oriented=True,\n", + " reliable=True,\n", + " preserves_boundaries=False,\n", + " typical_uses=[\"HTTP/HTTPS\", \"SSH\", \"SMTP\", \"FTP\", \"Database connections\"],\n", + " ),\n", + " ProtocolInfo(\n", + " name=\"UDP\",\n", + " socket_type=socket.SOCK_DGRAM,\n", + " connection_oriented=False,\n", + " reliable=False,\n", + " preserves_boundaries=True,\n", + " typical_uses=[\"DNS\", \"DHCP\", \"Video streaming\", \"Online gaming\", \"VoIP\"],\n", + " ),\n", + "]\n", + "\n", + "for proto in protocols:\n", + " print(f\"=== {proto.name} ({proto.socket_type.name}) ===\")\n", + " print(f\" Connection-oriented: {proto.connection_oriented}\")\n", + " print(f\" Reliable delivery: {proto.reliable}\")\n", + " print(f\" Message boundaries: {proto.preserves_boundaries}\")\n", + " print(f\" Typical uses: {', '.join(proto.typical_uses)}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Best Practices and Common Pitfalls\n", + "\n", + "The following helper demonstrates several best practices when creating server\n", + "sockets: using `SO_REUSEADDR`, proper error handling, port 0 for testing,\n", + "and the context manager pattern." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import socket\n", + "from contextlib import contextmanager\n", + "from typing import Generator\n", + "\n", + "\n", + "@contextmanager\n", + "def create_server_socket(\n", + " host: str = \"127.0.0.1\",\n", + " port: int = 0,\n", + " backlog: int = 5,\n", + " timeout: float | None = None,\n", + ") -> Generator[socket.socket, None, None]:\n", + " \"\"\"Create a properly configured TCP server socket.\n", + "\n", + " Best practices:\n", + " - Uses SO_REUSEADDR to avoid 'Address already in use' errors\n", + " - Port 0 lets the OS assign a free port (great for testing)\n", + " - Context manager ensures the socket is always closed\n", + " - Optional timeout prevents indefinite blocking\n", + " \"\"\"\n", + " server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", + " try:\n", + " server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", + " server.bind((host, port))\n", + " server.listen(backlog)\n", + " if timeout is not None:\n", + " server.settimeout(timeout)\n", + " actual_addr = server.getsockname()\n", + " print(f\"Server listening on {actual_addr[0]}:{actual_addr[1]}\")\n", + " yield server\n", + " finally:\n", + " server.close()\n", + " print(\"Server socket closed\")\n", + "\n", + "\n", + "# Usage demonstration\n", + "with create_server_socket(timeout=1.0) as srv:\n", + " host, port = srv.getsockname()\n", + " print(f\" Assigned port: {port}\")\n", + " print(f\" Timeout: {srv.gettimeout()}s\")\n", + " print(f\" SO_REUSEADDR: {srv.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)}\")\n", + "\n", + "# Common pitfalls to avoid:\n", + "print(\"\\n=== Common Pitfalls ===\")\n", + "pitfalls: dict[str, str] = {\n", + " \"Using send() instead of sendall()\": (\n", + " \"send() may not transmit all bytes. Always use sendall() for complete delivery.\"\n", + " ),\n", + " \"Assuming recv() returns full message\": (\n", + " \"TCP is a stream protocol. Use length-prefix or delimiter-based framing.\"\n", + " ),\n", + " \"Forgetting SO_REUSEADDR on servers\": (\n", + " \"Restarting a server may fail with 'Address in use' without this option.\"\n", + " ),\n", + " \"Not setting timeouts\": (\n", + " \"Blocking sockets without timeouts can hang indefinitely.\"\n", + " ),\n", + " \"Not closing sockets\": (\n", + " \"File descriptor leaks. Always use context managers.\"\n", + " ),\n", + "}\n", + "\n", + "for pitfall, explanation in pitfalls.items():\n", + " print(f\" - {pitfall}\")\n", + " print(f\" Fix: {explanation}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook covered the fundamentals of socket programming in Python:\n", + "\n", + "1. **Socket creation** with `AF_INET`/`SOCK_STREAM` (TCP) and `SOCK_DGRAM` (UDP)\n", + "2. **Context managers** for safe socket lifecycle management\n", + "3. **DNS resolution** with `getaddrinfo()` for protocol-agnostic address lookup\n", + "4. **Socket options** like `SO_REUSEADDR` and timeout configuration\n", + "5. **TCP client-server** echo pattern with `bind()`, `listen()`, `accept()`, `connect()`\n", + "6. **Partial send/receive handling** with `sendall()` and length-prefixed framing\n", + "7. **UDP datagrams** with `sendto()` and `recvfrom()`\n", + "8. **Multi-client servers** using threading for concurrent connections\n", + "\n", + "The next notebook covers HTTP and URL handling with `urllib`, building on these\n", + "socket fundamentals with higher-level abstractions." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_17/02_http_and_urls.ipynb b/src/chapter_17/02_http_and_urls.ipynb new file mode 100644 index 0000000..d68bf4a --- /dev/null +++ b/src/chapter_17/02_http_and_urls.ipynb @@ -0,0 +1,940 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 17: HTTP and URLs\n", + "\n", + "**Networking and Protocols**\n", + "\n", + "HTTP (Hypertext Transfer Protocol) is the foundation of data exchange on the web.\n", + "Python's standard library provides `urllib.parse` for URL manipulation and\n", + "`urllib.request` for making HTTP requests. Understanding URL structure, HTTP\n", + "semantics, and content encoding is essential for building web clients, APIs,\n", + "and data pipelines." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## URL Structure and `urllib.parse`\n", + "\n", + "A URL (Uniform Resource Locator) has a well-defined structure:\n", + "\n", + "```\n", + "scheme://netloc/path;params?query#fragment\n", + " | | | | | |\n", + " https host:port /api ;v=2 ?key=val #section\n", + "```\n", + "\n", + "`urllib.parse.urlparse()` decomposes a URL string into its components.\n", + "`urlunparse()` reconstructs a URL from components." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import urlparse, urlunparse, ParseResult\n", + "\n", + "\n", + "# Parse a full URL into components\n", + "url: str = \"https://api.example.com:8443/v2/users?page=1&limit=20#results\"\n", + "parsed: ParseResult = urlparse(url)\n", + "\n", + "print(f\"URL: {url}\")\n", + "print(f\"\\nParsed components:\")\n", + "print(f\" scheme: {parsed.scheme!r}\")\n", + "print(f\" netloc: {parsed.netloc!r}\")\n", + "print(f\" hostname: {parsed.hostname!r}\")\n", + "print(f\" port: {parsed.port}\")\n", + "print(f\" path: {parsed.path!r}\")\n", + "print(f\" params: {parsed.params!r}\")\n", + "print(f\" query: {parsed.query!r}\")\n", + "print(f\" fragment: {parsed.fragment!r}\")\n", + "\n", + "# Reconstruct the URL from its parts\n", + "reconstructed: str = urlunparse((\n", + " parsed.scheme,\n", + " parsed.netloc,\n", + " parsed.path,\n", + " parsed.params,\n", + " parsed.query,\n", + " parsed.fragment,\n", + "))\n", + "print(f\"\\nReconstructed: {reconstructed}\")\n", + "print(f\"Round-trip OK: {url == reconstructed}\")\n", + "\n", + "# Parse URLs with different schemes\n", + "print(\"\\n=== Various URL Schemes ===\")\n", + "test_urls: list[str] = [\n", + " \"ftp://files.example.com/pub/data.csv\",\n", + " \"mailto:user@example.com\",\n", + " \"file:///home/user/document.txt\",\n", + " \"postgresql://user:pass@db.local:5432/mydb\",\n", + "]\n", + "for u in test_urls:\n", + " p = urlparse(u)\n", + " print(f\" {u}\")\n", + " print(f\" scheme={p.scheme!r}, netloc={p.netloc!r}, path={p.path!r}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parsing and Building Query Strings\n", + "\n", + "`parse_qs()` converts a query string into a dictionary of lists (since keys\n", + "can repeat). `parse_qsl()` returns a list of (key, value) pairs.\n", + "`urlencode()` converts a dictionary or list of pairs back into a query string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import parse_qs, parse_qsl, urlencode, urlparse\n", + "\n", + "\n", + "# Parse query strings\n", + "url: str = \"https://search.example.com/results?q=python+sockets&page=2&lang=en&lang=fr\"\n", + "query_string: str = urlparse(url).query\n", + "print(f\"Query string: {query_string!r}\")\n", + "\n", + "# parse_qs: returns dict[str, list[str]] (values are always lists)\n", + "params_dict: dict[str, list[str]] = parse_qs(query_string)\n", + "print(f\"\\nparse_qs result:\")\n", + "for key, values in params_dict.items():\n", + " print(f\" {key!r}: {values}\")\n", + "\n", + "# parse_qsl: returns list of (key, value) tuples -- preserves order and duplicates\n", + "params_list: list[tuple[str, str]] = parse_qsl(query_string)\n", + "print(f\"\\nparse_qsl result:\")\n", + "for key, value in params_list:\n", + " print(f\" ({key!r}, {value!r})\")\n", + "\n", + "# Build a query string from a dictionary\n", + "print(\"\\n=== Building Query Strings ===\")\n", + "search_params: dict[str, str | int] = {\n", + " \"q\": \"python networking\",\n", + " \"page\": 1,\n", + " \"sort\": \"relevance\",\n", + "}\n", + "encoded: str = urlencode(search_params)\n", + "print(f\"From dict: {encoded}\")\n", + "\n", + "# urlencode with repeated keys using doseq=True\n", + "multi_value_params: dict[str, list[str]] = {\n", + " \"tag\": [\"python\", \"networking\", \"sockets\"],\n", + " \"status\": [\"published\"],\n", + "}\n", + "encoded_multi: str = urlencode(multi_value_params, doseq=True)\n", + "print(f\"Multi-value: {encoded_multi}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## URL Encoding and Decoding: `quote()` and `unquote()`\n", + "\n", + "URLs can only contain a limited set of ASCII characters. Special characters\n", + "must be percent-encoded (e.g., space becomes `%20`). `quote()` encodes a string\n", + "for safe inclusion in a URL, and `unquote()` reverses the encoding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import quote, unquote, quote_plus, unquote_plus\n", + "\n", + "\n", + "# quote() percent-encodes special characters (safe='/' by default)\n", + "raw_path: str = \"/api/search/hello world & more\"\n", + "encoded_path: str = quote(raw_path)\n", + "print(f\"Original: {raw_path!r}\")\n", + "print(f\"quote(): {encoded_path!r}\")\n", + "print(f\"unquote(): {unquote(encoded_path)!r}\")\n", + "\n", + "# quote with safe='' encodes EVERYTHING including /\n", + "fully_encoded: str = quote(raw_path, safe=\"\")\n", + "print(f\"\\nquote(safe=''): {fully_encoded!r}\")\n", + "\n", + "# quote_plus: like quote, but encodes spaces as '+' (used in query strings)\n", + "print(\"\\n=== quote_plus vs quote ===\")\n", + "query_value: str = \"hello world & friends\"\n", + "print(f\"Original: {query_value!r}\")\n", + "print(f\"quote(): {quote(query_value, safe='')!r}\")\n", + "print(f\"quote_plus(): {quote_plus(query_value)!r}\")\n", + "\n", + "# Handling Unicode characters\n", + "print(\"\\n=== Unicode in URLs ===\")\n", + "unicode_text: str = \"cafe\\u0301 r\\u00e9sum\\u00e9\"\n", + "encoded_unicode: str = quote(unicode_text)\n", + "print(f\"Original: {unicode_text!r}\")\n", + "print(f\"Encoded: {encoded_unicode!r}\")\n", + "print(f\"Decoded: {unquote(encoded_unicode)!r}\")\n", + "\n", + "# Practical example: building a safe URL\n", + "print(\"\\n=== Building a Safe URL ===\")\n", + "base: str = \"https://api.example.com\"\n", + "path: str = quote(\"/users/John Doe/profile\")\n", + "query: str = quote_plus(\"search term with spaces & symbols!\")\n", + "full_url: str = f\"{base}{path}?q={query}\"\n", + "print(f\"Safe URL: {full_url}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building URLs Programmatically\n", + "\n", + "Combining `urlunparse()`, `urlencode()`, and `quote()` lets you construct\n", + "URLs safely from parts. This is especially useful when building API clients\n", + "where URL components come from user input or configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import urlencode, urlunparse, urljoin, quote\n", + "\n", + "\n", + "def build_api_url(\n", + " host: str,\n", + " path: str,\n", + " params: dict[str, str | int | list[str]] | None = None,\n", + " *,\n", + " scheme: str = \"https\",\n", + " port: int | None = None,\n", + " fragment: str = \"\",\n", + ") -> str:\n", + " \"\"\"Build a URL from components, properly encoding each part.\"\"\"\n", + " netloc: str = f\"{host}:{port}\" if port else host\n", + " encoded_path: str = quote(path)\n", + " query: str = urlencode(params, doseq=True) if params else \"\"\n", + "\n", + " return urlunparse((\n", + " scheme,\n", + " netloc,\n", + " encoded_path,\n", + " \"\", # params (rarely used)\n", + " query,\n", + " fragment,\n", + " ))\n", + "\n", + "\n", + "# Build various API URLs\n", + "print(\"=== API URL Builder ===\")\n", + "\n", + "url1 = build_api_url(\"api.example.com\", \"/v2/users\", {\"page\": 1, \"limit\": 50})\n", + "print(f\" {url1}\")\n", + "\n", + "url2 = build_api_url(\n", + " \"localhost\", \"/search\",\n", + " {\"q\": \"python sockets\", \"tag\": [\"networking\", \"tutorial\"]},\n", + " scheme=\"http\", port=8080,\n", + ")\n", + "print(f\" {url2}\")\n", + "\n", + "url3 = build_api_url(\"docs.example.com\", \"/api/Chapter 17/overview\", fragment=\"intro\")\n", + "print(f\" {url3}\")\n", + "\n", + "# urljoin: resolve relative URLs against a base\n", + "print(\"\\n=== urljoin for Relative URLs ===\")\n", + "base_url: str = \"https://api.example.com/v2/\"\n", + "relative_paths: list[str] = [\n", + " \"users\",\n", + " \"users/123\",\n", + " \"../v1/legacy\",\n", + " \"/absolute/path\",\n", + "]\n", + "for rel in relative_paths:\n", + " resolved: str = urljoin(base_url, rel)\n", + " print(f\" urljoin({base_url!r}, {rel!r})\")\n", + " print(f\" -> {resolved}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Concepts: Methods, Headers, and Status Codes\n", + "\n", + "HTTP defines a set of request **methods** (verbs), **headers** for metadata,\n", + "and **status codes** to indicate the result of a request. Understanding these\n", + "semantics is critical for working with any web API.\n", + "\n", + "| Method | Purpose | Idempotent | Has Body |\n", + "|--------|---------|------------|----------|\n", + "| GET | Retrieve a resource | Yes | No |\n", + "| POST | Create a resource | No | Yes |\n", + "| PUT | Replace a resource entirely | Yes | Yes |\n", + "| PATCH | Partially update a resource | No | Yes |\n", + "| DELETE | Remove a resource | Yes | Optional |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass, field\n", + "from enum import IntEnum\n", + "\n", + "\n", + "class HTTPStatus(IntEnum):\n", + " \"\"\"Common HTTP status codes (subset of http.HTTPStatus).\"\"\"\n", + " # 2xx Success\n", + " OK = 200\n", + " CREATED = 201\n", + " NO_CONTENT = 204\n", + "\n", + " # 3xx Redirection\n", + " MOVED_PERMANENTLY = 301\n", + " NOT_MODIFIED = 304\n", + "\n", + " # 4xx Client Error\n", + " BAD_REQUEST = 400\n", + " UNAUTHORIZED = 401\n", + " FORBIDDEN = 403\n", + " NOT_FOUND = 404\n", + " METHOD_NOT_ALLOWED = 405\n", + " CONFLICT = 409\n", + " UNPROCESSABLE_ENTITY = 422\n", + " TOO_MANY_REQUESTS = 429\n", + "\n", + " # 5xx Server Error\n", + " INTERNAL_SERVER_ERROR = 500\n", + " BAD_GATEWAY = 502\n", + " SERVICE_UNAVAILABLE = 503\n", + " GATEWAY_TIMEOUT = 504\n", + "\n", + "\n", + "@dataclass\n", + "class HTTPMethod:\n", + " name: str\n", + " idempotent: bool\n", + " safe: bool # Does not modify server state\n", + " has_body: bool\n", + " description: str\n", + "\n", + "\n", + "methods: list[HTTPMethod] = [\n", + " HTTPMethod(\"GET\", idempotent=True, safe=True, has_body=False,\n", + " description=\"Retrieve a resource\"),\n", + " HTTPMethod(\"HEAD\", idempotent=True, safe=True, has_body=False,\n", + " description=\"Like GET but returns headers only\"),\n", + " HTTPMethod(\"POST\", idempotent=False, safe=False, has_body=True,\n", + " description=\"Create a new resource or trigger an action\"),\n", + " HTTPMethod(\"PUT\", idempotent=True, safe=False, has_body=True,\n", + " description=\"Replace a resource entirely\"),\n", + " HTTPMethod(\"PATCH\", idempotent=False, safe=False, has_body=True,\n", + " description=\"Partially update a resource\"),\n", + " HTTPMethod(\"DELETE\", idempotent=True, safe=False, has_body=False,\n", + " description=\"Remove a resource\"),\n", + "]\n", + "\n", + "print(\"=== HTTP Methods ===\")\n", + "print(f\"{'Method':<8} {'Idempotent':<12} {'Safe':<6} {'Body':<6} Description\")\n", + "print(\"-\" * 70)\n", + "for m in methods:\n", + " print(f\"{m.name:<8} {str(m.idempotent):<12} {str(m.safe):<6} {str(m.has_body):<6} {m.description}\")\n", + "\n", + "# Status code categories\n", + "print(\"\\n=== HTTP Status Code Categories ===\")\n", + "categories: dict[str, list[HTTPStatus]] = {\n", + " \"2xx Success\": [],\n", + " \"3xx Redirect\": [],\n", + " \"4xx Client Error\": [],\n", + " \"5xx Server Error\": [],\n", + "}\n", + "\n", + "for status in HTTPStatus:\n", + " code = status.value\n", + " if 200 <= code < 300:\n", + " categories[\"2xx Success\"].append(status)\n", + " elif 300 <= code < 400:\n", + " categories[\"3xx Redirect\"].append(status)\n", + " elif 400 <= code < 500:\n", + " categories[\"4xx Client Error\"].append(status)\n", + " elif 500 <= code < 600:\n", + " categories[\"5xx Server Error\"].append(status)\n", + "\n", + "for category, statuses in categories.items():\n", + " print(f\"\\n {category}:\")\n", + " for s in statuses:\n", + " print(f\" {s.value} {s.name}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `urllib.request`: Making HTTP Requests (Conceptual)\n", + "\n", + "The `urllib.request` module provides `urlopen()` for making HTTP requests.\n", + "While we avoid live network calls here, understanding the API is important.\n", + "`Request` objects allow full control over method, headers, and body.\n", + "\n", + "Note: For production code, the third-party `requests` or `httpx` libraries\n", + "are generally preferred for their cleaner APIs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.request import Request\n", + "from urllib.parse import urlencode\n", + "\n", + "\n", + "# Build a GET request with custom headers\n", + "get_request = Request(\n", + " url=\"https://api.example.com/v2/users?page=1\",\n", + " headers={\n", + " \"Accept\": \"application/json\",\n", + " \"Authorization\": \"Bearer token123\",\n", + " \"User-Agent\": \"PythonClient/1.0\",\n", + " },\n", + " method=\"GET\",\n", + ")\n", + "\n", + "print(\"=== GET Request ===\")\n", + "print(f\" URL: {get_request.full_url}\")\n", + "print(f\" Method: {get_request.method}\")\n", + "print(f\" Headers: {dict(get_request.headers)}\")\n", + "\n", + "# Build a POST request with JSON body\n", + "import json\n", + "\n", + "payload: dict[str, str | int] = {\"name\": \"Alice\", \"email\": \"alice@example.com\", \"age\": 30}\n", + "json_body: bytes = json.dumps(payload).encode(\"utf-8\")\n", + "\n", + "post_request = Request(\n", + " url=\"https://api.example.com/v2/users\",\n", + " data=json_body,\n", + " headers={\n", + " \"Content-Type\": \"application/json\",\n", + " \"Accept\": \"application/json\",\n", + " },\n", + " method=\"POST\",\n", + ")\n", + "\n", + "print(\"\\n=== POST Request ===\")\n", + "print(f\" URL: {post_request.full_url}\")\n", + "print(f\" Method: {post_request.method}\")\n", + "print(f\" Headers: {dict(post_request.headers)}\")\n", + "print(f\" Body: {post_request.data}\")\n", + "\n", + "# Build a form-encoded POST request\n", + "form_data: dict[str, str] = {\"username\": \"alice\", \"password\": \"secret123\"}\n", + "form_body: bytes = urlencode(form_data).encode(\"utf-8\")\n", + "\n", + "form_request = Request(\n", + " url=\"https://api.example.com/login\",\n", + " data=form_body,\n", + " headers={\"Content-Type\": \"application/x-www-form-urlencoded\"},\n", + " method=\"POST\",\n", + ")\n", + "\n", + "print(\"\\n=== Form-Encoded POST ===\")\n", + "print(f\" URL: {form_request.full_url}\")\n", + "print(f\" Body: {form_request.data}\")\n", + "\n", + "# Note: To actually send these requests, you would use:\n", + "# from urllib.request import urlopen\n", + "# response = urlopen(request)\n", + "# data = response.read()\n", + "# status = response.status\n", + "print(\"\\n(These Request objects are ready for urlopen() -- no live calls made)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSON over HTTP: Request/Response Serialization\n", + "\n", + "JSON is the dominant format for modern web APIs. A typical pattern involves:\n", + "1. Serialize a Python dict to JSON bytes for the request body\n", + "2. Set `Content-Type: application/json` header\n", + "3. Parse the JSON response body back to a Python dict\n", + "4. Handle encoding (usually UTF-8) correctly" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from dataclasses import dataclass, asdict\n", + "from typing import Any\n", + "\n", + "\n", + "@dataclass\n", + "class APIRequest:\n", + " \"\"\"Represents an HTTP request with JSON body.\"\"\"\n", + " method: str\n", + " url: str\n", + " headers: dict[str, str]\n", + " body: dict[str, Any] | None = None\n", + "\n", + " def to_bytes(self) -> bytes | None:\n", + " \"\"\"Serialize the body to JSON bytes for transmission.\"\"\"\n", + " if self.body is None:\n", + " return None\n", + " return json.dumps(self.body, ensure_ascii=False).encode(\"utf-8\")\n", + "\n", + "\n", + "@dataclass\n", + "class APIResponse:\n", + " \"\"\"Represents an HTTP response with parsed JSON body.\"\"\"\n", + " status_code: int\n", + " headers: dict[str, str]\n", + " body: dict[str, Any] | list[Any]\n", + "\n", + " @classmethod\n", + " def from_bytes(\n", + " cls,\n", + " status_code: int,\n", + " headers: dict[str, str],\n", + " raw_body: bytes,\n", + " ) -> \"APIResponse\":\n", + " \"\"\"Parse a JSON response from raw bytes.\"\"\"\n", + " # Determine encoding from Content-Type header\n", + " content_type: str = headers.get(\"Content-Type\", \"\")\n", + " encoding: str = \"utf-8\" # Default\n", + " if \"charset=\" in content_type:\n", + " encoding = content_type.split(\"charset=\")[1].split(\";\")[0].strip()\n", + "\n", + " text: str = raw_body.decode(encoding)\n", + " body = json.loads(text)\n", + " return cls(status_code=status_code, headers=headers, body=body)\n", + "\n", + "\n", + "# Simulate a request/response cycle\n", + "print(\"=== JSON Request ===\")\n", + "request = APIRequest(\n", + " method=\"POST\",\n", + " url=\"https://api.example.com/v2/users\",\n", + " headers={\"Content-Type\": \"application/json\", \"Accept\": \"application/json\"},\n", + " body={\"name\": \"Alice\", \"email\": \"alice@example.com\", \"roles\": [\"admin\", \"user\"]},\n", + ")\n", + "request_bytes: bytes | None = request.to_bytes()\n", + "print(f\" Method: {request.method}\")\n", + "print(f\" URL: {request.url}\")\n", + "print(f\" Body bytes: {request_bytes}\")\n", + "print(f\" Body size: {len(request_bytes or b'')} bytes\")\n", + "\n", + "# Simulate receiving a response\n", + "print(\"\\n=== JSON Response ===\")\n", + "simulated_response_bytes: bytes = json.dumps({\n", + " \"id\": 42,\n", + " \"name\": \"Alice\",\n", + " \"email\": \"alice@example.com\",\n", + " \"roles\": [\"admin\", \"user\"],\n", + " \"created_at\": \"2024-01-15T10:30:00Z\",\n", + "}).encode(\"utf-8\")\n", + "\n", + "response = APIResponse.from_bytes(\n", + " status_code=201,\n", + " headers={\"Content-Type\": \"application/json; charset=utf-8\"},\n", + " raw_body=simulated_response_bytes,\n", + ")\n", + "print(f\" Status: {response.status_code}\")\n", + "print(f\" Body: {json.dumps(response.body, indent=2)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Content-Type and Encoding Handling\n", + "\n", + "The `Content-Type` header tells the receiver how to interpret the body.\n", + "Common content types:\n", + "\n", + "- `application/json` -- JSON data (UTF-8 by default)\n", + "- `application/x-www-form-urlencoded` -- HTML form data\n", + "- `multipart/form-data` -- File uploads\n", + "- `text/html; charset=utf-8` -- HTML with explicit encoding\n", + "- `application/octet-stream` -- Raw binary data\n", + "\n", + "Always check the charset parameter to decode text correctly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class ContentType:\n", + " \"\"\"Parse and represent a Content-Type header.\"\"\"\n", + " media_type: str\n", + " charset: str | None = None\n", + " boundary: str | None = None\n", + "\n", + " @classmethod\n", + " def parse(cls, header: str) -> \"ContentType\":\n", + " \"\"\"Parse a Content-Type header string.\"\"\"\n", + " parts: list[str] = [p.strip() for p in header.split(\";\")]\n", + " media_type: str = parts[0]\n", + " charset: str | None = None\n", + " boundary: str | None = None\n", + "\n", + " for param in parts[1:]:\n", + " key, _, value = param.partition(\"=\")\n", + " key = key.strip().lower()\n", + " value = value.strip().strip('\"')\n", + " if key == \"charset\":\n", + " charset = value\n", + " elif key == \"boundary\":\n", + " boundary = value\n", + "\n", + " return cls(media_type=media_type, charset=charset, boundary=boundary)\n", + "\n", + " @property\n", + " def is_json(self) -> bool:\n", + " return self.media_type in (\"application/json\", \"application/ld+json\")\n", + "\n", + " @property\n", + " def is_text(self) -> bool:\n", + " return self.media_type.startswith(\"text/\")\n", + "\n", + " @property\n", + " def effective_charset(self) -> str:\n", + " \"\"\"Return the charset, defaulting based on media type.\"\"\"\n", + " if self.charset:\n", + " return self.charset\n", + " if self.is_json:\n", + " return \"utf-8\" # JSON defaults to UTF-8\n", + " if self.is_text:\n", + " return \"iso-8859-1\" # HTTP spec default for text/*\n", + " return \"utf-8\"\n", + "\n", + "\n", + "# Parse various Content-Type headers\n", + "print(\"=== Parsing Content-Type Headers ===\")\n", + "test_headers: list[str] = [\n", + " \"application/json\",\n", + " \"application/json; charset=utf-8\",\n", + " \"text/html; charset=iso-8859-1\",\n", + " 'multipart/form-data; boundary=\"----WebKitFormBoundary\"',\n", + " \"application/x-www-form-urlencoded\",\n", + " \"text/plain\",\n", + "]\n", + "\n", + "for header in test_headers:\n", + " ct = ContentType.parse(header)\n", + " print(f\"\\n Header: {header!r}\")\n", + " print(f\" media_type: {ct.media_type!r}\")\n", + " print(f\" charset: {ct.charset!r}\")\n", + " print(f\" effective: {ct.effective_charset!r}\")\n", + " if ct.boundary:\n", + " print(f\" boundary: {ct.boundary!r}\")\n", + " print(f\" is_json: {ct.is_json}\")\n", + " print(f\" is_text: {ct.is_text}\")\n", + "\n", + "\n", + "# Demonstrate decoding with the correct charset\n", + "print(\"\\n=== Decoding with Correct Charset ===\")\n", + "\n", + "\n", + "def decode_response_body(raw: bytes, content_type_header: str) -> str:\n", + " \"\"\"Decode response bytes using the charset from Content-Type.\"\"\"\n", + " ct = ContentType.parse(content_type_header)\n", + " return raw.decode(ct.effective_charset)\n", + "\n", + "\n", + "# UTF-8 JSON response\n", + "utf8_body: bytes = '{\"message\": \"caf\\u00e9\"}'.encode(\"utf-8\")\n", + "decoded = decode_response_body(utf8_body, \"application/json; charset=utf-8\")\n", + "print(f\" UTF-8 JSON: {decoded}\")\n", + "\n", + "# Latin-1 text response\n", + "latin1_body: bytes = \"R\\xe9sum\\xe9\".encode(\"iso-8859-1\")\n", + "decoded = decode_response_body(latin1_body, \"text/plain; charset=iso-8859-1\")\n", + "print(f\" Latin-1 text: {decoded}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Error Handling for Network Requests\n", + "\n", + "Network operations can fail in many ways: DNS resolution errors, connection\n", + "refused, timeouts, HTTP error status codes, and malformed responses. Robust\n", + "code must handle each case appropriately.\n", + "\n", + "The `urllib.error` module defines:\n", + "- `URLError`: Base class for URL-related errors (network issues, DNS failures)\n", + "- `HTTPError`: Subclass of `URLError` for HTTP error responses (4xx, 5xx)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.error import URLError, HTTPError\n", + "from urllib.request import Request\n", + "from dataclasses import dataclass\n", + "from typing import Any\n", + "import json\n", + "\n", + "\n", + "@dataclass\n", + "class APIError:\n", + " \"\"\"Structured representation of an API error.\"\"\"\n", + " error_type: str\n", + " message: str\n", + " status_code: int | None = None\n", + " retryable: bool = False\n", + "\n", + "\n", + "def classify_error(error: Exception) -> APIError:\n", + " \"\"\"Classify a network error into a structured APIError.\"\"\"\n", + " if isinstance(error, HTTPError):\n", + " status: int = error.code\n", + " retryable: bool = status in (429, 500, 502, 503, 504)\n", + " return APIError(\n", + " error_type=\"http_error\",\n", + " message=f\"HTTP {status}: {error.reason}\",\n", + " status_code=status,\n", + " retryable=retryable,\n", + " )\n", + " elif isinstance(error, URLError):\n", + " reason = str(error.reason)\n", + " if \"Name or service not known\" in reason or \"nodename nor servname\" in reason:\n", + " return APIError(\n", + " error_type=\"dns_error\",\n", + " message=f\"DNS resolution failed: {reason}\",\n", + " retryable=True,\n", + " )\n", + " elif \"Connection refused\" in reason:\n", + " return APIError(\n", + " error_type=\"connection_error\",\n", + " message=f\"Connection refused: {reason}\",\n", + " retryable=True,\n", + " )\n", + " elif \"timed out\" in reason:\n", + " return APIError(\n", + " error_type=\"timeout_error\",\n", + " message=f\"Request timed out: {reason}\",\n", + " retryable=True,\n", + " )\n", + " return APIError(\n", + " error_type=\"url_error\",\n", + " message=str(reason),\n", + " retryable=False,\n", + " )\n", + " elif isinstance(error, TimeoutError):\n", + " return APIError(\n", + " error_type=\"timeout_error\",\n", + " message=\"Request timed out\",\n", + " retryable=True,\n", + " )\n", + " else:\n", + " return APIError(\n", + " error_type=\"unknown_error\",\n", + " message=str(error),\n", + " retryable=False,\n", + " )\n", + "\n", + "\n", + "# Demonstrate error classification (without live network calls)\n", + "print(\"=== Error Classification ===\")\n", + "\n", + "# Simulate various errors\n", + "simulated_errors: list[tuple[str, Exception]] = [\n", + " (\"404 Not Found\", HTTPError(\n", + " url=\"https://api.example.com/users/999\",\n", + " code=404, msg=\"Not Found\", hdrs=None, fp=None, # type: ignore[arg-type]\n", + " )),\n", + " (\"429 Rate Limited\", HTTPError(\n", + " url=\"https://api.example.com/search\",\n", + " code=429, msg=\"Too Many Requests\", hdrs=None, fp=None, # type: ignore[arg-type]\n", + " )),\n", + " (\"503 Unavailable\", HTTPError(\n", + " url=\"https://api.example.com/health\",\n", + " code=503, msg=\"Service Unavailable\", hdrs=None, fp=None, # type: ignore[arg-type]\n", + " )),\n", + " (\"DNS Failure\", URLError(\"nodename nor servname provided\")),\n", + " (\"Connection Refused\", URLError(\"Connection refused\")),\n", + " (\"Timeout\", TimeoutError(\"Connection timed out\")),\n", + "]\n", + "\n", + "for scenario, error in simulated_errors:\n", + " classified = classify_error(error)\n", + " retry_label = \"RETRY\" if classified.retryable else \"FAIL\"\n", + " status_label = f\" [{classified.status_code}]\" if classified.status_code else \"\"\n", + " print(f\" [{retry_label}] {scenario}:{status_label} {classified.error_type} -- {classified.message}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retry Logic with Exponential Backoff\n", + "\n", + "When a request fails with a retryable error (timeout, 429, 5xx), a common\n", + "strategy is to retry with **exponential backoff**: wait 1s, 2s, 4s, 8s, etc.\n", + "This avoids overwhelming a struggling server while giving transient errors\n", + "a chance to resolve." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import random\n", + "from typing import Callable, TypeVar\n", + "\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "def retry_with_backoff(\n", + " func: Callable[[], T],\n", + " max_retries: int = 3,\n", + " base_delay: float = 1.0,\n", + " max_delay: float = 30.0,\n", + " jitter: bool = True,\n", + ") -> T:\n", + " \"\"\"Execute a function with exponential backoff retry.\n", + "\n", + " Args:\n", + " func: The callable to retry.\n", + " max_retries: Maximum number of retry attempts.\n", + " base_delay: Initial delay in seconds.\n", + " max_delay: Maximum delay cap in seconds.\n", + " jitter: Add randomness to prevent thundering herd.\n", + "\n", + " Returns:\n", + " The result of the successful function call.\n", + "\n", + " Raises:\n", + " The last exception if all retries are exhausted.\n", + " \"\"\"\n", + " last_exception: Exception | None = None\n", + "\n", + " for attempt in range(max_retries + 1):\n", + " try:\n", + " return func()\n", + " except Exception as e:\n", + " last_exception = e\n", + " if attempt == max_retries:\n", + " break\n", + "\n", + " delay: float = min(base_delay * (2 ** attempt), max_delay)\n", + " if jitter:\n", + " delay *= random.uniform(0.5, 1.5)\n", + "\n", + " print(f\" Attempt {attempt + 1} failed: {e}\")\n", + " print(f\" Retrying in {delay:.2f}s...\")\n", + " time.sleep(delay)\n", + "\n", + " raise last_exception # type: ignore[misc]\n", + "\n", + "\n", + "# Demonstrate retry with a simulated flaky function\n", + "call_count: int = 0\n", + "\n", + "\n", + "def flaky_api_call() -> dict[str, str]:\n", + " \"\"\"Simulates an API that fails twice before succeeding.\"\"\"\n", + " global call_count\n", + " call_count += 1\n", + " if call_count <= 2:\n", + " raise ConnectionError(f\"Connection reset (attempt {call_count})\")\n", + " return {\"status\": \"ok\", \"data\": \"success!\"}\n", + "\n", + "\n", + "print(\"=== Retry with Exponential Backoff ===\")\n", + "call_count = 0\n", + "result = retry_with_backoff(flaky_api_call, max_retries=3, base_delay=0.1)\n", + "print(f\" Final result: {result}\")\n", + "print(f\" Total attempts: {call_count}\")\n", + "\n", + "\n", + "# Demonstrate retry exhaustion\n", + "print(\"\\n=== Retry Exhaustion ===\")\n", + "\n", + "\n", + "def always_fails() -> str:\n", + " raise TimeoutError(\"Server not responding\")\n", + "\n", + "\n", + "try:\n", + " retry_with_backoff(always_fails, max_retries=2, base_delay=0.05)\n", + "except TimeoutError as e:\n", + " print(f\" All retries exhausted: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook covered HTTP and URL handling in Python:\n", + "\n", + "1. **URL parsing** with `urlparse()` to decompose URLs into components\n", + "2. **Query strings** with `parse_qs()`, `parse_qsl()`, and `urlencode()`\n", + "3. **URL encoding** with `quote()`/`unquote()` for safe special characters\n", + "4. **Building URLs** programmatically with `urlunparse()` and `urljoin()`\n", + "5. **HTTP semantics**: methods, headers, status codes, and idempotency\n", + "6. **`urllib.request`**: constructing `Request` objects for GET, POST, and form data\n", + "7. **JSON over HTTP**: serializing request bodies and parsing response bodies\n", + "8. **Content-Type handling**: parsing headers and decoding with correct charsets\n", + "9. **Error handling**: classifying network errors as retryable or fatal\n", + "10. **Retry logic**: exponential backoff with jitter for resilient clients\n", + "\n", + "The next notebook covers async networking with `asyncio`, combining the socket\n", + "concepts from notebook 01 with non-blocking I/O for high-concurrency applications." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_17/03_async_networking.ipynb b/src/chapter_17/03_async_networking.ipynb new file mode 100644 index 0000000..0990ae7 --- /dev/null +++ b/src/chapter_17/03_async_networking.ipynb @@ -0,0 +1,1003 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 17: Async Networking\n", + "\n", + "**Networking and Protocols**\n", + "\n", + "The `asyncio` module provides high-level APIs for asynchronous networking:\n", + "`open_connection()` for TCP clients and `start_server()` for TCP servers.\n", + "These use `StreamReader` and `StreamWriter` to abstract away the complexities\n", + "of non-blocking I/O, making it straightforward to handle thousands of\n", + "concurrent connections on a single thread." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Async TCP Client with `open_connection()`\n", + "\n", + "`asyncio.open_connection()` establishes a TCP connection and returns a\n", + "`(StreamReader, StreamWriter)` pair. The `StreamReader` provides async methods\n", + "for reading (`read()`, `readline()`, `readexactly()`), and the `StreamWriter`\n", + "provides `write()` and `drain()` for non-blocking sends.\n", + "\n", + "Key pattern: `writer.write()` buffers data, then `await writer.drain()` ensures\n", + "the buffer is flushed to the network." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "\n", + "\n", + "async def async_echo_server_handler(\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + ") -> None:\n", + " \"\"\"Handle a single client connection: echo lines back.\"\"\"\n", + " addr = writer.get_extra_info(\"peername\")\n", + " print(f\"[Server] Client connected from {addr}\")\n", + "\n", + " while True:\n", + " data: bytes = await reader.readline()\n", + " if not data:\n", + " break\n", + " message: str = data.decode().strip()\n", + " print(f\"[Server] Received: {message!r}\")\n", + "\n", + " response: bytes = f\"ECHO: {message}\\n\".encode()\n", + " writer.write(response)\n", + " await writer.drain() # Ensure data is sent\n", + "\n", + " print(f\"[Server] Client {addr} disconnected\")\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + "\n", + "async def async_echo_client(\n", + " host: str,\n", + " port: int,\n", + " messages: list[str],\n", + ") -> list[str]:\n", + " \"\"\"Connect to the echo server and exchange messages.\"\"\"\n", + " reader, writer = await asyncio.open_connection(host, port)\n", + " print(f\"[Client] Connected to {host}:{port}\")\n", + "\n", + " responses: list[str] = []\n", + " for msg in messages:\n", + " # Send a line-terminated message\n", + " writer.write(f\"{msg}\\n\".encode())\n", + " await writer.drain()\n", + "\n", + " # Read the echoed response\n", + " data: bytes = await reader.readline()\n", + " response: str = data.decode().strip()\n", + " print(f\"[Client] Sent: {msg!r}, Got: {response!r}\")\n", + " responses.append(response)\n", + "\n", + " writer.close()\n", + " await writer.wait_closed()\n", + " return responses\n", + "\n", + "\n", + "async def demo_client_server() -> None:\n", + " \"\"\"Run an async echo server and client.\"\"\"\n", + " # Start the server on a random port\n", + " server = await asyncio.start_server(\n", + " async_echo_server_handler, \"127.0.0.1\", 0\n", + " )\n", + " addr = server.sockets[0].getsockname()\n", + " port: int = addr[1]\n", + " print(f\"Server started on 127.0.0.1:{port}\")\n", + "\n", + " # Run the client\n", + " messages = [\"Hello async!\", \"Python networking\", \"Goodbye\"]\n", + " responses = await async_echo_client(\"127.0.0.1\", port, messages)\n", + "\n", + " # Shut down the server\n", + " server.close()\n", + " await server.wait_closed()\n", + "\n", + " expected = [f\"ECHO: {m}\" for m in messages]\n", + " print(f\"\\nAll responses correct: {responses == expected}\")\n", + "\n", + "\n", + "await demo_client_server()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## StreamReader and StreamWriter API\n", + "\n", + "The `StreamReader` and `StreamWriter` classes provide a clean abstraction\n", + "over raw sockets:\n", + "\n", + "**StreamReader methods:**\n", + "- `read(n)` -- Read up to `n` bytes\n", + "- `readline()` -- Read until `\\n` (or EOF)\n", + "- `readexactly(n)` -- Read exactly `n` bytes (raises `IncompleteReadError` if EOF)\n", + "- `readuntil(separator)` -- Read until a specific byte sequence\n", + "\n", + "**StreamWriter methods:**\n", + "- `write(data)` -- Buffer data for sending (synchronous, non-blocking)\n", + "- `writelines(data_list)` -- Buffer multiple chunks\n", + "- `drain()` -- Await until the write buffer is flushed\n", + "- `close()` / `wait_closed()` -- Close the connection gracefully\n", + "- `get_extra_info(name)` -- Get transport info (e.g., `peername`, `sockname`)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "\n", + "\n", + "async def demonstrate_stream_api() -> None:\n", + " \"\"\"Show various StreamReader/StreamWriter methods.\"\"\"\n", + "\n", + " async def server_handler(\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + " ) -> None:\n", + " \"\"\"Server that uses different read methods.\"\"\"\n", + " # 1. readexactly: read a fixed number of bytes\n", + " header: bytes = await reader.readexactly(4)\n", + " print(f\"[Server] readexactly(4): {header!r}\")\n", + "\n", + " # 2. readline: read until newline\n", + " line: bytes = await reader.readline()\n", + " print(f\"[Server] readline(): {line!r}\")\n", + "\n", + " # 3. read: read up to N bytes\n", + " chunk: bytes = await reader.read(100)\n", + " print(f\"[Server] read(100): {chunk!r}\")\n", + "\n", + " # Send a multi-part response using writelines\n", + " writer.writelines([\n", + " b\"HTTP/1.1 200 OK\\r\\n\",\n", + " b\"Content-Type: text/plain\\r\\n\",\n", + " b\"\\r\\n\",\n", + " b\"Response body here\",\n", + " ])\n", + " await writer.drain()\n", + "\n", + " # Get connection info\n", + " peername = writer.get_extra_info(\"peername\")\n", + " sockname = writer.get_extra_info(\"sockname\")\n", + " print(f\"[Server] peername={peername}, sockname={sockname}\")\n", + "\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + " # Start server\n", + " server = await asyncio.start_server(server_handler, \"127.0.0.1\", 0)\n", + " port: int = server.sockets[0].getsockname()[1]\n", + "\n", + " # Client side\n", + " reader, writer = await asyncio.open_connection(\"127.0.0.1\", port)\n", + "\n", + " # Send data that matches the server's read pattern\n", + " writer.write(b\"HDR!\") # 4 bytes for readexactly\n", + " writer.write(b\"A line of text\\n\") # For readline\n", + " writer.write(b\"Remaining data\") # For read\n", + " await writer.drain()\n", + "\n", + " # Signal end of writing\n", + " writer.write_eof()\n", + "\n", + " # Read the full response\n", + " response: bytes = await reader.read(-1) # Read until EOF\n", + " print(f\"\\n[Client] Full response:\\n{response.decode()}\")\n", + "\n", + " writer.close()\n", + " await writer.wait_closed()\n", + " server.close()\n", + " await server.wait_closed()\n", + "\n", + "\n", + "await demonstrate_stream_api()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `asyncio.start_server` for TCP Servers\n", + "\n", + "`asyncio.start_server()` creates a TCP server that calls a handler coroutine\n", + "for each new connection. Unlike threading-based servers, all connections are\n", + "handled on a single thread via the event loop. This scales efficiently to\n", + "thousands of simultaneous connections.\n", + "\n", + "The server object supports `serve_forever()` for long-running services,\n", + "or can be used as an async context manager." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "from datetime import datetime\n", + "\n", + "\n", + "# Track connected clients\n", + "connected_clients: set[str] = set()\n", + "\n", + "\n", + "async def chat_handler(\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + ") -> None:\n", + " \"\"\"Handle a client in a simple chat-like server.\"\"\"\n", + " addr = writer.get_extra_info(\"peername\")\n", + " client_id: str = f\"{addr[0]}:{addr[1]}\"\n", + " connected_clients.add(client_id)\n", + "\n", + " # Send welcome message\n", + " welcome: str = f\"Welcome! You are {client_id}. {len(connected_clients)} client(s) connected.\\n\"\n", + " writer.write(welcome.encode())\n", + " await writer.drain()\n", + "\n", + " try:\n", + " while True:\n", + " data: bytes = await reader.readline()\n", + " if not data:\n", + " break\n", + " message: str = data.decode().strip()\n", + " timestamp: str = datetime.now().strftime(\"%H:%M:%S\")\n", + " response: str = f\"[{timestamp}] {client_id}: {message}\\n\"\n", + " writer.write(response.encode())\n", + " await writer.drain()\n", + " except ConnectionResetError:\n", + " pass\n", + " finally:\n", + " connected_clients.discard(client_id)\n", + " writer.close()\n", + " await writer.wait_closed()\n", + " print(f\"[Server] {client_id} disconnected ({len(connected_clients)} remaining)\")\n", + "\n", + "\n", + "async def demo_chat_server() -> None:\n", + " \"\"\"Run a chat server with multiple concurrent clients.\"\"\"\n", + " server = await asyncio.start_server(chat_handler, \"127.0.0.1\", 0)\n", + " port: int = server.sockets[0].getsockname()[1]\n", + " print(f\"Chat server running on 127.0.0.1:{port}\")\n", + "\n", + " async def simulate_client(name: str, messages: list[str]) -> list[str]:\n", + " \"\"\"Simulate a chat client.\"\"\"\n", + " reader, writer = await asyncio.open_connection(\"127.0.0.1\", port)\n", + "\n", + " # Read welcome message\n", + " welcome: bytes = await reader.readline()\n", + " print(f\" [{name}] {welcome.decode().strip()}\")\n", + "\n", + " responses: list[str] = []\n", + " for msg in messages:\n", + " writer.write(f\"{msg}\\n\".encode())\n", + " await writer.drain()\n", + " response: bytes = await reader.readline()\n", + " resp_text: str = response.decode().strip()\n", + " print(f\" [{name}] Got: {resp_text}\")\n", + " responses.append(resp_text)\n", + " await asyncio.sleep(0.05) # Simulate think time\n", + "\n", + " writer.close()\n", + " await writer.wait_closed()\n", + " return responses\n", + "\n", + " # Run 3 clients concurrently\n", + " results = await asyncio.gather(\n", + " simulate_client(\"Alice\", [\"Hello!\", \"How is everyone?\"]),\n", + " simulate_client(\"Bob\", [\"Hi there!\", \"Doing great!\"]),\n", + " simulate_client(\"Carol\", [\"Hey!\", \"Good to be here!\"]),\n", + " )\n", + "\n", + " print(f\"\\nTotal messages exchanged: {sum(len(r) for r in results)}\")\n", + "\n", + " server.close()\n", + " await server.wait_closed()\n", + "\n", + "\n", + "connected_clients.clear()\n", + "await demo_chat_server()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Concurrent Connections with `asyncio.gather()`\n", + "\n", + "`asyncio.gather()` runs multiple coroutines concurrently on the event loop.\n", + "This is ideal for making many network requests in parallel -- for example,\n", + "querying multiple API endpoints simultaneously or connecting to several\n", + "servers at once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "import time\n", + "\n", + "\n", + "async def simulated_api_call(endpoint: str, delay: float) -> dict[str, str | float]:\n", + " \"\"\"Simulate an async API call with variable latency.\"\"\"\n", + " start: float = time.perf_counter()\n", + " await asyncio.sleep(delay) # Simulate network I/O\n", + " elapsed: float = time.perf_counter() - start\n", + " return {\n", + " \"endpoint\": endpoint,\n", + " \"status\": \"ok\",\n", + " \"latency\": round(elapsed, 3),\n", + " }\n", + "\n", + "\n", + "async def demo_concurrent_requests() -> None:\n", + " \"\"\"Show the performance difference between sequential and concurrent.\"\"\"\n", + " endpoints: list[tuple[str, float]] = [\n", + " (\"/api/users\", 0.3),\n", + " (\"/api/products\", 0.5),\n", + " (\"/api/orders\", 0.2),\n", + " (\"/api/inventory\", 0.4),\n", + " (\"/api/analytics\", 0.6),\n", + " ]\n", + "\n", + " # Sequential: await each one at a time\n", + " print(\"=== Sequential Requests ===\")\n", + " start = time.perf_counter()\n", + " sequential_results: list[dict[str, str | float]] = []\n", + " for endpoint, delay in endpoints:\n", + " result = await simulated_api_call(endpoint, delay)\n", + " sequential_results.append(result)\n", + " seq_time: float = time.perf_counter() - start\n", + "\n", + " for r in sequential_results:\n", + " print(f\" {r['endpoint']}: {r['latency']}s\")\n", + " print(f\" Total: {seq_time:.3f}s\\n\")\n", + "\n", + " # Concurrent: gather all at once\n", + " print(\"=== Concurrent Requests (asyncio.gather) ===\")\n", + " start = time.perf_counter()\n", + " concurrent_results = await asyncio.gather(\n", + " *[simulated_api_call(ep, delay) for ep, delay in endpoints]\n", + " )\n", + " con_time: float = time.perf_counter() - start\n", + "\n", + " for r in concurrent_results:\n", + " print(f\" {r['endpoint']}: {r['latency']}s\")\n", + " print(f\" Total: {con_time:.3f}s\")\n", + " print(f\"\\nSpeedup: {seq_time / con_time:.1f}x\")\n", + "\n", + "\n", + "await demo_concurrent_requests()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Timeouts with `asyncio.wait_for()`\n", + "\n", + "Network operations should always have timeouts to prevent indefinite hangs.\n", + "`asyncio.wait_for()` wraps any awaitable with a timeout: if the operation\n", + "does not complete within the specified duration, `asyncio.TimeoutError` is\n", + "raised and the wrapped task is cancelled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "\n", + "\n", + "async def slow_network_call(name: str, delay: float) -> str:\n", + " \"\"\"Simulate a network call that may be slow.\"\"\"\n", + " await asyncio.sleep(delay)\n", + " return f\"{name} completed in {delay}s\"\n", + "\n", + "\n", + "async def fetch_with_timeout(\n", + " name: str,\n", + " delay: float,\n", + " timeout: float,\n", + ") -> str:\n", + " \"\"\"Fetch with a timeout, returning an error message on timeout.\"\"\"\n", + " try:\n", + " result: str = await asyncio.wait_for(\n", + " slow_network_call(name, delay),\n", + " timeout=timeout,\n", + " )\n", + " return result\n", + " except asyncio.TimeoutError:\n", + " return f\"{name} TIMED OUT after {timeout}s (needed {delay}s)\"\n", + "\n", + "\n", + "async def demo_timeouts() -> None:\n", + " \"\"\"Demonstrate timeout behavior with various delays.\"\"\"\n", + " print(\"=== Individual Timeouts ===\")\n", + " tasks: list[tuple[str, float, float]] = [\n", + " (\"Fast API\", 0.1, 1.0), # Completes well within timeout\n", + " (\"Medium API\", 0.5, 1.0), # Completes within timeout\n", + " (\"Slow API\", 2.0, 0.5), # Exceeds timeout\n", + " (\"Dead API\", 10.0, 0.3), # Way over timeout\n", + " ]\n", + "\n", + " results = await asyncio.gather(\n", + " *[fetch_with_timeout(name, delay, timeout) for name, delay, timeout in tasks]\n", + " )\n", + "\n", + " for result in results:\n", + " status = \"TIMEOUT\" if \"TIMED OUT\" in result else \"OK\"\n", + " print(f\" [{status}] {result}\")\n", + "\n", + " # Timeout on the entire gather operation\n", + " print(\"\\n=== Timeout on Entire Batch ===\")\n", + " try:\n", + " batch_results = await asyncio.wait_for(\n", + " asyncio.gather(\n", + " slow_network_call(\"A\", 0.1),\n", + " slow_network_call(\"B\", 0.2),\n", + " slow_network_call(\"C\", 5.0), # This one is too slow\n", + " ),\n", + " timeout=0.5,\n", + " )\n", + " print(f\" All completed: {batch_results}\")\n", + " except asyncio.TimeoutError:\n", + " print(\" Entire batch timed out (one slow call held up the group)\")\n", + "\n", + "\n", + "await demo_timeouts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSON Message Framing: Length-Prefixed Protocol\n", + "\n", + "When sending structured data (like JSON) over TCP, we need a **framing protocol**\n", + "to know where one message ends and the next begins. A common approach is\n", + "**length-prefixed framing**:\n", + "\n", + "1. Send a 4-byte big-endian integer indicating the message length\n", + "2. Send the JSON payload of exactly that length\n", + "\n", + "This is more robust than newline-delimited framing because JSON can contain\n", + "newlines within string values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "import json\n", + "import struct\n", + "from typing import Any\n", + "\n", + "\n", + "async def send_json(\n", + " writer: asyncio.StreamWriter,\n", + " data: dict[str, Any],\n", + ") -> None:\n", + " \"\"\"Send a JSON message with a 4-byte length prefix.\"\"\"\n", + " payload: bytes = json.dumps(data, ensure_ascii=False).encode(\"utf-8\")\n", + " header: bytes = struct.pack(\"!I\", len(payload))\n", + " writer.write(header + payload)\n", + " await writer.drain()\n", + "\n", + "\n", + "async def recv_json(\n", + " reader: asyncio.StreamReader,\n", + ") -> dict[str, Any]:\n", + " \"\"\"Receive a length-prefixed JSON message.\"\"\"\n", + " header: bytes = await reader.readexactly(4)\n", + " (length,) = struct.unpack(\"!I\", header)\n", + " payload: bytes = await reader.readexactly(length)\n", + " return json.loads(payload.decode(\"utf-8\"))\n", + "\n", + "\n", + "async def json_rpc_server(\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + ") -> None:\n", + " \"\"\"A simple JSON-RPC-like server.\"\"\"\n", + " addr = writer.get_extra_info(\"peername\")\n", + " print(f\"[JSON Server] Client connected from {addr}\")\n", + "\n", + " try:\n", + " while True:\n", + " try:\n", + " request: dict[str, Any] = await recv_json(reader)\n", + " except asyncio.IncompleteReadError:\n", + " break # Client disconnected\n", + "\n", + " method: str = request.get(\"method\", \"unknown\")\n", + " params: dict[str, Any] = request.get(\"params\", {})\n", + " req_id: int = request.get(\"id\", 0)\n", + "\n", + " # Simple method dispatch\n", + " if method == \"add\":\n", + " result = params.get(\"a\", 0) + params.get(\"b\", 0)\n", + " elif method == \"greet\":\n", + " result = f\"Hello, {params.get('name', 'World')}!\"\n", + " elif method == \"echo\":\n", + " result = params\n", + " else:\n", + " result = {\"error\": f\"Unknown method: {method}\"}\n", + "\n", + " response: dict[str, Any] = {\"id\": req_id, \"result\": result}\n", + " await send_json(writer, response)\n", + " finally:\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + "\n", + "async def demo_json_protocol() -> None:\n", + " \"\"\"Demonstrate length-prefixed JSON messaging.\"\"\"\n", + " server = await asyncio.start_server(json_rpc_server, \"127.0.0.1\", 0)\n", + " port: int = server.sockets[0].getsockname()[1]\n", + " print(f\"JSON-RPC server on port {port}\\n\")\n", + "\n", + " reader, writer = await asyncio.open_connection(\"127.0.0.1\", port)\n", + "\n", + " # Send various requests\n", + " requests: list[dict[str, Any]] = [\n", + " {\"id\": 1, \"method\": \"add\", \"params\": {\"a\": 10, \"b\": 32}},\n", + " {\"id\": 2, \"method\": \"greet\", \"params\": {\"name\": \"Python\"}},\n", + " {\"id\": 3, \"method\": \"echo\", \"params\": {\"data\": [1, 2, 3], \"nested\": {\"key\": \"value\"}}},\n", + " {\"id\": 4, \"method\": \"unknown_method\", \"params\": {}},\n", + " ]\n", + "\n", + " for req in requests:\n", + " await send_json(writer, req)\n", + " resp: dict[str, Any] = await recv_json(reader)\n", + " print(f\" Request: {json.dumps(req)}\")\n", + " print(f\" Response: {json.dumps(resp)}\\n\")\n", + "\n", + " writer.close()\n", + " await writer.wait_closed()\n", + " server.close()\n", + " await server.wait_closed()\n", + "\n", + "\n", + "await demo_json_protocol()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scalability: Handling Many Concurrent Connections\n", + "\n", + "One of `asyncio`'s key advantages is handling thousands of concurrent connections\n", + "on a single thread. Each connection is a lightweight coroutine (not an OS thread),\n", + "so the overhead per connection is minimal. This cell demonstrates scaling to\n", + "many simultaneous clients." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "import time\n", + "\n", + "\n", + "request_count: int = 0\n", + "\n", + "\n", + "async def counting_handler(\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + ") -> None:\n", + " \"\"\"Simple handler that counts requests.\"\"\"\n", + " global request_count\n", + " data: bytes = await reader.readline()\n", + " if data:\n", + " request_count += 1\n", + " writer.write(b\"OK\\n\")\n", + " await writer.drain()\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + "\n", + "async def make_request(host: str, port: int, message: str) -> bool:\n", + " \"\"\"Make a single request and return success status.\"\"\"\n", + " try:\n", + " reader, writer = await asyncio.open_connection(host, port)\n", + " writer.write(f\"{message}\\n\".encode())\n", + " await writer.drain()\n", + " response: bytes = await reader.readline()\n", + " writer.close()\n", + " await writer.wait_closed()\n", + " return response.strip() == b\"OK\"\n", + " except Exception:\n", + " return False\n", + "\n", + "\n", + "async def demo_scalability() -> None:\n", + " \"\"\"Show async server handling many concurrent connections.\"\"\"\n", + " global request_count\n", + " request_count = 0\n", + "\n", + " server = await asyncio.start_server(counting_handler, \"127.0.0.1\", 0)\n", + " port: int = server.sockets[0].getsockname()[1]\n", + "\n", + " # Test with increasing numbers of concurrent clients\n", + " for num_clients in [10, 50, 100, 200]:\n", + " request_count = 0\n", + " start = time.perf_counter()\n", + "\n", + " results: list[bool] = await asyncio.gather(\n", + " *[make_request(\"127.0.0.1\", port, f\"req-{i}\") for i in range(num_clients)]\n", + " )\n", + "\n", + " elapsed: float = time.perf_counter() - start\n", + " successes: int = sum(results)\n", + " rps: float = num_clients / elapsed if elapsed > 0 else 0\n", + "\n", + " print(\n", + " f\" {num_clients:>4} clients: {successes}/{num_clients} succeeded, \"\n", + " f\"{elapsed:.3f}s, {rps:.0f} req/s\"\n", + " )\n", + "\n", + " server.close()\n", + " await server.wait_closed()\n", + "\n", + "\n", + "print(\"=== Async Server Scalability ===\")\n", + "await demo_scalability()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Comparison: Sync Sockets vs Async Streams\n", + "\n", + "The following comparison highlights the key differences between traditional\n", + "blocking socket programming and async stream-based networking.\n", + "\n", + "| Aspect | Sync Sockets | Async Streams |\n", + "|--------|-------------|---------------|\n", + "| **Module** | `socket` | `asyncio` |\n", + "| **Concurrency model** | One thread per connection | One event loop, many coroutines |\n", + "| **Connection setup** | `socket()` + `connect()` | `await open_connection()` |\n", + "| **Server setup** | `bind()` + `listen()` + `accept()` | `await start_server()` |\n", + "| **Reading** | `recv()` / `recv_into()` | `await reader.read()` / `readline()` |\n", + "| **Writing** | `sendall()` | `writer.write()` + `await drain()` |\n", + "| **Blocking** | Yes (unless non-blocking mode) | Never (cooperative yielding) |\n", + "| **Scalability** | ~100s of threads | ~10,000s of coroutines |\n", + "| **Error handling** | `socket.error`, `OSError` | `asyncio.IncompleteReadError`, etc. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "import socket\n", + "import threading\n", + "import time\n", + "\n", + "\n", + "NUM_CONNECTIONS: int = 50\n", + "SIMULATED_LATENCY: float = 0.05 # 50ms per request\n", + "\n", + "\n", + "# --- Sync (threading) approach ---\n", + "def sync_handle_client(conn: socket.socket) -> None:\n", + " with conn:\n", + " data = conn.recv(1024)\n", + " time.sleep(SIMULATED_LATENCY) # Simulate processing\n", + " conn.sendall(b\"OK\")\n", + "\n", + "\n", + "def run_sync_server(port: int, ready: threading.Event) -> None:\n", + " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv:\n", + " srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n", + " srv.bind((\"127.0.0.1\", port))\n", + " srv.listen(100)\n", + " srv.settimeout(5.0)\n", + " ready.set()\n", + "\n", + " threads: list[threading.Thread] = []\n", + " for _ in range(NUM_CONNECTIONS):\n", + " try:\n", + " conn, _ = srv.accept()\n", + " t = threading.Thread(target=sync_handle_client, args=(conn,), daemon=True)\n", + " t.start()\n", + " threads.append(t)\n", + " except socket.timeout:\n", + " break\n", + " for t in threads:\n", + " t.join(timeout=5)\n", + "\n", + "\n", + "def run_sync_clients(port: int) -> float:\n", + " start = time.perf_counter()\n", + " threads: list[threading.Thread] = []\n", + "\n", + " def client_task() -> None:\n", + " with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n", + " s.connect((\"127.0.0.1\", port))\n", + " s.sendall(b\"hello\")\n", + " s.recv(1024)\n", + "\n", + " for _ in range(NUM_CONNECTIONS):\n", + " t = threading.Thread(target=client_task, daemon=True)\n", + " threads.append(t)\n", + " t.start()\n", + " for t in threads:\n", + " t.join()\n", + " return time.perf_counter() - start\n", + "\n", + "\n", + "# --- Async approach ---\n", + "async def async_handle_client(\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + ") -> None:\n", + " await reader.read(1024)\n", + " await asyncio.sleep(SIMULATED_LATENCY) # Simulate processing\n", + " writer.write(b\"OK\")\n", + " await writer.drain()\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + "\n", + "async def run_async_benchmark() -> float:\n", + " server = await asyncio.start_server(async_handle_client, \"127.0.0.1\", 0)\n", + " port: int = server.sockets[0].getsockname()[1]\n", + "\n", + " async def client_task() -> None:\n", + " reader, writer = await asyncio.open_connection(\"127.0.0.1\", port)\n", + " writer.write(b\"hello\")\n", + " await writer.drain()\n", + " await reader.read(1024)\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + " start = time.perf_counter()\n", + " await asyncio.gather(*[client_task() for _ in range(NUM_CONNECTIONS)])\n", + " elapsed = time.perf_counter() - start\n", + "\n", + " server.close()\n", + " await server.wait_closed()\n", + " return elapsed\n", + "\n", + "\n", + "# Run sync benchmark\n", + "print(f\"=== Benchmark: {NUM_CONNECTIONS} Concurrent Connections ===\")\n", + "print(f\" Simulated latency: {SIMULATED_LATENCY * 1000:.0f}ms per request\\n\")\n", + "\n", + "# Find a free port for the sync server\n", + "with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as tmp:\n", + " tmp.bind((\"127.0.0.1\", 0))\n", + " sync_port: int = tmp.getsockname()[1]\n", + "\n", + "ready = threading.Event()\n", + "server_thread = threading.Thread(\n", + " target=run_sync_server, args=(sync_port, ready), daemon=True\n", + ")\n", + "server_thread.start()\n", + "ready.wait()\n", + "sync_time: float = run_sync_clients(sync_port)\n", + "server_thread.join(timeout=5)\n", + "print(f\" Threading: {sync_time:.3f}s\")\n", + "\n", + "# Run async benchmark\n", + "async_time: float = await run_async_benchmark()\n", + "print(f\" Asyncio: {async_time:.3f}s\")\n", + "\n", + "if sync_time > 0 and async_time > 0:\n", + " faster = \"asyncio\" if async_time < sync_time else \"threading\"\n", + " ratio = max(sync_time, async_time) / min(sync_time, async_time)\n", + " print(f\"\\n {faster} was {ratio:.1f}x faster\")\n", + " print(f\" (Both handle the same {NUM_CONNECTIONS} concurrent connections)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Graceful Shutdown Pattern\n", + "\n", + "Production async servers need to shut down gracefully: stop accepting new\n", + "connections, let in-flight requests complete, and then close. This pattern\n", + "uses `asyncio.Event` as a shutdown signal and `server.close()` +\n", + "`server.wait_closed()` for clean termination." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "from dataclasses import dataclass, field\n", + "from typing import Any\n", + "\n", + "\n", + "@dataclass\n", + "class ServerStats:\n", + " \"\"\"Track server statistics.\"\"\"\n", + " connections_accepted: int = 0\n", + " connections_active: int = 0\n", + " requests_processed: int = 0\n", + " errors: int = 0\n", + "\n", + "\n", + "class AsyncTCPServer:\n", + " \"\"\"A production-style async TCP server with graceful shutdown.\"\"\"\n", + "\n", + " def __init__(self, host: str = \"127.0.0.1\", port: int = 0) -> None:\n", + " self.host: str = host\n", + " self.port: int = port\n", + " self.stats: ServerStats = ServerStats()\n", + " self._shutdown_event: asyncio.Event = asyncio.Event()\n", + " self._server: asyncio.Server | None = None\n", + "\n", + " async def handle_client(\n", + " self,\n", + " reader: asyncio.StreamReader,\n", + " writer: asyncio.StreamWriter,\n", + " ) -> None:\n", + " \"\"\"Handle a client connection.\"\"\"\n", + " self.stats.connections_accepted += 1\n", + " self.stats.connections_active += 1\n", + " addr = writer.get_extra_info(\"peername\")\n", + "\n", + " try:\n", + " while not self._shutdown_event.is_set():\n", + " try:\n", + " data: bytes = await asyncio.wait_for(\n", + " reader.readline(), timeout=1.0\n", + " )\n", + " except asyncio.TimeoutError:\n", + " continue # Check shutdown flag\n", + "\n", + " if not data:\n", + " break\n", + "\n", + " self.stats.requests_processed += 1\n", + " message: str = data.decode().strip()\n", + " response: str = f\"Processed: {message}\\n\"\n", + " writer.write(response.encode())\n", + " await writer.drain()\n", + "\n", + " except Exception:\n", + " self.stats.errors += 1\n", + " finally:\n", + " self.stats.connections_active -= 1\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + " async def start(self) -> int:\n", + " \"\"\"Start the server and return the assigned port.\"\"\"\n", + " self._server = await asyncio.start_server(\n", + " self.handle_client, self.host, self.port\n", + " )\n", + " actual_port: int = self._server.sockets[0].getsockname()[1]\n", + " self.port = actual_port\n", + " print(f\"Server started on {self.host}:{actual_port}\")\n", + " return actual_port\n", + "\n", + " async def shutdown(self, grace_period: float = 2.0) -> None:\n", + " \"\"\"Gracefully shut down the server.\"\"\"\n", + " if self._server is None:\n", + " return\n", + "\n", + " print(\"Initiating graceful shutdown...\")\n", + " self._shutdown_event.set() # Signal handlers to stop\n", + "\n", + " # Stop accepting new connections\n", + " self._server.close()\n", + " await self._server.wait_closed()\n", + "\n", + " # Wait for active connections to finish\n", + " if self.stats.connections_active > 0:\n", + " print(f\" Waiting for {self.stats.connections_active} active connection(s)...\")\n", + " await asyncio.sleep(grace_period)\n", + "\n", + " print(f\"Server shut down. Stats: {self.stats}\")\n", + "\n", + "\n", + "# Demonstrate graceful shutdown\n", + "async def demo_graceful_shutdown() -> None:\n", + " server = AsyncTCPServer()\n", + " port = await server.start()\n", + "\n", + " async def client_session(client_id: int) -> None:\n", + " reader, writer = await asyncio.open_connection(\"127.0.0.1\", port)\n", + " for i in range(3):\n", + " writer.write(f\"Client-{client_id} msg-{i}\\n\".encode())\n", + " await writer.drain()\n", + " response = await reader.readline()\n", + " await asyncio.sleep(0.05)\n", + " writer.close()\n", + " await writer.wait_closed()\n", + "\n", + " # Run clients\n", + " await asyncio.gather(\n", + " *[client_session(i) for i in range(5)]\n", + " )\n", + "\n", + " # Graceful shutdown\n", + " await server.shutdown(grace_period=0.5)\n", + "\n", + "\n", + "await demo_graceful_shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook covered async networking with Python's `asyncio` module:\n", + "\n", + "1. **`asyncio.open_connection()`** for creating async TCP clients with `StreamReader`/`StreamWriter`\n", + "2. **`StreamReader` API**: `read()`, `readline()`, `readexactly()` for flexible async reads\n", + "3. **`StreamWriter` API**: `write()`, `drain()`, `close()`, `wait_closed()` for buffered sends\n", + "4. **`asyncio.start_server()`** for creating async TCP servers with per-connection handlers\n", + "5. **`asyncio.gather()`** for concurrent connections and parallel request execution\n", + "6. **`asyncio.wait_for()`** for adding timeouts to any async operation\n", + "7. **Length-prefixed JSON protocol** for structured message framing over TCP\n", + "8. **Scalability**: handling hundreds of concurrent connections on a single thread\n", + "9. **Sync vs async comparison**: threading-based sockets vs asyncio streams\n", + "10. **Graceful shutdown**: production patterns for clean server termination\n", + "\n", + "Together with notebooks 01 (socket fundamentals) and 02 (HTTP and URLs), this\n", + "completes the networking toolkit: low-level sockets for protocol control,\n", + "`urllib` for HTTP/URL operations, and `asyncio` for high-performance concurrent networking." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_17/README.md b/src/chapter_17/README.md new file mode 100644 index 0000000..b09c214 --- /dev/null +++ b/src/chapter_17/README.md @@ -0,0 +1,22 @@ +# Chapter 17: Networking and Protocols + +## Topics Covered +- Socket programming: TCP and UDP clients/servers +- `socket` module: `AF_INET`, `SOCK_STREAM`, `SOCK_DGRAM` +- HTTP with `urllib.request` and `http.client` +- `json` over HTTP: REST-style patterns +- `asyncio` networking: streams, protocols, transports +- `socketserver` for concurrent servers (threading/forking) +- SSL/TLS with `ssl` module +- Network error handling and timeouts + +## Notebooks +1. **01_socket_fundamentals.ipynb** — TCP/UDP clients and servers, socket options +2. **02_http_and_urls.ipynb** — urllib, http.client, REST patterns, JSON APIs +3. **03_async_networking.ipynb** — asyncio streams, concurrent connections, SSL + +## Key Takeaways +- Sockets are the foundation of all network communication +- Use higher-level abstractions (urllib, asyncio) for production code +- Always handle timeouts, connection errors, and partial reads +- asyncio enables high-concurrency networking without threads diff --git a/src/chapter_17/__init__.py b/src/chapter_17/__init__.py new file mode 100644 index 0000000..1b60d2c --- /dev/null +++ b/src/chapter_17/__init__.py @@ -0,0 +1 @@ +"""Chapter 17: Networking and Protocols.""" diff --git a/src/chapter_18/01_sqlite3_fundamentals.ipynb b/src/chapter_18/01_sqlite3_fundamentals.ipynb new file mode 100644 index 0000000..c00b5d9 --- /dev/null +++ b/src/chapter_18/01_sqlite3_fundamentals.ipynb @@ -0,0 +1,414 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 18: SQLite3 Fundamentals\n", + "\n", + "Python ships with `sqlite3` in the standard library, giving you a fully functional\n", + "relational database with zero external dependencies. SQLite stores the entire database\n", + "in a single file (or in memory), making it perfect for prototyping, testing, embedded\n", + "applications, and small-to-medium workloads.\n", + "\n", + "## What You Will Learn\n", + "- Connecting to SQLite databases (file-based and in-memory)\n", + "- Creating tables with `CREATE TABLE`\n", + "- Full CRUD operations: `INSERT`, `SELECT`, `UPDATE`, `DELETE`\n", + "- Parameterized queries to prevent SQL injection\n", + "- Batch inserts with `executemany()`\n", + "- Fetching results: `fetchone()`, `fetchall()`, `fetchmany()`\n", + "- Memory-efficient cursor iteration\n", + "- DB-API 2.0 (PEP 249) overview" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connecting to a Database\n", + "\n", + "Use `sqlite3.connect()` to open (or create) a database. The special string\n", + "`\":memory:\"` creates a temporary in-memory database that vanishes when the\n", + "connection is closed -- ideal for experimentation and testing." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sqlite3\n", + "\n", + "# In-memory database -- no file on disk\n", + "conn: sqlite3.Connection = sqlite3.connect(\":memory:\")\n", + "print(f\"Connection type: {type(conn)}\")\n", + "print(f\"SQLite version: {sqlite3.sqlite_version}\")\n", + "print(f\"DB-API version: {sqlite3.apilevel}\")\n", + "\n", + "# A cursor executes SQL statements and retrieves results\n", + "cur: sqlite3.Cursor = conn.cursor()\n", + "print(f\"Cursor type: {type(cur)}\")\n", + "\n", + "# File-based database (commented out to keep examples self-contained):\n", + "# conn = sqlite3.connect(\"my_app.db\") # creates the file if it doesn't exist" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating Tables with CREATE TABLE\n", + "\n", + "SQLite supports common SQL column types: `INTEGER`, `TEXT`, `REAL`, `BLOB`, and\n", + "`NULL`. The `INTEGER PRIMARY KEY` column acts as an auto-incrementing row ID." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Create a 'books' table\n", + "cur.execute(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS books (\n", + " id INTEGER PRIMARY KEY,\n", + " title TEXT NOT NULL,\n", + " author TEXT NOT NULL,\n", + " year INTEGER,\n", + " price REAL\n", + " )\n", + "\"\"\")\n", + "conn.commit()\n", + "\n", + "# Verify the table was created by querying sqlite_master\n", + "cur.execute(\"SELECT name FROM sqlite_master WHERE type='table'\")\n", + "tables: list[tuple[str]] = cur.fetchall()\n", + "print(f\"Tables in database: {[t[0] for t in tables]}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## INSERT: Adding Rows\n", + "\n", + "Use `INSERT INTO` to add rows. **Always** use parameterized queries (`?` placeholders)\n", + "instead of string formatting to prevent SQL injection attacks." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Single insert with parameterized query (? placeholders)\n", + "cur.execute(\n", + " \"INSERT INTO books (title, author, year, price) VALUES (?, ?, ?, ?)\",\n", + " (\"The Pragmatic Programmer\", \"David Thomas\", 1999, 49.99),\n", + ")\n", + "print(f\"Inserted row with id: {cur.lastrowid}\")\n", + "\n", + "# DANGER: Never do this -- vulnerable to SQL injection!\n", + "# title = \"'; DROP TABLE books; --\"\n", + "# cur.execute(f\"INSERT INTO books (title, author, year, price) VALUES ('{title}', ...)\")\n", + "\n", + "# Safe: parameterized queries escape all user input automatically\n", + "user_input: str = \"'; DROP TABLE books; --\"\n", + "cur.execute(\n", + " \"INSERT INTO books (title, author, year, price) VALUES (?, ?, ?, ?)\",\n", + " (user_input, \"Unknown\", 2024, 0.0),\n", + ")\n", + "print(f\"Safely inserted malicious string as data, id: {cur.lastrowid}\")\n", + "conn.commit()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## executemany(): Batch Inserts\n", + "\n", + "`executemany()` inserts multiple rows in a single call, which is more efficient\n", + "than calling `execute()` in a loop." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "books_to_insert: list[tuple[str, str, int, float]] = [\n", + " (\"Clean Code\", \"Robert C. Martin\", 2008, 39.99),\n", + " (\"Design Patterns\", \"Gang of Four\", 1994, 54.99),\n", + " (\"Refactoring\", \"Martin Fowler\", 1999, 47.99),\n", + " (\"Python Cookbook\", \"David Beazley\", 2013, 59.99),\n", + " (\"Fluent Python\", \"Luciano Ramalho\", 2015, 49.99),\n", + "]\n", + "\n", + "cur.executemany(\n", + " \"INSERT INTO books (title, author, year, price) VALUES (?, ?, ?, ?)\",\n", + " books_to_insert,\n", + ")\n", + "conn.commit()\n", + "\n", + "print(f\"Inserted {len(books_to_insert)} books\")\n", + "print(f\"Total rows affected: {cur.rowcount}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SELECT: Querying Data\n", + "\n", + "The `SELECT` statement retrieves rows. Use the cursor's fetch methods to\n", + "control how results are returned:\n", + "\n", + "| Method | Returns | Use When |\n", + "|--------|---------|----------|\n", + "| `fetchone()` | One row or `None` | You need exactly one result |\n", + "| `fetchall()` | List of all rows | Result set fits in memory |\n", + "| `fetchmany(n)` | Up to `n` rows | Processing in batches |\n", + "| Cursor iteration | One row at a time | Large result sets |" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# fetchall() -- get every row at once\n", + "cur.execute(\"SELECT id, title, author, year, price FROM books ORDER BY year\")\n", + "all_books: list[tuple] = cur.fetchall()\n", + "\n", + "print(f\"Total books: {len(all_books)}\\n\")\n", + "for book in all_books:\n", + " book_id, title, author, year, price = book\n", + " print(f\" [{book_id}] {title} by {author} ({year}) - ${price:.2f}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# fetchone() -- get a single row\n", + "cur.execute(\"SELECT title, price FROM books WHERE id = ?\", (1,))\n", + "row: tuple | None = cur.fetchone()\n", + "\n", + "if row is not None:\n", + " title, price = row\n", + " print(f\"Book 1: {title} (${price:.2f})\")\n", + "else:\n", + " print(\"Book not found\")\n", + "\n", + "# fetchone() returns None when no match\n", + "cur.execute(\"SELECT title FROM books WHERE id = ?\", (9999,))\n", + "missing: tuple | None = cur.fetchone()\n", + "print(f\"Missing book: {missing}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# fetchmany(n) -- process results in batches\n", + "cur.execute(\"SELECT title FROM books ORDER BY title\")\n", + "\n", + "batch_num: int = 1\n", + "while True:\n", + " batch: list[tuple] = cur.fetchmany(3)\n", + " if not batch:\n", + " break\n", + " print(f\"Batch {batch_num}: {[row[0] for row in batch]}\")\n", + " batch_num += 1" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Memory-Efficient Cursor Iteration\n", + "\n", + "For large result sets, iterate directly over the cursor. This processes one row\n", + "at a time without loading everything into memory." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Direct cursor iteration -- memory-efficient for large datasets\n", + "cur.execute(\"SELECT title, price FROM books WHERE price > ? ORDER BY price DESC\", (40.0,))\n", + "\n", + "print(\"Books over $40 (streamed one at a time):\")\n", + "for title, price in cur: # iterates row by row\n", + " print(f\" {title}: ${price:.2f}\")\n", + "\n", + "# The cursor protocol: __iter__ returns self, __next__ calls fetchone()\n", + "print(f\"\\nCursor supports iteration: {hasattr(cur, '__iter__')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## UPDATE and DELETE: Modifying Data\n", + "\n", + "`UPDATE` changes existing rows; `DELETE` removes them. Both support `WHERE`\n", + "clauses and parameterized queries. Always check `cursor.rowcount` to verify\n", + "how many rows were affected." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# UPDATE: change the price of a specific book\n", + "cur.execute(\n", + " \"UPDATE books SET price = ? WHERE title = ?\",\n", + " (44.99, \"Clean Code\"),\n", + ")\n", + "print(f\"Rows updated: {cur.rowcount}\")\n", + "\n", + "# Verify the update\n", + "cur.execute(\"SELECT title, price FROM books WHERE title = ?\", (\"Clean Code\",))\n", + "print(f\"After update: {cur.fetchone()}\")\n", + "\n", + "# DELETE: remove the malicious-string test row\n", + "cur.execute(\"DELETE FROM books WHERE author = ?\", (\"Unknown\",))\n", + "print(f\"\\nRows deleted: {cur.rowcount}\")\n", + "conn.commit()\n", + "\n", + "# Confirm deletion\n", + "cur.execute(\"SELECT COUNT(*) FROM books\")\n", + "count: int = cur.fetchone()[0]\n", + "print(f\"Remaining books: {count}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DB-API 2.0 (PEP 249) Overview\n", + "\n", + "Python's `sqlite3` module conforms to **PEP 249** (DB-API 2.0), the standard\n", + "interface for database access in Python. This means the patterns you learn here\n", + "apply to other databases too (PostgreSQL via `psycopg2`, MySQL via `mysql-connector`,\n", + "etc.).\n", + "\n", + "### Core DB-API 2.0 Concepts\n", + "\n", + "| Component | Description |\n", + "|-----------|-------------|\n", + "| `connect()` | Module-level function that returns a `Connection` |\n", + "| `Connection` | Represents a database session; manages transactions |\n", + "| `Cursor` | Executes SQL and fetches results |\n", + "| `execute()` | Run a single SQL statement with optional parameters |\n", + "| `executemany()` | Run a statement for each item in a sequence |\n", + "| `fetchone/all/many()` | Retrieve query results |\n", + "| `commit()` / `rollback()` | Transaction control |\n", + "| `close()` | Release the connection |\n", + "\n", + "### Module-Level Attributes" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# DB-API 2.0 module-level attributes\n", + "print(f\"apilevel: {sqlite3.apilevel!r}\") # '2.0'\n", + "print(f\"threadsafety: {sqlite3.threadsafety}\") # 1 = threads may share module\n", + "print(f\"paramstyle: {sqlite3.paramstyle!r}\") # 'qmark' means ? placeholders\n", + "\n", + "# Cursor description after a SELECT -- column metadata\n", + "cur.execute(\"SELECT id, title, price FROM books LIMIT 1\")\n", + "print(f\"\\nCursor description (column metadata):\")\n", + "if cur.description:\n", + " for col in cur.description:\n", + " # (name, type_code, display_size, internal_size, precision, scale, null_ok)\n", + " print(f\" Column: {col[0]!r}, type_code: {col[1]}\")\n", + "\n", + "# Clean up\n", + "conn.close()\n", + "print(\"\\nConnection closed.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Topic | What You Learned |\n", + "|-------|------------------|\n", + "| **Connecting** | `sqlite3.connect(\":memory:\")` for in-memory, or pass a file path |\n", + "| **Creating tables** | `CREATE TABLE IF NOT EXISTS` with typed columns |\n", + "| **INSERT** | Always use `?` placeholders to prevent SQL injection |\n", + "| **Batch inserts** | `executemany()` is cleaner and faster than looping |\n", + "| **SELECT** | `fetchone()`, `fetchall()`, `fetchmany()` for different needs |\n", + "| **Cursor iteration** | Iterate directly over the cursor for memory efficiency |\n", + "| **UPDATE / DELETE** | Check `rowcount` to verify how many rows were affected |\n", + "| **DB-API 2.0** | Standard interface (PEP 249) shared across database drivers |\n", + "\n", + "### Best Practices\n", + "- **Always use parameterized queries** -- never use f-strings or `%` formatting with SQL\n", + "- **Call `commit()`** after write operations to persist changes\n", + "- **Close connections** when done to release resources\n", + "- **Use `IF NOT EXISTS`** on `CREATE TABLE` to make scripts re-runnable\n", + "- **Prefer cursor iteration** over `fetchall()` for large result sets" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/chapter_18/02_transactions_and_patterns.ipynb b/src/chapter_18/02_transactions_and_patterns.ipynb new file mode 100644 index 0000000..213c85d --- /dev/null +++ b/src/chapter_18/02_transactions_and_patterns.ipynb @@ -0,0 +1,563 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 18: Transactions and Database Patterns\n", + "\n", + "Beyond basic CRUD operations, real-world database code needs reliable transaction\n", + "management, convenient row access patterns, and type handling for Python objects.\n", + "This notebook covers the patterns that make `sqlite3` code production-ready.\n", + "\n", + "## What You Will Learn\n", + "- Transaction control: `BEGIN`, `COMMIT`, `ROLLBACK`\n", + "- Using `Connection` as a context manager for automatic commit/rollback\n", + "- Isolation levels: `DEFERRED`, `IMMEDIATE`, `EXCLUSIVE`\n", + "- `sqlite3.Row` for dict-like access to query results\n", + "- Custom row factories\n", + "- Type adapters and converters (dates, custom types)\n", + "- Practical pattern: configuration store with upsert" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Transactions: BEGIN, COMMIT, ROLLBACK\n", + "\n", + "A **transaction** groups multiple SQL operations into an atomic unit: either\n", + "all succeed (`COMMIT`) or all are undone (`ROLLBACK`). SQLite starts transactions\n", + "implicitly, but you can control them explicitly for safety." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sqlite3\n", + "\n", + "conn: sqlite3.Connection = sqlite3.connect(\":memory:\")\n", + "cur: sqlite3.Cursor = conn.cursor()\n", + "\n", + "cur.execute(\"\"\"\n", + " CREATE TABLE accounts (\n", + " id INTEGER PRIMARY KEY,\n", + " name TEXT NOT NULL,\n", + " balance REAL NOT NULL DEFAULT 0.0\n", + " )\n", + "\"\"\")\n", + "cur.executemany(\n", + " \"INSERT INTO accounts (name, balance) VALUES (?, ?)\",\n", + " [(\"Alice\", 1000.0), (\"Bob\", 500.0)],\n", + ")\n", + "conn.commit()\n", + "\n", + "\n", + "def show_balances(connection: sqlite3.Connection) -> None:\n", + " \"\"\"Print all account balances.\"\"\"\n", + " for name, balance in connection.execute(\"SELECT name, balance FROM accounts\"):\n", + " print(f\" {name}: ${balance:.2f}\")\n", + "\n", + "\n", + "print(\"Initial balances:\")\n", + "show_balances(conn)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Simulate a bank transfer with explicit transaction control\n", + "transfer_amount: float = 200.0\n", + "\n", + "try:\n", + " cur.execute(\"UPDATE accounts SET balance = balance - ? WHERE name = ?\",\n", + " (transfer_amount, \"Alice\"))\n", + " cur.execute(\"UPDATE accounts SET balance = balance + ? WHERE name = ?\",\n", + " (transfer_amount, \"Bob\"))\n", + " conn.commit() # Both updates succeed together\n", + " print(\"Transfer committed successfully!\")\n", + "except Exception as e:\n", + " conn.rollback() # Undo both updates on any error\n", + " print(f\"Transfer rolled back: {e}\")\n", + "\n", + "print(\"\\nAfter transfer:\")\n", + "show_balances(conn)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Demonstrate ROLLBACK: simulate a failed transfer\n", + "try:\n", + " cur.execute(\"UPDATE accounts SET balance = balance - ? WHERE name = ?\",\n", + " (5000.0, \"Alice\")) # Would overdraw\n", + "\n", + " # Check constraint manually (SQLite lacks CHECK for this demo)\n", + " cur.execute(\"SELECT balance FROM accounts WHERE name = ?\", (\"Alice\",))\n", + " alice_balance: float = cur.fetchone()[0]\n", + " if alice_balance < 0:\n", + " raise ValueError(f\"Insufficient funds: Alice has ${alice_balance:.2f}\")\n", + "\n", + " conn.commit()\n", + "except (ValueError, sqlite3.Error) as e:\n", + " conn.rollback()\n", + " print(f\"Transaction rolled back: {e}\")\n", + "\n", + "print(\"\\nBalances unchanged after rollback:\")\n", + "show_balances(conn)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connection as Context Manager\n", + "\n", + "The recommended pattern is to use `conn` as a context manager in a `with` block.\n", + "On success, it auto-commits; on exception, it auto-rolls back. This eliminates\n", + "the try/except boilerplate.\n", + "\n", + "**Important**: The context manager handles `COMMIT`/`ROLLBACK`, but it does\n", + "**not** close the connection. You still need to close it yourself." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Successful transaction -- auto-committed\n", + "with conn:\n", + " conn.execute(\"UPDATE accounts SET balance = balance - ? WHERE name = ?\",\n", + " (100.0, \"Alice\"))\n", + " conn.execute(\"UPDATE accounts SET balance = balance + ? WHERE name = ?\",\n", + " (100.0, \"Bob\"))\n", + "\n", + "print(\"Auto-committed transfer:\")\n", + "show_balances(conn)\n", + "\n", + "# Failed transaction -- auto-rolled back\n", + "try:\n", + " with conn:\n", + " conn.execute(\"UPDATE accounts SET balance = balance + ? WHERE name = ?\",\n", + " (1000000.0, \"Bob\"))\n", + " raise RuntimeError(\"Something went wrong mid-transaction!\")\n", + "except RuntimeError as e:\n", + " print(f\"\\nCaught error: {e}\")\n", + "\n", + "print(\"Balances after auto-rollback (Bob's million reverted):\")\n", + "show_balances(conn)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Isolation Levels\n", + "\n", + "SQLite supports three transaction isolation levels that control **when** a\n", + "transaction acquires database locks:\n", + "\n", + "| Level | Lock Acquired | Use Case |\n", + "|-------|--------------|----------|\n", + "| `DEFERRED` (default) | On first read/write | General use |\n", + "| `IMMEDIATE` | On `BEGIN` | Writes that need early lock |\n", + "| `EXCLUSIVE` | On `BEGIN`, blocks all | Full exclusive access |\n", + "\n", + "In Python's `sqlite3`, set the isolation level via the `isolation_level` parameter\n", + "to `connect()`, or set it to `None` for **autocommit mode** (each statement is\n", + "its own transaction)." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Default isolation level\n", + "print(f\"Current isolation_level: {conn.isolation_level!r}\")\n", + "\n", + "# Create a new connection with explicit isolation level\n", + "conn_immediate: sqlite3.Connection = sqlite3.connect(\n", + " \":memory:\", isolation_level=\"IMMEDIATE\"\n", + ")\n", + "print(f\"IMMEDIATE connection: {conn_immediate.isolation_level!r}\")\n", + "conn_immediate.close()\n", + "\n", + "# Autocommit mode: isolation_level=None\n", + "conn_auto: sqlite3.Connection = sqlite3.connect(\n", + " \":memory:\", isolation_level=None\n", + ")\n", + "print(f\"Autocommit connection: {conn_auto.isolation_level!r}\")\n", + "\n", + "# In autocommit mode, each statement commits immediately\n", + "conn_auto.execute(\"CREATE TABLE demo (val TEXT)\")\n", + "conn_auto.execute(\"INSERT INTO demo VALUES (?)\", (\"auto-committed\",))\n", + "result = conn_auto.execute(\"SELECT val FROM demo\").fetchone()\n", + "print(f\"Auto-committed value: {result[0]}\")\n", + "conn_auto.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## sqlite3.Row: Dict-Like Row Access\n", + "\n", + "By default, `sqlite3` returns rows as plain tuples. Setting\n", + "`conn.row_factory = sqlite3.Row` gives you objects that support both\n", + "index-based **and** name-based access, plus `keys()` for column names." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Enable sqlite3.Row for dict-like access\n", + "conn.row_factory = sqlite3.Row\n", + "cur = conn.cursor()\n", + "\n", + "cur.execute(\"SELECT id, name, balance FROM accounts\")\n", + "rows: list[sqlite3.Row] = cur.fetchall()\n", + "\n", + "for row in rows:\n", + " # Access by column name\n", + " print(f\"Name: {row['name']}, Balance: ${row['balance']:.2f}\")\n", + "\n", + " # Access by index still works\n", + " assert row[1] == row['name']\n", + "\n", + "# Get column names\n", + "print(f\"\\nColumn names: {rows[0].keys()}\")\n", + "\n", + "# Convert to a regular dict\n", + "account_dict: dict[str, object] = dict(rows[0])\n", + "print(f\"As dict: {account_dict}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Custom Row Factories\n", + "\n", + "A **row factory** is a callable that receives `(cursor, row_tuple)` and returns\n", + "whatever representation you prefer: dicts, dataclasses, named tuples, etc." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class Account:\n", + " \"\"\"Represents a bank account.\"\"\"\n", + " id: int\n", + " name: str\n", + " balance: float\n", + "\n", + "\n", + "def account_factory(cursor: sqlite3.Cursor, row: tuple) -> Account:\n", + " \"\"\"Row factory that returns Account dataclass instances.\"\"\"\n", + " return Account(*row)\n", + "\n", + "\n", + "# Apply the custom row factory\n", + "conn.row_factory = account_factory\n", + "cur = conn.cursor()\n", + "\n", + "cur.execute(\"SELECT id, name, balance FROM accounts\")\n", + "accounts: list[Account] = cur.fetchall()\n", + "\n", + "for acct in accounts:\n", + " print(f\"{acct.name}'s balance: ${acct.balance:.2f} (type: {type(acct).__name__})\")\n", + "\n", + "# A generic dict factory\n", + "def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict[str, object]:\n", + " \"\"\"Row factory that returns plain dictionaries.\"\"\"\n", + " columns: list[str] = [col[0] for col in cursor.description]\n", + " return dict(zip(columns, row))\n", + "\n", + "conn.row_factory = dict_factory\n", + "result = conn.execute(\"SELECT id, name, balance FROM accounts\").fetchall()\n", + "print(f\"\\nAs dicts: {result}\")\n", + "\n", + "# Reset to default\n", + "conn.row_factory = None" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Type Adapters and Converters\n", + "\n", + "SQLite has limited native types (`INTEGER`, `REAL`, `TEXT`, `BLOB`, `NULL`).\n", + "Python's `sqlite3` module uses **adapters** and **converters** to bridge\n", + "between Python types and SQLite storage:\n", + "\n", + "- **Adapter**: Python object -> SQLite-compatible value (on write)\n", + "- **Converter**: SQLite value -> Python object (on read)\n", + "\n", + "`datetime` support is built in when you use `detect_types`." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sqlite3\n", + "from datetime import date, datetime\n", + "\n", + "# Enable built-in date/datetime converters\n", + "conn_typed: sqlite3.Connection = sqlite3.connect(\n", + " \":memory:\",\n", + " detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,\n", + ")\n", + "\n", + "conn_typed.execute(\"\"\"\n", + " CREATE TABLE events (\n", + " id INTEGER PRIMARY KEY,\n", + " name TEXT NOT NULL,\n", + " event_date date,\n", + " created_at timestamp\n", + " )\n", + "\"\"\")\n", + "\n", + "# Insert Python date and datetime objects -- adapted automatically\n", + "conn_typed.execute(\n", + " \"INSERT INTO events (name, event_date, created_at) VALUES (?, ?, ?)\",\n", + " (\"Conference\", date(2025, 6, 15), datetime(2025, 1, 10, 9, 30, 0)),\n", + ")\n", + "conn_typed.commit()\n", + "\n", + "# Read them back -- converted automatically to Python types\n", + "row = conn_typed.execute(\"SELECT name, event_date, created_at FROM events\").fetchone()\n", + "name, event_date, created_at = row\n", + "\n", + "print(f\"Event: {name}\")\n", + "print(f\"Date: {event_date} (type: {type(event_date).__name__})\")\n", + "print(f\"Created at: {created_at} (type: {type(created_at).__name__})\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "\n", + "\n", + "# Custom adapter and converter for storing JSON in SQLite\n", + "class Settings:\n", + " \"\"\"A custom type that stores structured settings.\"\"\"\n", + "\n", + " def __init__(self, data: dict[str, object]) -> None:\n", + " self.data = data\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Settings({self.data})\"\n", + "\n", + "\n", + "# Adapter: Settings -> TEXT (for storing in SQLite)\n", + "def adapt_settings(settings: Settings) -> str:\n", + " return json.dumps(settings.data)\n", + "\n", + "\n", + "# Converter: TEXT -> Settings (for reading from SQLite)\n", + "def convert_settings(raw: bytes) -> Settings:\n", + " return Settings(json.loads(raw.decode()))\n", + "\n", + "\n", + "# Register the adapter and converter\n", + "sqlite3.register_adapter(Settings, adapt_settings)\n", + "sqlite3.register_converter(\"settings\", convert_settings)\n", + "\n", + "# Use the custom type\n", + "conn_custom: sqlite3.Connection = sqlite3.connect(\n", + " \":memory:\", detect_types=sqlite3.PARSE_DECLTYPES\n", + ")\n", + "conn_custom.execute(\"CREATE TABLE prefs (id INTEGER PRIMARY KEY, config settings)\")\n", + "conn_custom.execute(\n", + " \"INSERT INTO prefs (config) VALUES (?)\",\n", + " (Settings({\"theme\": \"dark\", \"font_size\": 14, \"lang\": \"en\"}),),\n", + ")\n", + "conn_custom.commit()\n", + "\n", + "row = conn_custom.execute(\"SELECT config FROM prefs\").fetchone()\n", + "settings: Settings = row[0]\n", + "print(f\"Retrieved: {settings}\")\n", + "print(f\"Type: {type(settings).__name__}\")\n", + "print(f\"Theme: {settings.data['theme']}\")\n", + "\n", + "conn_custom.close()\n", + "conn_typed.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical Pattern: Configuration Store with Upsert\n", + "\n", + "An **upsert** (INSERT or UPDATE) is a common pattern for key-value stores.\n", + "SQLite 3.24+ supports `ON CONFLICT` for this. Here we build a reusable\n", + "configuration store." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sqlite3\n", + "\n", + "\n", + "class ConfigStore:\n", + " \"\"\"A persistent key-value configuration store backed by SQLite.\"\"\"\n", + "\n", + " def __init__(self, db_path: str = \":memory:\") -> None:\n", + " self.conn: sqlite3.Connection = sqlite3.connect(db_path)\n", + " self.conn.execute(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS config (\n", + " key TEXT PRIMARY KEY,\n", + " value TEXT NOT NULL\n", + " )\n", + " \"\"\")\n", + " self.conn.commit()\n", + "\n", + " def set(self, key: str, value: str) -> None:\n", + " \"\"\"Set a configuration value (insert or update).\"\"\"\n", + " with self.conn:\n", + " self.conn.execute(\n", + " \"INSERT INTO config (key, value) VALUES (?, ?) \"\n", + " \"ON CONFLICT(key) DO UPDATE SET value = excluded.value\",\n", + " (key, value),\n", + " )\n", + "\n", + " def get(self, key: str, default: str | None = None) -> str | None:\n", + " \"\"\"Get a configuration value, or return default if not found.\"\"\"\n", + " row = self.conn.execute(\n", + " \"SELECT value FROM config WHERE key = ?\", (key,)\n", + " ).fetchone()\n", + " return row[0] if row else default\n", + "\n", + " def delete(self, key: str) -> bool:\n", + " \"\"\"Delete a configuration key. Returns True if the key existed.\"\"\"\n", + " with self.conn:\n", + " cur = self.conn.execute(\"DELETE FROM config WHERE key = ?\", (key,))\n", + " return cur.rowcount > 0\n", + "\n", + " def all(self) -> dict[str, str]:\n", + " \"\"\"Return all configuration as a dictionary.\"\"\"\n", + " rows = self.conn.execute(\"SELECT key, value FROM config ORDER BY key\").fetchall()\n", + " return dict(rows)\n", + "\n", + " def close(self) -> None:\n", + " self.conn.close()\n", + "\n", + "\n", + "# Use the configuration store\n", + "store = ConfigStore()\n", + "\n", + "store.set(\"app.name\", \"MyApp\")\n", + "store.set(\"app.version\", \"1.0.0\")\n", + "store.set(\"app.debug\", \"true\")\n", + "print(f\"All config: {store.all()}\")\n", + "\n", + "# Upsert: update an existing key\n", + "store.set(\"app.version\", \"2.0.0\")\n", + "print(f\"\\nAfter upsert version: {store.get('app.version')}\")\n", + "\n", + "# Get with default\n", + "print(f\"Missing key: {store.get('app.secret', 'not-set')}\")\n", + "\n", + "# Delete\n", + "deleted: bool = store.delete(\"app.debug\")\n", + "print(f\"\\nDeleted 'app.debug': {deleted}\")\n", + "print(f\"Final config: {store.all()}\")\n", + "\n", + "store.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Topic | What You Learned |\n", + "|-------|------------------|\n", + "| **Transactions** | Group operations atomically with `COMMIT`/`ROLLBACK` |\n", + "| **Context manager** | `with conn:` auto-commits on success, rolls back on exception |\n", + "| **Isolation levels** | `DEFERRED`, `IMMEDIATE`, `EXCLUSIVE`, or `None` for autocommit |\n", + "| **sqlite3.Row** | Dict-like row access via `conn.row_factory = sqlite3.Row` |\n", + "| **Custom factories** | Return dataclasses, dicts, or any type from queries |\n", + "| **Adapters/converters** | Bridge Python types to/from SQLite storage (dates, JSON, etc.) |\n", + "| **Upsert pattern** | `INSERT ... ON CONFLICT DO UPDATE` for key-value stores |\n", + "\n", + "### Best Practices\n", + "- **Use `with conn:`** for all write operations to ensure atomic transactions\n", + "- **Set `row_factory`** early to avoid manual tuple unpacking throughout your code\n", + "- **Register adapters/converters** for domain types to keep SQL code clean\n", + "- **Use `detect_types`** with `PARSE_DECLTYPES` for automatic type handling\n", + "- **Prefer upsert** (`ON CONFLICT`) over check-then-insert patterns to avoid race conditions" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/chapter_18/03_advanced_database.ipynb b/src/chapter_18/03_advanced_database.ipynb new file mode 100644 index 0000000..1d35925 --- /dev/null +++ b/src/chapter_18/03_advanced_database.ipynb @@ -0,0 +1,782 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 18: Advanced Database Patterns\n", + "\n", + "This notebook brings together everything from the previous two notebooks and\n", + "introduces patterns used in real applications: schema migrations, the repository\n", + "pattern, aggregate queries, indexes, foreign keys, and a practical task tracker.\n", + "\n", + "## What You Will Learn\n", + "- Schema migrations with versioned `ALTER TABLE` patterns\n", + "- In-memory databases for fast, isolated testing\n", + "- The repository pattern for separating data access from business logic\n", + "- Aggregate queries: `COUNT`, `SUM`, `AVG`, `GROUP BY`\n", + "- Indexes and query performance\n", + "- Foreign keys and relationships\n", + "- Practical example: building a simple task tracker" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Schema Migrations: Versioned ALTER TABLE\n", + "\n", + "As your application evolves, the database schema needs to change. A **migration**\n", + "applies incremental, versioned changes. Here is a lightweight pattern that\n", + "tracks the current schema version in the database itself." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sqlite3\n", + "\n", + "\n", + "def get_schema_version(conn: sqlite3.Connection) -> int:\n", + " \"\"\"Read the current schema version from the database.\"\"\"\n", + " # user_version is a built-in SQLite pragma\n", + " return conn.execute(\"PRAGMA user_version\").fetchone()[0]\n", + "\n", + "\n", + "def set_schema_version(conn: sqlite3.Connection, version: int) -> None:\n", + " \"\"\"Set the schema version (cannot use parameter binding with PRAGMA).\"\"\"\n", + " conn.execute(f\"PRAGMA user_version = {version}\")\n", + "\n", + "\n", + "# Define migrations as a list of (version, description, sql) tuples\n", + "MIGRATIONS: list[tuple[int, str, str]] = [\n", + " (1, \"Create users table\", \"\"\"\n", + " CREATE TABLE users (\n", + " id INTEGER PRIMARY KEY,\n", + " name TEXT NOT NULL,\n", + " email TEXT NOT NULL UNIQUE\n", + " )\n", + " \"\"\"),\n", + " (2, \"Add created_at column to users\", \"\"\"\n", + " ALTER TABLE users ADD COLUMN created_at TEXT DEFAULT CURRENT_TIMESTAMP\n", + " \"\"\"),\n", + " (3, \"Add is_active column to users\", \"\"\"\n", + " ALTER TABLE users ADD COLUMN is_active INTEGER DEFAULT 1\n", + " \"\"\"),\n", + "]\n", + "\n", + "\n", + "def migrate(conn: sqlite3.Connection) -> None:\n", + " \"\"\"Apply all pending migrations.\"\"\"\n", + " current_version: int = get_schema_version(conn)\n", + " print(f\"Current schema version: {current_version}\")\n", + "\n", + " for version, description, sql in MIGRATIONS:\n", + " if version > current_version:\n", + " print(f\" Applying migration {version}: {description}\")\n", + " with conn:\n", + " conn.executescript(sql)\n", + " set_schema_version(conn, version)\n", + "\n", + " final: int = get_schema_version(conn)\n", + " print(f\"Schema version after migration: {final}\")\n", + "\n", + "\n", + "# Run migrations on a fresh database\n", + "conn: sqlite3.Connection = sqlite3.connect(\":memory:\")\n", + "migrate(conn)\n", + "\n", + "# Running again is a no-op (idempotent)\n", + "print(\"\\nRunning migrations again:\")\n", + "migrate(conn)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## In-Memory Databases for Testing\n", + "\n", + "In-memory databases are perfect for unit tests: they are fast, isolated, and\n", + "automatically cleaned up. Each `connect(\":memory:\")` call creates a completely\n", + "independent database." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def create_test_db() -> sqlite3.Connection:\n", + " \"\"\"Create a fresh test database with schema and sample data.\"\"\"\n", + " conn = sqlite3.connect(\":memory:\")\n", + " conn.row_factory = sqlite3.Row\n", + " conn.execute(\"\"\"\n", + " CREATE TABLE products (\n", + " id INTEGER PRIMARY KEY,\n", + " name TEXT NOT NULL,\n", + " price REAL NOT NULL,\n", + " stock INTEGER NOT NULL DEFAULT 0\n", + " )\n", + " \"\"\")\n", + " conn.executemany(\n", + " \"INSERT INTO products (name, price, stock) VALUES (?, ?, ?)\",\n", + " [\n", + " (\"Widget\", 9.99, 100),\n", + " (\"Gadget\", 24.99, 50),\n", + " (\"Doohickey\", 4.99, 200),\n", + " ],\n", + " )\n", + " conn.commit()\n", + " return conn\n", + "\n", + "\n", + "# Simulate test functions\n", + "def test_product_count() -> None:\n", + " \"\"\"Test that the database has the expected number of products.\"\"\"\n", + " db = create_test_db()\n", + " count: int = db.execute(\"SELECT COUNT(*) FROM products\").fetchone()[0]\n", + " assert count == 3, f\"Expected 3 products, got {count}\"\n", + " db.close()\n", + " print(\" test_product_count: PASSED\")\n", + "\n", + "\n", + "def test_update_stock() -> None:\n", + " \"\"\"Test that stock updates work correctly.\"\"\"\n", + " db = create_test_db()\n", + " db.execute(\"UPDATE products SET stock = stock - 10 WHERE name = ?\", (\"Widget\",))\n", + " db.commit()\n", + " row = db.execute(\"SELECT stock FROM products WHERE name = ?\", (\"Widget\",)).fetchone()\n", + " assert row[\"stock\"] == 90, f\"Expected 90, got {row['stock']}\"\n", + " db.close()\n", + " print(\" test_update_stock: PASSED\")\n", + "\n", + "\n", + "def test_isolation() -> None:\n", + " \"\"\"Each test gets its own database -- changes don't leak.\"\"\"\n", + " db = create_test_db()\n", + " # Widget should still have 100 stock (previous test's changes don't affect us)\n", + " row = db.execute(\"SELECT stock FROM products WHERE name = ?\", (\"Widget\",)).fetchone()\n", + " assert row[\"stock\"] == 100, f\"Expected 100, got {row['stock']}\"\n", + " db.close()\n", + " print(\" test_isolation: PASSED\")\n", + "\n", + "\n", + "print(\"Running tests with in-memory databases:\")\n", + "test_product_count()\n", + "test_update_stock()\n", + "test_isolation()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The Repository Pattern\n", + "\n", + "The **repository pattern** separates data access logic from business logic.\n", + "Instead of writing SQL throughout your application, you encapsulate all\n", + "database operations behind a clean interface. This makes your code easier\n", + "to test and maintain." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "from typing import Protocol\n", + "\n", + "\n", + "@dataclass\n", + "class User:\n", + " \"\"\"Domain model for a user.\"\"\"\n", + " id: int | None\n", + " name: str\n", + " email: str\n", + "\n", + "\n", + "class UserRepository(Protocol):\n", + " \"\"\"Interface for user data access.\"\"\"\n", + " def add(self, user: User) -> User: ...\n", + " def get_by_id(self, user_id: int) -> User | None: ...\n", + " def get_all(self) -> list[User]: ...\n", + " def update(self, user: User) -> None: ...\n", + " def delete(self, user_id: int) -> bool: ...\n", + "\n", + "\n", + "class SQLiteUserRepository:\n", + " \"\"\"SQLite implementation of the UserRepository.\"\"\"\n", + "\n", + " def __init__(self, conn: sqlite3.Connection) -> None:\n", + " self.conn = conn\n", + " self.conn.execute(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS users (\n", + " id INTEGER PRIMARY KEY,\n", + " name TEXT NOT NULL,\n", + " email TEXT NOT NULL UNIQUE\n", + " )\n", + " \"\"\")\n", + " self.conn.commit()\n", + "\n", + " def _row_to_user(self, row: tuple) -> User:\n", + " return User(id=row[0], name=row[1], email=row[2])\n", + "\n", + " def add(self, user: User) -> User:\n", + " with self.conn:\n", + " cur = self.conn.execute(\n", + " \"INSERT INTO users (name, email) VALUES (?, ?)\",\n", + " (user.name, user.email),\n", + " )\n", + " return User(id=cur.lastrowid, name=user.name, email=user.email)\n", + "\n", + " def get_by_id(self, user_id: int) -> User | None:\n", + " row = self.conn.execute(\n", + " \"SELECT id, name, email FROM users WHERE id = ?\", (user_id,)\n", + " ).fetchone()\n", + " return self._row_to_user(row) if row else None\n", + "\n", + " def get_all(self) -> list[User]:\n", + " rows = self.conn.execute(\"SELECT id, name, email FROM users ORDER BY name\").fetchall()\n", + " return [self._row_to_user(row) for row in rows]\n", + "\n", + " def update(self, user: User) -> None:\n", + " with self.conn:\n", + " self.conn.execute(\n", + " \"UPDATE users SET name = ?, email = ? WHERE id = ?\",\n", + " (user.name, user.email, user.id),\n", + " )\n", + "\n", + " def delete(self, user_id: int) -> bool:\n", + " with self.conn:\n", + " cur = self.conn.execute(\"DELETE FROM users WHERE id = ?\", (user_id,))\n", + " return cur.rowcount > 0\n", + "\n", + "\n", + "# Use the repository -- business logic never sees SQL\n", + "db: sqlite3.Connection = sqlite3.connect(\":memory:\")\n", + "repo: SQLiteUserRepository = SQLiteUserRepository(db)\n", + "\n", + "alice: User = repo.add(User(id=None, name=\"Alice\", email=\"alice@example.com\"))\n", + "bob: User = repo.add(User(id=None, name=\"Bob\", email=\"bob@example.com\"))\n", + "print(f\"Added: {alice}\")\n", + "print(f\"Added: {bob}\")\n", + "\n", + "# Read\n", + "found: User | None = repo.get_by_id(alice.id)\n", + "print(f\"\\nFound by id: {found}\")\n", + "\n", + "# Update\n", + "alice.name = \"Alice Smith\"\n", + "repo.update(alice)\n", + "print(f\"After update: {repo.get_by_id(alice.id)}\")\n", + "\n", + "# List all\n", + "print(f\"\\nAll users: {repo.get_all()}\")\n", + "\n", + "# Delete\n", + "deleted: bool = repo.delete(bob.id)\n", + "print(f\"\\nDeleted Bob: {deleted}\")\n", + "print(f\"All users after delete: {repo.get_all()}\")\n", + "\n", + "db.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Aggregate Queries: COUNT, SUM, AVG, GROUP BY\n", + "\n", + "SQL aggregate functions summarize data across rows. Combined with `GROUP BY`,\n", + "they produce powerful analytical results directly from the database." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "conn = sqlite3.connect(\":memory:\")\n", + "conn.row_factory = sqlite3.Row\n", + "\n", + "conn.execute(\"\"\"\n", + " CREATE TABLE sales (\n", + " id INTEGER PRIMARY KEY,\n", + " product TEXT NOT NULL,\n", + " category TEXT NOT NULL,\n", + " quantity INTEGER NOT NULL,\n", + " price REAL NOT NULL\n", + " )\n", + "\"\"\")\n", + "\n", + "sales_data: list[tuple[str, str, int, float]] = [\n", + " (\"Laptop\", \"Electronics\", 5, 999.99),\n", + " (\"Mouse\", \"Electronics\", 50, 29.99),\n", + " (\"Keyboard\", \"Electronics\", 30, 79.99),\n", + " (\"Desk\", \"Furniture\", 10, 299.99),\n", + " (\"Chair\", \"Furniture\", 15, 199.99),\n", + " (\"Monitor\", \"Electronics\", 20, 349.99),\n", + " (\"Bookshelf\", \"Furniture\", 8, 149.99),\n", + " (\"Headphones\", \"Electronics\", 40, 59.99),\n", + "]\n", + "\n", + "conn.executemany(\n", + " \"INSERT INTO sales (product, category, quantity, price) VALUES (?, ?, ?, ?)\",\n", + " sales_data,\n", + ")\n", + "conn.commit()\n", + "\n", + "# Basic aggregates\n", + "row = conn.execute(\"\"\"\n", + " SELECT\n", + " COUNT(*) AS total_products,\n", + " SUM(quantity) AS total_units,\n", + " ROUND(AVG(price), 2) AS avg_price,\n", + " MIN(price) AS cheapest,\n", + " MAX(price) AS most_expensive\n", + " FROM sales\n", + "\"\"\").fetchone()\n", + "\n", + "print(\"Overall statistics:\")\n", + "print(f\" Products: {row['total_products']}\")\n", + "print(f\" Total units: {row['total_units']}\")\n", + "print(f\" Average price: ${row['avg_price']}\")\n", + "print(f\" Cheapest: ${row['cheapest']}\")\n", + "print(f\" Most expensive: ${row['most_expensive']}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# GROUP BY -- aggregate per category\n", + "print(\"Sales by category:\")\n", + "print(f\" {'Category':<15} {'Products':>8} {'Units':>8} {'Avg Price':>10} {'Revenue':>12}\")\n", + "print(f\" {'-'*55}\")\n", + "\n", + "for row in conn.execute(\"\"\"\n", + " SELECT\n", + " category,\n", + " COUNT(*) AS num_products,\n", + " SUM(quantity) AS total_units,\n", + " ROUND(AVG(price), 2) AS avg_price,\n", + " ROUND(SUM(quantity * price), 2) AS revenue\n", + " FROM sales\n", + " GROUP BY category\n", + " ORDER BY revenue DESC\n", + "\"\"\"):\n", + " print(f\" {row['category']:<15} {row['num_products']:>8} \"\n", + " f\"{row['total_units']:>8} {row['avg_price']:>10.2f} \"\n", + " f\"{row['revenue']:>12.2f}\")\n", + "\n", + "# HAVING -- filter groups\n", + "print(\"\\nCategories with revenue over $5000:\")\n", + "for row in conn.execute(\"\"\"\n", + " SELECT category, ROUND(SUM(quantity * price), 2) AS revenue\n", + " FROM sales\n", + " GROUP BY category\n", + " HAVING revenue > 5000\n", + "\"\"\"):\n", + " print(f\" {row['category']}: ${row['revenue']:,.2f}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Indexes and Query Performance\n", + "\n", + "An **index** is a data structure that speeds up lookups on specific columns,\n", + "similar to a book's index. Without an index, SQLite performs a full table scan.\n", + "Use `EXPLAIN QUERY PLAN` to see how SQLite executes a query." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Check query plan BEFORE adding an index\n", + "print(\"Query plan WITHOUT index:\")\n", + "plan = conn.execute(\n", + " \"EXPLAIN QUERY PLAN SELECT * FROM sales WHERE category = ?\",\n", + " (\"Electronics\",)\n", + ").fetchall()\n", + "for row in plan:\n", + " print(f\" {row['detail']}\")\n", + "\n", + "# Create an index on the category column\n", + "conn.execute(\"CREATE INDEX IF NOT EXISTS idx_sales_category ON sales (category)\")\n", + "\n", + "# Check query plan AFTER adding an index\n", + "print(\"\\nQuery plan WITH index:\")\n", + "plan = conn.execute(\n", + " \"EXPLAIN QUERY PLAN SELECT * FROM sales WHERE category = ?\",\n", + " (\"Electronics\",)\n", + ").fetchall()\n", + "for row in plan:\n", + " print(f\" {row['detail']}\")\n", + "\n", + "# List all indexes in the database\n", + "print(\"\\nAll indexes:\")\n", + "for row in conn.execute(\"SELECT name, tbl_name FROM sqlite_master WHERE type='index'\"):\n", + " print(f\" {row['name']} on {row['tbl_name']}\")\n", + "\n", + "conn.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Foreign Keys and Relationships\n", + "\n", + "Foreign keys enforce referential integrity between tables. SQLite supports\n", + "foreign keys but **they are disabled by default**. You must enable them with\n", + "`PRAGMA foreign_keys = ON` for each connection." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "conn = sqlite3.connect(\":memory:\")\n", + "conn.row_factory = sqlite3.Row\n", + "\n", + "# Enable foreign key enforcement\n", + "conn.execute(\"PRAGMA foreign_keys = ON\")\n", + "fk_status = conn.execute(\"PRAGMA foreign_keys\").fetchone()[0]\n", + "print(f\"Foreign keys enabled: {bool(fk_status)}\")\n", + "\n", + "# Create related tables\n", + "conn.executescript(\"\"\"\n", + " CREATE TABLE authors (\n", + " id INTEGER PRIMARY KEY,\n", + " name TEXT NOT NULL\n", + " );\n", + "\n", + " CREATE TABLE books (\n", + " id INTEGER PRIMARY KEY,\n", + " title TEXT NOT NULL,\n", + " author_id INTEGER NOT NULL,\n", + " FOREIGN KEY (author_id) REFERENCES authors(id)\n", + " ON DELETE CASCADE\n", + " );\n", + "\"\"\")\n", + "\n", + "# Insert authors\n", + "conn.execute(\"INSERT INTO authors (name) VALUES (?)\", (\"Alice Walker\",))\n", + "conn.execute(\"INSERT INTO authors (name) VALUES (?)\", (\"Bob Martin\",))\n", + "\n", + "# Insert books linked to authors\n", + "conn.execute(\"INSERT INTO books (title, author_id) VALUES (?, ?)\",\n", + " (\"The Color Purple\", 1))\n", + "conn.execute(\"INSERT INTO books (title, author_id) VALUES (?, ?)\",\n", + " (\"Clean Code\", 2))\n", + "conn.execute(\"INSERT INTO books (title, author_id) VALUES (?, ?)\",\n", + " (\"Clean Architecture\", 2))\n", + "conn.commit()\n", + "\n", + "# JOIN query: get books with author names\n", + "print(\"\\nBooks with authors (JOIN):\")\n", + "for row in conn.execute(\"\"\"\n", + " SELECT b.title, a.name AS author\n", + " FROM books b\n", + " JOIN authors a ON b.author_id = a.id\n", + " ORDER BY a.name, b.title\n", + "\"\"\"):\n", + " print(f\" '{row['title']}' by {row['author']}\")\n", + "\n", + "# Foreign key constraint prevents invalid references\n", + "try:\n", + " conn.execute(\"INSERT INTO books (title, author_id) VALUES (?, ?)\",\n", + " (\"Ghost Book\", 999)) # author_id 999 doesn't exist\n", + "except sqlite3.IntegrityError as e:\n", + " print(f\"\\nFK violation caught: {e}\")\n", + "\n", + "# CASCADE delete: removing an author removes their books\n", + "conn.execute(\"DELETE FROM authors WHERE id = ?\", (2,))\n", + "conn.commit()\n", + "\n", + "remaining = conn.execute(\"SELECT title FROM books\").fetchall()\n", + "print(f\"\\nBooks after deleting Bob Martin (CASCADE): {[r['title'] for r in remaining]}\")\n", + "\n", + "conn.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical Example: Building a Simple Task Tracker\n", + "\n", + "Let's combine everything into a practical application: a task tracker with\n", + "projects, tasks, tags, migrations, and aggregate reporting." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sqlite3\n", + "from dataclasses import dataclass, field\n", + "from datetime import datetime\n", + "from enum import Enum\n", + "\n", + "\n", + "class TaskStatus(Enum):\n", + " TODO = \"todo\"\n", + " IN_PROGRESS = \"in_progress\"\n", + " DONE = \"done\"\n", + "\n", + "\n", + "@dataclass\n", + "class Task:\n", + " \"\"\"Domain model for a task.\"\"\"\n", + " id: int | None\n", + " project: str\n", + " title: str\n", + " status: TaskStatus = TaskStatus.TODO\n", + " priority: int = 1 # 1=low, 2=medium, 3=high\n", + " created_at: str = field(default_factory=lambda: datetime.now().isoformat())\n", + "\n", + "\n", + "class TaskTracker:\n", + " \"\"\"A simple task tracking application backed by SQLite.\"\"\"\n", + "\n", + " def __init__(self, db_path: str = \":memory:\") -> None:\n", + " self.conn = sqlite3.connect(db_path)\n", + " self.conn.row_factory = sqlite3.Row\n", + " self.conn.execute(\"PRAGMA foreign_keys = ON\")\n", + " self._create_schema()\n", + "\n", + " def _create_schema(self) -> None:\n", + " self.conn.executescript(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS tasks (\n", + " id INTEGER PRIMARY KEY,\n", + " project TEXT NOT NULL,\n", + " title TEXT NOT NULL,\n", + " status TEXT NOT NULL DEFAULT 'todo',\n", + " priority INTEGER NOT NULL DEFAULT 1,\n", + " created_at TEXT NOT NULL\n", + " );\n", + "\n", + " CREATE INDEX IF NOT EXISTS idx_tasks_project ON tasks (project);\n", + " CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks (status);\n", + " \"\"\")\n", + "\n", + " def _row_to_task(self, row: sqlite3.Row) -> Task:\n", + " return Task(\n", + " id=row[\"id\"],\n", + " project=row[\"project\"],\n", + " title=row[\"title\"],\n", + " status=TaskStatus(row[\"status\"]),\n", + " priority=row[\"priority\"],\n", + " created_at=row[\"created_at\"],\n", + " )\n", + "\n", + " def add_task(self, task: Task) -> Task:\n", + " \"\"\"Add a new task and return it with the assigned ID.\"\"\"\n", + " with self.conn:\n", + " cur = self.conn.execute(\n", + " \"INSERT INTO tasks (project, title, status, priority, created_at) \"\n", + " \"VALUES (?, ?, ?, ?, ?)\",\n", + " (task.project, task.title, task.status.value,\n", + " task.priority, task.created_at),\n", + " )\n", + " task.id = cur.lastrowid\n", + " return task\n", + "\n", + " def update_status(self, task_id: int, status: TaskStatus) -> None:\n", + " \"\"\"Change a task's status.\"\"\"\n", + " with self.conn:\n", + " self.conn.execute(\n", + " \"UPDATE tasks SET status = ? WHERE id = ?\",\n", + " (status.value, task_id),\n", + " )\n", + "\n", + " def get_tasks(self, project: str | None = None,\n", + " status: TaskStatus | None = None) -> list[Task]:\n", + " \"\"\"Retrieve tasks with optional filters.\"\"\"\n", + " query: str = \"SELECT * FROM tasks WHERE 1=1\"\n", + " params: list[str] = []\n", + " if project:\n", + " query += \" AND project = ?\"\n", + " params.append(project)\n", + " if status:\n", + " query += \" AND status = ?\"\n", + " params.append(status.value)\n", + " query += \" ORDER BY priority DESC, created_at\"\n", + " rows = self.conn.execute(query, params).fetchall()\n", + " return [self._row_to_task(r) for r in rows]\n", + "\n", + " def get_summary(self) -> list[dict[str, object]]:\n", + " \"\"\"Get task counts grouped by project and status.\"\"\"\n", + " rows = self.conn.execute(\"\"\"\n", + " SELECT\n", + " project,\n", + " SUM(CASE WHEN status = 'todo' THEN 1 ELSE 0 END) AS todo,\n", + " SUM(CASE WHEN status = 'in_progress' THEN 1 ELSE 0 END) AS in_progress,\n", + " SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) AS done,\n", + " COUNT(*) AS total\n", + " FROM tasks\n", + " GROUP BY project\n", + " ORDER BY project\n", + " \"\"\").fetchall()\n", + " return [dict(row) for row in rows]\n", + "\n", + " def close(self) -> None:\n", + " self.conn.close()\n", + "\n", + "\n", + "print(\"TaskTracker class defined successfully.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Build and use the task tracker\n", + "tracker = TaskTracker()\n", + "\n", + "# Add tasks to different projects\n", + "tasks_to_add: list[Task] = [\n", + " Task(id=None, project=\"Website\", title=\"Design homepage\", priority=3),\n", + " Task(id=None, project=\"Website\", title=\"Implement auth\", priority=3),\n", + " Task(id=None, project=\"Website\", title=\"Write tests\", priority=2),\n", + " Task(id=None, project=\"Website\", title=\"Deploy to staging\", priority=1),\n", + " Task(id=None, project=\"API\", title=\"Design endpoints\", priority=3),\n", + " Task(id=None, project=\"API\", title=\"Add rate limiting\", priority=2),\n", + " Task(id=None, project=\"API\", title=\"Write documentation\", priority=1),\n", + " Task(id=None, project=\"Mobile\", title=\"Setup project\", priority=3),\n", + " Task(id=None, project=\"Mobile\", title=\"Login screen\", priority=2),\n", + "]\n", + "\n", + "for task in tasks_to_add:\n", + " tracker.add_task(task)\n", + "\n", + "print(f\"Added {len(tasks_to_add)} tasks across projects.\\n\")\n", + "\n", + "# Update some statuses\n", + "tracker.update_status(1, TaskStatus.DONE) # Design homepage\n", + "tracker.update_status(2, TaskStatus.IN_PROGRESS) # Implement auth\n", + "tracker.update_status(5, TaskStatus.DONE) # Design endpoints\n", + "tracker.update_status(6, TaskStatus.IN_PROGRESS) # Add rate limiting\n", + "tracker.update_status(8, TaskStatus.DONE) # Setup project\n", + "\n", + "# Query tasks by project\n", + "print(\"Website tasks:\")\n", + "for t in tracker.get_tasks(project=\"Website\"):\n", + " priority_label = {1: \"LOW\", 2: \"MED\", 3: \"HIGH\"}[t.priority]\n", + " print(f\" [{t.status.value:12s}] [{priority_label:>4s}] {t.title}\")\n", + "\n", + "# Query tasks by status\n", + "print(\"\\nAll in-progress tasks:\")\n", + "for t in tracker.get_tasks(status=TaskStatus.IN_PROGRESS):\n", + " print(f\" {t.project}/{t.title}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Project summary report using aggregate queries\n", + "print(\"Project Summary\")\n", + "print(f\"{'Project':<12} {'Todo':>6} {'In Prog':>8} {'Done':>6} {'Total':>7}\")\n", + "print(f\"{'-'*41}\")\n", + "\n", + "for summary in tracker.get_summary():\n", + " print(f\"{summary['project']:<12} {summary['todo']:>6} \"\n", + " f\"{summary['in_progress']:>8} {summary['done']:>6} \"\n", + " f\"{summary['total']:>7}\")\n", + "\n", + "# Overall stats\n", + "all_tasks: list[Task] = tracker.get_tasks()\n", + "done_count: int = sum(1 for t in all_tasks if t.status == TaskStatus.DONE)\n", + "total_count: int = len(all_tasks)\n", + "completion: float = (done_count / total_count * 100) if total_count else 0.0\n", + "\n", + "print(f\"\\nOverall completion: {done_count}/{total_count} ({completion:.0f}%)\")\n", + "\n", + "tracker.close()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Topic | What You Learned |\n", + "|-------|------------------|\n", + "| **Schema migrations** | Version schema changes with `PRAGMA user_version` and ordered migration scripts |\n", + "| **In-memory testing** | `\":memory:\"` gives fast, isolated databases for each test |\n", + "| **Repository pattern** | Encapsulate SQL behind a clean Python interface with typed domain models |\n", + "| **Aggregates** | `COUNT`, `SUM`, `AVG`, `MIN`, `MAX` with `GROUP BY` and `HAVING` |\n", + "| **Indexes** | Speed up queries with `CREATE INDEX`; verify with `EXPLAIN QUERY PLAN` |\n", + "| **Foreign keys** | `PRAGMA foreign_keys = ON`, `REFERENCES`, `ON DELETE CASCADE` |\n", + "| **Task tracker** | Combines all patterns: schema, repository, queries, and reporting |\n", + "\n", + "### Architecture Guidelines\n", + "- **Separate concerns**: Keep SQL in repository classes, business logic outside\n", + "- **Use migrations**: Never manually alter a database; script every change\n", + "- **Index strategically**: Add indexes on columns you filter or join on\n", + "- **Enable foreign keys**: They catch bugs before bad data enters the system\n", + "- **Test with in-memory databases**: Fast, isolated, no cleanup needed\n", + "- **Use domain models**: Map rows to dataclasses or named tuples for type safety" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/chapter_18/README.md b/src/chapter_18/README.md new file mode 100644 index 0000000..98a9298 --- /dev/null +++ b/src/chapter_18/README.md @@ -0,0 +1,22 @@ +# Chapter 18: Database Access + +## Topics Covered +- `sqlite3` module: connections, cursors, queries +- DB-API 2.0 (PEP 249): the standard database interface +- Parameterized queries and SQL injection prevention +- Transactions: commit, rollback, autocommit, context managers +- Schema design and migrations +- Row factories and custom types +- In-memory databases for testing +- ORM patterns and the Repository pattern + +## Notebooks +1. **01_sqlite3_fundamentals.ipynb** — Connections, CRUD, parameterized queries +2. **02_transactions_and_patterns.ipynb** — Transactions, context managers, row factories +3. **03_advanced_database.ipynb** — Migrations, testing patterns, repository pattern + +## Key Takeaways +- Always use parameterized queries to prevent SQL injection +- Context managers ensure proper transaction handling +- In-memory databases make tests fast and isolated +- The Repository pattern decouples business logic from data access diff --git a/src/chapter_18/__init__.py b/src/chapter_18/__init__.py new file mode 100644 index 0000000..3af8940 --- /dev/null +++ b/src/chapter_18/__init__.py @@ -0,0 +1 @@ +"""Chapter 18: Database Access.""" diff --git a/src/chapter_19/01_virtual_environments.ipynb b/src/chapter_19/01_virtual_environments.ipynb new file mode 100644 index 0000000..23c55a4 --- /dev/null +++ b/src/chapter_19/01_virtual_environments.ipynb @@ -0,0 +1,617 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 19: Virtual Environments and Dependency Management\n", + "\n", + "This notebook covers Python virtual environments -- the essential mechanism for isolating\n", + "project dependencies and ensuring reproducible builds. We explore the built-in `venv` module,\n", + "pip fundamentals, dependency resolution, and best practices for professional Python development.\n", + "\n", + "## Key Concepts\n", + "- **Isolation**: Each project gets its own set of installed packages\n", + "- **Reproducibility**: Pin exact versions so builds are repeatable\n", + "- **venv module**: The standard library tool for creating virtual environments\n", + "- **pip**: The package installer for Python" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 1: Why Virtual Environments?\n", + "\n", + "Without virtual environments, every Python project on your system shares the same\n", + "global `site-packages` directory. This causes real problems:\n", + "\n", + "| Problem | Example |\n", + "|---|---|\n", + "| **Version conflicts** | Project A needs `requests==2.28` but Project B needs `requests==2.31` |\n", + "| **Polluted global state** | Installing packages globally can break system tools |\n", + "| **Non-reproducible builds** | No way to know which packages a project actually requires |\n", + "| **Permission issues** | System Python often requires `sudo` to install packages |\n", + "\n", + "A virtual environment creates a **self-contained directory tree** with its own Python\n", + "interpreter and `site-packages`. Packages installed inside the environment are invisible\n", + "to other environments." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sys\n", + "\n", + "# These paths reveal which Python environment is currently active.\n", + "# When running inside a virtual environment, sys.prefix differs from sys.base_prefix.\n", + "\n", + "print(f\"sys.executable: {sys.executable}\")\n", + "print(f\"sys.prefix: {sys.prefix}\")\n", + "print(f\"sys.base_prefix: {sys.base_prefix}\")\n", + "print()\n", + "\n", + "# The definitive check for whether you are inside a virtual environment\n", + "in_venv: bool = sys.prefix != sys.base_prefix\n", + "print(f\"Inside a virtual environment: {in_venv}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 2: Creating Virtual Environments with `venv`\n", + "\n", + "The `venv` module is part of the standard library (since Python 3.3). The typical\n", + "shell workflow is:\n", + "\n", + "```bash\n", + "# Create a new virtual environment\n", + "python -m venv .venv\n", + "\n", + "# Activate it (macOS / Linux)\n", + "source .venv/bin/activate\n", + "\n", + "# Activate it (Windows)\n", + ".venv\\Scripts\\activate\n", + "\n", + "# Deactivate when done\n", + "deactivate\n", + "```\n", + "\n", + "The `.venv` directory contains:\n", + "- A copy or symlink to the Python interpreter\n", + "- A `lib/pythonX.Y/site-packages/` directory for installed packages\n", + "- Activation scripts that adjust `PATH` and `VIRTUAL_ENV`\n", + "\n", + "**Convention**: Name the directory `.venv` (with a leading dot) so it is hidden on\n", + "Unix systems and easily matched by `.gitignore`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import os\n", + "import sys\n", + "import sysconfig\n", + "\n", + "# Explore the directory structure that venv creates.\n", + "# sysconfig.get_paths() shows where packages and scripts are installed.\n", + "\n", + "paths: dict[str, str] = sysconfig.get_paths()\n", + "\n", + "print(\"Key installation paths:\")\n", + "print(f\" purelib (pure Python packages): {paths['purelib']}\")\n", + "print(f\" scripts (console scripts): {paths['scripts']}\")\n", + "print(f\" include (C headers): {paths['include']}\")\n", + "print()\n", + "\n", + "# The VIRTUAL_ENV environment variable is set by the activation script\n", + "venv_path: str | None = os.environ.get(\"VIRTUAL_ENV\")\n", + "print(f\"VIRTUAL_ENV env var: {venv_path or '(not set -- not activated)'}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 3: Programmatic Environment Creation with `venv.EnvBuilder`\n", + "\n", + "For tooling, CI pipelines, or scripts that need to create environments on the fly,\n", + "`venv.EnvBuilder` provides a Python API instead of shelling out to `python -m venv`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import venv\n", + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "# EnvBuilder accepts the same options as the CLI\n", + "builder = venv.EnvBuilder(\n", + " system_site_packages=False, # Do NOT inherit global packages\n", + " clear=True, # Remove existing env dir first\n", + " with_pip=True, # Bootstrap pip into the new env\n", + " symlinks=True, # Use symlinks instead of copies (faster)\n", + " upgrade_deps=False, # Set True to upgrade pip/setuptools\n", + ")\n", + "\n", + "# Create a temporary environment for demonstration\n", + "tmp_dir: Path = Path(tempfile.mkdtemp()) / \"demo_env\"\n", + "builder.create(str(tmp_dir))\n", + "\n", + "print(f\"Environment created at: {tmp_dir}\")\n", + "print(f\"\\nDirectory contents:\")\n", + "for item in sorted(tmp_dir.iterdir()):\n", + " print(f\" {item.name}/\" if item.is_dir() else f\" {item.name}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Inspect the created environment more deeply\n", + "from pathlib import Path\n", + "import sys\n", + "\n", + "# The pyvenv.cfg file stores the environment's configuration\n", + "cfg_file: Path = tmp_dir / \"pyvenv.cfg\"\n", + "if cfg_file.exists():\n", + " print(\"Contents of pyvenv.cfg:\")\n", + " print(cfg_file.read_text())\n", + "\n", + "# Find the Python executable inside the new environment\n", + "if sys.platform == \"win32\":\n", + " python_path: Path = tmp_dir / \"Scripts\" / \"python.exe\"\n", + "else:\n", + " python_path: Path = tmp_dir / \"bin\" / \"python\"\n", + "\n", + "print(f\"Python executable exists: {python_path.exists()}\")\n", + "print(f\"Python executable path: {python_path}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 4: pip Basics -- Installing and Managing Packages\n", + "\n", + "Once inside a virtual environment, `pip` is the standard tool for installing packages.\n", + "\n", + "### Common Commands\n", + "\n", + "```bash\n", + "# Install a package (latest version)\n", + "pip install requests\n", + "\n", + "# Install a specific version\n", + "pip install requests==2.31.0\n", + "\n", + "# Install with version constraints\n", + "pip install \"requests>=2.28,<3.0\"\n", + "\n", + "# Install from a requirements file\n", + "pip install -r requirements.txt\n", + "\n", + "# List installed packages\n", + "pip list\n", + "\n", + "# Freeze current environment to a requirements file\n", + "pip freeze > requirements.txt\n", + "\n", + "# Uninstall a package\n", + "pip uninstall requests\n", + "```\n", + "\n", + "### `pip freeze` vs `pip list`\n", + "\n", + "| Command | Output | Purpose |\n", + "|---|---|---|\n", + "| `pip list` | Human-readable table | Quick inspection |\n", + "| `pip freeze` | `package==version` lines | Machine-readable, for `requirements.txt` |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# You can invoke pip programmatically via subprocess.\n", + "# IMPORTANT: Never use `import pip` directly -- it is not a public API.\n", + "\n", + "result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"list\", \"--format=json\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "import json\n", + "\n", + "packages: list[dict[str, str]] = json.loads(result.stdout)\n", + "print(f\"Installed packages in this environment: {len(packages)}\")\n", + "print()\n", + "\n", + "# Show first 10 packages\n", + "for pkg in packages[:10]:\n", + " print(f\" {pkg['name']:30s} {pkg['version']}\")\n", + "\n", + "if len(packages) > 10:\n", + " print(f\" ... and {len(packages) - 10} more\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 5: Requirements Files and Pinning\n", + "\n", + "A `requirements.txt` file lists the packages (and their versions) your project needs.\n", + "There are two common strategies:\n", + "\n", + "### 1. Abstract requirements (flexible)\n", + "```\n", + "requests>=2.28\n", + "click~=8.0\n", + "pydantic<3\n", + "```\n", + "Good for **libraries** -- allows downstream users to resolve compatible versions.\n", + "\n", + "### 2. Pinned requirements (exact)\n", + "```\n", + "requests==2.31.0\n", + "click==8.1.7\n", + "pydantic==2.5.3\n", + "```\n", + "Good for **applications** -- ensures identical installs across machines." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class Requirement:\n", + " \"\"\"Represents a single line in a requirements file.\"\"\"\n", + " name: str\n", + " specifier: str # e.g. \">=2.28\", \"==2.31.0\", \"~=8.0\"\n", + "\n", + " def __str__(self) -> str:\n", + " return f\"{self.name}{self.specifier}\"\n", + "\n", + "\n", + "def parse_requirements(text: str) -> list[Requirement]:\n", + " \"\"\"Parse a simple requirements.txt into Requirement objects.\"\"\"\n", + " requirements: list[Requirement] = []\n", + " for line in text.strip().splitlines():\n", + " line = line.strip()\n", + " if not line or line.startswith(\"#\"):\n", + " continue\n", + " # Split at the first version specifier character\n", + " for i, char in enumerate(line):\n", + " if char in (\">\", \"<\", \"=\", \"~\", \"!\"):\n", + " requirements.append(Requirement(line[:i], line[i:]))\n", + " break\n", + " else:\n", + " requirements.append(Requirement(line, \"\"))\n", + " return requirements\n", + "\n", + "\n", + "sample_requirements = \"\"\"\n", + "# Web framework and utilities\n", + "flask>=3.0,<4.0\n", + "requests==2.31.0\n", + "click~=8.0\n", + "pydantic>=2.0\n", + "\"\"\"\n", + "\n", + "reqs: list[Requirement] = parse_requirements(sample_requirements)\n", + "for req in reqs:\n", + " print(f\" Package: {req.name:15s} Specifier: {req.specifier}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 6: Dependency Resolution and Conflicts\n", + "\n", + "When you install a package, pip must resolve its **transitive dependencies** -- the\n", + "packages your dependencies themselves depend on.\n", + "\n", + "### How Conflicts Arise\n", + "\n", + "```\n", + "your-project\n", + " ├── package-A requires shared-lib>=2.0,<3.0\n", + " └── package-B requires shared-lib>=1.0,<2.0\n", + "```\n", + "\n", + "Here, `package-A` and `package-B` need **incompatible** versions of `shared-lib`.\n", + "pip will report a `ResolutionImpossible` error.\n", + "\n", + "### Strategies for Resolving Conflicts\n", + "1. **Upgrade/downgrade** one of the conflicting packages\n", + "2. **Find alternative packages** that have compatible dependency trees\n", + "3. **Use `pip install --dry-run`** to preview what would be installed\n", + "4. **Use `pip check`** to verify the current environment is consistent" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# pip check verifies that all installed packages have compatible dependencies\n", + "result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"check\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(\"pip check output:\")\n", + "if result.stdout.strip():\n", + " print(result.stdout)\n", + "else:\n", + " print(\" No broken requirements found.\")\n", + "\n", + "print(f\"Return code: {result.returncode} (0 = all OK)\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Simulating dependency resolution logic\n", + "from dataclasses import dataclass, field\n", + "\n", + "\n", + "@dataclass\n", + "class VersionRange:\n", + " \"\"\"A simplified version constraint.\"\"\"\n", + " min_version: tuple[int, ...]\n", + " max_version: tuple[int, ...]\n", + "\n", + " def contains(self, version: tuple[int, ...]) -> bool:\n", + " return self.min_version <= version < self.max_version\n", + "\n", + " def overlaps(self, other: \"VersionRange\") -> bool:\n", + " \"\"\"Check if two version ranges have any overlap.\"\"\"\n", + " return self.min_version < other.max_version and other.min_version < self.max_version\n", + "\n", + " def __repr__(self) -> str:\n", + " min_str = \".\".join(str(x) for x in self.min_version)\n", + " max_str = \".\".join(str(x) for x in self.max_version)\n", + " return f\">={min_str},<{max_str}\"\n", + "\n", + "\n", + "# Two packages require different versions of a shared dependency\n", + "constraint_a = VersionRange(min_version=(2, 0, 0), max_version=(3, 0, 0))\n", + "constraint_b = VersionRange(min_version=(1, 0, 0), max_version=(2, 0, 0))\n", + "constraint_c = VersionRange(min_version=(2, 5, 0), max_version=(4, 0, 0))\n", + "\n", + "print(f\"Constraint A: {constraint_a}\")\n", + "print(f\"Constraint B: {constraint_b}\")\n", + "print(f\"Constraint C: {constraint_c}\")\n", + "print()\n", + "print(f\"A overlaps B: {constraint_a.overlaps(constraint_b)} (conflict!)\")\n", + "print(f\"A overlaps C: {constraint_a.overlaps(constraint_c)} (compatible)\")\n", + "print()\n", + "\n", + "# Check if a specific version satisfies both A and C\n", + "test_version: tuple[int, ...] = (2, 7, 0)\n", + "print(f\"Version 2.7.0 satisfies A: {constraint_a.contains(test_version)}\")\n", + "print(f\"Version 2.7.0 satisfies C: {constraint_c.contains(test_version)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 7: Inspecting the Current Environment\n", + "\n", + "Several standard library modules help you introspect the running Python environment.\n", + "This is invaluable for debugging \"works on my machine\" issues." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sys\n", + "import os\n", + "import site\n", + "\n", + "# sys module: interpreter-level information\n", + "print(\"=== sys module ===\")\n", + "print(f\"Python version: {sys.version}\")\n", + "print(f\"Platform: {sys.platform}\")\n", + "print(f\"Executable: {sys.executable}\")\n", + "print(f\"Prefix: {sys.prefix}\")\n", + "print(f\"Base prefix: {sys.base_prefix}\")\n", + "print(f\"In virtual env: {sys.prefix != sys.base_prefix}\")\n", + "print()\n", + "\n", + "# site module: package installation locations\n", + "print(\"=== site module ===\")\n", + "user_site: str | None = site.getusersitepackages()\n", + "global_sites: list[str] = site.getsitepackages()\n", + "print(f\"User site-packages: {user_site}\")\n", + "print(f\"Global site-packages: {global_sites}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "# sys.path determines where Python looks for importable modules.\n", + "# In a virtual environment, the env's site-packages is at the front.\n", + "\n", + "print(\"Module search path (sys.path):\")\n", + "for i, path in enumerate(sys.path):\n", + " label = \"\"\n", + " if \"site-packages\" in path:\n", + " label = \" <-- site-packages\"\n", + " elif path == \"\":\n", + " label = \" <-- current directory\"\n", + " print(f\" [{i:2d}] {path}{label}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 8: Best Practices\n", + "\n", + "### One environment per project\n", + "Never share a virtual environment between unrelated projects. Each project should\n", + "have its own `.venv` directory at the project root.\n", + "\n", + "### Always `.gitignore` the environment directory\n", + "```gitignore\n", + "# Virtual environments\n", + ".venv/\n", + "venv/\n", + "env/\n", + "```\n", + "\n", + "### Pin your dependencies\n", + "For applications, use `pip freeze > requirements.txt` or better yet, a lock file\n", + "from a tool like `pip-tools`, `Poetry`, or `PDM`.\n", + "\n", + "### Use `python -m pip` instead of bare `pip`\n", + "This guarantees you are using pip from the correct Python installation:\n", + "```bash\n", + "python -m pip install requests # Always correct\n", + "pip install requests # May point to wrong Python\n", + "```\n", + "\n", + "### Automate environment setup\n", + "Provide a `Makefile` or script so teammates can get started quickly:\n", + "```makefile\n", + "venv:\n", + "\tpython -m venv .venv\n", + "\t.venv/bin/pip install -r requirements.txt\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import shutil\n", + "\n", + "# Clean up the temporary environment we created earlier\n", + "if tmp_dir.exists():\n", + " shutil.rmtree(tmp_dir)\n", + " print(f\"Cleaned up temporary environment: {tmp_dir}\")\n", + "\n", + "# Demonstrate a helper function for environment setup\n", + "def create_project_env(\n", + " project_dir: Path,\n", + " env_name: str = \".venv\",\n", + " with_pip: bool = True,\n", + " upgrade_deps: bool = True,\n", + ") -> Path:\n", + " \"\"\"Create a virtual environment for a project directory.\n", + "\n", + " Args:\n", + " project_dir: The root directory of the project.\n", + " env_name: Name of the environment directory.\n", + " with_pip: Whether to install pip in the environment.\n", + " upgrade_deps: Whether to upgrade pip and setuptools.\n", + "\n", + " Returns:\n", + " Path to the created environment directory.\n", + " \"\"\"\n", + " env_path: Path = project_dir / env_name\n", + " builder = venv.EnvBuilder(\n", + " system_site_packages=False,\n", + " clear=True,\n", + " with_pip=with_pip,\n", + " symlinks=True,\n", + " upgrade_deps=upgrade_deps,\n", + " )\n", + " builder.create(str(env_path))\n", + " return env_path\n", + "\n", + "\n", + "print(\"\\ncreate_project_env() is ready to use.\")\n", + "print(\"Usage: create_project_env(Path('/path/to/project'))\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Virtual Environments\n", + "- **Purpose**: Isolate project dependencies from the system Python and other projects\n", + "- **`python -m venv .venv`**: Creates a lightweight virtual environment\n", + "- **`sys.prefix != sys.base_prefix`**: Definitive check for active virtual environment\n", + "- **`venv.EnvBuilder`**: Programmatic API for creating environments in scripts and tools\n", + "\n", + "### pip and Dependencies\n", + "- **`pip install`**: Install packages; use version specifiers for control\n", + "- **`pip freeze`**: Export exact versions for reproducibility\n", + "- **`pip check`**: Verify dependency consistency in the current environment\n", + "- **Conflicts**: Arise when transitive dependencies require incompatible versions\n", + "\n", + "### Best Practices\n", + "- One `.venv` per project, always in `.gitignore`\n", + "- Use `python -m pip` to avoid PATH ambiguity\n", + "- Pin dependencies for applications; use flexible specifiers for libraries\n", + "- Automate environment creation with Makefiles or setup scripts" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_19/02_project_configuration.ipynb b/src/chapter_19/02_project_configuration.ipynb new file mode 100644 index 0000000..d57e7ff --- /dev/null +++ b/src/chapter_19/02_project_configuration.ipynb @@ -0,0 +1,687 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 19: Project Configuration with pyproject.toml\n", + "\n", + "This notebook covers the modern standard for configuring Python projects using\n", + "`pyproject.toml` (PEP 621). We explore project metadata, build systems, entry points,\n", + "optional dependencies, project layouts, and runtime metadata inspection.\n", + "\n", + "## Key Concepts\n", + "- **pyproject.toml**: The single configuration file for modern Python projects\n", + "- **PEP 621**: Standardized project metadata in `[project]` table\n", + "- **Build backends**: Pluggable systems that turn source into installable packages\n", + "- **Entry points**: Mechanism for registering CLI tools and plugins" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 1: pyproject.toml -- The Modern Standard\n", + "\n", + "Before `pyproject.toml`, Python projects used a patchwork of configuration files:\n", + "`setup.py`, `setup.cfg`, `MANIFEST.in`, `requirements.txt`, and tool-specific files\n", + "like `mypy.ini`, `pytest.ini`, `.flake8`, etc.\n", + "\n", + "**pyproject.toml** (introduced in PEP 518, refined by PEP 621) consolidates project\n", + "metadata and tool configuration into a single, declarative TOML file.\n", + "\n", + "### Minimal Example\n", + "\n", + "```toml\n", + "[project]\n", + "name = \"my-package\"\n", + "version = \"1.0.0\"\n", + "requires-python = \">=3.10\"\n", + "dependencies = [\n", + " \"requests>=2.28\",\n", + " \"click>=8.0\",\n", + "]\n", + "\n", + "[build-system]\n", + "requires = [\"setuptools>=68.0\"]\n", + "build-backend = \"setuptools.build_meta\"\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Python 3.11+ includes tomllib for reading TOML files\n", + "# For older versions, use the third-party 'tomli' package\n", + "try:\n", + " import tomllib # Python 3.11+\n", + "except ModuleNotFoundError:\n", + " import tomli as tomllib # type: ignore[no-redef]\n", + "\n", + "# Parse a pyproject.toml string to understand its structure\n", + "sample_toml: str = \"\"\"\n", + "[project]\n", + "name = \"example-cli\"\n", + "version = \"2.1.0\"\n", + "description = \"A demonstration CLI tool\"\n", + "requires-python = \">=3.10\"\n", + "license = \"MIT\"\n", + "authors = [\n", + " {name = \"Jane Developer\", email = \"jane@example.com\"},\n", + "]\n", + "dependencies = [\n", + " \"click>=8.0\",\n", + " \"rich>=13.0\",\n", + "]\n", + "\n", + "[project.optional-dependencies]\n", + "dev = [\"pytest>=7.0\", \"mypy>=1.0\"]\n", + "docs = [\"sphinx>=7.0\", \"furo\"]\n", + "\n", + "[project.scripts]\n", + "example-cli = \"example_cli.main:app\"\n", + "\n", + "[build-system]\n", + "requires = [\"setuptools>=68.0\"]\n", + "build-backend = \"setuptools.build_meta\"\n", + "\"\"\"\n", + "\n", + "config: dict = tomllib.loads(sample_toml)\n", + "\n", + "print(\"Parsed [project] table:\")\n", + "project = config[\"project\"]\n", + "for key, value in project.items():\n", + " print(f\" {key}: {value}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 2: The `[project]` Table (PEP 621)\n", + "\n", + "The `[project]` table contains all the metadata about your package:\n", + "\n", + "| Field | Required? | Description |\n", + "|---|---|---|\n", + "| `name` | Yes | Package name (used on PyPI) |\n", + "| `version` | Yes* | Semantic version string |\n", + "| `description` | No | One-line summary |\n", + "| `requires-python` | Recommended | Minimum Python version |\n", + "| `dependencies` | Recommended | Runtime dependencies |\n", + "| `license` | Recommended | SPDX license identifier |\n", + "| `authors` | No | List of `{name, email}` tables |\n", + "| `readme` | No | Path to README file |\n", + "| `classifiers` | No | PyPI trove classifiers |\n", + "\n", + "\\* Can be dynamic if set via `[project.dynamic]`" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass, field\n", + "\n", + "\n", + "@dataclass\n", + "class ProjectMetadata:\n", + " \"\"\"Represents the [project] table of pyproject.toml.\"\"\"\n", + " name: str\n", + " version: str\n", + " description: str = \"\"\n", + " requires_python: str = \">=3.10\"\n", + " dependencies: list[str] = field(default_factory=list)\n", + " optional_dependencies: dict[str, list[str]] = field(default_factory=dict)\n", + " scripts: dict[str, str] = field(default_factory=dict)\n", + "\n", + " @classmethod\n", + " def from_toml(cls, config: dict) -> \"ProjectMetadata\":\n", + " \"\"\"Parse a [project] table dict into a ProjectMetadata instance.\"\"\"\n", + " project: dict = config[\"project\"]\n", + " return cls(\n", + " name=project[\"name\"],\n", + " version=project[\"version\"],\n", + " description=project.get(\"description\", \"\"),\n", + " requires_python=project.get(\"requires-python\", \">=3.10\"),\n", + " dependencies=project.get(\"dependencies\", []),\n", + " optional_dependencies=project.get(\"optional-dependencies\", {}),\n", + " scripts=project.get(\"scripts\", {}),\n", + " )\n", + "\n", + "\n", + "metadata = ProjectMetadata.from_toml(config)\n", + "print(f\"Package: {metadata.name}\")\n", + "print(f\"Version: {metadata.version}\")\n", + "print(f\"Description: {metadata.description}\")\n", + "print(f\"Requires: Python {metadata.requires_python}\")\n", + "print(f\"Dependencies: {metadata.dependencies}\")\n", + "print(f\"Scripts: {metadata.scripts}\")\n", + "print(f\"Extras: {list(metadata.optional_dependencies.keys())}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 3: The `[build-system]` Table\n", + "\n", + "The `[build-system]` table tells tools like `pip` and `build` how to build your package.\n", + "\n", + "| Backend | Package | Strengths |\n", + "|---|---|---|\n", + "| **setuptools** | `setuptools>=68.0` | Mature, widely used, highly configurable |\n", + "| **hatchling** | `hatchling` | Modern, fast, good defaults |\n", + "| **flit-core** | `flit-core>=3.4` | Minimal, pure-Python only |\n", + "| **poetry-core** | `poetry-core>=1.0` | Used with Poetry workflow |\n", + "| **maturin** | `maturin>=1.0` | For Rust+Python (PyO3) extensions |\n", + "\n", + "### Example configurations for each backend\n", + "\n", + "```toml\n", + "# setuptools\n", + "[build-system]\n", + "requires = [\"setuptools>=68.0\", \"wheel\"]\n", + "build-backend = \"setuptools.build_meta\"\n", + "\n", + "# hatchling\n", + "[build-system]\n", + "requires = [\"hatchling\"]\n", + "build-backend = \"hatchling.build\"\n", + "\n", + "# flit\n", + "[build-system]\n", + "requires = [\"flit_core>=3.4\"]\n", + "build-backend = \"flit_core.api\"\n", + "\n", + "# poetry\n", + "[build-system]\n", + "requires = [\"poetry-core>=1.0.0\"]\n", + "build-backend = \"poetry.core.masonry.api\"\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Comparing build system configurations\n", + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class BuildSystem:\n", + " \"\"\"Represents the [build-system] table.\"\"\"\n", + " requires: list[str]\n", + " build_backend: str\n", + "\n", + " @property\n", + " def backend_name(self) -> str:\n", + " \"\"\"Extract the human-readable backend name.\"\"\"\n", + " mapping: dict[str, str] = {\n", + " \"setuptools.build_meta\": \"setuptools\",\n", + " \"hatchling.build\": \"hatchling\",\n", + " \"flit_core.api\": \"flit\",\n", + " \"poetry.core.masonry.api\": \"poetry\",\n", + " }\n", + " return mapping.get(self.build_backend, self.build_backend)\n", + "\n", + "\n", + "build_configs: list[BuildSystem] = [\n", + " BuildSystem([\"setuptools>=68.0\", \"wheel\"], \"setuptools.build_meta\"),\n", + " BuildSystem([\"hatchling\"], \"hatchling.build\"),\n", + " BuildSystem([\"flit_core>=3.4\"], \"flit_core.api\"),\n", + " BuildSystem([\"poetry-core>=1.0.0\"], \"poetry.core.masonry.api\"),\n", + "]\n", + "\n", + "print(f\"{'Backend':<15} {'Build Requires':<35} {'build-backend'}\")\n", + "print(\"-\" * 75)\n", + "for bs in build_configs:\n", + " requires_str = \", \".join(bs.requires)\n", + " print(f\"{bs.backend_name:<15} {requires_str:<35} {bs.build_backend}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 4: Entry Points -- Console Scripts\n", + "\n", + "Entry points let you register executable commands that get installed into the\n", + "environment's `bin/` (or `Scripts/`) directory.\n", + "\n", + "```toml\n", + "[project.scripts]\n", + "my-tool = \"my_package.cli:main\"\n", + "```\n", + "\n", + "This means: when the user types `my-tool` in their terminal, Python calls\n", + "`my_package.cli.main()`. The format is `module_path:callable`.\n", + "\n", + "### GUI Scripts\n", + "```toml\n", + "[project.gui-scripts]\n", + "my-gui = \"my_package.gui:launch\"\n", + "```\n", + "On Windows, `gui-scripts` suppresses the console window.\n", + "\n", + "### Plugin Entry Points\n", + "```toml\n", + "[project.entry-points.\"myapp.plugins\"]\n", + "csv-handler = \"myapp.plugins.csv:CsvPlugin\"\n", + "json-handler = \"myapp.plugins.json:JsonPlugin\"\n", + "```\n", + "This is the standard mechanism for **plugin discovery** in Python." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class EntryPoint:\n", + " \"\"\"Represents a console_scripts entry point.\"\"\"\n", + " command: str\n", + " module: str\n", + " callable: str\n", + "\n", + " @classmethod\n", + " def parse(cls, command: str, reference: str) -> \"EntryPoint\":\n", + " \"\"\"Parse an entry point from 'module.path:callable' format.\"\"\"\n", + " module, func = reference.split(\":\")\n", + " return cls(command=command, module=module, callable=func)\n", + "\n", + " def to_import_statement(self) -> str:\n", + " return f\"from {self.module} import {self.callable}\"\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"{self.command} -> {self.module}:{self.callable}\"\n", + "\n", + "\n", + "# Parse the entry point from our sample config\n", + "scripts: dict[str, str] = config[\"project\"][\"scripts\"]\n", + "\n", + "for cmd, ref in scripts.items():\n", + " ep = EntryPoint.parse(cmd, ref)\n", + " print(f\"Command: {ep.command}\")\n", + " print(f\"Module: {ep.module}\")\n", + " print(f\"Callable: {ep.callable}\")\n", + " print(f\"Import: {ep.to_import_statement()}\")\n", + " print()\n", + " print(f\"When a user runs `{ep.command}`, Python executes:\")\n", + " print(f\" {ep.to_import_statement()}\")\n", + " print(f\" {ep.callable}()\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 5: Optional Dependencies and Extras\n", + "\n", + "Optional dependencies let users install additional features only when needed:\n", + "\n", + "```toml\n", + "[project.optional-dependencies]\n", + "dev = [\"pytest>=7.0\", \"mypy>=1.0\", \"ruff>=0.1\"]\n", + "docs = [\"sphinx>=7.0\", \"furo\"]\n", + "postgres = [\"psycopg2>=2.9\"]\n", + "```\n", + "\n", + "Users install extras with square bracket syntax:\n", + "```bash\n", + "pip install my-package[dev] # Install dev extras\n", + "pip install my-package[dev,docs] # Install multiple extras\n", + "pip install -e \".[dev]\" # Editable install with extras\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Inspecting optional dependencies from our parsed config\n", + "optional_deps: dict[str, list[str]] = config[\"project\"].get(\n", + " \"optional-dependencies\", {}\n", + ")\n", + "\n", + "print(\"Available extras:\")\n", + "for extra_name, deps in optional_deps.items():\n", + " print(f\"\\n [{extra_name}]\")\n", + " for dep in deps:\n", + " print(f\" - {dep}\")\n", + "\n", + "# Show the install commands\n", + "package_name: str = config[\"project\"][\"name\"]\n", + "print(f\"\\nInstall commands:\")\n", + "for extra_name in optional_deps:\n", + " print(f\" pip install {package_name}[{extra_name}]\")\n", + "\n", + "all_extras: str = \",\".join(optional_deps.keys())\n", + "print(f\" pip install {package_name}[{all_extras}] # all extras\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 6: src-Layout vs Flat Layout\n", + "\n", + "Two common ways to organize your project source code:\n", + "\n", + "### Flat Layout\n", + "```\n", + "my-project/\n", + " my_package/\n", + " __init__.py\n", + " core.py\n", + " tests/\n", + " test_core.py\n", + " pyproject.toml\n", + "```\n", + "\n", + "### src-Layout (Recommended)\n", + "```\n", + "my-project/\n", + " src/\n", + " my_package/\n", + " __init__.py\n", + " core.py\n", + " tests/\n", + " test_core.py\n", + " pyproject.toml\n", + "```\n", + "\n", + "| Aspect | Flat Layout | src-Layout |\n", + "|---|---|---|\n", + "| **Import safety** | Can accidentally import uninstalled code | Forces you to install first |\n", + "| **Test isolation** | Tests may use local code, not installed | Tests always use installed package |\n", + "| **Simplicity** | Fewer directories | One extra `src/` level |\n", + "| **setuptools config** | Automatic discovery | Needs `[tool.setuptools.packages.find]` |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from pathlib import Path\n", + "\n", + "\n", + "def display_layout(name: str, structure: dict) -> None:\n", + " \"\"\"Display a project directory structure.\"\"\"\n", + " print(f\"\\n{name}:\")\n", + " _print_tree(structure, prefix=\" \")\n", + "\n", + "\n", + "def _print_tree(tree: dict, prefix: str = \"\") -> None:\n", + " \"\"\"Recursively print a directory tree.\"\"\"\n", + " items = list(tree.items())\n", + " for i, (name, children) in enumerate(items):\n", + " is_last: bool = i == len(items) - 1\n", + " connector: str = \"└── \" if is_last else \"├── \"\n", + " print(f\"{prefix}{connector}{name}\")\n", + " if isinstance(children, dict):\n", + " extension = \" \" if is_last else \"│ \"\n", + " _print_tree(children, prefix + extension)\n", + "\n", + "\n", + "# Flat layout\n", + "flat: dict = {\n", + " \"my_package/\": {\n", + " \"__init__.py\": None,\n", + " \"core.py\": None,\n", + " \"cli.py\": None,\n", + " },\n", + " \"tests/\": {\n", + " \"test_core.py\": None,\n", + " \"test_cli.py\": None,\n", + " },\n", + " \"pyproject.toml\": None,\n", + " \"README.md\": None,\n", + "}\n", + "\n", + "# src-layout\n", + "src_layout: dict = {\n", + " \"src/\": {\n", + " \"my_package/\": {\n", + " \"__init__.py\": None,\n", + " \"core.py\": None,\n", + " \"cli.py\": None,\n", + " },\n", + " },\n", + " \"tests/\": {\n", + " \"test_core.py\": None,\n", + " \"test_cli.py\": None,\n", + " },\n", + " \"pyproject.toml\": None,\n", + " \"README.md\": None,\n", + "}\n", + "\n", + "display_layout(\"Flat Layout\", flat)\n", + "display_layout(\"src-Layout\", src_layout)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# setuptools configuration for each layout\n", + "\n", + "flat_config: str = \"\"\"\\\n", + "# Flat layout -- setuptools auto-discovers packages\n", + "[tool.setuptools.packages.find]\n", + "exclude = [\"tests*\"]\n", + "\"\"\"\n", + "\n", + "src_config: str = \"\"\"\\\n", + "# src-layout -- tell setuptools where to find packages\n", + "[tool.setuptools.packages.find]\n", + "where = [\"src\"]\n", + "\"\"\"\n", + "\n", + "print(\"setuptools configuration:\")\n", + "print(flat_config)\n", + "print(src_config)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 7: `importlib.metadata` -- Reading Package Metadata at Runtime\n", + "\n", + "The `importlib.metadata` module (Python 3.8+) lets you inspect installed package\n", + "metadata at runtime -- version numbers, entry points, dependencies, and more.\n", + "This is the standard way to implement `--version` flags in CLI tools." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from importlib.metadata import (\n", + " metadata,\n", + " version,\n", + " requires,\n", + " packages_distributions,\n", + ")\n", + "\n", + "# Get the version of an installed package\n", + "pip_version: str = version(\"pip\")\n", + "print(f\"pip version: {pip_version}\")\n", + "\n", + "# Get full metadata for a package\n", + "pip_meta = metadata(\"pip\")\n", + "print(f\"\\nPackage metadata for 'pip':\")\n", + "print(f\" Name: {pip_meta['Name']}\")\n", + "print(f\" Version: {pip_meta['Version']}\")\n", + "print(f\" Summary: {pip_meta['Summary']}\")\n", + "print(f\" Home-page: {pip_meta.get('Home-page', 'N/A')}\")\n", + "print(f\" License: {pip_meta.get('License', 'N/A')}\")\n", + "\n", + "# Get the dependencies of a package\n", + "pip_deps: list[str] | None = requires(\"pip\")\n", + "if pip_deps:\n", + " print(f\"\\nDependencies ({len(pip_deps)} total):\")\n", + " for dep in pip_deps[:5]:\n", + " print(f\" - {dep}\")\n", + " if len(pip_deps) > 5:\n", + " print(f\" ... and {len(pip_deps) - 5} more\")\n", + "else:\n", + " print(\"\\nNo dependencies listed.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from importlib.metadata import entry_points, version, PackageNotFoundError\n", + "\n", + "# Discover console_scripts entry points installed in this environment\n", + "# entry_points() returns a dict-like object keyed by group name\n", + "console_eps = entry_points(group=\"console_scripts\")\n", + "\n", + "print(f\"Console scripts in this environment: {len(list(console_eps))}\")\n", + "print()\n", + "\n", + "for ep in list(console_eps)[:8]:\n", + " print(f\" {ep.name:25s} -> {ep.value}\")\n", + "\n", + "if len(list(console_eps)) > 8:\n", + " print(f\" ... and {len(list(console_eps)) - 8} more\")\n", + "\n", + "# Practical example: getting your own package version\n", + "def get_version(package_name: str) -> str:\n", + " \"\"\"Get the installed version of a package, or 'unknown'.\"\"\"\n", + " try:\n", + " return version(package_name)\n", + " except PackageNotFoundError:\n", + " return \"unknown (not installed)\"\n", + "\n", + "\n", + "print(f\"\\npip version: {get_version('pip')}\")\n", + "print(f\"setuptools version: {get_version('setuptools')}\")\n", + "print(f\"nonexistent: {get_version('nonexistent-package')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 8: `sysconfig` -- Understanding Installation Paths\n", + "\n", + "The `sysconfig` module reveals where Python installs files. This is essential for\n", + "understanding how packages are laid out on disk and for debugging import issues." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sysconfig\n", + "\n", + "# Get the installation scheme name\n", + "scheme: str = sysconfig.get_default_scheme()\n", + "print(f\"Default installation scheme: {scheme}\")\n", + "print()\n", + "\n", + "# All known paths in the current scheme\n", + "paths: dict[str, str] = sysconfig.get_paths()\n", + "print(\"Installation paths:\")\n", + "for name, path in sorted(paths.items()):\n", + " print(f\" {name:15s} -> {path}\")\n", + "\n", + "print(f\"\\nPlatform: {sysconfig.get_platform()}\")\n", + "print(f\"Python path: {sysconfig.get_python_lib()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sysconfig\n", + "\n", + "# sysconfig also exposes build-time configuration variables\n", + "# These are the values from when the Python interpreter was compiled\n", + "\n", + "interesting_vars: list[str] = [\n", + " \"prefix\",\n", + " \"exec_prefix\",\n", + " \"py_version\",\n", + " \"py_version_short\",\n", + " \"SOABI\", # Shared object ABI tag\n", + " \"EXT_SUFFIX\", # Extension module suffix (e.g. .cpython-312-darwin.so)\n", + "]\n", + "\n", + "print(\"Selected configuration variables:\")\n", + "for var in interesting_vars:\n", + " value = sysconfig.get_config_var(var)\n", + " print(f\" {var:20s} = {value}\")\n", + "\n", + "# Available installation schemes\n", + "print(f\"\\nAvailable schemes: {sysconfig.get_scheme_names()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### pyproject.toml\n", + "- **`[project]`**: Standardized metadata (PEP 621) -- name, version, dependencies\n", + "- **`[build-system]`**: Declares the build backend (setuptools, hatchling, flit, poetry)\n", + "- **`[project.scripts]`**: Register CLI commands via `module:callable` references\n", + "- **`[project.optional-dependencies]`**: Define extras installable with `pip install pkg[extra]`\n", + "\n", + "### Project Layout\n", + "- **src-layout** is recommended: forces installation before import, better test isolation\n", + "- **Flat layout** is simpler but can mask import errors during development\n", + "\n", + "### Runtime Introspection\n", + "- **`importlib.metadata`**: Read installed package versions, metadata, and entry points\n", + "- **`sysconfig`**: Inspect installation paths, platform info, and build configuration\n", + "- Use `importlib.metadata.version()` for `--version` flags in CLI tools" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_19/03_building_and_publishing.ipynb b/src/chapter_19/03_building_and_publishing.ipynb new file mode 100644 index 0000000..6328e7a --- /dev/null +++ b/src/chapter_19/03_building_and_publishing.ipynb @@ -0,0 +1,865 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 19: Building and Publishing Python Packages\n", + "\n", + "This notebook covers the process of turning your Python project into distributable\n", + "packages. We explore source distributions, wheels, semantic versioning, version\n", + "constraints, package data, and the publishing workflow.\n", + "\n", + "## Key Concepts\n", + "- **sdist**: Source distribution -- contains raw source code\n", + "- **wheel**: Pre-built distribution -- faster to install, no build step\n", + "- **Semantic versioning**: MAJOR.MINOR.PATCH version scheme\n", + "- **Publishing**: Upload packages to PyPI or private registries" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 1: Source Distributions vs Wheels\n", + "\n", + "Python packages are distributed in two primary formats:\n", + "\n", + "### Source Distribution (sdist)\n", + "- A `.tar.gz` archive of your project source code\n", + "- Must be **built** on the target machine during installation\n", + "- Includes `pyproject.toml`, source files, and any build scripts\n", + "- Format: `package-1.0.0.tar.gz`\n", + "\n", + "### Wheel (bdist_wheel)\n", + "- A `.whl` file (actually a ZIP archive with a special naming convention)\n", + "- **Pre-built** -- no compilation needed during installation\n", + "- Much faster to install than sdists\n", + "- Format: `package-1.0.0-py3-none-any.whl`\n", + "\n", + "| Aspect | sdist | wheel |\n", + "|---|---|---|\n", + "| **File extension** | `.tar.gz` | `.whl` |\n", + "| **Build required?** | Yes, on install | No, pre-built |\n", + "| **Install speed** | Slower | Faster |\n", + "| **C extensions** | Compiled on install | Pre-compiled for target platform |\n", + "| **Use case** | Fallback, source archive | Primary distribution format |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class WheelFilename:\n", + " \"\"\"Parse and represent a wheel filename per PEP 427.\n", + "\n", + " Format: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl\n", + " \"\"\"\n", + " name: str\n", + " version: str\n", + " python_tag: str # e.g. 'py3', 'cp312'\n", + " abi_tag: str # e.g. 'none', 'cp312'\n", + " platform_tag: str # e.g. 'any', 'manylinux_2_17_x86_64', 'macosx_14_0_arm64'\n", + "\n", + " @classmethod\n", + " def parse(cls, filename: str) -> \"WheelFilename\":\n", + " \"\"\"Parse a wheel filename into its components.\"\"\"\n", + " base = filename.removesuffix(\".whl\")\n", + " parts = base.split(\"-\")\n", + " # Handle optional build tag\n", + " if len(parts) == 6:\n", + " name, version, _build, python, abi, platform = parts\n", + " else:\n", + " name, version, python, abi, platform = parts\n", + " return cls(name, version, python, abi, platform)\n", + "\n", + " @property\n", + " def is_pure_python(self) -> bool:\n", + " \"\"\"Pure Python wheels work on any platform.\"\"\"\n", + " return self.abi_tag == \"none\" and self.platform_tag == \"any\"\n", + "\n", + " @property\n", + " def is_platform_specific(self) -> bool:\n", + " \"\"\"Platform-specific wheels contain compiled extensions.\"\"\"\n", + " return self.platform_tag != \"any\"\n", + "\n", + "\n", + "# Parse some real-world wheel filenames\n", + "examples: list[str] = [\n", + " \"requests-2.31.0-py3-none-any.whl\",\n", + " \"numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\n", + " \"pydantic_core-2.14.6-cp312-cp312-macosx_11_0_arm64.whl\",\n", + "]\n", + "\n", + "for filename in examples:\n", + " whl = WheelFilename.parse(filename)\n", + " print(f\"File: {filename}\")\n", + " print(f\" Name: {whl.name}\")\n", + " print(f\" Version: {whl.version}\")\n", + " print(f\" Python: {whl.python_tag}\")\n", + " print(f\" ABI: {whl.abi_tag}\")\n", + " print(f\" Platform: {whl.platform_tag}\")\n", + " print(f\" Pure: {whl.is_pure_python}\")\n", + " print()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 2: Building Packages with `python -m build`\n", + "\n", + "The `build` package (PyPA standard) creates both sdist and wheel from your project:\n", + "\n", + "```bash\n", + "# Install the build tool\n", + "pip install build\n", + "\n", + "# Build both sdist and wheel (output goes to dist/)\n", + "python -m build\n", + "\n", + "# Build only a wheel\n", + "python -m build --wheel\n", + "\n", + "# Build only a source distribution\n", + "python -m build --sdist\n", + "```\n", + "\n", + "After building, your `dist/` directory looks like:\n", + "```\n", + "dist/\n", + " my_package-1.0.0.tar.gz # sdist\n", + " my_package-1.0.0-py3-none-any.whl # wheel\n", + "```\n", + "\n", + "The `build` tool reads `[build-system]` from `pyproject.toml` and delegates to\n", + "the declared backend (setuptools, hatchling, etc.)." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from pathlib import Path\n", + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class BuildArtifact:\n", + " \"\"\"Represents a built distribution file.\"\"\"\n", + " path: Path\n", + " format: str # 'sdist' or 'wheel'\n", + "\n", + " @property\n", + " def size_kb(self) -> float:\n", + " if self.path.exists():\n", + " return self.path.stat().st_size / 1024\n", + " return 0.0\n", + "\n", + "\n", + "def classify_artifact(filename: str) -> str:\n", + " \"\"\"Determine if a dist file is an sdist or wheel.\"\"\"\n", + " if filename.endswith(\".whl\"):\n", + " return \"wheel\"\n", + " elif filename.endswith(\".tar.gz\"):\n", + " return \"sdist\"\n", + " elif filename.endswith(\".zip\"):\n", + " return \"sdist (zip)\"\n", + " return \"unknown\"\n", + "\n", + "\n", + "# Demonstrate classification\n", + "sample_files: list[str] = [\n", + " \"my_package-1.0.0.tar.gz\",\n", + " \"my_package-1.0.0-py3-none-any.whl\",\n", + " \"legacy_pkg-0.5.0.zip\",\n", + "]\n", + "\n", + "print(f\"{'Filename':<45} {'Format'}\")\n", + "print(\"-\" * 60)\n", + "for f in sample_files:\n", + " print(f\"{f:<45} {classify_artifact(f)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 3: Semantic Versioning (SemVer)\n", + "\n", + "The Python ecosystem widely follows **Semantic Versioning**:\n", + "\n", + "```\n", + "MAJOR.MINOR.PATCH\n", + " │ │ └── Bug fixes, no API changes\n", + " │ └──────── New features, backwards compatible\n", + " └────────────── Breaking changes\n", + "```\n", + "\n", + "### Version Progression Examples\n", + "- `1.0.0` -> `1.0.1`: Bug fix\n", + "- `1.0.1` -> `1.1.0`: New feature added\n", + "- `1.1.0` -> `2.0.0`: Breaking API change\n", + "\n", + "### Pre-release and Build Metadata\n", + "- `1.0.0a1` -- Alpha release\n", + "- `1.0.0b2` -- Beta release\n", + "- `1.0.0rc1` -- Release candidate\n", + "- `1.0.0.post1` -- Post-release (documentation fix, etc.)\n", + "- `1.0.0.dev3` -- Development release" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "from functools import total_ordering\n", + "\n", + "\n", + "@total_ordering\n", + "@dataclass(frozen=True)\n", + "class SemanticVersion:\n", + " \"\"\"A simplified semantic version implementation.\"\"\"\n", + " major: int\n", + " minor: int\n", + " patch: int\n", + " pre_release: str = \"\" # e.g. 'a1', 'b2', 'rc1'\n", + "\n", + " @classmethod\n", + " def parse(cls, version_str: str) -> \"SemanticVersion\":\n", + " \"\"\"Parse a version string like '1.2.3' or '2.0.0rc1'.\"\"\"\n", + " pre = \"\"\n", + " # Extract pre-release suffix\n", + " for marker in (\"rc\", \"b\", \"a\", \"dev\", \"post\"):\n", + " if marker in version_str:\n", + " idx = version_str.index(marker)\n", + " pre = version_str[idx:]\n", + " version_str = version_str[:idx].rstrip(\".\")\n", + " break\n", + " parts = version_str.split(\".\")\n", + " return cls(\n", + " major=int(parts[0]),\n", + " minor=int(parts[1]) if len(parts) > 1 else 0,\n", + " patch=int(parts[2]) if len(parts) > 2 else 0,\n", + " pre_release=pre,\n", + " )\n", + "\n", + " def __str__(self) -> str:\n", + " base = f\"{self.major}.{self.minor}.{self.patch}\"\n", + " return f\"{base}{self.pre_release}\" if self.pre_release else base\n", + "\n", + " def __eq__(self, other: object) -> bool:\n", + " if not isinstance(other, SemanticVersion):\n", + " return NotImplemented\n", + " return (self.major, self.minor, self.patch, self.pre_release) == (\n", + " other.major, other.minor, other.patch, other.pre_release\n", + " )\n", + "\n", + " def __lt__(self, other: \"SemanticVersion\") -> bool:\n", + " if not isinstance(other, SemanticVersion):\n", + " return NotImplemented\n", + " # Pre-release versions sort before their release\n", + " self_tuple = (self.major, self.minor, self.patch, self.pre_release == \"\", self.pre_release)\n", + " other_tuple = (other.major, other.minor, other.patch, other.pre_release == \"\", other.pre_release)\n", + " return self_tuple < other_tuple\n", + "\n", + " def bump_major(self) -> \"SemanticVersion\":\n", + " return SemanticVersion(self.major + 1, 0, 0)\n", + "\n", + " def bump_minor(self) -> \"SemanticVersion\":\n", + " return SemanticVersion(self.major, self.minor + 1, 0)\n", + "\n", + " def bump_patch(self) -> \"SemanticVersion\":\n", + " return SemanticVersion(self.major, self.minor, self.patch + 1)\n", + "\n", + "\n", + "# Parse and compare versions\n", + "versions: list[SemanticVersion] = [\n", + " SemanticVersion.parse(\"2.0.0rc1\"),\n", + " SemanticVersion.parse(\"1.0.0\"),\n", + " SemanticVersion.parse(\"2.0.0\"),\n", + " SemanticVersion.parse(\"1.1.0\"),\n", + " SemanticVersion.parse(\"1.0.1\"),\n", + " SemanticVersion.parse(\"2.0.0a1\"),\n", + "]\n", + "\n", + "print(\"Versions sorted by precedence:\")\n", + "for v in sorted(versions):\n", + " print(f\" {v}\")\n", + "\n", + "# Demonstrate version bumping\n", + "current = SemanticVersion.parse(\"1.4.2\")\n", + "print(f\"\\nCurrent version: {current}\")\n", + "print(f\"After patch bump: {current.bump_patch()}\")\n", + "print(f\"After minor bump: {current.bump_minor()}\")\n", + "print(f\"After major bump: {current.bump_major()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 4: Version Constraints and Specifiers\n", + "\n", + "When declaring dependencies, you use **version specifiers** to control which\n", + "versions are acceptable:\n", + "\n", + "| Specifier | Meaning | Example | Matches |\n", + "|---|---|---|---|\n", + "| `==` | Exact match | `==1.0.0` | Only 1.0.0 |\n", + "| `>=` | Minimum | `>=1.0` | 1.0, 1.1, 2.0, ... |\n", + "| `<=` | Maximum | `<=2.0` | 0.1, 1.5, 2.0 |\n", + "| `!=` | Exclude | `!=1.5.0` | Any except 1.5.0 |\n", + "| `~=` | Compatible release | `~=1.4` | >=1.4, <2.0 |\n", + "| `~=` | Compatible release | `~=1.4.2` | >=1.4.2, <1.5.0 |\n", + "\n", + "### The Compatible Release Operator (`~=`)\n", + "\n", + "The `~=` operator is particularly useful. It means \"compatible with this version\":\n", + "- `~=1.4` is equivalent to `>=1.4, <2.0`\n", + "- `~=1.4.2` is equivalent to `>=1.4.2, <1.5.0`\n", + "\n", + "It drops the last component and increments the previous one for the upper bound." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "from typing import Callable\n", + "\n", + "\n", + "@dataclass\n", + "class VersionSpecifier:\n", + " \"\"\"Represents a version constraint like '>=1.0,<2.0'.\"\"\"\n", + " raw: str\n", + "\n", + " def explain(self) -> str:\n", + " \"\"\"Provide a human-readable explanation of the specifier.\"\"\"\n", + " explanations: dict[str, str] = {\n", + " \"==\": \"exactly\",\n", + " \">=\": \"at least\",\n", + " \"<=\": \"at most\",\n", + " \">\": \"greater than\",\n", + " \"<\": \"less than\",\n", + " \"!=\": \"anything except\",\n", + " }\n", + " # Handle compatible release specially\n", + " if self.raw.startswith(\"~=\"):\n", + " version = self.raw[2:]\n", + " parts = version.split(\".\")\n", + " # Increment second-to-last, drop last\n", + " upper_parts = parts[:-1]\n", + " upper_parts[-1] = str(int(upper_parts[-1]) + 1)\n", + " upper = \".\".join(upper_parts)\n", + " return f\">={version}, <{upper}\"\n", + "\n", + " # Handle compound specifiers\n", + " parts_list: list[str] = []\n", + " for part in self.raw.split(\",\"):\n", + " part = part.strip()\n", + " for op, word in sorted(explanations.items(), key=lambda x: -len(x[0])):\n", + " if part.startswith(op):\n", + " ver = part[len(op):]\n", + " parts_list.append(f\"{word} {ver}\")\n", + " break\n", + " return \", \".join(parts_list)\n", + "\n", + "\n", + "# Demonstrate version specifiers\n", + "specifiers: list[str] = [\n", + " \"==1.0.0\",\n", + " \">=1.0,<2.0\",\n", + " \"~=1.4\",\n", + " \"~=1.4.2\",\n", + " \">=2.28,!=2.29.0\",\n", + " \">=3.10\",\n", + "]\n", + "\n", + "print(f\"{'Specifier':<25} {'Equivalent / Explanation'}\")\n", + "print(\"-\" * 60)\n", + "for spec in specifiers:\n", + " vs = VersionSpecifier(spec)\n", + " print(f\"{spec:<25} {vs.explain()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Using the packaging library (standard tool for version handling)\n", + "# This is what pip uses internally\n", + "try:\n", + " from packaging.version import Version\n", + " from packaging.specifiers import SpecifierSet\n", + "\n", + " # Create a specifier set\n", + " spec = SpecifierSet(\">=1.0,<2.0,!=1.5.0\")\n", + " print(f\"Specifier: {spec}\")\n", + " print()\n", + "\n", + " # Test versions against the specifier\n", + " test_versions: list[str] = [\"0.9\", \"1.0\", \"1.4.2\", \"1.5.0\", \"1.9.9\", \"2.0.0\"]\n", + " for v_str in test_versions:\n", + " v = Version(v_str)\n", + " matches: bool = v in spec\n", + " status = \"MATCH\" if matches else \"no match\"\n", + " print(f\" {v_str:10s} -> {status}\")\n", + "\n", + " # Compatible release operator\n", + " print(f\"\\n~=1.4.2 expands to: {SpecifierSet('~=1.4.2')}\")\n", + "\n", + "except ImportError:\n", + " print(\"The 'packaging' library is not installed.\")\n", + " print(\"Install it with: pip install packaging\")\n", + " print(\"It provides robust version parsing and comparison.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 5: MANIFEST.in and Package Data\n", + "\n", + "By default, only Python files are included in your package. To include other files\n", + "(templates, config files, data), you need to declare them.\n", + "\n", + "### Modern approach: pyproject.toml\n", + "```toml\n", + "# For setuptools\n", + "[tool.setuptools.package-data]\n", + "my_package = [\"templates/*.html\", \"data/*.json\", \"py.typed\"]\n", + "\n", + "# For hatchling\n", + "[tool.hatch.build.targets.wheel]\n", + "packages = [\"src/my_package\"]\n", + "```\n", + "\n", + "### Legacy approach: MANIFEST.in (for sdist)\n", + "```\n", + "include LICENSE\n", + "include README.md\n", + "recursive-include my_package/templates *.html\n", + "recursive-include my_package/data *.json\n", + "prune tests\n", + "```\n", + "\n", + "### Accessing package data at runtime\n", + "Use `importlib.resources` (Python 3.9+) instead of `__file__` hacks:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# importlib.resources -- the modern way to access package data\n", + "import importlib.resources as resources\n", + "import json\n", + "\n", + "# Demonstrate the API (using the email package as a real example)\n", + "# In your own package, you would do:\n", + "# files = resources.files(\"my_package\")\n", + "# template = (files / \"templates\" / \"base.html\").read_text()\n", + "\n", + "# Show the API pattern\n", + "print(\"importlib.resources usage patterns:\")\n", + "print()\n", + "print(\" # Access a file in your package\")\n", + "print(' files = importlib.resources.files(\"my_package\")')\n", + "print(' config = (files / \"config.json\").read_text()')\n", + "print()\n", + "print(\" # Access a subdirectory\")\n", + "print(' templates = files / \"templates\"')\n", + "print(' html = (templates / \"base.html\").read_text()')\n", + "print()\n", + "print(\" # Binary data\")\n", + "print(' icon = (files / \"icon.png\").read_bytes()')\n", + "print()\n", + "\n", + "# MANIFEST.in directives\n", + "directives: dict[str, str] = {\n", + " \"include\": \"Include specific files (e.g., include LICENSE)\",\n", + " \"exclude\": \"Exclude specific files\",\n", + " \"recursive-include\": \"Include files matching a pattern in a directory tree\",\n", + " \"recursive-exclude\": \"Exclude files matching a pattern\",\n", + " \"graft\": \"Include an entire directory tree\",\n", + " \"prune\": \"Exclude an entire directory tree\",\n", + "}\n", + "\n", + "print(\"MANIFEST.in directives:\")\n", + "for directive, desc in directives.items():\n", + " print(f\" {directive:<25s} {desc}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 6: The Publishing Workflow\n", + "\n", + "Publishing a package to PyPI follows a consistent workflow:\n", + "\n", + "```\n", + "1. Prepare --> Update version, changelog, metadata\n", + "2. Build --> python -m build (creates sdist + wheel)\n", + "3. Check --> twine check dist/* (validate metadata)\n", + "4. Test --> Upload to TestPyPI first\n", + "5. Publish --> twine upload dist/* (upload to PyPI)\n", + "```\n", + "\n", + "### Tools\n", + "- **`build`**: Creates sdist and wheel artifacts\n", + "- **`twine`**: Securely uploads packages to PyPI\n", + "- **`twine check`**: Validates package metadata before upload\n", + "\n", + "### TestPyPI\n", + "Always test your publishing workflow against TestPyPI first:\n", + "```bash\n", + "# Upload to TestPyPI\n", + "twine upload --repository testpypi dist/*\n", + "\n", + "# Install from TestPyPI to verify\n", + "pip install --index-url https://test.pypi.org/simple/ my-package\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass, field\n", + "from enum import Enum, auto\n", + "\n", + "\n", + "class PublishStep(Enum):\n", + " PREPARE = auto()\n", + " BUILD = auto()\n", + " CHECK = auto()\n", + " TEST_UPLOAD = auto()\n", + " PUBLISH = auto()\n", + "\n", + "\n", + "@dataclass\n", + "class PublishWorkflow:\n", + " \"\"\"Represents the steps to publish a Python package.\"\"\"\n", + " package_name: str\n", + " version: str\n", + " steps: list[tuple[PublishStep, str, str]] = field(default_factory=list)\n", + "\n", + " def __post_init__(self) -> None:\n", + " self.steps = [\n", + " (\n", + " PublishStep.PREPARE,\n", + " \"Update version and changelog\",\n", + " f'Set version = \"{self.version}\" in pyproject.toml',\n", + " ),\n", + " (\n", + " PublishStep.BUILD,\n", + " \"Build distributions\",\n", + " \"python -m build\",\n", + " ),\n", + " (\n", + " PublishStep.CHECK,\n", + " \"Validate package metadata\",\n", + " \"twine check dist/*\",\n", + " ),\n", + " (\n", + " PublishStep.TEST_UPLOAD,\n", + " \"Upload to TestPyPI\",\n", + " \"twine upload --repository testpypi dist/*\",\n", + " ),\n", + " (\n", + " PublishStep.PUBLISH,\n", + " \"Upload to PyPI\",\n", + " \"twine upload dist/*\",\n", + " ),\n", + " ]\n", + "\n", + " def display(self) -> None:\n", + " print(f\"Publishing workflow for {self.package_name} v{self.version}:\")\n", + " print(\"=\" * 60)\n", + " for i, (step, description, command) in enumerate(self.steps, 1):\n", + " print(f\"\\n Step {i}: {description}\")\n", + " print(f\" Command: {command}\")\n", + "\n", + "\n", + "workflow = PublishWorkflow(\"my-awesome-package\", \"1.0.0\")\n", + "workflow.display()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 7: Private Package Registries and Index URLs\n", + "\n", + "Not all packages belong on the public PyPI. Organizations often host **private\n", + "registries** for internal packages.\n", + "\n", + "### Common Private Registry Options\n", + "- **Azure Artifacts**: Integrated with Azure DevOps\n", + "- **AWS CodeArtifact**: Managed artifact repository\n", + "- **Google Artifact Registry**: GCP-hosted\n", + "- **GitLab Package Registry**: Built into GitLab\n", + "- **JFrog Artifactory**: Enterprise artifact management\n", + "- **devpi**: Self-hosted, open source\n", + "\n", + "### Configuration\n", + "\n", + "```bash\n", + "# Install from a private index\n", + "pip install --index-url https://private.registry.com/simple/ my-private-pkg\n", + "\n", + "# Use private index as extra (fallback to PyPI)\n", + "pip install --extra-index-url https://private.registry.com/simple/ my-pkg\n", + "```\n", + "\n", + "### pip.conf / pip.ini\n", + "```ini\n", + "[global]\n", + "index-url = https://private.registry.com/simple/\n", + "extra-index-url = https://pypi.org/simple/\n", + "trusted-host = private.registry.com\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "import os\n", + "\n", + "\n", + "@dataclass\n", + "class PipConfig:\n", + " \"\"\"Represents pip configuration for package indices.\"\"\"\n", + " index_url: str = \"https://pypi.org/simple/\"\n", + " extra_index_urls: list[str] | None = None\n", + " trusted_hosts: list[str] | None = None\n", + "\n", + " def to_pip_conf(self) -> str:\n", + " \"\"\"Generate pip.conf content.\"\"\"\n", + " lines: list[str] = [\"[global]\", f\"index-url = {self.index_url}\"]\n", + " if self.extra_index_urls:\n", + " for url in self.extra_index_urls:\n", + " lines.append(f\"extra-index-url = {url}\")\n", + " if self.trusted_hosts:\n", + " for host in self.trusted_hosts:\n", + " lines.append(f\"trusted-host = {host}\")\n", + " return \"\\n\".join(lines)\n", + "\n", + " def install_command(self, package: str) -> str:\n", + " \"\"\"Generate the pip install command.\"\"\"\n", + " parts: list[str] = [\"pip\", \"install\"]\n", + " if self.index_url != \"https://pypi.org/simple/\":\n", + " parts.extend([\"--index-url\", self.index_url])\n", + " if self.extra_index_urls:\n", + " for url in self.extra_index_urls:\n", + " parts.extend([\"--extra-index-url\", url])\n", + " parts.append(package)\n", + " return \" \".join(parts)\n", + "\n", + "\n", + "# Corporate setup with private registry\n", + "corporate_config = PipConfig(\n", + " index_url=\"https://artifacts.corp.example.com/pypi/simple/\",\n", + " extra_index_urls=[\"https://pypi.org/simple/\"],\n", + " trusted_hosts=[\"artifacts.corp.example.com\"],\n", + ")\n", + "\n", + "print(\"Generated pip.conf:\")\n", + "print(corporate_config.to_pip_conf())\n", + "print()\n", + "print(\"Install command:\")\n", + "print(f\" {corporate_config.install_command('internal-utils')}\")\n", + "print()\n", + "\n", + "# Default pip.conf location\n", + "if os.name == \"nt\":\n", + " config_path = Path.home() / \"pip\" / \"pip.ini\"\n", + "else:\n", + " config_path = Path.home() / \".config\" / \"pip\" / \"pip.conf\"\n", + "print(f\"Default pip config location: {config_path}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Section 8: Putting It All Together -- A Complete pyproject.toml\n", + "\n", + "Here is a complete, production-ready `pyproject.toml` that ties together everything\n", + "we have covered across all three notebooks in this chapter." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "complete_pyproject: str = \"\"\"\\\n", + "[project]\n", + "name = \"example-cli\"\n", + "version = \"1.0.0\"\n", + "description = \"A production-ready CLI tool example\"\n", + "readme = \"README.md\"\n", + "license = \"MIT\"\n", + "requires-python = \">=3.10\"\n", + "authors = [\n", + " {name = \"Jane Developer\", email = \"jane@example.com\"},\n", + "]\n", + "classifiers = [\n", + " \"Development Status :: 4 - Beta\",\n", + " \"Programming Language :: Python :: 3\",\n", + " \"Programming Language :: Python :: 3.10\",\n", + " \"Programming Language :: Python :: 3.11\",\n", + " \"Programming Language :: Python :: 3.12\",\n", + " \"Typing :: Typed\",\n", + "]\n", + "dependencies = [\n", + " \"click>=8.0\",\n", + " \"rich>=13.0\",\n", + " \"pydantic>=2.0,<3.0\",\n", + "]\n", + "\n", + "[project.optional-dependencies]\n", + "dev = [\n", + " \"pytest>=7.0\",\n", + " \"pytest-cov>=4.0\",\n", + " \"mypy>=1.0\",\n", + " \"ruff>=0.1\",\n", + "]\n", + "docs = [\n", + " \"sphinx>=7.0\",\n", + " \"furo\",\n", + "]\n", + "\n", + "[project.scripts]\n", + "example-cli = \"example_cli.main:app\"\n", + "\n", + "[project.urls]\n", + "Homepage = \"https://github.com/jane/example-cli\"\n", + "Documentation = \"https://example-cli.readthedocs.io\"\n", + "Repository = \"https://github.com/jane/example-cli\"\n", + "Issues = \"https://github.com/jane/example-cli/issues\"\n", + "\n", + "[build-system]\n", + "requires = [\"setuptools>=68.0\"]\n", + "build-backend = \"setuptools.build_meta\"\n", + "\n", + "[tool.setuptools.packages.find]\n", + "where = [\"src\"]\n", + "\n", + "[tool.setuptools.package-data]\n", + "example_cli = [\"py.typed\", \"templates/*.html\"]\n", + "\n", + "[tool.pytest.ini_options]\n", + "testpaths = [\"tests\"]\n", + "addopts = \"--strict-markers -v\"\n", + "\n", + "[tool.mypy]\n", + "strict = true\n", + "\n", + "[tool.ruff]\n", + "target-version = \"py310\"\n", + "line-length = 88\n", + "\"\"\"\n", + "\n", + "print(complete_pyproject)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Quick-reference checklist before publishing\n", + "\n", + "checklist: list[tuple[str, str]] = [\n", + " (\"Version updated\", 'version = \"X.Y.Z\" in pyproject.toml'),\n", + " (\"Changelog updated\", \"Document changes in CHANGELOG.md\"),\n", + " (\"Tests passing\", \"pytest --strict-markers\"),\n", + " (\"Type checks passing\", \"mypy src/\"),\n", + " (\"Linter clean\", \"ruff check src/\"),\n", + " (\"Build artifacts created\", \"python -m build\"),\n", + " (\"Metadata valid\", \"twine check dist/*\"),\n", + " (\"TestPyPI upload\", \"twine upload --repository testpypi dist/*\"),\n", + " (\"Test install works\", \"pip install --index-url https://test.pypi.org/simple/ pkg\"),\n", + " (\"PyPI upload\", \"twine upload dist/*\"),\n", + " (\"Git tag created\", 'git tag -a v1.0.0 -m \"Release 1.0.0\"'),\n", + "]\n", + "\n", + "print(\"Pre-publish checklist:\")\n", + "print(\"=\" * 65)\n", + "for i, (task, command) in enumerate(checklist, 1):\n", + " print(f\" [ ] {i:2d}. {task}\")\n", + " print(f\" {command}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Distribution Formats\n", + "- **sdist** (`.tar.gz`): Source archive, requires build on install\n", + "- **wheel** (`.whl`): Pre-built, fast to install, platform-specific for C extensions\n", + "- Always publish **both** sdist and wheel to PyPI\n", + "\n", + "### Versioning\n", + "- **Semantic versioning**: `MAJOR.MINOR.PATCH` with clear upgrade semantics\n", + "- **`~=` (compatible release)**: The recommended constraint for most dependencies\n", + "- **Pre-release tags**: `a` (alpha), `b` (beta), `rc` (release candidate)\n", + "\n", + "### Package Data\n", + "- Declare non-Python files in `[tool.setuptools.package-data]` or `MANIFEST.in`\n", + "- Access package data at runtime with `importlib.resources.files()`\n", + "\n", + "### Publishing\n", + "- Build with `python -m build`, validate with `twine check`, upload with `twine upload`\n", + "- Always test on **TestPyPI** before publishing to the real PyPI\n", + "- Use **private registries** for internal packages (`--index-url` or `pip.conf`)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_19/README.md b/src/chapter_19/README.md new file mode 100644 index 0000000..29ef64b --- /dev/null +++ b/src/chapter_19/README.md @@ -0,0 +1,22 @@ +# Chapter 19: Packaging and Distribution + +## Topics Covered +- Virtual environments: `venv`, isolation, activation +- `pyproject.toml`: modern project configuration (PEP 621) +- Build backends: setuptools, hatchling, flit, Poetry +- Package structure: src-layout vs flat layout +- Entry points: console scripts and plugins +- Dependency specification and version constraints +- Building distributions: sdist and wheel +- Publishing to PyPI and private registries + +## Notebooks +1. **01_virtual_environments.ipynb** — venv, isolation, pip, dependency resolution +2. **02_project_configuration.ipynb** — pyproject.toml, build backends, entry points +3. **03_building_and_publishing.ipynb** — sdist, wheels, PyPI, versioning strategies + +## Key Takeaways +- Always use virtual environments for project isolation +- `pyproject.toml` is the modern standard for Python project configuration +- src-layout prevents accidental imports from the source tree +- Semantic versioning communicates change impact to users diff --git a/src/chapter_19/__init__.py b/src/chapter_19/__init__.py new file mode 100644 index 0000000..52dea03 --- /dev/null +++ b/src/chapter_19/__init__.py @@ -0,0 +1 @@ +"""Chapter 19: Packaging and Distribution.""" diff --git a/src/chapter_20/01_memory_and_gc.ipynb b/src/chapter_20/01_memory_and_gc.ipynb new file mode 100644 index 0000000..a526b0a --- /dev/null +++ b/src/chapter_20/01_memory_and_gc.ipynb @@ -0,0 +1,517 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 20: Memory Management and Garbage Collection\n", + "\n", + "Understanding how CPython manages memory is essential for writing efficient Python code.\n", + "This notebook explores object identity, memory measurement, reference counting,\n", + "garbage collection, and weak references.\n", + "\n", + "## Topics Covered\n", + "- **Object identity**: `id()`, `is` vs `==`\n", + "- **Interning**: Small integer and string caching\n", + "- **Memory measurement**: `sys.getsizeof()`\n", + "- **Reference counting**: `sys.getrefcount()` and CPython's mechanism\n", + "- **Garbage collection**: The `gc` module and cycle detection\n", + "- **Weak references**: `weakref` module\n", + "- **Practical**: Tracking object lifecycles with `__del__`" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## id() and Identity: `is` vs `==`\n", + "\n", + "Every Python object has a unique identity (its memory address in CPython). The `id()` function\n", + "returns this identity. The `is` operator checks whether two names refer to the **same object**\n", + "(identity), while `==` checks whether two objects have the **same value** (equality)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "# Identity vs equality\n", + "a: list[int] = [1, 2, 3]\n", + "b: list[int] = [1, 2, 3]\n", + "c = a # c is an alias for a\n", + "\n", + "print(f\"a == b: {a == b}\") # True - same value\n", + "print(f\"a is b: {a is b}\") # False - different objects\n", + "print(f\"a is c: {a is c}\") # True - same object\n", + "\n", + "print(f\"\\nid(a): {id(a)}\")\n", + "print(f\"id(b): {id(b)}\")\n", + "print(f\"id(c): {id(c)}\")\n", + "\n", + "# Mutating through one alias affects the other\n", + "c.append(4)\n", + "print(f\"\\na after c.append(4): {a}\") # [1, 2, 3, 4]\n", + "print(f\"b is unaffected: {b}\") # [1, 2, 3]" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Small Integer Interning and String Interning\n", + "\n", + "CPython caches (\"interns\") small integers in the range **-5 to 256** at startup. Every variable\n", + "bound to one of these values points to the same pre-allocated object. Similarly, short strings\n", + "that look like identifiers are often interned automatically, and you can force interning with\n", + "`sys.intern()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5f6a7b8", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Small integer interning: -5 to 256 are cached singletons\n", + "x: int = 256\n", + "y: int = 256\n", + "print(f\"256 is 256: {x is y}\") # True - interned\n", + "print(f\"id(x)={id(x)}, id(y)={id(y)}\")\n", + "\n", + "# Outside the interning range, new objects are created\n", + "a: int = 257\n", + "b: int = 257\n", + "print(f\"\\n257 is 257: {a is b}\") # May be False (implementation detail)\n", + "print(f\"257 == 257: {a == b}\") # Always True\n", + "\n", + "# String interning for identifier-like strings\n", + "s1: str = \"hello\"\n", + "s2: str = \"hello\"\n", + "print(f\"\\n'hello' is 'hello': {s1 is s2}\") # Typically True (auto-interned)\n", + "\n", + "# Strings with spaces are usually NOT auto-interned\n", + "s3: str = \"hello world\"\n", + "s4: str = \"hello world\"\n", + "print(f\"'hello world' is 'hello world': {s3 is s4}\") # May be False\n", + "\n", + "# Force interning with sys.intern()\n", + "s5: str = sys.intern(\"hello world\")\n", + "s6: str = sys.intern(\"hello world\")\n", + "print(f\"\\nAfter sys.intern():\")\n", + "print(f\" s5 is s6: {s5 is s6}\") # True - forced interning" + ] + }, + { + "cell_type": "markdown", + "id": "f6a7b8c9", + "metadata": {}, + "source": [ + "## sys.getsizeof(): Measuring Object Memory\n", + "\n", + "`sys.getsizeof()` returns the memory consumption of an object in bytes. Note that it only\n", + "measures the **shallow** size of the object itself, not the objects it references. For containers,\n", + "it measures the container overhead plus its internal array of pointers, but not the elements." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7b8c9d0", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Size of basic types\n", + "print(\"Basic type sizes (bytes):\")\n", + "print(f\" int(0): {sys.getsizeof(0)}\")\n", + "print(f\" int(1): {sys.getsizeof(1)}\")\n", + "print(f\" int(2**30): {sys.getsizeof(2**30)}\")\n", + "print(f\" int(2**100): {sys.getsizeof(2**100)}\")\n", + "print(f\" float(3.14): {sys.getsizeof(3.14)}\")\n", + "print(f\" bool(True): {sys.getsizeof(True)}\")\n", + "print(f\" None: {sys.getsizeof(None)}\")\n", + "\n", + "# Strings grow with length\n", + "print(f\"\\nString sizes:\")\n", + "print(f\" '': {sys.getsizeof('')}\")\n", + "print(f\" 'a': {sys.getsizeof('a')}\")\n", + "print(f\" 'hello': {sys.getsizeof('hello')}\")\n", + "print(f\" 'a' * 100: {sys.getsizeof('a' * 100)}\")\n", + "\n", + "# Container sizes (shallow - does NOT include element sizes)\n", + "print(f\"\\nContainer sizes:\")\n", + "print(f\" []: {sys.getsizeof([])}\")\n", + "print(f\" [1,2,3]: {sys.getsizeof([1, 2, 3])}\")\n", + "print(f\" list(range(100)): {sys.getsizeof(list(range(100)))}\")\n", + "print(f\" (): {sys.getsizeof(())}\")\n", + "print(f\" (1,2,3): {sys.getsizeof((1, 2, 3))}\")\n", + "print(f\" {{}}: {sys.getsizeof({})}\")\n", + "print(f\" set(): {sys.getsizeof(set())}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b8c9d0e1", + "metadata": {}, + "source": [ + "## sys.getrefcount(): Reference Counting\n", + "\n", + "CPython uses **reference counting** as its primary memory management strategy. Every object\n", + "has a reference count that tracks how many names, containers, or other objects point to it.\n", + "When the count drops to zero, the memory is immediately freed.\n", + "\n", + "`sys.getrefcount(obj)` returns the current reference count. Note that passing `obj` to the\n", + "function itself creates a temporary reference, so the count is always at least one higher\n", + "than you might expect." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9d0e1f2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Basic reference counting\n", + "a = [1, 2, 3]\n", + "print(f\"After 'a = [1,2,3]': refcount = {sys.getrefcount(a)}\")\n", + "# The count includes: a + the argument to getrefcount = 2\n", + "\n", + "b = a # Another reference to the same list\n", + "print(f\"After 'b = a': refcount = {sys.getrefcount(a)}\")\n", + "\n", + "c = a # Yet another reference\n", + "print(f\"After 'c = a': refcount = {sys.getrefcount(a)}\")\n", + "\n", + "del b # Remove one reference\n", + "print(f\"After 'del b': refcount = {sys.getrefcount(a)}\")\n", + "\n", + "del c # Remove another\n", + "print(f\"After 'del c': refcount = {sys.getrefcount(a)}\")\n", + "\n", + "# Small integers have very high refcounts due to interning\n", + "print(f\"\\nRefcount of int 1: {sys.getrefcount(1)}\")\n", + "print(f\"Refcount of int 0: {sys.getrefcount(0)}\")\n", + "print(f\"Refcount of None: {sys.getrefcount(None)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d0e1f2a3", + "metadata": {}, + "source": [ + "## Garbage Collection: The gc Module\n", + "\n", + "Reference counting alone cannot handle **circular references** (object A references B, and B\n", + "references A). CPython's garbage collector uses a **generational** algorithm to detect and\n", + "break reference cycles.\n", + "\n", + "The `gc` module provides control over the garbage collector:\n", + "- `gc.collect()` - Force a collection cycle\n", + "- `gc.get_referrers(obj)` - Find objects that reference `obj`\n", + "- `gc.get_referents(obj)` - Find objects that `obj` references\n", + "- `gc.get_threshold()` - View collection thresholds\n", + "- `gc.isenabled()` / `gc.disable()` / `gc.enable()` - Control the collector" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# Check GC status and thresholds\n", + "print(f\"GC enabled: {gc.isenabled()}\")\n", + "print(f\"GC thresholds (gen0, gen1, gen2): {gc.get_threshold()}\")\n", + "print(f\"GC counts (gen0, gen1, gen2): {gc.get_count()}\")\n", + "\n", + "# Create a circular reference\n", + "class Node:\n", + " \"\"\"A node that can create circular references.\"\"\"\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + " self.partner: Node | None = None\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Node({self.name!r})\"\n", + "\n", + "# Create a cycle: a -> b -> a\n", + "a = Node(\"A\")\n", + "b = Node(\"B\")\n", + "a.partner = b\n", + "b.partner = a\n", + "\n", + "print(f\"\\na.partner = {a.partner}\")\n", + "print(f\"b.partner = {b.partner}\")\n", + "print(f\"Cycle: a.partner.partner is a = {a.partner.partner is a}\")\n", + "\n", + "# Without GC, deleting a and b would leak memory\n", + "# because the cycle keeps refcounts above zero\n", + "del a, b\n", + "\n", + "# Force garbage collection to clean up the cycle\n", + "collected = gc.collect()\n", + "print(f\"\\nForced GC collected {collected} objects\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "# gc.get_referrers() and gc.get_referents()\n", + "target: list[int] = [10, 20, 30]\n", + "container: dict[str, list[int]] = {\"data\": target}\n", + "alias = target\n", + "\n", + "# Find what refers TO our target list\n", + "referrers = gc.get_referrers(target)\n", + "print(f\"Number of referrers to target: {len(referrers)}\")\n", + "for ref in referrers:\n", + " if isinstance(ref, dict) and \"data\" in ref:\n", + " print(f\" Found container dict: {ref}\")\n", + "\n", + "# Find what target refers TO (its contents)\n", + "referents = gc.get_referents(target)\n", + "print(f\"\\nObjects that target references: {referents}\")\n", + "\n", + "# gc.get_referents on a dict shows keys and values\n", + "dict_referents = gc.get_referents(container)\n", + "print(f\"Dict referents: {dict_referents}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## weakref: References That Don't Prevent GC\n", + "\n", + "A **weak reference** allows you to refer to an object without incrementing its reference count.\n", + "When the original object is garbage collected, the weak reference returns `None` instead of\n", + "keeping the object alive.\n", + "\n", + "Weak references are useful for caches, observer patterns, and avoiding circular reference leaks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class ExpensiveResource:\n", + " \"\"\"Simulates a resource-heavy object.\"\"\"\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + " print(f\" Created: {self.name}\")\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"ExpensiveResource({self.name!r})\"\n", + "\n", + " def __del__(self) -> None:\n", + " print(f\" Destroyed: {self.name}\")\n", + "\n", + "\n", + "# Create a strong reference and a weak reference\n", + "print(\"Creating resource:\")\n", + "obj = ExpensiveResource(\"big_data\")\n", + "weak = weakref.ref(obj)\n", + "\n", + "print(f\"\\nWeak ref alive: {weak()}\")\n", + "print(f\"weak() is obj: {weak() is obj}\")\n", + "\n", + "# Delete the strong reference\n", + "print(\"\\nDeleting strong reference:\")\n", + "del obj\n", + "\n", + "# The weak reference now returns None\n", + "print(f\"Weak ref after del: {weak()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5d6e7f8", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class CachedItem:\n", + " \"\"\"An item that can be weakly cached.\"\"\"\n", + " def __init__(self, item_id: int, data: str) -> None:\n", + " self.item_id = item_id\n", + " self.data = data\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"CachedItem(id={self.item_id}, data={self.data!r})\"\n", + "\n", + "\n", + "# WeakValueDictionary: values are weak references\n", + "# Items are automatically removed when no strong refs remain\n", + "cache: weakref.WeakValueDictionary[int, CachedItem] = weakref.WeakValueDictionary()\n", + "\n", + "# Create items and cache them\n", + "item1 = CachedItem(1, \"first\")\n", + "item2 = CachedItem(2, \"second\")\n", + "item3 = CachedItem(3, \"third\")\n", + "\n", + "cache[1] = item1\n", + "cache[2] = item2\n", + "cache[3] = item3\n", + "\n", + "print(f\"Cache keys: {list(cache.keys())}\")\n", + "print(f\"Cache[1]: {cache[1]}\")\n", + "\n", + "# Delete a strong reference - the entry disappears from cache\n", + "del item2\n", + "print(f\"\\nAfter del item2, cache keys: {list(cache.keys())}\")\n", + "\n", + "del item1\n", + "print(f\"After del item1, cache keys: {list(cache.keys())}\")\n", + "\n", + "# item3 is still alive because we hold a strong reference\n", + "print(f\"cache[3]: {cache[3]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d6e7f8a9", + "metadata": {}, + "source": [ + "## Practical: Tracking Object Lifecycles with `__del__`\n", + "\n", + "The `__del__` method (finalizer) is called when an object is about to be garbage collected.\n", + "While it should not be relied on for critical cleanup (use context managers for that), it is\n", + "useful for understanding when objects are actually destroyed.\n", + "\n", + "**Warning**: `__del__` can cause issues with circular references if not used carefully, because\n", + "the garbage collector cannot determine a safe order to call finalizers in a cycle." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7f8a9b0", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "import weakref\n", + "\n", + "\n", + "class TrackedObject:\n", + " \"\"\"An object that reports its own lifecycle events.\"\"\"\n", + " _instances: list[str] = [] # Class-level tracking\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + " TrackedObject._instances.append(name)\n", + " print(f\" [CREATED] {name} (total alive: {len(TrackedObject._instances)})\")\n", + "\n", + " def __del__(self) -> None:\n", + " if self.name in TrackedObject._instances:\n", + " TrackedObject._instances.remove(self.name)\n", + " print(f\" [DESTROYED] {self.name} (total alive: {len(TrackedObject._instances)})\")\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"TrackedObject({self.name!r})\"\n", + "\n", + "\n", + "# Watch the lifecycle\n", + "print(\"Creating objects:\")\n", + "obj_a = TrackedObject(\"alpha\")\n", + "obj_b = TrackedObject(\"beta\")\n", + "obj_c = TrackedObject(\"gamma\")\n", + "\n", + "print(f\"\\nAlive: {TrackedObject._instances}\")\n", + "\n", + "print(\"\\nDeleting beta:\")\n", + "del obj_b\n", + "\n", + "print(f\"\\nAlive: {TrackedObject._instances}\")\n", + "\n", + "print(\"\\nDeleting remaining objects:\")\n", + "del obj_a\n", + "del obj_c\n", + "\n", + "print(f\"\\nAlive: {TrackedObject._instances}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8a9b0c1", + "metadata": {}, + "outputs": [], + "source": "import sys\nimport weakref\n\n\nclass LifecycleDemo:\n \"\"\"Demonstrates ref counting + weak refs + finalizers together.\"\"\"\n def __init__(self, name: str) -> None:\n self.name = name\n\n def __repr__(self) -> str:\n return f\"LifecycleDemo({self.name!r})\"\n\n def __del__(self) -> None:\n print(f\" Finalizer called for {self.name}\")\n\n\n# Weak reference with a callback\ndef weak_callback(ref: weakref.ref) -> None:\n \"\"\"Called when the weakly-referenced object is collected.\"\"\"\n print(f\" Weak reference callback triggered: object was collected\")\n\n\nprint(\"Step 1: Create object with weak ref + callback\")\nobj = LifecycleDemo(\"demo\")\nweak = weakref.ref(obj, weak_callback)\nprint(f\" obj = {obj}\")\nprint(f\" weak() = {weak()}\")\nprint(f\" refcount = {sys.getrefcount(obj)}\")\n\nprint(\"\\nStep 2: Delete the strong reference\")\ndel obj\n\nprint(\"\\nStep 3: Check weak reference\")\nprint(f\" weak() = {weak()}\")" + }, + { + "cell_type": "markdown", + "id": "a9b0c1d2", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Concept | Tool | Purpose |\n", + "|---------|------|---------|\n", + "| **Identity** | `id()`, `is` | Check if two names refer to the same object |\n", + "| **Equality** | `==` | Check if two objects have the same value |\n", + "| **Interning** | Small ints, `sys.intern()` | Reuse immutable objects to save memory |\n", + "| **Memory size** | `sys.getsizeof()` | Measure shallow memory of an object |\n", + "| **Ref counting** | `sys.getrefcount()` | See how many references point to an object |\n", + "| **GC control** | `gc.collect()`, `gc.get_referrers()` | Force collection, inspect object graph |\n", + "| **Weak refs** | `weakref.ref()`, `WeakValueDictionary` | Reference objects without preventing GC |\n", + "| **Finalizers** | `__del__` | Run cleanup code when object is collected |\n", + "\n", + "### Best Practices\n", + "- Use `==` for value comparison, `is` only for `None` checks and identity checks\n", + "- Never rely on interning behavior in application logic -- it is an implementation detail\n", + "- Use `sys.getsizeof()` for quick estimates, but remember it is shallow\n", + "- Prefer context managers over `__del__` for resource cleanup\n", + "- Use `weakref` for caches to avoid memory leaks\n", + "- Let the garbage collector do its job; only call `gc.collect()` when you have a specific reason" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_20/02_profiling_and_timeit.ipynb b/src/chapter_20/02_profiling_and_timeit.ipynb new file mode 100644 index 0000000..f2ae87a --- /dev/null +++ b/src/chapter_20/02_profiling_and_timeit.ipynb @@ -0,0 +1,702 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1010101", + "metadata": {}, + "source": [ + "# Chapter 20: Profiling and Timing Python Code\n", + "\n", + "Before optimizing code, you must **measure** it. This notebook covers Python's built-in\n", + "tools for timing code snippets, profiling function calls, and identifying performance\n", + "bottlenecks.\n", + "\n", + "## Topics Covered\n", + "- **timeit**: Precise micro-benchmarking of small code snippets\n", + "- **cProfile**: Function-level profiling with call counts and timing\n", + "- **pstats**: Analyzing and sorting profile results\n", + "- **time.perf_counter()**: Manual high-resolution timing\n", + "- **Decorator-based profiling**: Reusable profiling patterns\n", + "- **Common performance traps**: String concatenation, repeated lookups, and more" + ] + }, + { + "cell_type": "markdown", + "id": "b2020202", + "metadata": {}, + "source": [ + "## The timeit Module: Micro-Benchmarking\n", + "\n", + "`timeit` runs a code snippet many times and reports the total (or average) execution time.\n", + "It automatically disables garbage collection during timing and uses the best available timer\n", + "for your platform, making it ideal for comparing small code alternatives." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3030303", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# timeit.timeit() runs the statement `number` times and returns total seconds\n", + "# Default number is 1_000_000\n", + "\n", + "# Compare list creation approaches\n", + "time_literal = timeit.timeit(\"[1, 2, 3]\", number=1_000_000)\n", + "time_constructor = timeit.timeit(\"list((1, 2, 3))\", number=1_000_000)\n", + "\n", + "print(\"Creating [1, 2, 3] one million times:\")\n", + "print(f\" List literal: {time_literal:.4f}s\")\n", + "print(f\" list() constructor: {time_constructor:.4f}s\")\n", + "print(f\" Literal is {time_constructor / time_literal:.1f}x faster\")\n", + "\n", + "# Compare dict creation\n", + "time_dict_literal = timeit.timeit('{\"a\": 1, \"b\": 2}', number=1_000_000)\n", + "time_dict_constructor = timeit.timeit('dict(a=1, b=2)', number=1_000_000)\n", + "\n", + "print(f\"\\nCreating dict one million times:\")\n", + "print(f\" Dict literal: {time_dict_literal:.4f}s\")\n", + "print(f\" dict(): {time_dict_constructor:.4f}s\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4040404", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# timeit.repeat() runs multiple trials and returns a list of times\n", + "# This helps identify variance in measurements\n", + "\n", + "results_comp = timeit.repeat(\n", + " \"[x**2 for x in range(100)]\",\n", + " repeat=5,\n", + " number=10_000,\n", + ")\n", + "\n", + "results_map = timeit.repeat(\n", + " \"list(map(lambda x: x**2, range(100)))\",\n", + " repeat=5,\n", + " number=10_000,\n", + ")\n", + "\n", + "print(\"Squaring 0-99, 10k iterations, 5 trials:\")\n", + "print(f\" List comprehension: {[f'{t:.4f}' for t in results_comp]}\")\n", + "print(f\" map() + lambda: {[f'{t:.4f}' for t in results_map]}\")\n", + "print(f\"\\n Best comprehension: {min(results_comp):.4f}s\")\n", + "print(f\" Best map+lambda: {min(results_map):.4f}s\")\n", + "print(\"\\nNote: use min() of repeat results, not average (min reflects best case)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5050505", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# Using setup parameter for imports and initialization\n", + "# The setup code runs once, then the statement is timed\n", + "\n", + "setup = \"from collections import deque; data = list(range(1000))\"\n", + "\n", + "time_list_pop = timeit.timeit(\n", + " \"data.pop(0)\",\n", + " setup=\"data = list(range(1000))\",\n", + " number=1_000,\n", + ")\n", + "\n", + "time_deque_pop = timeit.timeit(\n", + " \"d.popleft()\",\n", + " setup=\"from collections import deque; d = deque(range(1000))\",\n", + " number=1_000,\n", + ")\n", + "\n", + "print(\"Popping from the left, 1000 times:\")\n", + "print(f\" list.pop(0): {time_list_pop:.6f}s\")\n", + "print(f\" deque.popleft(): {time_deque_pop:.6f}s\")\n", + "print(f\" deque is {time_list_pop / time_deque_pop:.1f}x faster\")\n", + "\n", + "# Timing a callable directly with timeit.timeit()\n", + "def fibonacci(n: int) -> int:\n", + " \"\"\"Naive recursive fibonacci.\"\"\"\n", + " if n < 2:\n", + " return n\n", + " return fibonacci(n - 1) + fibonacci(n - 2)\n", + "\n", + "time_fib = timeit.timeit(lambda: fibonacci(20), number=100)\n", + "print(f\"\\nfibonacci(20) x 100: {time_fib:.4f}s\")\n", + "print(f\" Average per call: {time_fib / 100 * 1000:.2f}ms\")" + ] + }, + { + "cell_type": "markdown", + "id": "f6060606", + "metadata": {}, + "source": [ + "## cProfile: Function-Level Profiling\n", + "\n", + "`cProfile` is a deterministic profiler that records every function call, how many times it was\n", + "called, and how long each call took. It reports both:\n", + "- **tottime**: Time spent *inside* the function (excluding sub-calls)\n", + "- **cumtime**: Cumulative time including all sub-calls\n", + "\n", + "This is the go-to tool for identifying which functions consume the most time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7070707", + "metadata": {}, + "outputs": [], + "source": [ + "import cProfile\n", + "\n", + "\n", + "def process_data(n: int) -> list[int]:\n", + " \"\"\"Simulate a multi-step data pipeline.\"\"\"\n", + " raw = generate_data(n)\n", + " filtered = filter_data(raw)\n", + " result = transform_data(filtered)\n", + " return result\n", + "\n", + "\n", + "def generate_data(n: int) -> list[int]:\n", + " \"\"\"Generate a list of numbers.\"\"\"\n", + " return [i * 3 for i in range(n)]\n", + "\n", + "\n", + "def filter_data(data: list[int]) -> list[int]:\n", + " \"\"\"Filter to even numbers only.\"\"\"\n", + " return [x for x in data if x % 2 == 0]\n", + "\n", + "\n", + "def transform_data(data: list[int]) -> list[int]:\n", + " \"\"\"Apply an expensive transformation.\"\"\"\n", + " result: list[int] = []\n", + " for x in data:\n", + " result.append(slow_computation(x))\n", + " return result\n", + "\n", + "\n", + "def slow_computation(x: int) -> int:\n", + " \"\"\"Simulate an expensive per-element computation.\"\"\"\n", + " total = 0\n", + " for i in range(100):\n", + " total += x * i\n", + " return total\n", + "\n", + "\n", + "# Profile the pipeline\n", + "print(\"Profiling process_data(5000):\")\n", + "print(\"=\" * 70)\n", + "cProfile.run(\"process_data(5000)\", sort=\"cumulative\")" + ] + }, + { + "cell_type": "markdown", + "id": "b8080808", + "metadata": {}, + "source": [ + "## pstats: Analyzing Profile Results\n", + "\n", + "For more control over profile output, save the results to a `pstats.Stats` object.\n", + "You can sort by different columns, filter by function name, and strip directory paths\n", + "for cleaner output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9090909", + "metadata": {}, + "outputs": [], + "source": [ + "import cProfile\n", + "import pstats\n", + "import io\n", + "\n", + "\n", + "def compute_primes(limit: int) -> list[int]:\n", + " \"\"\"Find primes up to limit using trial division.\"\"\"\n", + " primes: list[int] = []\n", + " for num in range(2, limit):\n", + " if is_prime(num):\n", + " primes.append(num)\n", + " return primes\n", + "\n", + "\n", + "def is_prime(n: int) -> bool:\n", + " \"\"\"Check if n is prime via trial division.\"\"\"\n", + " if n < 2:\n", + " return False\n", + " for i in range(2, int(n**0.5) + 1):\n", + " if n % i == 0:\n", + " return False\n", + " return True\n", + "\n", + "\n", + "# Profile into a Stats object for analysis\n", + "profiler = cProfile.Profile()\n", + "profiler.enable()\n", + "result = compute_primes(5000)\n", + "profiler.disable()\n", + "\n", + "print(f\"Found {len(result)} primes under 5000\")\n", + "print()\n", + "\n", + "# Analyze with pstats\n", + "stream = io.StringIO()\n", + "stats = pstats.Stats(profiler, stream=stream)\n", + "stats.strip_dirs() # Remove directory paths for readability\n", + "stats.sort_stats(\"tottime\") # Sort by time spent in function\n", + "stats.print_stats(10) # Top 10 functions\n", + "print(stream.getvalue())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0101010", + "metadata": {}, + "outputs": [], + "source": [ + "import cProfile\n", + "import pstats\n", + "import io\n", + "\n", + "\n", + "def run_simulation(iterations: int) -> float:\n", + " \"\"\"Run a multi-step simulation.\"\"\"\n", + " total: float = 0.0\n", + " for i in range(iterations):\n", + " value = step_one(i)\n", + " value = step_two(value)\n", + " total += step_three(value)\n", + " return total\n", + "\n", + "\n", + "def step_one(x: int) -> float:\n", + " \"\"\"First processing step.\"\"\"\n", + " return sum(i * 0.1 for i in range(x % 50))\n", + "\n", + "\n", + "def step_two(x: float) -> float:\n", + " \"\"\"Second processing step (deliberately slow).\"\"\"\n", + " result = x\n", + " for _ in range(200):\n", + " result = (result + 1.0) * 0.999\n", + " return result\n", + "\n", + "\n", + "def step_three(x: float) -> float:\n", + " \"\"\"Third processing step.\"\"\"\n", + " return x ** 0.5\n", + "\n", + "\n", + "# Identifying bottlenecks: sort by cumtime vs tottime\n", + "profiler = cProfile.Profile()\n", + "profiler.enable()\n", + "run_simulation(500)\n", + "profiler.disable()\n", + "\n", + "# Sort by cumulative time (includes sub-calls)\n", + "stream = io.StringIO()\n", + "stats = pstats.Stats(profiler, stream=stream)\n", + "stats.strip_dirs()\n", + "print(\"Sorted by CUMULATIVE time (cumtime):\")\n", + "stats.sort_stats(\"cumulative\")\n", + "stats.print_stats(8)\n", + "print(stream.getvalue())\n", + "\n", + "# Sort by total time in function (excludes sub-calls)\n", + "stream2 = io.StringIO()\n", + "stats2 = pstats.Stats(profiler, stream=stream2)\n", + "stats2.strip_dirs()\n", + "print(\"\\nSorted by TOTAL time in function (tottime):\")\n", + "stats2.sort_stats(\"tottime\")\n", + "stats2.print_stats(8)\n", + "print(stream2.getvalue())" + ] + }, + { + "cell_type": "markdown", + "id": "e1111111", + "metadata": {}, + "source": [ + "## time.perf_counter(): Manual High-Resolution Timing\n", + "\n", + "When you need to time a specific section of code (not just a function call), use\n", + "`time.perf_counter()`. It provides the highest resolution clock available on your system\n", + "and includes time spent sleeping. For measuring only CPU time, use `time.process_time()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2121212", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "# time.perf_counter() for wall-clock time\n", + "start = time.perf_counter()\n", + "total = sum(i * i for i in range(1_000_000))\n", + "end = time.perf_counter()\n", + "print(f\"Sum of squares (1M): {end - start:.4f}s\")\n", + "\n", + "# time.perf_counter_ns() for nanosecond precision\n", + "start_ns = time.perf_counter_ns()\n", + "result = sorted([5, 3, 1, 4, 2])\n", + "end_ns = time.perf_counter_ns()\n", + "print(f\"Sorting 5 elements: {end_ns - start_ns} ns\")\n", + "\n", + "# time.process_time() measures only CPU time (not sleep/IO)\n", + "cpu_start = time.process_time()\n", + "data = [x ** 2 for x in range(500_000)]\n", + "cpu_end = time.process_time()\n", + "print(f\"\\nCPU time for 500k squares: {cpu_end - cpu_start:.4f}s\")\n", + "\n", + "# Demonstrate the difference: perf_counter includes sleep, process_time does not\n", + "wall_start = time.perf_counter()\n", + "cpu_start = time.process_time()\n", + "time.sleep(0.1) # Sleep 100ms\n", + "_ = sum(range(100_000))\n", + "wall_end = time.perf_counter()\n", + "cpu_end = time.process_time()\n", + "\n", + "print(f\"\\nWith 100ms sleep + computation:\")\n", + "print(f\" Wall clock (perf_counter): {wall_end - wall_start:.4f}s\")\n", + "print(f\" CPU time (process_time): {cpu_end - cpu_start:.4f}s\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3131313", + "metadata": {}, + "source": [ + "## Profiling Patterns: Decorator-Based Profiling\n", + "\n", + "A profiling decorator makes it easy to measure any function without modifying its code.\n", + "This is a reusable pattern you can drop into any project." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4141414", + "metadata": {}, + "outputs": [], + "source": [ + "import functools\n", + "import time\n", + "from typing import Any, Callable, TypeVar\n", + "\n", + "F = TypeVar(\"F\", bound=Callable[..., Any])\n", + "\n", + "\n", + "def timer(func: F) -> F:\n", + " \"\"\"Decorator that prints execution time of the wrapped function.\"\"\"\n", + " @functools.wraps(func)\n", + " def wrapper(*args: Any, **kwargs: Any) -> Any:\n", + " start = time.perf_counter()\n", + " result = func(*args, **kwargs)\n", + " elapsed = time.perf_counter() - start\n", + " print(f\" {func.__name__}() took {elapsed:.4f}s\")\n", + " return result\n", + " return wrapper # type: ignore[return-value]\n", + "\n", + "\n", + "def call_counter(func: F) -> F:\n", + " \"\"\"Decorator that counts how many times a function is called.\"\"\"\n", + " @functools.wraps(func)\n", + " def wrapper(*args: Any, **kwargs: Any) -> Any:\n", + " wrapper.call_count += 1 # type: ignore[attr-defined]\n", + " return func(*args, **kwargs)\n", + " wrapper.call_count = 0 # type: ignore[attr-defined]\n", + " return wrapper # type: ignore[return-value]\n", + "\n", + "\n", + "# Using the timer decorator\n", + "@timer\n", + "def slow_sort(data: list[int]) -> list[int]:\n", + " \"\"\"Bubble sort - deliberately slow for demonstration.\"\"\"\n", + " arr = data.copy()\n", + " n = len(arr)\n", + " for i in range(n):\n", + " for j in range(0, n - i - 1):\n", + " if arr[j] > arr[j + 1]:\n", + " arr[j], arr[j + 1] = arr[j + 1], arr[j]\n", + " return arr\n", + "\n", + "\n", + "@timer\n", + "def fast_sort(data: list[int]) -> list[int]:\n", + " \"\"\"Built-in sort (Timsort).\"\"\"\n", + " return sorted(data)\n", + "\n", + "\n", + "import random\n", + "test_data = [random.randint(0, 10_000) for _ in range(5_000)]\n", + "\n", + "print(\"Sorting 5,000 random integers:\")\n", + "result1 = slow_sort(test_data)\n", + "result2 = fast_sort(test_data)\n", + "print(f\" Results match: {result1 == result2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5151515", + "metadata": {}, + "outputs": [], + "source": [ + "import functools\n", + "import time\n", + "from typing import Any, Callable\n", + "\n", + "\n", + "def profile_stats(func: Callable[..., Any]) -> Callable[..., Any]:\n", + " \"\"\"Decorator that accumulates timing statistics across multiple calls.\"\"\"\n", + " @functools.wraps(func)\n", + " def wrapper(*args: Any, **kwargs: Any) -> Any:\n", + " start = time.perf_counter()\n", + " result = func(*args, **kwargs)\n", + " elapsed = time.perf_counter() - start\n", + " wrapper._times.append(elapsed) # type: ignore[attr-defined]\n", + " return result\n", + "\n", + " wrapper._times: list[float] = [] # type: ignore[attr-defined]\n", + "\n", + " def get_stats() -> dict[str, float]:\n", + " times = wrapper._times # type: ignore[attr-defined]\n", + " if not times:\n", + " return {\"calls\": 0}\n", + " return {\n", + " \"calls\": len(times),\n", + " \"total\": sum(times),\n", + " \"avg\": sum(times) / len(times),\n", + " \"min\": min(times),\n", + " \"max\": max(times),\n", + " }\n", + "\n", + " wrapper.get_stats = get_stats # type: ignore[attr-defined]\n", + " return wrapper\n", + "\n", + "\n", + "@profile_stats\n", + "def process_item(x: int) -> int:\n", + " \"\"\"Simulate variable-cost processing.\"\"\"\n", + " return sum(range(x))\n", + "\n", + "\n", + "# Run the function many times with varying input sizes\n", + "for i in range(100):\n", + " process_item(i * 100)\n", + "\n", + "# Inspect accumulated statistics\n", + "stats = process_item.get_stats() # type: ignore[attr-defined]\n", + "print(\"Profile statistics for process_item():\")\n", + "for key, value in stats.items():\n", + " if isinstance(value, float):\n", + " print(f\" {key}: {value:.6f}s\")\n", + " else:\n", + " print(f\" {key}: {value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d6161616", + "metadata": {}, + "source": [ + "## Common Performance Traps\n", + "\n", + "Certain coding patterns are deceptively slow. Measuring them with `timeit` makes the\n", + "cost obvious and helps you choose better alternatives." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7171717", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# Trap 1: String concatenation in a loop\n", + "# Strings are immutable, so += creates a new string each time: O(n^2)\n", + "\n", + "def concat_with_plus(n: int) -> str:\n", + " \"\"\"Build a string using += (slow).\"\"\"\n", + " result = \"\"\n", + " for i in range(n):\n", + " result += str(i) + \" \"\n", + " return result\n", + "\n", + "\n", + "def concat_with_join(n: int) -> str:\n", + " \"\"\"Build a string using join (fast).\"\"\"\n", + " return \" \".join(str(i) for i in range(n))\n", + "\n", + "\n", + "def concat_with_list(n: int) -> str:\n", + " \"\"\"Build a string by appending to a list, then joining.\"\"\"\n", + " parts: list[str] = []\n", + " for i in range(n):\n", + " parts.append(str(i))\n", + " return \" \".join(parts)\n", + "\n", + "\n", + "n = 10_000\n", + "t_plus = timeit.timeit(lambda: concat_with_plus(n), number=10)\n", + "t_join = timeit.timeit(lambda: concat_with_join(n), number=10)\n", + "t_list = timeit.timeit(lambda: concat_with_list(n), number=10)\n", + "\n", + "print(f\"String concatenation ({n} items, 10 runs):\")\n", + "print(f\" += loop: {t_plus:.4f}s\")\n", + "print(f\" join(genexpr): {t_join:.4f}s\")\n", + "print(f\" list + join: {t_list:.4f}s\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8181818", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# Trap 2: Repeated attribute/method lookups in tight loops\n", + "# Each dot lookup (e.g., math.sqrt) is a dictionary lookup\n", + "\n", + "setup = \"import math; data = list(range(1, 10001))\"\n", + "\n", + "# Slow: repeated global + attribute lookup\n", + "slow_code = \"\"\"\n", + "result = []\n", + "for x in data:\n", + " result.append(math.sqrt(x))\n", + "\"\"\"\n", + "\n", + "# Fast: cache the method lookup and the append method\n", + "fast_code = \"\"\"\n", + "sqrt = math.sqrt\n", + "result = []\n", + "append = result.append\n", + "for x in data:\n", + " append(sqrt(x))\n", + "\"\"\"\n", + "\n", + "# Fastest: use a list comprehension (implicit optimization)\n", + "fastest_code = \"result = [math.sqrt(x) for x in data]\"\n", + "\n", + "t_slow = timeit.timeit(slow_code, setup=setup, number=1000)\n", + "t_fast = timeit.timeit(fast_code, setup=setup, number=1000)\n", + "t_fastest = timeit.timeit(fastest_code, setup=setup, number=1000)\n", + "\n", + "print(\"sqrt() over 10k items, 1000 runs:\")\n", + "print(f\" Repeated lookups: {t_slow:.4f}s\")\n", + "print(f\" Cached lookups: {t_fast:.4f}s\")\n", + "print(f\" List comprehension: {t_fastest:.4f}s\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9191919", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# Trap 3: Checking membership in a list vs a set\n", + "# list membership is O(n), set membership is O(1)\n", + "\n", + "setup_list = \"data = list(range(10000))\"\n", + "setup_set = \"data = set(range(10000))\"\n", + "\n", + "# Worst case: item not found (must scan entire list)\n", + "t_list = timeit.timeit(\"9999 in data\", setup=setup_list, number=100_000)\n", + "t_set = timeit.timeit(\"9999 in data\", setup=setup_set, number=100_000)\n", + "\n", + "print(\"Membership test (item at end), 100k checks:\")\n", + "print(f\" list: {t_list:.4f}s\")\n", + "print(f\" set: {t_set:.4f}s\")\n", + "print(f\" set is {t_list / t_set:.0f}x faster\")\n", + "\n", + "# Trap 4: Creating objects unnecessarily in loops\n", + "t_create = timeit.timeit(\n", + " \"[tuple(range(10)) for _ in range(1000)]\",\n", + " number=1000,\n", + ")\n", + "t_reuse = timeit.timeit(\n", + " \"[t for _ in range(1000)]\",\n", + " setup=\"t = tuple(range(10))\",\n", + " number=1000,\n", + ")\n", + "\n", + "print(f\"\\nCreating vs reusing a tuple, 1000x1000:\")\n", + "print(f\" Create each time: {t_create:.4f}s\")\n", + "print(f\" Reuse one object: {t_reuse:.4f}s\")" + ] + }, + { + "cell_type": "markdown", + "id": "b0202020", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Tool | Use Case | Key Function |\n", + "|------|----------|-------------|\n", + "| **timeit** | Micro-benchmark small snippets | `timeit.timeit()`, `timeit.repeat()` |\n", + "| **cProfile** | Profile all function calls | `cProfile.run()`, `cProfile.Profile()` |\n", + "| **pstats** | Analyze profile results | `Stats.sort_stats()`, `Stats.print_stats()` |\n", + "| **time.perf_counter()** | Manual wall-clock timing | High-resolution, includes sleep |\n", + "| **time.process_time()** | CPU-only timing | Excludes sleep and I/O wait |\n", + "\n", + "### Profiling Workflow\n", + "1. **Measure first** -- never guess where bottlenecks are\n", + "2. **Use `cProfile`** to find which functions take the most time\n", + "3. **Sort by `cumtime`** to find the \"big picture\" bottlenecks\n", + "4. **Sort by `tottime`** to find where CPU time is actually spent\n", + "5. **Use `timeit`** to compare alternative implementations\n", + "6. **Re-measure** after each change to verify improvement\n", + "\n", + "### Common Traps to Avoid\n", + "- String concatenation with `+=` in loops (use `\"\".join()` instead)\n", + "- Repeated attribute lookups in tight loops (cache with local variables)\n", + "- Membership testing in `list` when `set` would suffice\n", + "- Creating identical objects repeatedly instead of reusing them" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_20/03_optimization_techniques.ipynb b/src/chapter_20/03_optimization_techniques.ipynb new file mode 100644 index 0000000..a69dde5 --- /dev/null +++ b/src/chapter_20/03_optimization_techniques.ipynb @@ -0,0 +1,723 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "aa110011", + "metadata": {}, + "source": [ + "# Chapter 20: Optimization Techniques\n", + "\n", + "Once you have profiled your code and identified bottlenecks, this notebook covers the\n", + "techniques available to make Python code faster and more memory-efficient. The golden rule:\n", + "**profile first, optimize algorithmic complexity, then micro-optimize**.\n", + "\n", + "## Topics Covered\n", + "- **`__slots__`**: Memory optimization for classes\n", + "- **Algorithmic improvements**: O(n) vs O(n^2) with measurements\n", + "- **`dis` module**: Bytecode disassembly\n", + "- **Code object attributes**: `co_varnames`, `co_consts`, etc.\n", + "- **Generators vs list comprehensions**: Memory trade-offs\n", + "- **Local vs global variable access**: `LOAD_FAST` vs `LOAD_GLOBAL`\n", + "- **`collections` and built-in optimizations**\n", + "- **The GIL**: Threading vs multiprocessing\n", + "- **`ctypes` overview**: Calling C from Python\n", + "- **Optimization guidelines**: A practical workflow" + ] + }, + { + "cell_type": "markdown", + "id": "bb220022", + "metadata": {}, + "source": [ + "## `__slots__` for Memory Optimization\n", + "\n", + "By default, Python stores instance attributes in a per-object `__dict__` dictionary. For\n", + "classes with many instances, this is wasteful. Defining `__slots__` replaces `__dict__` with\n", + "a fixed-size struct, reducing memory per instance significantly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc330033", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "class PointDict:\n", + " \"\"\"Regular class with __dict__.\"\"\"\n", + " def __init__(self, x: float, y: float, z: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + " self.z = z\n", + "\n", + "\n", + "class PointSlots:\n", + " \"\"\"Slotted class - no __dict__.\"\"\"\n", + " __slots__ = (\"x\", \"y\", \"z\")\n", + "\n", + " def __init__(self, x: float, y: float, z: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + " self.z = z\n", + "\n", + "\n", + "# Compare single-instance memory\n", + "pd = PointDict(1.0, 2.0, 3.0)\n", + "ps = PointSlots(1.0, 2.0, 3.0)\n", + "\n", + "print(\"Single instance memory:\")\n", + "print(f\" PointDict: {sys.getsizeof(pd)} + __dict__: {sys.getsizeof(pd.__dict__)} bytes\")\n", + "print(f\" PointSlots: {sys.getsizeof(ps)} bytes (no __dict__)\")\n", + "\n", + "# Measure memory for many instances\n", + "n = 100_000\n", + "dict_objects = [PointDict(1.0, 2.0, 3.0) for _ in range(n)]\n", + "slots_objects = [PointSlots(1.0, 2.0, 3.0) for _ in range(n)]\n", + "\n", + "# Approximate total memory (instance + __dict__ for dict version)\n", + "dict_total = sum(sys.getsizeof(o) + sys.getsizeof(o.__dict__) for o in dict_objects[:100]) / 100\n", + "slots_total = sum(sys.getsizeof(o) for o in slots_objects[:100]) / 100\n", + "\n", + "print(f\"\\nEstimated per-object cost ({n:,} instances):\")\n", + "print(f\" PointDict: ~{dict_total:.0f} bytes/obj = ~{dict_total * n / 1024 / 1024:.1f} MB total\")\n", + "print(f\" PointSlots: ~{slots_total:.0f} bytes/obj = ~{slots_total * n / 1024 / 1024:.1f} MB total\")\n", + "print(f\" Savings: ~{(1 - slots_total / dict_total) * 100:.0f}%\")\n", + "\n", + "# __slots__ also prevents adding arbitrary attributes\n", + "try:\n", + " ps.w = 4.0 # type: ignore[attr-defined]\n", + "except AttributeError as e:\n", + " print(f\"\\nCannot add attributes to slotted class: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "dd440044", + "metadata": {}, + "source": [ + "## Algorithmic Improvements: O(n) vs O(n^2)\n", + "\n", + "The single most impactful optimization is choosing a better algorithm. A constant-factor\n", + "micro-optimization cannot save code that has the wrong Big-O complexity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee550055", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "\n", + "def has_duplicates_quadratic(data: list[int]) -> bool:\n", + " \"\"\"O(n^2): Check every pair for duplicates.\"\"\"\n", + " n = len(data)\n", + " for i in range(n):\n", + " for j in range(i + 1, n):\n", + " if data[i] == data[j]:\n", + " return True\n", + " return False\n", + "\n", + "\n", + "def has_duplicates_linear(data: list[int]) -> bool:\n", + " \"\"\"O(n): Use a set for constant-time lookup.\"\"\"\n", + " seen: set[int] = set()\n", + " for item in data:\n", + " if item in seen:\n", + " return True\n", + " seen.add(item)\n", + " return False\n", + "\n", + "\n", + "def has_duplicates_pythonic(data: list[int]) -> bool:\n", + " \"\"\"O(n): Idiomatic Python -- compare len to set.\"\"\"\n", + " return len(data) != len(set(data))\n", + "\n", + "\n", + "# Benchmark with increasing sizes\n", + "print(f\"{'Size':>8} {'O(n^2)':>12} {'O(n) set':>12} {'Pythonic':>12}\")\n", + "print(\"-\" * 48)\n", + "\n", + "for size in [1_000, 5_000, 10_000, 20_000]:\n", + " # No duplicates (worst case for quadratic)\n", + " data = list(range(size))\n", + "\n", + " start = time.perf_counter()\n", + " has_duplicates_quadratic(data)\n", + " t_quad = time.perf_counter() - start\n", + "\n", + " start = time.perf_counter()\n", + " has_duplicates_linear(data)\n", + " t_linear = time.perf_counter() - start\n", + "\n", + " start = time.perf_counter()\n", + " has_duplicates_pythonic(data)\n", + " t_pythonic = time.perf_counter() - start\n", + "\n", + " print(f\"{size:>8,} {t_quad:>11.4f}s {t_linear:>11.6f}s {t_pythonic:>11.6f}s\")\n", + "\n", + "print(\"\\nNote how O(n^2) grows ~4x when n doubles, while O(n) grows ~2x.\")" + ] + }, + { + "cell_type": "markdown", + "id": "ff660066", + "metadata": {}, + "source": [ + "## The `dis` Module: Bytecode Disassembly\n", + "\n", + "Python compiles source code to bytecode, which the CPython virtual machine then executes.\n", + "The `dis` module lets you inspect this bytecode. This is useful for understanding *why*\n", + "one approach is faster than another at the lowest level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa770077", + "metadata": {}, + "outputs": [], + "source": [ + "import dis\n", + "\n", + "\n", + "def add_numbers(a: int, b: int) -> int:\n", + " \"\"\"Simple addition function.\"\"\"\n", + " return a + b\n", + "\n", + "\n", + "def add_with_temp(a: int, b: int) -> int:\n", + " \"\"\"Addition with a temporary variable.\"\"\"\n", + " result = a + b\n", + " return result\n", + "\n", + "\n", + "print(\"Bytecode for add_numbers:\")\n", + "dis.dis(add_numbers)\n", + "\n", + "print(\"\\nBytecode for add_with_temp:\")\n", + "dis.dis(add_with_temp)\n", + "\n", + "print(\"\\nKey instructions:\")\n", + "print(\" LOAD_FAST - Load a local variable (fast, indexed by position)\")\n", + "print(\" BINARY_OP - Perform a binary operation (+, -, *, etc.)\")\n", + "print(\" STORE_FAST - Store to a local variable\")\n", + "print(\" RETURN_VALUE - Return the top of the stack\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb880088", + "metadata": {}, + "outputs": [], + "source": [ + "import dis\n", + "\n", + "# Compare: list comprehension vs manual loop\n", + "def with_comprehension() -> list[int]:\n", + " return [x * 2 for x in range(10)]\n", + "\n", + "def with_loop() -> list[int]:\n", + " result: list[int] = []\n", + " for x in range(10):\n", + " result.append(x * 2)\n", + " return result\n", + "\n", + "print(\"List comprehension bytecode:\")\n", + "dis.dis(with_comprehension)\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"\\nManual loop bytecode:\")\n", + "dis.dis(with_loop)\n", + "\n", + "print(\"\\nThe comprehension uses a specialized BUILD_LIST + LIST_APPEND\")\n", + "print(\"path that avoids the overhead of LOAD_ATTR for .append().\")" + ] + }, + { + "cell_type": "markdown", + "id": "cc990099", + "metadata": {}, + "source": [ + "## Code Object Attributes\n", + "\n", + "Every function has a `__code__` object containing its compiled bytecode and metadata.\n", + "Inspecting these attributes reveals how Python organizes function internals." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd001100", + "metadata": {}, + "outputs": [], + "source": [ + "def example_function(x: int, y: int, z: int = 10) -> int:\n", + " \"\"\"A function with locals, constants, and a default.\"\"\"\n", + " multiplier = 2\n", + " offset = 100\n", + " result = (x + y + z) * multiplier + offset\n", + " return result\n", + "\n", + "\n", + "code = example_function.__code__\n", + "\n", + "print(\"Code object attributes:\")\n", + "print(f\" co_argcount: {code.co_argcount}\") # Number of positional args\n", + "print(f\" co_varnames: {code.co_varnames}\") # Local variable names (args first)\n", + "print(f\" co_consts: {code.co_consts}\") # Constant values used\n", + "print(f\" co_names: {code.co_names}\") # Names used (global lookups)\n", + "print(f\" co_nlocals: {code.co_nlocals}\") # Number of local variables\n", + "print(f\" co_stacksize: {code.co_stacksize}\") # Max stack depth needed\n", + "print(f\" co_filename: {code.co_filename}\") # Source file\n", + "\n", + "# Constants are pre-computed at compile time\n", + "print(f\"\\nConstants include: docstring, default values, and literal numbers\")\n", + "print(f\"Note: co_varnames includes args first, then locals\")" + ] + }, + { + "cell_type": "markdown", + "id": "ee111111", + "metadata": {}, + "source": [ + "## Generator Expressions vs List Comprehensions: Memory\n", + "\n", + "A list comprehension builds the entire list in memory. A generator expression yields\n", + "one item at a time, using near-constant memory regardless of data size. When you only\n", + "need to iterate once, generators are the better choice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff221122", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# List comprehension: builds entire list in memory\n", + "list_comp = [x * x for x in range(1_000_000)]\n", + "\n", + "# Generator expression: lazy, constant memory\n", + "gen_expr = (x * x for x in range(1_000_000))\n", + "\n", + "print(\"Memory comparison (1M items):\")\n", + "print(f\" List comprehension: {sys.getsizeof(list_comp):>12,} bytes\")\n", + "print(f\" Generator expr: {sys.getsizeof(gen_expr):>12,} bytes\")\n", + "\n", + "# Both produce the same result when consumed\n", + "print(f\"\\n sum(list_comp): {sum(list_comp)}\")\n", + "print(f\" sum(gen_expr): {sum(gen_expr)}\")\n", + "\n", + "# Functions like sum(), min(), max(), any(), all() accept iterables\n", + "# so you can feed them generators directly\n", + "import timeit\n", + "\n", + "t_list = timeit.timeit(\"sum([x*x for x in range(10_000)])\", number=1000)\n", + "t_gen = timeit.timeit(\"sum(x*x for x in range(10_000))\", number=1000)\n", + "\n", + "print(f\"\\nsum() with 10k items, 1000 runs:\")\n", + "print(f\" List comp: {t_list:.4f}s\")\n", + "print(f\" Generator: {t_gen:.4f}s\")\n", + "print(\" Generator avoids allocating and freeing a temporary list.\")" + ] + }, + { + "cell_type": "markdown", + "id": "aa331133", + "metadata": {}, + "source": [ + "## Local Variable Access vs Global: LOAD_FAST vs LOAD_GLOBAL\n", + "\n", + "Local variables are stored in a fixed-size array and accessed by index (`LOAD_FAST`).\n", + "Global and built-in lookups require dictionary lookups (`LOAD_GLOBAL`). In tight loops,\n", + "this difference matters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb441144", + "metadata": {}, + "outputs": [], + "source": [ + "import dis\n", + "import timeit\n", + "\n", + "GLOBAL_MULTIPLIER: int = 3\n", + "\n", + "\n", + "def use_global(data: list[int]) -> list[int]:\n", + " \"\"\"Access a global variable in a loop.\"\"\"\n", + " return [x * GLOBAL_MULTIPLIER for x in data]\n", + "\n", + "\n", + "def use_local(data: list[int]) -> list[int]:\n", + " \"\"\"Cache the global as a local variable.\"\"\"\n", + " multiplier = GLOBAL_MULTIPLIER # One LOAD_GLOBAL, then LOAD_FAST\n", + " return [x * multiplier for x in data]\n", + "\n", + "\n", + "# Show the bytecode difference\n", + "print(\"use_global bytecode:\")\n", + "dis.dis(use_global)\n", + "\n", + "print(\"\\nuse_local bytecode:\")\n", + "dis.dis(use_local)\n", + "\n", + "# Benchmark\n", + "data = list(range(100_000))\n", + "t_global = timeit.timeit(lambda: use_global(data), number=100)\n", + "t_local = timeit.timeit(lambda: use_local(data), number=100)\n", + "\n", + "print(f\"\\n100k items, 100 runs:\")\n", + "print(f\" Global lookup: {t_global:.4f}s\")\n", + "print(f\" Local (cached): {t_local:.4f}s\")" + ] + }, + { + "cell_type": "markdown", + "id": "cc551155", + "metadata": {}, + "source": [ + "## Collections and Built-in Optimizations\n", + "\n", + "Python's standard library is written in C and heavily optimized. Using built-in functions\n", + "and `collections` types is almost always faster than reimplementing their logic in pure Python." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd661166", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "from collections import Counter, defaultdict, deque\n", + "\n", + "# Built-in sum() vs manual loop\n", + "setup = \"data = list(range(10_000))\"\n", + "\n", + "t_builtin = timeit.timeit(\"sum(data)\", setup=setup, number=10_000)\n", + "t_manual = timeit.timeit(\n", + " \"total = 0\\nfor x in data:\\n total += x\",\n", + " setup=setup,\n", + " number=10_000,\n", + ")\n", + "\n", + "print(\"Summing 10k ints, 10k runs:\")\n", + "print(f\" Built-in sum(): {t_builtin:.4f}s\")\n", + "print(f\" Manual loop: {t_manual:.4f}s\")\n", + "print(f\" Built-in is {t_manual / t_builtin:.1f}x faster\")\n", + "\n", + "# Counter vs manual counting\n", + "words = [\"apple\", \"banana\", \"cherry\"] * 3000\n", + "\n", + "def manual_count(words: list[str]) -> dict[str, int]:\n", + " counts: dict[str, int] = {}\n", + " for w in words:\n", + " counts[w] = counts.get(w, 0) + 1\n", + " return counts\n", + "\n", + "t_counter = timeit.timeit(lambda: Counter(words), number=1_000)\n", + "t_manual_c = timeit.timeit(lambda: manual_count(words), number=1_000)\n", + "\n", + "print(f\"\\nCounting 9k words, 1k runs:\")\n", + "print(f\" Counter(): {t_counter:.4f}s\")\n", + "print(f\" Manual dict: {t_manual_c:.4f}s\")\n", + "\n", + "# deque.appendleft() vs list.insert(0, x)\n", + "t_deque = timeit.timeit(\n", + " \"d.appendleft(1)\",\n", + " setup=\"from collections import deque; d = deque(range(10000))\",\n", + " number=100_000,\n", + ")\n", + "t_list_ins = timeit.timeit(\n", + " \"lst.insert(0, 1)\",\n", + " setup=\"lst = list(range(10000))\",\n", + " number=100_000,\n", + ")\n", + "\n", + "print(f\"\\nPrepend 100k times to a 10k-item container:\")\n", + "print(f\" deque.appendleft(): {t_deque:.4f}s\")\n", + "print(f\" list.insert(0): {t_list_ins:.4f}s\")" + ] + }, + { + "cell_type": "markdown", + "id": "ee771177", + "metadata": {}, + "source": [ + "## The GIL: Global Interpreter Lock\n", + "\n", + "CPython has a **Global Interpreter Lock (GIL)** that allows only one thread to execute\n", + "Python bytecode at a time. This means:\n", + "\n", + "- **Threading** is useful for I/O-bound work (network, disk) where threads spend most\n", + " time waiting, but **not** for CPU-bound work.\n", + "- **Multiprocessing** spawns separate processes, each with its own GIL, enabling true\n", + " parallelism for CPU-bound tasks.\n", + "\n", + "> **Note**: Python 3.13 introduced an experimental free-threaded build (PEP 703) that\n", + "> can disable the GIL. This is still experimental and not the default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff881188", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import threading\n", + "import multiprocessing\n", + "\n", + "\n", + "def cpu_bound_work(n: int) -> int:\n", + " \"\"\"CPU-intensive computation.\"\"\"\n", + " total = 0\n", + " for i in range(n):\n", + " total += i * i\n", + " return total\n", + "\n", + "\n", + "WORK_SIZE = 5_000_000\n", + "NUM_TASKS = 4\n", + "\n", + "# Sequential execution\n", + "start = time.perf_counter()\n", + "for _ in range(NUM_TASKS):\n", + " cpu_bound_work(WORK_SIZE)\n", + "t_sequential = time.perf_counter() - start\n", + "\n", + "# Threaded execution (GIL limits parallelism for CPU work)\n", + "start = time.perf_counter()\n", + "threads = [threading.Thread(target=cpu_bound_work, args=(WORK_SIZE,)) for _ in range(NUM_TASKS)]\n", + "for t in threads:\n", + " t.start()\n", + "for t in threads:\n", + " t.join()\n", + "t_threaded = time.perf_counter() - start\n", + "\n", + "# Multiprocessing execution (true parallelism)\n", + "start = time.perf_counter()\n", + "with multiprocessing.Pool(NUM_TASKS) as pool:\n", + " pool.map(cpu_bound_work, [WORK_SIZE] * NUM_TASKS)\n", + "t_multiproc = time.perf_counter() - start\n", + "\n", + "print(f\"CPU-bound work: {NUM_TASKS} tasks x {WORK_SIZE:,} iterations\")\n", + "print(f\" Sequential: {t_sequential:.3f}s\")\n", + "print(f\" Threading: {t_threaded:.3f}s (GIL prevents speedup)\")\n", + "print(f\" Multiprocessing: {t_multiproc:.3f}s (true parallelism)\")\n", + "print(f\"\\nThreading speedup: {t_sequential / t_threaded:.2f}x\")\n", + "print(f\"Multiprocessing speedup: {t_sequential / t_multiproc:.2f}x\")" + ] + }, + { + "cell_type": "markdown", + "id": "aa991199", + "metadata": {}, + "source": [ + "## ctypes Overview: Calling C Functions from Python\n", + "\n", + "`ctypes` allows Python to call functions in shared C libraries (`.so` on Linux, `.dylib`\n", + "on macOS, `.dll` on Windows) without writing any C extension code. This is useful for\n", + "performance-critical sections or interfacing with existing C libraries.\n", + "\n", + "This is a **conceptual overview** -- we will call functions from the C standard library\n", + "to demonstrate the mechanism." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb002200", + "metadata": {}, + "outputs": [], + "source": [ + "import ctypes\n", + "import ctypes.util\n", + "import sys\n", + "\n", + "# Load the C standard library\n", + "if sys.platform == \"darwin\":\n", + " libc = ctypes.CDLL(\"libSystem.B.dylib\")\n", + "elif sys.platform == \"linux\":\n", + " libc_path = ctypes.util.find_library(\"c\")\n", + " libc = ctypes.CDLL(libc_path)\n", + "else:\n", + " libc = ctypes.CDLL(\"msvcrt\") # Windows\n", + "\n", + "# Call C's abs() function\n", + "libc.abs.restype = ctypes.c_int\n", + "libc.abs.argtypes = [ctypes.c_int]\n", + "\n", + "result = libc.abs(-42)\n", + "print(f\"C abs(-42) = {result}\")\n", + "\n", + "# Call C's strlen() on a byte string\n", + "libc.strlen.restype = ctypes.c_size_t\n", + "libc.strlen.argtypes = [ctypes.c_char_p]\n", + "\n", + "length = libc.strlen(b\"Hello, ctypes!\")\n", + "print(f\"C strlen('Hello, ctypes!') = {length}\")\n", + "\n", + "# Conceptual: for real performance gains, you would:\n", + "# 1. Write a C function that performs your hot loop\n", + "# 2. Compile it into a shared library\n", + "# 3. Load it with ctypes and call it from Python\n", + "print(\"\\nctypes workflow:\")\n", + "print(\" 1. Write C code -> compile to .so / .dylib / .dll\")\n", + "print(\" 2. ctypes.CDLL('mylib.so')\")\n", + "print(\" 3. Set .restype and .argtypes for type safety\")\n", + "print(\" 4. Call functions directly from Python\")\n", + "print(\"\\nAlternatives: cffi, Cython, pybind11, or the new C API\")" + ] + }, + { + "cell_type": "markdown", + "id": "cc113311", + "metadata": {}, + "source": [ + "## Optimization Guidelines: A Practical Workflow\n", + "\n", + "Follow this order when optimizing Python code:\n", + "\n", + "1. **Write correct code first** -- premature optimization is the root of all evil\n", + "2. **Profile** with `cProfile` to find actual bottlenecks (do not guess)\n", + "3. **Fix algorithmic complexity** -- O(n) vs O(n^2) matters far more than micro-tuning\n", + "4. **Use built-in functions and C-implemented types** (`sum`, `sorted`, `Counter`, `deque`)\n", + "5. **Reduce memory pressure** -- `__slots__`, generators, avoid unnecessary copies\n", + "6. **Micro-optimize hot loops** -- local variable caching, list comprehensions\n", + "7. **Consider C extensions** -- `ctypes`, `Cython`, `pybind11` for truly hot code\n", + "8. **Use multiprocessing** for CPU-bound parallelism (not threading)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd224422", + "metadata": {}, + "outputs": [], + "source": [ + "import timeit\n", + "\n", + "# Putting it all together: optimizing a word frequency counter\n", + "# Step by step, from naive to optimized\n", + "\n", + "sample_text = \"the quick brown fox jumps over the lazy dog \" * 10_000\n", + "\n", + "\n", + "def v1_naive(text: str) -> dict[str, int]:\n", + " \"\"\"Version 1: Naive implementation.\"\"\"\n", + " words = text.split()\n", + " counts: dict[str, int] = {}\n", + " for word in words:\n", + " if word in counts:\n", + " counts[word] = counts[word] + 1\n", + " else:\n", + " counts[word] = 1\n", + " return counts\n", + "\n", + "\n", + "def v2_get_default(text: str) -> dict[str, int]:\n", + " \"\"\"Version 2: Use dict.get() with default.\"\"\"\n", + " words = text.split()\n", + " counts: dict[str, int] = {}\n", + " for word in words:\n", + " counts[word] = counts.get(word, 0) + 1\n", + " return counts\n", + "\n", + "\n", + "def v3_defaultdict(text: str) -> dict[str, int]:\n", + " \"\"\"Version 3: Use defaultdict(int).\"\"\"\n", + " from collections import defaultdict\n", + " words = text.split()\n", + " counts: defaultdict[str, int] = defaultdict(int)\n", + " for word in words:\n", + " counts[word] += 1\n", + " return dict(counts)\n", + "\n", + "\n", + "def v4_counter(text: str) -> dict[str, int]:\n", + " \"\"\"Version 4: Use Counter (C-optimized).\"\"\"\n", + " from collections import Counter\n", + " return dict(Counter(text.split()))\n", + "\n", + "\n", + "# Benchmark all versions\n", + "versions = [\n", + " (\"v1: naive if/else\", v1_naive),\n", + " (\"v2: dict.get()\", v2_get_default),\n", + " (\"v3: defaultdict\", v3_defaultdict),\n", + " (\"v4: Counter\", v4_counter),\n", + "]\n", + "\n", + "print(f\"Word frequency counting: {len(sample_text.split()):,} words\")\n", + "print(\"=\" * 45)\n", + "\n", + "for name, func in versions:\n", + " t = timeit.timeit(lambda f=func: f(sample_text), number=50)\n", + " print(f\" {name:<22s} {t:.4f}s (50 runs)\")\n", + "\n", + "# Verify all produce the same result\n", + "results = [func(sample_text) for _, func in versions]\n", + "print(f\"\\nAll versions agree: {all(r == results[0] for r in results)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ee335533", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Optimization Hierarchy (Most to Least Impact)\n", + "\n", + "| Priority | Technique | Typical Speedup |\n", + "|----------|-----------|----------------|\n", + "| 1 | **Algorithm choice** (O(n) vs O(n^2)) | 10x - 1000x+ |\n", + "| 2 | **Use C-implemented built-ins** (sum, sorted, Counter) | 2x - 10x |\n", + "| 3 | **Reduce memory** (__slots__, generators) | 30-50% memory savings |\n", + "| 4 | **Micro-optimize** (local vars, comprehensions) | 5-30% |\n", + "| 5 | **C extensions** (ctypes, Cython) | 10x - 100x for hot loops |\n", + "| 6 | **Parallelism** (multiprocessing for CPU-bound) | Up to Nx on N cores |\n", + "\n", + "### Key Tools\n", + "- **`dis.dis()`**: See exactly what bytecode Python generates\n", + "- **`__code__` attributes**: Inspect function internals\n", + "- **`sys.getsizeof()`**: Measure memory of individual objects\n", + "- **`__slots__`**: Eliminate per-instance `__dict__` overhead\n", + "- **`ctypes`**: Call C functions without writing C extensions\n", + "\n", + "### The Golden Rule\n", + "**\"Make it work, make it right, make it fast -- in that order.\"**\n", + "\n", + "Always profile before optimizing. The bottleneck is almost never where you think it is." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_20/README.md b/src/chapter_20/README.md new file mode 100644 index 0000000..5734cab --- /dev/null +++ b/src/chapter_20/README.md @@ -0,0 +1,22 @@ +# Chapter 20: Python Internals and Performance + +## Topics Covered +- CPython internals: reference counting, garbage collection +- The GIL (Global Interpreter Lock): what it is, when it matters +- Memory model: `id()`, `is` vs `==`, interning, `sys.getsizeof()` +- `__slots__` for memory optimization +- Profiling: `cProfile`, `timeit`, `line_profiler` concepts +- Optimization techniques: algorithmic improvements, caching, lazy evaluation +- `dis` module: bytecode disassembly +- C extensions overview: `ctypes`, `cffi`, Cython concepts + +## Notebooks +1. **01_memory_and_gc.ipynb** — Reference counting, garbage collection, memory inspection +2. **02_profiling_and_timeit.ipynb** — cProfile, timeit, identifying bottlenecks +3. **03_optimization_techniques.ipynb** — Algorithmic improvements, bytecode, C extensions + +## Key Takeaways +- Profile before optimizing — measure, don't guess +- The GIL affects CPU-bound threads but not I/O-bound or multiprocessing +- `__slots__` and generators reduce memory usage significantly +- Algorithmic improvements beat micro-optimizations every time diff --git a/src/chapter_20/__init__.py b/src/chapter_20/__init__.py new file mode 100644 index 0000000..55ba304 --- /dev/null +++ b/src/chapter_20/__init__.py @@ -0,0 +1 @@ +"""Chapter 20: Python Internals and Performance.""" diff --git a/tests/test_chapter_16.py b/tests/test_chapter_16.py new file mode 100644 index 0000000..7973f9d --- /dev/null +++ b/tests/test_chapter_16.py @@ -0,0 +1,129 @@ +"""Tests for Chapter 16: Type Hints and Static Analysis.""" + +from typing import ( + Generic, + Literal, + Protocol, + TypeVar, + get_type_hints, + overload, + runtime_checkable, +) + + +class TestAnnotationFundamentals: + """Test basic type annotation patterns.""" + + def test_function_annotations(self) -> None: + """Functions store type hints in __annotations__.""" + + def greet(name: str, times: int = 1) -> str: + return (name + "! ") * times + + hints = get_type_hints(greet) + assert hints["name"] is str + assert hints["times"] is int + assert hints["return"] is str + assert greet("Hello", 2) == "Hello! Hello! " + + def test_variable_annotations(self) -> None: + """Variable annotations are stored in the class/module.""" + x: int = 42 + y: str = "hello" + assert isinstance(x, int) + assert isinstance(y, str) + + def test_generic_types(self) -> None: + """Built-in generics allow parameterized types.""" + numbers: list[int] = [1, 2, 3] + mapping: dict[str, int] = {"a": 1, "b": 2} + pair: tuple[str, int] = ("hello", 42) + + assert numbers == [1, 2, 3] + assert mapping["a"] == 1 + assert pair == ("hello", 42) + + +class TestAdvancedTyping: + """Test advanced typing features.""" + + def test_typevar_generic_function(self) -> None: + """TypeVar enables generic functions.""" + T = TypeVar("T") + + def first(items: list[T]) -> T: + return items[0] + + assert first([1, 2, 3]) == 1 + assert first(["a", "b"]) == "a" + + def test_generic_class(self) -> None: + """Generic classes are parameterized with TypeVar.""" + T = TypeVar("T") + + class Stack(Generic[T]): + def __init__(self) -> None: + self._items: list[T] = [] + + def push(self, item: T) -> None: + self._items.append(item) + + def pop(self) -> T: + return self._items.pop() + + def is_empty(self) -> bool: + return len(self._items) == 0 + + stack: Stack[int] = Stack() + stack.push(1) + stack.push(2) + assert stack.pop() == 2 + assert stack.pop() == 1 + assert stack.is_empty() + + def test_protocol_structural_typing(self) -> None: + """Protocol enables structural subtyping.""" + + @runtime_checkable + class Drawable(Protocol): + def draw(self) -> str: ... + + class Circle: + def draw(self) -> str: + return "Drawing circle" + + class Square: + def draw(self) -> str: + return "Drawing square" + + def render(shape: Drawable) -> str: + return shape.draw() + + assert render(Circle()) == "Drawing circle" + assert render(Square()) == "Drawing square" + assert isinstance(Circle(), Drawable) + + def test_overload_signatures(self) -> None: + """@overload documents multiple call signatures.""" + + @overload + def process(data: str) -> list[str]: ... + @overload + def process(data: list[str]) -> str: ... + + def process(data: str | list[str]) -> list[str] | str: + if isinstance(data, str): + return data.split() + return " ".join(data) + + assert process("hello world") == ["hello", "world"] + assert process(["hello", "world"]) == "hello world" + + def test_literal_type(self) -> None: + """Literal restricts values to specific constants.""" + + def set_color(color: Literal["red", "green", "blue"]) -> str: + return f"Color set to {color}" + + assert set_color("red") == "Color set to red" + assert set_color("blue") == "Color set to blue" diff --git a/tests/test_chapter_17.py b/tests/test_chapter_17.py new file mode 100644 index 0000000..53565b2 --- /dev/null +++ b/tests/test_chapter_17.py @@ -0,0 +1,125 @@ +"""Tests for Chapter 17: Networking and Protocols.""" + +import json +import socket +import threading +import urllib.parse + + +class TestSocketFundamentals: + """Test socket basics.""" + + def test_tcp_echo_server(self) -> None: + """TCP client-server communication works.""" + received: list[bytes] = [] + + def echo_server(server_sock: socket.socket) -> None: + conn, _ = server_sock.accept() + data = conn.recv(1024) + received.append(data) + conn.sendall(data) + conn.close() + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port = server.getsockname()[1] + server.listen(1) + + t = threading.Thread(target=echo_server, args=(server,)) + t.start() + + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.connect(("127.0.0.1", port)) + client.sendall(b"Hello, TCP!") + response = client.recv(1024) + client.close() + server.close() + t.join() + + assert response == b"Hello, TCP!" + assert received[0] == b"Hello, TCP!" + + def test_udp_communication(self) -> None: + """UDP send and receive works.""" + received: list[bytes] = [] + + def udp_receiver(sock: socket.socket) -> None: + data, _ = sock.recvfrom(1024) + received.append(data) + + server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + server.bind(("127.0.0.1", 0)) + port = server.getsockname()[1] + + t = threading.Thread(target=udp_receiver, args=(server,)) + t.start() + + client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + client.sendto(b"Hello, UDP!", ("127.0.0.1", port)) + client.close() + t.join() + server.close() + + assert received[0] == b"Hello, UDP!" + + def test_socket_address_info(self) -> None: + """getaddrinfo resolves host and service names.""" + results = socket.getaddrinfo("localhost", None, socket.AF_INET) + assert len(results) > 0 + assert results[0][0] == socket.AF_INET + + +class TestURLHandling: + """Test URL parsing and construction.""" + + def test_url_parsing(self) -> None: + """urllib.parse breaks URLs into components.""" + url = "https://example.com:8080/path?key=value&q=test#section" + parsed = urllib.parse.urlparse(url) + + assert parsed.scheme == "https" + assert parsed.hostname == "example.com" + assert parsed.port == 8080 + assert parsed.path == "/path" + assert parsed.fragment == "section" + + def test_query_string_parsing(self) -> None: + """parse_qs extracts query parameters.""" + query = "name=Alice&age=30&tag=python&tag=coding" + params = urllib.parse.parse_qs(query) + + assert params["name"] == ["Alice"] + assert params["age"] == ["30"] + assert params["tag"] == ["python", "coding"] + + def test_url_encoding(self) -> None: + """urlencode builds query strings from dicts.""" + params = {"name": "Alice Bob", "city": "New York"} + encoded = urllib.parse.urlencode(params) + assert "name=Alice+Bob" in encoded + assert "city=New+York" in encoded + + +class TestJSONProtocol: + """Test JSON-based communication patterns.""" + + def test_json_request_response(self) -> None: + """JSON serialization for network messages.""" + request = {"action": "greet", "name": "Alice"} + encoded: bytes = json.dumps(request).encode("utf-8") + + decoded: dict = json.loads(encoded.decode("utf-8")) + assert decoded == request + assert decoded["action"] == "greet" + + def test_json_message_framing(self) -> None: + """Length-prefixed framing for JSON messages.""" + message = {"type": "chat", "text": "Hello!"} + payload: bytes = json.dumps(message).encode("utf-8") + frame: bytes = len(payload).to_bytes(4, "big") + payload + + # Parse the frame + length = int.from_bytes(frame[:4], "big") + body = json.loads(frame[4 : 4 + length].decode("utf-8")) + assert body == message diff --git a/tests/test_chapter_18.py b/tests/test_chapter_18.py new file mode 100644 index 0000000..6771751 --- /dev/null +++ b/tests/test_chapter_18.py @@ -0,0 +1,117 @@ +"""Tests for Chapter 18: Database Access.""" + +import sqlite3 + + +class TestSQLiteFundamentals: + """Test sqlite3 basics.""" + + def test_create_and_query(self) -> None: + """Create a table and query it.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, age INTEGER)") + conn.execute("INSERT INTO users (name, age) VALUES (?, ?)", ("Alice", 30)) + conn.execute("INSERT INTO users (name, age) VALUES (?, ?)", ("Bob", 25)) + + rows = conn.execute("SELECT name, age FROM users ORDER BY name").fetchall() + assert rows == [("Alice", 30), ("Bob", 25)] + conn.close() + + def test_parameterized_queries(self) -> None: + """Parameterized queries prevent SQL injection.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)") + conn.execute("INSERT INTO items (name) VALUES (?)", ("Widget",)) + + # Parameterized query safely handles special characters + malicious = "'; DROP TABLE items; --" + conn.execute("INSERT INTO items (name) VALUES (?)", (malicious,)) + rows = conn.execute("SELECT name FROM items").fetchall() + assert len(rows) == 2 # Table still exists with both rows + + conn.close() + + def test_executemany_batch_insert(self) -> None: + """executemany efficiently inserts multiple rows.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE numbers (value INTEGER)") + + data = [(i,) for i in range(100)] + conn.executemany("INSERT INTO numbers (value) VALUES (?)", data) + + count = conn.execute("SELECT COUNT(*) FROM numbers").fetchone()[0] + assert count == 100 + conn.close() + + +class TestTransactions: + """Test transaction handling.""" + + def test_commit_persists_data(self) -> None: + """Committed data persists.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE t (val TEXT)") + conn.execute("INSERT INTO t (val) VALUES (?)", ("kept",)) + conn.commit() + + rows = conn.execute("SELECT val FROM t").fetchall() + assert rows == [("kept",)] + conn.close() + + def test_rollback_discards_changes(self) -> None: + """Rollback discards uncommitted changes.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE t (val TEXT)") + conn.execute("INSERT INTO t (val) VALUES (?)", ("initial",)) + conn.commit() + + conn.execute("INSERT INTO t (val) VALUES (?)", ("discarded",)) + conn.rollback() + + rows = conn.execute("SELECT val FROM t").fetchall() + assert rows == [("initial",)] + conn.close() + + def test_context_manager_auto_commit(self) -> None: + """Connection as context manager auto-commits on success.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE t (val TEXT)") + + with conn: + conn.execute("INSERT INTO t (val) VALUES (?)", ("auto-committed",)) + + rows = conn.execute("SELECT val FROM t").fetchall() + assert rows == [("auto-committed",)] + conn.close() + + +class TestRowFactories: + """Test custom row factories.""" + + def test_row_factory_dict(self) -> None: + """sqlite3.Row provides dict-like access.""" + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, age INTEGER)") + conn.execute("INSERT INTO users (name, age) VALUES (?, ?)", ("Alice", 30)) + + row = conn.execute("SELECT * FROM users").fetchone() + assert row["name"] == "Alice" + assert row["age"] == 30 + assert list(row.keys()) == ["id", "name", "age"] + conn.close() + + def test_custom_row_factory(self) -> None: + """Custom row factories transform query results.""" + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE points (x REAL, y REAL)") + conn.execute("INSERT INTO points (x, y) VALUES (?, ?)", (1.0, 2.0)) + conn.execute("INSERT INTO points (x, y) VALUES (?, ?)", (3.0, 4.0)) + + def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict: + return {col[0]: value for col, value in zip(cursor.description, row)} + + conn.row_factory = dict_factory + rows = conn.execute("SELECT x, y FROM points ORDER BY x").fetchall() + assert rows == [{"x": 1.0, "y": 2.0}, {"x": 3.0, "y": 4.0}] + conn.close() diff --git a/tests/test_chapter_19.py b/tests/test_chapter_19.py new file mode 100644 index 0000000..b14a509 --- /dev/null +++ b/tests/test_chapter_19.py @@ -0,0 +1,72 @@ +"""Tests for Chapter 19: Packaging and Distribution.""" + +import importlib.metadata +import sys +import sysconfig +import venv +from pathlib import Path + + +class TestVirtualEnvironments: + """Test virtual environment concepts.""" + + def test_sys_prefix_identifies_environment(self) -> None: + """sys.prefix points to the active environment.""" + assert Path(sys.prefix).exists() + assert Path(sys.executable).exists() + + def test_venv_builder_exists(self) -> None: + """venv module provides EnvBuilder for creating environments.""" + builder = venv.EnvBuilder(with_pip=False) + assert hasattr(builder, "create") + + def test_sysconfig_paths(self) -> None: + """sysconfig provides installation paths.""" + paths = sysconfig.get_paths() + assert "purelib" in paths + assert "scripts" in paths + assert "include" in paths + + +class TestPackageMetadata: + """Test package metadata access.""" + + def test_installed_package_version(self) -> None: + """importlib.metadata reads installed package info.""" + version = importlib.metadata.version("pip") + assert version is not None + parts = version.split(".") + assert len(parts) >= 2 # Major.minor at minimum + + def test_package_metadata_fields(self) -> None: + """Package metadata contains standard fields.""" + meta = importlib.metadata.metadata("pip") + assert meta["Name"] == "pip" + assert "Version" in meta + assert "Summary" in meta + + def test_entry_points(self) -> None: + """Entry points expose console scripts and plugins.""" + eps = importlib.metadata.entry_points() + # entry_points() returns a SelectableGroups or dict-like + assert eps is not None + + +class TestVersionParsing: + """Test version string handling.""" + + def test_version_comparison(self) -> None: + """Version strings can be compared with packaging conventions.""" + from importlib.metadata import version + + pip_version = version("pip") + # Version string is parseable + parts = pip_version.split(".") + major = int(parts[0]) + assert major >= 1 + + def test_python_version(self) -> None: + """sys.version_info provides structured version info.""" + assert sys.version_info.major == 3 + assert sys.version_info.minor >= 12 + assert isinstance(sys.version_info.releaselevel, str) diff --git a/tests/test_chapter_20.py b/tests/test_chapter_20.py new file mode 100644 index 0000000..c4d0bef --- /dev/null +++ b/tests/test_chapter_20.py @@ -0,0 +1,157 @@ +"""Tests for Chapter 20: Python Internals and Performance.""" + +import dis +import gc +import sys +import weakref +from io import StringIO + + +class TestMemoryModel: + """Test Python memory model concepts.""" + + def test_id_and_identity(self) -> None: + """id() returns unique identity; 'is' checks identity.""" + a = [1, 2, 3] + b = a + c = [1, 2, 3] + + assert a is b # Same object + assert a is not c # Different objects + assert a == c # Equal values + assert id(a) == id(b) + assert id(a) != id(c) + + def test_small_integer_interning(self) -> None: + """CPython interns small integers (-5 to 256).""" + a = 256 + b = 256 + assert a is b # Interned + + def test_sys_getsizeof(self) -> None: + """sys.getsizeof returns object memory in bytes.""" + empty_list = [] + small_list = [1, 2, 3] + assert sys.getsizeof(empty_list) < sys.getsizeof(small_list) + + empty_dict: dict = {} + assert sys.getsizeof(empty_dict) > 0 + + def test_sys_getrefcount(self) -> None: + """sys.getrefcount shows reference count (includes the call arg).""" + a = [1, 2, 3] + count = sys.getrefcount(a) + assert count >= 2 # 'a' + argument to getrefcount + + +class TestGarbageCollection: + """Test garbage collection behavior.""" + + def test_reference_counting(self) -> None: + """Objects are freed when reference count reaches zero.""" + deleted: list[bool] = [] + + class Tracked: + def __del__(self) -> None: + deleted.append(True) + + obj = Tracked() + assert len(deleted) == 0 + del obj + # After del, __del__ should have been called + assert len(deleted) == 1 + + def test_weakref_doesnt_prevent_gc(self) -> None: + """Weak references don't prevent garbage collection.""" + + class MyObj: + pass + + obj = MyObj() + ref = weakref.ref(obj) + assert ref() is obj + + del obj + gc.collect() + assert ref() is None # Object was collected + + def test_gc_cycle_detection(self) -> None: + """gc module detects and collects reference cycles.""" + + class Node: + def __init__(self) -> None: + self.ref: "Node | None" = None + + a = Node() + b = Node() + a.ref = b + b.ref = a # Circular reference + + del a, b + collected = gc.collect() + assert collected >= 0 # gc handled the cycle + + +class TestSlotsOptimization: + """Test __slots__ for memory efficiency.""" + + def test_slots_restricts_attributes(self) -> None: + """__slots__ restricts which attributes can be set.""" + + class Point: + __slots__ = ("x", "y") + + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y + + p = Point(1.0, 2.0) + assert p.x == 1.0 + assert not hasattr(p, "__dict__") + + def test_slots_saves_memory(self) -> None: + """Slotted classes use less memory than regular classes.""" + + class Regular: + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y + + class Slotted: + __slots__ = ("x", "y") + + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y + + regular = Regular(1.0, 2.0) + slotted = Slotted(1.0, 2.0) + assert sys.getsizeof(slotted) <= sys.getsizeof(regular) + + +class TestBytecodeInspection: + """Test dis module for bytecode inspection.""" + + def test_dis_produces_output(self) -> None: + """dis.dis disassembles functions to bytecode.""" + + def add(a: int, b: int) -> int: + return a + b + + output = StringIO() + dis.dis(add, file=output) + bytecode = output.getvalue() + assert "RETURN_VALUE" in bytecode + + def test_code_object_attributes(self) -> None: + """Function code objects expose bytecode metadata.""" + + def example(x: int, y: int) -> int: + z = x + y + return z * 2 + + code = example.__code__ + assert code.co_argcount == 2 + assert "x" in code.co_varnames + assert "y" in code.co_varnames + assert "z" in code.co_varnames