diff --git a/README.md b/README.md index de3c1e7..2d9dbb9 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,11 @@ src/ ├── chapter_18/ # Database Access ├── chapter_19/ # Packaging and Distribution ├── chapter_20/ # Python Internals and Performance +├── chapter_21/ # Logging and Debugging +├── chapter_22/ # Web Development Fundamentals +├── chapter_23/ # Security and Cryptography +├── chapter_24/ # Metaprogramming +├── chapter_25/ # Python Ecosystem and Best Practices └── ... tests/ ├── conftest.py # Shared pytest fixtures @@ -116,6 +121,11 @@ tests/ | 18 | Database Access | sqlite3, DB-API, transactions, row factories | Done | | 19 | Packaging and Distribution | venv, pyproject.toml, wheels, entry points, PyPI | Done | | 20 | Python Internals and Performance | Memory model, GIL, profiling, bytecode, optimization | Done | +| 21 | Logging and Debugging | logging module, handlers, formatters, pdb, traceback | Done | +| 22 | Web Development Fundamentals | WSGI, HTTP servers, routing, templates, REST APIs | Done | +| 23 | Security and Cryptography | hashlib, hmac, secrets, input validation, secure coding | Done | +| 24 | Metaprogramming | Dynamic attributes, class decorators, introspection, codegen | Done | +| 25 | Python Ecosystem and Best Practices | Code style, linting, testing patterns, CI/CD, project org | Done | ## Running Notebooks diff --git a/src/chapter_21/01_logging_fundamentals.ipynb b/src/chapter_21/01_logging_fundamentals.ipynb new file mode 100644 index 0000000..91e5760 --- /dev/null +++ b/src/chapter_21/01_logging_fundamentals.ipynb @@ -0,0 +1,597 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 21: Logging Fundamentals\n", + "\n", + "The `logging` module is Python's built-in framework for emitting structured, leveled log\n", + "messages. It provides far more control and flexibility than `print()` statements, and is\n", + "the standard way to record what your program is doing at runtime.\n", + "\n", + "## Topics Covered\n", + "- **Why logging over print()**: Advantages of structured logging\n", + "- **Log levels**: DEBUG, INFO, WARNING, ERROR, CRITICAL with numeric values\n", + "- **Loggers**: Creating loggers with `getLogger()`, setting levels\n", + "- **Handlers**: `StreamHandler` and basic handler setup\n", + "- **Formatters**: Format strings and common placeholders\n", + "- **basicConfig()**: Quick setup for simple use cases\n", + "- **LogRecord attributes**: How log records flow through the system\n", + "- **Practical**: Setting up logging for an application module" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Why Logging Over print()\n", + "\n", + "Using `print()` for debugging and monitoring has significant limitations:\n", + "\n", + "| Feature | `print()` | `logging` |\n", + "|---------|-----------|----------|\n", + "| Severity levels | No | DEBUG, INFO, WARNING, ERROR, CRITICAL |\n", + "| Output destination | stdout only | Console, files, network, email, etc. |\n", + "| Enable/disable | Must remove/comment code | Change level or disable handler |\n", + "| Timestamps | Manual | Built-in with `%(asctime)s` |\n", + "| Source tracking | Manual | Automatic module/line number |\n", + "| Thread safety | No | Yes |\n", + "| Configuration | Hardcoded | Runtime-configurable |\n", + "\n", + "The `logging` module lets you leave diagnostic code in your program permanently and control\n", + "its verbosity through configuration rather than code changes." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# The problem with print() debugging\n", + "def calculate_discount(price: float, discount_pct: float) -> float:\n", + " \"\"\"Calculate discounted price -- print version.\"\"\"\n", + " print(f\"DEBUG: price={price}, discount_pct={discount_pct}\") # Must remove later\n", + " if discount_pct < 0 or discount_pct > 100:\n", + " print(f\"WARNING: Invalid discount {discount_pct}\") # No severity level\n", + " return price\n", + " result = price * (1 - discount_pct / 100)\n", + " print(f\"DEBUG: result={result}\") # Clutters output\n", + " return result\n", + "\n", + "\n", + "print(calculate_discount(100.0, 20.0))\n", + "print(calculate_discount(50.0, -5.0))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Log Levels: Severity Classification\n", + "\n", + "The logging module defines five standard levels, each with a numeric value. Messages are\n", + "only emitted if their level meets or exceeds the logger's configured threshold.\n", + "\n", + "| Level | Numeric Value | Purpose |\n", + "|-------|--------------|--------|\n", + "| `DEBUG` | 10 | Detailed diagnostic information |\n", + "| `INFO` | 20 | Confirmation that things are working |\n", + "| `WARNING` | 30 | Something unexpected but not an error (default level) |\n", + "| `ERROR` | 40 | A serious problem; some functionality failed |\n", + "| `CRITICAL` | 50 | A very serious error; the program may not continue |" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "\n", + "# Numeric values of each level\n", + "levels: list[tuple[str, int]] = [\n", + " (\"DEBUG\", logging.DEBUG),\n", + " (\"INFO\", logging.INFO),\n", + " (\"WARNING\", logging.WARNING),\n", + " (\"ERROR\", logging.ERROR),\n", + " (\"CRITICAL\", logging.CRITICAL),\n", + "]\n", + "\n", + "for name, value in levels:\n", + " print(f\"{name:>10} = {value}\")\n", + "\n", + "# You can also look up level names and values\n", + "print(f\"\\nLevel name for 30: {logging.getLevelName(30)}\")\n", + "print(f\"Level value for 'ERROR': {logging.getLevelName('ERROR')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating Loggers with getLogger()\n", + "\n", + "`logging.getLogger(name)` returns a logger with the given name, creating it if necessary.\n", + "Calling it again with the same name always returns the **same** logger instance (singleton\n", + "pattern). The convention is to use `__name__` as the logger name, which gives you the\n", + "module's fully qualified name.\n", + "\n", + "The **root logger** is obtained by calling `getLogger()` with no arguments. All other loggers\n", + "are descendants of the root logger." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "\n", + "# Create a named logger (convention: use __name__)\n", + "logger = logging.getLogger(\"myapp.utils\")\n", + "\n", + "# getLogger with the same name returns the same instance\n", + "same_logger = logging.getLogger(\"myapp.utils\")\n", + "print(f\"Same instance: {logger is same_logger}\") # True\n", + "\n", + "# The root logger\n", + "root = logging.getLogger()\n", + "print(f\"Root logger name: {root.name!r}\")\n", + "print(f\"Root logger level: {root.level} ({logging.getLevelName(root.level)})\")\n", + "\n", + "# Default level for new loggers is NOTSET (0), which defers to parent\n", + "print(f\"\\nLogger name: {logger.name!r}\")\n", + "print(f\"Logger level: {logger.level} ({logging.getLevelName(logger.level)})\")\n", + "print(f\"Effective level: {logger.getEffectiveLevel()} \"\n", + " f\"({logging.getLevelName(logger.getEffectiveLevel())})\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "\n", + "# Setting the level on a logger controls which messages pass through\n", + "logger = logging.getLogger(\"myapp.demo\")\n", + "logger.setLevel(logging.DEBUG)\n", + "\n", + "# We need a handler to see output -- add a StreamHandler\n", + "handler = logging.StreamHandler()\n", + "handler.setLevel(logging.DEBUG)\n", + "logger.addHandler(handler)\n", + "\n", + "# All five levels\n", + "logger.debug(\"This is a DEBUG message\")\n", + "logger.info(\"This is an INFO message\")\n", + "logger.warning(\"This is a WARNING message\")\n", + "logger.error(\"This is an ERROR message\")\n", + "logger.critical(\"This is a CRITICAL message\")\n", + "\n", + "# Now raise the threshold to WARNING\n", + "print(\"\\n--- After setting level to WARNING ---\")\n", + "logger.setLevel(logging.WARNING)\n", + "logger.debug(\"This DEBUG will NOT appear\")\n", + "logger.info(\"This INFO will NOT appear\")\n", + "logger.warning(\"This WARNING will appear\")\n", + "logger.error(\"This ERROR will appear\")\n", + "\n", + "# Clean up: remove handler to avoid duplicate output in later cells\n", + "logger.removeHandler(handler)\n", + "logger.setLevel(logging.NOTSET)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Handlers: Directing Log Output\n", + "\n", + "**Handlers** determine where log records go. A logger can have multiple handlers, each\n", + "directing records to a different destination with its own level threshold.\n", + "\n", + "Common handler types:\n", + "- `StreamHandler` -- writes to `sys.stderr` (default) or any file-like object\n", + "- `FileHandler` -- writes to a file on disk\n", + "- `RotatingFileHandler` -- file handler with automatic rotation\n", + "- `SocketHandler`, `SMTPHandler`, `HTTPHandler` -- network-based handlers\n", + "- `NullHandler` -- discards all records (used in libraries)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "# Create a logger with a StreamHandler writing to stdout\n", + "logger = logging.getLogger(\"myapp.handlers\")\n", + "logger.setLevel(logging.DEBUG)\n", + "\n", + "# StreamHandler to stdout (default is stderr)\n", + "console_handler = logging.StreamHandler(sys.stdout)\n", + "console_handler.setLevel(logging.INFO) # Only INFO and above on console\n", + "\n", + "logger.addHandler(console_handler)\n", + "\n", + "# DEBUG won't appear (below handler's threshold)\n", + "logger.debug(\"Debug details -- will not appear on console\")\n", + "\n", + "# INFO and above will appear\n", + "logger.info(\"Application started\")\n", + "logger.warning(\"Disk space low\")\n", + "logger.error(\"Connection failed\")\n", + "\n", + "# Each handler has its own level -- the logger level is the first gate,\n", + "# then each handler applies its own level filter\n", + "print(f\"\\nLogger level: {logging.getLevelName(logger.level)}\")\n", + "print(f\"Handler level: {logging.getLevelName(console_handler.level)}\")\n", + "\n", + "# Clean up\n", + "logger.removeHandler(console_handler)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Formatters: Controlling Log Output Format\n", + "\n", + "**Formatters** define the layout of each log message. You attach a formatter to a handler\n", + "using `handler.setFormatter(formatter)`.\n", + "\n", + "Common format placeholders:\n", + "\n", + "| Placeholder | Description |\n", + "|-------------|-------------|\n", + "| `%(levelname)s` | Level name (DEBUG, INFO, etc.) |\n", + "| `%(name)s` | Logger name |\n", + "| `%(message)s` | The log message |\n", + "| `%(asctime)s` | Timestamp |\n", + "| `%(filename)s` | Source file name |\n", + "| `%(lineno)d` | Line number |\n", + "| `%(funcName)s` | Function name |\n", + "| `%(module)s` | Module name |\n", + "| `%(pathname)s` | Full file path |\n", + "| `%(process)d` | Process ID |\n", + "| `%(thread)d` | Thread ID |" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "logger = logging.getLogger(\"myapp.formatters\")\n", + "logger.setLevel(logging.DEBUG)\n", + "\n", + "# Create a handler with a detailed formatter\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setLevel(logging.DEBUG)\n", + "\n", + "# Format: timestamp - level - logger name - message\n", + "formatter = logging.Formatter(\n", + " fmt=\"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s\",\n", + " datefmt=\"%Y-%m-%d %H:%M:%S\"\n", + ")\n", + "handler.setFormatter(formatter)\n", + "logger.addHandler(handler)\n", + "\n", + "logger.debug(\"Initializing database connection\")\n", + "logger.info(\"Server started on port 8080\")\n", + "logger.warning(\"Cache miss for key 'user:42'\")\n", + "logger.error(\"Failed to write to /var/log/app.log\")\n", + "\n", + "# Switch to a minimal formatter\n", + "print(\"\\n--- Minimal format ---\")\n", + "minimal_formatter = logging.Formatter(\"%(levelname)s: %(message)s\")\n", + "handler.setFormatter(minimal_formatter)\n", + "\n", + "logger.info(\"Same handler, different format\")\n", + "logger.error(\"Errors are now more concise\")\n", + "\n", + "# Clean up\n", + "logger.removeHandler(handler)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## basicConfig(): Quick Setup\n", + "\n", + "`logging.basicConfig()` provides a convenient one-call setup for the **root logger**.\n", + "It creates a `StreamHandler` with a default formatter and attaches it to the root logger.\n", + "\n", + "**Important**: `basicConfig()` only works if the root logger has no handlers yet. Calling\n", + "it a second time has no effect unless you pass `force=True` (Python 3.8+)." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "\n", + "# Reset root logger for this demo\n", + "root = logging.getLogger()\n", + "for h in root.handlers[:]:\n", + " root.removeHandler(h)\n", + "\n", + "# basicConfig with common options\n", + "logging.basicConfig(\n", + " level=logging.DEBUG,\n", + " format=\"%(asctime)s [%(levelname)s] %(name)s: %(message)s\",\n", + " datefmt=\"%H:%M:%S\",\n", + " force=True, # Override any existing configuration\n", + ")\n", + "\n", + "# Now any logger will output through the root's handler\n", + "logging.debug(\"Root logger debug message\")\n", + "logging.info(\"Root logger info message\")\n", + "logging.warning(\"Root logger warning message\")\n", + "\n", + "# Named loggers also propagate to the root by default\n", + "app_logger = logging.getLogger(\"myapp\")\n", + "app_logger.info(\"Message from myapp logger (propagated to root)\")\n", + "\n", + "# Clean up root logger\n", + "for h in root.handlers[:]:\n", + " root.removeHandler(h)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LogRecord Attributes and the Logging Flow\n", + "\n", + "When you call `logger.info(\"message\")`, the following sequence occurs:\n", + "\n", + "1. The logger checks if the message's level meets the logger's threshold\n", + "2. A `LogRecord` object is created containing all metadata\n", + "3. Logger filters are applied (if any)\n", + "4. The record is passed to each handler on this logger\n", + "5. Each handler checks its own level threshold and filters\n", + "6. The handler's formatter converts the `LogRecord` to a string\n", + "7. The handler emits the formatted string to its destination\n", + "8. If `propagate=True` (default), the record is also passed to parent loggers\n", + "\n", + "A `LogRecord` carries rich metadata that formatters and filters can use." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "\n", + "# Create a LogRecord manually to inspect its attributes\n", + "record: logging.LogRecord = logging.LogRecord(\n", + " name=\"myapp.module\",\n", + " level=logging.WARNING,\n", + " pathname=\"/app/myapp/module.py\",\n", + " lineno=42,\n", + " msg=\"Connection to %s timed out after %d seconds\",\n", + " args=(\"db.example.com\", 30),\n", + " exc_info=None,\n", + ")\n", + "\n", + "# Key attributes on every LogRecord\n", + "attributes: list[str] = [\n", + " \"name\", \"levelname\", \"levelno\", \"pathname\", \"filename\",\n", + " \"module\", \"lineno\", \"funcName\", \"created\", \"msecs\",\n", + " \"relativeCreated\", \"thread\", \"threadName\", \"process\",\n", + "]\n", + "\n", + "print(\"LogRecord attributes:\")\n", + "for attr in attributes:\n", + " print(f\" {attr:>20}: {getattr(record, attr)}\")\n", + "\n", + "# The formatted message (msg % args)\n", + "print(f\"\\n {'getMessage()':>20}: {record.getMessage()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "\n", + "class InspectingHandler(logging.Handler):\n", + " \"\"\"A custom handler that prints LogRecord attributes for learning.\"\"\"\n", + "\n", + " def emit(self, record: logging.LogRecord) -> None:\n", + " \"\"\"Print key attributes from the log record.\"\"\"\n", + " print(f\" LogRecord received:\")\n", + " print(f\" name = {record.name}\")\n", + " print(f\" levelname = {record.levelname}\")\n", + " print(f\" message = {record.getMessage()}\")\n", + " print(f\" created = {record.created:.2f}\")\n", + " print(f\" thread = {record.threadName}\")\n", + " print()\n", + "\n", + "\n", + "# Attach our inspecting handler to a logger\n", + "logger = logging.getLogger(\"myapp.inspect\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.propagate = False # Don't send to root logger\n", + "\n", + "inspector = InspectingHandler()\n", + "logger.addHandler(inspector)\n", + "\n", + "# Each call creates and routes a LogRecord\n", + "logger.info(\"User %s logged in from %s\", \"alice\", \"192.168.1.10\")\n", + "logger.error(\"Failed to process order #%d\", 1234)\n", + "\n", + "# Clean up\n", + "logger.removeHandler(inspector)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: Setting Up Logging for an Application Module\n", + "\n", + "Here is a complete example showing how to set up proper logging for an application\n", + "module. This pattern is the recommended approach for production code:\n", + "\n", + "1. Each module creates its own logger with `logging.getLogger(__name__)`\n", + "2. The application entry point configures handlers and formatters\n", + "3. Loggers use lazy string formatting with `%s` placeholders (not f-strings)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "\n", + "# --- Module code (e.g., myapp/payment.py) ---\n", + "# In a real module, this would be: logger = logging.getLogger(__name__)\n", + "payment_logger = logging.getLogger(\"myapp.payment\")\n", + "\n", + "\n", + "def process_payment(user_id: int, amount: float, currency: str = \"USD\") -> bool:\n", + " \"\"\"Process a payment for a user.\"\"\"\n", + " payment_logger.info(\"Processing payment: user=%d, amount=%.2f %s\", user_id, amount, currency)\n", + "\n", + " if amount <= 0:\n", + " payment_logger.error(\"Invalid amount %.2f for user %d\", amount, user_id)\n", + " return False\n", + "\n", + " if amount > 10_000:\n", + " payment_logger.warning(\"Large payment detected: %.2f %s for user %d\", amount, currency, user_id)\n", + "\n", + " # Simulate processing\n", + " payment_logger.debug(\"Connecting to payment gateway for user %d\", user_id)\n", + " payment_logger.debug(\"Payment authorized, transaction committed\")\n", + " payment_logger.info(\"Payment of %.2f %s completed for user %d\", amount, currency, user_id)\n", + " return True\n", + "\n", + "\n", + "# --- Application entry point (e.g., main.py) ---\n", + "def configure_logging(level: int = logging.DEBUG) -> None:\n", + " \"\"\"Configure application-wide logging.\"\"\"\n", + " root = logging.getLogger()\n", + " root.setLevel(level)\n", + "\n", + " # Remove any existing handlers\n", + " for h in root.handlers[:]:\n", + " root.removeHandler(h)\n", + "\n", + " # Console handler with detailed format\n", + " console = logging.StreamHandler(sys.stdout)\n", + " console.setLevel(level)\n", + " console.setFormatter(logging.Formatter(\n", + " fmt=\"%(asctime)s [%(levelname)-8s] %(name)s: %(message)s\",\n", + " datefmt=\"%H:%M:%S\"\n", + " ))\n", + " root.addHandler(console)\n", + "\n", + "\n", + "# Configure and run\n", + "configure_logging(logging.DEBUG)\n", + "\n", + "app_logger = logging.getLogger(\"myapp\")\n", + "app_logger.info(\"Application starting\")\n", + "\n", + "process_payment(user_id=42, amount=99.99)\n", + "process_payment(user_id=7, amount=-10.00)\n", + "process_payment(user_id=13, amount=15_000.00, currency=\"EUR\")\n", + "\n", + "app_logger.info(\"Application shutting down\")\n", + "\n", + "# Clean up root logger\n", + "for h in logging.getLogger().handlers[:]:\n", + " logging.getLogger().removeHandler(h)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Concept | Tool | Purpose |\n", + "|---------|------|--------|\n", + "| **Log levels** | `DEBUG` through `CRITICAL` | Classify message severity (10-50) |\n", + "| **Loggers** | `logging.getLogger(name)` | Create/retrieve named logger instances |\n", + "| **Handlers** | `StreamHandler`, `FileHandler` | Direct log output to destinations |\n", + "| **Formatters** | `logging.Formatter(fmt)` | Control the layout of log messages |\n", + "| **Quick setup** | `logging.basicConfig()` | One-call configuration for the root logger |\n", + "| **LogRecord** | Automatic | Carries all metadata through the logging pipeline |\n", + "\n", + "### Best Practices\n", + "- Use `logging.getLogger(__name__)` in every module for automatic naming\n", + "- Use lazy formatting with `%s` placeholders, not f-strings, for log messages\n", + "- Configure handlers and formatters at the application level, not in libraries\n", + "- Set appropriate log levels: `DEBUG` for development, `INFO` or `WARNING` for production\n", + "- Always clean up handlers when reconfiguring to avoid duplicate output\n", + "- Prefer `logging` over `print()` for any diagnostic output that may need to persist" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_21/02_advanced_logging.ipynb b/src/chapter_21/02_advanced_logging.ipynb new file mode 100644 index 0000000..2a604ec --- /dev/null +++ b/src/chapter_21/02_advanced_logging.ipynb @@ -0,0 +1,788 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 21: Advanced Logging\n", + "\n", + "Building on logging fundamentals, this notebook covers the advanced features of Python's\n", + "logging module: logger hierarchies, multiple handlers, custom filters, rotating file\n", + "handlers, structured logging, configuration via dictionaries, and best practices for\n", + "logging in libraries vs applications.\n", + "\n", + "## Topics Covered\n", + "- **Logger hierarchy**: Parent-child relationships and propagation\n", + "- **Multiple handlers**: Console + file on a single logger\n", + "- **Filters**: Filtering log records by custom criteria\n", + "- **Rotating handlers**: `RotatingFileHandler` and `TimedRotatingFileHandler`\n", + "- **Structured logging**: JSON-formatted log output\n", + "- **dictConfig and fileConfig**: Declarative configuration\n", + "- **Libraries vs applications**: Different logging strategies\n", + "- **LoggerAdapter**: Adding context with extra fields" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Logger Hierarchy: Parent-Child Relationships\n", + "\n", + "Logger names form a hierarchy using dot-separated names, just like Python packages.\n", + "The logger `myapp.db.connection` is a child of `myapp.db`, which is a child of `myapp`,\n", + "which is a child of the **root** logger.\n", + "\n", + "When a logger processes a record, it passes the record to its own handlers, then\n", + "(if `propagate=True`, which is the default) passes it up to the parent logger. This\n", + "continues all the way to the root." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "# Clean slate\n", + "for name in [\"webapp\", \"webapp.auth\", \"webapp.auth.oauth\"]:\n", + " lg = logging.getLogger(name)\n", + " lg.handlers.clear()\n", + " lg.setLevel(logging.NOTSET)\n", + " lg.propagate = True\n", + "\n", + "# Build a three-level hierarchy\n", + "parent = logging.getLogger(\"webapp\")\n", + "child = logging.getLogger(\"webapp.auth\")\n", + "grandchild = logging.getLogger(\"webapp.auth.oauth\")\n", + "\n", + "# Verify the hierarchy\n", + "print(f\"child.parent: {child.parent}\")\n", + "print(f\"grandchild.parent: {grandchild.parent}\")\n", + "print(f\"parent.parent: {parent.parent}\")\n", + "\n", + "# Set level only on the parent -- children inherit it\n", + "parent.setLevel(logging.WARNING)\n", + "\n", + "print(f\"\\nparent effective level: {logging.getLevelName(parent.getEffectiveLevel())}\")\n", + "print(f\"child effective level: {logging.getLevelName(child.getEffectiveLevel())}\")\n", + "print(f\"grandchild effective level: {logging.getLevelName(grandchild.getEffectiveLevel())}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "# Demonstrate propagation: records bubble up to parent handlers\n", + "parent = logging.getLogger(\"webapp\")\n", + "parent.setLevel(logging.DEBUG)\n", + "parent.handlers.clear()\n", + "\n", + "child = logging.getLogger(\"webapp.auth\")\n", + "child.handlers.clear()\n", + "\n", + "# Add a handler ONLY to the parent\n", + "parent_handler = logging.StreamHandler(sys.stdout)\n", + "parent_handler.setFormatter(logging.Formatter(\"[%(name)s] %(levelname)s: %(message)s\"))\n", + "parent.addHandler(parent_handler)\n", + "parent.propagate = False # Stop at parent, don't go to root\n", + "\n", + "# Child has no handler, but propagation sends records to parent's handler\n", + "print(\"--- Child logs propagate to parent handler ---\")\n", + "child.warning(\"Login attempt failed for user 'admin'\")\n", + "child.info(\"Session created for user 'alice'\")\n", + "\n", + "# Disable propagation on child\n", + "print(\"\\n--- After child.propagate = False ---\")\n", + "child.propagate = False\n", + "child.warning(\"This will NOT appear (no handler, no propagation)\")\n", + "\n", + "# Give child its own handler\n", + "child_handler = logging.StreamHandler(sys.stdout)\n", + "child_handler.setFormatter(logging.Formatter(\" AUTH >> %(levelname)s: %(message)s\"))\n", + "child.addHandler(child_handler)\n", + "\n", + "print(\"\\n--- Child with its own handler, propagation off ---\")\n", + "child.warning(\"Appears only through child's handler\")\n", + "\n", + "# Re-enable propagation: message appears in BOTH handlers\n", + "child.propagate = True\n", + "print(\"\\n--- Child with own handler + propagation on ---\")\n", + "child.error(\"Appears in BOTH child and parent handlers\")\n", + "\n", + "# Clean up\n", + "parent.removeHandler(parent_handler)\n", + "child.removeHandler(child_handler)\n", + "child.propagate = True\n", + "parent.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multiple Handlers Per Logger\n", + "\n", + "A common pattern is to attach multiple handlers to a single logger, each with a different\n", + "level and destination. For example: verbose `DEBUG` output to a file, and only `WARNING`+\n", + "to the console." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "import io\n", + "\n", + "logger = logging.getLogger(\"myapp.multi\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.handlers.clear()\n", + "logger.propagate = False\n", + "\n", + "# Handler 1: Console -- only WARNING and above\n", + "console_handler = logging.StreamHandler(sys.stdout)\n", + "console_handler.setLevel(logging.WARNING)\n", + "console_handler.setFormatter(logging.Formatter(\n", + " \"CONSOLE | %(levelname)-8s | %(message)s\"\n", + "))\n", + "\n", + "# Handler 2: Simulated file (StringIO) -- all levels from DEBUG\n", + "file_buffer = io.StringIO()\n", + "file_handler = logging.StreamHandler(file_buffer)\n", + "file_handler.setLevel(logging.DEBUG)\n", + "file_handler.setFormatter(logging.Formatter(\n", + " \"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s\",\n", + " datefmt=\"%Y-%m-%d %H:%M:%S\"\n", + "))\n", + "\n", + "logger.addHandler(console_handler)\n", + "logger.addHandler(file_handler)\n", + "\n", + "# Emit messages at various levels\n", + "print(\"=== Console output (WARNING+) ===\")\n", + "logger.debug(\"Connecting to database\")\n", + "logger.info(\"Query executed in 45ms\")\n", + "logger.warning(\"Slow query detected: 2300ms\")\n", + "logger.error(\"Connection pool exhausted\")\n", + "\n", + "# Show what the \"file\" captured (all levels)\n", + "print(\"\\n=== File output (DEBUG+) ===\")\n", + "print(file_buffer.getvalue())\n", + "\n", + "# Clean up\n", + "logger.removeHandler(console_handler)\n", + "logger.removeHandler(file_handler)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Filters: Custom Log Record Filtering\n", + "\n", + "Filters provide fine-grained control over which records a logger or handler processes.\n", + "A filter can be:\n", + "- A `logging.Filter` instance (filters by logger name prefix)\n", + "- Any object with a `filter(record)` method that returns `True`/`False`\n", + "- A callable that takes a `LogRecord` and returns `True`/`False`" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "\n", + "class SensitiveDataFilter(logging.Filter):\n", + " \"\"\"Filter that redacts sensitive fields from log messages.\"\"\"\n", + "\n", + " SENSITIVE_KEYWORDS: list[str] = [\"password\", \"token\", \"secret\", \"api_key\"]\n", + "\n", + " def filter(self, record: logging.LogRecord) -> bool:\n", + " \"\"\"Redact sensitive data but allow the record through.\"\"\"\n", + " message: str = record.getMessage()\n", + " for keyword in self.SENSITIVE_KEYWORDS:\n", + " if keyword in message.lower():\n", + " record.msg = f\"[REDACTED - contains '{keyword}']\"\n", + " record.args = None\n", + " break\n", + " return True # Always allow through (but redacted)\n", + "\n", + "\n", + "class LevelRangeFilter(logging.Filter):\n", + " \"\"\"Only allow records within a specific level range.\"\"\"\n", + "\n", + " def __init__(self, low: int = logging.DEBUG, high: int = logging.CRITICAL) -> None:\n", + " super().__init__()\n", + " self.low = low\n", + " self.high = high\n", + "\n", + " def filter(self, record: logging.LogRecord) -> bool:\n", + " return self.low <= record.levelno <= self.high\n", + "\n", + "\n", + "# Setup logger\n", + "logger = logging.getLogger(\"myapp.filters\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.handlers.clear()\n", + "logger.propagate = False\n", + "\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setFormatter(logging.Formatter(\"%(levelname)-8s: %(message)s\"))\n", + "logger.addHandler(handler)\n", + "\n", + "# Add the sensitive data filter\n", + "sensitive_filter = SensitiveDataFilter()\n", + "handler.addFilter(sensitive_filter)\n", + "\n", + "print(\"--- With SensitiveDataFilter ---\")\n", + "logger.info(\"User alice logged in successfully\")\n", + "logger.info(\"Authenticating with password=s3cret123\")\n", + "logger.info(\"Setting API_KEY=abc-def-ghi\")\n", + "logger.warning(\"Token expired for user bob\")\n", + "\n", + "# Add a level range filter: only DEBUG and INFO\n", + "handler.removeFilter(sensitive_filter)\n", + "range_filter = LevelRangeFilter(low=logging.DEBUG, high=logging.INFO)\n", + "handler.addFilter(range_filter)\n", + "\n", + "print(\"\\n--- With LevelRangeFilter (DEBUG-INFO only) ---\")\n", + "logger.debug(\"This DEBUG message passes\")\n", + "logger.info(\"This INFO message passes\")\n", + "logger.warning(\"This WARNING will be filtered out\")\n", + "logger.error(\"This ERROR will be filtered out\")\n", + "\n", + "# Clean up\n", + "handler.removeFilter(range_filter)\n", + "logger.removeHandler(handler)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## RotatingFileHandler and TimedRotatingFileHandler\n", + "\n", + "For long-running applications, log files can grow very large. The `logging.handlers`\n", + "module provides rotating handlers that automatically manage file sizes:\n", + "\n", + "- **`RotatingFileHandler`**: Rotates when the file reaches a size limit\n", + " - `maxBytes`: Maximum size per file\n", + " - `backupCount`: Number of rotated backup files to keep\n", + " \n", + "- **`TimedRotatingFileHandler`**: Rotates based on time intervals\n", + " - `when`: `'S'` (seconds), `'M'` (minutes), `'H'` (hours), `'D'` (days), `'midnight'`\n", + " - `interval`: Number of time units between rotations\n", + " - `backupCount`: Number of backup files to keep" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler\n", + "import logging\n", + "import tempfile\n", + "import os\n", + "\n", + "# Create a temporary directory for demo log files\n", + "log_dir: str = tempfile.mkdtemp(prefix=\"logging_demo_\")\n", + "log_file: str = os.path.join(log_dir, \"app.log\")\n", + "\n", + "# RotatingFileHandler: rotate after 1KB, keep 3 backups\n", + "rotating_handler = RotatingFileHandler(\n", + " filename=log_file,\n", + " maxBytes=1024, # Rotate after 1KB\n", + " backupCount=3, # Keep app.log.1, app.log.2, app.log.3\n", + ")\n", + "rotating_handler.setFormatter(logging.Formatter(\n", + " \"%(asctime)s [%(levelname)s] %(message)s\"\n", + "))\n", + "\n", + "logger = logging.getLogger(\"myapp.rotating\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.handlers.clear()\n", + "logger.propagate = False\n", + "logger.addHandler(rotating_handler)\n", + "\n", + "# Write enough messages to trigger rotation\n", + "for i in range(50):\n", + " logger.info(\"Log entry number %d: some application event data here\", i)\n", + "\n", + "# Close the handler so files are flushed\n", + "rotating_handler.close()\n", + "logger.removeHandler(rotating_handler)\n", + "\n", + "# Show the rotated files\n", + "print(f\"Log directory: {log_dir}\")\n", + "print(f\"\\nFiles created:\")\n", + "for filename in sorted(os.listdir(log_dir)):\n", + " filepath = os.path.join(log_dir, filename)\n", + " size = os.path.getsize(filepath)\n", + " print(f\" {filename}: {size} bytes\")\n", + "\n", + "# Show content of the most recent log file\n", + "print(f\"\\nLast 3 lines of app.log:\")\n", + "with open(log_file) as f:\n", + " lines = f.readlines()\n", + " for line in lines[-3:]:\n", + " print(f\" {line.rstrip()}\")\n", + "\n", + "# TimedRotatingFileHandler (conceptual -- just show configuration)\n", + "print(\"\\nTimedRotatingFileHandler configuration example:\")\n", + "print(\" when='midnight' -- rotate at midnight each day\")\n", + "print(\" interval=1 -- every 1 unit of 'when'\")\n", + "print(\" backupCount=7 -- keep one week of logs\")\n", + "\n", + "# Clean up temp directory\n", + "import shutil\n", + "shutil.rmtree(log_dir)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Logging: JSON-Formatted Output\n", + "\n", + "Plain-text logs are human-readable but hard to parse programmatically. **Structured logging**\n", + "outputs records in a machine-readable format like JSON, making it easy to index, search,\n", + "and analyze logs with tools like Elasticsearch, Splunk, or CloudWatch." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import logging\n", + "import sys\n", + "from datetime import datetime, timezone\n", + "\n", + "\n", + "class JSONFormatter(logging.Formatter):\n", + " \"\"\"Formats log records as JSON lines.\"\"\"\n", + "\n", + " def format(self, record: logging.LogRecord) -> str:\n", + " \"\"\"Convert a LogRecord to a JSON string.\"\"\"\n", + " log_data: dict[str, object] = {\n", + " \"timestamp\": datetime.fromtimestamp(\n", + " record.created, tz=timezone.utc\n", + " ).isoformat(),\n", + " \"level\": record.levelname,\n", + " \"logger\": record.name,\n", + " \"message\": record.getMessage(),\n", + " \"module\": record.module,\n", + " \"function\": record.funcName,\n", + " \"line\": record.lineno,\n", + " }\n", + "\n", + " # Include exception info if present\n", + " if record.exc_info and record.exc_info[0] is not None:\n", + " log_data[\"exception\"] = self.formatException(record.exc_info)\n", + "\n", + " # Include any extra fields\n", + " if hasattr(record, \"request_id\"):\n", + " log_data[\"request_id\"] = record.request_id # type: ignore[attr-defined]\n", + " if hasattr(record, \"user_id\"):\n", + " log_data[\"user_id\"] = record.user_id # type: ignore[attr-defined]\n", + "\n", + " return json.dumps(log_data, default=str)\n", + "\n", + "\n", + "# Set up logger with JSON formatter\n", + "logger = logging.getLogger(\"myapp.json\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.handlers.clear()\n", + "logger.propagate = False\n", + "\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setFormatter(JSONFormatter())\n", + "logger.addHandler(handler)\n", + "\n", + "# Regular log messages become JSON\n", + "logger.info(\"Application started\")\n", + "logger.warning(\"Cache miss ratio above threshold: 45%%\")\n", + "\n", + "# Log with extra fields\n", + "logger.info(\n", + " \"User authenticated\",\n", + " extra={\"request_id\": \"req-abc-123\", \"user_id\": 42}\n", + ")\n", + "\n", + "# Log with exception\n", + "try:\n", + " result: float = 1 / 0\n", + "except ZeroDivisionError:\n", + " logger.error(\"Calculation failed\", exc_info=True)\n", + "\n", + "# Clean up\n", + "logger.removeHandler(handler)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## dictConfig: Declarative Logging Configuration\n", + "\n", + "`logging.config.dictConfig()` lets you configure the entire logging system from a\n", + "dictionary. This is the **recommended** approach for complex setups because:\n", + "\n", + "- Configuration can be stored in YAML/JSON/TOML files\n", + "- Easy to change without modifying code\n", + "- All loggers, handlers, formatters, and filters defined in one place\n", + "\n", + "`logging.config.fileConfig()` is the older file-based approach using INI format." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import logging.config\n", + "import sys\n", + "\n", + "# Define the entire logging configuration as a dictionary\n", + "LOGGING_CONFIG: dict = {\n", + " \"version\": 1, # Required, always 1\n", + " \"disable_existing_loggers\": False, # Don't disable loggers created before config\n", + "\n", + " \"formatters\": {\n", + " \"standard\": {\n", + " \"format\": \"%(asctime)s [%(levelname)-8s] %(name)s: %(message)s\",\n", + " \"datefmt\": \"%H:%M:%S\",\n", + " },\n", + " \"brief\": {\n", + " \"format\": \"%(levelname)s: %(message)s\",\n", + " },\n", + " },\n", + "\n", + " \"handlers\": {\n", + " \"console\": {\n", + " \"class\": \"logging.StreamHandler\",\n", + " \"level\": \"INFO\",\n", + " \"formatter\": \"brief\",\n", + " \"stream\": \"ext://sys.stdout\",\n", + " },\n", + " \"detailed_console\": {\n", + " \"class\": \"logging.StreamHandler\",\n", + " \"level\": \"DEBUG\",\n", + " \"formatter\": \"standard\",\n", + " \"stream\": \"ext://sys.stdout\",\n", + " },\n", + " },\n", + "\n", + " \"loggers\": {\n", + " \"myapp\": {\n", + " \"level\": \"DEBUG\",\n", + " \"handlers\": [\"detailed_console\"],\n", + " \"propagate\": False,\n", + " },\n", + " \"myapp.db\": {\n", + " \"level\": \"WARNING\", # Only warnings+ from the db module\n", + " \"handlers\": [\"console\"],\n", + " \"propagate\": False,\n", + " },\n", + " },\n", + "\n", + " \"root\": {\n", + " \"level\": \"WARNING\",\n", + " \"handlers\": [\"console\"],\n", + " },\n", + "}\n", + "\n", + "# Apply the configuration\n", + "logging.config.dictConfig(LOGGING_CONFIG)\n", + "\n", + "# Test the configured loggers\n", + "app_logger = logging.getLogger(\"myapp\")\n", + "db_logger = logging.getLogger(\"myapp.db\")\n", + "\n", + "print(\"--- myapp logger (detailed format, DEBUG+) ---\")\n", + "app_logger.debug(\"Initializing application\")\n", + "app_logger.info(\"Configuration loaded\")\n", + "\n", + "print(\"\\n--- myapp.db logger (brief format, WARNING+) ---\")\n", + "db_logger.info(\"Executing SELECT query\") # Filtered out\n", + "db_logger.warning(\"Query took 5.2 seconds\")\n", + "db_logger.error(\"Connection lost to primary DB\")\n", + "\n", + "# Clean up\n", + "for name in [\"myapp\", \"myapp.db\"]:\n", + " lg = logging.getLogger(name)\n", + " lg.handlers.clear()\n", + " lg.setLevel(logging.NOTSET)\n", + " lg.propagate = True\n", + "root = logging.getLogger()\n", + "root.handlers.clear()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Logging in Libraries vs Applications\n", + "\n", + "Libraries and applications have different logging responsibilities:\n", + "\n", + "**Libraries** should:\n", + "- Create loggers using `getLogger(__name__)`\n", + "- Add a `NullHandler` to suppress \"No handlers could be found\" warnings\n", + "- **Never** configure handlers, formatters, or levels\n", + "- Let the application decide how to handle log output\n", + "\n", + "**Applications** should:\n", + "- Configure the root logger or specific loggers with handlers and formatters\n", + "- Use `dictConfig()` or `basicConfig()` at the entry point\n", + "- Set appropriate levels for different components" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "\n", + "# === Library code (what a library author writes) ===\n", + "# File: mylib/__init__.py\n", + "lib_logger = logging.getLogger(\"mylib\")\n", + "lib_logger.addHandler(logging.NullHandler()) # Prevent \"no handler\" warning\n", + "\n", + "\n", + "def library_function(data: str) -> str:\n", + " \"\"\"A function in a third-party library.\"\"\"\n", + " lib_logger.debug(\"Processing data: %s\", data[:50])\n", + " if not data:\n", + " lib_logger.warning(\"Empty data received\")\n", + " return \"\"\n", + " lib_logger.info(\"Data processed successfully, length=%d\", len(data))\n", + " return data.upper()\n", + "\n", + "\n", + "# === Without application configuration, library logs are silently discarded ===\n", + "print(\"--- Without app configuration (NullHandler absorbs logs) ---\")\n", + "result = library_function(\"hello world\")\n", + "print(f\"Result: {result}\")\n", + "print(\"(No log output -- NullHandler absorbed everything)\")\n", + "\n", + "# === Application code: configure logging to see library output ===\n", + "print(\"\\n--- With app configuration ---\")\n", + "app_handler = logging.StreamHandler(sys.stdout)\n", + "app_handler.setFormatter(logging.Formatter(\"[%(name)s] %(levelname)s: %(message)s\"))\n", + "\n", + "# Configure the library's logger from the application side\n", + "lib_logger.addHandler(app_handler)\n", + "lib_logger.setLevel(logging.DEBUG)\n", + "\n", + "result = library_function(\"hello world\")\n", + "print(f\"Result: {result}\")\n", + "\n", + "# Clean up\n", + "lib_logger.removeHandler(app_handler)\n", + "lib_logger.setLevel(logging.NOTSET)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LoggerAdapter: Adding Context with Extra Fields\n", + "\n", + "`LoggerAdapter` wraps a logger and injects extra context into every log record.\n", + "This is useful for adding request IDs, user IDs, or other contextual information\n", + "without passing them to every log call." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "\n", + "# Set up a logger with a format that includes 'extra' fields\n", + "logger = logging.getLogger(\"myapp.adapter\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.handlers.clear()\n", + "logger.propagate = False\n", + "\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setFormatter(logging.Formatter(\n", + " \"%(asctime)s [%(levelname)s] request_id=%(request_id)s user=%(user)s: %(message)s\",\n", + " datefmt=\"%H:%M:%S\"\n", + "))\n", + "logger.addHandler(handler)\n", + "\n", + "# Create a LoggerAdapter that injects request context\n", + "request_context: dict[str, str] = {\n", + " \"request_id\": \"req-7f3a-b2c1\",\n", + " \"user\": \"alice\",\n", + "}\n", + "adapter = logging.LoggerAdapter(logger, extra=request_context)\n", + "\n", + "# Every log call through the adapter automatically includes the context\n", + "adapter.info(\"Handling GET /api/users\")\n", + "adapter.debug(\"Querying database for user list\")\n", + "adapter.info(\"Returning 25 users\")\n", + "\n", + "# A different request gets a different adapter\n", + "print()\n", + "another_context: dict[str, str] = {\n", + " \"request_id\": \"req-9d4e-f8a2\",\n", + " \"user\": \"bob\",\n", + "}\n", + "another_adapter = logging.LoggerAdapter(logger, extra=another_context)\n", + "another_adapter.warning(\"Rate limit approaching for user\")\n", + "another_adapter.error(\"Permission denied for /admin endpoint\")\n", + "\n", + "# Clean up\n", + "logger.removeHandler(handler)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "\n", + "class RequestAdapter(logging.LoggerAdapter):\n", + " \"\"\"Custom LoggerAdapter that prefixes messages with request context.\"\"\"\n", + "\n", + " def process(\n", + " self, msg: str, kwargs: dict\n", + " ) -> tuple[str, dict]:\n", + " \"\"\"Prepend request context to every log message.\"\"\"\n", + " request_id: str = self.extra.get(\"request_id\", \"unknown\") # type: ignore[union-attr]\n", + " client_ip: str = self.extra.get(\"client_ip\", \"unknown\") # type: ignore[union-attr]\n", + " return f\"[{request_id}] [{client_ip}] {msg}\", kwargs\n", + "\n", + "\n", + "# Set up logger with a simple format (context is in the message itself)\n", + "logger = logging.getLogger(\"myapp.custom_adapter\")\n", + "logger.setLevel(logging.DEBUG)\n", + "logger.handlers.clear()\n", + "logger.propagate = False\n", + "\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setFormatter(logging.Formatter(\"%(levelname)-8s %(message)s\"))\n", + "logger.addHandler(handler)\n", + "\n", + "# Simulate handling two concurrent requests\n", + "req1 = RequestAdapter(logger, {\"request_id\": \"a1b2c3\", \"client_ip\": \"192.168.1.10\"})\n", + "req2 = RequestAdapter(logger, {\"request_id\": \"d4e5f6\", \"client_ip\": \"10.0.0.5\"})\n", + "\n", + "req1.info(\"Received GET /api/products\")\n", + "req2.info(\"Received POST /api/orders\")\n", + "req1.debug(\"Fetching products from cache\")\n", + "req2.warning(\"Order validation failed: missing field 'quantity'\")\n", + "req1.info(\"Returning 150 products\")\n", + "req2.error(\"Order creation failed\")\n", + "\n", + "# Clean up\n", + "logger.removeHandler(handler)\n", + "logger.propagate = True" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Concept | Tool | Purpose |\n", + "|---------|------|--------|\n", + "| **Logger hierarchy** | Dot-separated names | Parent-child log propagation |\n", + "| **Multiple handlers** | `addHandler()` | Send logs to console + file simultaneously |\n", + "| **Filters** | `logging.Filter`, callables | Fine-grained record filtering and modification |\n", + "| **Rotating handlers** | `RotatingFileHandler` | Automatic log file rotation by size or time |\n", + "| **Structured logging** | Custom `Formatter` | JSON output for machine parsing |\n", + "| **dictConfig** | `logging.config.dictConfig()` | Declarative logging configuration |\n", + "| **NullHandler** | `logging.NullHandler()` | Silent default handler for libraries |\n", + "| **LoggerAdapter** | `logging.LoggerAdapter` | Inject context into every log message |\n", + "\n", + "### Best Practices\n", + "- Use `propagate=False` on loggers with their own handlers to avoid duplicate output\n", + "- Libraries should only add `NullHandler`; applications configure everything else\n", + "- Use `dictConfig()` for complex setups -- store config in external files\n", + "- Use `RotatingFileHandler` in production to prevent disk exhaustion\n", + "- Use structured (JSON) logging when logs will be parsed by aggregation tools\n", + "- Use `LoggerAdapter` or the `extra` parameter to add request/session context\n", + "- Set `disable_existing_loggers: False` in `dictConfig` to avoid silencing third-party loggers" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_21/03_debugging_techniques.ipynb b/src/chapter_21/03_debugging_techniques.ipynb new file mode 100644 index 0000000..4e57a4a --- /dev/null +++ b/src/chapter_21/03_debugging_techniques.ipynb @@ -0,0 +1,860 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 21: Debugging Techniques\n", + "\n", + "Debugging is the systematic process of finding and fixing errors in code. Python provides\n", + "a rich set of tools for understanding what went wrong, including the `traceback` module,\n", + "the `pdb` debugger, the `warnings` module, and `assert` statements. This notebook covers\n", + "these tools and practical strategies for efficient debugging.\n", + "\n", + "## Topics Covered\n", + "- **traceback module**: `format_exc()`, `extract_stack()`, `print_exception()`\n", + "- **Exception chaining**: `__cause__` and `__context__`\n", + "- **breakpoint()**: The built-in debugger hook and `PYTHONBREAKPOINT`\n", + "- **pdb commands**: `n`, `s`, `c`, `p`, `l`, `w`, `b` (conceptual overview)\n", + "- **Post-mortem debugging**: Debugging after an exception\n", + "- **warnings module**: `warn()`, `filterwarnings()`, warning categories\n", + "- **assert statements**: When and when not to use them\n", + "- **Debugging strategies**: Binary search, rubber duck, logging-based" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The traceback Module\n", + "\n", + "The `traceback` module provides utilities for extracting, formatting, and printing\n", + "stack traces. This is essential for logging exceptions, sending error reports, or\n", + "building custom error handling.\n", + "\n", + "Key functions:\n", + "- `traceback.format_exc()` -- Returns the current exception's traceback as a string\n", + "- `traceback.print_exception()` -- Prints exception info to a file (default: stderr)\n", + "- `traceback.extract_stack()` -- Returns the current call stack as a list of frames\n", + "- `traceback.extract_tb()` -- Extracts frames from a traceback object\n", + "- `traceback.format_exception()` -- Formats exception info as a list of strings" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import traceback\n", + "import sys\n", + "\n", + "\n", + "def level_three() -> None:\n", + " \"\"\"The deepest function that raises an exception.\"\"\"\n", + " data: dict[str, int] = {\"a\": 1, \"b\": 2}\n", + " return data[\"missing_key\"] # KeyError!\n", + "\n", + "\n", + "def level_two() -> None:\n", + " \"\"\"An intermediate function.\"\"\"\n", + " level_three()\n", + "\n", + "\n", + "def level_one() -> None:\n", + " \"\"\"The top-level function.\"\"\"\n", + " level_two()\n", + "\n", + "\n", + "# traceback.format_exc() -- capture traceback as a string\n", + "print(\"=== traceback.format_exc() ===\")\n", + "try:\n", + " level_one()\n", + "except KeyError:\n", + " tb_string: str = traceback.format_exc()\n", + " print(tb_string)\n", + "\n", + "# traceback.print_exception() -- print to stdout instead of stderr\n", + "print(\"=== traceback.print_exception() ===\")\n", + "try:\n", + " level_one()\n", + "except KeyError:\n", + " exc_type, exc_value, exc_tb = sys.exc_info()\n", + " traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import traceback\n", + "\n", + "\n", + "def show_stack() -> None:\n", + " \"\"\"Demonstrate extract_stack() to inspect the call stack.\"\"\"\n", + " print(\"=== traceback.extract_stack() ===\")\n", + " stack = traceback.extract_stack()\n", + " for frame in stack:\n", + " print(f\" File: {frame.filename}\")\n", + " print(f\" Line: {frame.lineno}, in {frame.name}\")\n", + " if frame.line:\n", + " print(f\" Code: {frame.line}\")\n", + " print()\n", + "\n", + "\n", + "def caller() -> None:\n", + " \"\"\"Calls show_stack to demonstrate the call chain.\"\"\"\n", + " show_stack()\n", + "\n", + "\n", + "caller()\n", + "\n", + "# extract_tb() works with traceback objects from exceptions\n", + "print(\"=== traceback.extract_tb() ===\")\n", + "try:\n", + " result: float = 1 / 0\n", + "except ZeroDivisionError:\n", + " import sys\n", + " _, _, tb = sys.exc_info()\n", + " frames = traceback.extract_tb(tb)\n", + " for frame in frames:\n", + " print(f\" {frame.filename}:{frame.lineno} in {frame.name}\")\n", + " print(f\" {frame.line}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exception Chaining: `__cause__` and `__context__`\n", + "\n", + "When one exception leads to another, Python tracks the relationship through two\n", + "attributes:\n", + "\n", + "- **`__cause__`** (explicit chaining) -- Set by `raise X from Y`. Indicates that `Y`\n", + " directly caused `X`.\n", + "- **`__context__`** (implicit chaining) -- Automatically set when an exception is raised\n", + " while handling another exception.\n", + "- **`__suppress_context__`** -- Set to `True` by `raise X from None` to hide the\n", + " implicit context." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import traceback\n", + "\n", + "\n", + "# Explicit chaining with 'raise ... from ...'\n", + "class DatabaseError(Exception):\n", + " \"\"\"Error accessing the database.\"\"\"\n", + "\n", + "\n", + "class ServiceError(Exception):\n", + " \"\"\"Error in the service layer.\"\"\"\n", + "\n", + "\n", + "def query_database(query: str) -> list[dict]:\n", + " \"\"\"Simulate a database query that fails.\"\"\"\n", + " raise DatabaseError(f\"Connection refused for query: {query}\")\n", + "\n", + "\n", + "def get_users() -> list[dict]:\n", + " \"\"\"Service function that wraps database errors.\"\"\"\n", + " try:\n", + " return query_database(\"SELECT * FROM users\")\n", + " except DatabaseError as db_err:\n", + " # Explicit chaining: ServiceError caused by DatabaseError\n", + " raise ServiceError(\"Failed to fetch users\") from db_err\n", + "\n", + "\n", + "print(\"=== Explicit chaining (raise ... from ...) ===\")\n", + "try:\n", + " get_users()\n", + "except ServiceError as e:\n", + " print(f\"Caught: {e}\")\n", + " print(f\" __cause__: {e.__cause__}\")\n", + " print(f\" __context__: {e.__context__}\")\n", + " print()\n", + " # Full traceback shows the chain\n", + " traceback.print_exc()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import traceback\n", + "\n", + "\n", + "# Implicit chaining: exception raised while handling another\n", + "def implicit_chain_example() -> None:\n", + " \"\"\"Demonstrate implicit exception chaining.\"\"\"\n", + " try:\n", + " int(\"not_a_number\")\n", + " except ValueError:\n", + " # This raises a new exception WHILE handling ValueError\n", + " data: dict[str, int] = {}\n", + " _ = data[\"missing\"] # KeyError with implicit context\n", + "\n", + "\n", + "print(\"=== Implicit chaining (__context__) ===\")\n", + "try:\n", + " implicit_chain_example()\n", + "except KeyError as e:\n", + " print(f\"Caught: KeyError({e})\")\n", + " print(f\" __cause__: {e.__cause__}\") # None (not explicit)\n", + " print(f\" __context__: {e.__context__}\") # The original ValueError\n", + "\n", + "\n", + "# Suppressing context with 'raise ... from None'\n", + "def suppressed_context_example() -> None:\n", + " \"\"\"Suppress the original exception context.\"\"\"\n", + " try:\n", + " int(\"not_a_number\")\n", + " except ValueError:\n", + " raise RuntimeError(\"Invalid input provided\") from None\n", + "\n", + "\n", + "print(\"\\n=== Suppressed context (raise ... from None) ===\")\n", + "try:\n", + " suppressed_context_example()\n", + "except RuntimeError as e:\n", + " print(f\"Caught: {e}\")\n", + " print(f\" __cause__: {e.__cause__}\")\n", + " print(f\" __suppress_context__: {e.__suppress_context__}\")\n", + " print(f\" __context__: {e.__context__}\") # Still set, but suppressed" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## breakpoint() and PYTHONBREAKPOINT\n", + "\n", + "The `breakpoint()` built-in (Python 3.7+) is the standard way to drop into an\n", + "interactive debugger. By default, it calls `pdb.set_trace()`, but you can customize\n", + "its behavior through the `PYTHONBREAKPOINT` environment variable:\n", + "\n", + "| `PYTHONBREAKPOINT` value | Behavior |\n", + "|--------------------------|----------|\n", + "| *(not set)* | Calls `pdb.set_trace()` (default) |\n", + "| `\"0\"` | Disables all breakpoints (no-op) |\n", + "| `\"ipdb.set_trace\"` | Uses ipdb instead of pdb |\n", + "| `\"pudb.set_trace\"` | Uses pudb (full-screen debugger) |\n", + "| Any callable path | Calls that function |\n", + "\n", + "**Note**: We will not actually invoke `breakpoint()` in this notebook since it would\n", + "pause execution and require interactive input. The examples below are conceptual." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import os\n", + "import sys\n", + "\n", + "# Show the current PYTHONBREAKPOINT setting\n", + "current_setting: str = os.environ.get(\"PYTHONBREAKPOINT\", \"(not set -- defaults to pdb)\")\n", + "print(f\"Current PYTHONBREAKPOINT: {current_setting}\")\n", + "\n", + "# The breakpoint() hook function\n", + "print(f\"sys.breakpointhook: {sys.breakpointhook}\")\n", + "\n", + "# How you would use breakpoint() in practice:\n", + "print(\"\\n--- Example usage (conceptual, not executed) ---\")\n", + "print(\"\"\"\n", + "def process_data(items: list[str]) -> list[str]:\n", + " results = []\n", + " for item in items:\n", + " transformed = item.strip().lower()\n", + " breakpoint() # Execution pauses here; you can inspect variables\n", + " results.append(transformed)\n", + " return results\n", + "\"\"\")\n", + "\n", + "# Disabling breakpoints via environment variable\n", + "print(\"To disable all breakpoints:\")\n", + "print(\" export PYTHONBREAKPOINT=0\")\n", + "print(\" python my_script.py\")\n", + "\n", + "# Using a different debugger\n", + "print(\"\\nTo use ipdb instead:\")\n", + "print(\" export PYTHONBREAKPOINT=ipdb.set_trace\")\n", + "print(\" python my_script.py\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## pdb Commands Overview\n", + "\n", + "The Python Debugger (`pdb`) provides an interactive command-line interface for stepping\n", + "through code, inspecting variables, and setting breakpoints. Here are the essential\n", + "commands:\n", + "\n", + "| Command | Full Name | Description |\n", + "|---------|-----------|------------|\n", + "| `n` | next | Execute the next line (step **over** function calls) |\n", + "| `s` | step | Step **into** the next function call |\n", + "| `c` | continue | Continue execution until the next breakpoint |\n", + "| `p expr` | print | Print the value of an expression |\n", + "| `pp expr` | pretty-print | Pretty-print the value of an expression |\n", + "| `l` | list | Show source code around the current line |\n", + "| `ll` | longlist | Show the entire current function |\n", + "| `w` | where | Print the full stack trace |\n", + "| `b N` | break | Set a breakpoint at line N |\n", + "| `b func` | break | Set a breakpoint at a function |\n", + "| `cl N` | clear | Clear breakpoint number N |\n", + "| `r` | return | Continue until the current function returns |\n", + "| `q` | quit | Quit the debugger |\n", + "| `h` | help | Show help for commands |\n", + "| `!stmt` | execute | Execute a Python statement |" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Conceptual pdb session walkthrough\n", + "# This shows what a typical debugging session looks like\n", + "\n", + "sample_code: str = \"\"\"\n", + "def find_average(numbers: list[float]) -> float:\n", + " total = sum(numbers)\n", + " count = len(numbers)\n", + " average = total / count # BUG: crashes if numbers is empty\n", + " return average\n", + "\n", + "# Running with pdb:\n", + "# $ python -m pdb my_script.py\n", + "\n", + "# Or with breakpoint():\n", + "# breakpoint()\n", + "# result = find_average([])\n", + "\"\"\"\n", + "\n", + "pdb_session: str = \"\"\"\n", + "Typical pdb session:\n", + "\n", + "> my_script.py(2)find_average()\n", + "-> total = sum(numbers)\n", + "(Pdb) p numbers # Print the argument\n", + "[]\n", + "(Pdb) n # Step to next line\n", + "> my_script.py(3)find_average()\n", + "-> count = len(numbers)\n", + "(Pdb) n # Step to next line\n", + "> my_script.py(4)find_average()\n", + "-> average = total / count\n", + "(Pdb) p total, count # Inspect variables\n", + "(0, 0)\n", + "(Pdb) p count == 0 # Evaluate expressions\n", + "True\n", + "(Pdb) w # Show call stack\n", + " my_script.py(8)()\n", + "-> result = find_average([])\n", + "> my_script.py(4)find_average()\n", + "-> average = total / count\n", + "(Pdb) l # List source code\n", + " 1 def find_average(numbers: list[float]) -> float:\n", + " 2 total = sum(numbers)\n", + " 3 count = len(numbers)\n", + " 4 -> average = total / count\n", + " 5 return average\n", + "(Pdb) q # Quit debugger\n", + "\"\"\"\n", + "\n", + "print(\"Sample code with a bug:\")\n", + "print(sample_code)\n", + "print(pdb_session)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Post-Mortem Debugging\n", + "\n", + "**Post-mortem debugging** lets you inspect the program state at the point where an\n", + "unhandled exception occurred. Instead of setting breakpoints beforehand, you debug\n", + "**after** the crash.\n", + "\n", + "- `pdb.post_mortem(tb)` -- Start pdb at the frame where the exception occurred\n", + "- `pdb.pm()` -- Shortcut that uses `sys.last_traceback`\n", + "- `python -m pdb script.py` -- Automatically enters post-mortem on crash" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sys\n", + "import traceback\n", + "\n", + "\n", + "def buggy_function(data: dict[str, int]) -> int:\n", + " \"\"\"A function with a bug we want to debug post-mortem.\"\"\"\n", + " total: int = 0\n", + " for key in [\"x\", \"y\", \"z\"]:\n", + " total += data[key] # KeyError if key is missing\n", + " return total\n", + "\n", + "\n", + "# Capture the exception for post-mortem analysis\n", + "print(\"=== Capturing exception for post-mortem analysis ===\")\n", + "try:\n", + " result = buggy_function({\"x\": 10, \"y\": 20}) # Missing \"z\"\n", + "except KeyError:\n", + " exc_type, exc_value, exc_tb = sys.exc_info()\n", + "\n", + " # In a real debugging session, you would call:\n", + " # import pdb; pdb.post_mortem(exc_tb)\n", + " # This drops you into pdb at the exact line that crashed\n", + "\n", + " # For this notebook, we inspect the traceback programmatically\n", + " print(f\"Exception type: {exc_type.__name__}\")\n", + " print(f\"Exception value: {exc_value}\")\n", + " print(f\"\\nTraceback frames:\")\n", + " for frame_info in traceback.extract_tb(exc_tb):\n", + " print(f\" {frame_info.name}() at line {frame_info.lineno}\")\n", + " print(f\" Code: {frame_info.line}\")\n", + "\n", + " # Access local variables from the crashed frame\n", + " if exc_tb is not None:\n", + " crashed_frame = exc_tb.tb_next # The frame where the error occurred\n", + " if crashed_frame is not None:\n", + " local_vars = crashed_frame.tb_frame.f_locals\n", + " print(f\"\\nLocal variables at crash point:\")\n", + " for var_name, var_value in local_vars.items():\n", + " print(f\" {var_name} = {var_value!r}\")\n", + "\n", + "print(\"\\nPost-mortem usage:\")\n", + "print(\" import pdb; pdb.post_mortem(exc_tb) # Interactive debugging\")\n", + "print(\" python -m pdb script.py # Auto post-mortem on crash\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The warnings Module\n", + "\n", + "The `warnings` module issues alerts about potential issues that are not errors. Warnings\n", + "are useful for deprecation notices, questionable coding patterns, and other situations\n", + "where you want to notify developers without crashing the program.\n", + "\n", + "Warning categories:\n", + "\n", + "| Category | Purpose |\n", + "|----------|--------|\n", + "| `UserWarning` | Default category for user-issued warnings |\n", + "| `DeprecationWarning` | Feature will be removed in a future version |\n", + "| `FutureWarning` | Behavior will change in a future version |\n", + "| `PendingDeprecationWarning` | Feature will be deprecated soon |\n", + "| `RuntimeWarning` | Questionable runtime behavior |\n", + "| `SyntaxWarning` | Questionable syntax |\n", + "| `ResourceWarning` | Resource usage issues (unclosed files, etc.) |" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import warnings\n", + "\n", + "# Reset warning filters for this demo\n", + "warnings.resetwarnings()\n", + "\n", + "# Basic warning\n", + "print(\"=== Basic warnings ===\")\n", + "warnings.warn(\"This is a default UserWarning\")\n", + "warnings.warn(\"This function is deprecated\", DeprecationWarning, stacklevel=1)\n", + "warnings.warn(\"Behavior will change in v3.0\", FutureWarning)\n", + "\n", + "# Custom warning category\n", + "class SecurityWarning(UserWarning):\n", + " \"\"\"Warning about potential security issues.\"\"\"\n", + "\n", + "warnings.warn(\"Using default encryption key\", SecurityWarning)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import warnings\n", + "\n", + "# filterwarnings() controls which warnings are shown\n", + "# Actions: 'default', 'ignore', 'always', 'error', 'once', 'module'\n", + "\n", + "warnings.resetwarnings()\n", + "\n", + "# 'ignore' suppresses warnings\n", + "print(\"=== Ignoring DeprecationWarning ===\")\n", + "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n", + "warnings.warn(\"This deprecation warning is hidden\", DeprecationWarning)\n", + "print(\"(No output -- DeprecationWarning is ignored)\")\n", + "\n", + "# 'error' turns warnings into exceptions\n", + "print(\"\\n=== Turning warnings into errors ===\")\n", + "warnings.resetwarnings()\n", + "warnings.filterwarnings(\"error\", category=RuntimeWarning)\n", + "\n", + "try:\n", + " warnings.warn(\"Suspicious runtime behavior\", RuntimeWarning)\n", + "except RuntimeWarning as e:\n", + " print(f\"Caught as exception: {e}\")\n", + "\n", + "# 'always' shows the warning every time (default shows once per location)\n", + "print(\"\\n=== 'always' filter ===\")\n", + "warnings.resetwarnings()\n", + "warnings.filterwarnings(\"always\", category=UserWarning)\n", + "\n", + "for i in range(3):\n", + " warnings.warn(f\"Repeated warning #{i}\")\n", + "\n", + "# Filter by message pattern (regex)\n", + "print(\"\\n=== Filter by message pattern ===\")\n", + "warnings.resetwarnings()\n", + "warnings.filterwarnings(\"ignore\", message=\".*deprecated.*\")\n", + "warnings.warn(\"This is deprecated\") # Ignored (matches pattern)\n", + "warnings.warn(\"This is important\") # Shown\n", + "\n", + "warnings.resetwarnings()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Assert Statements: When and When Not to Use Them\n", + "\n", + "The `assert` statement checks a condition and raises `AssertionError` if it is `False`.\n", + "Assertions are a debugging aid, **not** a mechanism for input validation or error handling.\n", + "\n", + "```python\n", + "assert condition, \"optional error message\"\n", + "```\n", + "\n", + "**Critical**: Assertions are removed when Python runs with optimization (`-O` flag).\n", + "Never use assertions for:\n", + "- Input validation\n", + "- Security checks\n", + "- Conditions that might legitimately fail in production" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# GOOD uses of assert: checking internal invariants and preconditions\n", + "\n", + "def calculate_average(values: list[float]) -> float:\n", + " \"\"\"Calculate the average. Caller must ensure non-empty list.\"\"\"\n", + " # Internal precondition: the caller guarantees a non-empty list\n", + " assert len(values) > 0, \"Internal error: values must not be empty\"\n", + " return sum(values) / len(values)\n", + "\n", + "\n", + "def normalize(data: list[float]) -> list[float]:\n", + " \"\"\"Normalize data to [0, 1] range.\"\"\"\n", + " min_val: float = min(data)\n", + " max_val: float = max(data)\n", + " spread: float = max_val - min_val\n", + "\n", + " assert spread >= 0, f\"Internal error: negative spread {spread}\"\n", + "\n", + " if spread == 0:\n", + " return [0.5] * len(data)\n", + "\n", + " result: list[float] = [(x - min_val) / spread for x in data]\n", + "\n", + " # Post-condition: all values should be in [0, 1]\n", + " assert all(0.0 <= v <= 1.0 for v in result), \"Internal error: normalization failed\"\n", + "\n", + " return result\n", + "\n", + "\n", + "# Good: assert catches internal bugs\n", + "print(\"Average:\", calculate_average([10.0, 20.0, 30.0]))\n", + "print(\"Normalized:\", normalize([2.0, 5.0, 8.0, 11.0]))\n", + "\n", + "# Triggering an assertion\n", + "print(\"\\n--- Triggering an assertion ---\")\n", + "try:\n", + " calculate_average([]) # Violates precondition\n", + "except AssertionError as e:\n", + " print(f\"AssertionError: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# BAD uses of assert (NEVER do this in production code)\n", + "\n", + "# BAD: Using assert for input validation\n", + "def bad_validate_age(age: int) -> None:\n", + " \"\"\"BAD: Assertions are removed with -O flag!\"\"\"\n", + " assert isinstance(age, int), \"Age must be an integer\" # WRONG\n", + " assert 0 <= age <= 150, \"Age must be between 0 and 150\" # WRONG\n", + "\n", + "\n", + "# GOOD: Using proper exceptions for input validation\n", + "def good_validate_age(age: int) -> None:\n", + " \"\"\"GOOD: Proper validation that works even with -O flag.\"\"\"\n", + " if not isinstance(age, int):\n", + " raise TypeError(f\"Age must be an integer, got {type(age).__name__}\")\n", + " if not 0 <= age <= 150:\n", + " raise ValueError(f\"Age must be between 0 and 150, got {age}\")\n", + "\n", + "\n", + "# BAD: Assert with side effects (the tuple form is always truthy!)\n", + "print(\"=== Common assert pitfall ===\")\n", + "x: int = -1\n", + "\n", + "# This is ALWAYS True because a non-empty tuple is truthy\n", + "assert (x > 0, \"x must be positive\") # BUG: This never fails!\n", + "print(f\"x = {x} (the assert above did NOT catch x < 0!)\")\n", + "\n", + "# Correct form (no parentheses around the condition + message):\n", + "try:\n", + " assert x > 0, \"x must be positive\"\n", + "except AssertionError as e:\n", + " print(f\"Correct assertion caught: {e}\")\n", + "\n", + "print(\"\\n--- Summary ---\")\n", + "print(\"USE assert for: internal invariants, postconditions, debug checks\")\n", + "print(\"AVOID assert for: input validation, security, any side-effect logic\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Debugging Strategies\n", + "\n", + "Beyond tools, effective debugging requires systematic strategies:\n", + "\n", + "1. **Reproduce first** -- Create the smallest input that triggers the bug\n", + "2. **Read the error message** -- Python tracebacks are detailed and informative\n", + "3. **Binary search** -- Narrow down the problem by testing at the midpoint\n", + "4. **Rubber duck debugging** -- Explain the code line-by-line to an inanimate object\n", + "5. **Logging-based debugging** -- Add targeted log statements to trace execution\n", + "6. **Simplify** -- Remove complexity until the bug disappears, then add it back" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import logging\n", + "import sys\n", + "\n", + "# Strategy: Logging-based debugging\n", + "# Add temporary logging to trace execution flow and variable state\n", + "\n", + "# Set up a debug logger\n", + "debug_logger = logging.getLogger(\"debug\")\n", + "debug_logger.setLevel(logging.DEBUG)\n", + "debug_logger.handlers.clear()\n", + "debug_logger.propagate = False\n", + "\n", + "handler = logging.StreamHandler(sys.stdout)\n", + "handler.setFormatter(logging.Formatter(\" DEBUG | %(message)s\"))\n", + "debug_logger.addHandler(handler)\n", + "\n", + "\n", + "def merge_sorted(left: list[int], right: list[int]) -> list[int]:\n", + " \"\"\"Merge two sorted lists into one sorted list.\"\"\"\n", + " debug_logger.debug(\"merge_sorted called: left=%s, right=%s\", left, right)\n", + "\n", + " result: list[int] = []\n", + " i: int = 0\n", + " j: int = 0\n", + "\n", + " while i < len(left) and j < len(right):\n", + " debug_logger.debug(\" comparing left[%d]=%d vs right[%d]=%d\", i, left[i], j, right[j])\n", + " if left[i] <= right[j]:\n", + " result.append(left[i])\n", + " i += 1\n", + " else:\n", + " result.append(right[j])\n", + " j += 1\n", + "\n", + " # Append remaining elements\n", + " result.extend(left[i:])\n", + " result.extend(right[j:])\n", + "\n", + " debug_logger.debug(\" result=%s\", result)\n", + " return result\n", + "\n", + "\n", + "# Run the function with logging enabled\n", + "print(\"=== Logging-based debugging ===\")\n", + "merged = merge_sorted([1, 3, 5, 7], [2, 4, 6, 8])\n", + "print(f\"\\nFinal result: {merged}\")\n", + "\n", + "# To disable debug logging without removing code:\n", + "print(\"\\n=== Same function, logging disabled ===\")\n", + "debug_logger.setLevel(logging.WARNING) # Suppress DEBUG messages\n", + "merged = merge_sorted([10, 20], [15, 25])\n", + "print(f\"Result: {merged}\")\n", + "\n", + "# Clean up\n", + "debug_logger.removeHandler(handler)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Strategy: Binary search debugging\n", + "# When you have a large codebase or data pipeline, narrow down the bug\n", + "# by testing at the midpoint of the suspected problem area.\n", + "\n", + "def pipeline_step_1(data: list[int]) -> list[int]:\n", + " \"\"\"Filter out negative numbers.\"\"\"\n", + " return [x for x in data if x >= 0]\n", + "\n", + "\n", + "def pipeline_step_2(data: list[int]) -> list[int]:\n", + " \"\"\"Double each value.\"\"\"\n", + " return [x * 2 for x in data]\n", + "\n", + "\n", + "def pipeline_step_3(data: list[int]) -> list[int]:\n", + " \"\"\"BUG: Accidentally squares instead of adding 1.\"\"\"\n", + " return [x ** 2 for x in data] # Should be [x + 1 for x in data]\n", + "\n", + "\n", + "def pipeline_step_4(data: list[int]) -> list[int]:\n", + " \"\"\"Sort the results.\"\"\"\n", + " return sorted(data)\n", + "\n", + "\n", + "def run_pipeline(data: list[int]) -> list[int]:\n", + " \"\"\"Run a multi-step data pipeline.\"\"\"\n", + " result = data\n", + " result = pipeline_step_1(result)\n", + " result = pipeline_step_2(result)\n", + " result = pipeline_step_3(result)\n", + " result = pipeline_step_4(result)\n", + " return result\n", + "\n", + "\n", + "# We know the output is wrong, but which step has the bug?\n", + "input_data: list[int] = [3, -1, 2, -5, 4]\n", + "expected: list[int] = [5, 5, 7, 9] # filter -> double -> add1 -> sort\n", + "actual: list[int] = run_pipeline(input_data)\n", + "print(f\"Input: {input_data}\")\n", + "print(f\"Expected: {expected}\")\n", + "print(f\"Actual: {actual}\")\n", + "print(f\"Match: {expected == actual}\")\n", + "\n", + "# Binary search: check the midpoint (after step 2)\n", + "print(\"\\n--- Binary search debugging ---\")\n", + "after_step_1: list[int] = pipeline_step_1(input_data)\n", + "print(f\"After step 1 (filter): {after_step_1}\") # [3, 2, 4] -- correct\n", + "\n", + "after_step_2: list[int] = pipeline_step_2(after_step_1)\n", + "print(f\"After step 2 (double): {after_step_2}\") # [6, 4, 8] -- correct\n", + "\n", + "after_step_3: list[int] = pipeline_step_3(after_step_2)\n", + "print(f\"After step 3 (add 1?): {after_step_3}\") # [36, 16, 64] -- BUG HERE!\n", + "print(f\" Expected after step 3: [7, 5, 9]\")\n", + "print(f\" Bug found: step 3 squares instead of adding 1!\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Tool | Purpose | When to Use |\n", + "|------|---------|------------|\n", + "| **traceback** | Format and inspect stack traces | Logging exceptions, error reports |\n", + "| **Exception chaining** | `raise X from Y`, `__cause__`, `__context__` | Wrapping exceptions while preserving the cause |\n", + "| **breakpoint()** | Drop into interactive debugger | Interactive investigation of live state |\n", + "| **pdb** | Step through code, inspect variables | Complex bugs requiring line-by-line analysis |\n", + "| **Post-mortem** | Debug after an exception | When the crash is hard to reproduce |\n", + "| **warnings** | Non-fatal alerts about potential issues | Deprecations, questionable patterns |\n", + "| **assert** | Check internal invariants | Debug-time verification of assumptions |\n", + "\n", + "### Debugging Strategy Checklist\n", + "1. Read the traceback carefully -- the answer is often in the error message\n", + "2. Reproduce the bug with the smallest possible input\n", + "3. Use `logging` to trace execution flow without stopping the program\n", + "4. Use `breakpoint()` and pdb when you need to inspect live state interactively\n", + "5. Use binary search to narrow down which step or component contains the bug\n", + "6. Use `assert` for internal invariants, but never for input validation\n", + "7. Use `warnings.warn()` for deprecations rather than `print()` or logging\n", + "8. Explain the code out loud (rubber duck debugging) when you are stuck" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_21/README.md b/src/chapter_21/README.md new file mode 100644 index 0000000..6a62a2e --- /dev/null +++ b/src/chapter_21/README.md @@ -0,0 +1,22 @@ +# Chapter 21: Logging and Debugging + +## Topics Covered +- `logging` module: loggers, handlers, formatters, filters +- Log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL +- Handler types: StreamHandler, FileHandler, RotatingFileHandler +- Formatter patterns and structured logging +- Logger hierarchy and propagation +- `pdb` debugger: breakpoints, stepping, inspecting +- `breakpoint()` built-in (Python 3.7+) +- Debugging strategies and common patterns + +## Notebooks +1. **01_logging_fundamentals.ipynb** — Loggers, handlers, formatters, log levels +2. **02_advanced_logging.ipynb** — Hierarchy, filters, rotating files, structured logging +3. **03_debugging_techniques.ipynb** — pdb, breakpoint(), debugging strategies, traceback + +## Key Takeaways +- Use logging instead of print() for production code +- Configure logging once at application startup +- Logger hierarchy enables fine-grained control per module +- pdb and breakpoint() are essential debugging tools diff --git a/src/chapter_21/__init__.py b/src/chapter_21/__init__.py new file mode 100644 index 0000000..6d8960f --- /dev/null +++ b/src/chapter_21/__init__.py @@ -0,0 +1 @@ +"""Chapter 21: Logging and Debugging.""" diff --git a/src/chapter_22/01_wsgi_and_http.ipynb b/src/chapter_22/01_wsgi_and_http.ipynb new file mode 100644 index 0000000..64c9db4 --- /dev/null +++ b/src/chapter_22/01_wsgi_and_http.ipynb @@ -0,0 +1,998 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 22: WSGI and HTTP Fundamentals\n", + "\n", + "**Web Development Fundamentals**\n", + "\n", + "The Hypertext Transfer Protocol (HTTP) is the foundation of data communication on the\n", + "World Wide Web. Python provides built-in modules for working with HTTP at every level,\n", + "from low-level status codes (`http.HTTPStatus`) to development servers (`http.server`)\n", + "to the WSGI specification that underpins frameworks like Flask and Django. Understanding\n", + "these primitives is essential before reaching for higher-level abstractions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Protocol Basics: Request/Response Cycle\n", + "\n", + "HTTP is a stateless, text-based protocol built on TCP. Every interaction follows\n", + "the same pattern:\n", + "\n", + "1. **Client** sends a **request**: method, path, headers, optional body\n", + "2. **Server** processes the request\n", + "3. **Server** sends a **response**: status code, headers, body\n", + "\n", + "The request and response are composed of a start line, headers (key-value pairs),\n", + "a blank line separator, and an optional body." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Anatomy of an HTTP request and response as raw text\n", + "\n", + "http_request: str = (\n", + " \"GET /api/users?page=2 HTTP/1.1\\r\\n\"\n", + " \"Host: example.com\\r\\n\"\n", + " \"Accept: application/json\\r\\n\"\n", + " \"User-Agent: PythonClient/1.0\\r\\n\"\n", + " \"\\r\\n\" # Blank line separates headers from body\n", + ")\n", + "\n", + "http_response: str = (\n", + " \"HTTP/1.1 200 OK\\r\\n\"\n", + " \"Content-Type: application/json\\r\\n\"\n", + " \"Content-Length: 27\\r\\n\"\n", + " \"\\r\\n\"\n", + " '{\"users\": [], \"page\": 2}\\r\\n'\n", + ")\n", + "\n", + "print(\"=== HTTP Request ===\")\n", + "print(http_request)\n", + "\n", + "print(\"=== HTTP Response ===\")\n", + "print(http_response)\n", + "\n", + "# Parse the request line\n", + "request_line = http_request.split(\"\\r\\n\")[0]\n", + "method, path, version = request_line.split(\" \")\n", + "print(f\"Method: {method}\")\n", + "print(f\"Path: {path}\")\n", + "print(f\"Version: {version}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Methods and Their Semantics\n", + "\n", + "HTTP defines several request methods, each with specific semantics:\n", + "\n", + "| Method | Purpose | Safe | Idempotent | Body |\n", + "|--------|---------|------|------------|------|\n", + "| GET | Retrieve a resource | Yes | Yes | No |\n", + "| POST | Create a resource / submit data | No | No | Yes |\n", + "| PUT | Replace a resource entirely | No | Yes | Yes |\n", + "| PATCH | Partially update a resource | No | No | Yes |\n", + "| DELETE | Remove a resource | No | Yes | Optional |\n", + "| HEAD | GET without the body | Yes | Yes | No |\n", + "| OPTIONS | Describe communication options | Yes | Yes | Optional |\n", + "\n", + "**Safe** means the method does not modify server state. **Idempotent** means\n", + "calling it multiple times produces the same result as calling it once." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class HTTPMethod:\n", + " \"\"\"Describes an HTTP method and its properties.\"\"\"\n", + " name: str\n", + " safe: bool\n", + " idempotent: bool\n", + " has_body: bool\n", + " description: str\n", + "\n", + "\n", + "methods: list[HTTPMethod] = [\n", + " HTTPMethod(\"GET\", safe=True, idempotent=True, has_body=False,\n", + " description=\"Retrieve a resource\"),\n", + " HTTPMethod(\"POST\", safe=False, idempotent=False, has_body=True,\n", + " description=\"Create a resource or submit data\"),\n", + " HTTPMethod(\"PUT\", safe=False, idempotent=True, has_body=True,\n", + " description=\"Replace a resource entirely\"),\n", + " HTTPMethod(\"PATCH\", safe=False, idempotent=False, has_body=True,\n", + " description=\"Partially update a resource\"),\n", + " HTTPMethod(\"DELETE\", safe=False, idempotent=True, has_body=False,\n", + " description=\"Remove a resource\"),\n", + " HTTPMethod(\"HEAD\", safe=True, idempotent=True, has_body=False,\n", + " description=\"GET without the response body\"),\n", + " HTTPMethod(\"OPTIONS\", safe=True, idempotent=True, has_body=False,\n", + " description=\"Describe communication options\"),\n", + "]\n", + "\n", + "for m in methods:\n", + " flags = []\n", + " if m.safe:\n", + " flags.append(\"safe\")\n", + " if m.idempotent:\n", + " flags.append(\"idempotent\")\n", + " if m.has_body:\n", + " flags.append(\"has body\")\n", + " print(f\"{m.name:8s} -- {m.description:40s} [{', '.join(flags)}]\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Status Codes with `http.HTTPStatus`\n", + "\n", + "Python's `http` module provides the `HTTPStatus` enum with all standard status\n", + "codes. Status codes are grouped into five categories:\n", + "\n", + "- **1xx Informational**: Request received, processing continues\n", + "- **2xx Success**: Request successfully received and processed\n", + "- **3xx Redirection**: Further action needed to complete the request\n", + "- **4xx Client Error**: The request contains bad syntax or cannot be fulfilled\n", + "- **5xx Server Error**: The server failed to fulfill a valid request" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from http import HTTPStatus\n", + "from collections import defaultdict\n", + "\n", + "\n", + "# Explore commonly used status codes\n", + "common_codes: list[HTTPStatus] = [\n", + " HTTPStatus.OK,\n", + " HTTPStatus.CREATED,\n", + " HTTPStatus.NO_CONTENT,\n", + " HTTPStatus.MOVED_PERMANENTLY,\n", + " HTTPStatus.NOT_MODIFIED,\n", + " HTTPStatus.BAD_REQUEST,\n", + " HTTPStatus.UNAUTHORIZED,\n", + " HTTPStatus.FORBIDDEN,\n", + " HTTPStatus.NOT_FOUND,\n", + " HTTPStatus.METHOD_NOT_ALLOWED,\n", + " HTTPStatus.CONFLICT,\n", + " HTTPStatus.UNPROCESSABLE_ENTITY,\n", + " HTTPStatus.INTERNAL_SERVER_ERROR,\n", + " HTTPStatus.BAD_GATEWAY,\n", + " HTTPStatus.SERVICE_UNAVAILABLE,\n", + "]\n", + "\n", + "for status in common_codes:\n", + " print(f\"{status.value:3d} {status.phrase:30s} {status.description}\")\n", + "\n", + "# Group all status codes by category\n", + "print(\"\\n=== Status Code Categories ===\")\n", + "categories: dict[str, list[HTTPStatus]] = defaultdict(list)\n", + "category_names: dict[int, str] = {\n", + " 1: \"Informational\",\n", + " 2: \"Success\",\n", + " 3: \"Redirection\",\n", + " 4: \"Client Error\",\n", + " 5: \"Server Error\",\n", + "}\n", + "\n", + "for status in HTTPStatus:\n", + " group = status.value // 100\n", + " categories[category_names[group]].append(status)\n", + "\n", + "for name, statuses in categories.items():\n", + " print(f\" {name}: {len(statuses)} status codes\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTTP Headers: Common Request and Response Headers\n", + "\n", + "Headers carry metadata about the request or response. They are case-insensitive\n", + "key-value pairs separated by a colon. Some important headers include:\n", + "\n", + "- **Content-Type**: MIME type of the body (e.g., `application/json`)\n", + "- **Content-Length**: Size of the body in bytes\n", + "- **Accept**: What content types the client can handle\n", + "- **Authorization**: Credentials for authentication\n", + "- **Cache-Control**: Caching directives" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass, field\n", + "\n", + "\n", + "@dataclass\n", + "class HTTPHeaders:\n", + " \"\"\"A simple case-insensitive header collection.\"\"\"\n", + " _headers: dict[str, str] = field(default_factory=dict)\n", + "\n", + " def set(self, name: str, value: str) -> None:\n", + " \"\"\"Set a header (case-insensitive key).\"\"\"\n", + " self._headers[name.lower()] = value\n", + "\n", + " def get(self, name: str, default: str = \"\") -> str:\n", + " \"\"\"Get a header value (case-insensitive key).\"\"\"\n", + " return self._headers.get(name.lower(), default)\n", + "\n", + " def items(self) -> list[tuple[str, str]]:\n", + " return list(self._headers.items())\n", + "\n", + " def __repr__(self) -> str:\n", + " lines = [f\" {k}: {v}\" for k, v in self._headers.items()]\n", + " return \"Headers(\\n\" + \"\\n\".join(lines) + \"\\n)\"\n", + "\n", + "\n", + "# Build typical request headers\n", + "request_headers = HTTPHeaders()\n", + "request_headers.set(\"Host\", \"api.example.com\")\n", + "request_headers.set(\"Accept\", \"application/json\")\n", + "request_headers.set(\"Authorization\", \"Bearer eyJhbGci...\")\n", + "request_headers.set(\"User-Agent\", \"PythonClient/1.0\")\n", + "request_headers.set(\"Accept-Encoding\", \"gzip, deflate\")\n", + "\n", + "print(\"=== Request Headers ===\")\n", + "print(request_headers)\n", + "\n", + "# Build typical response headers\n", + "response_headers = HTTPHeaders()\n", + "response_headers.set(\"Content-Type\", \"application/json; charset=utf-8\")\n", + "response_headers.set(\"Content-Length\", \"1024\")\n", + "response_headers.set(\"Cache-Control\", \"no-cache, no-store\")\n", + "response_headers.set(\"X-Request-Id\", \"abc-123-def\")\n", + "\n", + "print(\"=== Response Headers ===\")\n", + "print(response_headers)\n", + "\n", + "# Case-insensitive access\n", + "print(f\"Content-Type (lowercase lookup): {response_headers.get('content-type')}\")\n", + "print(f\"Content-Type (mixed case): {response_headers.get('Content-Type')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The WSGI Specification\n", + "\n", + "WSGI (Web Server Gateway Interface, PEP 3333) is the standard interface between\n", + "Python web servers and web applications. It defines a simple contract:\n", + "\n", + "- The **application** is a callable that takes two arguments:\n", + " 1. `environ` -- a dictionary with CGI-style environment variables\n", + " 2. `start_response` -- a callback to begin the HTTP response\n", + "- The application returns an iterable of byte strings (the response body).\n", + "\n", + "This simple interface is what allows any WSGI server (Gunicorn, uWSGI, mod_wsgi)\n", + "to run any WSGI application (Flask, Django, Pyramid)." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Callable, Iterable\n", + "\n", + "\n", + "# Type aliases for WSGI\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "def hello_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " \"\"\"The simplest possible WSGI application.\"\"\"\n", + " status: str = \"200 OK\"\n", + " headers: list[tuple[str, str]] = [\n", + " (\"Content-Type\", \"text/plain; charset=utf-8\"),\n", + " ]\n", + " start_response(status, headers)\n", + " return [b\"Hello, WSGI World!\"]\n", + "\n", + "\n", + "# Simulate calling the WSGI app (what a server does internally)\n", + "captured_status: str = \"\"\n", + "captured_headers: list[tuple[str, str]] = []\n", + "\n", + "\n", + "def mock_start_response(\n", + " status: str,\n", + " response_headers: list[tuple[str, str]],\n", + ") -> Callable[..., object]:\n", + " \"\"\"Mock start_response that captures the status and headers.\"\"\"\n", + " global captured_status, captured_headers\n", + " captured_status = status\n", + " captured_headers = response_headers\n", + " return lambda *args: None # write() callable (rarely used)\n", + "\n", + "\n", + "# Build a minimal environ dict\n", + "environ: Environ = {\n", + " \"REQUEST_METHOD\": \"GET\",\n", + " \"PATH_INFO\": \"/\",\n", + " \"SERVER_NAME\": \"localhost\",\n", + " \"SERVER_PORT\": \"8000\",\n", + " \"HTTP_HOST\": \"localhost:8000\",\n", + "}\n", + "\n", + "# Call the application\n", + "body_parts: Iterable[bytes] = hello_app(environ, mock_start_response)\n", + "body: bytes = b\"\".join(body_parts)\n", + "\n", + "print(f\"Status: {captured_status}\")\n", + "print(f\"Headers: {captured_headers}\")\n", + "print(f\"Body: {body.decode()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The WSGI `environ` Dictionary\n", + "\n", + "The `environ` dict contains CGI-style variables plus WSGI-specific keys.\n", + "Understanding its contents is crucial for parsing requests:\n", + "\n", + "- **REQUEST_METHOD**: GET, POST, PUT, DELETE, etc.\n", + "- **PATH_INFO**: The URL path (e.g., `/api/users`)\n", + "- **QUERY_STRING**: Everything after `?` in the URL\n", + "- **CONTENT_TYPE**: MIME type of the request body\n", + "- **CONTENT_LENGTH**: Size of the request body\n", + "- **HTTP_***: Request headers (e.g., `HTTP_HOST`, `HTTP_ACCEPT`)\n", + "- **wsgi.input**: A file-like object for reading the request body\n", + "- **wsgi.errors**: A file-like object for error output" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import io\n", + "from urllib.parse import parse_qs\n", + "\n", + "\n", + "def build_environ(\n", + " method: str = \"GET\",\n", + " path: str = \"/\",\n", + " query_string: str = \"\",\n", + " body: bytes = b\"\",\n", + " content_type: str = \"\",\n", + " headers: dict[str, str] | None = None,\n", + ") -> Environ:\n", + " \"\"\"Build a realistic WSGI environ dict for testing.\"\"\"\n", + " environ: Environ = {\n", + " # CGI variables\n", + " \"REQUEST_METHOD\": method,\n", + " \"SCRIPT_NAME\": \"\",\n", + " \"PATH_INFO\": path,\n", + " \"QUERY_STRING\": query_string,\n", + " \"SERVER_NAME\": \"localhost\",\n", + " \"SERVER_PORT\": \"8000\",\n", + " \"SERVER_PROTOCOL\": \"HTTP/1.1\",\n", + " \"CONTENT_TYPE\": content_type,\n", + " \"CONTENT_LENGTH\": str(len(body)),\n", + " # WSGI variables\n", + " \"wsgi.version\": (1, 0),\n", + " \"wsgi.url_scheme\": \"http\",\n", + " \"wsgi.input\": io.BytesIO(body),\n", + " \"wsgi.errors\": io.StringIO(),\n", + " \"wsgi.multithread\": False,\n", + " \"wsgi.multiprocess\": False,\n", + " \"wsgi.run_once\": False,\n", + " }\n", + " # Add HTTP_ headers\n", + " if headers:\n", + " for key, value in headers.items():\n", + " environ_key = \"HTTP_\" + key.upper().replace(\"-\", \"_\")\n", + " environ[environ_key] = value\n", + " return environ\n", + "\n", + "\n", + "# Build environ for a GET request with query parameters\n", + "env = build_environ(\n", + " method=\"GET\",\n", + " path=\"/api/users\",\n", + " query_string=\"page=2&limit=10&sort=name\",\n", + " headers={\n", + " \"Host\": \"api.example.com\",\n", + " \"Accept\": \"application/json\",\n", + " \"Authorization\": \"Bearer token123\",\n", + " },\n", + ")\n", + "\n", + "print(\"=== WSGI Environ (CGI Variables) ===\")\n", + "cgi_keys = [\n", + " \"REQUEST_METHOD\", \"PATH_INFO\", \"QUERY_STRING\",\n", + " \"SERVER_NAME\", \"SERVER_PORT\", \"SERVER_PROTOCOL\",\n", + "]\n", + "for key in cgi_keys:\n", + " print(f\" {key}: {env[key]}\")\n", + "\n", + "print(\"\\n=== HTTP Headers (from environ) ===\")\n", + "for key, value in env.items():\n", + " if isinstance(key, str) and key.startswith(\"HTTP_\"):\n", + " header_name = key[5:].replace(\"_\", \"-\").title()\n", + " print(f\" {header_name}: {value}\")\n", + "\n", + "# Parse query string\n", + "print(\"\\n=== Parsed Query Parameters ===\")\n", + "qs = str(env[\"QUERY_STRING\"])\n", + "params: dict[str, list[str]] = parse_qs(qs)\n", + "for key, values in params.items():\n", + " print(f\" {key}: {values}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building a WSGI App That Parses Requests\n", + "\n", + "A practical WSGI application inspects the method, path, query string, and\n", + "body to decide what to do. Below we build an application that routes based\n", + "on `PATH_INFO` and handles different HTTP methods." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import io\n", + "from http import HTTPStatus\n", + "from typing import Callable, Iterable\n", + "from urllib.parse import parse_qs\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "\n", + "def read_body(environ: Environ) -> bytes:\n", + " \"\"\"Read the request body from wsgi.input.\"\"\"\n", + " content_length_str = str(environ.get(\"CONTENT_LENGTH\", \"0\") or \"0\")\n", + " try:\n", + " content_length = int(content_length_str)\n", + " except ValueError:\n", + " content_length = 0\n", + " wsgi_input = environ.get(\"wsgi.input\")\n", + " if wsgi_input and content_length > 0:\n", + " assert hasattr(wsgi_input, \"read\")\n", + " return wsgi_input.read(content_length) # type: ignore[union-attr]\n", + " return b\"\"\n", + "\n", + "\n", + "def json_response(\n", + " data: object,\n", + " status: HTTPStatus,\n", + " start_response: StartResponse,\n", + ") -> Iterable[bytes]:\n", + " \"\"\"Helper to return a JSON response.\"\"\"\n", + " body: bytes = json.dumps(data, indent=2).encode(\"utf-8\")\n", + " start_response(\n", + " f\"{status.value} {status.phrase}\",\n", + " [\n", + " (\"Content-Type\", \"application/json; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ],\n", + " )\n", + " return [body]\n", + "\n", + "\n", + "def demo_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " \"\"\"A WSGI app that inspects and echoes request details.\"\"\"\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " query_string = str(environ.get(\"QUERY_STRING\", \"\"))\n", + "\n", + " if path == \"/echo\":\n", + " body = read_body(environ)\n", + " response_data = {\n", + " \"method\": method,\n", + " \"path\": path,\n", + " \"query_params\": parse_qs(query_string),\n", + " \"body\": body.decode(\"utf-8\", errors=\"replace\"),\n", + " \"content_type\": str(environ.get(\"CONTENT_TYPE\", \"\")),\n", + " }\n", + " return json_response(response_data, HTTPStatus.OK, start_response)\n", + "\n", + " elif path == \"/health\":\n", + " return json_response({\"status\": \"healthy\"}, HTTPStatus.OK, start_response)\n", + "\n", + " else:\n", + " return json_response(\n", + " {\"error\": \"Not Found\", \"path\": path},\n", + " HTTPStatus.NOT_FOUND,\n", + " start_response,\n", + " )\n", + "\n", + "\n", + "# Test the app with different requests\n", + "def call_app(app: Callable[..., Iterable[bytes]], environ: Environ) -> None:\n", + " \"\"\"Simulate calling a WSGI app and print the results.\"\"\"\n", + " result_status = \"\"\n", + " result_headers: list[tuple[str, str]] = []\n", + "\n", + " def start_response(\n", + " status: str, headers: list[tuple[str, str]]\n", + " ) -> Callable[..., object]:\n", + " nonlocal result_status, result_headers\n", + " result_status = status\n", + " result_headers = headers\n", + " return lambda *args: None\n", + "\n", + " body = b\"\".join(app(environ, start_response))\n", + " print(f\" Status: {result_status}\")\n", + " print(f\" Body: {body.decode()}\")\n", + "\n", + "\n", + "print(\"=== GET /health ===\")\n", + "call_app(demo_app, build_environ(path=\"/health\"))\n", + "\n", + "print(\"\\n=== GET /echo?name=alice&role=admin ===\")\n", + "call_app(demo_app, build_environ(path=\"/echo\", query_string=\"name=alice&role=admin\"))\n", + "\n", + "print(\"\\n=== POST /echo with JSON body ===\")\n", + "call_app(\n", + " demo_app,\n", + " build_environ(\n", + " method=\"POST\",\n", + " path=\"/echo\",\n", + " body=b'{\"message\": \"hello\"}',\n", + " content_type=\"application/json\",\n", + " ),\n", + ")\n", + "\n", + "print(\"\\n=== GET /unknown ===\")\n", + "call_app(demo_app, build_environ(path=\"/unknown\"))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Response Building: Status Line, Headers, Body\n", + "\n", + "When constructing HTTP responses, you must:\n", + "\n", + "1. Set the correct **status code** (matching the semantic outcome)\n", + "2. Include appropriate **headers** (Content-Type, Content-Length, caching, etc.)\n", + "3. Encode the **body** as bytes (HTTP transmits raw bytes, not strings)\n", + "\n", + "Below is a `ResponseBuilder` that makes response construction explicit and safe." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "from http import HTTPStatus\n", + "from dataclasses import dataclass, field\n", + "\n", + "\n", + "@dataclass\n", + "class Response:\n", + " \"\"\"Encapsulates an HTTP response for WSGI.\"\"\"\n", + " status: HTTPStatus = HTTPStatus.OK\n", + " headers: dict[str, str] = field(default_factory=dict)\n", + " body: bytes = b\"\"\n", + "\n", + " @property\n", + " def status_line(self) -> str:\n", + " return f\"{self.status.value} {self.status.phrase}\"\n", + "\n", + " @property\n", + " def header_list(self) -> list[tuple[str, str]]:\n", + " return list(self.headers.items())\n", + "\n", + " @classmethod\n", + " def text(cls, content: str, status: HTTPStatus = HTTPStatus.OK) -> \"Response\":\n", + " \"\"\"Create a plain text response.\"\"\"\n", + " body = content.encode(\"utf-8\")\n", + " return cls(\n", + " status=status,\n", + " headers={\n", + " \"Content-Type\": \"text/plain; charset=utf-8\",\n", + " \"Content-Length\": str(len(body)),\n", + " },\n", + " body=body,\n", + " )\n", + "\n", + " @classmethod\n", + " def html(cls, content: str, status: HTTPStatus = HTTPStatus.OK) -> \"Response\":\n", + " \"\"\"Create an HTML response.\"\"\"\n", + " body = content.encode(\"utf-8\")\n", + " return cls(\n", + " status=status,\n", + " headers={\n", + " \"Content-Type\": \"text/html; charset=utf-8\",\n", + " \"Content-Length\": str(len(body)),\n", + " },\n", + " body=body,\n", + " )\n", + "\n", + " @classmethod\n", + " def json_data(\n", + " cls, data: object, status: HTTPStatus = HTTPStatus.OK\n", + " ) -> \"Response\":\n", + " \"\"\"Create a JSON response.\"\"\"\n", + " body = json.dumps(data, indent=2).encode(\"utf-8\")\n", + " return cls(\n", + " status=status,\n", + " headers={\n", + " \"Content-Type\": \"application/json; charset=utf-8\",\n", + " \"Content-Length\": str(len(body)),\n", + " },\n", + " body=body,\n", + " )\n", + "\n", + " @classmethod\n", + " def redirect(cls, location: str, permanent: bool = False) -> \"Response\":\n", + " \"\"\"Create a redirect response.\"\"\"\n", + " status = (\n", + " HTTPStatus.MOVED_PERMANENTLY if permanent\n", + " else HTTPStatus.FOUND\n", + " )\n", + " return cls(\n", + " status=status,\n", + " headers={\"Location\": location, \"Content-Length\": \"0\"},\n", + " body=b\"\",\n", + " )\n", + "\n", + "\n", + "# Demonstrate different response types\n", + "responses: list[tuple[str, Response]] = [\n", + " (\"Plain text\", Response.text(\"Hello, World!\")),\n", + " (\"HTML\", Response.html(\"

Welcome

Hello!

\")),\n", + " (\"JSON\", Response.json_data({\"users\": [\"alice\", \"bob\"], \"count\": 2})),\n", + " (\"Redirect\", Response.redirect(\"/new-location\")),\n", + " (\"Not Found\", Response.text(\"Page not found\", HTTPStatus.NOT_FOUND)),\n", + "]\n", + "\n", + "for label, resp in responses:\n", + " print(f\"=== {label} ===\")\n", + " print(f\" Status: {resp.status_line}\")\n", + " print(f\" Headers: {resp.header_list}\")\n", + " print(f\" Body: {resp.body[:80]!r}\")\n", + " print()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Development Server with `http.server`\n", + "\n", + "Python's `http.server` module provides a simple HTTP server suitable for\n", + "development and testing. It can serve static files out of the box, and can\n", + "be extended with custom request handlers. The module also includes\n", + "`CGIHTTPRequestHandler` for CGI scripts and can be used as a quick WSGI\n", + "host via `wsgiref`." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from http.server import HTTPServer, BaseHTTPRequestHandler\n", + "import threading\n", + "import urllib.request\n", + "import json\n", + "\n", + "\n", + "class SimpleAPIHandler(BaseHTTPRequestHandler):\n", + " \"\"\"A minimal HTTP request handler demonstrating the http.server API.\"\"\"\n", + "\n", + " def do_GET(self) -> None:\n", + " \"\"\"Handle GET requests.\"\"\"\n", + " if self.path == \"/api/info\":\n", + " data = {\n", + " \"server\": \"SimpleAPIHandler\",\n", + " \"method\": \"GET\",\n", + " \"path\": self.path,\n", + " \"client\": self.client_address[0],\n", + " }\n", + " body = json.dumps(data, indent=2).encode(\"utf-8\")\n", + " self.send_response(200)\n", + " self.send_header(\"Content-Type\", \"application/json\")\n", + " self.send_header(\"Content-Length\", str(len(body)))\n", + " self.end_headers()\n", + " self.wfile.write(body)\n", + " else:\n", + " self.send_error(404, \"Not Found\")\n", + "\n", + " def log_message(self, format: str, *args: object) -> None:\n", + " \"\"\"Suppress default logging to keep notebook output clean.\"\"\"\n", + " pass\n", + "\n", + "\n", + "# Start a development server in a background thread\n", + "server = HTTPServer((\"127.0.0.1\", 0), SimpleAPIHandler)\n", + "port = server.server_address[1]\n", + "print(f\"Development server running on http://127.0.0.1:{port}\")\n", + "\n", + "server_thread = threading.Thread(target=server.serve_forever, daemon=True)\n", + "server_thread.start()\n", + "\n", + "# Make a request to the server\n", + "try:\n", + " url = f\"http://127.0.0.1:{port}/api/info\"\n", + " with urllib.request.urlopen(url) as response:\n", + " print(f\"\\nResponse status: {response.status}\")\n", + " print(f\"Content-Type: {response.headers['Content-Type']}\")\n", + " body = json.loads(response.read())\n", + " print(f\"Body: {json.dumps(body, indent=2)}\")\n", + "\n", + " # Request a non-existent path\n", + " try:\n", + " url_404 = f\"http://127.0.0.1:{port}/nonexistent\"\n", + " urllib.request.urlopen(url_404)\n", + " except urllib.error.HTTPError as e:\n", + " print(f\"\\n404 test: status={e.code}, reason={e.reason}\")\n", + "\n", + "finally:\n", + " server.shutdown()\n", + " print(\"\\nServer shut down.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running a WSGI App with `wsgiref`\n", + "\n", + "The `wsgiref` module in the standard library provides a reference WSGI server\n", + "implementation. It is suitable for development and testing. The `make_server()`\n", + "function creates a server that hosts any WSGI-compliant application." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from wsgiref.simple_server import make_server\n", + "import threading\n", + "import urllib.request\n", + "import json\n", + "from typing import Callable, Iterable\n", + "from http import HTTPStatus\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "\n", + "def wsgiref_demo_app(\n", + " environ: Environ, start_response: StartResponse\n", + ") -> Iterable[bytes]:\n", + " \"\"\"A WSGI app served by wsgiref.\"\"\"\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + "\n", + " response_data = {\n", + " \"message\": f\"Handled {method} {path} via wsgiref\",\n", + " \"server\": \"wsgiref.simple_server\",\n", + " \"wsgi_version\": list(environ.get(\"wsgi.version\", (1, 0))), # type: ignore[arg-type]\n", + " }\n", + " body = json.dumps(response_data, indent=2).encode(\"utf-8\")\n", + "\n", + " start_response(\n", + " f\"{HTTPStatus.OK.value} {HTTPStatus.OK.phrase}\",\n", + " [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ],\n", + " )\n", + " return [body]\n", + "\n", + "\n", + "# Create and start the wsgiref server\n", + "httpd = make_server(\"127.0.0.1\", 0, wsgiref_demo_app)\n", + "port = httpd.server_address[1]\n", + "print(f\"wsgiref server running on http://127.0.0.1:{port}\")\n", + "\n", + "server_thread = threading.Thread(target=httpd.serve_forever, daemon=True)\n", + "server_thread.start()\n", + "\n", + "# Test the WSGI app through the actual HTTP server\n", + "try:\n", + " for path in [\"/\", \"/api/data\", \"/users/123\"]:\n", + " url = f\"http://127.0.0.1:{port}{path}\"\n", + " with urllib.request.urlopen(url) as resp:\n", + " data = json.loads(resp.read())\n", + " print(f\"GET {path} -> {data['message']}\")\n", + "finally:\n", + " httpd.shutdown()\n", + " print(\"\\nwsgiref server shut down.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Request Parsing: A Complete Example\n", + "\n", + "Pulling it all together, here is a `Request` dataclass that extracts all\n", + "relevant information from a WSGI `environ` dict -- method, path, query\n", + "parameters, headers, and body -- into a clean, typed interface." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import io\n", + "import json\n", + "from dataclasses import dataclass, field\n", + "from urllib.parse import parse_qs\n", + "\n", + "\n", + "@dataclass\n", + "class Request:\n", + " \"\"\"Parsed HTTP request extracted from a WSGI environ.\"\"\"\n", + " method: str\n", + " path: str\n", + " query_params: dict[str, list[str]]\n", + " headers: dict[str, str]\n", + " body: bytes\n", + " content_type: str\n", + "\n", + " @classmethod\n", + " def from_environ(cls, environ: dict[str, object]) -> \"Request\":\n", + " \"\"\"Parse a WSGI environ dict into a Request object.\"\"\"\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " query_string = str(environ.get(\"QUERY_STRING\", \"\"))\n", + " content_type = str(environ.get(\"CONTENT_TYPE\", \"\"))\n", + "\n", + " # Parse content length and read body\n", + " try:\n", + " content_length = int(str(environ.get(\"CONTENT_LENGTH\", \"0\") or \"0\"))\n", + " except ValueError:\n", + " content_length = 0\n", + "\n", + " wsgi_input = environ.get(\"wsgi.input\")\n", + " body = b\"\"\n", + " if wsgi_input and content_length > 0 and hasattr(wsgi_input, \"read\"):\n", + " body = wsgi_input.read(content_length) # type: ignore[union-attr]\n", + "\n", + " # Extract HTTP headers from environ\n", + " headers: dict[str, str] = {}\n", + " for key, value in environ.items():\n", + " if isinstance(key, str) and key.startswith(\"HTTP_\"):\n", + " header_name = key[5:].replace(\"_\", \"-\").title()\n", + " headers[header_name] = str(value)\n", + "\n", + " return cls(\n", + " method=method,\n", + " path=path,\n", + " query_params=parse_qs(query_string),\n", + " headers=headers,\n", + " body=body,\n", + " content_type=content_type,\n", + " )\n", + "\n", + " @property\n", + " def json(self) -> object:\n", + " \"\"\"Parse the body as JSON.\"\"\"\n", + " return json.loads(self.body) if self.body else None\n", + "\n", + " def get_query(self, key: str, default: str = \"\") -> str:\n", + " \"\"\"Get a single query parameter value.\"\"\"\n", + " values = self.query_params.get(key, [])\n", + " return values[0] if values else default\n", + "\n", + "\n", + "# Test with a simulated POST request\n", + "post_body = json.dumps({\"username\": \"alice\", \"email\": \"alice@example.com\"}).encode()\n", + "test_environ: dict[str, object] = {\n", + " \"REQUEST_METHOD\": \"POST\",\n", + " \"PATH_INFO\": \"/api/users\",\n", + " \"QUERY_STRING\": \"notify=true&format=json\",\n", + " \"CONTENT_TYPE\": \"application/json\",\n", + " \"CONTENT_LENGTH\": str(len(post_body)),\n", + " \"wsgi.input\": io.BytesIO(post_body),\n", + " \"HTTP_HOST\": \"api.example.com\",\n", + " \"HTTP_ACCEPT\": \"application/json\",\n", + " \"HTTP_AUTHORIZATION\": \"Bearer secret-token\",\n", + " \"HTTP_X_REQUEST_ID\": \"req-abc-123\",\n", + "}\n", + "\n", + "req = Request.from_environ(test_environ)\n", + "print(f\"Method: {req.method}\")\n", + "print(f\"Path: {req.path}\")\n", + "print(f\"Content-Type: {req.content_type}\")\n", + "print(f\"Query params: {req.query_params}\")\n", + "print(f\" notify: {req.get_query('notify')}\")\n", + "print(f\" format: {req.get_query('format')}\")\n", + "print(f\" missing: {req.get_query('missing', 'default-value')}\")\n", + "print(f\"Headers: {req.headers}\")\n", + "print(f\"Body (raw): {req.body}\")\n", + "print(f\"Body (JSON): {req.json}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook covered the foundations of HTTP and WSGI in Python:\n", + "\n", + "1. **HTTP request/response cycle**: The text-based protocol with method, path,\n", + " headers, status codes, and body\n", + "2. **HTTP methods**: GET, POST, PUT, PATCH, DELETE and their semantic properties\n", + " (safe, idempotent, body)\n", + "3. **Status codes**: The `http.HTTPStatus` enum organized into 1xx-5xx categories\n", + "4. **Headers**: Case-insensitive key-value metadata for requests and responses\n", + "5. **WSGI specification**: The `environ` dict, `start_response` callable, and\n", + " the application callable interface (PEP 3333)\n", + "6. **Request parsing**: Extracting method, path, query string, headers, and body\n", + " from the WSGI environ\n", + "7. **Response building**: Constructing status lines, headers, and encoded bodies\n", + "8. **Development servers**: `http.server` for simple handlers and `wsgiref` for\n", + " hosting WSGI applications\n", + "\n", + "The next notebook covers URL routing, template rendering, and form handling --\n", + "building on these WSGI primitives to create more structured web applications." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_22/02_routing_and_templates.ipynb b/src/chapter_22/02_routing_and_templates.ipynb new file mode 100644 index 0000000..2cd8142 --- /dev/null +++ b/src/chapter_22/02_routing_and_templates.ipynb @@ -0,0 +1,1226 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 22: Routing and Templates\n", + "\n", + "**Web Development Fundamentals**\n", + "\n", + "Building on the WSGI foundation from the previous notebook, this notebook explores\n", + "how web frameworks map URLs to handler functions (routing), extract parameters from\n", + "paths and query strings, render dynamic HTML with templates, and handle form\n", + "submissions. All examples use only the Python standard library to illustrate the\n", + "patterns that frameworks like Flask and Django implement under the hood." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## URL Routing: Mapping Paths to Handlers\n", + "\n", + "Routing is the process of matching an incoming URL path to a handler function.\n", + "The simplest approach is a dictionary mapping path strings to callables. More\n", + "advanced routers use regex patterns to extract path parameters." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Callable, Iterable\n", + "from http import HTTPStatus\n", + "import json\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "Handler = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class SimpleRouter:\n", + " \"\"\"A basic router that maps exact paths to handler functions.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._routes: dict[str, Handler] = {}\n", + "\n", + " def add_route(self, path: str, handler: Handler) -> None:\n", + " \"\"\"Register a handler for an exact path.\"\"\"\n", + " self._routes[path] = handler\n", + "\n", + " def resolve(self, path: str) -> Handler | None:\n", + " \"\"\"Find a handler for the given path.\"\"\"\n", + " return self._routes.get(path)\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " \"\"\"WSGI application interface.\"\"\"\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " handler = self.resolve(path)\n", + "\n", + " if handler is None:\n", + " body = json.dumps({\"error\": \"Not Found\"}).encode(\"utf-8\")\n", + " start_response(\n", + " f\"{HTTPStatus.NOT_FOUND.value} {HTTPStatus.NOT_FOUND.phrase}\",\n", + " [(\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body)))],\n", + " )\n", + " return [body]\n", + "\n", + " return handler(environ, start_response)\n", + "\n", + "\n", + "# Define handlers\n", + "def home_handler(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " body = b\"Welcome to the home page!\"\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "def about_handler(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " body = b\"About this application.\"\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Set up the router\n", + "router = SimpleRouter()\n", + "router.add_route(\"/\", home_handler)\n", + "router.add_route(\"/about\", about_handler)\n", + "\n", + "\n", + "# Test helper\n", + "def test_route(app: SimpleRouter, path: str) -> None:\n", + " \"\"\"Simulate a GET request and print the result.\"\"\"\n", + " result_status = \"\"\n", + "\n", + " def start_response(status: str, headers: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal result_status\n", + " result_status = status\n", + " return lambda *a: None\n", + "\n", + " environ: Environ = {\"REQUEST_METHOD\": \"GET\", \"PATH_INFO\": path}\n", + " body = b\"\".join(app(environ, start_response))\n", + " print(f\"GET {path:20s} -> {result_status:15s} | {body.decode()}\")\n", + "\n", + "\n", + "test_route(router, \"/\")\n", + "test_route(router, \"/about\")\n", + "test_route(router, \"/missing\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Route Decorator Pattern\n", + "\n", + "Frameworks like Flask use a `@app.route(\"/path\")` decorator to register handlers.\n", + "This pattern is syntactic sugar over `add_route()` -- the decorator registers\n", + "the function in the routing table and returns it unchanged." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Callable, Iterable\n", + "from http import HTTPStatus\n", + "import json\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "Handler = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class App:\n", + " \"\"\"A mini web framework with Flask-style route decorators.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._routes: dict[str, dict[str, Handler]] = {} # path -> {method -> handler}\n", + "\n", + " def route(\n", + " self, path: str, methods: list[str] | None = None\n", + " ) -> Callable[[Handler], Handler]:\n", + " \"\"\"Decorator to register a route handler.\n", + "\n", + " Usage:\n", + " @app.route(\"/users\", methods=[\"GET\", \"POST\"])\n", + " def users_handler(environ, start_response):\n", + " ...\n", + " \"\"\"\n", + " if methods is None:\n", + " methods = [\"GET\"]\n", + "\n", + " def decorator(func: Handler) -> Handler:\n", + " if path not in self._routes:\n", + " self._routes[path] = {}\n", + " for method in methods:\n", + " self._routes[path][method.upper()] = func\n", + " return func\n", + "\n", + " return decorator\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " \"\"\"WSGI interface: dispatch to the matching handler.\"\"\"\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + "\n", + " route_methods = self._routes.get(path)\n", + " if route_methods is None:\n", + " return self._error(HTTPStatus.NOT_FOUND, start_response)\n", + "\n", + " handler = route_methods.get(method)\n", + " if handler is None:\n", + " allowed = \", \".join(sorted(route_methods.keys()))\n", + " body = json.dumps({\"error\": \"Method Not Allowed\", \"allowed\": allowed}).encode()\n", + " start_response(\n", + " f\"{HTTPStatus.METHOD_NOT_ALLOWED.value} {HTTPStatus.METHOD_NOT_ALLOWED.phrase}\",\n", + " [(\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " (\"Allow\", allowed)],\n", + " )\n", + " return [body]\n", + "\n", + " return handler(environ, start_response)\n", + "\n", + " @staticmethod\n", + " def _error(\n", + " status: HTTPStatus, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " body = json.dumps({\"error\": status.phrase}).encode()\n", + " start_response(\n", + " f\"{status.value} {status.phrase}\",\n", + " [(\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body)))],\n", + " )\n", + " return [body]\n", + "\n", + "\n", + "# Create the app and register routes using decorators\n", + "app = App()\n", + "\n", + "\n", + "@app.route(\"/\")\n", + "def index(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " body = b\"Home page\"\n", + " start_response(\"200 OK\", [(\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body)))])\n", + " return [body]\n", + "\n", + "\n", + "@app.route(\"/items\", methods=[\"GET\", \"POST\"])\n", + "def items(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " method = str(environ.get(\"REQUEST_METHOD\"))\n", + " body = f\"Items endpoint: {method}\".encode()\n", + " start_response(\"200 OK\", [(\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body)))])\n", + " return [body]\n", + "\n", + "\n", + "# Test the decorator-based routing\n", + "def test_app(app: App, method: str, path: str) -> None:\n", + " result_status = \"\"\n", + "\n", + " def start_response(status: str, headers: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal result_status\n", + " result_status = status\n", + " return lambda *a: None\n", + "\n", + " environ: Environ = {\"REQUEST_METHOD\": method, \"PATH_INFO\": path}\n", + " body = b\"\".join(app(environ, start_response))\n", + " print(f\"{method:6s} {path:15s} -> {result_status:30s} | {body.decode()}\")\n", + "\n", + "\n", + "test_app(app, \"GET\", \"/\")\n", + "test_app(app, \"GET\", \"/items\")\n", + "test_app(app, \"POST\", \"/items\")\n", + "test_app(app, \"DELETE\", \"/items\") # Method not allowed\n", + "test_app(app, \"GET\", \"/unknown\") # Not found" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Path Parameters: Extracting Values from URLs\n", + "\n", + "Dynamic routes like `/users/42` or `/posts/hello-world` require extracting\n", + "variable segments from the URL path. Frameworks use regex patterns (or custom\n", + "parsers) to match and capture these segments. Below we build a regex-based\n", + "router that supports `` placeholders in route patterns." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import re\n", + "from typing import Callable, Iterable\n", + "from dataclasses import dataclass, field\n", + "from http import HTTPStatus\n", + "import json\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "# Handlers now receive path params as a dict\n", + "ParamHandler = Callable[[Environ, StartResponse, dict[str, str]], Iterable[bytes]]\n", + "\n", + "\n", + "@dataclass\n", + "class Route:\n", + " \"\"\"A route with a regex pattern compiled from a path template.\"\"\"\n", + " template: str # Original template, e.g., \"/users/\"\n", + " pattern: re.Pattern[str] # Compiled regex\n", + " handler: ParamHandler\n", + " methods: set[str] = field(default_factory=lambda: {\"GET\"})\n", + "\n", + "\n", + "def compile_route_pattern(template: str) -> re.Pattern[str]:\n", + " \"\"\"Convert a route template like '/users/' to a regex.\n", + "\n", + " captures a path segment as a named group matching [^/]+.\n", + " \"\"\"\n", + " # Replace with named regex groups\n", + " regex = re.sub(r\"<(\\w+)>\", r\"(?P<\\1>[^/]+)\", template)\n", + " return re.compile(f\"^{regex}$\")\n", + "\n", + "\n", + "class RegexRouter:\n", + " \"\"\"A router that supports path parameters via regex.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._routes: list[Route] = []\n", + "\n", + " def route(\n", + " self, template: str, methods: list[str] | None = None\n", + " ) -> Callable[[ParamHandler], ParamHandler]:\n", + " if methods is None:\n", + " methods = [\"GET\"]\n", + "\n", + " def decorator(func: ParamHandler) -> ParamHandler:\n", + " route = Route(\n", + " template=template,\n", + " pattern=compile_route_pattern(template),\n", + " handler=func,\n", + " methods=set(m.upper() for m in methods),\n", + " )\n", + " self._routes.append(route)\n", + " return func\n", + "\n", + " return decorator\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + "\n", + " for route in self._routes:\n", + " match = route.pattern.match(path)\n", + " if match:\n", + " if method not in route.methods:\n", + " body = json.dumps({\"error\": \"Method Not Allowed\"}).encode()\n", + " start_response(\"405 Method Not Allowed\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + " params = match.groupdict()\n", + " return route.handler(environ, start_response, params)\n", + "\n", + " body = json.dumps({\"error\": \"Not Found\"}).encode()\n", + " start_response(\"404 Not Found\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Demonstrate regex-based routing\n", + "print(\"=== Pattern Compilation ===\")\n", + "test_templates = [\"/users/\", \"/posts//comments/\"]\n", + "for tmpl in test_templates:\n", + " pattern = compile_route_pattern(tmpl)\n", + " print(f\" {tmpl:45s} -> {pattern.pattern}\")\n", + "\n", + "# Build an app with parameterized routes\n", + "api = RegexRouter()\n", + "\n", + "\n", + "@api.route(\"/users/\")\n", + "def get_user(\n", + " environ: Environ, start_response: StartResponse, params: dict[str, str]\n", + ") -> Iterable[bytes]:\n", + " body = json.dumps({\"user_id\": params[\"user_id\"], \"name\": \"Alice\"}).encode()\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "@api.route(\"/posts//comments/\")\n", + "def get_comment(\n", + " environ: Environ, start_response: StartResponse, params: dict[str, str]\n", + ") -> Iterable[bytes]:\n", + " body = json.dumps({\n", + " \"post_slug\": params[\"slug\"],\n", + " \"comment_id\": params[\"comment_id\"],\n", + " }).encode()\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Test parameterized routing\n", + "print(\"\\n=== Parameterized Route Tests ===\")\n", + "for path in [\"/users/42\", \"/users/alice\", \"/posts/hello-world/comments/7\", \"/nope\"]:\n", + " captured_status = \"\"\n", + "\n", + " def sr(status: str, headers: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal captured_status\n", + " captured_status = status\n", + " return lambda *a: None\n", + "\n", + " environ: Environ = {\"REQUEST_METHOD\": \"GET\", \"PATH_INFO\": path}\n", + " body = b\"\".join(api(environ, sr))\n", + " print(f\" GET {path:45s} -> {captured_status:15s} | {body.decode()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query Parameter Parsing\n", + "\n", + "Query parameters appear after `?` in a URL (e.g., `/search?q=python&page=2`).\n", + "The `urllib.parse` module provides `parse_qs()` and `parse_qsl()` for parsing\n", + "these into dictionaries or lists of pairs. Note that a key can appear multiple\n", + "times, so `parse_qs()` returns lists of values." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from urllib.parse import parse_qs, parse_qsl, urlencode, quote, unquote\n", + "\n", + "\n", + "# Basic query string parsing\n", + "query_string = \"q=python+web&page=2&tag=wsgi&tag=http&sort=relevance\"\n", + "\n", + "print(\"=== parse_qs (dict of lists) ===\")\n", + "params: dict[str, list[str]] = parse_qs(query_string)\n", + "for key, values in params.items():\n", + " print(f\" {key}: {values}\")\n", + "\n", + "print(\"\\n=== parse_qsl (list of pairs) ===\")\n", + "pairs: list[tuple[str, str]] = parse_qsl(query_string)\n", + "for key, value in pairs:\n", + " print(f\" {key} = {value}\")\n", + "\n", + "# URL encoding and decoding\n", + "print(\"\\n=== URL Encoding ===\")\n", + "raw_text = \"hello world & special=chars/here\"\n", + "encoded = quote(raw_text)\n", + "decoded = unquote(encoded)\n", + "print(f\" Raw: {raw_text}\")\n", + "print(f\" Encoded: {encoded}\")\n", + "print(f\" Decoded: {decoded}\")\n", + "\n", + "# Build a query string from parameters\n", + "print(\"\\n=== Building Query Strings ===\")\n", + "search_params: dict[str, str | list[str]] = {\n", + " \"q\": \"python web\",\n", + " \"page\": \"1\",\n", + " \"tags\": [\"wsgi\", \"http\", \"rest\"], # type: ignore[dict-item]\n", + "}\n", + "# urlencode with doseq=True handles list values\n", + "qs = urlencode(search_params, doseq=True)\n", + "print(f\" Encoded: {qs}\")\n", + "\n", + "\n", + "# Helper function for extracting typed query params\n", + "def get_query_param(\n", + " query_string: str,\n", + " key: str,\n", + " default: str = \"\",\n", + " as_int: bool = False,\n", + ") -> str | int:\n", + " \"\"\"Extract a single query parameter with optional int conversion.\"\"\"\n", + " params = parse_qs(query_string)\n", + " values = params.get(key, [])\n", + " value = values[0] if values else default\n", + " if as_int:\n", + " try:\n", + " return int(value)\n", + " except (ValueError, TypeError):\n", + " return 0\n", + " return value\n", + "\n", + "\n", + "print(\"\\n=== Typed Query Param Extraction ===\")\n", + "qs_example = \"page=3&limit=25&search=python\"\n", + "print(f\" page (int): {get_query_param(qs_example, 'page', as_int=True)}\")\n", + "print(f\" limit (int): {get_query_param(qs_example, 'limit', as_int=True)}\")\n", + "print(f\" search (str): {get_query_param(qs_example, 'search')}\")\n", + "print(f\" missing (default): {get_query_param(qs_example, 'missing', 'fallback')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Template Rendering with `string.Template`\n", + "\n", + "Before reaching for Jinja2 or Mako, Python's standard library offers\n", + "`string.Template` for simple variable substitution in templates. It uses\n", + "`$name` or `${name}` placeholders and is safe against code injection\n", + "(unlike f-strings, which execute arbitrary expressions)." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from string import Template\n", + "\n", + "\n", + "# Basic string.Template usage\n", + "page_template = Template(\"\"\"\\\n", + "\n", + "\n", + "$title\n", + "\n", + "

$heading

\n", + "

Welcome, $username!

\n", + "

You have $message_count new messages.

\n", + "\n", + "\"\"\")\n", + "\n", + "# Substitute values into the template\n", + "rendered = page_template.substitute(\n", + " title=\"Dashboard\",\n", + " heading=\"User Dashboard\",\n", + " username=\"Alice\",\n", + " message_count=\"5\",\n", + ")\n", + "print(\"=== Rendered Template ===\")\n", + "print(rendered)\n", + "\n", + "# safe_substitute does not raise on missing keys\n", + "print(\"\\n=== Safe Substitute (missing keys) ===\")\n", + "partial = Template(\"Hello, $name! Your role is $role.\")\n", + "result = partial.safe_substitute(name=\"Bob\")\n", + "print(f\" Result: {result}\")\n", + "print(\" Note: $role was left as-is because it was missing\")\n", + "\n", + "# Template with ${braced} syntax for adjacent text\n", + "print(\"\\n=== Braced Syntax ===\")\n", + "braced = Template(\"File: ${filename}_v${version}.txt\")\n", + "print(f\" {braced.substitute(filename='report', version='2')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HTML Escaping for Security\n", + "\n", + "Injecting user-provided data into HTML without escaping creates **Cross-Site\n", + "Scripting (XSS)** vulnerabilities. The `html.escape()` function converts\n", + "dangerous characters (`<`, `>`, `&`, `\"`, `'`) into HTML entities. Always\n", + "escape user input before inserting it into HTML templates." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import html\n", + "from string import Template\n", + "\n", + "\n", + "# Demonstrate the XSS problem\n", + "malicious_input = ''\n", + "\n", + "print(\"=== Without Escaping (DANGEROUS) ===\")\n", + "unsafe_template = Template(\"

Welcome, $username!

\")\n", + "unsafe_html = unsafe_template.substitute(username=malicious_input)\n", + "print(f\" {unsafe_html}\")\n", + "print(\" ^ The script tag would execute in a browser!\")\n", + "\n", + "print(\"\\n=== With html.escape() (SAFE) ===\")\n", + "safe_username = html.escape(malicious_input)\n", + "safe_html = unsafe_template.substitute(username=safe_username)\n", + "print(f\" {safe_html}\")\n", + "print(\" ^ The script tag is rendered as harmless text.\")\n", + "\n", + "# What html.escape does to each character\n", + "print(\"\\n=== Character Escaping ===\")\n", + "test_chars: list[str] = [\"<\", \">\", \"&\", '\"', \"'\", \"normal text\"]\n", + "for char in test_chars:\n", + " escaped = html.escape(char, quote=True)\n", + " print(f\" {char!r:20s} -> {escaped!r}\")\n", + "\n", + "\n", + "# A safe template rendering function\n", + "def render_template(template_str: str, **kwargs: str) -> str:\n", + " \"\"\"Render a template with all values HTML-escaped.\"\"\"\n", + " escaped_kwargs = {key: html.escape(str(value)) for key, value in kwargs.items()}\n", + " return Template(template_str).substitute(escaped_kwargs)\n", + "\n", + "\n", + "print(\"\\n=== Safe Rendering Function ===\")\n", + "result = render_template(\n", + " \"

Hello, $name! Search for: $query

\",\n", + " name=\"Alice \",\n", + " query='test\" onclick=\"alert(1)',\n", + ")\n", + "print(f\" {result}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A Template Engine with Loops and Conditionals\n", + "\n", + "`string.Template` only supports simple substitution. For rendering lists and\n", + "conditional content, we need something more. Below is a minimal template engine\n", + "that uses Python string formatting with helper functions to generate HTML\n", + "from data structures -- a pattern common in server-side rendering." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import html\n", + "from dataclasses import dataclass\n", + "\n", + "\n", + "@dataclass\n", + "class User:\n", + " name: str\n", + " email: str\n", + " is_admin: bool = False\n", + "\n", + "\n", + "def render_user_list(users: list[User], title: str = \"User List\") -> str:\n", + " \"\"\"Render an HTML page listing users, with escaping.\"\"\"\n", + " rows: list[str] = []\n", + " for user in users:\n", + " name = html.escape(user.name)\n", + " email = html.escape(user.email)\n", + " badge = ' Admin' if user.is_admin else \"\"\n", + " rows.append(\n", + " f\" {name}{badge}{email}\"\n", + " )\n", + "\n", + " table_body = \"\\n\".join(rows) if rows else \" No users\"\n", + " safe_title = html.escape(title)\n", + "\n", + " return f\"\"\"\n", + "\n", + "{safe_title}\n", + "\n", + "

{safe_title}

\n", + "

Total: {len(users)} user(s)

\n", + " \n", + " \n", + "{table_body}\n", + "
NameEmail
\n", + "\n", + "\"\"\"\n", + "\n", + "\n", + "# Render with normal data\n", + "users = [\n", + " User(\"Alice\", \"alice@example.com\", is_admin=True),\n", + " User(\"Bob\", \"bob@example.com\"),\n", + " User(\"Charlie \", \"charlie@example.com\"),\n", + "]\n", + "\n", + "print(render_user_list(users, \"Team Members\"))\n", + "print(\"\\n--- Note: Charlie's malicious name is safely escaped. ---\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Form Handling: Parsing POST Data\n", + "\n", + "HTML forms submit data as URL-encoded key-value pairs in the request body\n", + "(Content-Type: `application/x-www-form-urlencoded`). The `urllib.parse.parse_qs()`\n", + "function parses this format, just as it does query strings. For file uploads,\n", + "forms use `multipart/form-data`, which requires more complex parsing." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import io\n", + "from urllib.parse import parse_qs, urlencode\n", + "from typing import Callable, Iterable\n", + "import json\n", + "import html\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "\n", + "def parse_form_data(environ: Environ) -> dict[str, list[str]]:\n", + " \"\"\"Parse URL-encoded form data from a POST request body.\"\"\"\n", + " content_type = str(environ.get(\"CONTENT_TYPE\", \"\"))\n", + "\n", + " if \"application/x-www-form-urlencoded\" not in content_type:\n", + " return {}\n", + "\n", + " try:\n", + " content_length = int(str(environ.get(\"CONTENT_LENGTH\", \"0\") or \"0\"))\n", + " except ValueError:\n", + " content_length = 0\n", + "\n", + " wsgi_input = environ.get(\"wsgi.input\")\n", + " if not wsgi_input or content_length == 0:\n", + " return {}\n", + "\n", + " body = wsgi_input.read(content_length) # type: ignore[union-attr]\n", + " return parse_qs(body.decode(\"utf-8\"))\n", + "\n", + "\n", + "# Simulate a form submission\n", + "form_body = urlencode({\n", + " \"username\": \"alice\",\n", + " \"email\": \"alice@example.com\",\n", + " \"role\": \"admin\",\n", + "}).encode(\"utf-8\")\n", + "\n", + "form_environ: Environ = {\n", + " \"REQUEST_METHOD\": \"POST\",\n", + " \"PATH_INFO\": \"/register\",\n", + " \"CONTENT_TYPE\": \"application/x-www-form-urlencoded\",\n", + " \"CONTENT_LENGTH\": str(len(form_body)),\n", + " \"wsgi.input\": io.BytesIO(form_body),\n", + "}\n", + "\n", + "print(\"=== Raw Form Body ===\")\n", + "print(f\" {form_body.decode()}\")\n", + "\n", + "print(\"\\n=== Parsed Form Data ===\")\n", + "form_data = parse_form_data(form_environ)\n", + "for key, values in form_data.items():\n", + " print(f\" {key}: {values}\")\n", + "\n", + "\n", + "# A WSGI handler that renders a form and processes submissions\n", + "def form_handler(\n", + " environ: Environ, start_response: StartResponse\n", + ") -> Iterable[bytes]:\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + "\n", + " if method == \"GET\":\n", + " # Render the form\n", + " form_html = \"\"\"\n", + "\n", + "

Registration

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "\"\"\"\n", + " body = form_html.encode(\"utf-8\")\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"text/html; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + " elif method == \"POST\":\n", + " # Process the form data\n", + " data = parse_form_data(environ)\n", + " username = html.escape(data.get(\"username\", [\"\"])[0])\n", + " email = html.escape(data.get(\"email\", [\"\"])[0])\n", + "\n", + " response_html = f\"\"\"\n", + "\n", + "

Registration Successful

\n", + "

Username: {username}

\n", + "

Email: {email}

\n", + "\"\"\"\n", + " body = response_html.encode(\"utf-8\")\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"text/html; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + " else:\n", + " body = b\"Method Not Allowed\"\n", + " start_response(\"405 Method Not Allowed\", [\n", + " (\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Test GET (show form)\n", + "print(\"\\n=== GET /register (show form) ===\")\n", + "status_result = \"\"\n", + "def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " global status_result\n", + " status_result = s\n", + " return lambda *a: None\n", + "\n", + "body_out = b\"\".join(form_handler({\"REQUEST_METHOD\": \"GET\", \"PATH_INFO\": \"/register\"}, sr))\n", + "print(f\" Status: {status_result}\")\n", + "print(f\" Contains
: {'\\r\\n\"\n", + " f\"--{boundary}--\\r\\n\"\n", + ")\n", + "\n", + "print(\"=== Multipart Form Data Structure ===\")\n", + "print(f\"Content-Type: multipart/form-data; boundary={boundary}\")\n", + "print(f\"Content-Length: {len(multipart_body)}\")\n", + "print()\n", + "print(multipart_body)\n", + "\n", + "# Parse the parts manually (for educational purposes)\n", + "print(\"=== Parsing Parts ===\")\n", + "parts = multipart_body.split(f\"--{boundary}\")\n", + "for i, part in enumerate(parts):\n", + " part = part.strip()\n", + " if not part or part == \"--\":\n", + " continue\n", + " # Split headers from body at the blank line\n", + " if \"\\r\\n\\r\\n\" in part:\n", + " header_section, body_section = part.split(\"\\r\\n\\r\\n\", 1)\n", + " print(f\"\\nPart {i}:\")\n", + " print(f\" Headers: {header_section}\")\n", + " print(f\" Body: {body_section[:50]}\")\n", + "\n", + "print(\"\\n--- In practice, use the 'multipart' package or framework utilities. ---\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Static File Serving Patterns\n", + "\n", + "During development, Python applications often need to serve static files (CSS,\n", + "JavaScript, images). In production, a reverse proxy (nginx, Caddy) handles this.\n", + "Below is a WSGI middleware that serves static files from a directory, with\n", + "content type detection and basic security (preventing directory traversal)." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import mimetypes\n", + "import os\n", + "from pathlib import Path\n", + "from typing import Callable, Iterable\n", + "from http import HTTPStatus\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class StaticFileMiddleware:\n", + " \"\"\"WSGI middleware that serves static files from a directory.\n", + "\n", + " Requests matching the url_prefix are served from static_dir.\n", + " All other requests are passed through to the wrapped app.\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " app: WSGIApp,\n", + " static_dir: str | Path,\n", + " url_prefix: str = \"/static\",\n", + " ) -> None:\n", + " self._app = app\n", + " self._static_dir = Path(static_dir).resolve()\n", + " self._url_prefix = url_prefix.rstrip(\"/\")\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + "\n", + " if not path.startswith(self._url_prefix + \"/\"):\n", + " return self._app(environ, start_response)\n", + "\n", + " # Extract the file path relative to the static directory\n", + " relative_path = path[len(self._url_prefix) + 1:]\n", + " file_path = (self._static_dir / relative_path).resolve()\n", + "\n", + " # Security: prevent directory traversal\n", + " if not str(file_path).startswith(str(self._static_dir)):\n", + " body = b\"Forbidden\"\n", + " start_response(\n", + " f\"{HTTPStatus.FORBIDDEN.value} {HTTPStatus.FORBIDDEN.phrase}\",\n", + " [(\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body)))],\n", + " )\n", + " return [body]\n", + "\n", + " if not file_path.is_file():\n", + " body = b\"Not Found\"\n", + " start_response(\n", + " f\"{HTTPStatus.NOT_FOUND.value} {HTTPStatus.NOT_FOUND.phrase}\",\n", + " [(\"Content-Type\", \"text/plain\"),\n", + " (\"Content-Length\", str(len(body)))],\n", + " )\n", + " return [body]\n", + "\n", + " # Serve the file\n", + " content_type, _ = mimetypes.guess_type(str(file_path))\n", + " if content_type is None:\n", + " content_type = \"application/octet-stream\"\n", + "\n", + " body = file_path.read_bytes()\n", + " start_response(\n", + " f\"{HTTPStatus.OK.value} {HTTPStatus.OK.phrase}\",\n", + " [\n", + " (\"Content-Type\", content_type),\n", + " (\"Content-Length\", str(len(body))),\n", + " (\"Cache-Control\", \"public, max-age=3600\"),\n", + " ],\n", + " )\n", + " return [body]\n", + "\n", + "\n", + "# Demonstrate MIME type detection\n", + "print(\"=== MIME Type Detection ===\")\n", + "test_files = [\"style.css\", \"app.js\", \"image.png\", \"data.json\", \"page.html\", \"doc.pdf\"]\n", + "for filename in test_files:\n", + " mime_type, encoding = mimetypes.guess_type(filename)\n", + " print(f\" {filename:15s} -> {mime_type or 'unknown'}\")\n", + "\n", + "# Demonstrate path traversal prevention\n", + "print(\"\\n=== Directory Traversal Prevention ===\")\n", + "static_root = Path(\"/var/www/static\").resolve()\n", + "test_paths = [\n", + " \"style.css\",\n", + " \"../../../etc/passwd\",\n", + " \"images/../../../secret.txt\",\n", + " \"valid/nested/file.js\",\n", + "]\n", + "for rel_path in test_paths:\n", + " resolved = (static_root / rel_path).resolve()\n", + " is_safe = str(resolved).startswith(str(static_root))\n", + " status = \"SAFE\" if is_safe else \"BLOCKED\"\n", + " print(f\" {rel_path:35s} -> [{status}] {resolved}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Putting It Together: A Mini Web Application\n", + "\n", + "Below we combine routing, templates, query parameters, form handling, and\n", + "static file patterns into a cohesive mini web application. This demonstrates\n", + "how frameworks organize these concerns." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import re\n", + "import json\n", + "import html\n", + "import io\n", + "from http import HTTPStatus\n", + "from typing import Callable, Iterable\n", + "from urllib.parse import parse_qs\n", + "from string import Template\n", + "from dataclasses import dataclass, field\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "\n", + "# Simple in-memory \"database\"\n", + "USERS: dict[str, dict[str, str]] = {\n", + " \"1\": {\"name\": \"Alice\", \"email\": \"alice@example.com\"},\n", + " \"2\": {\"name\": \"Bob\", \"email\": \"bob@example.com\"},\n", + "}\n", + "\n", + "# HTML templates\n", + "BASE_TEMPLATE = Template(\"\"\"\n", + "\n", + "$title\n", + "\n", + " \n", + "
\n", + " $content\n", + "\n", + "\"\"\")\n", + "\n", + "\n", + "def render_page(title: str, content: str) -> bytes:\n", + " \"\"\"Render content inside the base template.\"\"\"\n", + " page = BASE_TEMPLATE.substitute(title=html.escape(title), content=content)\n", + " return page.encode(\"utf-8\")\n", + "\n", + "\n", + "def html_response(\n", + " body: bytes,\n", + " start_response: StartResponse,\n", + " status: HTTPStatus = HTTPStatus.OK,\n", + ") -> Iterable[bytes]:\n", + " start_response(f\"{status.value} {status.phrase}\", [\n", + " (\"Content-Type\", \"text/html; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Route handlers\n", + "def handle_home(\n", + " environ: Environ, start_response: StartResponse, params: dict[str, str]\n", + ") -> Iterable[bytes]:\n", + " body = render_page(\"Home\", \"

Welcome!

A mini web app built with WSGI.

\")\n", + " return html_response(body, start_response)\n", + "\n", + "\n", + "def handle_user_list(\n", + " environ: Environ, start_response: StartResponse, params: dict[str, str]\n", + ") -> Iterable[bytes]:\n", + " # Support optional search query parameter\n", + " qs = str(environ.get(\"QUERY_STRING\", \"\"))\n", + " search = parse_qs(qs).get(\"q\", [\"\"])[0].lower()\n", + "\n", + " rows: list[str] = []\n", + " for uid, user in USERS.items():\n", + " if search and search not in user[\"name\"].lower():\n", + " continue\n", + " name = html.escape(user[\"name\"])\n", + " email = html.escape(user[\"email\"])\n", + " rows.append(f\"{name}{email}\")\n", + "\n", + " table = \"\" + \"\".join(rows) + \"
NameEmail
\"\n", + " search_form = (\n", + " ''\n", + " )\n", + " content = f\"

Users

{search_form}{table}\"\n", + " body = render_page(\"Users\", content)\n", + " return html_response(body, start_response)\n", + "\n", + "\n", + "def handle_user_detail(\n", + " environ: Environ, start_response: StartResponse, params: dict[str, str]\n", + ") -> Iterable[bytes]:\n", + " user_id = params[\"user_id\"]\n", + " user = USERS.get(user_id)\n", + " if user is None:\n", + " body = render_page(\"Not Found\", \"

User not found

\")\n", + " return html_response(body, start_response, HTTPStatus.NOT_FOUND)\n", + "\n", + " name = html.escape(user[\"name\"])\n", + " email = html.escape(user[\"email\"])\n", + " content = f\"

{name}

Email: {email}

Back to list\"\n", + " body = render_page(name, content)\n", + " return html_response(body, start_response)\n", + "\n", + "\n", + "# Build the router using our RegexRouter from earlier\n", + "@dataclass\n", + "class MiniRoute:\n", + " pattern: re.Pattern[str]\n", + " handler: Callable[..., Iterable[bytes]]\n", + "\n", + "\n", + "routes: list[MiniRoute] = [\n", + " MiniRoute(re.compile(r\"^/$\"), handle_home),\n", + " MiniRoute(re.compile(r\"^/users$\"), handle_user_list),\n", + " MiniRoute(re.compile(r\"^/users/(?P\\w+)$\"), handle_user_detail),\n", + "]\n", + "\n", + "\n", + "def mini_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " for route in routes:\n", + " match = route.pattern.match(path)\n", + " if match:\n", + " return route.handler(environ, start_response, match.groupdict())\n", + "\n", + " body = render_page(\"Not Found\", \"

404 - Page not found

\")\n", + " return html_response(body, start_response, HTTPStatus.NOT_FOUND)\n", + "\n", + "\n", + "# Test the mini application\n", + "def test_mini(method: str, path: str, query_string: str = \"\") -> None:\n", + " status_line = \"\"\n", + " def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal status_line\n", + " status_line = s\n", + " return lambda *a: None\n", + "\n", + " environ: Environ = {\n", + " \"REQUEST_METHOD\": method,\n", + " \"PATH_INFO\": path,\n", + " \"QUERY_STRING\": query_string,\n", + " }\n", + " body = b\"\".join(mini_app(environ, sr))\n", + " # Show just a snippet of the body\n", + " decoded = body.decode()\n", + " has_users_table = \" {status_line:15s} (table: {has_users_table}, len: {len(body)})\")\n", + "\n", + "\n", + "print(\"=== Mini Web App Tests ===\")\n", + "test_mini(\"GET\", \"/\")\n", + "test_mini(\"GET\", \"/users\")\n", + "test_mini(\"GET\", \"/users\", \"q=alice\")\n", + "test_mini(\"GET\", \"/users/1\")\n", + "test_mini(\"GET\", \"/users/2\")\n", + "test_mini(\"GET\", \"/users/999\")\n", + "test_mini(\"GET\", \"/nonexistent\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook covered the core patterns used by web frameworks for routing and\n", + "rendering:\n", + "\n", + "1. **URL routing**: Mapping paths to handler functions using dictionaries and\n", + " regex patterns\n", + "2. **Route decorators**: The `@app.route(\"/path\")` pattern that registers\n", + " handlers declaratively (as used by Flask)\n", + "3. **Path parameters**: Extracting dynamic values from URL segments with\n", + " named regex groups (``)\n", + "4. **Query parameters**: Parsing `?key=value` pairs with `urllib.parse.parse_qs()`\n", + "5. **Template rendering**: Using `string.Template` for simple substitution and\n", + " building HTML programmatically\n", + "6. **HTML escaping**: Preventing XSS attacks with `html.escape()` -- always\n", + " escape user input before inserting it into HTML\n", + "7. **Form handling**: Parsing URL-encoded POST data and understanding multipart\n", + " form structure for file uploads\n", + "8. **Static file serving**: MIME type detection, directory traversal prevention,\n", + " and middleware patterns for serving files\n", + "\n", + "The next notebook covers REST API design, JSON handling, middleware, error\n", + "handling, and CORS -- building on these routing fundamentals to create\n", + "API-focused web applications." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_22/03_rest_apis.ipynb b/src/chapter_22/03_rest_apis.ipynb new file mode 100644 index 0000000..cc6f754 --- /dev/null +++ b/src/chapter_22/03_rest_apis.ipynb @@ -0,0 +1,1385 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 22: REST APIs\n", + "\n", + "**Web Development Fundamentals**\n", + "\n", + "REST (Representational State Transfer) is an architectural style for building web\n", + "APIs that leverages HTTP semantics. This notebook covers REST conventions, JSON\n", + "request/response handling, content negotiation, middleware patterns, error handling,\n", + "CORS, API versioning, and culminates in a complete in-memory CRUD API -- all built\n", + "with standard library tools on top of the WSGI foundation from the previous notebooks." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## REST Conventions: Resources, Methods, Status Codes\n", + "\n", + "REST maps HTTP methods to CRUD operations on resources:\n", + "\n", + "| Operation | Method | URL Pattern | Success Code | Body |\n", + "|-----------|--------|-------------|-------------|------|\n", + "| List all | GET | `/resources` | 200 OK | Collection |\n", + "| Create | POST | `/resources` | 201 Created | New resource |\n", + "| Read one | GET | `/resources/{id}` | 200 OK | Single resource |\n", + "| Replace | PUT | `/resources/{id}` | 200 OK | Updated resource |\n", + "| Update | PATCH | `/resources/{id}` | 200 OK | Updated resource |\n", + "| Delete | DELETE | `/resources/{id}` | 204 No Content | Empty |\n", + "\n", + "Key principles:\n", + "- **Resources** are nouns (e.g., `/users`, `/posts`), not verbs\n", + "- **HTTP methods** express the action\n", + "- **Status codes** communicate the outcome\n", + "- **JSON** is the standard interchange format" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass\n", + "from http import HTTPStatus\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class RESTEndpoint:\n", + " \"\"\"Documents a REST API endpoint.\"\"\"\n", + " operation: str\n", + " method: str\n", + " url_pattern: str\n", + " success_status: HTTPStatus\n", + " has_request_body: bool\n", + " has_response_body: bool\n", + "\n", + "\n", + "# Standard CRUD endpoints for a \"books\" resource\n", + "book_endpoints: list[RESTEndpoint] = [\n", + " RESTEndpoint(\"List all\", \"GET\", \"/api/books\",\n", + " HTTPStatus.OK, False, True),\n", + " RESTEndpoint(\"Create\", \"POST\", \"/api/books\",\n", + " HTTPStatus.CREATED, True, True),\n", + " RESTEndpoint(\"Read one\", \"GET\", \"/api/books/{id}\",\n", + " HTTPStatus.OK, False, True),\n", + " RESTEndpoint(\"Full update\", \"PUT\", \"/api/books/{id}\",\n", + " HTTPStatus.OK, True, True),\n", + " RESTEndpoint(\"Partial update\", \"PATCH\", \"/api/books/{id}\",\n", + " HTTPStatus.OK, True, True),\n", + " RESTEndpoint(\"Delete\", \"DELETE\", \"/api/books/{id}\",\n", + " HTTPStatus.NO_CONTENT, False, False),\n", + "]\n", + "\n", + "print(\"=== REST Endpoints for 'books' Resource ===\")\n", + "print(f\"{'Operation':<17s} {'Method':<8s} {'URL':<22s} {'Status':<20s} {'Req Body':<10s} {'Resp Body'}\")\n", + "print(\"-\" * 90)\n", + "for ep in book_endpoints:\n", + " status_str = f\"{ep.success_status.value} {ep.success_status.phrase}\"\n", + " print(\n", + " f\"{ep.operation:<17s} {ep.method:<8s} {ep.url_pattern:<22s} \"\n", + " f\"{status_str:<20s} {str(ep.has_request_body):<10s} {ep.has_response_body}\"\n", + " )" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSON Request/Response Handling\n", + "\n", + "REST APIs communicate using JSON. On the server side, this means:\n", + "- **Requests**: Parse JSON from the request body (`wsgi.input`)\n", + "- **Responses**: Serialize Python objects to JSON, set `Content-Type: application/json`\n", + "\n", + "Below are helper functions that handle JSON serialization, content-type\n", + "validation, and error handling for malformed input." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import io\n", + "from http import HTTPStatus\n", + "from typing import Any, Callable, Iterable\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "\n", + "class JSONParseError(Exception):\n", + " \"\"\"Raised when request body cannot be parsed as JSON.\"\"\"\n", + " pass\n", + "\n", + "\n", + "def read_json_body(environ: Environ) -> Any:\n", + " \"\"\"Read and parse JSON from the request body.\n", + "\n", + " Raises JSONParseError if the body is not valid JSON.\n", + " Returns None if the body is empty.\n", + " \"\"\"\n", + " try:\n", + " content_length = int(str(environ.get(\"CONTENT_LENGTH\", \"0\") or \"0\"))\n", + " except ValueError:\n", + " content_length = 0\n", + "\n", + " if content_length == 0:\n", + " return None\n", + "\n", + " wsgi_input = environ.get(\"wsgi.input\")\n", + " if not wsgi_input or not hasattr(wsgi_input, \"read\"):\n", + " return None\n", + "\n", + " raw: bytes = wsgi_input.read(content_length) # type: ignore[union-attr]\n", + " try:\n", + " return json.loads(raw)\n", + " except (json.JSONDecodeError, UnicodeDecodeError) as e:\n", + " raise JSONParseError(f\"Invalid JSON: {e}\") from e\n", + "\n", + "\n", + "def json_response(\n", + " data: Any,\n", + " status: HTTPStatus,\n", + " start_response: StartResponse,\n", + " extra_headers: list[tuple[str, str]] | None = None,\n", + ") -> Iterable[bytes]:\n", + " \"\"\"Build a JSON response with proper headers.\"\"\"\n", + " body = json.dumps(data, indent=2, default=str).encode(\"utf-8\")\n", + " headers: list[tuple[str, str]] = [\n", + " (\"Content-Type\", \"application/json; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ]\n", + " if extra_headers:\n", + " headers.extend(extra_headers)\n", + " start_response(f\"{status.value} {status.phrase}\", headers)\n", + " return [body]\n", + "\n", + "\n", + "def no_content_response(start_response: StartResponse) -> Iterable[bytes]:\n", + " \"\"\"Build a 204 No Content response (for DELETE).\"\"\"\n", + " start_response(\n", + " f\"{HTTPStatus.NO_CONTENT.value} {HTTPStatus.NO_CONTENT.phrase}\",\n", + " [],\n", + " )\n", + " return []\n", + "\n", + "\n", + "# Test JSON parsing\n", + "print(\"=== Valid JSON ===\")\n", + "valid_body = json.dumps({\"title\": \"Python Cookbook\", \"author\": \"Beazley\"}).encode()\n", + "env: Environ = {\n", + " \"CONTENT_LENGTH\": str(len(valid_body)),\n", + " \"wsgi.input\": io.BytesIO(valid_body),\n", + "}\n", + "parsed = read_json_body(env)\n", + "print(f\" Parsed: {parsed}\")\n", + "print(f\" Type: {type(parsed).__name__}\")\n", + "\n", + "print(\"\\n=== Invalid JSON ===\")\n", + "invalid_body = b\"{not valid json}\"\n", + "env_bad: Environ = {\n", + " \"CONTENT_LENGTH\": str(len(invalid_body)),\n", + " \"wsgi.input\": io.BytesIO(invalid_body),\n", + "}\n", + "try:\n", + " read_json_body(env_bad)\n", + "except JSONParseError as e:\n", + " print(f\" Error: {e}\")\n", + "\n", + "print(\"\\n=== Empty Body ===\")\n", + "env_empty: Environ = {\"CONTENT_LENGTH\": \"0\", \"wsgi.input\": io.BytesIO(b\"\")}\n", + "result = read_json_body(env_empty)\n", + "print(f\" Result: {result}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Content Negotiation: Accept and Content-Type Headers\n", + "\n", + "Content negotiation allows clients and servers to agree on the data format:\n", + "\n", + "- **`Accept`** (request header): What formats the client can handle\n", + "- **`Content-Type`** (request/response header): What format the body is in\n", + "\n", + "A well-designed API checks the `Accept` header and returns `406 Not Acceptable`\n", + "if it cannot produce the requested format. It also validates the `Content-Type`\n", + "of incoming requests." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from http import HTTPStatus\n", + "\n", + "\n", + "def parse_accept_header(accept: str) -> list[tuple[str, float]]:\n", + " \"\"\"Parse an Accept header into a list of (media_type, quality) tuples.\n", + "\n", + " Example: 'text/html, application/json;q=0.9, */*;q=0.1'\n", + " Returns: [('text/html', 1.0), ('application/json', 0.9), ('*/*', 0.1)]\n", + " \"\"\"\n", + " result: list[tuple[str, float]] = []\n", + " for item in accept.split(\",\"):\n", + " parts = item.strip().split(\";\")\n", + " media_type = parts[0].strip()\n", + " quality = 1.0\n", + " for param in parts[1:]:\n", + " param = param.strip()\n", + " if param.startswith(\"q=\"):\n", + " try:\n", + " quality = float(param[2:])\n", + " except ValueError:\n", + " quality = 0.0\n", + " result.append((media_type, quality))\n", + " # Sort by quality descending\n", + " result.sort(key=lambda x: x[1], reverse=True)\n", + " return result\n", + "\n", + "\n", + "def accepts_json(environ: dict[str, object]) -> bool:\n", + " \"\"\"Check if the client accepts JSON responses.\"\"\"\n", + " accept = str(environ.get(\"HTTP_ACCEPT\", \"*/*\"))\n", + " preferences = parse_accept_header(accept)\n", + " for media_type, quality in preferences:\n", + " if quality <= 0:\n", + " continue\n", + " if media_type in (\"application/json\", \"*/*\", \"application/*\"):\n", + " return True\n", + " return False\n", + "\n", + "\n", + "def has_json_content_type(environ: dict[str, object]) -> bool:\n", + " \"\"\"Check if the request body is JSON.\"\"\"\n", + " content_type = str(environ.get(\"CONTENT_TYPE\", \"\")).lower()\n", + " return \"application/json\" in content_type\n", + "\n", + "\n", + "# Parse various Accept headers\n", + "print(\"=== Parsing Accept Headers ===\")\n", + "test_headers = [\n", + " \"application/json\",\n", + " \"text/html, application/json;q=0.9\",\n", + " \"text/html, application/xhtml+xml, */*;q=0.8\",\n", + " \"application/xml;q=0.9, application/json;q=1.0\",\n", + " \"text/plain;q=0.5, text/html;q=0.8\",\n", + "]\n", + "\n", + "for header in test_headers:\n", + " parsed = parse_accept_header(header)\n", + " accepts = accepts_json({\"HTTP_ACCEPT\": header})\n", + " print(f\" Accept: {header}\")\n", + " print(f\" Parsed: {parsed}\")\n", + " print(f\" JSON OK: {accepts}\")\n", + " print()\n", + "\n", + "# Content-Type validation\n", + "print(\"=== Content-Type Validation ===\")\n", + "test_content_types = [\n", + " \"application/json\",\n", + " \"application/json; charset=utf-8\",\n", + " \"text/plain\",\n", + " \"application/x-www-form-urlencoded\",\n", + " \"\",\n", + "]\n", + "\n", + "for ct in test_content_types:\n", + " is_json = has_json_content_type({\"CONTENT_TYPE\": ct})\n", + " print(f\" Content-Type: {ct or '(empty)':45s} -> JSON: {is_json}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Middleware Pattern: Wrapping WSGI Apps\n", + "\n", + "Middleware is a WSGI app that wraps another WSGI app, adding cross-cutting\n", + "behavior (logging, authentication, error handling, CORS) without modifying\n", + "the inner application. Middleware follows the decorator/wrapper pattern:\n", + "it receives a request, optionally modifies it, calls the inner app, and\n", + "optionally modifies the response." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import time\n", + "import json\n", + "from typing import Callable, Iterable\n", + "from http import HTTPStatus\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class LoggingMiddleware:\n", + " \"\"\"Middleware that logs request method, path, status, and duration.\"\"\"\n", + "\n", + " def __init__(self, app: WSGIApp) -> None:\n", + " self._app = app\n", + " self.log: list[str] = [] # Capture logs for demonstration\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"?\"))\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " start = time.perf_counter()\n", + "\n", + " captured_status = \"\"\n", + "\n", + " def logging_start_response(\n", + " status: str, headers: list[tuple[str, str]]\n", + " ) -> Callable[..., object]:\n", + " nonlocal captured_status\n", + " captured_status = status\n", + " return start_response(status, headers)\n", + "\n", + " result = self._app(environ, logging_start_response)\n", + " duration_ms = (time.perf_counter() - start) * 1000\n", + "\n", + " log_entry = f\"{method} {path} -> {captured_status} ({duration_ms:.1f}ms)\"\n", + " self.log.append(log_entry)\n", + " return result\n", + "\n", + "\n", + "class AuthMiddleware:\n", + " \"\"\"Middleware that checks for an Authorization header.\"\"\"\n", + "\n", + " def __init__(self, app: WSGIApp, valid_tokens: set[str]) -> None:\n", + " self._app = app\n", + " self._valid_tokens = valid_tokens\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " auth_header = str(environ.get(\"HTTP_AUTHORIZATION\", \"\"))\n", + "\n", + " if auth_header.startswith(\"Bearer \"):\n", + " token = auth_header[7:]\n", + " if token in self._valid_tokens:\n", + " # Authenticated -- pass through\n", + " return self._app(environ, start_response)\n", + "\n", + " # Unauthorized\n", + " body = json.dumps({\"error\": \"Unauthorized\", \"detail\": \"Invalid or missing token\"}).encode()\n", + " start_response(\n", + " f\"{HTTPStatus.UNAUTHORIZED.value} {HTTPStatus.UNAUTHORIZED.phrase}\",\n", + " [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " (\"WWW-Authenticate\", 'Bearer realm=\"api\"'),\n", + " ],\n", + " )\n", + " return [body]\n", + "\n", + "\n", + "# Simple inner app\n", + "def inner_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " body = json.dumps({\"message\": \"Secret data\"}).encode()\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Stack middleware: logging -> auth -> app\n", + "valid_tokens = {\"secret-token-123\", \"admin-token-456\"}\n", + "app = LoggingMiddleware(AuthMiddleware(inner_app, valid_tokens))\n", + "\n", + "\n", + "def call_with_auth(token: str | None, path: str = \"/api/data\") -> None:\n", + " status_result = \"\"\n", + " def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal status_result\n", + " status_result = s\n", + " return lambda *a: None\n", + "\n", + " environ: Environ = {\n", + " \"REQUEST_METHOD\": \"GET\",\n", + " \"PATH_INFO\": path,\n", + " }\n", + " if token:\n", + " environ[\"HTTP_AUTHORIZATION\"] = f\"Bearer {token}\"\n", + "\n", + " body = b\"\".join(app(environ, sr))\n", + " token_display = token[:15] + \"...\" if token else \"(none)\"\n", + " print(f\" Token: {token_display:20s} -> {status_result:20s} | {body.decode()}\")\n", + "\n", + "\n", + "print(\"=== Middleware Chain Tests ===\")\n", + "call_with_auth(None) # No token\n", + "call_with_auth(\"wrong-token\") # Invalid token\n", + "call_with_auth(\"secret-token-123\") # Valid token\n", + "call_with_auth(\"admin-token-456\") # Another valid token\n", + "\n", + "print(\"\\n=== Request Log ===\")\n", + "for entry in app.log:\n", + " print(f\" {entry}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Error Handling: JSON Errors and Exception Mapping\n", + "\n", + "A robust REST API returns structured JSON error responses with appropriate\n", + "status codes. An error-handling middleware can catch exceptions and convert\n", + "them to proper HTTP responses, preventing stack traces from leaking to clients." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import traceback\n", + "from http import HTTPStatus\n", + "from typing import Callable, Iterable\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class APIError(Exception):\n", + " \"\"\"Base exception for API errors that map to HTTP status codes.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " status: HTTPStatus,\n", + " message: str,\n", + " detail: str | None = None,\n", + " ) -> None:\n", + " super().__init__(message)\n", + " self.status = status\n", + " self.message = message\n", + " self.detail = detail\n", + "\n", + "\n", + "class NotFoundError(APIError):\n", + " def __init__(self, resource: str, resource_id: str) -> None:\n", + " super().__init__(\n", + " HTTPStatus.NOT_FOUND,\n", + " f\"{resource} not found\",\n", + " f\"{resource} with id '{resource_id}' does not exist\",\n", + " )\n", + "\n", + "\n", + "class ValidationError(APIError):\n", + " def __init__(self, message: str, detail: str | None = None) -> None:\n", + " super().__init__(HTTPStatus.UNPROCESSABLE_ENTITY, message, detail)\n", + "\n", + "\n", + "class ConflictError(APIError):\n", + " def __init__(self, message: str) -> None:\n", + " super().__init__(HTTPStatus.CONFLICT, message)\n", + "\n", + "\n", + "# Exception-to-status mapping for standard Python exceptions\n", + "EXCEPTION_STATUS_MAP: dict[type, HTTPStatus] = {\n", + " ValueError: HTTPStatus.BAD_REQUEST,\n", + " KeyError: HTTPStatus.NOT_FOUND,\n", + " PermissionError: HTTPStatus.FORBIDDEN,\n", + " NotImplementedError: HTTPStatus.NOT_IMPLEMENTED,\n", + "}\n", + "\n", + "\n", + "class ErrorHandlingMiddleware:\n", + " \"\"\"Middleware that catches exceptions and returns JSON error responses.\"\"\"\n", + "\n", + " def __init__(self, app: WSGIApp, debug: bool = False) -> None:\n", + " self._app = app\n", + " self._debug = debug\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " try:\n", + " return self._app(environ, start_response)\n", + " except APIError as e:\n", + " return self._error_response(e.status, e.message, e.detail, start_response)\n", + " except Exception as e:\n", + " # Map known exceptions to status codes\n", + " status = EXCEPTION_STATUS_MAP.get(\n", + " type(e), HTTPStatus.INTERNAL_SERVER_ERROR\n", + " )\n", + " detail = traceback.format_exc() if self._debug else None\n", + " return self._error_response(status, str(e), detail, start_response)\n", + "\n", + " @staticmethod\n", + " def _error_response(\n", + " status: HTTPStatus,\n", + " message: str,\n", + " detail: str | None,\n", + " start_response: StartResponse,\n", + " ) -> Iterable[bytes]:\n", + " error_data: dict[str, object] = {\n", + " \"error\": status.phrase,\n", + " \"status\": status.value,\n", + " \"message\": message,\n", + " }\n", + " if detail:\n", + " error_data[\"detail\"] = detail\n", + "\n", + " body = json.dumps(error_data, indent=2).encode(\"utf-8\")\n", + " start_response(\n", + " f\"{status.value} {status.phrase}\",\n", + " [\n", + " (\"Content-Type\", \"application/json; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ],\n", + " )\n", + " return [body]\n", + "\n", + "\n", + "# Test with various exceptions\n", + "def failing_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " if path == \"/not-found\":\n", + " raise NotFoundError(\"Book\", \"42\")\n", + " elif path == \"/validation\":\n", + " raise ValidationError(\"Title is required\", \"Field 'title' cannot be empty\")\n", + " elif path == \"/value-error\":\n", + " raise ValueError(\"Invalid page number\")\n", + " elif path == \"/crash\":\n", + " raise RuntimeError(\"Unexpected server error\")\n", + " body = b'{\"ok\": true}'\n", + " start_response(\"200 OK\", [(\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body)))])\n", + " return [body]\n", + "\n", + "\n", + "wrapped = ErrorHandlingMiddleware(failing_app, debug=True)\n", + "\n", + "for path in [\"/ok\", \"/not-found\", \"/validation\", \"/value-error\", \"/crash\"]:\n", + " captured = \"\"\n", + " def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal captured\n", + " captured = s\n", + " return lambda *a: None\n", + "\n", + " body = b\"\".join(wrapped({\"PATH_INFO\": path, \"REQUEST_METHOD\": \"GET\"}, sr))\n", + " parsed = json.loads(body)\n", + " msg = parsed.get(\"message\", parsed.get(\"ok\", \"\"))\n", + " print(f\"GET {path:17s} -> {captured:30s} | {msg}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CORS: Cross-Origin Resource Sharing\n", + "\n", + "Browsers enforce the **Same-Origin Policy**, blocking requests from one origin\n", + "(e.g., `http://frontend.com`) to a different origin (e.g., `http://api.com`).\n", + "CORS headers tell browsers which cross-origin requests are allowed.\n", + "\n", + "Key CORS headers:\n", + "- `Access-Control-Allow-Origin`: Which origins can access the API\n", + "- `Access-Control-Allow-Methods`: Which HTTP methods are allowed\n", + "- `Access-Control-Allow-Headers`: Which request headers are allowed\n", + "- `Access-Control-Max-Age`: How long preflight results can be cached\n", + "\n", + "Browsers send a **preflight** `OPTIONS` request before certain cross-origin\n", + "requests to check permissions." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "from typing import Callable, Iterable\n", + "from http import HTTPStatus\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class CORSMiddleware:\n", + " \"\"\"Middleware that adds CORS headers to responses.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " app: WSGIApp,\n", + " allowed_origins: list[str] | None = None,\n", + " allowed_methods: list[str] | None = None,\n", + " allowed_headers: list[str] | None = None,\n", + " max_age: int = 3600,\n", + " ) -> None:\n", + " self._app = app\n", + " self._allowed_origins = set(allowed_origins or [\"*\"])\n", + " self._allowed_methods = allowed_methods or [\n", + " \"GET\", \"POST\", \"PUT\", \"PATCH\", \"DELETE\", \"OPTIONS\"\n", + " ]\n", + " self._allowed_headers = allowed_headers or [\n", + " \"Content-Type\", \"Authorization\", \"Accept\"\n", + " ]\n", + " self._max_age = max_age\n", + "\n", + " def _get_cors_headers(self, origin: str) -> list[tuple[str, str]]:\n", + " \"\"\"Build CORS headers based on the request origin.\"\"\"\n", + " if \"*\" in self._allowed_origins:\n", + " allow_origin = \"*\"\n", + " elif origin in self._allowed_origins:\n", + " allow_origin = origin\n", + " else:\n", + " return [] # Origin not allowed\n", + "\n", + " return [\n", + " (\"Access-Control-Allow-Origin\", allow_origin),\n", + " (\"Access-Control-Allow-Methods\", \", \".join(self._allowed_methods)),\n", + " (\"Access-Control-Allow-Headers\", \", \".join(self._allowed_headers)),\n", + " (\"Access-Control-Max-Age\", str(self._max_age)),\n", + " ]\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + " origin = str(environ.get(\"HTTP_ORIGIN\", \"\"))\n", + " cors_headers = self._get_cors_headers(origin)\n", + "\n", + " # Handle preflight OPTIONS request\n", + " if method == \"OPTIONS\" and cors_headers:\n", + " start_response(\n", + " f\"{HTTPStatus.NO_CONTENT.value} {HTTPStatus.NO_CONTENT.phrase}\",\n", + " cors_headers,\n", + " )\n", + " return []\n", + "\n", + " # For normal requests, add CORS headers to the response\n", + " def cors_start_response(\n", + " status: str, headers: list[tuple[str, str]]\n", + " ) -> Callable[..., object]:\n", + " return start_response(status, headers + cors_headers)\n", + "\n", + " return self._app(environ, cors_start_response)\n", + "\n", + "\n", + "# Inner API app\n", + "def api_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " body = json.dumps({\"data\": \"API response\"}).encode()\n", + " start_response(\"200 OK\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "# Wrap with CORS middleware\n", + "cors_app = CORSMiddleware(\n", + " api_app,\n", + " allowed_origins=[\"http://frontend.example.com\", \"http://localhost:3000\"],\n", + ")\n", + "\n", + "\n", + "def test_cors(method: str, origin: str) -> None:\n", + " captured_headers: list[tuple[str, str]] = []\n", + " captured_status = \"\"\n", + "\n", + " def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal captured_status, captured_headers\n", + " captured_status = s\n", + " captured_headers = h\n", + " return lambda *a: None\n", + "\n", + " environ: Environ = {\n", + " \"REQUEST_METHOD\": method,\n", + " \"PATH_INFO\": \"/api/data\",\n", + " \"HTTP_ORIGIN\": origin,\n", + " }\n", + " body = b\"\".join(cors_app(environ, sr))\n", + "\n", + " cors_related = {k: v for k, v in captured_headers if k.startswith(\"Access-Control\")}\n", + " print(f\"\\n {method} from {origin or '(no origin)'}\")\n", + " print(f\" Status: {captured_status}\")\n", + " print(f\" CORS headers: {cors_related}\")\n", + "\n", + "\n", + "print(\"=== CORS Middleware Tests ===\")\n", + "test_cors(\"OPTIONS\", \"http://frontend.example.com\") # Preflight, allowed\n", + "test_cors(\"GET\", \"http://frontend.example.com\") # Normal, allowed\n", + "test_cors(\"GET\", \"http://evil.com\") # Not allowed\n", + "test_cors(\"OPTIONS\", \"http://localhost:3000\") # Preflight, allowed" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API Versioning Strategies\n", + "\n", + "As APIs evolve, breaking changes need to be managed. Common versioning strategies:\n", + "\n", + "1. **URL path versioning**: `/api/v1/users` vs `/api/v2/users`\n", + "2. **Header versioning**: `Accept: application/vnd.myapi.v2+json`\n", + "3. **Query parameter versioning**: `/api/users?version=2`\n", + "\n", + "URL path versioning is the most common and most explicit approach." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import re\n", + "from typing import Callable, Iterable\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes]]\n", + "\n", + "\n", + "class VersionedAPI:\n", + " \"\"\"Router that dispatches to versioned API implementations.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._versions: dict[str, WSGIApp] = {}\n", + " self._version_pattern = re.compile(r\"^/api/(v\\d+)(/.*)?$\")\n", + "\n", + " def register_version(self, version: str, app: WSGIApp) -> None:\n", + " \"\"\"Register a WSGI app for a specific API version.\"\"\"\n", + " self._versions[version] = app\n", + "\n", + " def __call__(\n", + " self, environ: Environ, start_response: StartResponse\n", + " ) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " match = self._version_pattern.match(path)\n", + "\n", + " if not match:\n", + " body = json.dumps({\n", + " \"error\": \"Not Found\",\n", + " \"message\": \"Use /api/v1/ or /api/v2/ prefix\",\n", + " \"available_versions\": sorted(self._versions.keys()),\n", + " }).encode()\n", + " start_response(\"404 Not Found\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + " version = match.group(1)\n", + " remaining_path = match.group(2) or \"/\"\n", + "\n", + " version_app = self._versions.get(version)\n", + " if version_app is None:\n", + " body = json.dumps({\n", + " \"error\": f\"API version '{version}' not supported\",\n", + " \"available_versions\": sorted(self._versions.keys()),\n", + " }).encode()\n", + " start_response(\"404 Not Found\", [\n", + " (\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + " # Rewrite PATH_INFO to strip the version prefix\n", + " environ[\"PATH_INFO\"] = remaining_path\n", + " environ[\"API_VERSION\"] = version\n", + " return version_app(environ, start_response)\n", + "\n", + "\n", + "# V1 API: returns user as {\"name\": \"...\"}\n", + "def v1_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " if path == \"/users\":\n", + " data = {\"users\": [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}]}\n", + " else:\n", + " data = {\"version\": \"v1\", \"path\": path}\n", + " body = json.dumps(data).encode()\n", + " start_response(\"200 OK\", [(\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body)))])\n", + " return [body]\n", + "\n", + "\n", + "# V2 API: returns user as {\"full_name\": \"...\", \"id\": N} (breaking change)\n", + "def v2_app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " if path == \"/users\":\n", + " data = {\n", + " \"users\": [\n", + " {\"id\": 1, \"full_name\": \"Alice Smith\", \"email\": \"alice@example.com\"},\n", + " {\"id\": 2, \"full_name\": \"Bob Jones\", \"email\": \"bob@example.com\"},\n", + " ],\n", + " \"total\": 2,\n", + " }\n", + " else:\n", + " data = {\"version\": \"v2\", \"path\": path}\n", + " body = json.dumps(data).encode()\n", + " start_response(\"200 OK\", [(\"Content-Type\", \"application/json\"),\n", + " (\"Content-Length\", str(len(body)))])\n", + " return [body]\n", + "\n", + "\n", + "# Set up versioned API\n", + "api = VersionedAPI()\n", + "api.register_version(\"v1\", v1_app)\n", + "api.register_version(\"v2\", v2_app)\n", + "\n", + "# Test versioned routing\n", + "def test_version(path: str) -> None:\n", + " captured = \"\"\n", + " def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal captured\n", + " captured = s\n", + " return lambda *a: None\n", + "\n", + " body = b\"\".join(api({\"REQUEST_METHOD\": \"GET\", \"PATH_INFO\": path}, sr))\n", + " data = json.loads(body)\n", + " print(f\"GET {path:25s} -> {captured:15s} | {json.dumps(data)}\")\n", + "\n", + "\n", + "print(\"=== API Versioning ===\")\n", + "test_version(\"/api/v1/users\")\n", + "test_version(\"/api/v2/users\")\n", + "test_version(\"/api/v3/users\") # Unsupported version\n", + "test_version(\"/other/path\") # Non-API path" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: Building a Complete Mini REST API\n", + "\n", + "Now we combine everything into a fully functional in-memory CRUD API for\n", + "managing \"books\". It includes:\n", + "- Proper routing with path parameters\n", + "- JSON request/response handling\n", + "- HTTP method dispatch\n", + "- Appropriate status codes\n", + "- Input validation\n", + "- Error handling" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import re\n", + "import io\n", + "from http import HTTPStatus\n", + "from typing import Any, Callable, Iterable\n", + "from dataclasses import dataclass, field, asdict\n", + "\n", + "\n", + "Environ = dict[str, object]\n", + "StartResponse = Callable[[str, list[tuple[str, str]]], Callable[..., object]]\n", + "\n", + "\n", + "# --- Domain Model ---\n", + "\n", + "@dataclass\n", + "class Book:\n", + " id: int\n", + " title: str\n", + " author: str\n", + " year: int\n", + " isbn: str = \"\"\n", + "\n", + "\n", + "class BookStore:\n", + " \"\"\"In-memory book storage with CRUD operations.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self._books: dict[int, Book] = {}\n", + " self._next_id: int = 1\n", + "\n", + " def list_all(self) -> list[Book]:\n", + " return list(self._books.values())\n", + "\n", + " def get(self, book_id: int) -> Book | None:\n", + " return self._books.get(book_id)\n", + "\n", + " def create(self, title: str, author: str, year: int, isbn: str = \"\") -> Book:\n", + " book = Book(id=self._next_id, title=title, author=author, year=year, isbn=isbn)\n", + " self._books[book.id] = book\n", + " self._next_id += 1\n", + " return book\n", + "\n", + " def update(self, book_id: int, **kwargs: Any) -> Book | None:\n", + " book = self._books.get(book_id)\n", + " if book is None:\n", + " return None\n", + " for key, value in kwargs.items():\n", + " if hasattr(book, key) and key != \"id\":\n", + " setattr(book, key, value)\n", + " return book\n", + "\n", + " def delete(self, book_id: int) -> bool:\n", + " return self._books.pop(book_id, None) is not None\n", + "\n", + "\n", + "# --- Request Helpers ---\n", + "\n", + "def read_json(environ: Environ) -> Any:\n", + " \"\"\"Read JSON body from request.\"\"\"\n", + " try:\n", + " length = int(str(environ.get(\"CONTENT_LENGTH\", \"0\") or \"0\"))\n", + " except ValueError:\n", + " length = 0\n", + " if length == 0:\n", + " return None\n", + " wsgi_input = environ.get(\"wsgi.input\")\n", + " if not wsgi_input or not hasattr(wsgi_input, \"read\"):\n", + " return None\n", + " raw = wsgi_input.read(length) # type: ignore[union-attr]\n", + " return json.loads(raw)\n", + "\n", + "\n", + "def respond_json(\n", + " data: Any, status: HTTPStatus, start_response: StartResponse\n", + ") -> Iterable[bytes]:\n", + " body = json.dumps(data, indent=2, default=str).encode(\"utf-8\")\n", + " start_response(f\"{status.value} {status.phrase}\", [\n", + " (\"Content-Type\", \"application/json; charset=utf-8\"),\n", + " (\"Content-Length\", str(len(body))),\n", + " ])\n", + " return [body]\n", + "\n", + "\n", + "def respond_no_content(start_response: StartResponse) -> Iterable[bytes]:\n", + " start_response(f\"{HTTPStatus.NO_CONTENT.value} {HTTPStatus.NO_CONTENT.phrase}\", [])\n", + " return []\n", + "\n", + "\n", + "def respond_error(\n", + " status: HTTPStatus, message: str, start_response: StartResponse\n", + ") -> Iterable[bytes]:\n", + " return respond_json(\n", + " {\"error\": status.phrase, \"status\": status.value, \"message\": message},\n", + " status,\n", + " start_response,\n", + " )\n", + "\n", + "\n", + "# --- API Application ---\n", + "\n", + "# URL patterns\n", + "BOOKS_LIST = re.compile(r\"^/api/books$\")\n", + "BOOKS_DETAIL = re.compile(r\"^/api/books/(?P\\d+)$\")\n", + "\n", + "\n", + "def create_book_api(store: BookStore) -> Callable[..., Iterable[bytes]]:\n", + " \"\"\"Create a WSGI application for the books REST API.\"\"\"\n", + "\n", + " def app(environ: Environ, start_response: StartResponse) -> Iterable[bytes]:\n", + " path = str(environ.get(\"PATH_INFO\", \"/\"))\n", + " method = str(environ.get(\"REQUEST_METHOD\", \"GET\"))\n", + "\n", + " # --- /api/books ---\n", + " match = BOOKS_LIST.match(path)\n", + " if match:\n", + " if method == \"GET\":\n", + " books = [asdict(b) for b in store.list_all()]\n", + " return respond_json({\"books\": books, \"count\": len(books)},\n", + " HTTPStatus.OK, start_response)\n", + "\n", + " elif method == \"POST\":\n", + " data = read_json(environ)\n", + " if not data or not isinstance(data, dict):\n", + " return respond_error(\n", + " HTTPStatus.BAD_REQUEST, \"JSON body required\", start_response\n", + " )\n", + " # Validate required fields\n", + " missing = [f for f in (\"title\", \"author\", \"year\") if f not in data]\n", + " if missing:\n", + " return respond_error(\n", + " HTTPStatus.UNPROCESSABLE_ENTITY,\n", + " f\"Missing required fields: {', '.join(missing)}\",\n", + " start_response,\n", + " )\n", + " book = store.create(\n", + " title=data[\"title\"],\n", + " author=data[\"author\"],\n", + " year=int(data[\"year\"]),\n", + " isbn=data.get(\"isbn\", \"\"),\n", + " )\n", + " return respond_json(asdict(book), HTTPStatus.CREATED, start_response)\n", + "\n", + " else:\n", + " return respond_error(\n", + " HTTPStatus.METHOD_NOT_ALLOWED,\n", + " f\"Method {method} not allowed on /api/books\",\n", + " start_response,\n", + " )\n", + "\n", + " # --- /api/books/ ---\n", + " match = BOOKS_DETAIL.match(path)\n", + " if match:\n", + " book_id = int(match.group(\"book_id\"))\n", + "\n", + " if method == \"GET\":\n", + " book = store.get(book_id)\n", + " if book is None:\n", + " return respond_error(\n", + " HTTPStatus.NOT_FOUND,\n", + " f\"Book {book_id} not found\",\n", + " start_response,\n", + " )\n", + " return respond_json(asdict(book), HTTPStatus.OK, start_response)\n", + "\n", + " elif method == \"PUT\":\n", + " data = read_json(environ)\n", + " if not data or not isinstance(data, dict):\n", + " return respond_error(\n", + " HTTPStatus.BAD_REQUEST, \"JSON body required\", start_response\n", + " )\n", + " book = store.update(book_id, **data)\n", + " if book is None:\n", + " return respond_error(\n", + " HTTPStatus.NOT_FOUND,\n", + " f\"Book {book_id} not found\",\n", + " start_response,\n", + " )\n", + " return respond_json(asdict(book), HTTPStatus.OK, start_response)\n", + "\n", + " elif method == \"PATCH\":\n", + " data = read_json(environ)\n", + " if not data or not isinstance(data, dict):\n", + " return respond_error(\n", + " HTTPStatus.BAD_REQUEST, \"JSON body required\", start_response\n", + " )\n", + " book = store.update(book_id, **data)\n", + " if book is None:\n", + " return respond_error(\n", + " HTTPStatus.NOT_FOUND,\n", + " f\"Book {book_id} not found\",\n", + " start_response,\n", + " )\n", + " return respond_json(asdict(book), HTTPStatus.OK, start_response)\n", + "\n", + " elif method == \"DELETE\":\n", + " if not store.delete(book_id):\n", + " return respond_error(\n", + " HTTPStatus.NOT_FOUND,\n", + " f\"Book {book_id} not found\",\n", + " start_response,\n", + " )\n", + " return respond_no_content(start_response)\n", + "\n", + " else:\n", + " return respond_error(\n", + " HTTPStatus.METHOD_NOT_ALLOWED,\n", + " f\"Method {method} not allowed\",\n", + " start_response,\n", + " )\n", + "\n", + " return respond_error(HTTPStatus.NOT_FOUND, \"Endpoint not found\", start_response)\n", + "\n", + " return app\n", + "\n", + "\n", + "print(\"Book API created. See the next cell for testing.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Testing the REST API\n", + "\n", + "We exercise every CRUD operation on our books API: creating, listing,\n", + "reading, updating (PUT and PATCH), and deleting resources. Each test\n", + "verifies the correct status code and response body." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "import io\n", + "from http import HTTPStatus\n", + "from typing import Callable, Any, Iterable\n", + "\n", + "\n", + "# Create a fresh store and API\n", + "store = BookStore()\n", + "api = create_book_api(store)\n", + "\n", + "\n", + "def api_request(\n", + " method: str,\n", + " path: str,\n", + " body: dict[str, Any] | None = None,\n", + ") -> tuple[str, dict[str, Any] | None]:\n", + " \"\"\"Make a request to the API and return (status, parsed_body).\"\"\"\n", + " environ: dict[str, object] = {\n", + " \"REQUEST_METHOD\": method,\n", + " \"PATH_INFO\": path,\n", + " }\n", + "\n", + " if body is not None:\n", + " encoded = json.dumps(body).encode(\"utf-8\")\n", + " environ[\"CONTENT_TYPE\"] = \"application/json\"\n", + " environ[\"CONTENT_LENGTH\"] = str(len(encoded))\n", + " environ[\"wsgi.input\"] = io.BytesIO(encoded)\n", + "\n", + " captured_status = \"\"\n", + "\n", + " def sr(s: str, h: list[tuple[str, str]]) -> Callable[..., object]:\n", + " nonlocal captured_status\n", + " captured_status = s\n", + " return lambda *a: None\n", + "\n", + " response_body = b\"\".join(api(environ, sr))\n", + " parsed = json.loads(response_body) if response_body else None\n", + " return captured_status, parsed\n", + "\n", + "\n", + "# --- Test CRUD Operations ---\n", + "\n", + "print(\"=== 1. Create Books (POST /api/books) ===\")\n", + "status, data = api_request(\"POST\", \"/api/books\", {\n", + " \"title\": \"Fluent Python\",\n", + " \"author\": \"Luciano Ramalho\",\n", + " \"year\": 2022,\n", + " \"isbn\": \"978-1492056355\",\n", + "})\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "status, data = api_request(\"POST\", \"/api/books\", {\n", + " \"title\": \"Python Cookbook\",\n", + " \"author\": \"David Beazley\",\n", + " \"year\": 2013,\n", + "})\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "status, data = api_request(\"POST\", \"/api/books\", {\n", + " \"title\": \"Effective Python\",\n", + " \"author\": \"Brett Slatkin\",\n", + " \"year\": 2019,\n", + "})\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "print(\"\\n=== 2. List All Books (GET /api/books) ===\")\n", + "status, data = api_request(\"GET\", \"/api/books\")\n", + "print(f\" {status} -> {data['count']} books\")\n", + "for book in data[\"books\"]:\n", + " print(f\" [{book['id']}] {book['title']} by {book['author']} ({book['year']})\")\n", + "\n", + "print(\"\\n=== 3. Get Single Book (GET /api/books/1) ===\")\n", + "status, data = api_request(\"GET\", \"/api/books/1\")\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "print(\"\\n=== 4. Get Non-Existent Book (GET /api/books/999) ===\")\n", + "status, data = api_request(\"GET\", \"/api/books/999\")\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "print(\"\\n=== 5. Update Book (PUT /api/books/2) ===\")\n", + "status, data = api_request(\"PUT\", \"/api/books/2\", {\n", + " \"title\": \"Python Cookbook, 3rd Ed.\",\n", + " \"author\": \"David Beazley & Brian K. Jones\",\n", + " \"year\": 2013,\n", + "})\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "print(\"\\n=== 6. Partial Update (PATCH /api/books/3) ===\")\n", + "status, data = api_request(\"PATCH\", \"/api/books/3\", {\n", + " \"year\": 2020,\n", + " \"isbn\": \"978-0134854717\",\n", + "})\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "print(\"\\n=== 7. Delete Book (DELETE /api/books/2) ===\")\n", + "status, data = api_request(\"DELETE\", \"/api/books/2\")\n", + "print(f\" {status} -> (no body)\")\n", + "\n", + "print(\"\\n=== 8. Verify Deletion (GET /api/books) ===\")\n", + "status, data = api_request(\"GET\", \"/api/books\")\n", + "print(f\" {status} -> {data['count']} books remaining\")\n", + "for book in data[\"books\"]:\n", + " print(f\" [{book['id']}] {book['title']}\")\n", + "\n", + "print(\"\\n=== 9. Validation Error (POST without required fields) ===\")\n", + "status, data = api_request(\"POST\", \"/api/books\", {\"title\": \"Incomplete\"})\n", + "print(f\" {status} -> {json.dumps(data)}\")\n", + "\n", + "print(\"\\n=== 10. Delete Non-Existent (DELETE /api/books/999) ===\")\n", + "status, data = api_request(\"DELETE\", \"/api/books/999\")\n", + "print(f\" {status} -> {json.dumps(data)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running the API with a Real HTTP Server\n", + "\n", + "Finally, we can serve our REST API over HTTP using `wsgiref` and make real\n", + "HTTP requests with `urllib`. This demonstrates the complete end-to-end flow\n", + "with middleware stacked on top of the core application." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from wsgiref.simple_server import make_server\n", + "import threading\n", + "import urllib.request\n", + "import json\n", + "from typing import Any\n", + "\n", + "\n", + "# Create a fresh store and API, wrapped with error handling\n", + "live_store = BookStore()\n", + "live_api = ErrorHandlingMiddleware(create_book_api(live_store))\n", + "\n", + "# Start the server\n", + "httpd = make_server(\"127.0.0.1\", 0, live_api)\n", + "port = httpd.server_address[1]\n", + "base_url = f\"http://127.0.0.1:{port}\"\n", + "print(f\"API server running at {base_url}\")\n", + "\n", + "server_thread = threading.Thread(target=httpd.serve_forever, daemon=True)\n", + "server_thread.start()\n", + "\n", + "\n", + "def http_request(\n", + " method: str, path: str, body: dict[str, Any] | None = None\n", + ") -> tuple[int, Any]:\n", + " \"\"\"Make an HTTP request and return (status_code, parsed_json).\"\"\"\n", + " url = f\"{base_url}{path}\"\n", + " data = json.dumps(body).encode(\"utf-8\") if body else None\n", + " req = urllib.request.Request(\n", + " url, data=data, method=method,\n", + " headers={\"Content-Type\": \"application/json\"} if data else {},\n", + " )\n", + " try:\n", + " with urllib.request.urlopen(req) as resp:\n", + " body_bytes = resp.read()\n", + " return resp.status, json.loads(body_bytes) if body_bytes else None\n", + " except urllib.error.HTTPError as e:\n", + " body_bytes = e.read()\n", + " return e.code, json.loads(body_bytes) if body_bytes else None\n", + "\n", + "\n", + "try:\n", + " # Create a book\n", + " print(\"\\n--- POST /api/books ---\")\n", + " status, data = http_request(\"POST\", \"/api/books\", {\n", + " \"title\": \"Architecture Patterns with Python\",\n", + " \"author\": \"Harry Percival\",\n", + " \"year\": 2020,\n", + " })\n", + " print(f\" {status}: {json.dumps(data)}\")\n", + "\n", + " # List books\n", + " print(\"\\n--- GET /api/books ---\")\n", + " status, data = http_request(\"GET\", \"/api/books\")\n", + " print(f\" {status}: {data['count']} book(s)\")\n", + "\n", + " # Get the book\n", + " print(\"\\n--- GET /api/books/1 ---\")\n", + " status, data = http_request(\"GET\", \"/api/books/1\")\n", + " print(f\" {status}: {data['title']} by {data['author']}\")\n", + "\n", + " # Update the book\n", + " print(\"\\n--- PATCH /api/books/1 ---\")\n", + " status, data = http_request(\"PATCH\", \"/api/books/1\", {\"year\": 2021})\n", + " print(f\" {status}: year updated to {data['year']}\")\n", + "\n", + " # Delete the book\n", + " print(\"\\n--- DELETE /api/books/1 ---\")\n", + " status, data = http_request(\"DELETE\", \"/api/books/1\")\n", + " print(f\" {status}: deleted\")\n", + "\n", + " # Verify deletion\n", + " print(\"\\n--- GET /api/books/1 (after delete) ---\")\n", + " status, data = http_request(\"GET\", \"/api/books/1\")\n", + " print(f\" {status}: {data}\")\n", + "\n", + "finally:\n", + " httpd.shutdown()\n", + " print(\"\\nServer shut down.\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook covered the complete lifecycle of building REST APIs in Python:\n", + "\n", + "1. **REST conventions**: Resources as nouns, HTTP methods as verbs, status codes\n", + " as outcomes (GET=read, POST=create, PUT/PATCH=update, DELETE=remove)\n", + "2. **JSON handling**: Parsing request bodies with `json.loads()`, serializing\n", + " responses with `json.dumps()`, and proper Content-Type headers\n", + "3. **Content negotiation**: Parsing `Accept` headers and validating `Content-Type`\n", + " to ensure client-server format agreement\n", + "4. **Middleware pattern**: Wrapping WSGI apps for cross-cutting concerns --\n", + " logging, authentication, error handling, and CORS\n", + "5. **Error handling**: Custom exception hierarchy mapping to HTTP status codes,\n", + " structured JSON error responses, and an error-handling middleware\n", + "6. **CORS**: Understanding the Same-Origin Policy, preflight `OPTIONS` requests,\n", + " and the `Access-Control-*` headers\n", + "7. **API versioning**: URL path versioning with a version-dispatching router\n", + "8. **Complete CRUD API**: An end-to-end books API with routing, validation,\n", + " proper status codes, and real HTTP server integration\n", + "\n", + "These patterns form the foundation that frameworks like Flask, Django REST\n", + "Framework, and FastAPI build upon. Understanding them at the WSGI level gives\n", + "you deep insight into how web frameworks work internally." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_22/README.md b/src/chapter_22/README.md new file mode 100644 index 0000000..331b359 --- /dev/null +++ b/src/chapter_22/README.md @@ -0,0 +1,22 @@ +# Chapter 22: Web Development Fundamentals + +## Topics Covered +- WSGI: the standard interface between web servers and Python apps +- `http.server`: built-in HTTP server for development +- Request/response cycle: methods, headers, status codes, body +- URL routing patterns and dispatching +- Template rendering concepts +- JSON APIs and REST conventions +- Form handling and query parameters +- Middleware patterns + +## Notebooks +1. **01_wsgi_and_http.ipynb** — WSGI protocol, http.server, request/response cycle +2. **02_routing_and_templates.ipynb** — URL dispatching, template patterns, form handling +3. **03_rest_apis.ipynb** — JSON APIs, REST conventions, middleware, error handling + +## Key Takeaways +- WSGI is the foundation of Python web frameworks +- HTTP is stateless: each request is independent +- REST conventions provide consistent API design +- Middleware enables cross-cutting concerns (auth, logging, CORS) diff --git a/src/chapter_22/__init__.py b/src/chapter_22/__init__.py new file mode 100644 index 0000000..def5237 --- /dev/null +++ b/src/chapter_22/__init__.py @@ -0,0 +1 @@ +"""Chapter 22: Web Development Fundamentals.""" diff --git a/src/chapter_23/01_hashing_and_hmac.ipynb b/src/chapter_23/01_hashing_and_hmac.ipynb new file mode 100644 index 0000000..143f69d --- /dev/null +++ b/src/chapter_23/01_hashing_and_hmac.ipynb @@ -0,0 +1,615 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 23: Hashing and HMAC\n", + "\n", + "Cryptographic hashing is the foundation of data integrity verification, password storage,\n", + "and message authentication. This notebook explores Python's `hashlib` and `hmac` modules\n", + "for computing secure digests and verifying data authenticity.\n", + "\n", + "## Topics Covered\n", + "- **hashlib module**: Available algorithms, SHA-256, SHA-512, MD5, BLAKE2b\n", + "- **Creating digests**: `hexdigest()`, `digest()`, `digest_size`\n", + "- **Incremental hashing**: `update()` for large files\n", + "- **File integrity checking** with SHA-256\n", + "- **HMAC**: Message authentication codes with the `hmac` module\n", + "- **hmac.new()**: Key, message, and hash algorithm\n", + "- **hmac.compare_digest()**: Constant-time comparison\n", + "- **Practical**: Verifying data integrity and authenticity" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## hashlib Module: Available Algorithms\n", + "\n", + "The `hashlib` module provides a common interface to many secure hash and message digest\n", + "algorithms. `hashlib.algorithms_guaranteed` lists algorithms available on all platforms,\n", + "while `hashlib.algorithms_available` lists everything the current interpreter supports\n", + "(which may include additional algorithms from OpenSSL)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "\n", + "# Algorithms guaranteed on all platforms\n", + "print(\"Guaranteed algorithms:\")\n", + "print(sorted(hashlib.algorithms_guaranteed))\n", + "\n", + "# Algorithms available on this platform (may include extras from OpenSSL)\n", + "print(f\"\\nAvailable algorithms ({len(hashlib.algorithms_available)} total):\")\n", + "print(sorted(hashlib.algorithms_available))\n", + "\n", + "# Extra algorithms only available on this platform\n", + "extras: set[str] = hashlib.algorithms_available - hashlib.algorithms_guaranteed\n", + "if extras:\n", + " print(f\"\\nPlatform-specific extras: {sorted(extras)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating Digests: SHA-256, SHA-512, MD5, BLAKE2b\n", + "\n", + "Each hash algorithm produces a fixed-size output (the digest) regardless of input size.\n", + "- `hexdigest()` returns the digest as a hexadecimal string\n", + "- `digest()` returns the raw bytes\n", + "- `digest_size` gives the output length in bytes\n", + "- `block_size` gives the internal block size in bytes\n", + "\n", + "**Important**: MD5 and SHA-1 are considered cryptographically broken for collision\n", + "resistance. Use SHA-256 or better for security-sensitive applications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "\n", + "message: bytes = b\"The quick brown fox jumps over the lazy dog\"\n", + "\n", + "# SHA-256: the workhorse of modern hashing (32 bytes / 256 bits)\n", + "sha256_hash = hashlib.sha256(message)\n", + "print(\"SHA-256:\")\n", + "print(f\" hexdigest: {sha256_hash.hexdigest()}\")\n", + "print(f\" digest_size: {sha256_hash.digest_size} bytes\")\n", + "print(f\" block_size: {sha256_hash.block_size} bytes\")\n", + "print(f\" name: {sha256_hash.name}\")\n", + "\n", + "# SHA-512: stronger variant (64 bytes / 512 bits)\n", + "sha512_hash = hashlib.sha512(message)\n", + "print(f\"\\nSHA-512:\")\n", + "print(f\" hexdigest: {sha512_hash.hexdigest()}\")\n", + "print(f\" digest_size: {sha512_hash.digest_size} bytes\")\n", + "\n", + "# MD5: fast but NOT cryptographically secure (16 bytes / 128 bits)\n", + "md5_hash = hashlib.md5(message)\n", + "print(f\"\\nMD5 (not secure -- legacy use only):\")\n", + "print(f\" hexdigest: {md5_hash.hexdigest()}\")\n", + "print(f\" digest_size: {md5_hash.digest_size} bytes\")\n", + "\n", + "# BLAKE2b: modern, fast, and secure (configurable size, default 64 bytes)\n", + "blake2_hash = hashlib.blake2b(message)\n", + "print(f\"\\nBLAKE2b:\")\n", + "print(f\" hexdigest: {blake2_hash.hexdigest()}\")\n", + "print(f\" digest_size: {blake2_hash.digest_size} bytes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "\n", + "message: bytes = b\"Hello, world!\"\n", + "\n", + "# hexdigest() vs digest(): string representation vs raw bytes\n", + "h = hashlib.sha256(message)\n", + "\n", + "hex_result: str = h.hexdigest()\n", + "raw_result: bytes = h.digest()\n", + "\n", + "print(f\"hexdigest() type: {type(hex_result).__name__}\")\n", + "print(f\"hexdigest(): {hex_result}\")\n", + "print(f\"Length: {len(hex_result)} characters\\n\")\n", + "\n", + "print(f\"digest() type: {type(raw_result).__name__}\")\n", + "print(f\"digest(): {raw_result}\")\n", + "print(f\"Length: {len(raw_result)} bytes\\n\")\n", + "\n", + "# They represent the same value -- hex is just the human-readable form\n", + "print(f\"Hex of raw bytes: {raw_result.hex()}\")\n", + "print(f\"Match: {raw_result.hex() == hex_result}\")\n", + "\n", + "# BLAKE2b with custom digest size (e.g., 16 bytes for a short hash)\n", + "short_hash = hashlib.blake2b(message, digest_size=16)\n", + "print(f\"\\nBLAKE2b (16-byte): {short_hash.hexdigest()}\")\n", + "print(f\"Digest size: {short_hash.digest_size} bytes\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Incremental Hashing with update()\n", + "\n", + "For large data that cannot fit in memory, you can feed chunks incrementally using\n", + "`update()`. Calling `h.update(a); h.update(b)` is equivalent to `h.update(a + b)`.\n", + "This is essential for hashing large files without loading them entirely into memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "\n", + "# Demonstrate that update() is equivalent to hashing concatenated data\n", + "chunk1: bytes = b\"Hello, \"\n", + "chunk2: bytes = b\"world!\"\n", + "full_message: bytes = chunk1 + chunk2\n", + "\n", + "# Method 1: Hash all at once\n", + "h_once = hashlib.sha256(full_message)\n", + "print(f\"All at once: {h_once.hexdigest()}\")\n", + "\n", + "# Method 2: Hash incrementally with update()\n", + "h_incremental = hashlib.sha256()\n", + "h_incremental.update(chunk1)\n", + "h_incremental.update(chunk2)\n", + "print(f\"Incremental: {h_incremental.hexdigest()}\")\n", + "\n", + "# They produce identical digests\n", + "print(f\"Match: {h_once.hexdigest() == h_incremental.hexdigest()}\")\n", + "\n", + "# copy() creates a snapshot of the hash state\n", + "h_base = hashlib.sha256(b\"base data\")\n", + "h_copy = h_base.copy()\n", + "h_base.update(b\" + extra\")\n", + "\n", + "print(f\"\\nOriginal (with extra): {h_base.hexdigest()[:32]}...\")\n", + "print(f\"Copy (without extra): {h_copy.hexdigest()[:32]}...\")\n", + "print(f\"Different: {h_base.hexdigest() != h_copy.hexdigest()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## File Integrity Checking with SHA-256\n", + "\n", + "A common use of hashing is verifying file integrity. By computing a SHA-256 checksum\n", + "of a file and comparing it to a known-good value, you can detect corruption or tampering.\n", + "The `update()` approach lets you hash files of any size efficiently." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "\n", + "def compute_file_hash(filepath: Path, algorithm: str = \"sha256\",\n", + " chunk_size: int = 8192) -> str:\n", + " \"\"\"Compute the hash of a file by reading it in chunks.\n", + "\n", + " Args:\n", + " filepath: Path to the file to hash.\n", + " algorithm: Hash algorithm name (default: sha256).\n", + " chunk_size: Number of bytes to read per iteration.\n", + "\n", + " Returns:\n", + " Hex digest string of the file contents.\n", + " \"\"\"\n", + " h = hashlib.new(algorithm)\n", + " with open(filepath, \"rb\") as f:\n", + " while True:\n", + " chunk: bytes = f.read(chunk_size)\n", + " if not chunk:\n", + " break\n", + " h.update(chunk)\n", + " return h.hexdigest()\n", + "\n", + "\n", + "def verify_file_integrity(filepath: Path, expected_hash: str,\n", + " algorithm: str = \"sha256\") -> bool:\n", + " \"\"\"Verify a file matches an expected hash.\"\"\"\n", + " actual_hash: str = compute_file_hash(filepath, algorithm)\n", + " return actual_hash == expected_hash\n", + "\n", + "\n", + "# Create a temporary file to demonstrate\n", + "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".txt\", delete=False) as f:\n", + " f.write(\"This is important data that must not be tampered with.\\n\")\n", + " temp_path = Path(f.name)\n", + "\n", + "# Compute the checksum\n", + "original_hash: str = compute_file_hash(temp_path)\n", + "print(f\"File: {temp_path.name}\")\n", + "print(f\"SHA-256: {original_hash}\")\n", + "\n", + "# Verify integrity (should pass)\n", + "is_valid: bool = verify_file_integrity(temp_path, original_hash)\n", + "print(f\"\\nIntegrity check (unmodified): {'PASS' if is_valid else 'FAIL'}\")\n", + "\n", + "# Tamper with the file and re-check\n", + "with open(temp_path, \"a\") as f:\n", + " f.write(\"sneaky modification\")\n", + "\n", + "is_valid_after = verify_file_integrity(temp_path, original_hash)\n", + "print(f\"Integrity check (tampered): {'PASS' if is_valid_after else 'FAIL'}\")\n", + "\n", + "new_hash: str = compute_file_hash(temp_path)\n", + "print(f\"\\nNew hash: {new_hash}\")\n", + "print(f\"Hashes differ: {original_hash != new_hash}\")\n", + "\n", + "# Cleanup\n", + "temp_path.unlink()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## HMAC: Message Authentication Codes\n", + "\n", + "A plain hash verifies **integrity** (data was not corrupted), but it cannot verify\n", + "**authenticity** (data came from a trusted source). An attacker who modifies data can\n", + "simply recompute the hash.\n", + "\n", + "**HMAC** (Hash-based Message Authentication Code) solves this by incorporating a secret\n", + "key into the hash. Only someone who knows the key can produce a valid HMAC. The `hmac`\n", + "module implements this using any hash algorithm from `hashlib`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hmac\n", + "import hashlib\n", + "\n", + "# A shared secret key (in practice, use secrets.token_bytes())\n", + "secret_key: bytes = b\"my-super-secret-key-2024\"\n", + "message: bytes = b\"Transfer $500 to account 12345\"\n", + "\n", + "# Create an HMAC with SHA-256\n", + "mac = hmac.new(key=secret_key, msg=message, digestmod=hashlib.sha256)\n", + "\n", + "print(f\"Message: {message.decode()}\")\n", + "print(f\"HMAC-SHA256: {mac.hexdigest()}\")\n", + "print(f\"Digest size: {mac.digest_size} bytes\")\n", + "print(f\"Block size: {mac.block_size} bytes\")\n", + "print(f\"Name: {mac.name}\")\n", + "\n", + "# HMAC also supports incremental update()\n", + "mac2 = hmac.new(key=secret_key, digestmod=hashlib.sha256)\n", + "mac2.update(b\"Transfer $500 \")\n", + "mac2.update(b\"to account 12345\")\n", + "print(f\"\\nIncremental: {mac2.hexdigest()}\")\n", + "print(f\"Match: {mac.hexdigest() == mac2.hexdigest()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hmac\n", + "import hashlib\n", + "\n", + "secret_key: bytes = b\"my-super-secret-key-2024\"\n", + "message: bytes = b\"Transfer $500 to account 12345\"\n", + "\n", + "# Different keys produce completely different HMACs\n", + "mac_correct = hmac.new(secret_key, message, hashlib.sha256)\n", + "mac_wrong = hmac.new(b\"wrong-key\", message, hashlib.sha256)\n", + "\n", + "print(\"Same message, different keys:\")\n", + "print(f\" Correct key: {mac_correct.hexdigest()[:32]}...\")\n", + "print(f\" Wrong key: {mac_wrong.hexdigest()[:32]}...\")\n", + "\n", + "# Different messages with the same key also differ\n", + "mac_tampered = hmac.new(secret_key, b\"Transfer $5000 to account 12345\", hashlib.sha256)\n", + "print(f\"\\nSame key, tampered message:\")\n", + "print(f\" Original: {mac_correct.hexdigest()[:32]}...\")\n", + "print(f\" Tampered: {mac_tampered.hexdigest()[:32]}...\")\n", + "\n", + "# You can also use string names for the digestmod\n", + "mac_str = hmac.new(secret_key, message, \"sha256\")\n", + "print(f\"\\nUsing string digestmod: {mac_str.hexdigest()[:32]}...\")\n", + "print(f\"Same result: {mac_str.hexdigest() == mac_correct.hexdigest()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## hmac.compare_digest(): Constant-Time Comparison\n", + "\n", + "When verifying an HMAC, you must **never** use `==` to compare digests. A regular string\n", + "comparison short-circuits on the first differing character, which leaks timing information\n", + "that an attacker can exploit to guess the correct HMAC byte by byte.\n", + "\n", + "`hmac.compare_digest()` performs a **constant-time** comparison: it always takes the same\n", + "amount of time regardless of how many bytes match, preventing timing attacks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hmac\n", + "import hashlib\n", + "\n", + "secret_key: bytes = b\"server-secret-key\"\n", + "\n", + "\n", + "def sign_message(key: bytes, message: bytes) -> str:\n", + " \"\"\"Create an HMAC signature for a message.\"\"\"\n", + " return hmac.new(key, message, hashlib.sha256).hexdigest()\n", + "\n", + "\n", + "def verify_message(key: bytes, message: bytes, signature: str) -> bool:\n", + " \"\"\"Verify a message signature using constant-time comparison.\n", + "\n", + " IMPORTANT: Always use hmac.compare_digest() instead of == to prevent\n", + " timing attacks.\n", + " \"\"\"\n", + " expected: str = hmac.new(key, message, hashlib.sha256).hexdigest()\n", + " return hmac.compare_digest(expected, signature)\n", + "\n", + "\n", + "# Simulate sender creating a signed message\n", + "original_message: bytes = b\"user_id=42&action=withdraw&amount=100\"\n", + "signature: str = sign_message(secret_key, original_message)\n", + "print(f\"Message: {original_message.decode()}\")\n", + "print(f\"Signature: {signature}\")\n", + "\n", + "# Receiver verifies the message\n", + "is_authentic: bool = verify_message(secret_key, original_message, signature)\n", + "print(f\"\\nVerification (authentic): {is_authentic}\")\n", + "\n", + "# Attacker tries to tamper with the message\n", + "tampered_message: bytes = b\"user_id=42&action=withdraw&amount=10000\"\n", + "is_tampered_valid: bool = verify_message(secret_key, tampered_message, signature)\n", + "print(f\"Verification (tampered): {is_tampered_valid}\")\n", + "\n", + "# Attacker tries a forged signature\n", + "forged_sig: str = \"a\" * 64\n", + "is_forged_valid: bool = verify_message(secret_key, original_message, forged_sig)\n", + "print(f\"Verification (forged): {is_forged_valid}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: Verifying Data Integrity and Authenticity\n", + "\n", + "This example combines file hashing and HMAC to build a simple system that signs\n", + "files with a secret key and later verifies both integrity (content unchanged) and\n", + "authenticity (signed by someone with the key)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import hmac\n", + "import json\n", + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "\n", + "def sign_file(filepath: Path, key: bytes,\n", + " chunk_size: int = 8192) -> dict[str, str]:\n", + " \"\"\"Create a signed manifest for a file.\n", + "\n", + " Returns a dict with the filename, SHA-256 hash, and HMAC signature.\n", + " \"\"\"\n", + " sha256 = hashlib.sha256()\n", + " mac = hmac.new(key, digestmod=hashlib.sha256)\n", + "\n", + " with open(filepath, \"rb\") as f:\n", + " while True:\n", + " chunk: bytes = f.read(chunk_size)\n", + " if not chunk:\n", + " break\n", + " sha256.update(chunk)\n", + " mac.update(chunk)\n", + "\n", + " return {\n", + " \"filename\": filepath.name,\n", + " \"sha256\": sha256.hexdigest(),\n", + " \"hmac_sha256\": mac.hexdigest(),\n", + " }\n", + "\n", + "\n", + "def verify_file(filepath: Path, manifest: dict[str, str],\n", + " key: bytes, chunk_size: int = 8192) -> dict[str, bool]:\n", + " \"\"\"Verify a file against a signed manifest.\n", + "\n", + " Returns a dict indicating whether integrity and authenticity checks pass.\n", + " \"\"\"\n", + " sha256 = hashlib.sha256()\n", + " mac = hmac.new(key, digestmod=hashlib.sha256)\n", + "\n", + " with open(filepath, \"rb\") as f:\n", + " while True:\n", + " chunk: bytes = f.read(chunk_size)\n", + " if not chunk:\n", + " break\n", + " sha256.update(chunk)\n", + " mac.update(chunk)\n", + "\n", + " integrity_ok: bool = sha256.hexdigest() == manifest[\"sha256\"]\n", + " authenticity_ok: bool = hmac.compare_digest(\n", + " mac.hexdigest(), manifest[\"hmac_sha256\"]\n", + " )\n", + "\n", + " return {\n", + " \"integrity\": integrity_ok,\n", + " \"authenticity\": authenticity_ok,\n", + " }\n", + "\n", + "\n", + "# Create a test file\n", + "secret: bytes = b\"release-signing-key-v1\"\n", + "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".dat\", delete=False) as f:\n", + " f.write(\"Critical configuration data: server=prod, port=443\\n\")\n", + " test_file = Path(f.name)\n", + "\n", + "# Sign the file\n", + "manifest: dict[str, str] = sign_file(test_file, secret)\n", + "print(\"Signed manifest:\")\n", + "print(json.dumps(manifest, indent=2))\n", + "\n", + "# Verify unmodified file\n", + "result = verify_file(test_file, manifest, secret)\n", + "print(f\"\\nVerification (original): {result}\")\n", + "\n", + "# Tamper with the file\n", + "with open(test_file, \"a\") as f:\n", + " f.write(\"server=evil-server\\n\")\n", + "\n", + "result_tampered = verify_file(test_file, manifest, secret)\n", + "print(f\"Verification (tampered): {result_tampered}\")\n", + "\n", + "# Verify with wrong key (authenticity fails even if hash matches)\n", + "# Re-create the original file to test key mismatch\n", + "with open(test_file, \"w\") as f:\n", + " f.write(\"Critical configuration data: server=prod, port=443\\n\")\n", + "\n", + "result_wrong_key = verify_file(test_file, manifest, b\"wrong-key\")\n", + "print(f\"Verification (wrong key): {result_wrong_key}\")\n", + "\n", + "# Cleanup\n", + "test_file.unlink()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hashing Pitfalls and Best Practices\n", + "\n", + "Understanding when and how to use hashing correctly is just as important as knowing\n", + "the API. Here are common mistakes and their solutions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import hmac\n", + "\n", + "# Pitfall 1: Hashing strings directly (must encode to bytes first)\n", + "text: str = \"Hello, world!\"\n", + "try:\n", + " hashlib.sha256(text) # type: ignore[arg-type]\n", + "except TypeError as e:\n", + " print(f\"Pitfall 1 - Cannot hash str directly: {e}\")\n", + "\n", + "# Correct: encode the string to bytes\n", + "correct_hash: str = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + "print(f\"Correct: {correct_hash[:32]}...\")\n", + "\n", + "# Pitfall 2: Encoding matters! Different encodings produce different hashes\n", + "text_special: str = \"caf\\u00e9\" # cafe with accent\n", + "hash_utf8: str = hashlib.sha256(text_special.encode(\"utf-8\")).hexdigest()\n", + "hash_latin1: str = hashlib.sha256(text_special.encode(\"latin-1\")).hexdigest()\n", + "print(f\"\\nPitfall 2 - Encoding matters:\")\n", + "print(f\" UTF-8: {hash_utf8[:32]}...\")\n", + "print(f\" Latin-1: {hash_latin1[:32]}...\")\n", + "print(f\" Same? {hash_utf8 == hash_latin1}\")\n", + "\n", + "# Pitfall 3: Using == instead of compare_digest for security checks\n", + "sig_a: str = \"abc123\"\n", + "sig_b: str = \"abc123\"\n", + "\n", + "# BAD: vulnerable to timing attacks\n", + "insecure: bool = sig_a == sig_b\n", + "\n", + "# GOOD: constant-time comparison\n", + "secure: bool = hmac.compare_digest(sig_a, sig_b)\n", + "\n", + "print(f\"\\nPitfall 3 - Use compare_digest for security:\")\n", + "print(f\" == (insecure): {insecure}\")\n", + "print(f\" compare_digest (secure): {secure}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Concept | Tool | Purpose |\n", + "|---------|------|---------|\n", + "| **Hash algorithms** | `hashlib.sha256()`, `sha512()`, `blake2b()` | Produce fixed-size digests from arbitrary data |\n", + "| **Digest output** | `hexdigest()`, `digest()` | Get hash as hex string or raw bytes |\n", + "| **Incremental hashing** | `update()` | Hash large data in chunks without loading it all |\n", + "| **File integrity** | `compute_file_hash()` pattern | Verify files have not been corrupted |\n", + "| **HMAC** | `hmac.new(key, msg, digestmod)` | Authenticate messages with a shared secret |\n", + "| **Safe comparison** | `hmac.compare_digest()` | Prevent timing attacks when verifying signatures |\n", + "\n", + "### Best Practices\n", + "- Use SHA-256 or BLAKE2b for new applications; avoid MD5 and SHA-1 for security\n", + "- Always encode strings to bytes before hashing, and be consistent about the encoding\n", + "- Use `update()` to hash large files in chunks rather than loading them entirely\n", + "- Use HMAC (not plain hashing) when you need to verify both integrity and authenticity\n", + "- Always use `hmac.compare_digest()` for security-sensitive comparisons\n", + "- Never use hashing alone for passwords -- use `hashlib.pbkdf2_hmac()` or similar (see next notebook)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_23/02_secrets_and_tokens.ipynb b/src/chapter_23/02_secrets_and_tokens.ipynb new file mode 100644 index 0000000..23af77b --- /dev/null +++ b/src/chapter_23/02_secrets_and_tokens.ipynb @@ -0,0 +1,705 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 23: Secrets, Tokens, and Password Hashing\n", + "\n", + "Generating cryptographically secure random values is critical for passwords, session\n", + "tokens, API keys, and other security-sensitive data. This notebook covers Python's\n", + "`secrets` module for secure randomness and `hashlib.pbkdf2_hmac()` for password hashing.\n", + "\n", + "## Topics Covered\n", + "- **secrets vs random**: When to use which module\n", + "- **Token generation**: `token_hex()`, `token_bytes()`, `token_urlsafe()`\n", + "- **Secure passwords**: `secrets.choice()` for password generation\n", + "- **Session tokens and API keys**: Generation patterns\n", + "- **Password hashing**: Salt, iteration, key derivation with `pbkdf2_hmac()`\n", + "- **Password verification**: Storing and checking hashed passwords\n", + "- **Secure random numbers**: `secrets.randbelow()`, `secrets.SystemRandom`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## secrets vs random: Choosing the Right Module\n", + "\n", + "Python has two random modules with very different purposes:\n", + "\n", + "- **`random`**: Uses a Mersenne Twister PRNG. Fast and reproducible with seeds, but\n", + " **NOT cryptographically secure**. Use for simulations, games, and statistical sampling.\n", + "- **`secrets`**: Uses the OS-provided CSPRNG (`/dev/urandom` on Unix, `CryptGenRandom`\n", + " on Windows). Slower but **unpredictable**. Use for passwords, tokens, and keys.\n", + "\n", + "**Rule of thumb**: If the value must be unguessable, use `secrets`. If it just needs to\n", + "be \"random enough\" for non-security purposes, use `random`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import secrets\n", + "\n", + "# random module: reproducible with a seed (NOT secure)\n", + "random.seed(42)\n", + "print(\"random module (seeded with 42):\")\n", + "print(f\" random.randint(0, 100): {random.randint(0, 100)}\")\n", + "print(f\" random.randint(0, 100): {random.randint(0, 100)}\")\n", + "\n", + "# Re-seeding produces the same sequence -- predictable!\n", + "random.seed(42)\n", + "print(f\" After re-seed(42): {random.randint(0, 100)}\")\n", + "\n", + "# secrets module: always unpredictable, no seed\n", + "print(\"\\nsecrets module (cryptographically secure):\")\n", + "print(f\" secrets.randbelow(100): {secrets.randbelow(100)}\")\n", + "print(f\" secrets.randbelow(100): {secrets.randbelow(100)}\")\n", + "print(f\" secrets.randbelow(100): {secrets.randbelow(100)}\")\n", + "\n", + "# Demonstrate the danger: random's state can be recovered\n", + "print(\"\\nKey difference:\")\n", + "print(\" random: deterministic PRNG -- state can be reconstructed from output\")\n", + "print(\" secrets: OS entropy source -- no internal state to reconstruct\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Token Generation: token_hex, token_bytes, token_urlsafe\n", + "\n", + "The `secrets` module provides three convenient functions for generating random tokens:\n", + "\n", + "- `token_bytes(n)`: Returns `n` random bytes\n", + "- `token_hex(n)`: Returns `2n` hex characters (representing `n` random bytes)\n", + "- `token_urlsafe(n)`: Returns a URL-safe base64-encoded string from `n` random bytes\n", + "\n", + "The default `n` is chosen to provide reasonable security (currently 32 bytes = 256 bits)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import secrets\n", + "\n", + "# token_bytes: raw random bytes\n", + "raw_token: bytes = secrets.token_bytes(16)\n", + "print(f\"token_bytes(16): {raw_token}\")\n", + "print(f\" Type: {type(raw_token).__name__}\")\n", + "print(f\" Len: {len(raw_token)} bytes\")\n", + "\n", + "# token_hex: hex-encoded string (2 chars per byte)\n", + "hex_token: str = secrets.token_hex(16)\n", + "print(f\"\\ntoken_hex(16): {hex_token}\")\n", + "print(f\" Type: {type(hex_token).__name__}\")\n", + "print(f\" Len: {len(hex_token)} characters\")\n", + "\n", + "# token_urlsafe: base64-encoded, safe for URLs and filenames\n", + "safe_token: str = secrets.token_urlsafe(16)\n", + "print(f\"\\ntoken_urlsafe(16): {safe_token}\")\n", + "print(f\" Type: {type(safe_token).__name__}\")\n", + "print(f\" Len: {len(safe_token)} characters\")\n", + "\n", + "# Default size (no argument) uses a reasonable default\n", + "default_token: str = secrets.token_hex()\n", + "print(f\"\\ntoken_hex() default: {default_token}\")\n", + "print(f\" Len: {len(default_token)} characters ({len(default_token) // 2} bytes)\")\n", + "\n", + "# Each call produces a unique value\n", + "print(\"\\nUniqueness check:\")\n", + "tokens: list[str] = [secrets.token_hex(8) for _ in range(5)]\n", + "for i, t in enumerate(tokens):\n", + " print(f\" Token {i}: {t}\")\n", + "print(f\" All unique: {len(set(tokens)) == len(tokens)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generating Secure Passwords with secrets.choice()\n", + "\n", + "`secrets.choice()` selects a random element from a sequence using the CSPRNG.\n", + "Combined with a character alphabet, it can generate strong passwords. Unlike\n", + "`random.choice()`, the selection is cryptographically unpredictable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import secrets\n", + "import string\n", + "\n", + "\n", + "def generate_password(length: int = 16,\n", + " use_uppercase: bool = True,\n", + " use_digits: bool = True,\n", + " use_punctuation: bool = True) -> str:\n", + " \"\"\"Generate a cryptographically secure random password.\n", + "\n", + " Args:\n", + " length: Number of characters in the password.\n", + " use_uppercase: Include uppercase letters.\n", + " use_digits: Include digits.\n", + " use_punctuation: Include punctuation characters.\n", + "\n", + " Returns:\n", + " A random password string.\n", + " \"\"\"\n", + " alphabet: str = string.ascii_lowercase\n", + " if use_uppercase:\n", + " alphabet += string.ascii_uppercase\n", + " if use_digits:\n", + " alphabet += string.digits\n", + " if use_punctuation:\n", + " alphabet += string.punctuation\n", + "\n", + " # Ensure at least one character from each required category\n", + " password: str = \"\"\n", + " while True:\n", + " password = \"\".join(secrets.choice(alphabet) for _ in range(length))\n", + " # Verify the password meets complexity requirements\n", + " has_lower: bool = any(c in string.ascii_lowercase for c in password)\n", + " has_upper: bool = not use_uppercase or any(c in string.ascii_uppercase for c in password)\n", + " has_digit: bool = not use_digits or any(c in string.digits for c in password)\n", + " has_punct: bool = not use_punctuation or any(c in string.punctuation for c in password)\n", + " if has_lower and has_upper and has_digit and has_punct:\n", + " break\n", + "\n", + " return password\n", + "\n", + "\n", + "# Generate various passwords\n", + "print(\"Generated passwords:\")\n", + "for i in range(5):\n", + " pwd: str = generate_password(length=20)\n", + " print(f\" {i + 1}. {pwd}\")\n", + "\n", + "# Simpler password (letters and digits only)\n", + "simple: str = generate_password(length=12, use_punctuation=False)\n", + "print(f\"\\nSimple (no punctuation): {simple}\")\n", + "\n", + "# Passphrase generation (using word lists is often more memorable)\n", + "words: list[str] = [\n", + " \"correct\", \"horse\", \"battery\", \"staple\", \"cloud\", \"river\",\n", + " \"mountain\", \"forest\", \"bridge\", \"castle\", \"thunder\", \"crystal\",\n", + " \"garden\", \"lantern\", \"voyage\", \"anchor\", \"falcon\", \"marble\",\n", + " \"willow\", \"canyon\", \"ember\", \"harbor\", \"meadow\", \"summit\",\n", + "]\n", + "passphrase: str = \"-\".join(secrets.choice(words) for _ in range(4))\n", + "print(f\"\\nPassphrase: {passphrase}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Session Tokens and API Key Generation Patterns\n", + "\n", + "Tokens and API keys need to be unique, unguessable, and appropriately sized.\n", + "Here are practical patterns for generating them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import secrets\n", + "import hashlib\n", + "from datetime import datetime, timezone\n", + "\n", + "\n", + "def generate_session_token() -> str:\n", + " \"\"\"Generate a session token (URL-safe, 32 bytes of entropy).\"\"\"\n", + " return secrets.token_urlsafe(32)\n", + "\n", + "\n", + "def generate_api_key(prefix: str = \"sk\") -> str:\n", + " \"\"\"Generate an API key with a prefix for easy identification.\n", + "\n", + " Format: prefix_hex (e.g., sk_a1b2c3d4...)\n", + " The prefix helps identify the key type without revealing the secret.\n", + " \"\"\"\n", + " token: str = secrets.token_hex(24) # 48 hex chars = 192 bits\n", + " return f\"{prefix}_{token}\"\n", + "\n", + "\n", + "def generate_reset_token(user_id: int) -> dict[str, str]:\n", + " \"\"\"Generate a password reset token with metadata.\n", + "\n", + " Returns the token and its hash. Store the HASH in the database,\n", + " send the TOKEN to the user. On verification, hash the submitted\n", + " token and compare it to the stored hash.\n", + " \"\"\"\n", + " token: str = secrets.token_urlsafe(32)\n", + " token_hash: str = hashlib.sha256(token.encode()).hexdigest()\n", + "\n", + " return {\n", + " \"token\": token, # Send this to the user (via email)\n", + " \"token_hash\": token_hash, # Store this in the database\n", + " \"user_id\": str(user_id),\n", + " \"created_at\": datetime.now(timezone.utc).isoformat(),\n", + " }\n", + "\n", + "\n", + "# Generate examples\n", + "print(\"Session token:\")\n", + "session: str = generate_session_token()\n", + "print(f\" {session}\")\n", + "print(f\" Length: {len(session)} chars\")\n", + "\n", + "print(\"\\nAPI keys:\")\n", + "for key_type in [\"sk\", \"pk\", \"test\"]:\n", + " api_key: str = generate_api_key(prefix=key_type)\n", + " print(f\" {api_key}\")\n", + "\n", + "print(\"\\nPassword reset token:\")\n", + "reset_info: dict[str, str] = generate_reset_token(user_id=42)\n", + "print(f\" Token (for user): {reset_info['token'][:32]}...\")\n", + "print(f\" Hash (for DB): {reset_info['token_hash'][:32]}...\")\n", + "print(f\" User ID: {reset_info['user_id']}\")\n", + "print(f\" Created at: {reset_info['created_at']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Password Hashing: Salt, Iteration, and Key Derivation\n", + "\n", + "Passwords must **never** be stored as plain text or simple hashes. Attackers who obtain\n", + "a database can use rainbow tables and brute force to crack simple hashes almost instantly.\n", + "\n", + "**Key Derivation Functions** (KDFs) like PBKDF2 are designed for password hashing:\n", + "- **Salt**: A random value unique to each password, preventing rainbow table attacks\n", + "- **Iterations**: Deliberate slowness (hundreds of thousands of rounds) to make brute\n", + " force expensive\n", + "- **Key stretching**: Transforms a weak password into a strong derived key\n", + "\n", + "`hashlib.pbkdf2_hmac()` implements PBKDF2 with HMAC as the pseudorandom function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import os\n", + "import secrets\n", + "\n", + "# Why plain hashing is dangerous\n", + "password: str = \"mysecretpassword\"\n", + "plain_hash: str = hashlib.sha256(password.encode()).hexdigest()\n", + "print(\"BAD: Plain SHA-256 hash of password\")\n", + "print(f\" Hash: {plain_hash}\")\n", + "print(\" Problems: no salt, too fast, vulnerable to rainbow tables\\n\")\n", + "\n", + "# Same password always produces the same hash (bad!)\n", + "plain_hash2: str = hashlib.sha256(password.encode()).hexdigest()\n", + "print(f\" Same password, same hash: {plain_hash == plain_hash2}\")\n", + "\n", + "# GOOD: PBKDF2 with salt and iterations\n", + "salt: bytes = os.urandom(32) # 32 bytes of random salt\n", + "iterations: int = 600_000 # OWASP recommended minimum for PBKDF2-SHA256\n", + "\n", + "derived_key: bytes = hashlib.pbkdf2_hmac(\n", + " hash_name=\"sha256\",\n", + " password=password.encode(\"utf-8\"),\n", + " salt=salt,\n", + " iterations=iterations,\n", + " dklen=32, # Derived key length in bytes\n", + ")\n", + "\n", + "print(\"\\nGOOD: PBKDF2-HMAC-SHA256\")\n", + "print(f\" Salt (hex): {salt.hex()}\")\n", + "print(f\" Iterations: {iterations:,}\")\n", + "print(f\" Derived key (hex): {derived_key.hex()}\")\n", + "\n", + "# Same password, different salt = different derived key\n", + "salt2: bytes = os.urandom(32)\n", + "derived_key2: bytes = hashlib.pbkdf2_hmac(\n", + " \"sha256\", password.encode(), salt2, iterations, dklen=32\n", + ")\n", + "print(f\"\\n Different salt, same password:\")\n", + "print(f\" Key 1: {derived_key.hex()[:32]}...\")\n", + "print(f\" Key 2: {derived_key2.hex()[:32]}...\")\n", + "print(f\" Same? {derived_key == derived_key2}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Password Verification Workflow\n", + "\n", + "A complete password hashing system needs two operations:\n", + "1. **Hash**: When creating/changing a password, generate a salt, hash the password,\n", + " and store the salt + hash together\n", + "2. **Verify**: When authenticating, retrieve the stored salt, re-hash the submitted\n", + " password with that salt, and compare" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import hmac\n", + "import os\n", + "\n", + "# Configuration\n", + "HASH_ALGORITHM: str = \"sha256\"\n", + "ITERATIONS: int = 600_000\n", + "SALT_LENGTH: int = 32\n", + "KEY_LENGTH: int = 32\n", + "\n", + "\n", + "def hash_password(password: str) -> str:\n", + " \"\"\"Hash a password for storage.\n", + "\n", + " Returns a string in the format: algorithm$iterations$salt_hex$key_hex\n", + " This format stores everything needed to verify the password later.\n", + " \"\"\"\n", + " salt: bytes = os.urandom(SALT_LENGTH)\n", + " key: bytes = hashlib.pbkdf2_hmac(\n", + " HASH_ALGORITHM,\n", + " password.encode(\"utf-8\"),\n", + " salt,\n", + " ITERATIONS,\n", + " dklen=KEY_LENGTH,\n", + " )\n", + " return f\"{HASH_ALGORITHM}${ITERATIONS}${salt.hex()}${key.hex()}\"\n", + "\n", + "\n", + "def verify_password(password: str, stored_hash: str) -> bool:\n", + " \"\"\"Verify a password against a stored hash.\n", + "\n", + " Uses constant-time comparison to prevent timing attacks.\n", + " \"\"\"\n", + " algorithm, iterations_str, salt_hex, key_hex = stored_hash.split(\"$\")\n", + " salt: bytes = bytes.fromhex(salt_hex)\n", + " stored_key: bytes = bytes.fromhex(key_hex)\n", + " iterations: int = int(iterations_str)\n", + "\n", + " computed_key: bytes = hashlib.pbkdf2_hmac(\n", + " algorithm,\n", + " password.encode(\"utf-8\"),\n", + " salt,\n", + " iterations,\n", + " dklen=len(stored_key),\n", + " )\n", + "\n", + " # Constant-time comparison to prevent timing attacks\n", + " return hmac.compare_digest(computed_key, stored_key)\n", + "\n", + "\n", + "# Simulate user registration\n", + "print(\"User registration:\")\n", + "user_password: str = \"correct-horse-battery-staple\"\n", + "stored: str = hash_password(user_password)\n", + "print(f\" Password: {user_password}\")\n", + "print(f\" Stored hash: {stored[:60]}...\")\n", + "print(f\" Components: algorithm$iterations$salt$key\")\n", + "\n", + "# Simulate login attempt (correct password)\n", + "print(\"\\nLogin (correct password):\")\n", + "is_valid: bool = verify_password(\"correct-horse-battery-staple\", stored)\n", + "print(f\" Verified: {is_valid}\")\n", + "\n", + "# Simulate login attempt (wrong password)\n", + "print(\"\\nLogin (wrong password):\")\n", + "is_valid_wrong: bool = verify_password(\"wrong-password\", stored)\n", + "print(f\" Verified: {is_valid_wrong}\")\n", + "\n", + "# Show that same password produces different stored hashes (due to salt)\n", + "stored2: str = hash_password(user_password)\n", + "print(f\"\\nSame password, different stored hashes:\")\n", + "print(f\" Hash 1: {stored[:48]}...\")\n", + "print(f\" Hash 2: {stored2[:48]}...\")\n", + "print(f\" Same? {stored == stored2}\")\n", + "print(f\" Both verify: {verify_password(user_password, stored) and verify_password(user_password, stored2)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Secure Random Numbers: randbelow() and SystemRandom\n", + "\n", + "`secrets.randbelow(n)` returns a secure random integer in the range `[0, n)`. For more\n", + "complex random operations (shuffling, sampling), `secrets.SystemRandom` provides a\n", + "drop-in replacement for `random.Random` that uses the OS CSPRNG." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import secrets\n", + "\n", + "# randbelow(n): secure random integer in [0, n)\n", + "print(\"secrets.randbelow():\")\n", + "for _ in range(5):\n", + " value: int = secrets.randbelow(100)\n", + " print(f\" randbelow(100) = {value}\")\n", + "\n", + "# Simulating a fair coin flip\n", + "flips: list[str] = [\n", + " \"heads\" if secrets.randbelow(2) == 0 else \"tails\"\n", + " for _ in range(10)\n", + "]\n", + "print(f\"\\nSecure coin flips: {flips}\")\n", + "\n", + "# Simulating a fair die roll\n", + "rolls: list[int] = [secrets.randbelow(6) + 1 for _ in range(10)]\n", + "print(f\"Secure die rolls: {rolls}\")\n", + "\n", + "# secrets.choice() -- pick a random element\n", + "colors: list[str] = [\"red\", \"green\", \"blue\", \"yellow\", \"purple\"]\n", + "print(f\"\\nSecure choice: {secrets.choice(colors)}\")\n", + "\n", + "# SystemRandom: a full Random interface backed by the CSPRNG\n", + "secure_rng = secrets.SystemRandom()\n", + "\n", + "print(\"\\nsecrets.SystemRandom() methods:\")\n", + "print(f\" randint(1, 100): {secure_rng.randint(1, 100)}\")\n", + "print(f\" random(): {secure_rng.random():.6f}\")\n", + "print(f\" uniform(1, 10): {secure_rng.uniform(1, 10):.4f}\")\n", + "\n", + "# Secure shuffle (in-place)\n", + "deck: list[str] = [\"A\", \"K\", \"Q\", \"J\", \"10\", \"9\", \"8\", \"7\"]\n", + "secure_rng.shuffle(deck)\n", + "print(f\" Shuffled deck: {deck}\")\n", + "\n", + "# Secure sample (without replacement)\n", + "sample: list[str] = secure_rng.sample(colors, k=3)\n", + "print(f\" Sample 3 colors: {sample}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import secrets\n", + "import string\n", + "\n", + "\n", + "def generate_otp(length: int = 6) -> str:\n", + " \"\"\"Generate a numeric one-time password (OTP).\n", + "\n", + " Uses secrets.choice for cryptographic security rather than\n", + " random.randint which is predictable.\n", + " \"\"\"\n", + " return \"\".join(secrets.choice(string.digits) for _ in range(length))\n", + "\n", + "\n", + "def generate_verification_code(length: int = 8) -> str:\n", + " \"\"\"Generate an alphanumeric verification code (uppercase + digits).\n", + "\n", + " Excludes ambiguous characters (0/O, 1/I/l) for readability.\n", + " \"\"\"\n", + " safe_chars: str = \"ABCDEFGHJKLMNPQRSTUVWXYZ23456789\"\n", + " return \"\".join(secrets.choice(safe_chars) for _ in range(length))\n", + "\n", + "\n", + "# Generate OTPs\n", + "print(\"One-Time Passwords (6-digit):\")\n", + "for i in range(5):\n", + " print(f\" OTP {i + 1}: {generate_otp()}\")\n", + "\n", + "# Generate verification codes\n", + "print(\"\\nVerification codes (no ambiguous chars):\")\n", + "for i in range(5):\n", + " code: str = generate_verification_code()\n", + " # Format with dashes for readability: ABCD-EFGH\n", + " formatted: str = f\"{code[:4]}-{code[4:]}\"\n", + " print(f\" Code {i + 1}: {formatted}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: Complete Token Management System\n", + "\n", + "This example demonstrates a token management pattern that generates, stores (as hashes),\n", + "and verifies tokens. This is the same pattern used by real-world systems for API keys,\n", + "password reset links, and email verification tokens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import hmac\n", + "import secrets\n", + "from dataclasses import dataclass, field\n", + "from datetime import datetime, timedelta, timezone\n", + "\n", + "\n", + "@dataclass\n", + "class StoredToken:\n", + " \"\"\"A token as stored in the database (hashed, never plain text).\"\"\"\n", + " token_hash: str\n", + " user_id: int\n", + " purpose: str\n", + " created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))\n", + " expires_at: datetime | None = None\n", + " used: bool = False\n", + "\n", + "\n", + "class TokenManager:\n", + " \"\"\"Manages secure token generation, storage, and verification.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " # In a real app, this would be a database\n", + " self._store: dict[str, StoredToken] = {}\n", + "\n", + " @staticmethod\n", + " def _hash_token(token: str) -> str:\n", + " \"\"\"Hash a token for storage. We store hashes, not raw tokens.\"\"\"\n", + " return hashlib.sha256(token.encode(\"utf-8\")).hexdigest()\n", + "\n", + " def create_token(self, user_id: int, purpose: str,\n", + " ttl_hours: int = 24) -> str:\n", + " \"\"\"Create a new token and store its hash.\n", + "\n", + " Returns the raw token (only shown once to the user).\n", + " \"\"\"\n", + " raw_token: str = secrets.token_urlsafe(32)\n", + " token_hash: str = self._hash_token(raw_token)\n", + " now: datetime = datetime.now(timezone.utc)\n", + "\n", + " self._store[token_hash] = StoredToken(\n", + " token_hash=token_hash,\n", + " user_id=user_id,\n", + " purpose=purpose,\n", + " created_at=now,\n", + " expires_at=now + timedelta(hours=ttl_hours),\n", + " )\n", + "\n", + " return raw_token\n", + "\n", + " def verify_token(self, raw_token: str, purpose: str) -> int | None:\n", + " \"\"\"Verify a token and return the user_id if valid, None otherwise.\"\"\"\n", + " token_hash: str = self._hash_token(raw_token)\n", + "\n", + " stored = self._store.get(token_hash)\n", + " if stored is None:\n", + " return None # Token not found\n", + "\n", + " if stored.used:\n", + " return None # Token already used\n", + "\n", + " if stored.purpose != purpose:\n", + " return None # Wrong purpose\n", + "\n", + " now: datetime = datetime.now(timezone.utc)\n", + " if stored.expires_at and now > stored.expires_at:\n", + " return None # Token expired\n", + "\n", + " # Mark as used (one-time tokens)\n", + " stored.used = True\n", + " return stored.user_id\n", + "\n", + "\n", + "# Demonstrate the token lifecycle\n", + "manager = TokenManager()\n", + "\n", + "# Create a password reset token\n", + "reset_token: str = manager.create_token(user_id=42, purpose=\"password_reset\", ttl_hours=1)\n", + "print(f\"Reset token (shown to user once): {reset_token[:24]}...\")\n", + "print(f\"Tokens in store: {len(manager._store)}\")\n", + "\n", + "# Verify with the correct token and purpose\n", + "user_id = manager.verify_token(reset_token, purpose=\"password_reset\")\n", + "print(f\"\\nVerify (correct): user_id = {user_id}\")\n", + "\n", + "# Try to reuse the same token (should fail -- already used)\n", + "user_id_reuse = manager.verify_token(reset_token, purpose=\"password_reset\")\n", + "print(f\"Verify (reuse): user_id = {user_id_reuse}\")\n", + "\n", + "# Try a fake token\n", + "fake_user = manager.verify_token(\"fake-token-abc123\", purpose=\"password_reset\")\n", + "print(f\"Verify (fake): user_id = {fake_user}\")\n", + "\n", + "# Create and verify an email verification token\n", + "email_token: str = manager.create_token(user_id=99, purpose=\"email_verify\", ttl_hours=48)\n", + "wrong_purpose = manager.verify_token(email_token, purpose=\"password_reset\")\n", + "right_purpose = manager.verify_token(email_token, purpose=\"email_verify\")\n", + "print(f\"\\nWrong purpose: user_id = {wrong_purpose}\")\n", + "# Note: the token was consumed by the wrong_purpose check internally,\n", + "# but since purpose didn't match, used was not set -- let's check again\n", + "right_purpose2 = manager.verify_token(email_token, purpose=\"email_verify\")\n", + "print(f\"Right purpose: user_id = {right_purpose2}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Concept | Tool | Purpose |\n", + "|---------|------|---------|\n", + "| **Secure randomness** | `secrets` module | Generate unpredictable tokens, passwords, keys |\n", + "| **Token generation** | `token_hex()`, `token_urlsafe()` | Create session tokens, API keys |\n", + "| **Secure choice** | `secrets.choice()` | Pick random elements securely |\n", + "| **Random integers** | `secrets.randbelow(n)` | Secure alternative to `random.randint()` |\n", + "| **Full CSPRNG** | `secrets.SystemRandom` | Drop-in replacement for `random.Random` |\n", + "| **Password hashing** | `hashlib.pbkdf2_hmac()` | Derive keys from passwords with salt + iterations |\n", + "| **Password storage** | `algorithm$iterations$salt$key` | Store all parameters needed for verification |\n", + "\n", + "### Best Practices\n", + "- Always use `secrets` (not `random`) for security-sensitive values\n", + "- Never store passwords as plain text or simple hashes\n", + "- Use `pbkdf2_hmac()` with at least 600,000 iterations for password hashing\n", + "- Generate a unique random salt for every password\n", + "- Store token hashes in the database, never the raw tokens\n", + "- Use `hmac.compare_digest()` for all security comparisons\n", + "- Set expiration times on all tokens\n", + "- Mark tokens as used after verification to prevent replay attacks" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_23/03_secure_coding.ipynb b/src/chapter_23/03_secure_coding.ipynb new file mode 100644 index 0000000..5c33022 --- /dev/null +++ b/src/chapter_23/03_secure_coding.ipynb @@ -0,0 +1,880 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 23: Secure Coding Practices\n", + "\n", + "Writing secure code goes beyond cryptography. It requires disciplined input validation,\n", + "proper output encoding, safe handling of external data, and following established security\n", + "principles. This notebook covers practical secure coding patterns in Python.\n", + "\n", + "## Topics Covered\n", + "- **Input validation**: Whitelisting vs blacklisting approaches\n", + "- **Common vulnerabilities**: Injection and XSS concepts\n", + "- **Output encoding**: `html.escape()` for HTML safety\n", + "- **Shell safety**: `shlex.quote()` for command construction\n", + "- **SQL injection prevention**: Parameterized queries\n", + "- **SSL/TLS basics**: Creating SSL contexts, certificate verification\n", + "- **Secrets management**: Environment variables with `os.environ`\n", + "- **Secure coding checklist**: Principles and guidelines" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Input Validation: Whitelisting vs Blacklisting\n", + "\n", + "Input validation is the first line of defense. There are two fundamental approaches:\n", + "\n", + "- **Whitelisting** (allow-list): Define exactly what IS allowed and reject everything\n", + " else. This is the preferred approach because it is secure by default.\n", + "- **Blacklisting** (deny-list): Define what is NOT allowed and accept everything else.\n", + " This is fragile because attackers can often find inputs you forgot to block.\n", + "\n", + "**Rule**: Always prefer whitelisting. Only use blacklisting as an additional layer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "\n", + "# WHITELIST approach: define exactly what is allowed\n", + "def validate_username(username: str) -> bool:\n", + " \"\"\"Validate a username using a whitelist pattern.\n", + "\n", + " Only allows lowercase letters, digits, underscores, and hyphens.\n", + " Must be 3-20 characters long.\n", + " \"\"\"\n", + " pattern: str = r\"^[a-z0-9_-]{3,20}$\"\n", + " return bool(re.match(pattern, username))\n", + "\n", + "\n", + "def validate_email(email: str) -> bool:\n", + " \"\"\"Basic email validation using a whitelist pattern.\n", + "\n", + " Note: For production use, consider the email-validator library\n", + " or actually sending a verification email.\n", + " \"\"\"\n", + " pattern: str = r\"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$\"\n", + " return bool(re.match(pattern, email)) and len(email) <= 254\n", + "\n", + "\n", + "def validate_age(age_str: str) -> int | None:\n", + " \"\"\"Validate and convert age input with strict bounds checking.\"\"\"\n", + " try:\n", + " age: int = int(age_str)\n", + " except ValueError:\n", + " return None\n", + " if 0 <= age <= 150: # Reasonable range\n", + " return age\n", + " return None\n", + "\n", + "\n", + "# Test username validation\n", + "test_usernames: list[tuple[str, bool]] = [\n", + " (\"alice\", True),\n", + " (\"bob_123\", True),\n", + " (\"my-name\", True),\n", + " (\"AB\", False), # Too short\n", + " (\"Alice\", False), # Uppercase not allowed\n", + " (\"user@name\", False), # Special chars not allowed\n", + " (\"'; DROP TABLE--\", False), # SQL injection attempt\n", + "]\n", + "\n", + "print(\"Username validation (whitelist):\")\n", + "for username, expected in test_usernames:\n", + " result: bool = validate_username(username)\n", + " status: str = \"PASS\" if result == expected else \"FAIL\"\n", + " print(f\" [{status}] {username!r:25s} -> valid={result}\")\n", + "\n", + "# Test email validation\n", + "print(\"\\nEmail validation:\")\n", + "test_emails: list[str] = [\"user@example.com\", \"bad@\", \"no-at-sign\", \"a@b.co\"]\n", + "for email in test_emails:\n", + " print(f\" {email!r:25s} -> valid={validate_email(email)}\")\n", + "\n", + "# Test age validation\n", + "print(\"\\nAge validation:\")\n", + "for age_input in [\"25\", \"0\", \"150\", \"-1\", \"200\", \"abc\", \"25; DROP TABLE\"]:\n", + " print(f\" {age_input!r:20s} -> {validate_age(age_input)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "import re\n", + "\n", + "\n", + "@dataclass\n", + "class ValidationResult:\n", + " \"\"\"Result of a validation check.\"\"\"\n", + " is_valid: bool\n", + " value: str\n", + " errors: list[str]\n", + "\n", + "\n", + "def validate_and_sanitize(raw_input: str, field_name: str,\n", + " max_length: int = 100,\n", + " allowed_pattern: str = r\"^[\\w\\s.,!?-]+$\") -> ValidationResult:\n", + " \"\"\"Validate and sanitize user input with detailed error reporting.\n", + "\n", + " Steps:\n", + " 1. Strip leading/trailing whitespace\n", + " 2. Check length constraints\n", + " 3. Validate against allowed character pattern\n", + " 4. Normalize whitespace\n", + " \"\"\"\n", + " errors: list[str] = []\n", + "\n", + " # Step 1: Strip whitespace\n", + " cleaned: str = raw_input.strip()\n", + "\n", + " # Step 2: Length checks\n", + " if len(cleaned) == 0:\n", + " errors.append(f\"{field_name} cannot be empty\")\n", + " if len(cleaned) > max_length:\n", + " errors.append(f\"{field_name} exceeds maximum length of {max_length}\")\n", + "\n", + " # Step 3: Character whitelist check\n", + " if cleaned and not re.match(allowed_pattern, cleaned):\n", + " errors.append(f\"{field_name} contains disallowed characters\")\n", + "\n", + " # Step 4: Normalize internal whitespace\n", + " cleaned = re.sub(r\"\\s+\", \" \", cleaned)\n", + "\n", + " return ValidationResult(\n", + " is_valid=len(errors) == 0,\n", + " value=cleaned,\n", + " errors=errors,\n", + " )\n", + "\n", + "\n", + "# Test cases\n", + "test_inputs: list[str] = [\n", + " \" Hello, World! \",\n", + " \"Normal input text.\",\n", + " \"\",\n", + " \"\",\n", + " \"A\" * 150,\n", + " \"Line one\\n\\nLine two\",\n", + "]\n", + "\n", + "print(\"Input validation and sanitization:\")\n", + "for raw in test_inputs:\n", + " result: ValidationResult = validate_and_sanitize(raw, \"comment\")\n", + " display: str = raw[:40] + \"...\" if len(raw) > 40 else raw\n", + " print(f\"\\n Input: {display!r}\")\n", + " print(f\" Valid: {result.is_valid}\")\n", + " if result.errors:\n", + " print(f\" Errors: {result.errors}\")\n", + " else:\n", + " print(f\" Clean: {result.value!r}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Common Vulnerability Types: Injection and XSS\n", + "\n", + "**Injection attacks** occur when untrusted data is sent to an interpreter as part of a\n", + "command or query. The two most common types in web applications are:\n", + "\n", + "- **SQL Injection**: Malicious SQL code in user input modifies database queries\n", + "- **Cross-Site Scripting (XSS)**: Malicious JavaScript in user input executes in\n", + " other users' browsers\n", + "\n", + "The defense is always the same: **never trust user input**, and always use the appropriate\n", + "encoding or parameterization for the output context." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Demonstrating WHY injection is dangerous (conceptual examples)\n", + "\n", + "# BAD: String formatting for SQL (NEVER do this)\n", + "def bad_login_query(username: str, password: str) -> str:\n", + " \"\"\"INSECURE: Demonstrates SQL injection vulnerability.\n", + "\n", + " WARNING: This is an example of what NOT to do.\n", + " \"\"\"\n", + " return f\"SELECT * FROM users WHERE name='{username}' AND pass='{password}'\"\n", + "\n", + "\n", + "# Normal input\n", + "normal_query: str = bad_login_query(\"alice\", \"secret123\")\n", + "print(\"Normal query:\")\n", + "print(f\" {normal_query}\")\n", + "\n", + "# SQL injection: the attacker's input changes the query structure\n", + "injection_query: str = bad_login_query(\"' OR '1'='1\", \"' OR '1'='1\")\n", + "print(\"\\nInjected query (bypasses authentication):\")\n", + "print(f\" {injection_query}\")\n", + "\n", + "# Destructive injection\n", + "destructive_query: str = bad_login_query(\"'; DROP TABLE users;--\", \"anything\")\n", + "print(\"\\nDestructive injection:\")\n", + "print(f\" {destructive_query}\")\n", + "\n", + "# XSS example: user input rendered as HTML\n", + "def bad_greeting(name: str) -> str:\n", + " \"\"\"INSECURE: Demonstrates XSS vulnerability.\"\"\"\n", + " return f\"

Welcome, {name}!

\"\n", + "\n", + "\n", + "normal_html: str = bad_greeting(\"Alice\")\n", + "print(f\"\\nNormal HTML: {normal_html}\")\n", + "\n", + "xss_html: str = bad_greeting(\"\")\n", + "print(f\"XSS HTML: {xss_html}\")\n", + "print(\" ^ The script tag would execute in a browser!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## html.escape(): HTML Output Encoding\n", + "\n", + "When rendering user-supplied data in HTML, you must escape special characters to prevent\n", + "XSS attacks. `html.escape()` converts characters like `<`, `>`, `&`, and `\"` into their\n", + "HTML entity equivalents so they are displayed as text rather than interpreted as HTML." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import html\n", + "\n", + "\n", + "def safe_greeting(name: str) -> str:\n", + " \"\"\"SECURE: Escapes user input before embedding in HTML.\"\"\"\n", + " safe_name: str = html.escape(name)\n", + " return f\"

Welcome, {safe_name}!

\"\n", + "\n", + "\n", + "# html.escape() converts special HTML characters to entities\n", + "test_strings: list[str] = [\n", + " \"Alice\",\n", + " \"\",\n", + " 'Bob\" onmouseover=\"alert(1)',\n", + " \"Tom & Jerry \",\n", + " \"O'Brien\",\n", + "]\n", + "\n", + "print(\"html.escape() examples:\")\n", + "for s in test_strings:\n", + " escaped: str = html.escape(s)\n", + " print(f\"\\n Original: {s}\")\n", + " print(f\" Escaped: {escaped}\")\n", + "\n", + "# Safe rendering\n", + "print(\"\\nSafe HTML output:\")\n", + "print(f\" {safe_greeting('Alice')}\")\n", + "print(f\" {safe_greeting('')}\")\n", + "\n", + "# html.escape() with quote=True (default) escapes both \" and '\n", + "# quote=False leaves quotes unescaped (rarely what you want)\n", + "attr_value: str = 'value\" onclick=\"alert(1)'\n", + "print(f\"\\nAttribute escaping:\")\n", + "print(f\" Original: {attr_value}\")\n", + "print(f\" quote=True: {html.escape(attr_value, quote=True)}\")\n", + "print(f\" quote=False: {html.escape(attr_value, quote=False)}\")\n", + "\n", + "# html.unescape() reverses the process\n", + "encoded: str = \"<b>Bold</b> & "quoted"\"\n", + "print(f\"\\nhtml.unescape():\")\n", + "print(f\" Encoded: {encoded}\")\n", + "print(f\" Unescaped: {html.unescape(encoded)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## shlex.quote(): Shell Command Safety\n", + "\n", + "When you must construct shell commands that include user input (which you should avoid\n", + "whenever possible), `shlex.quote()` properly escapes the input to prevent shell injection.\n", + "\n", + "**Best practice**: Use `subprocess.run()` with a **list of arguments** instead of a shell\n", + "string. This avoids shell interpretation entirely. Use `shlex.quote()` only when you must\n", + "construct a shell command string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import shlex\n", + "import subprocess\n", + "\n", + "\n", + "# Dangerous: shell injection through string formatting\n", + "def bad_file_search(filename: str) -> str:\n", + " \"\"\"INSECURE: Demonstrates shell injection vulnerability.\"\"\"\n", + " return f\"find /tmp -name '{filename}'\"\n", + "\n", + "\n", + "# Normal input\n", + "print(\"Shell command construction:\")\n", + "print(f\" Normal: {bad_file_search('report.txt')}\")\n", + "\n", + "# Shell injection: the attacker breaks out of the quotes\n", + "malicious: str = \"'; rm -rf /; echo '\"\n", + "print(f\" Injected: {bad_file_search(malicious)}\")\n", + "print(\" ^ Would execute: rm -rf / (catastrophic!)\")\n", + "\n", + "# SAFE: Using shlex.quote()\n", + "def safe_file_search_cmd(filename: str) -> str:\n", + " \"\"\"SECURE: Uses shlex.quote() to prevent shell injection.\"\"\"\n", + " safe_name: str = shlex.quote(filename)\n", + " return f\"find /tmp -name {safe_name}\"\n", + "\n", + "\n", + "print(f\"\\n Safe cmd: {safe_file_search_cmd(malicious)}\")\n", + "print(\" ^ Malicious input is safely quoted as a single argument\")\n", + "\n", + "# shlex.quote() examples\n", + "dangerous_inputs: list[str] = [\n", + " \"normal.txt\",\n", + " \"file with spaces.txt\",\n", + " \"file;rm -rf /\",\n", + " \"$(whoami)\",\n", + " \"`id`\",\n", + " \"file\\nnewline\",\n", + "]\n", + "\n", + "print(\"\\nshlex.quote() examples:\")\n", + "for inp in dangerous_inputs:\n", + " print(f\" {inp!r:30s} -> {shlex.quote(inp)}\")\n", + "\n", + "# BEST: Use subprocess with a list (no shell at all)\n", + "print(\"\\nBest practice: subprocess with list args (no shell):\")\n", + "result = subprocess.run(\n", + " [\"echo\", \"Hello from\", \"subprocess!\"],\n", + " capture_output=True, text=True, check=False,\n", + ")\n", + "print(f\" Output: {result.stdout.strip()}\")\n", + "print(\" No shell interpretation = no shell injection possible\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parameterized Queries: SQL Injection Prevention\n", + "\n", + "The correct defense against SQL injection is **parameterized queries** (also called\n", + "prepared statements). Instead of formatting values into the SQL string, you pass them\n", + "as separate parameters. The database driver handles escaping automatically.\n", + "\n", + "Python's `sqlite3` module uses `?` as the parameter placeholder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3\n", + "\n", + "# Create an in-memory database for demonstration\n", + "conn = sqlite3.connect(\":memory:\")\n", + "cursor = conn.cursor()\n", + "\n", + "# Set up a users table\n", + "cursor.execute(\"\"\"\n", + " CREATE TABLE users (\n", + " id INTEGER PRIMARY KEY,\n", + " username TEXT NOT NULL,\n", + " email TEXT NOT NULL\n", + " )\n", + "\"\"\")\n", + "cursor.executemany(\n", + " \"INSERT INTO users (username, email) VALUES (?, ?)\",\n", + " [\n", + " (\"alice\", \"alice@example.com\"),\n", + " (\"bob\", \"bob@example.com\"),\n", + " (\"charlie\", \"charlie@example.com\"),\n", + " ],\n", + ")\n", + "conn.commit()\n", + "\n", + "# BAD: String formatting (vulnerable to SQL injection)\n", + "def bad_lookup(username: str) -> list[tuple[int, str, str]]:\n", + " \"\"\"INSECURE: Do NOT use string formatting for SQL.\"\"\"\n", + " query: str = f\"SELECT * FROM users WHERE username = '{username}'\"\n", + " return cursor.execute(query).fetchall()\n", + "\n", + "\n", + "# GOOD: Parameterized query (safe)\n", + "def safe_lookup(username: str) -> list[tuple[int, str, str]]:\n", + " \"\"\"SECURE: Uses parameterized query.\"\"\"\n", + " return cursor.execute(\n", + " \"SELECT * FROM users WHERE username = ?\", (username,)\n", + " ).fetchall()\n", + "\n", + "\n", + "# Normal lookup works with both\n", + "print(\"Normal lookup:\")\n", + "print(f\" Bad: {bad_lookup('alice')}\")\n", + "print(f\" Safe: {safe_lookup('alice')}\")\n", + "\n", + "# Injection attempt: returns all users with bad method\n", + "injection: str = \"' OR '1'='1\"\n", + "print(f\"\\nInjection attempt with: {injection!r}\")\n", + "print(f\" Bad (returns all): {bad_lookup(injection)}\")\n", + "print(f\" Safe (returns none): {safe_lookup(injection)}\")\n", + "\n", + "# The parameterized query treats the entire input as a literal value\n", + "# It looks for a user literally named \"' OR '1'='1\" (which doesn't exist)\n", + "\n", + "conn.close()\n", + "print(\"\\nKey takeaway: ALWAYS use parameterized queries (? placeholders)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ssl Module Basics: SSL Contexts and Certificate Verification\n", + "\n", + "The `ssl` module provides TLS/SSL support for network connections. An `SSLContext`\n", + "encapsulates the configuration for secure connections: protocol version, certificate\n", + "verification, cipher suites, etc.\n", + "\n", + "**Always** use `ssl.create_default_context()` unless you have a very specific reason\n", + "not to. It enables certificate verification, uses modern protocols, and disables\n", + "known-insecure options." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ssl\n", + "\n", + "# create_default_context(): the recommended way to create an SSL context\n", + "# It enables certificate verification and uses secure defaults\n", + "ctx: ssl.SSLContext = ssl.create_default_context()\n", + "\n", + "print(\"Default SSL context settings:\")\n", + "print(f\" Protocol: {ctx.protocol}\")\n", + "print(f\" Verify mode: {ctx.verify_mode}\")\n", + "print(f\" Check hostname: {ctx.check_hostname}\")\n", + "print(f\" Minimum TLS version: {ctx.minimum_version}\")\n", + "print(f\" Maximum TLS version: {ctx.maximum_version}\")\n", + "\n", + "# The default context loads the system's trusted CA certificates\n", + "stats: dict[str, int] = ctx.cert_store_stats()\n", + "print(f\"\\nCertificate store stats:\")\n", + "print(f\" x509 certificates: {stats['x509']}\")\n", + "print(f\" x509 CA certificates: {stats['x509_ca']}\")\n", + "print(f\" CRL entries: {stats['crl']}\")\n", + "\n", + "# List available ciphers (first 5 for brevity)\n", + "ciphers: list[dict[str, object]] = ctx.get_ciphers()\n", + "print(f\"\\nAvailable ciphers: {len(ciphers)} total\")\n", + "for cipher in ciphers[:5]:\n", + " print(f\" {cipher['name']}\")\n", + "print(\" ...\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ssl\n", + "import urllib.request\n", + "\n", + "# SECURE: Using default context (verifies certificates)\n", + "print(\"Secure HTTPS request (certificate verification enabled):\")\n", + "secure_ctx: ssl.SSLContext = ssl.create_default_context()\n", + "\n", + "try:\n", + " with urllib.request.urlopen(\"https://www.python.org\",\n", + " context=secure_ctx) as response:\n", + " print(f\" Status: {response.status}\")\n", + " print(f\" URL: {response.url}\")\n", + " # Get the server's certificate info\n", + " cert: dict | None = response.fp.raw._sock.getpeercert() # type: ignore[union-attr]\n", + " if cert:\n", + " subject: tuple = cert.get(\"subject\", ())\n", + " for field in subject:\n", + " for key, value in field:\n", + " if key == \"commonName\":\n", + " print(f\" Cert CN: {value}\")\n", + " not_after: str | None = cert.get(\"notAfter\")\n", + " if not_after:\n", + " print(f\" Expires: {not_after}\")\n", + "except Exception as e:\n", + " print(f\" Connection error: {e}\")\n", + "\n", + "# INSECURE: Disabling verification (NEVER do this in production)\n", + "print(\"\\nWARNING: Creating an insecure context (for demonstration only):\")\n", + "insecure_ctx = ssl.create_default_context()\n", + "insecure_ctx.check_hostname = False\n", + "insecure_ctx.verify_mode = ssl.CERT_NONE\n", + "print(f\" Verify mode: {insecure_ctx.verify_mode}\")\n", + "print(f\" Check hostname: {insecure_ctx.check_hostname}\")\n", + "print(\" NEVER disable certificate verification in production!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Environment Variables for Secrets (os.environ)\n", + "\n", + "Secrets like API keys, database passwords, and encryption keys should **never** be\n", + "hardcoded in source code. The standard approach is to use environment variables, which\n", + "can be set per-deployment without changing code.\n", + "\n", + "Common tools for managing environment variables:\n", + "- `.env` files loaded with `python-dotenv` (for development)\n", + "- System environment variables (for production)\n", + "- Secret management services (AWS Secrets Manager, HashiCorp Vault, etc.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "\n", + "def get_required_env(name: str) -> str:\n", + " \"\"\"Get a required environment variable or raise an error.\n", + "\n", + " Use this for secrets that MUST be set for the application to run.\n", + " \"\"\"\n", + " value: str | None = os.environ.get(name)\n", + " if value is None:\n", + " raise RuntimeError(\n", + " f\"Required environment variable {name!r} is not set. \"\n", + " f\"Set it before running the application.\"\n", + " )\n", + " return value\n", + "\n", + "\n", + "def get_optional_env(name: str, default: str = \"\") -> str:\n", + " \"\"\"Get an optional environment variable with a default.\"\"\"\n", + " return os.environ.get(name, default)\n", + "\n", + "\n", + "class AppConfig:\n", + " \"\"\"Application configuration loaded from environment variables.\n", + "\n", + " Centralizing config access makes it easy to audit which secrets\n", + " your application uses and where they come from.\n", + " \"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " # Set some demo variables for this example\n", + " os.environ[\"APP_SECRET_KEY\"] = \"demo-secret-key-12345\"\n", + " os.environ[\"APP_DATABASE_URL\"] = \"postgresql://user:pass@localhost/db\"\n", + " os.environ[\"APP_DEBUG\"] = \"false\"\n", + "\n", + " self.secret_key: str = get_required_env(\"APP_SECRET_KEY\")\n", + " self.database_url: str = get_required_env(\"APP_DATABASE_URL\")\n", + " self.debug: bool = get_optional_env(\"APP_DEBUG\", \"false\").lower() == \"true\"\n", + " self.log_level: str = get_optional_env(\"APP_LOG_LEVEL\", \"INFO\")\n", + "\n", + " def __repr__(self) -> str:\n", + " \"\"\"Repr that masks sensitive values.\"\"\"\n", + " return (\n", + " f\"AppConfig(\"\n", + " f\"secret_key='***', \"\n", + " f\"database_url='***', \"\n", + " f\"debug={self.debug}, \"\n", + " f\"log_level={self.log_level!r})\"\n", + " )\n", + "\n", + "\n", + "# Load configuration\n", + "config = AppConfig()\n", + "print(f\"Config: {config}\")\n", + "print(f\" Note: __repr__ masks secrets to prevent accidental logging\")\n", + "\n", + "# Missing required variable\n", + "print(\"\\nMissing required variable:\")\n", + "try:\n", + " _ = get_required_env(\"NONEXISTENT_SECRET\")\n", + "except RuntimeError as e:\n", + " print(f\" Error: {e}\")\n", + "\n", + "# BAD: Hardcoded secrets (NEVER do this)\n", + "print(\"\\nAnti-patterns to avoid:\")\n", + "print(' BAD: API_KEY = \"sk-1234567890abcdef\" # Hardcoded in source')\n", + "print(' BAD: password = \"admin123\" # Literal password')\n", + "print(' GOOD: API_KEY = os.environ[\"API_KEY\"] # From environment')\n", + "print(' GOOD: password = get_required_env(\"DB_PASSWORD\") # Validated')\n", + "\n", + "# Cleanup demo variables\n", + "for key in [\"APP_SECRET_KEY\", \"APP_DATABASE_URL\", \"APP_DEBUG\"]:\n", + " os.environ.pop(key, None)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Secure Coding Checklist and Principles\n", + "\n", + "Security is not a feature you add at the end -- it is a practice woven into every\n", + "line of code. Below are essential principles to follow." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Principle 1: Defense in Depth\n", + "# Apply multiple layers of security -- never rely on a single check\n", + "\n", + "import re\n", + "import html\n", + "\n", + "\n", + "def process_comment(raw_comment: str) -> str:\n", + " \"\"\"Process a user comment with multiple security layers.\n", + "\n", + " Layer 1: Input validation (reject clearly invalid input)\n", + " Layer 2: Sanitization (clean the input)\n", + " Layer 3: Output encoding (escape for the target context)\n", + " \"\"\"\n", + " # Layer 1: Reject if too long or empty\n", + " if not raw_comment.strip() or len(raw_comment) > 10_000:\n", + " return \"\"\n", + "\n", + " # Layer 2: Strip control characters (except newlines)\n", + " cleaned: str = re.sub(r\"[\\x00-\\x09\\x0b-\\x1f\\x7f]\", \"\", raw_comment)\n", + " cleaned = cleaned.strip()\n", + "\n", + " # Layer 3: HTML-escape for safe rendering\n", + " safe: str = html.escape(cleaned)\n", + "\n", + " return safe\n", + "\n", + "\n", + "test_comments: list[str] = [\n", + " \"Great article! Thanks for sharing.\",\n", + " \"\",\n", + " \"Normal text with \\x00null\\x01bytes\",\n", + "]\n", + "\n", + "print(\"Defense in Depth: Processing user comments\")\n", + "for comment in test_comments:\n", + " result: str = process_comment(comment)\n", + " print(f\" Input: {comment!r}\")\n", + " print(f\" Output: {result!r}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Principle 2: Least Privilege and Fail Securely\n", + "\n", + "import os\n", + "import stat\n", + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "\n", + "def create_secure_temp_file(content: str) -> Path:\n", + " \"\"\"Create a temporary file with restrictive permissions.\n", + "\n", + " Only the owner can read/write the file (mode 0o600).\n", + " \"\"\"\n", + " fd, path_str = tempfile.mkstemp(suffix=\".secret\", prefix=\"app_\")\n", + " path = Path(path_str)\n", + " try:\n", + " # Set restrictive permissions BEFORE writing content\n", + " os.chmod(fd, stat.S_IRUSR | stat.S_IWUSR) # 0o600\n", + " os.write(fd, content.encode(\"utf-8\"))\n", + " finally:\n", + " os.close(fd)\n", + " return path\n", + "\n", + "\n", + "# Create a secure temp file\n", + "secret_file: Path = create_secure_temp_file(\"secret_api_key=abc123\")\n", + "file_stat = secret_file.stat()\n", + "mode: str = oct(file_stat.st_mode)[-3:]\n", + "print(f\"Secure temp file: {secret_file}\")\n", + "print(f\" Permissions: {mode} (owner read/write only)\")\n", + "print(f\" Contents: {secret_file.read_text()}\")\n", + "\n", + "# Fail securely: never expose internals in error messages\n", + "def safe_division(a: float, b: float) -> dict[str, object]:\n", + " \"\"\"Division that fails securely with a generic error message.\n", + "\n", + " Internal details are logged, not shown to the user.\n", + " \"\"\"\n", + " try:\n", + " result: float = a / b\n", + " return {\"success\": True, \"result\": result}\n", + " except ZeroDivisionError:\n", + " # Log the full error internally (would use logging in production)\n", + " # logger.error(f\"Division by zero: {a}/{b}\", exc_info=True)\n", + "\n", + " # Return a generic message to the user\n", + " return {\"success\": False, \"error\": \"Invalid operation\"}\n", + "\n", + "\n", + "print(f\"\\nFail securely:\")\n", + "print(f\" 10 / 3 = {safe_division(10, 3)}\")\n", + "print(f\" 10 / 0 = {safe_division(10, 0)}\")\n", + "print(\" Note: error message is generic, no internal details exposed\")\n", + "\n", + "# Cleanup\n", + "secret_file.unlink()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Principle 3: Sensitive Data Handling\n", + "# Minimize exposure of sensitive data in memory and logs\n", + "\n", + "import secrets\n", + "\n", + "\n", + "class SensitiveString:\n", + " \"\"\"A string wrapper that prevents accidental exposure in logs.\n", + "\n", + " The actual value is only accessible through the .get_secret_value() method.\n", + " __repr__ and __str__ always return a masked version.\n", + " \"\"\"\n", + "\n", + " def __init__(self, value: str) -> None:\n", + " self._value: str = value\n", + "\n", + " def get_secret_value(self) -> str:\n", + " \"\"\"Explicitly retrieve the secret value.\"\"\"\n", + " return self._value\n", + "\n", + " def __repr__(self) -> str:\n", + " return \"SensitiveString('***')\"\n", + "\n", + " def __str__(self) -> str:\n", + " return \"***\"\n", + "\n", + " def __eq__(self, other: object) -> bool:\n", + " if isinstance(other, SensitiveString):\n", + " # Use constant-time comparison\n", + " import hmac\n", + " return hmac.compare_digest(\n", + " self._value.encode(), other._value.encode()\n", + " )\n", + " return NotImplemented\n", + "\n", + " def __hash__(self) -> int:\n", + " return hash(self._value)\n", + "\n", + "\n", + "# Usage\n", + "api_key = SensitiveString(secrets.token_hex(16))\n", + "db_password = SensitiveString(\"super-secret-password\")\n", + "\n", + "# Safe to log or print\n", + "print(f\"API Key: {api_key}\") # Prints: ***\n", + "print(f\"DB Pass: {db_password!r}\") # Prints: SensitiveString('***')\n", + "\n", + "# Must explicitly request the value\n", + "print(f\"\\nActual API Key (explicit): {api_key.get_secret_value()[:16]}...\")\n", + "\n", + "# Safe in data structures\n", + "config_data: dict[str, object] = {\n", + " \"host\": \"localhost\",\n", + " \"port\": 5432,\n", + " \"password\": db_password,\n", + "}\n", + "print(f\"\\nConfig dict: {config_data}\")\n", + "print(\" Password is masked even in dict repr\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Principle | Tool / Technique | Purpose |\n", + "|-----------|------------------|---------|\n", + "| **Input validation** | Whitelist patterns, `re.match()` | Reject invalid input at the boundary |\n", + "| **HTML safety** | `html.escape()` | Prevent XSS in rendered HTML |\n", + "| **Shell safety** | `shlex.quote()`, `subprocess.run(list)` | Prevent shell injection |\n", + "| **SQL safety** | Parameterized queries (`?` placeholders) | Prevent SQL injection |\n", + "| **TLS/SSL** | `ssl.create_default_context()` | Secure network connections with cert verification |\n", + "| **Secrets management** | `os.environ`, config classes | Keep secrets out of source code |\n", + "| **Defense in depth** | Validate + sanitize + encode | Multiple security layers |\n", + "| **Fail securely** | Generic error messages | Never expose internals to users |\n", + "\n", + "### Secure Coding Checklist\n", + "- Validate all external input using whitelist patterns\n", + "- Use parameterized queries for all database operations\n", + "- Escape output for the target context (HTML, shell, SQL, etc.)\n", + "- Store secrets in environment variables, never in source code\n", + "- Use `ssl.create_default_context()` for all TLS connections\n", + "- Never disable certificate verification in production\n", + "- Mask sensitive data in logs and error messages\n", + "- Apply the principle of least privilege to file permissions and access\n", + "- Prefer `subprocess.run()` with list arguments over shell strings\n", + "- Use constant-time comparison (`hmac.compare_digest()`) for security tokens" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_23/README.md b/src/chapter_23/README.md new file mode 100644 index 0000000..7ca2c76 --- /dev/null +++ b/src/chapter_23/README.md @@ -0,0 +1,22 @@ +# Chapter 23: Security and Cryptography + +## Topics Covered +- `hashlib`: SHA-256, MD5, BLAKE2, file hashing +- `hmac`: message authentication codes +- `secrets`: cryptographically secure random values +- Password hashing and verification patterns +- Input validation and sanitization +- Common vulnerabilities: injection, XSS concepts +- `ssl` module basics and certificate verification +- Secure coding principles + +## Notebooks +1. **01_hashing_and_hmac.ipynb** — hashlib, HMAC, file integrity, password hashing +2. **02_secrets_and_tokens.ipynb** — secrets module, token generation, secure random +3. **03_secure_coding.ipynb** — Input validation, common vulnerabilities, SSL basics + +## Key Takeaways +- Never store passwords in plain text — use proper hashing +- Use `secrets` module for tokens, not `random` +- HMAC provides message integrity and authentication +- Validate all input at system boundaries diff --git a/src/chapter_23/__init__.py b/src/chapter_23/__init__.py new file mode 100644 index 0000000..9a6036d --- /dev/null +++ b/src/chapter_23/__init__.py @@ -0,0 +1 @@ +"""Chapter 23: Security and Cryptography.""" diff --git a/src/chapter_24/01_dynamic_attributes.ipynb b/src/chapter_24/01_dynamic_attributes.ipynb new file mode 100644 index 0000000..b81cc09 --- /dev/null +++ b/src/chapter_24/01_dynamic_attributes.ipynb @@ -0,0 +1,615 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 24: Dynamic Attributes and Metaprogramming\n", + "\n", + "Python gives you fine-grained control over attribute access through a family of dunder\n", + "methods: `__getattr__`, `__getattribute__`, `__setattr__`, and `__delattr__`. These hooks\n", + "let you intercept how attributes are read, written, and deleted, enabling patterns like\n", + "proxy objects, lazy loading, validation, and dynamic configuration.\n", + "\n", + "## Topics Covered\n", + "- **`__getattr__`**: Called when normal attribute lookup fails\n", + "- **`__getattribute__`**: Called on every attribute access (use carefully)\n", + "- **`__setattr__`**: Intercepting attribute assignment and validation\n", + "- **`__delattr__`**: Intercepting attribute deletion\n", + "- **Proxy objects** with `__getattr__`\n", + "- **Lazy attribute loading** patterns\n", + "- **Property factories**: Creating properties dynamically\n", + "- **Practical**: A configuration object with dot notation access" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `__getattr__`: Fallback for Missing Attributes\n", + "\n", + "`__getattr__` is called **only** when the normal attribute lookup mechanism fails --\n", + "that is, when the attribute is not found in the instance `__dict__`, the class, or any\n", + "base class. This makes it safe and efficient: it does not interfere with attributes\n", + "that actually exist." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class FlexibleObject:\n", + " \"\"\"Demonstrates __getattr__ as a fallback for missing attributes.\"\"\"\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name # This is a real attribute\n", + "\n", + " def __getattr__(self, attr: str) -> str:\n", + " \"\"\"Called ONLY when normal lookup fails.\"\"\"\n", + " print(f\" __getattr__ called for: {attr!r}\")\n", + " return f\"\"\n", + "\n", + "\n", + "obj = FlexibleObject(\"demo\")\n", + "\n", + "# Accessing a real attribute -- __getattr__ is NOT called\n", + "print(f\"obj.name = {obj.name!r}\")\n", + "\n", + "# Accessing a missing attribute -- __getattr__ IS called\n", + "print(f\"obj.color = {obj.color!r}\")\n", + "print(f\"obj.size = {obj.size!r}\")\n", + "\n", + "# hasattr uses __getattr__ under the hood\n", + "print(f\"\\nhasattr(obj, 'name'): {hasattr(obj, 'name')}\")\n", + "print(f\"hasattr(obj, 'anything'): {hasattr(obj, 'anything')}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `__getattribute__`: Intercepting Every Attribute Access\n", + "\n", + "`__getattribute__` is called on **every** attribute access, even for attributes that\n", + "exist. This is powerful but dangerous -- if implemented incorrectly it can cause infinite\n", + "recursion. Always use `super().__getattribute__()` or `object.__getattribute__()` to\n", + "perform the actual lookup inside your override." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class AuditedAccess:\n", + " \"\"\"Logs every attribute access using __getattribute__.\"\"\"\n", + "\n", + " def __init__(self, x: int, y: int) -> None:\n", + " self.x = x\n", + " self.y = y\n", + " self._access_log: list[str] = []\n", + "\n", + " def __getattribute__(self, name: str) -> object:\n", + " \"\"\"Called on EVERY attribute access, including existing ones.\"\"\"\n", + " # Must use object.__getattribute__ to avoid infinite recursion\n", + " if not name.startswith(\"_\"):\n", + " log = object.__getattribute__(self, \"_access_log\")\n", + " log.append(name)\n", + " return object.__getattribute__(self, name)\n", + "\n", + " def get_log(self) -> list[str]:\n", + " return self._access_log\n", + "\n", + "\n", + "audited = AuditedAccess(10, 20)\n", + "print(f\"audited.x = {audited.x}\")\n", + "print(f\"audited.y = {audited.y}\")\n", + "print(f\"audited.x = {audited.x}\")\n", + "\n", + "# The access log recorded every attribute read\n", + "print(f\"\\nAccess log: {audited.get_log()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `__setattr__`: Intercepting Attribute Assignment\n", + "\n", + "`__setattr__` is called on **every** attribute assignment, including during `__init__`.\n", + "This makes it ideal for validation but requires care: use `object.__setattr__()` or\n", + "`self.__dict__[name] = value` to actually store the attribute without recursion." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class ValidatedPoint:\n", + " \"\"\"Validates that x and y are always numeric.\"\"\"\n", + "\n", + " def __init__(self, x: float, y: float) -> None:\n", + " self.x = x # Goes through __setattr__\n", + " self.y = y\n", + "\n", + " def __setattr__(self, name: str, value: object) -> None:\n", + " if name in (\"x\", \"y\"):\n", + " if not isinstance(value, (int, float)):\n", + " raise TypeError(\n", + " f\"{name!r} must be numeric, got {type(value).__name__}\"\n", + " )\n", + " # Use object.__setattr__ to actually store the value\n", + " object.__setattr__(self, name, float(value))\n", + " else:\n", + " object.__setattr__(self, name, value)\n", + "\n", + "\n", + "p = ValidatedPoint(3, 4)\n", + "print(f\"p.x = {p.x}, p.y = {p.y}\")\n", + "\n", + "p.x = 10 # Validated and converted to float\n", + "print(f\"After p.x = 10: p.x = {p.x}\")\n", + "\n", + "# Invalid assignment raises TypeError\n", + "try:\n", + " p.x = \"hello\"\n", + "except TypeError as e:\n", + " print(f\"\\nTypeError: {e}\")\n", + "\n", + "# Other attributes are accepted without validation\n", + "p.label = \"origin\"\n", + "print(f\"p.label = {p.label!r}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `__delattr__`: Intercepting Attribute Deletion\n", + "\n", + "`__delattr__` is called when `del obj.attr` is executed. You can use it to\n", + "prevent deletion of critical attributes or to perform cleanup." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class ProtectedAttributes:\n", + " \"\"\"Prevents deletion of protected attributes.\"\"\"\n", + "\n", + " _protected: frozenset[str] = frozenset({\"id\", \"created_at\"})\n", + "\n", + " def __init__(self, id: int, created_at: str, name: str) -> None:\n", + " self.id = id\n", + " self.created_at = created_at\n", + " self.name = name\n", + "\n", + " def __delattr__(self, name: str) -> None:\n", + " if name in self._protected:\n", + " raise AttributeError(\n", + " f\"Cannot delete protected attribute {name!r}\"\n", + " )\n", + " print(f\" Deleting attribute {name!r}\")\n", + " object.__delattr__(self, name)\n", + "\n", + "\n", + "record = ProtectedAttributes(42, \"2025-01-01\", \"Alice\")\n", + "print(f\"record.name = {record.name!r}\")\n", + "\n", + "# Deleting an unprotected attribute works\n", + "del record.name\n", + "print(f\"hasattr(record, 'name'): {hasattr(record, 'name')}\")\n", + "\n", + "# Deleting a protected attribute is blocked\n", + "try:\n", + " del record.id\n", + "except AttributeError as e:\n", + " print(f\"\\nAttributeError: {e}\")\n", + "\n", + "print(f\"record.id still = {record.id}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building Proxy Objects with `__getattr__`\n", + "\n", + "A **proxy** wraps another object and forwards attribute access to it. `__getattr__`\n", + "is perfect for this because it only fires when the proxy itself does not have the\n", + "requested attribute, naturally delegating to the wrapped object." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "class LoggingProxy:\n", + " \"\"\"Wraps any object and logs all attribute access and method calls.\"\"\"\n", + "\n", + " def __init__(self, target: Any) -> None:\n", + " # Store target in __dict__ directly to avoid triggering __setattr__\n", + " object.__setattr__(self, \"_target\", target)\n", + " object.__setattr__(self, \"_log\", [])\n", + "\n", + " def __getattr__(self, name: str) -> Any:\n", + " target = object.__getattribute__(self, \"_target\")\n", + " log = object.__getattribute__(self, \"_log\")\n", + "\n", + " attr = getattr(target, name)\n", + " if callable(attr):\n", + " def logged_method(*args: Any, **kwargs: Any) -> Any:\n", + " log.append(f\"CALL: {name}({args}, {kwargs})\")\n", + " return attr(*args, **kwargs)\n", + " return logged_method\n", + " else:\n", + " log.append(f\"GET: {name} -> {attr!r}\")\n", + " return attr\n", + "\n", + "\n", + "# Proxy a regular list\n", + "real_list: list[int] = [1, 2, 3]\n", + "proxy = LoggingProxy(real_list)\n", + "\n", + "# All operations are forwarded to the real list\n", + "proxy.append(4)\n", + "proxy.extend([5, 6])\n", + "length = proxy.__len__()\n", + "\n", + "print(f\"Real list: {real_list}\")\n", + "print(f\"\\nProxy log:\")\n", + "for entry in proxy._log:\n", + " print(f\" {entry}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Lazy Attribute Loading Patterns\n", + "\n", + "Lazy loading defers expensive computation until the attribute is first accessed.\n", + "You can implement this with `__getattr__` -- the attribute starts missing, gets\n", + "computed on first access, and then is stored so `__getattr__` is never called again\n", + "for that attribute." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import time\n", + "from typing import Any\n", + "\n", + "\n", + "class LazyDataLoader:\n", + " \"\"\"Loads expensive data on first access, then caches it.\"\"\"\n", + "\n", + " def __init__(self, source: str) -> None:\n", + " self.source = source\n", + " # Note: 'data' and 'summary' are NOT set here -- they are lazy\n", + "\n", + " def _load_data(self) -> list[dict[str, Any]]:\n", + " \"\"\"Simulate expensive data loading.\"\"\"\n", + " print(f\" Loading data from {self.source!r} (expensive operation)...\")\n", + " time.sleep(0.1) # Simulate I/O delay\n", + " return [\n", + " {\"id\": 1, \"value\": 100},\n", + " {\"id\": 2, \"value\": 200},\n", + " {\"id\": 3, \"value\": 300},\n", + " ]\n", + "\n", + " def _compute_summary(self) -> dict[str, float]:\n", + " \"\"\"Compute a summary from the loaded data.\"\"\"\n", + " print(\" Computing summary (depends on data)...\")\n", + " values = [row[\"value\"] for row in self.data]\n", + " return {\"count\": len(values), \"total\": sum(values), \"mean\": sum(values) / len(values)}\n", + "\n", + " def __getattr__(self, name: str) -> Any:\n", + " \"\"\"Lazy-load attributes on first access.\"\"\"\n", + " loaders: dict[str, Any] = {\n", + " \"data\": self._load_data,\n", + " \"summary\": self._compute_summary,\n", + " }\n", + " if name in loaders:\n", + " value = loaders[name]()\n", + " # Store in __dict__ so __getattr__ is not called again\n", + " self.__dict__[name] = value\n", + " return value\n", + " raise AttributeError(f\"{type(self).__name__!r} has no attribute {name!r}\")\n", + "\n", + "\n", + "loader = LazyDataLoader(\"database\")\n", + "print(\"Created loader -- no data loaded yet\")\n", + "print(f\"loader.__dict__ keys: {list(loader.__dict__.keys())}\\n\")\n", + "\n", + "# First access triggers loading\n", + "print(f\"loader.data = {loader.data}\\n\")\n", + "\n", + "# Second access is instant -- cached in __dict__\n", + "print(\"Accessing again (should be cached):\")\n", + "print(f\"loader.data = {loader.data}\\n\")\n", + "\n", + "# Summary depends on data (which is already loaded)\n", + "print(f\"loader.summary = {loader.summary}\")\n", + "\n", + "print(f\"\\nFinal __dict__ keys: {list(loader.__dict__.keys())}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Property Factories: Creating Properties Dynamically\n", + "\n", + "You can create `property` objects programmatically and attach them to classes at\n", + "runtime. This is useful when you want many similar properties with the same pattern\n", + "(e.g., validated, type-checked, or logged)." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "def make_validated_property(\n", + " name: str,\n", + " expected_type: type,\n", + " min_value: float | None = None,\n", + " max_value: float | None = None,\n", + ") -> property:\n", + " \"\"\"Factory that creates a validated property with range checking.\"\"\"\n", + " storage_name = f\"_{name}\"\n", + "\n", + " def getter(self: Any) -> Any:\n", + " return getattr(self, storage_name, None)\n", + "\n", + " def setter(self: Any, value: Any) -> None:\n", + " if not isinstance(value, expected_type):\n", + " raise TypeError(\n", + " f\"{name!r} must be {expected_type.__name__}, got {type(value).__name__}\"\n", + " )\n", + " if min_value is not None and value < min_value:\n", + " raise ValueError(f\"{name!r} must be >= {min_value}, got {value}\")\n", + " if max_value is not None and value > max_value:\n", + " raise ValueError(f\"{name!r} must be <= {max_value}, got {value}\")\n", + " setattr(self, storage_name, value)\n", + "\n", + " return property(getter, setter, doc=f\"Validated {expected_type.__name__} property\")\n", + "\n", + "\n", + "class Sensor:\n", + " \"\"\"A sensor with dynamically created validated properties.\"\"\"\n", + " temperature = make_validated_property(\"temperature\", (int, float), -273.15, 1000.0)\n", + " humidity = make_validated_property(\"humidity\", (int, float), 0.0, 100.0)\n", + " label = make_validated_property(\"label\", str)\n", + "\n", + "\n", + "sensor = Sensor()\n", + "sensor.temperature = 22.5\n", + "sensor.humidity = 65\n", + "sensor.label = \"living_room\"\n", + "\n", + "print(f\"temperature: {sensor.temperature}\")\n", + "print(f\"humidity: {sensor.humidity}\")\n", + "print(f\"label: {sensor.label!r}\")\n", + "\n", + "# Validation in action\n", + "for desc, action in [\n", + " (\"temperature = 'hot'\", lambda: setattr(sensor, \"temperature\", \"hot\")),\n", + " (\"temperature = -300\", lambda: setattr(sensor, \"temperature\", -300)),\n", + " (\"humidity = 150\", lambda: setattr(sensor, \"humidity\", 150)),\n", + "]:\n", + " try:\n", + " action()\n", + " except (TypeError, ValueError) as e:\n", + " print(f\"\\n{desc} -> {type(e).__name__}: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: Configuration Object with Dot Notation Access\n", + "\n", + "A common use case for dynamic attributes is building a configuration object that\n", + "allows nested dot notation (`config.database.host`) backed by a plain dictionary.\n", + "This combines `__getattr__`, `__setattr__`, and `__delattr__` into a cohesive pattern." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "class DotConfig:\n", + " \"\"\"Configuration object that supports nested dot notation access.\n", + "\n", + " Wraps a dictionary so you can write `config.database.host` instead\n", + " of `config['database']['host']`.\n", + " \"\"\"\n", + "\n", + " def __init__(self, data: dict[str, Any] | None = None, **kwargs: Any) -> None:\n", + " # Use object.__setattr__ to bypass our custom __setattr__\n", + " object.__setattr__(self, \"_data\", {})\n", + " source = data if data is not None else kwargs\n", + " for key, value in source.items():\n", + " self[key] = value # Uses __setitem__ -> recursion for nested dicts\n", + "\n", + " def __getattr__(self, name: str) -> Any:\n", + " \"\"\"Access config values with dot notation.\"\"\"\n", + " try:\n", + " return self._data[name]\n", + " except KeyError:\n", + " raise AttributeError(f\"No config key {name!r}\") from None\n", + "\n", + " def __setattr__(self, name: str, value: Any) -> None:\n", + " \"\"\"Set config values with dot notation.\"\"\"\n", + " self[name] = value\n", + "\n", + " def __delattr__(self, name: str) -> None:\n", + " \"\"\"Delete config values with dot notation.\"\"\"\n", + " try:\n", + " del self._data[name]\n", + " except KeyError:\n", + " raise AttributeError(f\"No config key {name!r}\") from None\n", + "\n", + " def __setitem__(self, key: str, value: Any) -> None:\n", + " \"\"\"Recursively wrap nested dicts as DotConfig.\"\"\"\n", + " if isinstance(value, dict):\n", + " value = DotConfig(value)\n", + " self._data[key] = value\n", + "\n", + " def __contains__(self, key: str) -> bool:\n", + " return key in self._data\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"DotConfig({self._data!r})\"\n", + "\n", + " def to_dict(self) -> dict[str, Any]:\n", + " \"\"\"Convert back to a plain dictionary.\"\"\"\n", + " result: dict[str, Any] = {}\n", + " for key, value in self._data.items():\n", + " result[key] = value.to_dict() if isinstance(value, DotConfig) else value\n", + " return result\n", + "\n", + "\n", + "# Build config from a nested dictionary\n", + "raw_config: dict[str, Any] = {\n", + " \"app_name\": \"my_service\",\n", + " \"debug\": True,\n", + " \"database\": {\n", + " \"host\": \"localhost\",\n", + " \"port\": 5432,\n", + " \"credentials\": {\n", + " \"user\": \"admin\",\n", + " \"password\": \"secret\",\n", + " },\n", + " },\n", + " \"features\": {\n", + " \"enable_cache\": True,\n", + " \"max_retries\": 3,\n", + " },\n", + "}\n", + "\n", + "config = DotConfig(raw_config)\n", + "\n", + "# Dot notation access, including deeply nested values\n", + "print(f\"App name: {config.app_name!r}\")\n", + "print(f\"Debug: {config.debug}\")\n", + "print(f\"DB host: {config.database.host!r}\")\n", + "print(f\"DB port: {config.database.port}\")\n", + "print(f\"DB user: {config.database.credentials.user!r}\")\n", + "print(f\"Cache on: {config.features.enable_cache}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Mutating the config with dot notation\n", + "config.database.port = 3306\n", + "config.database.credentials.password = \"new_secret\"\n", + "config.features.max_retries = 5\n", + "\n", + "# Adding new nested sections\n", + "config.logging = {\"level\": \"INFO\", \"file\": \"/var/log/app.log\"}\n", + "\n", + "print(f\"Updated DB port: {config.database.port}\")\n", + "print(f\"Updated password: {config.database.credentials.password!r}\")\n", + "print(f\"Updated retries: {config.features.max_retries}\")\n", + "print(f\"New log level: {config.logging.level!r}\")\n", + "\n", + "# Deleting a config key\n", + "del config.debug\n", + "print(f\"\\n'debug' in config: {'debug' in config}\")\n", + "\n", + "# Convert back to plain dict\n", + "print(f\"\\nAs dict: {config.to_dict()}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Hook | When Called | Use Case |\n", + "|------|------------|----------|\n", + "| `__getattr__` | Attribute not found by normal lookup | Defaults, proxies, lazy loading |\n", + "| `__getattribute__` | Every attribute access | Auditing, access control (use sparingly) |\n", + "| `__setattr__` | Every attribute assignment | Validation, logging, read-only attrs |\n", + "| `__delattr__` | Every `del obj.attr` | Protecting critical attributes |\n", + "\n", + "### Best Practices\n", + "- Prefer `__getattr__` over `__getattribute__` -- it is safer and more efficient\n", + "- In `__getattribute__` and `__setattr__`, use `object.__getattribute__()` or\n", + " `object.__setattr__()` to avoid infinite recursion\n", + "- Cache lazily loaded values in `__dict__` so the hook is only called once\n", + "- Property factories reduce boilerplate when many attributes share the same validation\n", + "- Proxy objects with `__getattr__` are great for logging, caching, and access control\n", + "- For nested dict access, a `DotConfig` wrapper can make configuration code much cleaner" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_24/02_class_decorators.ipynb b/src/chapter_24/02_class_decorators.ipynb new file mode 100644 index 0000000..f2e5577 --- /dev/null +++ b/src/chapter_24/02_class_decorators.ipynb @@ -0,0 +1,672 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 24: Class Decorators and Subclass Hooks\n", + "\n", + "Class decorators and `__init_subclass__` are powerful tools for modifying classes after\n", + "they are created. They sit in the sweet spot between simple function decorators and the\n", + "full complexity of metaclasses, handling the vast majority of class-customization needs\n", + "with clear, readable code.\n", + "\n", + "## Topics Covered\n", + "- **Class decorators**: Modifying classes after creation\n", + "- **Adding methods and attributes** with decorators\n", + "- **`@dataclass`** as the canonical class decorator example\n", + "- **`__init_subclass__`**: Hook called when a class is subclassed\n", + "- **Plugin registration** with `__init_subclass__`\n", + "- **Comparing**: Class decorator vs metaclass vs `__init_subclass__`\n", + "- **Decorator stacking** order and composition\n", + "- **Practical**: Auto-registering command handlers" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Class Decorators: Modifying Classes After Creation\n", + "\n", + "A class decorator is a callable that takes a class object as its argument and returns\n", + "a (usually modified) class. The syntax `@decorator` before `class Foo:` is equivalent\n", + "to `Foo = decorator(Foo)` after the class body executes." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any, TypeVar\n", + "\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "def add_repr(cls: type[T]) -> type[T]:\n", + " \"\"\"Class decorator that adds a __repr__ based on __init__ parameters.\"\"\"\n", + "\n", + " def __repr__(self: Any) -> str:\n", + " attrs = \", \".join(\n", + " f\"{k}={v!r}\"\n", + " for k, v in self.__dict__.items()\n", + " if not k.startswith(\"_\")\n", + " )\n", + " return f\"{type(self).__name__}({attrs})\"\n", + "\n", + " cls.__repr__ = __repr__ # type: ignore[attr-defined]\n", + " return cls\n", + "\n", + "\n", + "def add_eq(cls: type[T]) -> type[T]:\n", + " \"\"\"Class decorator that adds __eq__ based on public attributes.\"\"\"\n", + "\n", + " def __eq__(self: Any, other: Any) -> bool:\n", + " if type(self) is not type(other):\n", + " return NotImplemented\n", + " return self.__dict__ == other.__dict__\n", + "\n", + " cls.__eq__ = __eq__ # type: ignore[attr-defined]\n", + " return cls\n", + "\n", + "\n", + "@add_repr\n", + "@add_eq\n", + "class Color:\n", + " def __init__(self, r: int, g: int, b: int) -> None:\n", + " self.r = r\n", + " self.g = g\n", + " self.b = b\n", + "\n", + "\n", + "c1 = Color(255, 128, 0)\n", + "c2 = Color(255, 128, 0)\n", + "c3 = Color(0, 0, 0)\n", + "\n", + "print(f\"c1 = {c1!r}\")\n", + "print(f\"c2 = {c2!r}\")\n", + "print(f\"c1 == c2: {c1 == c2}\")\n", + "print(f\"c1 == c3: {c1 == c3}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adding Methods and Attributes with Decorators\n", + "\n", + "Class decorators can inject methods, class attributes, or even modify the class\n", + "hierarchy. A parameterized class decorator (a decorator factory) takes arguments\n", + "and returns the actual decorator." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any, TypeVar\n", + "from datetime import datetime, timezone\n", + "\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "def timestamped(cls: type[T]) -> type[T]:\n", + " \"\"\"Inject created_at tracking into a class.\"\"\"\n", + " original_init = cls.__init__\n", + "\n", + " def new_init(self: Any, *args: Any, **kwargs: Any) -> None:\n", + " original_init(self, *args, **kwargs)\n", + " self.created_at = datetime.now(timezone.utc)\n", + "\n", + " cls.__init__ = new_init # type: ignore[attr-defined]\n", + " return cls\n", + "\n", + "\n", + "def with_version(version: str):\n", + " \"\"\"Parameterized class decorator that adds a version attribute.\"\"\"\n", + " def decorator(cls: type[T]) -> type[T]:\n", + " cls.version = version # type: ignore[attr-defined]\n", + "\n", + " def get_info(self: Any) -> str:\n", + " return f\"{type(self).__name__} v{cls.version}\" # type: ignore[attr-defined]\n", + "\n", + " cls.get_info = get_info # type: ignore[attr-defined]\n", + " return cls\n", + " return decorator\n", + "\n", + "\n", + "@timestamped\n", + "@with_version(\"2.1.0\")\n", + "class UserService:\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + "\n", + "\n", + "svc = UserService(\"auth\")\n", + "print(f\"svc.name = {svc.name!r}\")\n", + "print(f\"svc.created_at = {svc.created_at}\")\n", + "print(f\"UserService.version = {UserService.version!r}\")\n", + "print(f\"svc.get_info() = {svc.get_info()!r}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## @dataclass as the Canonical Class Decorator\n", + "\n", + "`@dataclasses.dataclass` is the most widely used class decorator in the standard library.\n", + "It inspects class annotations to automatically generate `__init__`, `__repr__`, `__eq__`,\n", + "and other methods. Understanding how it works demystifies class decorators in general." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from dataclasses import dataclass, field, fields\n", + "\n", + "\n", + "@dataclass(frozen=True, order=True)\n", + "class Coordinate:\n", + " \"\"\"An immutable, orderable 2D coordinate.\"\"\"\n", + " x: float\n", + " y: float\n", + " label: str = field(default=\"\", compare=False, repr=False)\n", + "\n", + "\n", + "p1 = Coordinate(1.0, 2.0, \"origin-ish\")\n", + "p2 = Coordinate(3.0, 4.0)\n", + "p3 = Coordinate(1.0, 2.0)\n", + "\n", + "print(f\"p1 = {p1!r}\")\n", + "print(f\"p2 = {p2!r}\")\n", + "print(f\"p1 == p3: {p1 == p3}\") # True (label excluded from compare)\n", + "print(f\"p1 < p2: {p1 < p2}\") # True (compared as tuples of (x, y))\n", + "\n", + "# Inspect what @dataclass generated\n", + "print(f\"\\nFields: {[f.name for f in fields(Coordinate)]}\")\n", + "print(f\"Generated methods: {[m for m in dir(Coordinate) if not m.startswith('_') or m in ('__init__', '__repr__', '__eq__', '__lt__')]}\")\n", + "\n", + "# Frozen means immutable\n", + "try:\n", + " p1.x = 99\n", + "except AttributeError as e:\n", + " print(f\"\\nCannot mutate frozen: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `__init_subclass__`: Hook Called When a Class Is Subclassed\n", + "\n", + "Introduced in Python 3.6, `__init_subclass__` is a class method that is automatically\n", + "called on the **parent** class whenever it is subclassed. This provides a simple hook\n", + "for customizing subclass creation without needing a metaclass.\n", + "\n", + "Key points:\n", + "- Defined on the parent class, called when a child is created\n", + "- Receives the new subclass as its first argument (after `cls`)\n", + "- Can accept keyword arguments from the class statement" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class Validated:\n", + " \"\"\"Base class that enforces subclasses must define required_fields.\"\"\"\n", + "\n", + " def __init_subclass__(cls, *, required_fields: tuple[str, ...] = (), **kwargs: object) -> None:\n", + " super().__init_subclass__(**kwargs)\n", + " cls._required_fields = required_fields\n", + " print(f\" Registered {cls.__name__} with required fields: {required_fields}\")\n", + "\n", + " # Inject a validate() method\n", + " def validate(self: object) -> bool:\n", + " missing = [\n", + " f for f in cls._required_fields\n", + " if not hasattr(self, f) or getattr(self, f) is None\n", + " ]\n", + " if missing:\n", + " raise ValueError(f\"Missing required fields: {missing}\")\n", + " return True\n", + "\n", + " cls.validate = validate # type: ignore[attr-defined]\n", + "\n", + "\n", + "print(\"Defining subclasses:\")\n", + "\n", + "\n", + "class User(Validated, required_fields=(\"name\", \"email\")):\n", + " def __init__(self, name: str | None = None, email: str | None = None) -> None:\n", + " self.name = name\n", + " self.email = email\n", + "\n", + "\n", + "class Product(Validated, required_fields=(\"sku\", \"price\")):\n", + " def __init__(self, sku: str | None = None, price: float | None = None) -> None:\n", + " self.sku = sku\n", + " self.price = price\n", + "\n", + "\n", + "# Valid instance\n", + "user = User(\"Alice\", \"alice@example.com\")\n", + "print(f\"\\nuser.validate() = {user.validate()}\")\n", + "\n", + "# Invalid instance -- missing required field\n", + "incomplete = User(\"Bob\", None)\n", + "try:\n", + " incomplete.validate()\n", + "except ValueError as e:\n", + " print(f\"Validation error: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plugin Registration with `__init_subclass__`\n", + "\n", + "One of the most practical uses of `__init_subclass__` is automatic plugin registration.\n", + "Subclasses register themselves simply by existing -- no manual registration step needed." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import ClassVar\n", + "\n", + "\n", + "class Serializer:\n", + " \"\"\"Base serializer that auto-registers subclasses by format name.\"\"\"\n", + " _registry: ClassVar[dict[str, type[\"Serializer\"]]] = {}\n", + "\n", + " def __init_subclass__(cls, *, format_name: str = \"\", **kwargs: object) -> None:\n", + " super().__init_subclass__(**kwargs)\n", + " if format_name:\n", + " cls._registry[format_name] = cls\n", + " cls.format_name = format_name # type: ignore[attr-defined]\n", + "\n", + " @classmethod\n", + " def get_serializer(cls, format_name: str) -> \"Serializer\":\n", + " \"\"\"Look up a serializer by format name.\"\"\"\n", + " if format_name not in cls._registry:\n", + " raise ValueError(f\"Unknown format: {format_name!r}\")\n", + " return cls._registry[format_name]()\n", + "\n", + " def serialize(self, data: dict) -> str:\n", + " raise NotImplementedError\n", + "\n", + "\n", + "class JSONSerializer(Serializer, format_name=\"json\"):\n", + " def serialize(self, data: dict) -> str:\n", + " import json\n", + " return json.dumps(data)\n", + "\n", + "\n", + "class CSVSerializer(Serializer, format_name=\"csv\"):\n", + " def serialize(self, data: dict) -> str:\n", + " header = \",\".join(data.keys())\n", + " values = \",\".join(str(v) for v in data.values())\n", + " return f\"{header}\\n{values}\"\n", + "\n", + "\n", + "class XMLSerializer(Serializer, format_name=\"xml\"):\n", + " def serialize(self, data: dict) -> str:\n", + " items = \"\".join(f\"<{k}>{v}\" for k, v in data.items())\n", + " return f\"{items}\"\n", + "\n", + "\n", + "# All serializers registered automatically\n", + "print(f\"Registered formats: {list(Serializer._registry.keys())}\")\n", + "\n", + "# Use the registry to dispatch\n", + "sample_data = {\"name\": \"Alice\", \"age\": 30, \"city\": \"Paris\"}\n", + "\n", + "for fmt in [\"json\", \"csv\", \"xml\"]:\n", + " s = Serializer.get_serializer(fmt)\n", + " print(f\"\\n[{fmt}]:\\n{s.serialize(sample_data)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Comparing: Class Decorator vs Metaclass vs `__init_subclass__`\n", + "\n", + "Python offers three main mechanisms for customizing class creation. Here is when\n", + "to use each one:\n", + "\n", + "| Mechanism | Applied To | When It Runs | Complexity |\n", + "|-----------|-----------|-------------|------------|\n", + "| Class decorator | The decorated class only | After class body executes | Low |\n", + "| `__init_subclass__` | All subclasses of a base | When each subclass is created | Medium |\n", + "| Metaclass | All instances of the metaclass | During class creation | High |\n", + "\n", + "**Rule of thumb**: Start with class decorators. Move to `__init_subclass__` when you\n", + "need automatic behavior in all subclasses. Only reach for metaclasses when you need\n", + "to control the class creation process itself (e.g., modifying `__new__` or the class\n", + "namespace)." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "# Approach 1: Class decorator -- explicit, opt-in\n", + "def add_greeting(cls: type) -> type:\n", + " cls.greet = lambda self: f\"Hello from {type(self).__name__}!\" # type: ignore[attr-defined]\n", + " return cls\n", + "\n", + "\n", + "@add_greeting\n", + "class ServiceA:\n", + " pass\n", + "\n", + "\n", + "# Approach 2: __init_subclass__ -- automatic for all subclasses\n", + "class GreetingBase:\n", + " def __init_subclass__(cls, **kwargs: Any) -> None:\n", + " super().__init_subclass__(**kwargs)\n", + " cls.greet = lambda self: f\"Hello from {type(self).__name__}!\" # type: ignore[attr-defined]\n", + "\n", + "\n", + "class ServiceB(GreetingBase):\n", + " pass\n", + "\n", + "\n", + "class ServiceC(GreetingBase):\n", + " pass\n", + "\n", + "\n", + "# Approach 3: Metaclass -- most powerful, most complex\n", + "class GreetingMeta(type):\n", + " def __new__(mcs, name: str, bases: tuple, namespace: dict[str, Any]) -> type:\n", + " cls = super().__new__(mcs, name, bases, namespace)\n", + " cls.greet = lambda self: f\"Hello from {type(self).__name__}!\" # type: ignore[attr-defined]\n", + " return cls\n", + "\n", + "\n", + "class ServiceD(metaclass=GreetingMeta):\n", + " pass\n", + "\n", + "\n", + "# All three approaches achieve the same result\n", + "for cls in [ServiceA, ServiceB, ServiceC, ServiceD]:\n", + " instance = cls()\n", + " print(f\"{cls.__name__}: {instance.greet()}\")\n", + "\n", + "print(f\"\\nServiceB uses: __init_subclass__ (auto for all subclasses)\")\n", + "print(f\"ServiceA uses: class decorator (explicit opt-in)\")\n", + "print(f\"ServiceD uses: metaclass (full control, but rarely needed)\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Decorator Stacking Order and Composition\n", + "\n", + "When multiple class decorators are stacked, they are applied **bottom-up** (innermost\n", + "first). This is the same rule as function decorators. Understanding the order matters\n", + "when decorators depend on or modify each other's work." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import TypeVar\n", + "\n", + "T = TypeVar(\"T\")\n", + "\n", + "\n", + "def decorator_a(cls: type[T]) -> type[T]:\n", + " \"\"\"Adds method_a and records application order.\"\"\"\n", + " order = getattr(cls, \"_decorator_order\", [])\n", + " cls._decorator_order = order + [\"A\"] # type: ignore[attr-defined]\n", + " cls.method_a = lambda self: \"from A\" # type: ignore[attr-defined]\n", + " print(f\" Applied decorator A to {cls.__name__}\")\n", + " return cls\n", + "\n", + "\n", + "def decorator_b(cls: type[T]) -> type[T]:\n", + " \"\"\"Adds method_b and records application order.\"\"\"\n", + " order = getattr(cls, \"_decorator_order\", [])\n", + " cls._decorator_order = order + [\"B\"] # type: ignore[attr-defined]\n", + " cls.method_b = lambda self: \"from B\" # type: ignore[attr-defined]\n", + " print(f\" Applied decorator B to {cls.__name__}\")\n", + " return cls\n", + "\n", + "\n", + "def decorator_c(cls: type[T]) -> type[T]:\n", + " \"\"\"Adds method_c and records application order.\"\"\"\n", + " order = getattr(cls, \"_decorator_order\", [])\n", + " cls._decorator_order = order + [\"C\"] # type: ignore[attr-defined]\n", + " cls.method_c = lambda self: \"from C\" # type: ignore[attr-defined]\n", + " print(f\" Applied decorator C to {cls.__name__}\")\n", + " return cls\n", + "\n", + "\n", + "# Stacking: bottom-up application order\n", + "# Equivalent to: MyClass = decorator_a(decorator_b(decorator_c(MyClass)))\n", + "print(\"Applying decorators (bottom-up):\")\n", + "\n", + "\n", + "@decorator_a # Applied third (outermost)\n", + "@decorator_b # Applied second\n", + "@decorator_c # Applied first (innermost)\n", + "class MyClass:\n", + " pass\n", + "\n", + "\n", + "obj = MyClass()\n", + "print(f\"\\nApplication order: {MyClass._decorator_order}\")\n", + "print(f\"obj.method_a() = {obj.method_a()!r}\")\n", + "print(f\"obj.method_b() = {obj.method_b()!r}\")\n", + "print(f\"obj.method_c() = {obj.method_c()!r}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: Auto-Registering Command Handlers\n", + "\n", + "This practical example builds a command dispatcher that uses `__init_subclass__`\n", + "to automatically register handler classes, and a class decorator to tag individual\n", + "methods as command handlers. This is a pattern found in CLI frameworks, chatbots,\n", + "and game engines." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any, Callable, ClassVar\n", + "import functools\n", + "\n", + "\n", + "# Step 1: A method decorator to tag individual methods as commands\n", + "def command(name: str | None = None, description: str = \"\"):\n", + " \"\"\"Mark a method as a command handler.\"\"\"\n", + " def decorator(func: Callable[..., Any]) -> Callable[..., Any]:\n", + " func._command_name = name or func.__name__ # type: ignore[attr-defined]\n", + " func._command_desc = description # type: ignore[attr-defined]\n", + " return func\n", + " return decorator\n", + "\n", + "\n", + "# Step 2: Base class with __init_subclass__ for auto-registration\n", + "class CommandHandler:\n", + " \"\"\"Base class that auto-discovers @command methods in subclasses.\"\"\"\n", + " _handlers: ClassVar[dict[str, tuple[type, Callable[..., Any], str]]] = {}\n", + "\n", + " def __init_subclass__(cls, **kwargs: Any) -> None:\n", + " super().__init_subclass__(**kwargs)\n", + " # Scan the subclass for methods decorated with @command\n", + " for attr_name in dir(cls):\n", + " method = getattr(cls, attr_name, None)\n", + " if callable(method) and hasattr(method, \"_command_name\"):\n", + " cmd_name = method._command_name\n", + " cmd_desc = method._command_desc\n", + " cls._handlers[cmd_name] = (cls, method, cmd_desc)\n", + "\n", + " @classmethod\n", + " def dispatch(cls, command_str: str) -> str:\n", + " \"\"\"Parse and dispatch a command string.\"\"\"\n", + " parts = command_str.strip().split()\n", + " cmd_name = parts[0]\n", + " args = parts[1:]\n", + "\n", + " if cmd_name not in cls._handlers:\n", + " return f\"Unknown command: {cmd_name!r}\"\n", + "\n", + " handler_cls, method, _ = cls._handlers[cmd_name]\n", + " instance = handler_cls()\n", + " return method(instance, *args)\n", + "\n", + " @classmethod\n", + " def help_text(cls) -> str:\n", + " \"\"\"Generate help text from all registered commands.\"\"\"\n", + " lines = [\"Available commands:\"]\n", + " for name, (handler_cls, _, desc) in sorted(cls._handlers.items()):\n", + " lines.append(f\" {name:15s} - {desc} [{handler_cls.__name__}]\")\n", + " return \"\\n\".join(lines)\n", + "\n", + "\n", + "print(\"Defining command handler classes...\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Step 3: Define handler subclasses -- they register automatically\n", + "\n", + "class FileCommands(CommandHandler):\n", + " \"\"\"File-related commands.\"\"\"\n", + "\n", + " @command(name=\"ls\", description=\"List files in a directory\")\n", + " def list_files(self, path: str = \".\") -> str:\n", + " return f\"Listing files in {path!r}: file1.txt, file2.py, data.csv\"\n", + "\n", + " @command(name=\"cat\", description=\"Display file contents\")\n", + " def show_file(self, filename: str) -> str:\n", + " return f\"Contents of {filename!r}: [simulated file content]\"\n", + "\n", + "\n", + "class SystemCommands(CommandHandler):\n", + " \"\"\"System-related commands.\"\"\"\n", + "\n", + " @command(name=\"status\", description=\"Show system status\")\n", + " def system_status(self) -> str:\n", + " return \"System OK | CPU: 45% | Memory: 62%\"\n", + "\n", + " @command(name=\"uptime\", description=\"Show system uptime\")\n", + " def system_uptime(self) -> str:\n", + " return \"Uptime: 14 days, 3 hours, 22 minutes\"\n", + "\n", + "\n", + "class UserCommands(CommandHandler):\n", + " \"\"\"User management commands.\"\"\"\n", + "\n", + " @command(name=\"whoami\", description=\"Show current user\")\n", + " def who_am_i(self) -> str:\n", + " return \"Current user: admin (role: superuser)\"\n", + "\n", + "\n", + "# Show the auto-generated help text\n", + "print(CommandHandler.help_text())\n", + "\n", + "# Dispatch some commands\n", + "print()\n", + "for cmd in [\"ls /home\", \"cat readme.md\", \"status\", \"whoami\", \"unknown_cmd\"]:\n", + " result = CommandHandler.dispatch(cmd)\n", + " print(f\" > {cmd}\")\n", + " print(f\" {result}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Mechanism | Best For | Example |\n", + "|-----------|----------|--------|\n", + "| **Class decorator** | One-off modifications, adding methods/attrs | `@add_repr`, `@dataclass` |\n", + "| **Parameterized decorator** | Configurable class modifications | `@with_version(\"2.0\")` |\n", + "| **`__init_subclass__`** | Auto-registration, subclass validation | Plugin systems, serializers |\n", + "| **Metaclass** | Deep class creation control | ORMs, API frameworks |\n", + "| **`@command` + `__init_subclass__`** | Command/plugin dispatch | CLI tools, chatbots |\n", + "\n", + "### Best Practices\n", + "- Start with class decorators for simple, explicit modifications\n", + "- Use `__init_subclass__` when all subclasses should automatically participate\n", + "- Remember decorator stacking order: bottom-up (innermost applied first)\n", + "- Reserve metaclasses for cases where you truly need to intercept class creation\n", + "- Combine method decorators with `__init_subclass__` for powerful plugin patterns\n", + "- Always call `super().__init_subclass__(**kwargs)` to support cooperative inheritance" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_24/03_introspection_and_codegen.ipynb b/src/chapter_24/03_introspection_and_codegen.ipynb new file mode 100644 index 0000000..7409f1a --- /dev/null +++ b/src/chapter_24/03_introspection_and_codegen.ipynb @@ -0,0 +1,888 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 24: Introspection and Code Generation\n", + "\n", + "Python's rich introspection capabilities let you examine objects, functions, and classes\n", + "at runtime. Combined with dynamic code generation tools like `type()`, `exec()`, and\n", + "the `ast` module, you can build powerful abstractions such as ORMs, validators, and\n", + "code analysis tools.\n", + "\n", + "## Topics Covered\n", + "- **`inspect` module**: `signature()`, `getsource()`, `getmembers()`\n", + "- **`inspect` predicates**: `isclass()`, `isfunction()`, `ismethod()`\n", + "- **`inspect.Parameter`** and **`Signature`** objects\n", + "- **`type()`** as a class factory (3-argument form)\n", + "- **`exec()` and `eval()`**: Dynamic code execution and dangers\n", + "- **`ast` module**: Parsing and analyzing Python source\n", + "- **Code generation patterns**: Creating classes and functions at runtime\n", + "- **Practical**: Building a simple ORM-style field validator with introspection" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The `inspect` Module: Examining Objects at Runtime\n", + "\n", + "The `inspect` module provides tools to examine live objects: retrieve source code,\n", + "inspect function signatures, list class members, and determine what kind of object\n", + "you are looking at." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import inspect\n", + "from typing import Any\n", + "\n", + "\n", + "class SampleClass:\n", + " \"\"\"A sample class for introspection demos.\"\"\"\n", + " class_var: str = \"shared\"\n", + "\n", + " def __init__(self, x: int, y: int = 10) -> None:\n", + " self.x = x\n", + " self.y = y\n", + "\n", + " def add(self) -> int:\n", + " \"\"\"Return the sum of x and y.\"\"\"\n", + " return self.x + self.y\n", + "\n", + " @classmethod\n", + " def from_tuple(cls, pair: tuple[int, int]) -> \"SampleClass\":\n", + " return cls(pair[0], pair[1])\n", + "\n", + " @staticmethod\n", + " def description() -> str:\n", + " return \"A sample class\"\n", + "\n", + " @property\n", + " def magnitude(self) -> float:\n", + " return (self.x**2 + self.y**2) ** 0.5\n", + "\n", + "\n", + "# getsource: retrieve the actual source code\n", + "print(\"Source of SampleClass.add:\")\n", + "print(inspect.getsource(SampleClass.add))\n", + "\n", + "# getmembers: list all members with optional predicate filtering\n", + "print(\"Methods of SampleClass:\")\n", + "methods = inspect.getmembers(SampleClass, predicate=inspect.isfunction)\n", + "for name, obj in methods:\n", + " print(f\" {name}: {obj}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inspect Predicates: Classifying Objects\n", + "\n", + "The `inspect` module provides predicates to determine what kind of object you have.\n", + "These are essential when writing generic tools that need to handle different object\n", + "types differently." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import inspect\n", + "import os\n", + "\n", + "\n", + "def sample_function(x: int) -> int:\n", + " return x * 2\n", + "\n", + "\n", + "sample_lambda = lambda x: x * 2\n", + "\n", + "\n", + "class Example:\n", + " def method(self) -> None:\n", + " pass\n", + "\n", + " @classmethod\n", + " def cls_method(cls) -> None:\n", + " pass\n", + "\n", + " @staticmethod\n", + " def static_method() -> None:\n", + " pass\n", + "\n", + "\n", + "instance = Example()\n", + "\n", + "# Test various predicates\n", + "objects_to_test: dict[str, object] = {\n", + " \"sample_function\": sample_function,\n", + " \"sample_lambda\": sample_lambda,\n", + " \"Example (class)\": Example,\n", + " \"instance.method\": instance.method,\n", + " \"os (module)\": os,\n", + " \"42 (int)\": 42,\n", + "}\n", + "\n", + "predicates = [\n", + " (\"isfunction\", inspect.isfunction),\n", + " (\"ismethod\", inspect.ismethod),\n", + " (\"isclass\", inspect.isclass),\n", + " (\"ismodule\", inspect.ismodule),\n", + " (\"isbuiltin\", inspect.isbuiltin),\n", + "]\n", + "\n", + "# Print a truth table\n", + "header = f\"{'Object':<22s}\" + \"\".join(f\"{name:<14s}\" for name, _ in predicates)\n", + "print(header)\n", + "print(\"-\" * len(header))\n", + "\n", + "for label, obj in objects_to_test.items():\n", + " row = f\"{label:<22s}\"\n", + " for _, pred in predicates:\n", + " row += f\"{str(pred(obj)):<14s}\"\n", + " print(row)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `inspect.Parameter` and `Signature` Objects\n", + "\n", + "`inspect.signature()` returns a `Signature` object that provides structured access to\n", + "a callable's parameters: their names, kinds (positional, keyword, etc.), defaults,\n", + "and annotations. This is the foundation for building tools that work with arbitrary\n", + "function signatures." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import inspect\n", + "\n", + "\n", + "def complex_function(\n", + " pos_only: int,\n", + " /,\n", + " normal: str,\n", + " default_val: float = 3.14,\n", + " *args: int,\n", + " keyword_only: bool = False,\n", + " **kwargs: str,\n", + ") -> dict:\n", + " \"\"\"A function with every kind of parameter.\"\"\"\n", + " return {}\n", + "\n", + "\n", + "sig = inspect.signature(complex_function)\n", + "print(f\"Signature: {sig}\")\n", + "print(f\"Return annotation: {sig.return_annotation}\")\n", + "\n", + "# Map parameter kind enum values to readable names\n", + "kind_names: dict[int, str] = {\n", + " inspect.Parameter.POSITIONAL_ONLY: \"POSITIONAL_ONLY\",\n", + " inspect.Parameter.POSITIONAL_OR_KEYWORD: \"POSITIONAL_OR_KEYWORD\",\n", + " inspect.Parameter.VAR_POSITIONAL: \"VAR_POSITIONAL (*args)\",\n", + " inspect.Parameter.KEYWORD_ONLY: \"KEYWORD_ONLY\",\n", + " inspect.Parameter.VAR_KEYWORD: \"VAR_KEYWORD (**kwargs)\",\n", + "}\n", + "\n", + "print(f\"\\n{'Name':<16s} {'Kind':<28s} {'Default':<16s} {'Annotation'}\")\n", + "print(\"-\" * 76)\n", + "for name, param in sig.parameters.items():\n", + " default = (\n", + " repr(param.default)\n", + " if param.default is not inspect.Parameter.empty\n", + " else \"(none)\"\n", + " )\n", + " annotation = (\n", + " param.annotation.__name__\n", + " if isinstance(param.annotation, type)\n", + " else str(param.annotation)\n", + " if param.annotation is not inspect.Parameter.empty\n", + " else \"(none)\"\n", + " )\n", + " kind = kind_names.get(param.kind, str(param.kind))\n", + " print(f\"{name:<16s} {kind:<28s} {default:<16s} {annotation}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import inspect\n", + "\n", + "\n", + "# Practical use: build a signature-aware wrapper\n", + "def describe_callable(func: object) -> str:\n", + " \"\"\"Generate a human-readable description of a callable's signature.\"\"\"\n", + " sig = inspect.signature(func) # type: ignore[arg-type]\n", + " parts: list[str] = []\n", + "\n", + " required = []\n", + " optional = []\n", + "\n", + " for name, param in sig.parameters.items():\n", + " if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):\n", + " continue\n", + " if param.default is inspect.Parameter.empty:\n", + " required.append(name)\n", + " else:\n", + " optional.append(f\"{name}={param.default!r}\")\n", + "\n", + " parts.append(f\"Required: {', '.join(required) or '(none)'}\")\n", + " parts.append(f\"Optional: {', '.join(optional) or '(none)'}\")\n", + " parts.append(f\"Accepts *args: {any(p.kind == p.VAR_POSITIONAL for p in sig.parameters.values())}\")\n", + " parts.append(f\"Accepts **kwargs: {any(p.kind == p.VAR_KEYWORD for p in sig.parameters.values())}\")\n", + "\n", + " return \"\\n\".join(parts)\n", + "\n", + "\n", + "def example_func(host: str, port: int, *, ssl: bool = True, **options: str) -> None:\n", + " pass\n", + "\n", + "\n", + "print(f\"Describing: {inspect.signature(example_func)}\")\n", + "print(describe_callable(example_func))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `type()` as a Class Factory (3-Argument Form)\n", + "\n", + "Beyond checking an object's type, `type()` has a 3-argument form that **creates new\n", + "classes** at runtime: `type(name, bases, namespace)`. This is exactly what Python does\n", + "behind the scenes when it encounters a `class` statement." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "# Creating a class with the class statement\n", + "class DogStatic:\n", + " species: str = \"Canis familiaris\"\n", + "\n", + " def __init__(self, name: str, age: int) -> None:\n", + " self.name = name\n", + " self.age = age\n", + "\n", + " def speak(self) -> str:\n", + " return f\"{self.name} says Woof!\"\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Dog({self.name!r}, age={self.age})\"\n", + "\n", + "\n", + "# Creating the EXACT same class dynamically with type()\n", + "def dog_init(self: Any, name: str, age: int) -> None:\n", + " self.name = name\n", + " self.age = age\n", + "\n", + "\n", + "def dog_speak(self: Any) -> str:\n", + " return f\"{self.name} says Woof!\"\n", + "\n", + "\n", + "def dog_repr(self: Any) -> str:\n", + " return f\"Dog({self.name!r}, age={self.age})\"\n", + "\n", + "\n", + "DogDynamic = type(\n", + " \"DogDynamic\", # class name\n", + " (), # base classes (tuple)\n", + " { # namespace (class body as a dict)\n", + " \"species\": \"Canis familiaris\",\n", + " \"__init__\": dog_init,\n", + " \"speak\": dog_speak,\n", + " \"__repr__\": dog_repr,\n", + " },\n", + ")\n", + "\n", + "# Both classes work identically\n", + "for cls in [DogStatic, DogDynamic]:\n", + " d = cls(\"Rex\", 5)\n", + " print(f\"{cls.__name__}: {d!r}, {d.speak()}, species={d.species!r}\")\n", + "\n", + "print(f\"\\ntype(DogDynamic) = {type(DogDynamic)}\")\n", + "print(f\"isinstance check: {isinstance(DogDynamic('Spot', 3), DogDynamic)}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "# Practical use: a class factory function\n", + "def make_struct(name: str, fields: list[str]) -> type:\n", + " \"\"\"Create a simple struct-like class with named fields.\"\"\"\n", + "\n", + " def __init__(self: Any, **kwargs: Any) -> None:\n", + " for field in fields:\n", + " if field not in kwargs:\n", + " raise TypeError(f\"Missing required field: {field!r}\")\n", + " setattr(self, field, kwargs[field])\n", + "\n", + " def __repr__(self: Any) -> str:\n", + " attrs = \", \".join(f\"{f}={getattr(self, f)!r}\" for f in fields)\n", + " return f\"{name}({attrs})\"\n", + "\n", + " def __eq__(self: Any, other: Any) -> bool:\n", + " if type(self) is not type(other):\n", + " return NotImplemented\n", + " return all(getattr(self, f) == getattr(other, f) for f in fields)\n", + "\n", + " return type(\n", + " name,\n", + " (),\n", + " {\n", + " \"_fields\": tuple(fields),\n", + " \"__init__\": __init__,\n", + " \"__repr__\": __repr__,\n", + " \"__eq__\": __eq__,\n", + " },\n", + " )\n", + "\n", + "\n", + "# Create struct classes dynamically\n", + "Point = make_struct(\"Point\", [\"x\", \"y\"])\n", + "Color = make_struct(\"Color\", [\"r\", \"g\", \"b\"])\n", + "\n", + "p = Point(x=3, y=4)\n", + "c = Color(r=255, g=128, b=0)\n", + "\n", + "print(f\"{p!r}\")\n", + "print(f\"{c!r}\")\n", + "print(f\"Point._fields = {Point._fields}\")\n", + "print(f\"Point(x=3, y=4) == Point(x=3, y=4): {p == Point(x=3, y=4)}\")\n", + "\n", + "# Missing field raises TypeError\n", + "try:\n", + " Point(x=1)\n", + "except TypeError as e:\n", + " print(f\"\\nTypeError: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `exec()` and `eval()`: Dynamic Code Execution\n", + "\n", + "`eval()` evaluates a single expression and returns its value. `exec()` executes\n", + "arbitrary statements. Both are powerful but dangerous -- never use them with\n", + "untrusted input.\n", + "\n", + "**When to use them**: Code generation in frameworks (like `@dataclass` does internally),\n", + "plugin systems with trusted code, and REPL-like tools.\n", + "\n", + "**When NOT to use them**: Anything involving user-supplied strings (use `ast.literal_eval()`\n", + "for safe evaluation of literals)." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import ast\n", + "from typing import Any\n", + "\n", + "# eval() evaluates an expression and returns the result\n", + "result = eval(\"2 ** 10 + len('hello')\")\n", + "print(f\"eval('2 ** 10 + len(\\'hello\\')') = {result}\")\n", + "\n", + "# eval() with a controlled namespace\n", + "safe_globals: dict[str, Any] = {\"__builtins__\": {}} # Strip all builtins\n", + "safe_locals: dict[str, Any] = {\"x\": 10, \"y\": 20}\n", + "result = eval(\"x + y\", safe_globals, safe_locals)\n", + "print(f\"eval('x + y', restricted) = {result}\")\n", + "\n", + "# exec() executes statements (no return value)\n", + "namespace: dict[str, Any] = {}\n", + "exec(\n", + " \"\"\"\n", + "def greet(name: str) -> str:\n", + " return f\"Hello, {name}!\"\n", + "\n", + "result = greet(\"World\")\n", + "\"\"\",\n", + " namespace,\n", + ")\n", + "print(f\"\\nexec created function: {namespace['greet']}\")\n", + "print(f\"exec result: {namespace['result']!r}\")\n", + "\n", + "# SAFE alternative for parsing literals: ast.literal_eval\n", + "# Only allows: strings, bytes, numbers, tuples, lists, dicts, sets, bools, None\n", + "safe_data = ast.literal_eval(\"{'name': 'Alice', 'scores': [95, 87, 92]}\")\n", + "print(f\"\\nast.literal_eval: {safe_data}\")\n", + "\n", + "# ast.literal_eval rejects anything dangerous\n", + "try:\n", + " ast.literal_eval(\"__import__('os').system('echo hacked')\")\n", + "except (ValueError, SyntaxError) as e:\n", + " print(f\"ast.literal_eval blocked: {type(e).__name__}: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The `ast` Module: Parsing and Analyzing Python Source\n", + "\n", + "The `ast` module parses Python source code into an Abstract Syntax Tree -- a tree of\n", + "node objects representing the structure of the code. This enables static analysis,\n", + "code transformation, and safe code inspection without executing anything." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import ast\n", + "\n", + "# Parse source code into an AST\n", + "source_code = \"\"\"\n", + "def calculate(x: int, y: int) -> int:\n", + " result = x * 2 + y\n", + " return result\n", + "\n", + "class Config:\n", + " debug: bool = True\n", + " max_retries: int = 3\n", + "\n", + " def validate(self) -> bool:\n", + " return self.max_retries > 0\n", + "\"\"\"\n", + "\n", + "tree = ast.parse(source_code)\n", + "\n", + "# Walk the tree to find all function and class definitions\n", + "print(\"Top-level AST nodes:\")\n", + "for node in ast.iter_child_nodes(tree):\n", + " print(f\" {type(node).__name__}: \", end=\"\")\n", + " if isinstance(node, ast.FunctionDef):\n", + " args = [arg.arg for arg in node.args.args]\n", + " print(f\"function '{node.name}' with args {args}\")\n", + " elif isinstance(node, ast.ClassDef):\n", + " print(f\"class '{node.name}'\")\n", + " for item in node.body:\n", + " if isinstance(item, ast.FunctionDef):\n", + " print(f\" method: '{item.name}'\")\n", + " elif isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):\n", + " print(f\" field: '{item.target.id}'\")\n", + "\n", + "# ast.dump shows the full tree structure\n", + "simple_tree = ast.parse(\"x = 1 + 2\")\n", + "print(f\"\\nAST dump of 'x = 1 + 2':\")\n", + "print(ast.dump(simple_tree, indent=2))" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import ast\n", + "\n", + "\n", + "class FunctionAnalyzer(ast.NodeVisitor):\n", + " \"\"\"Walks an AST and collects information about function definitions.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.functions: list[dict[str, object]] = []\n", + "\n", + " def visit_FunctionDef(self, node: ast.FunctionDef) -> None:\n", + " info: dict[str, object] = {\n", + " \"name\": node.name,\n", + " \"args\": [arg.arg for arg in node.args.args],\n", + " \"decorators\": [\n", + " ast.dump(d) for d in node.decorator_list\n", + " ],\n", + " \"line\": node.lineno,\n", + " \"has_return\": any(\n", + " isinstance(n, ast.Return) and n.value is not None\n", + " for n in ast.walk(node)\n", + " ),\n", + " \"num_statements\": sum(\n", + " 1 for n in ast.walk(node)\n", + " if isinstance(n, ast.stmt) and n is not node\n", + " ),\n", + " }\n", + " self.functions.append(info)\n", + " self.generic_visit(node) # Continue visiting child nodes\n", + "\n", + "\n", + "# Analyze a code sample\n", + "code = \"\"\"\n", + "def add(a, b):\n", + " return a + b\n", + "\n", + "def log(message):\n", + " timestamp = get_time()\n", + " formatted = f\"[{timestamp}] {message}\"\n", + " print(formatted)\n", + "\n", + "def noop():\n", + " pass\n", + "\"\"\"\n", + "\n", + "analyzer = FunctionAnalyzer()\n", + "analyzer.visit(ast.parse(code))\n", + "\n", + "print(f\"Found {len(analyzer.functions)} functions:\\n\")\n", + "for func in analyzer.functions:\n", + " print(f\" {func['name']}({', '.join(func['args'])})\")\n", + " print(f\" Line: {func['line']}, Statements: {func['num_statements']}, Returns value: {func['has_return']}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Code Generation Patterns: Creating Functions at Runtime\n", + "\n", + "Python frameworks like `dataclasses` and `attrs` use `exec()` internally to generate\n", + "methods with correct signatures. Here is how this pattern works and why it is\n", + "sometimes preferred over closures." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from typing import Any\n", + "\n", + "\n", + "def make_init(fields: list[tuple[str, type, Any]]) -> Any:\n", + " \"\"\"Generate an __init__ method from a list of (name, type, default) triples.\n", + "\n", + " This is similar to what @dataclass does internally.\n", + " \"\"\"\n", + " # Build the argument list\n", + " args_parts: list[str] = [\"self\"]\n", + " body_parts: list[str] = []\n", + " local_ns: dict[str, Any] = {}\n", + "\n", + " for name, field_type, default in fields:\n", + " if default is not ...: # Ellipsis means no default\n", + " default_name = f\"_default_{name}\"\n", + " local_ns[default_name] = default\n", + " args_parts.append(f\"{name} = {default_name}\")\n", + " else:\n", + " args_parts.append(name)\n", + "\n", + " body_parts.append(f\" self.{name} = {name}\")\n", + "\n", + " args_str = \", \".join(args_parts)\n", + " body_str = \"\\n\".join(body_parts) if body_parts else \" pass\"\n", + "\n", + " func_code = f\"def __init__({args_str}):\\n{body_str}\"\n", + " print(f\"Generated code:\\n{func_code}\\n\")\n", + "\n", + " exec(func_code, local_ns)\n", + " return local_ns[\"__init__\"]\n", + "\n", + "\n", + "# Generate an __init__ for a User class\n", + "init_method = make_init([\n", + " (\"name\", str, ...), # Required (no default)\n", + " (\"email\", str, ...), # Required\n", + " (\"active\", bool, True), # Optional with default\n", + " (\"role\", str, \"viewer\"), # Optional with default\n", + "])\n", + "\n", + "# Attach the generated __init__ to a class\n", + "class User:\n", + " __init__ = init_method\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"User(name={self.name!r}, email={self.email!r}, active={self.active}, role={self.role!r})\"\n", + "\n", + "\n", + "u1 = User(\"Alice\", \"alice@example.com\")\n", + "u2 = User(\"Bob\", \"bob@example.com\", active=False, role=\"admin\")\n", + "\n", + "print(f\"{u1!r}\")\n", + "print(f\"{u2!r}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Practical: ORM-Style Field Validator with Introspection\n", + "\n", + "This practical example combines introspection, `__init_subclass__`, and dynamic\n", + "attribute access to build a mini ORM-style system. Fields are declared as class\n", + "annotations with `Field` descriptors, and the framework automatically generates\n", + "validation logic by inspecting the class." + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import inspect\n", + "from typing import Any, get_type_hints\n", + "\n", + "\n", + "class Field:\n", + " \"\"\"A descriptor that validates type and optional constraints.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " *,\n", + " required: bool = True,\n", + " default: Any = ...,\n", + " min_value: float | None = None,\n", + " max_value: float | None = None,\n", + " min_length: int | None = None,\n", + " max_length: int | None = None,\n", + " ) -> None:\n", + " self.required = required\n", + " self.default = default\n", + " self.min_value = min_value\n", + " self.max_value = max_value\n", + " self.min_length = min_length\n", + " self.max_length = max_length\n", + " # These are set by __set_name__\n", + " self.name: str = \"\"\n", + " self.storage_name: str = \"\"\n", + " self.expected_type: type = object\n", + "\n", + " def __set_name__(self, owner: type, name: str) -> None:\n", + " self.name = name\n", + " self.storage_name = f\"_field_{name}\"\n", + "\n", + " def __get__(self, obj: Any, objtype: type | None = None) -> Any:\n", + " if obj is None:\n", + " return self\n", + " return getattr(obj, self.storage_name, self.default)\n", + "\n", + " def __set__(self, obj: Any, value: Any) -> None:\n", + " self.validate(value)\n", + " setattr(obj, self.storage_name, value)\n", + "\n", + " def validate(self, value: Any) -> None:\n", + " \"\"\"Validate the value against all constraints.\"\"\"\n", + " if value is ... or value is None:\n", + " if self.required:\n", + " raise ValueError(f\"Field {self.name!r} is required\")\n", + " return\n", + "\n", + " if self.expected_type is not object and not isinstance(value, self.expected_type):\n", + " raise TypeError(\n", + " f\"Field {self.name!r}: expected {self.expected_type.__name__}, \"\n", + " f\"got {type(value).__name__}\"\n", + " )\n", + "\n", + " if self.min_value is not None and value < self.min_value:\n", + " raise ValueError(f\"Field {self.name!r}: {value} < min {self.min_value}\")\n", + " if self.max_value is not None and value > self.max_value:\n", + " raise ValueError(f\"Field {self.name!r}: {value} > max {self.max_value}\")\n", + " if self.min_length is not None and len(value) < self.min_length:\n", + " raise ValueError(f\"Field {self.name!r}: length {len(value)} < min {self.min_length}\")\n", + " if self.max_length is not None and len(value) > self.max_length:\n", + " raise ValueError(f\"Field {self.name!r}: length {len(value)} > max {self.max_length}\")\n", + "\n", + "\n", + "class ValidatedModel:\n", + " \"\"\"Base class that auto-configures Field descriptors using introspection.\"\"\"\n", + "\n", + " def __init_subclass__(cls, **kwargs: Any) -> None:\n", + " super().__init_subclass__(**kwargs)\n", + "\n", + " # Use get_type_hints to resolve annotations (including forward refs)\n", + " hints = get_type_hints(cls)\n", + "\n", + " # Wire up each Field descriptor with its annotated type\n", + " for attr_name, attr_value in cls.__dict__.items():\n", + " if isinstance(attr_value, Field) and attr_name in hints:\n", + " attr_value.expected_type = hints[attr_name]\n", + "\n", + " def __init__(self, **kwargs: Any) -> None:\n", + " # Discover all Field descriptors in the class\n", + " for attr_name in dir(type(self)):\n", + " descriptor = getattr(type(self), attr_name, None)\n", + " if isinstance(descriptor, Field):\n", + " if attr_name in kwargs:\n", + " setattr(self, attr_name, kwargs[attr_name])\n", + " elif descriptor.default is not ...:\n", + " setattr(self, attr_name, descriptor.default)\n", + " elif descriptor.required:\n", + " raise ValueError(f\"Missing required field: {attr_name!r}\")\n", + "\n", + " def __repr__(self) -> str:\n", + " fields = []\n", + " for attr_name in dir(type(self)):\n", + " descriptor = getattr(type(self), attr_name, None)\n", + " if isinstance(descriptor, Field):\n", + " value = getattr(self, attr_name)\n", + " fields.append(f\"{attr_name}={value!r}\")\n", + " return f\"{type(self).__name__}({', '.join(fields)})\"\n", + "\n", + "\n", + "print(\"ValidatedModel base class defined. Now define models...\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Define concrete models with validated fields\n", + "\n", + "class Employee(ValidatedModel):\n", + " name: str = Field(min_length=1, max_length=100)\n", + " email: str = Field(min_length=5, max_length=255)\n", + " age: int = Field(min_value=18, max_value=120)\n", + " department: str = Field(required=False, default=\"unassigned\")\n", + "\n", + "\n", + "class Product(ValidatedModel):\n", + " sku: str = Field(min_length=3, max_length=20)\n", + " price: float = Field(min_value=0.01)\n", + " quantity: int = Field(min_value=0, default=0)\n", + "\n", + "\n", + "# Introspect the fields\n", + "print(\"Employee fields:\")\n", + "for attr_name in sorted(dir(Employee)):\n", + " descriptor = getattr(Employee, attr_name, None)\n", + " if isinstance(descriptor, Field):\n", + " constraints = []\n", + " if descriptor.required:\n", + " constraints.append(\"required\")\n", + " if descriptor.min_value is not None:\n", + " constraints.append(f\"min={descriptor.min_value}\")\n", + " if descriptor.max_value is not None:\n", + " constraints.append(f\"max={descriptor.max_value}\")\n", + " if descriptor.min_length is not None:\n", + " constraints.append(f\"minlen={descriptor.min_length}\")\n", + " if descriptor.max_length is not None:\n", + " constraints.append(f\"maxlen={descriptor.max_length}\")\n", + " print(f\" {attr_name}: {descriptor.expected_type.__name__} [{', '.join(constraints)}]\")\n", + "\n", + "# Create valid instances\n", + "emp = Employee(name=\"Alice Johnson\", email=\"alice@example.com\", age=30)\n", + "prod = Product(sku=\"WDG-001\", price=29.99, quantity=100)\n", + "\n", + "print(f\"\\n{emp!r}\")\n", + "print(f\"{prod!r}\")\n", + "\n", + "# Demonstrate validation errors\n", + "test_cases: list[tuple[str, dict[str, Any]]] = [\n", + " (\"Employee with empty name\", {\"name\": \"\", \"email\": \"a@b.com\", \"age\": 25}),\n", + " (\"Employee too young\", {\"name\": \"Bob\", \"email\": \"b@c.com\", \"age\": 15}),\n", + " (\"Product negative price\", {\"sku\": \"ABC\", \"price\": -5.0}),\n", + " (\"Product wrong type for price\", {\"sku\": \"ABC\", \"price\": \"free\"}),\n", + "]\n", + "\n", + "print(\"\\nValidation errors:\")\n", + "for desc, kwargs in test_cases:\n", + " try:\n", + " if \"sku\" in kwargs:\n", + " Product(**kwargs)\n", + " else:\n", + " Employee(**kwargs)\n", + " except (TypeError, ValueError) as e:\n", + " print(f\" {desc}: {type(e).__name__}: {e}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Tool | Purpose | Key Functions |\n", + "|------|---------|---------------|\n", + "| **`inspect`** | Examine live objects | `signature()`, `getsource()`, `getmembers()` |\n", + "| **`inspect` predicates** | Classify objects | `isclass()`, `isfunction()`, `ismethod()` |\n", + "| **`Signature`/`Parameter`** | Structured parameter info | `sig.parameters`, `param.kind`, `param.default` |\n", + "| **`type()` (3-arg)** | Create classes dynamically | `type(name, bases, namespace)` |\n", + "| **`exec()`/`eval()`** | Dynamic code execution | Use sparingly, never with untrusted input |\n", + "| **`ast`** | Parse and analyze source | `ast.parse()`, `ast.walk()`, `NodeVisitor` |\n", + "| **Code generation** | Generate methods at runtime | Used by `@dataclass`, `attrs`, ORMs |\n", + "\n", + "### Best Practices\n", + "- Use `inspect.signature()` instead of manually parsing `__code__` attributes\n", + "- Prefer `ast.literal_eval()` over `eval()` for parsing data literals safely\n", + "- Use `type()` for simple dynamic class creation; use metaclasses for complex cases\n", + "- When using `exec()` for code generation, keep the generated code minimal and well-tested\n", + "- Combine `__init_subclass__` with `get_type_hints()` for annotation-driven frameworks\n", + "- The `ast.NodeVisitor` pattern is the idiomatic way to analyze Python source trees" + ], + "outputs": [], + "execution_count": null + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_24/README.md b/src/chapter_24/README.md new file mode 100644 index 0000000..8fcd94c --- /dev/null +++ b/src/chapter_24/README.md @@ -0,0 +1,22 @@ +# Chapter 24: Metaprogramming + +## Topics Covered +- Dynamic attribute access: `__getattr__`, `__getattribute__`, `__setattr__` +- Property factories and dynamic properties +- Class decorators: modifying classes after creation +- `__init_subclass__`: hook for subclass creation +- Import hooks: `sys.meta_path`, custom finders and loaders +- `exec()` and `eval()`: dynamic code execution (and its dangers) +- `inspect` module: introspecting live objects +- Code generation patterns + +## Notebooks +1. **01_dynamic_attributes.ipynb** — `__getattr__`, `__setattr__`, property factories +2. **02_class_decorators.ipynb** — Class decorators, `__init_subclass__`, registration +3. **03_introspection_and_codegen.ipynb** — inspect module, import hooks, code generation + +## Key Takeaways +- `__getattr__` enables proxy objects and lazy attributes +- Class decorators are a simpler alternative to metaclasses +- `__init_subclass__` handles most subclass registration needs +- Use metaprogramming sparingly — readability trumps cleverness diff --git a/src/chapter_24/__init__.py b/src/chapter_24/__init__.py new file mode 100644 index 0000000..c11efec --- /dev/null +++ b/src/chapter_24/__init__.py @@ -0,0 +1 @@ +"""Chapter 24: Metaprogramming.""" diff --git a/src/chapter_25/01_code_style_and_docs.ipynb b/src/chapter_25/01_code_style_and_docs.ipynb new file mode 100644 index 0000000..35f9295 --- /dev/null +++ b/src/chapter_25/01_code_style_and_docs.ipynb @@ -0,0 +1,818 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 25: Code Style and Documentation\n", + "\n", + "Writing clean, readable, and well-documented Python code is as important as making it work.\n", + "This notebook covers PEP 8 style guidelines, naming conventions, docstring standards,\n", + "type hints as documentation, the `textwrap` module, and code organization best practices.\n", + "\n", + "## Topics Covered\n", + "- **PEP 8**: The Python style guide and naming conventions\n", + "- **Naming conventions**: `snake_case`, `PascalCase`, `UPPER_SNAKE_CASE`\n", + "- **Formatting rules**: Line length, indentation, blank lines, import ordering\n", + "- **Docstrings**: PEP 257, one-line vs multi-line, Google/NumPy styles\n", + "- **`__doc__` attribute** and `help()` function\n", + "- **Type hints as documentation** (revisited)\n", + "- **`textwrap` module**: `dedent()`, `fill()`, `wrap()`\n", + "- **Code organization**: Module layout, `__all__`, public vs private" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PEP 8: The Python Style Guide\n", + "\n", + "**PEP 8** is the official style guide for Python code. It covers naming, formatting,\n", + "whitespace, comments, and more. Consistency within a project is the most important principle.\n", + "\n", + "Key principles:\n", + "- Code is read much more often than it is written\n", + "- Consistency with PEP 8 matters, but consistency within a project matters more\n", + "- Sometimes PEP 8 guidelines should be broken (e.g., to maintain backward compatibility)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# PEP 8 Naming Conventions\n", + "\n", + "# Variables and functions: snake_case\n", + "user_name: str = \"alice\"\n", + "item_count: int = 42\n", + "\n", + "\n", + "def calculate_total_price(unit_price: float, quantity: int) -> float:\n", + " \"\"\"Calculate the total price for a given quantity.\"\"\"\n", + " return unit_price * quantity\n", + "\n", + "\n", + "# Classes: PascalCase (also called CapWords or CamelCase)\n", + "class ShoppingCart:\n", + " \"\"\"A shopping cart that holds items and computes totals.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.items: list[tuple[str, float, int]] = []\n", + "\n", + " def add_item(self, name: str, price: float, qty: int = 1) -> None:\n", + " \"\"\"Add an item to the cart.\"\"\"\n", + " self.items.append((name, price, qty))\n", + "\n", + "\n", + "# Constants: UPPER_SNAKE_CASE\n", + "MAX_RETRY_COUNT: int = 3\n", + "DEFAULT_TIMEOUT_SECONDS: float = 30.0\n", + "BASE_API_URL: str = \"https://api.example.com/v1\"\n", + "\n", + "# Module-level \"private\" names: leading underscore\n", + "_internal_cache: dict[str, str] = {}\n", + "\n", + "\n", + "def _helper_function() -> None:\n", + " \"\"\"Not part of the public API.\"\"\"\n", + " pass\n", + "\n", + "\n", + "print(\"Naming convention examples:\")\n", + "print(f\" snake_case variable: user_name = {user_name!r}\")\n", + "print(f\" snake_case function: calculate_total_price(9.99, 3) = {calculate_total_price(9.99, 3)}\")\n", + "print(f\" PascalCase class: {ShoppingCart.__name__}\")\n", + "print(f\" UPPER_SNAKE constant: MAX_RETRY_COUNT = {MAX_RETRY_COUNT}\")\n", + "print(f\" _private name: _internal_cache = {_internal_cache}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# PEP 8: Line Length, Indentation, and Blank Lines\n", + "\n", + "# Rule: Maximum line length is 79 characters (72 for docstrings/comments)\n", + "# Many projects use 88 (black) or 100 as their limit\n", + "\n", + "# Line continuation with parentheses (preferred over backslash)\n", + "total = (\n", + " 1\n", + " + 2\n", + " + 3\n", + " + 4\n", + " + 5\n", + ")\n", + "\n", + "# Long function signatures: align or use hanging indent\n", + "def create_user(\n", + " username: str,\n", + " email: str,\n", + " age: int,\n", + " is_active: bool = True,\n", + ") -> dict[str, str | int | bool]:\n", + " \"\"\"Create a user dictionary with the given attributes.\"\"\"\n", + " return {\n", + " \"username\": username,\n", + " \"email\": email,\n", + " \"age\": age,\n", + " \"is_active\": is_active,\n", + " }\n", + "\n", + "\n", + "# Long conditions: break after the boolean operator\n", + "user = create_user(\"alice\", \"alice@example.com\", 30)\n", + "is_valid = (\n", + " user[\"is_active\"]\n", + " and isinstance(user[\"age\"], int)\n", + " and user[\"age\"] >= 18\n", + ")\n", + "\n", + "print(f\"total = {total}\")\n", + "print(f\"user = {user}\")\n", + "print(f\"is_valid = {is_valid}\")\n", + "\n", + "# Blank lines:\n", + "# - 2 blank lines before/after top-level definitions (functions, classes)\n", + "# - 1 blank line between methods inside a class\n", + "# - Use blank lines sparingly inside functions to indicate logical sections\n", + "print(\"\\nIndentation: always use 4 spaces per level (never tabs)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# PEP 8: Import Ordering\n", + "#\n", + "# Imports should be grouped in this order, separated by a blank line:\n", + "# 1. Standard library imports\n", + "# 2. Related third-party imports\n", + "# 3. Local application/library-specific imports\n", + "#\n", + "# Within each group, imports should be alphabetically sorted.\n", + "# Absolute imports are preferred over relative imports.\n", + "\n", + "# Example of properly ordered imports (shown as a string since\n", + "# we are in a notebook and cannot demonstrate all import types)\n", + "\n", + "import_example = \"\"\"\n", + "# 1. Standard library\n", + "import os\n", + "import sys\n", + "from collections import defaultdict\n", + "from pathlib import Path\n", + "\n", + "# 2. Third-party\n", + "import requests\n", + "from flask import Flask, jsonify\n", + "\n", + "# 3. Local application\n", + "from myapp.models import User\n", + "from myapp.utils import validate_email\n", + "\"\"\"\n", + "\n", + "print(\"PEP 8 import ordering:\")\n", + "print(import_example)\n", + "\n", + "# Imports to avoid:\n", + "print(\"Imports to AVOID:\")\n", + "print(\" from module import * # Pollutes namespace\")\n", + "print(\" import os, sys # Put on separate lines\")\n", + "print(\" import os; import sys # No semicolons\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Docstrings: PEP 257 Conventions\n", + "\n", + "**PEP 257** defines conventions for Python docstrings. A docstring is a string literal\n", + "that appears as the first statement in a module, class, method, or function body.\n", + "\n", + "Key rules:\n", + "- Use triple double quotes `\"\"\"...\"\"\"`\n", + "- One-line docstrings: opening and closing quotes on the same line\n", + "- Multi-line docstrings: summary line, blank line, then description\n", + "- The closing `\"\"\"` should be on its own line for multi-line docstrings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# One-line vs multi-line docstrings\n", + "\n", + "def square(n: int) -> int:\n", + " \"\"\"Return the square of n.\"\"\"\n", + " return n * n\n", + "\n", + "\n", + "def calculate_bmi(\n", + " weight_kg: float,\n", + " height_m: float,\n", + ") -> float:\n", + " \"\"\"Calculate Body Mass Index (BMI) from weight and height.\n", + "\n", + " BMI is calculated as weight in kilograms divided by the square\n", + " of height in meters. This is a standard measure used to classify\n", + " underweight, normal, overweight, and obese categories.\n", + "\n", + " The formula is: BMI = weight_kg / (height_m ** 2)\n", + " \"\"\"\n", + " if height_m <= 0:\n", + " raise ValueError(\"Height must be positive\")\n", + " return weight_kg / (height_m ** 2)\n", + "\n", + "\n", + "class Temperature:\n", + " \"\"\"A temperature value with conversion methods.\n", + "\n", + " Stores temperature internally in Celsius and provides\n", + " conversion to Fahrenheit and Kelvin.\n", + " \"\"\"\n", + "\n", + " def __init__(self, celsius: float) -> None:\n", + " \"\"\"Initialize with a temperature in Celsius.\"\"\"\n", + " self.celsius = celsius\n", + "\n", + " def to_fahrenheit(self) -> float:\n", + " \"\"\"Convert the stored temperature to Fahrenheit.\"\"\"\n", + " return self.celsius * 9 / 5 + 32\n", + "\n", + " def to_kelvin(self) -> float:\n", + " \"\"\"Convert the stored temperature to Kelvin.\"\"\"\n", + " return self.celsius + 273.15\n", + "\n", + "\n", + "print(f\"square(5) = {square(5)}\")\n", + "print(f\"BMI(70, 1.75) = {calculate_bmi(70, 1.75):.1f}\")\n", + "\n", + "temp = Temperature(100)\n", + "print(f\"\\n100C = {temp.to_fahrenheit()}F = {temp.to_kelvin()}K\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Google-style vs NumPy-style docstrings\n", + "\n", + "# Google style: uses indented sections with colons\n", + "def google_style_example(\n", + " name: str,\n", + " scores: list[float],\n", + " normalize: bool = False,\n", + ") -> dict[str, float]:\n", + " \"\"\"Compute summary statistics for a student's scores.\n", + "\n", + " Takes a list of scores and returns a dictionary with the mean,\n", + " minimum, and maximum values. Optionally normalizes scores to\n", + " the 0-1 range.\n", + "\n", + " Args:\n", + " name: The student's name.\n", + " scores: A list of numerical scores.\n", + " normalize: If True, normalize scores to [0, 1] before\n", + " computing statistics.\n", + "\n", + " Returns:\n", + " A dictionary with keys 'name', 'mean', 'min', 'max'.\n", + "\n", + " Raises:\n", + " ValueError: If scores is empty.\n", + " \"\"\"\n", + " if not scores:\n", + " raise ValueError(\"scores must not be empty\")\n", + " if normalize:\n", + " lo, hi = min(scores), max(scores)\n", + " spread = hi - lo if hi != lo else 1.0\n", + " scores = [(s - lo) / spread for s in scores]\n", + " return {\n", + " \"name\": name,\n", + " \"mean\": sum(scores) / len(scores),\n", + " \"min\": min(scores),\n", + " \"max\": max(scores),\n", + " }\n", + "\n", + "\n", + "# NumPy style: uses underlined section headers\n", + "def numpy_style_example(\n", + " data: list[float],\n", + " window_size: int = 3,\n", + ") -> list[float]:\n", + " \"\"\"Compute a simple moving average over a data series.\n", + "\n", + " Parameters\n", + " ----------\n", + " data : list[float]\n", + " The input data series.\n", + " window_size : int, optional\n", + " The number of points to average over (default is 3).\n", + "\n", + " Returns\n", + " -------\n", + " list[float]\n", + " The smoothed data series, shorter by (window_size - 1) elements.\n", + "\n", + " Examples\n", + " --------\n", + " >>> numpy_style_example([1.0, 2.0, 3.0, 4.0, 5.0], window_size=3)\n", + " [2.0, 3.0, 4.0]\n", + " \"\"\"\n", + " return [\n", + " sum(data[i : i + window_size]) / window_size\n", + " for i in range(len(data) - window_size + 1)\n", + " ]\n", + "\n", + "\n", + "print(\"Google style:\")\n", + "result = google_style_example(\"Alice\", [85, 92, 78, 95, 88])\n", + "print(f\" {result}\")\n", + "\n", + "print(\"\\nNumPy style:\")\n", + "smoothed = numpy_style_example([1.0, 2.0, 3.0, 4.0, 5.0], window_size=3)\n", + "print(f\" {smoothed}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# __doc__ attribute and help() function\n", + "\n", + "# Every documented object has a __doc__ attribute\n", + "def greet(name: str) -> str:\n", + " \"\"\"Return a greeting message for the given name.\"\"\"\n", + " return f\"Hello, {name}!\"\n", + "\n", + "\n", + "class Circle:\n", + " \"\"\"A circle defined by its radius.\n", + "\n", + " Provides methods to compute area and circumference.\n", + " \"\"\"\n", + "\n", + " import math\n", + "\n", + " def __init__(self, radius: float) -> None:\n", + " \"\"\"Initialize the circle with the given radius.\"\"\"\n", + " self.radius = radius\n", + "\n", + " def area(self) -> float:\n", + " \"\"\"Return the area of the circle.\"\"\"\n", + " return self.math.pi * self.radius ** 2\n", + "\n", + "\n", + "# Access __doc__ directly\n", + "print(\"__doc__ attribute:\")\n", + "print(f\" greet.__doc__ = {greet.__doc__!r}\")\n", + "print(f\" Circle.__doc__ = {Circle.__doc__!r}\")\n", + "print(f\" Circle.area.__doc__ = {Circle.area.__doc__!r}\")\n", + "\n", + "# Built-in objects have docstrings too\n", + "print(f\"\\n len.__doc__ = {len.__doc__!r}\")\n", + "print(f\" list.append.__doc__ = {list.append.__doc__!r}\")\n", + "\n", + "# help() renders the docstring with formatting\n", + "print(\"\\n--- help(greet) ---\")\n", + "help(greet)\n", + "\n", + "print(\"--- help(Circle) ---\")\n", + "help(Circle)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Type Hints as Documentation (Revisited)\n", + "\n", + "Type hints serve as a form of **executable documentation** that can be verified by tools\n", + "like `mypy`. They communicate the expected types of function parameters, return values,\n", + "and variables without requiring the reader to infer them from usage." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Protocol, TypeAlias\n", + "from collections.abc import Callable, Sequence\n", + "\n", + "\n", + "# Type aliases make complex types readable and self-documenting\n", + "UserId: TypeAlias = int\n", + "Coordinate: TypeAlias = tuple[float, float]\n", + "ErrorHandler: TypeAlias = Callable[[Exception], None]\n", + "Matrix: TypeAlias = list[list[float]]\n", + "\n", + "\n", + "# Without type hints - what do these parameters expect?\n", + "def find_nearest_bad(point, locations):\n", + " \"\"\"Find the nearest location to a point.\"\"\"\n", + " pass # What is a point? What is locations?\n", + "\n", + "\n", + "# With type hints - immediately clear\n", + "def find_nearest(\n", + " point: Coordinate,\n", + " locations: Sequence[Coordinate],\n", + ") -> Coordinate | None:\n", + " \"\"\"Find the nearest location to a point.\"\"\"\n", + " if not locations:\n", + " return None\n", + " return min(\n", + " locations,\n", + " key=lambda loc: (loc[0] - point[0]) ** 2 + (loc[1] - point[1]) ** 2,\n", + " )\n", + "\n", + "\n", + "# Protocol as documentation: specifies required interface\n", + "class Loggable(Protocol):\n", + " \"\"\"Any object that can produce a log-friendly string.\"\"\"\n", + "\n", + " def to_log_string(self) -> str: ...\n", + "\n", + "\n", + "def log_items(items: Sequence[Loggable]) -> None:\n", + " \"\"\"Log each item using its to_log_string method.\"\"\"\n", + " for item in items:\n", + " print(f\" LOG: {item.to_log_string()}\")\n", + "\n", + "\n", + "class Order:\n", + " \"\"\"A simple order that satisfies the Loggable protocol.\"\"\"\n", + "\n", + " def __init__(self, order_id: int, total: float) -> None:\n", + " self.order_id = order_id\n", + " self.total = total\n", + "\n", + " def to_log_string(self) -> str:\n", + " return f\"Order#{self.order_id}: ${self.total:.2f}\"\n", + "\n", + "\n", + "# Demonstration\n", + "locations: list[Coordinate] = [(1.0, 2.0), (3.0, 4.0), (0.5, 0.5)]\n", + "nearest = find_nearest((1.0, 1.0), locations)\n", + "print(f\"Nearest to (1.0, 1.0): {nearest}\")\n", + "\n", + "orders = [Order(1, 29.99), Order(2, 149.50)]\n", + "log_items(orders)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The textwrap Module\n", + "\n", + "The `textwrap` module provides utilities for wrapping, filling, indenting, and\n", + "cleaning up text. It is especially useful for formatting docstrings, help messages,\n", + "and CLI output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "# textwrap.dedent() - remove common leading whitespace\n", + "# Essential for multi-line strings defined inside indented code\n", + "def get_help_text() -> str:\n", + " return textwrap.dedent(\"\"\"\\\n", + " Usage: myapp [OPTIONS] COMMAND\n", + "\n", + " A command-line tool for managing projects.\n", + "\n", + " Options:\n", + " --help Show this help message\n", + " --version Show version number\n", + " --verbose Enable verbose output\n", + " \"\"\")\n", + "\n", + "\n", + "print(\"dedent() removes common leading whitespace:\")\n", + "print(get_help_text())\n", + "\n", + "# Without dedent, the text would have unwanted indentation\n", + "raw = \"\"\"\n", + " This text has\n", + " extra indentation\n", + " from the source code.\n", + " \"\"\"\n", + "print(\"Before dedent:\")\n", + "print(repr(raw))\n", + "print(\"\\nAfter dedent:\")\n", + "print(repr(textwrap.dedent(raw)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "long_text = (\n", + " \"Python is a high-level, general-purpose programming language. \"\n", + " \"Its design philosophy emphasizes code readability with the use of \"\n", + " \"significant indentation. Python is dynamically typed and garbage-collected. \"\n", + " \"It supports multiple programming paradigms, including structured, \"\n", + " \"object-oriented, and functional programming.\"\n", + ")\n", + "\n", + "# textwrap.wrap() - returns a list of wrapped lines\n", + "lines = textwrap.wrap(long_text, width=50)\n", + "print(\"wrap(width=50) returns a list of lines:\")\n", + "for i, line in enumerate(lines):\n", + " print(f\" [{i}] {line!r}\")\n", + "\n", + "# textwrap.fill() - returns a single string with newlines\n", + "print(\"\\nfill(width=50) returns a single string:\")\n", + "print(textwrap.fill(long_text, width=50))\n", + "\n", + "# fill() with initial_indent and subsequent_indent\n", + "print(\"\\nfill() with indentation options:\")\n", + "print(textwrap.fill(\n", + " long_text,\n", + " width=60,\n", + " initial_indent=\" * \",\n", + " subsequent_indent=\" \",\n", + "))\n", + "\n", + "# textwrap.shorten() - truncate text to a maximum width\n", + "print(\"\\nshorten(width=40):\")\n", + "print(f\" {textwrap.shorten(long_text, width=40)}\")\n", + "print(f\" {textwrap.shorten(long_text, width=40, placeholder='...')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Code Organization: Module Layout\n", + "\n", + "A well-organized Python module follows a standard layout. The order of elements\n", + "matters for readability and tooling compatibility.\n", + "\n", + "**Recommended module layout:**\n", + "1. Module docstring\n", + "2. `__all__` definition (if used)\n", + "3. Imports (stdlib, third-party, local)\n", + "4. Module-level constants\n", + "5. Module-level \"private\" helpers\n", + "6. Public classes and functions\n", + "7. `if __name__ == \"__main__\":` block (if applicable)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# __all__: Controlling What Gets Exported\n", + "#\n", + "# __all__ is a list of strings that defines the public API of a module.\n", + "# When someone does `from module import *`, only names in __all__ are imported.\n", + "# It also serves as documentation of the module's intended public interface.\n", + "\n", + "# Simulating a module's __all__ within this notebook\n", + "# In a real module file (e.g., utils.py), this would be at the top:\n", + "\n", + "module_example = '''\n", + "\"\"\"Utility functions for string processing.\"\"\"\n", + "\n", + "__all__ = [\"slugify\", \"truncate\", \"StringProcessor\"]\n", + "\n", + "import re\n", + "import unicodedata\n", + "\n", + "# Not in __all__ -- this is an internal helper\n", + "def _normalize_whitespace(text: str) -> str:\n", + " return re.sub(r\"\\\\s+\", \" \", text).strip()\n", + "\n", + "# In __all__ -- part of the public API\n", + "def slugify(text: str) -> str:\n", + " \"\"\"Convert text to a URL-friendly slug.\"\"\"\n", + " text = unicodedata.normalize(\"NFKD\", text)\n", + " text = text.lower().strip()\n", + " text = re.sub(r\"[^\\\\w\\\\s-]\", \"\", text)\n", + " return re.sub(r\"[-\\\\s]+\", \"-\", text)\n", + "\n", + "def truncate(text: str, max_length: int = 100) -> str:\n", + " \"\"\"Truncate text to max_length, adding ellipsis if needed.\"\"\"\n", + " if len(text) <= max_length:\n", + " return text\n", + " return text[:max_length - 3] + \"...\"\n", + "\n", + "class StringProcessor:\n", + " \"\"\"A configurable string processor.\"\"\"\n", + " ...\n", + "'''\n", + "\n", + "print(\"Module layout example:\")\n", + "print(module_example)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Public vs Private: Naming Conventions in Practice\n", + "#\n", + "# Python uses naming conventions (not access modifiers) to signal visibility:\n", + "# public_name - part of the public API\n", + "# _private_name - internal implementation detail (convention)\n", + "# __mangled_name - name-mangled to _ClassName__mangled_name (rarely needed)\n", + "# __dunder__ - reserved for Python's special methods\n", + "\n", + "class BankAccount:\n", + " \"\"\"A bank account demonstrating public vs private conventions.\"\"\"\n", + "\n", + " # Class-level constant (public)\n", + " MIN_BALANCE: float = 0.0\n", + "\n", + " def __init__(self, owner: str, balance: float = 0.0) -> None:\n", + " # Public attribute: part of the interface\n", + " self.owner = owner\n", + " # \"Private\" attribute: implementation detail\n", + " self._balance = balance\n", + " # Tracks transaction history internally\n", + " self._transactions: list[tuple[str, float]] = []\n", + "\n", + " # Public method: part of the API\n", + " def deposit(self, amount: float) -> None:\n", + " \"\"\"Deposit money into the account.\"\"\"\n", + " self._validate_amount(amount)\n", + " self._balance += amount\n", + " self._record_transaction(\"deposit\", amount)\n", + "\n", + " def get_balance(self) -> float:\n", + " \"\"\"Return the current balance.\"\"\"\n", + " return self._balance\n", + "\n", + " # Private methods: internal helpers\n", + " def _validate_amount(self, amount: float) -> None:\n", + " \"\"\"Validate that an amount is positive.\"\"\"\n", + " if amount <= 0:\n", + " raise ValueError(f\"Amount must be positive, got {amount}\")\n", + "\n", + " def _record_transaction(self, kind: str, amount: float) -> None:\n", + " \"\"\"Record a transaction in the internal log.\"\"\"\n", + " self._transactions.append((kind, amount))\n", + "\n", + " # Dunder method: Python special protocol\n", + " def __repr__(self) -> str:\n", + " return f\"BankAccount(owner={self.owner!r}, balance={self._balance:.2f})\"\n", + "\n", + "\n", + "account = BankAccount(\"Alice\", 100.0)\n", + "account.deposit(50.0)\n", + "print(f\"Account: {account}\")\n", + "print(f\"Balance: ${account.get_balance():.2f}\")\n", + "\n", + "# _private attributes are accessible but signal 'do not use directly'\n", + "print(f\"\\nDirect access (not recommended): account._balance = {account._balance}\")\n", + "print(f\"Transaction log: {account._transactions}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Putting it all together: a well-structured mini-module\n", + "\n", + "import textwrap\n", + "from collections.abc import Sequence\n", + "from typing import TypeAlias\n", + "\n", + "# Type aliases\n", + "Score: TypeAlias = float\n", + "GradeLetter: TypeAlias = str\n", + "\n", + "# Constants\n", + "GRADE_THRESHOLDS: dict[GradeLetter, Score] = {\n", + " \"A\": 90.0,\n", + " \"B\": 80.0,\n", + " \"C\": 70.0,\n", + " \"D\": 60.0,\n", + " \"F\": 0.0,\n", + "}\n", + "\n", + "\n", + "def _find_grade(score: Score) -> GradeLetter:\n", + " \"\"\"Map a numeric score to a letter grade.\"\"\"\n", + " for letter, threshold in GRADE_THRESHOLDS.items():\n", + " if score >= threshold:\n", + " return letter\n", + " return \"F\"\n", + "\n", + "\n", + "def generate_report(\n", + " student_name: str,\n", + " scores: Sequence[Score],\n", + ") -> str:\n", + " \"\"\"Generate a formatted grade report for a student.\n", + "\n", + " Args:\n", + " student_name: The student's full name.\n", + " scores: A sequence of numeric scores (0-100).\n", + "\n", + " Returns:\n", + " A formatted multi-line report string.\n", + " \"\"\"\n", + " if not scores:\n", + " return f\"No scores recorded for {student_name}.\"\n", + "\n", + " avg: Score = sum(scores) / len(scores)\n", + " grade: GradeLetter = _find_grade(avg)\n", + "\n", + " report = textwrap.dedent(f\"\"\"\\\n", + " Student Report\n", + " ==============\n", + " Name: {student_name}\n", + " Scores: {', '.join(f'{s:.0f}' for s in scores)}\n", + " Average: {avg:.1f}\n", + " Grade: {grade}\n", + " \"\"\")\n", + " return report\n", + "\n", + "\n", + "# Generate and display the report\n", + "print(generate_report(\"Alice Johnson\", [92, 85, 78, 95, 88]))\n", + "print(generate_report(\"Bob Smith\", [62, 71, 55, 68, 73]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Topic | Convention / Tool | Purpose |\n", + "|-------|------------------|----------|\n", + "| **Variables/functions** | `snake_case` | Readability, PEP 8 standard |\n", + "| **Classes** | `PascalCase` | Distinguish types from values |\n", + "| **Constants** | `UPPER_SNAKE_CASE` | Signal immutability |\n", + "| **Private names** | `_leading_underscore` | Signal internal implementation |\n", + "| **Imports** | stdlib, third-party, local | Organized dependency sections |\n", + "| **One-line docstring** | `\"\"\"Return the result.\"\"\"` | Simple functions |\n", + "| **Multi-line docstring** | Google or NumPy style | Complex functions with args/returns |\n", + "| **`__doc__`** | Attribute access | Programmatic docstring access |\n", + "| **Type hints** | `TypeAlias`, `Protocol` | Executable, verifiable documentation |\n", + "| **`textwrap.dedent()`** | Remove indentation | Clean multi-line strings |\n", + "| **`textwrap.fill()`** | Wrap text | Format text to a given width |\n", + "| **`__all__`** | Export control | Define the public API of a module |\n", + "\n", + "### Best Practices\n", + "- Follow PEP 8 consistently, but prioritize project-level consistency\n", + "- Write docstrings for all public modules, classes, functions, and methods\n", + "- Use type hints to make function signatures self-documenting\n", + "- Choose one docstring style (Google or NumPy) and use it throughout your project\n", + "- Use `__all__` to explicitly declare your module's public API\n", + "- Prefix internal helpers with an underscore to signal they are not public\n", + "- Use `textwrap.dedent()` for multi-line strings defined inside indented code" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_25/02_linting_and_testing.ipynb b/src/chapter_25/02_linting_and_testing.ipynb new file mode 100644 index 0000000..7ee94a4 --- /dev/null +++ b/src/chapter_25/02_linting_and_testing.ipynb @@ -0,0 +1,831 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 25: Linting, Formatting, and Testing\n", + "\n", + "Automated tooling for code quality and testing is essential for professional Python\n", + "development. This notebook covers linting with `ruff`, auto-formatting with `black`,\n", + "import sorting, and comprehensive testing with `pytest`.\n", + "\n", + "## Topics Covered\n", + "- **Linting**: `ruff`, `flake8` concepts, `pycodestyle`\n", + "- **Auto-formatting**: `black`, `ruff format`\n", + "- **Import sorting**: `isort`, `ruff`'s isort rules\n", + "- **pytest fundamentals**: Test discovery, assertions, fixtures\n", + "- **`@pytest.mark.parametrize`**: Data-driven tests\n", + "- **Fixture patterns**: Scope, `autouse`, factory fixtures\n", + "- **Protocol-based mocking** for testability\n", + "- **Test organization**: `conftest.py`, test classes, naming" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Linting: Catching Errors Before Runtime\n", + "\n", + "A **linter** analyzes source code for potential errors, style violations, and suspicious\n", + "constructs without actually running the code. Modern Python projects typically use\n", + "`ruff` as a fast, unified linter that replaces `flake8`, `pycodestyle`, `pyflakes`,\n", + "and many other tools.\n", + "\n", + "**Common lint categories:**\n", + "- **E/W** (pycodestyle): Style violations (whitespace, line length, etc.)\n", + "- **F** (pyflakes): Logical errors (unused imports, undefined names)\n", + "- **I** (isort): Import ordering violations\n", + "- **UP** (pyupgrade): Code that can use newer Python syntax\n", + "- **B** (bugbear): Likely bugs and design problems" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Examples of common lint violations and their fixes\n", + "#\n", + "# We cannot run ruff directly in a notebook, but we can demonstrate\n", + "# the kinds of issues linters catch.\n", + "\n", + "# --- E501: Line too long ---\n", + "# BAD: exceeds 79 characters\n", + "# result = some_function(first_argument, second_argument, third_argument, fourth_argument, fifth_argument)\n", + "\n", + "# GOOD: break into multiple lines\n", + "# result = some_function(\n", + "# first_argument,\n", + "# second_argument,\n", + "# third_argument,\n", + "# )\n", + "\n", + "# --- F401: Unused import ---\n", + "# BAD:\n", + "# import os # never used in the module\n", + "\n", + "# --- F841: Local variable assigned but never used ---\n", + "# BAD:\n", + "# x = compute_something() # x is never read\n", + "\n", + "# --- E711: Comparison to None ---\n", + "# BAD:\n", + "value: str | None = \"hello\"\n", + "# if value == None: # noqa: E711\n", + "# pass\n", + "\n", + "# GOOD:\n", + "if value is None:\n", + " pass\n", + "\n", + "# --- UP035: Use newer import path ---\n", + "# BAD (Python < 3.9 style):\n", + "# from typing import List, Dict\n", + "\n", + "# GOOD (Python 3.9+):\n", + "# Use list[str], dict[str, int] directly\n", + "\n", + "# --- B006: Mutable default argument ---\n", + "# BAD:\n", + "def bad_default(items: list[str] = []) -> list[str]: # noqa: B006\n", + " items.append(\"new\")\n", + " return items\n", + "\n", + "# GOOD:\n", + "def good_default(items: list[str] | None = None) -> list[str]:\n", + " if items is None:\n", + " items = []\n", + " items.append(\"new\")\n", + " return items\n", + "\n", + "\n", + "print(\"Mutable default argument bug:\")\n", + "print(f\" bad_default() = {bad_default()}\")\n", + "print(f\" bad_default() = {bad_default()}\") # Bug: returns ['new', 'new']\n", + "print(f\" bad_default() = {bad_default()}\") # Bug: returns ['new', 'new', 'new']\n", + "\n", + "print(f\"\\n good_default() = {good_default()}\")\n", + "print(f\" good_default() = {good_default()}\") # Correct: always ['new']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ruff configuration in pyproject.toml\n", + "#\n", + "# ruff is configured via pyproject.toml (or ruff.toml).\n", + "# Here is a typical configuration:\n", + "\n", + "ruff_config = \"\"\"\n", + "[tool.ruff]\n", + "# Target Python version\n", + "target-version = \"py312\"\n", + "\n", + "# Line length (matching black's default)\n", + "line-length = 88\n", + "\n", + "[tool.ruff.lint]\n", + "# Enable rule categories\n", + "select = [\n", + " \"E\", # pycodestyle errors\n", + " \"W\", # pycodestyle warnings\n", + " \"F\", # pyflakes\n", + " \"I\", # isort\n", + " \"UP\", # pyupgrade\n", + " \"B\", # flake8-bugbear\n", + " \"SIM\", # flake8-simplify\n", + " \"RUF\", # ruff-specific rules\n", + "]\n", + "\n", + "# Ignore specific rules\n", + "ignore = [\n", + " \"E501\", # line too long (handled by formatter)\n", + "]\n", + "\n", + "[tool.ruff.lint.per-file-ignores]\n", + "# Allow unused imports in __init__.py (re-exports)\n", + "\"__init__.py\" = [\"F401\"]\n", + "# Allow assert in tests\n", + "\"tests/**\" = [\"S101\"]\n", + "\n", + "[tool.ruff.lint.isort]\n", + "# Force single-line imports for cleaner diffs\n", + "force-single-line = true\n", + "\"\"\"\n", + "\n", + "print(\"Typical ruff configuration (pyproject.toml):\")\n", + "print(ruff_config)\n", + "\n", + "# Command-line usage:\n", + "print(\"Common ruff commands:\")\n", + "print(\" ruff check . # Lint all files\")\n", + "print(\" ruff check --fix . # Lint and auto-fix safe issues\")\n", + "print(\" ruff check --select E,W . # Check only pycodestyle rules\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Auto-Formatting: black and ruff format\n", + "\n", + "Auto-formatters rewrite your code to conform to a consistent style, eliminating\n", + "debates about formatting. **black** is the most popular Python formatter, and\n", + "**ruff format** provides a near-identical output at much higher speed.\n", + "\n", + "Key principles of black:\n", + "- Deterministic: same input always produces same output\n", + "- Line length: 88 characters by default\n", + "- Trailing commas trigger vertical formatting\n", + "- Double quotes for strings by default" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# How black/ruff format transforms code\n", + "#\n", + "# Before formatting:\n", + "before = '''\n", + "x = { 'a':37,'b':42, 'c':927}\n", + "y = 'hello ''world'\n", + "z = 'hello '+'world'\n", + "a = [1,2, 3, 4]\n", + "zzz = {k: v for k,v in y.items() if k == 'a' or k == 'b'}\n", + "def f (a, b, c):\n", + " return a+b+c\n", + "'''\n", + "\n", + "# After formatting (what black/ruff format produces):\n", + "after = '''\n", + "x = {\"a\": 37, \"b\": 42, \"c\": 927}\n", + "y = \"hello \" \"world\"\n", + "z = \"hello \" + \"world\"\n", + "a = [1, 2, 3, 4]\n", + "zzz = {k: v for k, v in y.items() if k == \"a\" or k == \"b\"}\n", + "\n", + "\n", + "def f(a, b, c):\n", + " return a + b + c\n", + "'''\n", + "\n", + "print(\"Before auto-formatting:\")\n", + "print(before)\n", + "print(\"After auto-formatting:\")\n", + "print(after)\n", + "\n", + "# Trailing comma magic: forces vertical layout\n", + "print(\"Trailing comma triggers vertical formatting:\")\n", + "print(\"\"\" # Stays on one line (no trailing comma):\"\"\")\n", + "print(\"\"\" result = func(arg1, arg2, arg3)\"\"\")\n", + "print(\"\"\"\"\"\")\n", + "print(\"\"\" # Expands vertically (trailing comma present):\"\"\")\n", + "print(\"\"\" result = func(\n", + " arg1,\n", + " arg2,\n", + " arg3, # <-- trailing comma\n", + " )\"\"\")\n", + "\n", + "# Configuration in pyproject.toml\n", + "print(\"\\nFormatter config (pyproject.toml):\")\n", + "print(\" [tool.black]\")\n", + "print(' line-length = 88')\n", + "print(' target-version = [\"py312\"]')\n", + "print(\"\\n # Or for ruff format:\")\n", + "print(\" [tool.ruff.format]\")\n", + "print(' quote-style = \"double\"')\n", + "print(' indent-style = \"space\"')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import Sorting: isort and ruff's isort rules\n", + "#\n", + "# isort (or ruff's I rules) automatically sorts and groups imports\n", + "# according to PEP 8 conventions.\n", + "\n", + "# Before isort:\n", + "before_isort = \"\"\"\n", + "from myapp.utils import helper\n", + "import sys\n", + "from pathlib import Path\n", + "import requests\n", + "import os\n", + "from collections import defaultdict\n", + "from flask import Flask\n", + "from myapp.models import User\n", + "\"\"\"\n", + "\n", + "# After isort:\n", + "after_isort = \"\"\"\n", + "import os\n", + "import sys\n", + "from collections import defaultdict\n", + "from pathlib import Path\n", + "\n", + "import requests\n", + "from flask import Flask\n", + "\n", + "from myapp.models import User\n", + "from myapp.utils import helper\n", + "\"\"\"\n", + "\n", + "print(\"Before isort:\")\n", + "print(before_isort)\n", + "print(\"After isort:\")\n", + "print(after_isort)\n", + "\n", + "# isort configuration compatible with black\n", + "print(\"isort config (pyproject.toml):\")\n", + "print(\" [tool.isort]\")\n", + "print(' profile = \"black\"')\n", + "print(\" force_single_line = true\")\n", + "print()\n", + "print(\"Common commands:\")\n", + "print(\" isort . # Sort all imports\")\n", + "print(\" isort --check-only . # Check without modifying\")\n", + "print(\" ruff check --select I . # Check imports with ruff\")\n", + "print(\" ruff check --fix --select I . # Fix imports with ruff\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## pytest Fundamentals\n", + "\n", + "`pytest` is the standard testing framework for Python. It provides simple assertion\n", + "syntax, powerful fixtures, parameterized tests, and a plugin ecosystem.\n", + "\n", + "**Test discovery rules:**\n", + "- Test files: `test_*.py` or `*_test.py`\n", + "- Test functions: `test_*` prefix\n", + "- Test classes: `Test*` prefix (no `__init__` method)\n", + "- Test methods: `test_*` prefix inside `Test*` classes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Code under test: a simple calculator module\n", + "\n", + "class Calculator:\n", + " \"\"\"A basic calculator with history tracking.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.history: list[str] = []\n", + "\n", + " def add(self, a: float, b: float) -> float:\n", + " \"\"\"Return the sum of a and b.\"\"\"\n", + " result = a + b\n", + " self.history.append(f\"{a} + {b} = {result}\")\n", + " return result\n", + "\n", + " def divide(self, a: float, b: float) -> float:\n", + " \"\"\"Return a divided by b.\n", + "\n", + " Raises:\n", + " ZeroDivisionError: If b is zero.\n", + " \"\"\"\n", + " if b == 0:\n", + " raise ZeroDivisionError(\"Cannot divide by zero\")\n", + " result = a / b\n", + " self.history.append(f\"{a} / {b} = {result}\")\n", + " return result\n", + "\n", + " def clear_history(self) -> None:\n", + " \"\"\"Clear the calculation history.\"\"\"\n", + " self.history.clear()\n", + "\n", + "\n", + "# Demonstrate what pytest tests look like\n", + "# In a real project, this would be in tests/test_calculator.py\n", + "\n", + "def test_add() -> None:\n", + " \"\"\"Test basic addition.\"\"\"\n", + " calc = Calculator()\n", + " assert calc.add(2, 3) == 5\n", + " assert calc.add(-1, 1) == 0\n", + " assert calc.add(0.1, 0.2) - 0.3 < 1e-10 # float comparison\n", + "\n", + "\n", + "def test_divide() -> None:\n", + " \"\"\"Test basic division.\"\"\"\n", + " calc = Calculator()\n", + " assert calc.divide(10, 2) == 5.0\n", + " assert calc.divide(7, 2) == 3.5\n", + "\n", + "\n", + "def test_divide_by_zero() -> None:\n", + " \"\"\"Test that dividing by zero raises ZeroDivisionError.\"\"\"\n", + " calc = Calculator()\n", + " try:\n", + " calc.divide(1, 0)\n", + " assert False, \"Should have raised ZeroDivisionError\"\n", + " except ZeroDivisionError as e:\n", + " assert str(e) == \"Cannot divide by zero\"\n", + "\n", + "\n", + "def test_history() -> None:\n", + " \"\"\"Test that operations are recorded in history.\"\"\"\n", + " calc = Calculator()\n", + " calc.add(1, 2)\n", + " calc.divide(10, 5)\n", + " assert len(calc.history) == 2\n", + " assert \"1 + 2 = 3\" in calc.history[0]\n", + "\n", + "\n", + "# Run the tests manually in the notebook\n", + "for test_func in [test_add, test_divide, test_divide_by_zero, test_history]:\n", + " try:\n", + " test_func()\n", + " print(f\" PASSED: {test_func.__name__}\")\n", + " except AssertionError as e:\n", + " print(f\" FAILED: {test_func.__name__} - {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# @pytest.mark.parametrize: Data-Driven Tests\n", + "#\n", + "# Parametrize lets you run the same test with multiple sets of inputs.\n", + "# This avoids duplicating test logic and makes it easy to add cases.\n", + "\n", + "# In a real test file, you would use:\n", + "# import pytest\n", + "# @pytest.mark.parametrize(\"a, b, expected\", [\n", + "# (2, 3, 5),\n", + "# (-1, 1, 0),\n", + "# (0, 0, 0),\n", + "# (100, 200, 300),\n", + "# ])\n", + "# def test_add(a: float, b: float, expected: float) -> None:\n", + "# calc = Calculator()\n", + "# assert calc.add(a, b) == expected\n", + "\n", + "# Simulating parametrize behavior in the notebook\n", + "test_cases_add: list[tuple[float, float, float]] = [\n", + " (2, 3, 5),\n", + " (-1, 1, 0),\n", + " (0, 0, 0),\n", + " (100, 200, 300),\n", + " (-5, -3, -8),\n", + "]\n", + "\n", + "print(\"Parametrized test_add:\")\n", + "calc = Calculator()\n", + "for a, b, expected in test_cases_add:\n", + " result = calc.add(a, b)\n", + " status = \"PASSED\" if result == expected else \"FAILED\"\n", + " print(f\" {status}: add({a}, {b}) == {expected} (got {result})\")\n", + "\n", + "# Parametrize with IDs for readable output\n", + "# @pytest.mark.parametrize(\"input_str, expected\", [\n", + "# pytest.param(\"hello\", \"HELLO\", id=\"simple\"),\n", + "# pytest.param(\"\", \"\", id=\"empty\"),\n", + "# pytest.param(\"Hello World\", \"HELLO WORLD\", id=\"mixed-case\"),\n", + "# ])\n", + "# def test_upper(input_str: str, expected: str) -> None:\n", + "# assert input_str.upper() == expected\n", + "\n", + "test_cases_upper: list[tuple[str, str, str]] = [\n", + " (\"hello\", \"HELLO\", \"simple\"),\n", + " (\"\", \"\", \"empty\"),\n", + " (\"Hello World\", \"HELLO WORLD\", \"mixed-case\"),\n", + "]\n", + "\n", + "print(\"\\nParametrized test_upper (with IDs):\")\n", + "for input_str, expected, test_id in test_cases_upper:\n", + " result = input_str.upper()\n", + " status = \"PASSED\" if result == expected else \"FAILED\"\n", + " print(f\" {status} [{test_id}]: {input_str!r}.upper() == {expected!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Fixtures: Setup and Teardown for Tests\n", + "#\n", + "# pytest fixtures provide reusable setup/teardown logic.\n", + "# They are dependency-injected by name into test functions.\n", + "\n", + "# In a real test file:\n", + "fixture_example = \"\"\"\n", + "import pytest\n", + "from myapp.calculator import Calculator\n", + "\n", + "\n", + "# Basic fixture: creates a fresh Calculator for each test\n", + "@pytest.fixture\n", + "def calc() -> Calculator:\n", + " return Calculator()\n", + "\n", + "\n", + "# Fixture with teardown (using yield)\n", + "@pytest.fixture\n", + "def calc_with_history() -> Calculator:\n", + " calc = Calculator()\n", + " calc.add(1, 2)\n", + " calc.add(3, 4)\n", + " yield calc # Test runs here\n", + " calc.clear_history() # Teardown runs after the test\n", + "\n", + "\n", + "# Fixture scopes: \"function\" (default), \"class\", \"module\", \"session\"\n", + "@pytest.fixture(scope=\"module\")\n", + "def db_connection():\n", + " \"\"\"Created once per module, shared across all tests in the module.\"\"\"\n", + " conn = create_connection()\n", + " yield conn\n", + " conn.close()\n", + "\n", + "\n", + "# autouse: automatically applied to all tests in scope\n", + "@pytest.fixture(autouse=True)\n", + "def reset_global_state():\n", + " \"\"\"Reset global state before each test.\"\"\"\n", + " global_config.reset()\n", + " yield\n", + "\n", + "\n", + "# Factory fixture: returns a factory function for flexible setup\n", + "@pytest.fixture\n", + "def make_calculator():\n", + " \"\"\"Factory that creates calculators with optional preset history.\"\"\"\n", + " created: list[Calculator] = []\n", + "\n", + " def _factory(preload: list[tuple[float, float]] | None = None) -> Calculator:\n", + " calc = Calculator()\n", + " if preload:\n", + " for a, b in preload:\n", + " calc.add(a, b)\n", + " created.append(calc)\n", + " return calc\n", + "\n", + " yield _factory\n", + "\n", + " # Teardown: clear all created calculators\n", + " for c in created:\n", + " c.clear_history()\n", + "\n", + "\n", + "# Using fixtures in tests\n", + "def test_add_simple(calc: Calculator) -> None:\n", + " assert calc.add(2, 3) == 5\n", + "\n", + "\n", + "def test_with_preloaded_history(make_calculator) -> None:\n", + " calc = make_calculator(preload=[(1, 2), (3, 4)])\n", + " assert len(calc.history) == 2\n", + " assert calc.add(5, 6) == 11\n", + " assert len(calc.history) == 3\n", + "\"\"\"\n", + "\n", + "print(\"pytest fixture patterns:\")\n", + "print(fixture_example)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Protocol-Based Mocking for Testability\n", + "#\n", + "# Instead of patching with unittest.mock, use Protocols to define\n", + "# interfaces and inject test implementations. This makes tests\n", + "# simpler and less brittle.\n", + "\n", + "from typing import Protocol\n", + "\n", + "\n", + "# Define the interface using Protocol\n", + "class EmailSender(Protocol):\n", + " \"\"\"Protocol for sending emails.\"\"\"\n", + "\n", + " def send(self, to: str, subject: str, body: str) -> bool: ...\n", + "\n", + "\n", + "# Production implementation\n", + "class SmtpEmailSender:\n", + " \"\"\"Real email sender using SMTP.\"\"\"\n", + "\n", + " def __init__(self, host: str, port: int) -> None:\n", + " self.host = host\n", + " self.port = port\n", + "\n", + " def send(self, to: str, subject: str, body: str) -> bool:\n", + " # In production, this would actually send an email\n", + " print(f\" [SMTP] Sending to {to}: {subject}\")\n", + " return True\n", + "\n", + "\n", + "# Test implementation (no mocking library needed!)\n", + "class FakeEmailSender:\n", + " \"\"\"Fake email sender for testing.\"\"\"\n", + "\n", + " def __init__(self) -> None:\n", + " self.sent: list[tuple[str, str, str]] = []\n", + "\n", + " def send(self, to: str, subject: str, body: str) -> bool:\n", + " self.sent.append((to, subject, body))\n", + " return True\n", + "\n", + "\n", + "# Business logic depends on the Protocol, not the implementation\n", + "class UserService:\n", + " \"\"\"Service that manages users and sends notifications.\"\"\"\n", + "\n", + " def __init__(self, email_sender: EmailSender) -> None:\n", + " self.email_sender = email_sender\n", + "\n", + " def register_user(self, email: str, name: str) -> dict[str, str]:\n", + " \"\"\"Register a new user and send a welcome email.\"\"\"\n", + " user = {\"email\": email, \"name\": name, \"status\": \"active\"}\n", + " self.email_sender.send(\n", + " to=email,\n", + " subject=\"Welcome!\",\n", + " body=f\"Hello {name}, welcome to our service!\",\n", + " )\n", + " return user\n", + "\n", + "\n", + "# Test using the fake -- no mock.patch needed\n", + "def test_register_user_sends_welcome_email() -> None:\n", + " \"\"\"Test that registering a user sends a welcome email.\"\"\"\n", + " fake_sender = FakeEmailSender()\n", + " service = UserService(email_sender=fake_sender)\n", + "\n", + " user = service.register_user(\"alice@example.com\", \"Alice\")\n", + "\n", + " assert user[\"status\"] == \"active\"\n", + " assert len(fake_sender.sent) == 1\n", + " to, subject, body = fake_sender.sent[0]\n", + " assert to == \"alice@example.com\"\n", + " assert subject == \"Welcome!\"\n", + " assert \"Alice\" in body\n", + "\n", + "\n", + "# Run the test\n", + "test_register_user_sends_welcome_email()\n", + "print(\"PASSED: test_register_user_sends_welcome_email\")\n", + "\n", + "# Show production usage\n", + "print(\"\\nProduction usage:\")\n", + "prod_sender = SmtpEmailSender(\"smtp.example.com\", 587)\n", + "prod_service = UserService(email_sender=prod_sender)\n", + "prod_service.register_user(\"bob@example.com\", \"Bob\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test Organization: conftest.py, Test Classes, and Naming\n", + "#\n", + "# A well-organized test suite follows conventions that make tests\n", + "# easy to find, understand, and maintain.\n", + "\n", + "project_layout = \"\"\"\n", + "Project test organization:\n", + "\n", + "my_project/\n", + " src/\n", + " my_project/\n", + " __init__.py\n", + " calculator.py\n", + " user_service.py\n", + " email/\n", + " __init__.py\n", + " sender.py\n", + " tests/\n", + " conftest.py # Shared fixtures for all tests\n", + " test_calculator.py # Tests for calculator module\n", + " test_user_service.py # Tests for user_service module\n", + " email/\n", + " conftest.py # Fixtures specific to email tests\n", + " test_sender.py # Tests for email sender\n", + "\"\"\"\n", + "\n", + "conftest_example = '''\n", + "# tests/conftest.py\n", + "#\n", + "# conftest.py files are automatically loaded by pytest.\n", + "# Fixtures defined here are available to all tests in the\n", + "# same directory and subdirectories.\n", + "\n", + "import pytest\n", + "from my_project.calculator import Calculator\n", + "from my_project.email.sender import FakeEmailSender\n", + "\n", + "\n", + "@pytest.fixture\n", + "def calculator() -> Calculator:\n", + " \"\"\"Provide a fresh Calculator instance.\"\"\"\n", + " return Calculator()\n", + "\n", + "\n", + "@pytest.fixture\n", + "def fake_email_sender() -> FakeEmailSender:\n", + " \"\"\"Provide a fake email sender for testing.\"\"\"\n", + " return FakeEmailSender()\n", + "'''\n", + "\n", + "test_class_example = '''\n", + "# tests/test_calculator.py\n", + "#\n", + "# Test classes group related tests. No __init__ method needed.\n", + "\n", + "\n", + "class TestAdd:\n", + " \"\"\"Tests for Calculator.add().\"\"\"\n", + "\n", + " def test_positive_numbers(self, calculator: Calculator) -> None:\n", + " assert calculator.add(2, 3) == 5\n", + "\n", + " def test_negative_numbers(self, calculator: Calculator) -> None:\n", + " assert calculator.add(-2, -3) == -5\n", + "\n", + " def test_mixed_signs(self, calculator: Calculator) -> None:\n", + " assert calculator.add(-1, 1) == 0\n", + "\n", + "\n", + "class TestDivide:\n", + " \"\"\"Tests for Calculator.divide().\"\"\"\n", + "\n", + " def test_even_division(self, calculator: Calculator) -> None:\n", + " assert calculator.divide(10, 2) == 5.0\n", + "\n", + " def test_float_result(self, calculator: Calculator) -> None:\n", + " assert calculator.divide(7, 2) == 3.5\n", + "\n", + " def test_divide_by_zero_raises(self, calculator: Calculator) -> None:\n", + " with pytest.raises(ZeroDivisionError, match=\"Cannot divide by zero\"):\n", + " calculator.divide(1, 0)\n", + "'''\n", + "\n", + "print(project_layout)\n", + "print(\"conftest.py:\")\n", + "print(conftest_example)\n", + "print(\"Test classes:\")\n", + "print(test_class_example)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# pytest Command-Line Usage and Configuration\n", + "\n", + "pytest_config = \"\"\"\n", + "# pyproject.toml pytest configuration\n", + "\n", + "[tool.pytest.ini_options]\n", + "# Minimum pytest version required\n", + "minversion = \"7.0\"\n", + "\n", + "# Default command-line options\n", + "addopts = [\n", + " \"-ra\", # Show extra test summary for all except passed\n", + " \"--strict-markers\", # Error on unregistered markers\n", + " \"--tb=short\", # Shorter traceback format\n", + "]\n", + "\n", + "# Test discovery paths\n", + "testpaths = [\"tests\"]\n", + "\n", + "# Register custom markers\n", + "markers = [\n", + " \"slow: marks tests as slow (deselect with '-m not slow')\",\n", + " \"integration: marks integration tests\",\n", + "]\n", + "\"\"\"\n", + "\n", + "print(pytest_config)\n", + "\n", + "print(\"Common pytest commands:\")\n", + "print(\" pytest # Run all tests\")\n", + "print(\" pytest tests/test_calc.py # Run specific file\")\n", + "print(\" pytest -k 'test_add' # Run tests matching pattern\")\n", + "print(\" pytest -m 'not slow' # Skip tests marked as slow\")\n", + "print(\" pytest -x # Stop on first failure\")\n", + "print(\" pytest --lf # Rerun only last failures\")\n", + "print(\" pytest -v # Verbose output\")\n", + "print(\" pytest --co # Collect only (show what would run)\")\n", + "print(\" pytest --cov=src # Measure code coverage\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Tool / Concept | Purpose | Key Command |\n", + "|---------------|---------|-------------|\n", + "| **ruff** | Fast, unified linter | `ruff check .` |\n", + "| **ruff --fix** | Auto-fix lint issues | `ruff check --fix .` |\n", + "| **black / ruff format** | Auto-format code | `black .` or `ruff format .` |\n", + "| **isort / ruff I rules** | Sort imports | `isort .` or `ruff check --select I .` |\n", + "| **pytest** | Run tests | `pytest` |\n", + "| **@pytest.mark.parametrize** | Data-driven tests | Multiple inputs, one test function |\n", + "| **Fixtures** | Reusable setup/teardown | `@pytest.fixture` |\n", + "| **Factory fixtures** | Flexible object creation | Return a factory callable |\n", + "| **Protocol mocking** | Testable dependencies | Inject fake implementations |\n", + "| **conftest.py** | Shared fixtures | Auto-loaded by pytest |\n", + "\n", + "### Best Practices\n", + "- Use `ruff` as your single linting and formatting tool for simplicity\n", + "- Configure all tools in `pyproject.toml` for a single source of truth\n", + "- Write tests that are independent, deterministic, and fast\n", + "- Use `@pytest.mark.parametrize` to avoid duplicating test logic\n", + "- Prefer Protocol-based dependency injection over `mock.patch` for cleaner tests\n", + "- Organize tests to mirror your source code structure\n", + "- Use `conftest.py` for fixtures shared across multiple test files\n", + "- Run `pytest --cov` to track test coverage and find untested code" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_25/03_cicd_and_project_org.ipynb b/src/chapter_25/03_cicd_and_project_org.ipynb new file mode 100644 index 0000000..26c53cd --- /dev/null +++ b/src/chapter_25/03_cicd_and_project_org.ipynb @@ -0,0 +1,857 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 25: CI/CD and Project Organization\n", + "\n", + "A professional Python project needs more than just code. This notebook covers the\n", + "infrastructure around your code: pre-commit hooks, GitHub Actions CI/CD, project\n", + "structure, Makefiles, dependency management, and release workflows.\n", + "\n", + "## Topics Covered\n", + "- **Pre-commit hooks**: Configuration and common hooks\n", + "- **GitHub Actions**: Workflow YAML structure, jobs, steps\n", + "- **CI pipeline**: Lint, type-check, test (parallel jobs)\n", + "- **Branch protection** and required status checks\n", + "- **Project structure**: src-layout, tests, docs, Makefile\n", + "- **Makefile patterns**: Prereqs, install, check, test\n", + "- **Dependency management**: Lock files, security scanning\n", + "- **Release workflow**: Tagging, changelog, semantic versioning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-commit Hooks\n", + "\n", + "**Pre-commit** is a framework for managing git hook scripts that run automatically\n", + "before each commit. It catches issues early, before code enters the repository.\n", + "\n", + "How it works:\n", + "1. Install `pre-commit`: `pip install pre-commit`\n", + "2. Create a `.pre-commit-config.yaml` configuration file\n", + "3. Install the hooks: `pre-commit install`\n", + "4. Hooks run automatically on `git commit`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# .pre-commit-config.yaml: Typical configuration\n", + "\n", + "pre_commit_config = \"\"\"\n", + "# .pre-commit-config.yaml\n", + "repos:\n", + " # General file checks\n", + " - repo: https://github.com/pre-commit/pre-commit-hooks\n", + " rev: v5.0.0\n", + " hooks:\n", + " - id: trailing-whitespace # Remove trailing whitespace\n", + " - id: end-of-file-fixer # Ensure files end with newline\n", + " - id: check-yaml # Validate YAML syntax\n", + " - id: check-toml # Validate TOML syntax\n", + " - id: check-added-large-files # Prevent committing large files\n", + " args: ['--maxkb=500']\n", + " - id: check-merge-conflict # Detect unresolved merge markers\n", + " - id: debug-statements # Flag print()/pdb left in code\n", + "\n", + " # Ruff: linting and formatting\n", + " - repo: https://github.com/astral-sh/ruff-pre-commit\n", + " rev: v0.8.6\n", + " hooks:\n", + " - id: ruff # Lint\n", + " args: ['--fix'] # Auto-fix safe issues\n", + " - id: ruff-format # Format\n", + "\n", + " # Type checking with mypy\n", + " - repo: https://github.com/pre-commit/mirrors-mypy\n", + " rev: v1.14.1\n", + " hooks:\n", + " - id: mypy\n", + " additional_dependencies: [] # Add stubs here if needed\n", + "\"\"\"\n", + "\n", + "print(\"Typical .pre-commit-config.yaml:\")\n", + "print(pre_commit_config)\n", + "\n", + "print(\"Common pre-commit commands:\")\n", + "print(\" pre-commit install # Install hooks into .git/hooks\")\n", + "print(\" pre-commit run --all-files # Run all hooks on all files\")\n", + "print(\" pre-commit autoupdate # Update hook versions\")\n", + "print(\" git commit --no-verify # Skip hooks (emergency only)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Pre-commit hook lifecycle: what happens on git commit\n", + "\n", + "hook_lifecycle = \"\"\"\n", + "Pre-commit hook lifecycle:\n", + "\n", + " $ git add my_file.py\n", + " $ git commit -m \"Add feature\"\n", + " \n", + " [pre-commit] running hooks...\n", + " \n", + " Step 1: trailing-whitespace ..... Passed\n", + " Step 2: end-of-file-fixer ...... Passed\n", + " Step 3: check-yaml ............. Passed (skipped - no YAML files staged)\n", + " Step 4: ruff ................... Failed\n", + " - src/main.py:10:1 F401 'os' imported but unused\n", + " - Fixed 1 error (auto-fix applied)\n", + " Step 5: ruff-format ........... Failed\n", + " - 1 file reformatted\n", + " Step 6: mypy .................. Passed\n", + " \n", + " [pre-commit] Some hooks modified files.\n", + " [pre-commit] Commit aborted. Review changes and re-commit.\n", + " \n", + " $ git diff # See what hooks changed\n", + " $ git add my_file.py # Stage the fixes\n", + " $ git commit -m \"Add feature\" # Retry -- all hooks pass now\n", + " \n", + " [pre-commit] running hooks...\n", + " All hooks passed.\n", + " [main abc1234] Add feature\n", + "\"\"\"\n", + "\n", + "print(hook_lifecycle)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## GitHub Actions: CI/CD Workflows\n", + "\n", + "**GitHub Actions** is a CI/CD platform built into GitHub. Workflows are defined in\n", + "YAML files under `.github/workflows/`. Each workflow consists of **jobs** that contain\n", + "**steps**, and jobs can run in parallel or depend on each other.\n", + "\n", + "Key concepts:\n", + "- **Workflow**: Triggered by events (push, PR, schedule, etc.)\n", + "- **Job**: Runs on a virtual machine (runner)\n", + "- **Step**: Individual command or action within a job\n", + "- **Action**: Reusable unit of CI logic (e.g., `actions/checkout@v4`)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# GitHub Actions workflow: CI pipeline with parallel jobs\n", + "\n", + "ci_workflow = \"\"\"\n", + "# .github/workflows/ci.yml\n", + "name: CI\n", + "\n", + "on:\n", + " push:\n", + " branches: [main]\n", + " pull_request:\n", + " branches: [main]\n", + "\n", + "# Cancel in-progress runs for the same branch\n", + "concurrency:\n", + " group: ci-${{ github.ref }}\n", + " cancel-in-progress: true\n", + "\n", + "jobs:\n", + " # --- Job 1: Lint ---\n", + " lint:\n", + " name: Lint\n", + " runs-on: ubuntu-latest\n", + " steps:\n", + " - uses: actions/checkout@v4\n", + "\n", + " - name: Set up Python\n", + " uses: actions/setup-python@v5\n", + " with:\n", + " python-version: \"3.12\"\n", + "\n", + " - name: Install dependencies\n", + " run: |\n", + " python -m pip install --upgrade pip\n", + " pip install ruff\n", + "\n", + " - name: Run ruff check\n", + " run: ruff check .\n", + "\n", + " - name: Run ruff format check\n", + " run: ruff format --check .\n", + "\n", + " # --- Job 2: Type Check (runs in parallel with lint) ---\n", + " type-check:\n", + " name: Type Check\n", + " runs-on: ubuntu-latest\n", + " steps:\n", + " - uses: actions/checkout@v4\n", + "\n", + " - name: Set up Python\n", + " uses: actions/setup-python@v5\n", + " with:\n", + " python-version: \"3.12\"\n", + "\n", + " - name: Install dependencies\n", + " run: |\n", + " python -m pip install --upgrade pip\n", + " pip install mypy\n", + " pip install -e \".[dev]\"\n", + "\n", + " - name: Run mypy\n", + " run: mypy src/\n", + "\n", + " # --- Job 3: Test (runs in parallel with lint and type-check) ---\n", + " test:\n", + " name: Test (Python ${{ matrix.python-version }})\n", + " runs-on: ubuntu-latest\n", + " strategy:\n", + " matrix:\n", + " python-version: [\"3.11\", \"3.12\", \"3.13\"]\n", + " fail-fast: false\n", + "\n", + " steps:\n", + " - uses: actions/checkout@v4\n", + "\n", + " - name: Set up Python ${{ matrix.python-version }}\n", + " uses: actions/setup-python@v5\n", + " with:\n", + " python-version: ${{ matrix.python-version }}\n", + "\n", + " - name: Install dependencies\n", + " run: |\n", + " python -m pip install --upgrade pip\n", + " pip install -e \".[dev]\"\n", + "\n", + " - name: Run tests with coverage\n", + " run: pytest --cov=src --cov-report=xml\n", + "\n", + " - name: Upload coverage\n", + " if: matrix.python-version == '3.12'\n", + " uses: codecov/codecov-action@v4\n", + " with:\n", + " file: coverage.xml\n", + "\"\"\"\n", + "\n", + "print(\"GitHub Actions CI workflow:\")\n", + "print(ci_workflow)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Branch Protection and Required Status Checks\n", + "#\n", + "# Branch protection rules ensure code quality by requiring certain\n", + "# conditions before merging to protected branches.\n", + "\n", + "branch_protection = \"\"\"\n", + "Branch protection for 'main':\n", + "\n", + " Settings > Branches > Branch protection rules > main\n", + "\n", + " [x] Require a pull request before merging\n", + " [x] Require approvals: 1\n", + " [x] Dismiss stale pull request approvals when new commits are pushed\n", + "\n", + " [x] Require status checks to pass before merging\n", + " Required checks:\n", + " - lint (from CI workflow)\n", + " - type-check (from CI workflow)\n", + " - test (3.11) (from CI workflow matrix)\n", + " - test (3.12) (from CI workflow matrix)\n", + " - test (3.13) (from CI workflow matrix)\n", + "\n", + " [x] Require branches to be up to date before merging\n", + "\n", + " [x] Require conversation resolution before merging\n", + "\n", + " [ ] Include administrators (optional -- enforce rules on admins too)\n", + "\n", + "\n", + "Effect: A PR to main can only be merged when:\n", + " 1. At least 1 approval from a reviewer\n", + " 2. All CI checks pass (lint, type-check, all test matrix entries)\n", + " 3. Branch is up-to-date with main\n", + " 4. All review comments are resolved\n", + "\"\"\"\n", + "\n", + "print(branch_protection)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Project Structure: src-layout\n", + "\n", + "The **src-layout** is the recommended project structure for Python packages. It prevents\n", + "accidental imports from the source directory during testing and ensures your installed\n", + "package is what gets tested." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# src-layout project structure\n", + "\n", + "project_structure = \"\"\"\n", + "my-project/\n", + "|\n", + "|-- .github/\n", + "| |-- workflows/\n", + "| |-- ci.yml # CI pipeline\n", + "| |-- release.yml # Release automation\n", + "|\n", + "|-- src/\n", + "| |-- my_project/ # Package source code\n", + "| |-- __init__.py # Package init (version, public API)\n", + "| |-- core.py # Core business logic\n", + "| |-- models.py # Data models\n", + "| |-- utils.py # Utility functions\n", + "| |-- py.typed # PEP 561 marker for type info\n", + "|\n", + "|-- tests/\n", + "| |-- conftest.py # Shared test fixtures\n", + "| |-- test_core.py # Tests for core module\n", + "| |-- test_models.py # Tests for models module\n", + "| |-- test_utils.py # Tests for utils module\n", + "|\n", + "|-- docs/ # Documentation (optional)\n", + "| |-- index.md\n", + "| |-- api.md\n", + "|\n", + "|-- .pre-commit-config.yaml # Pre-commit hook configuration\n", + "|-- .gitignore # Git ignore rules\n", + "|-- .editorconfig # Editor configuration\n", + "|-- pyproject.toml # Project metadata, build config, tool config\n", + "|-- Makefile # Development task automation\n", + "|-- README.md # Project documentation\n", + "|-- LICENSE # License file\n", + "\"\"\"\n", + "\n", + "print(\"Recommended project structure (src-layout):\")\n", + "print(project_structure)\n", + "\n", + "print(\"Why src-layout?\")\n", + "print(\" 1. Prevents importing source directly during testing\")\n", + "print(\" 2. Forces you to install the package (pip install -e .)\")\n", + "print(\" 3. Tests run against the installed package, not loose files\")\n", + "print(\" 4. Matches how end users will import your code\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# pyproject.toml: The single source of truth\n", + "#\n", + "# pyproject.toml consolidates project metadata, build configuration,\n", + "# and tool settings into a single file.\n", + "\n", + "pyproject_example = \"\"\"\n", + "[build-system]\n", + "requires = [\"hatchling\"]\n", + "build-backend = \"hatchling.build\"\n", + "\n", + "[project]\n", + "name = \"my-project\"\n", + "version = \"1.2.0\"\n", + "description = \"A well-structured Python project\"\n", + "readme = \"README.md\"\n", + "license = {text = \"MIT\"}\n", + "requires-python = \">=3.11\"\n", + "authors = [\n", + " {name = \"Your Name\", email = \"you@example.com\"},\n", + "]\n", + "\n", + "# Runtime dependencies\n", + "dependencies = [\n", + " \"httpx>=0.27\",\n", + " \"pydantic>=2.0\",\n", + "]\n", + "\n", + "# Optional dependency groups\n", + "[project.optional-dependencies]\n", + "dev = [\n", + " \"pytest>=8.0\",\n", + " \"pytest-cov>=6.0\",\n", + " \"mypy>=1.13\",\n", + " \"ruff>=0.8\",\n", + " \"pre-commit>=4.0\",\n", + "]\n", + "docs = [\n", + " \"mkdocs>=1.6\",\n", + " \"mkdocs-material>=9.5\",\n", + "]\n", + "\n", + "# Entry points (CLI commands)\n", + "[project.scripts]\n", + "my-tool = \"my_project.cli:main\"\n", + "\n", + "# --- Tool configurations ---\n", + "\n", + "[tool.ruff]\n", + "target-version = \"py311\"\n", + "line-length = 88\n", + "\n", + "[tool.ruff.lint]\n", + "select = [\"E\", \"W\", \"F\", \"I\", \"UP\", \"B\", \"SIM\", \"RUF\"]\n", + "\n", + "[tool.mypy]\n", + "python_version = \"3.11\"\n", + "strict = true\n", + "warn_return_any = true\n", + "warn_unused_configs = true\n", + "\n", + "[tool.pytest.ini_options]\n", + "addopts = [\"-ra\", \"--strict-markers\", \"--tb=short\"]\n", + "testpaths = [\"tests\"]\n", + "\"\"\"\n", + "\n", + "print(\"pyproject.toml (comprehensive example):\")\n", + "print(pyproject_example)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Makefile Patterns\n", + "\n", + "A **Makefile** provides a simple, standard interface for common development tasks.\n", + "Developers can type `make test` or `make check` without memorizing tool-specific\n", + "commands. Makefiles use tabs for indentation (not spaces)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Makefile for a Python project\n", + "\n", + "makefile_content = \"\"\"\n", + "# Makefile\n", + ".PHONY: help install check lint format typecheck test clean\n", + "\n", + "# Default target: show help\n", + "help: ## Show this help message\n", + "\\t@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \\\\\n", + "\\t\\tsort | \\\\\n", + "\\t\\tawk 'BEGIN {FS = \":.*?## \"}; {printf \" \\\\033[36m%-15s\\\\033[0m %s\\\\n\", $$1, $$2}'\n", + "\n", + "install: ## Install the project and dev dependencies\n", + "\\tpython -m pip install --upgrade pip\n", + "\\tpip install -e \".[dev]\"\n", + "\\tpre-commit install\n", + "\n", + "lint: ## Run linter (ruff check)\n", + "\\truff check .\n", + "\n", + "format: ## Auto-format code (ruff format)\n", + "\\truff format .\n", + "\\truff check --fix .\n", + "\n", + "typecheck: ## Run type checker (mypy)\n", + "\\tmypy src/\n", + "\n", + "test: ## Run tests with coverage\n", + "\\tpytest --cov=src --cov-report=term-missing\n", + "\n", + "check: lint typecheck test ## Run all checks (lint + typecheck + test)\n", + "\n", + "clean: ## Remove build artifacts and caches\n", + "\\trm -rf build/ dist/ .eggs/ *.egg-info/\n", + "\\trm -rf .pytest_cache/ .mypy_cache/ .ruff_cache/\n", + "\\trm -rf htmlcov/ .coverage coverage.xml\n", + "\\tfind . -type d -name __pycache__ -exec rm -rf {} +\n", + "\"\"\"\n", + "\n", + "print(\"Makefile for Python development:\")\n", + "print(makefile_content)\n", + "\n", + "print(\"Usage:\")\n", + "print(\" make help # Show available targets\")\n", + "print(\" make install # Set up the development environment\")\n", + "print(\" make check # Run all quality checks\")\n", + "print(\" make format # Auto-format the codebase\")\n", + "print(\" make test # Run the test suite\")\n", + "print(\" make clean # Clean up generated files\")\n", + "print()\n", + "print(\"Key Makefile concepts:\")\n", + "print(\" .PHONY # Declare targets that are not files\")\n", + "print(\" target: prereqs # prereqs run before the target\")\n", + "print(\" check: lint typecheck test # 'check' depends on all three\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dependency Management\n", + "\n", + "Managing dependencies properly ensures reproducible builds, prevents supply chain\n", + "attacks, and avoids version conflicts. Modern Python offers several approaches\n", + "to dependency locking and security scanning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Dependency management strategies\n", + "\n", + "dep_management = \"\"\"\n", + "Dependency Management Strategies:\n", + "\n", + "1. ABSTRACT DEPENDENCIES (pyproject.toml)\n", + " - Specify minimum version constraints\n", + " - Used for libraries (consumed by other packages)\n", + " \n", + " dependencies = [\n", + " \"httpx>=0.27\", # Minimum version\n", + " \"pydantic>=2.0,<3\", # Version range\n", + " ]\n", + "\n", + "2. LOCK FILES (pip-tools, uv, poetry)\n", + " - Pin exact versions for reproducible installs\n", + " - Used for applications (deployed directly)\n", + " \n", + " # Using pip-tools:\n", + " pip-compile pyproject.toml -o requirements.lock\n", + " pip install -r requirements.lock\n", + " \n", + " # Using uv (faster alternative):\n", + " uv lock\n", + " uv sync\n", + "\n", + "3. LOCK FILE EXAMPLE (requirements.lock)\n", + " # Generated by pip-compile -- DO NOT EDIT\n", + " certifi==2024.12.14\n", + " h11==0.14.0\n", + " httpcore==1.0.7\n", + " httpx==0.28.1\n", + " idna==3.10\n", + " pydantic==2.10.4\n", + " pydantic-core==2.27.2\n", + "\"\"\"\n", + "\n", + "print(dep_management)\n", + "\n", + "security = \"\"\"\n", + "Dependency Security Scanning:\n", + "\n", + " # pip-audit: check for known vulnerabilities\n", + " pip install pip-audit\n", + " pip-audit\n", + " \n", + " # GitHub Dependabot (automatic):\n", + " # .github/dependabot.yml\n", + " version: 2\n", + " updates:\n", + " - package-ecosystem: pip\n", + " directory: \"/\"\n", + " schedule:\n", + " interval: weekly\n", + " open-pull-requests-limit: 10\n", + " \n", + " # Safety check in CI:\n", + " - name: Security audit\n", + " run: pip-audit --strict\n", + "\"\"\"\n", + "\n", + "print(security)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Release Workflow: Tagging and Semantic Versioning\n", + "\n", + "**Semantic Versioning** (SemVer) uses a `MAJOR.MINOR.PATCH` scheme:\n", + "- **MAJOR**: Breaking changes (incompatible API changes)\n", + "- **MINOR**: New features (backward-compatible)\n", + "- **PATCH**: Bug fixes (backward-compatible)\n", + "\n", + "A release workflow automates tagging, changelog generation, and package publishing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Semantic versioning examples\n", + "\n", + "semver_examples = \"\"\"\n", + "Semantic Versioning (SemVer): MAJOR.MINOR.PATCH\n", + "\n", + " 1.0.0 Initial stable release\n", + " 1.0.1 Bug fix (patch)\n", + " 1.1.0 New feature added (minor)\n", + " 1.1.1 Bug fix in new feature (patch)\n", + " 2.0.0 Breaking API change (major)\n", + " 2.0.0-rc.1 Release candidate (pre-release)\n", + "\n", + "When to bump each number:\n", + " PATCH (1.0.x): Bug fixes, documentation updates\n", + " MINOR (1.x.0): New features, deprecations\n", + " MAJOR (x.0.0): Removed features, changed APIs, breaking changes\n", + "\n", + "Pre-release versions:\n", + " 1.0.0-alpha.1 Early development\n", + " 1.0.0-beta.1 Feature complete, testing\n", + " 1.0.0-rc.1 Release candidate\n", + "\"\"\"\n", + "\n", + "print(semver_examples)\n", + "\n", + "# Working with git tags for releases\n", + "print(\"Git tagging for releases:\")\n", + "print(\" git tag v1.2.0 # Create a lightweight tag\")\n", + "print(\" git tag -a v1.2.0 -m 'Release 1.2.0' # Create an annotated tag\")\n", + "print(\" git push origin v1.2.0 # Push tag to remote\")\n", + "print(\" git push origin --tags # Push all tags\")\n", + "print(\" git tag -l 'v1.*' # List matching tags\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# GitHub Actions release workflow\n", + "\n", + "release_workflow = \"\"\"\n", + "# .github/workflows/release.yml\n", + "name: Release\n", + "\n", + "on:\n", + " push:\n", + " tags:\n", + " - 'v*' # Trigger on version tags (v1.0.0, v2.1.3, etc.)\n", + "\n", + "permissions:\n", + " contents: write # Needed to create GitHub releases\n", + " id-token: write # Needed for trusted publishing to PyPI\n", + "\n", + "jobs:\n", + " # First, run all CI checks\n", + " ci:\n", + " uses: ./.github/workflows/ci.yml # Reuse the CI workflow\n", + "\n", + " # Then build and publish\n", + " release:\n", + " name: Build and Publish\n", + " needs: ci # Only run after CI passes\n", + " runs-on: ubuntu-latest\n", + "\n", + " steps:\n", + " - uses: actions/checkout@v4\n", + "\n", + " - name: Set up Python\n", + " uses: actions/setup-python@v5\n", + " with:\n", + " python-version: \"3.12\"\n", + "\n", + " - name: Install build tools\n", + " run: pip install build\n", + "\n", + " - name: Build package\n", + " run: python -m build\n", + "\n", + " - name: Create GitHub Release\n", + " uses: softprops/action-gh-release@v2\n", + " with:\n", + " files: dist/*\n", + " generate_release_notes: true\n", + "\n", + " - name: Publish to PyPI\n", + " uses: pypa/gh-action-pypi-publish@release/v1\n", + " # Uses trusted publishing (no API token needed)\n", + "\"\"\"\n", + "\n", + "print(\"GitHub Actions release workflow:\")\n", + "print(release_workflow)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Changelog management: keeping users informed\n", + "\n", + "changelog_example = \"\"\"\n", + "# CHANGELOG.md\n", + "#\n", + "# Format based on Keep a Changelog (https://keepachangelog.com)\n", + "\n", + "## [Unreleased]\n", + "### Added\n", + "- Support for Python 3.13\n", + "\n", + "## [1.2.0] - 2025-12-15\n", + "### Added\n", + "- New `export_csv()` method on Report class\n", + "- CLI flag `--output-format` for choosing export format\n", + "\n", + "### Changed\n", + "- Improved error messages for invalid configuration files\n", + "\n", + "### Fixed\n", + "- Race condition in concurrent file processing (#142)\n", + "\n", + "## [1.1.0] - 2025-09-01\n", + "### Added\n", + "- Initial support for async operations\n", + "- Configuration file validation\n", + "\n", + "### Deprecated\n", + "- `process_sync()` -- use `process()` with `async=False` instead\n", + "\n", + "## [1.0.0] - 2025-06-01\n", + "### Added\n", + "- Initial stable release\n", + "- Core processing pipeline\n", + "- CLI interface\n", + "- Full documentation\n", + "\"\"\"\n", + "\n", + "print(\"Changelog (Keep a Changelog format):\")\n", + "print(changelog_example)\n", + "\n", + "print(\"Release process summary:\")\n", + "print(\" 1. Update version in pyproject.toml\")\n", + "print(\" 2. Update CHANGELOG.md (move Unreleased -> new version)\")\n", + "print(\" 3. Commit: git commit -m 'Release v1.2.0'\")\n", + "print(\" 4. Tag: git tag -a v1.2.0 -m 'Release 1.2.0'\")\n", + "print(\" 5. Push: git push origin main --tags\")\n", + "print(\" 6. CI runs, then release workflow builds and publishes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Putting it all together: the development workflow\n", + "\n", + "full_workflow = \"\"\"\n", + "Complete Development Workflow:\n", + "\n", + " SETUP (one time)\n", + " ================\n", + " $ git clone https://github.com/user/my-project.git\n", + " $ cd my-project\n", + " $ python -m venv .venv\n", + " $ source .venv/bin/activate\n", + " $ make install # pip install -e \".[dev]\" + pre-commit install\n", + "\n", + " DAILY DEVELOPMENT\n", + " =================\n", + " $ git checkout -b feature/my-feature # Create feature branch\n", + " \n", + " # ... write code and tests ...\n", + " \n", + " $ make check # Run lint + typecheck + tests locally\n", + " $ git add -p # Stage changes interactively\n", + " $ git commit # Pre-commit hooks run automatically\n", + " $ git push -u origin feature/my-feature\n", + " \n", + " # Open a Pull Request on GitHub\n", + " # CI runs lint, type-check, and tests in parallel\n", + " # Reviewer approves\n", + " # Merge to main\n", + "\n", + " RELEASE\n", + " =======\n", + " $ vim pyproject.toml # Bump version\n", + " $ vim CHANGELOG.md # Update changelog\n", + " $ git commit -m 'Release v1.2.0'\n", + " $ git tag -a v1.2.0 -m 'Release 1.2.0'\n", + " $ git push origin main --tags\n", + " # Release workflow builds, tests, and publishes automatically\n", + "\"\"\"\n", + "\n", + "print(full_workflow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Key Takeaways\n", + "\n", + "| Area | Tool / Practice | Purpose |\n", + "|------|----------------|----------|\n", + "| **Pre-commit** | `.pre-commit-config.yaml` | Catch issues before they enter the repo |\n", + "| **CI/CD** | GitHub Actions (`.github/workflows/`) | Automated lint, type-check, test |\n", + "| **Parallel jobs** | `lint`, `type-check`, `test` | Fast feedback on PRs |\n", + "| **Matrix testing** | `strategy.matrix` | Test across Python versions |\n", + "| **Branch protection** | Required status checks | Enforce quality gates |\n", + "| **src-layout** | `src/my_project/` | Correct package structure |\n", + "| **pyproject.toml** | Single config file | Metadata + all tool configs |\n", + "| **Makefile** | `make check`, `make test` | Standard developer interface |\n", + "| **Lock files** | `pip-compile`, `uv lock` | Reproducible installs |\n", + "| **Security** | `pip-audit`, Dependabot | Vulnerability scanning |\n", + "| **SemVer** | `MAJOR.MINOR.PATCH` | Communicate change impact |\n", + "| **Release** | Git tags + GitHub Actions | Automated build and publish |\n", + "\n", + "### Best Practices\n", + "- Install pre-commit hooks as the first step when joining a project\n", + "- Run `make check` locally before pushing to catch issues early\n", + "- Keep CI jobs parallel to minimize feedback time on pull requests\n", + "- Use the src-layout to prevent import confusion during testing\n", + "- Consolidate all configuration in `pyproject.toml` where possible\n", + "- Pin dependencies with lock files for applications, use ranges for libraries\n", + "- Follow semantic versioning to communicate the impact of changes\n", + "- Automate releases with CI/CD to eliminate manual packaging errors" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_25/README.md b/src/chapter_25/README.md new file mode 100644 index 0000000..20ff03c --- /dev/null +++ b/src/chapter_25/README.md @@ -0,0 +1,22 @@ +# Chapter 25: Python Ecosystem and Best Practices + +## Topics Covered +- Code style: PEP 8, naming conventions, code organization +- Documentation: docstrings, PEP 257, Sphinx basics +- Linting and formatting: ruff, black, flake8, isort +- Testing best practices: pytest patterns, fixtures, parametrize +- CI/CD pipelines: GitHub Actions, pre-commit hooks +- Project organization: monorepo vs multi-repo, src-layout +- Dependency management: pinning, lock files, security updates +- Python version management and compatibility + +## Notebooks +1. **01_code_style_and_docs.ipynb** — PEP 8, naming conventions, docstrings, documentation +2. **02_linting_and_testing.ipynb** — Ruff, formatters, pytest patterns, parametrize +3. **03_cicd_and_project_org.ipynb** — GitHub Actions, pre-commit, project structure, workflows + +## Key Takeaways +- Consistent code style reduces cognitive overhead +- Automated linting catches issues before review +- pytest fixtures and parametrize reduce test boilerplate +- CI/CD ensures quality gates are never skipped diff --git a/src/chapter_25/__init__.py b/src/chapter_25/__init__.py new file mode 100644 index 0000000..6880eac --- /dev/null +++ b/src/chapter_25/__init__.py @@ -0,0 +1 @@ +"""Chapter 25: Python Ecosystem and Best Practices.""" diff --git a/tests/test_chapter_21.py b/tests/test_chapter_21.py new file mode 100644 index 0000000..4a582cb --- /dev/null +++ b/tests/test_chapter_21.py @@ -0,0 +1,75 @@ +"""Tests for Chapter 21: Logging and Debugging.""" + +import logging +import traceback + + +class TestLoggingFundamentals: + """Test logging module basics.""" + + def test_log_levels_ordering(self) -> None: + """Log levels have a numeric ordering.""" + assert logging.DEBUG < logging.INFO + assert logging.INFO < logging.WARNING + assert logging.WARNING < logging.ERROR + assert logging.ERROR < logging.CRITICAL + + def test_logger_creation(self) -> None: + """getLogger returns a named logger.""" + logger = logging.getLogger("test.module") + assert logger.name == "test.module" + assert isinstance(logger, logging.Logger) + + def test_handler_and_formatter(self) -> None: + """Handlers format and route log records.""" + logger = logging.getLogger("test.handler") + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler() + formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s") + handler.setFormatter(formatter) + + record = logger.makeRecord("test.handler", logging.INFO, "", 0, "hello", (), None) + formatted = handler.format(record) + assert "INFO" in formatted + assert "hello" in formatted + + def test_logger_hierarchy(self) -> None: + """Child loggers inherit from parent loggers.""" + parent = logging.getLogger("myapp") + child = logging.getLogger("myapp.module") + assert child.parent is parent + + +class TestDebugging: + """Test debugging utilities.""" + + def test_traceback_format(self) -> None: + """traceback module formats exception information.""" + try: + raise ValueError("test error") + except ValueError: + tb = traceback.format_exc() + assert "ValueError" in tb + assert "test error" in tb + + def test_traceback_extract(self) -> None: + """traceback.extract_stack captures the call stack.""" + stack = traceback.extract_stack() + assert len(stack) > 0 + # Last frame is this function + assert "test_traceback_extract" in stack[-1].name + + def test_breakpoint_function_exists(self) -> None: + """breakpoint() is a built-in function.""" + assert callable(breakpoint) + + def test_exception_chaining(self) -> None: + """Exception chaining preserves the original cause.""" + try: + try: + raise KeyError("original") + except KeyError as e: + raise ValueError("wrapper") from e + except ValueError as e: + assert isinstance(e.__cause__, KeyError) + assert str(e.__cause__) == "'original'" diff --git a/tests/test_chapter_22.py b/tests/test_chapter_22.py new file mode 100644 index 0000000..51391f9 --- /dev/null +++ b/tests/test_chapter_22.py @@ -0,0 +1,116 @@ +"""Tests for Chapter 22: Web Development Fundamentals.""" + +import http +import json +from urllib.parse import parse_qs, urlencode + + +class TestHTTPConcepts: + """Test HTTP fundamentals.""" + + def test_http_status_codes(self) -> None: + """HTTPStatus provides standard status codes.""" + assert http.HTTPStatus.OK == 200 + assert http.HTTPStatus.NOT_FOUND == 404 + assert http.HTTPStatus.INTERNAL_SERVER_ERROR == 500 + assert http.HTTPStatus.OK.phrase == "OK" + + def test_http_methods(self) -> None: + """Standard HTTP methods are well-defined strings.""" + methods = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} + assert "GET" in methods + assert "POST" in methods + + +class TestWSGI: + """Test WSGI concepts.""" + + def test_wsgi_app_callable(self) -> None: + """A WSGI app is a callable that takes environ and start_response.""" + responses: list[tuple[str, list]] = [] + + def start_response(status: str, headers: list) -> None: + responses.append((status, headers)) + + def app(environ: dict, start_response) -> list[bytes]: + start_response("200 OK", [("Content-Type", "text/plain")]) + return [b"Hello, World!"] + + environ = {"REQUEST_METHOD": "GET", "PATH_INFO": "/"} + body = app(environ, start_response) + + assert body == [b"Hello, World!"] + assert responses[0][0] == "200 OK" + + def test_wsgi_environ_keys(self) -> None: + """WSGI environ contains standard CGI variables.""" + environ = { + "REQUEST_METHOD": "GET", + "PATH_INFO": "/api/users", + "QUERY_STRING": "page=1&limit=10", + "SERVER_NAME": "localhost", + "SERVER_PORT": "8000", + } + assert environ["REQUEST_METHOD"] == "GET" + assert environ["PATH_INFO"] == "/api/users" + params = parse_qs(environ["QUERY_STRING"]) + assert params["page"] == ["1"] + + +class TestRouting: + """Test URL routing patterns.""" + + def test_simple_router(self) -> None: + """A router maps URL paths to handler functions.""" + routes: dict[str, callable] = {} + + def route(path: str): + def decorator(func): + routes[path] = func + return func + + return decorator + + @route("/") + def index() -> str: + return "Home" + + @route("/about") + def about() -> str: + return "About" + + assert routes["/"]() == "Home" + assert routes["/about"]() == "About" + assert "/missing" not in routes + + def test_json_response_building(self) -> None: + """JSON responses combine status, headers, and body.""" + data = {"users": [{"name": "Alice"}, {"name": "Bob"}]} + body: bytes = json.dumps(data).encode("utf-8") + headers = [ + ("Content-Type", "application/json"), + ("Content-Length", str(len(body))), + ] + + assert json.loads(body) == data + assert any(k == "Content-Type" for k, v in headers) + + +class TestFormHandling: + """Test form data processing.""" + + def test_url_encoded_form_data(self) -> None: + """URL-encoded form data is parsed with parse_qs.""" + form_data = "username=alice&password=secret123&remember=on" + parsed = parse_qs(form_data) + + assert parsed["username"] == ["alice"] + assert parsed["password"] == ["secret123"] + assert parsed["remember"] == ["on"] + + def test_form_data_encoding(self) -> None: + """urlencode builds form-encoded strings from dicts.""" + data = {"name": "Alice Bob", "email": "alice@example.com"} + encoded = urlencode(data) + assert "name=Alice+Bob" in encoded + assert "email=alice%40example.com" in encoded diff --git a/tests/test_chapter_23.py b/tests/test_chapter_23.py new file mode 100644 index 0000000..d990d5a --- /dev/null +++ b/tests/test_chapter_23.py @@ -0,0 +1,94 @@ +"""Tests for Chapter 23: Security and Cryptography.""" + +import hashlib +import hmac +import secrets + + +class TestHashing: + """Test hashlib and hashing patterns.""" + + def test_sha256_digest(self) -> None: + """SHA-256 produces a 64-character hex digest.""" + digest = hashlib.sha256(b"hello world").hexdigest() + assert len(digest) == 64 + # Same input always produces same hash + assert digest == hashlib.sha256(b"hello world").hexdigest() + + def test_different_inputs_different_hashes(self) -> None: + """Different inputs produce different hashes.""" + h1 = hashlib.sha256(b"hello").hexdigest() + h2 = hashlib.sha256(b"world").hexdigest() + assert h1 != h2 + + def test_blake2b_digest(self) -> None: + """BLAKE2b is a fast, secure hash function.""" + h = hashlib.blake2b(b"hello", digest_size=32) + assert len(h.hexdigest()) == 64 + + def test_incremental_hashing(self) -> None: + """update() allows incremental hashing of large data.""" + h1 = hashlib.sha256(b"helloworld").hexdigest() + + h2 = hashlib.sha256() + h2.update(b"hello") + h2.update(b"world") + + assert h1 == h2.hexdigest() + + +class TestHMAC: + """Test HMAC message authentication.""" + + def test_hmac_creation(self) -> None: + """HMAC combines a key with a message hash.""" + key = b"secret-key" + message = b"important data" + mac = hmac.new(key, message, hashlib.sha256) + assert len(mac.hexdigest()) == 64 + + def test_hmac_verification(self) -> None: + """HMAC verification uses constant-time comparison.""" + key = b"secret-key" + message = b"important data" + mac1 = hmac.new(key, message, hashlib.sha256).hexdigest() + mac2 = hmac.new(key, message, hashlib.sha256).hexdigest() + assert hmac.compare_digest(mac1, mac2) + + def test_hmac_different_keys(self) -> None: + """Different keys produce different MACs.""" + message = b"same message" + mac1 = hmac.new(b"key1", message, hashlib.sha256).hexdigest() + mac2 = hmac.new(b"key2", message, hashlib.sha256).hexdigest() + assert mac1 != mac2 + + +class TestSecrets: + """Test secrets module for secure random values.""" + + def test_token_hex(self) -> None: + """token_hex generates a random hex string.""" + token = secrets.token_hex(32) + assert len(token) == 64 # 32 bytes = 64 hex chars + assert isinstance(token, str) + + def test_token_bytes(self) -> None: + """token_bytes generates random bytes.""" + token = secrets.token_bytes(16) + assert len(token) == 16 + assert isinstance(token, bytes) + + def test_token_urlsafe(self) -> None: + """token_urlsafe generates URL-safe base64 tokens.""" + token = secrets.token_urlsafe(32) + assert isinstance(token, str) + # URL-safe characters only + safe_chars = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_") + assert all(c in safe_chars for c in token) + + def test_compare_digest_constant_time(self) -> None: + """compare_digest prevents timing attacks.""" + a = "correct_token" + b = "correct_token" + assert secrets.compare_digest(a, b) + assert not secrets.compare_digest(a, "wrong_token") diff --git a/tests/test_chapter_24.py b/tests/test_chapter_24.py new file mode 100644 index 0000000..073adf0 --- /dev/null +++ b/tests/test_chapter_24.py @@ -0,0 +1,160 @@ +"""Tests for Chapter 24: Metaprogramming.""" + +import inspect + + +class TestDynamicAttributes: + """Test dynamic attribute access.""" + + def test_getattr_fallback(self) -> None: + """__getattr__ is called when normal lookup fails.""" + + class DefaultDict: + def __init__(self, **kwargs: str) -> None: + self.__dict__.update(kwargs) + + def __getattr__(self, name: str) -> str: + return f"default_{name}" + + obj = DefaultDict(color="red") + assert obj.color == "red" + assert obj.missing == "default_missing" + + def test_setattr_validation(self) -> None: + """__setattr__ can validate attribute assignments.""" + + class Validated: + def __setattr__(self, name: str, value: object) -> None: + if name == "age" and isinstance(value, int) and value < 0: + raise ValueError("age must be non-negative") + super().__setattr__(name, value) + + v = Validated() + v.age = 25 + assert v.age == 25 + + try: + v.age = -1 + assert False, "Should have raised ValueError" + except ValueError: + pass + + def test_property_dynamic(self) -> None: + """Properties can be created dynamically.""" + + class Circle: + def __init__(self, radius: float) -> None: + self._radius = radius + + @property + def radius(self) -> float: + return self._radius + + @radius.setter + def radius(self, value: float) -> None: + if value < 0: + raise ValueError("radius must be non-negative") + self._radius = value + + @property + def area(self) -> float: + return 3.14159 * self._radius**2 + + c = Circle(5.0) + assert c.radius == 5.0 + assert abs(c.area - 78.53975) < 0.001 + + +class TestClassDecorators: + """Test class decorator patterns.""" + + def test_class_decorator_adds_method(self) -> None: + """Class decorators can add methods to classes.""" + + def add_repr(cls): + def __repr__(self) -> str: + attrs = ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items()) + return f"{cls.__name__}({attrs})" + + cls.__repr__ = __repr__ + return cls + + @add_repr + class Point: + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y + + p = Point(1.0, 2.0) + assert "Point(" in repr(p) + assert "x=1.0" in repr(p) + + def test_init_subclass_registration(self) -> None: + """__init_subclass__ registers subclasses automatically.""" + registry: list[type] = [] + + class Plugin: + def __init_subclass__(cls, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + registry.append(cls) + + class AuthPlugin(Plugin): + pass + + class CachePlugin(Plugin): + pass + + assert len(registry) == 2 + assert AuthPlugin in registry + assert CachePlugin in registry + + +class TestIntrospection: + """Test inspect module capabilities.""" + + def test_inspect_signature(self) -> None: + """inspect.signature reveals function parameters.""" + + def greet(name: str, greeting: str = "Hello") -> str: + return f"{greeting}, {name}!" + + sig = inspect.signature(greet) + params = list(sig.parameters.keys()) + assert params == ["name", "greeting"] + assert sig.parameters["greeting"].default == "Hello" + + def test_inspect_members(self) -> None: + """inspect.getmembers lists object attributes.""" + + class MyClass: + x: int = 10 + + def method(self) -> None: + pass + + methods = [name for name, _ in inspect.getmembers(MyClass, predicate=inspect.isfunction)] + assert "method" in methods + + def test_inspect_source(self) -> None: + """inspect.getsource retrieves source code.""" + + def add(a: int, b: int) -> int: + return a + b + + source = inspect.getsource(add) + assert "def add" in source + assert "return a + b" in source + + def test_inspect_isclass_isfunction(self) -> None: + """inspect provides type-checking predicates.""" + + class Foo: + pass + + def bar() -> None: + pass + + assert inspect.isclass(Foo) + assert inspect.isfunction(bar) + assert not inspect.isclass(bar) + assert not inspect.isfunction(Foo) diff --git a/tests/test_chapter_25.py b/tests/test_chapter_25.py new file mode 100644 index 0000000..cfe016b --- /dev/null +++ b/tests/test_chapter_25.py @@ -0,0 +1,126 @@ +"""Tests for Chapter 25: Python Ecosystem and Best Practices.""" + +import ast +import textwrap +from typing import Protocol + + +class TestCodeStyle: + """Test code style concepts.""" + + def test_pep8_naming_conventions(self) -> None: + """PEP 8 naming conventions are standard Python style.""" + # snake_case for functions and variables + my_variable = 42 + assert my_variable == 42 + + # PascalCase for classes + class MyClass: + pass + + assert MyClass.__name__ == "MyClass" + + # UPPER_SNAKE_CASE for constants + MAX_RETRIES = 3 + assert MAX_RETRIES == 3 + + def test_docstring_access(self) -> None: + """Docstrings are accessible via __doc__.""" + + def my_function() -> str: + """This function returns a greeting.""" + return "hello" + + assert my_function.__doc__ is not None + assert "greeting" in my_function.__doc__ + + def test_textwrap_dedent(self) -> None: + """textwrap.dedent cleans up indented strings.""" + indented = """\ + Line 1 + Line 2 + Line 3""" + cleaned = textwrap.dedent(indented) + assert cleaned == "Line 1\nLine 2\nLine 3" + + +class TestTestingPatterns: + """Test pytest patterns and best practices.""" + + def test_parametrize_concept(self) -> None: + """Parameterized tests cover multiple cases.""" + cases = [ + (2, 3, 5), + (0, 0, 0), + (-1, 1, 0), + (100, 200, 300), + ] + for a, b, expected in cases: + assert a + b == expected + + def test_fixture_concept(self) -> None: + """Fixtures provide reusable test setup.""" + + def create_user(name: str = "Alice", age: int = 30) -> dict: + return {"name": name, "age": age, "active": True} + + user = create_user() + assert user["name"] == "Alice" + assert user["active"] is True + + custom_user = create_user("Bob", 25) + assert custom_user["name"] == "Bob" + + def test_protocol_for_testability(self) -> None: + """Protocols enable easy mocking and testing.""" + + class DataStore(Protocol): + def get(self, key: str) -> str | None: ... + def put(self, key: str, value: str) -> None: ... + + class InMemoryStore: + def __init__(self) -> None: + self._data: dict[str, str] = {} + + def get(self, key: str) -> str | None: + return self._data.get(key) + + def put(self, key: str, value: str) -> None: + self._data[key] = value + + store: DataStore = InMemoryStore() + store.put("key1", "value1") + assert store.get("key1") == "value1" + assert store.get("missing") is None + + +class TestASTAndCodeAnalysis: + """Test AST-based code analysis.""" + + def test_ast_parse(self) -> None: + """ast.parse creates an AST from source code.""" + source = "x = 1 + 2" + tree = ast.parse(source) + assert isinstance(tree, ast.Module) + assert len(tree.body) == 1 + assert isinstance(tree.body[0], ast.Assign) + + def test_ast_walk_finds_functions(self) -> None: + """ast.walk traverses all nodes in the AST.""" + source = textwrap.dedent("""\ + def foo(): + pass + def bar(): + pass + """) + tree = ast.parse(source) + functions = [node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)] + assert functions == ["foo", "bar"] + + def test_compile_and_exec(self) -> None: + """Compiled AST can be executed.""" + source = "result = 2 ** 10" + code = compile(source, "", "exec") + namespace: dict = {} + exec(code, namespace) + assert namespace["result"] == 1024