diff --git a/README.md b/README.md index 62a910e..5c4f965 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,11 @@ src/ ├── chapter_28/ # Command-Line Interfaces ├── chapter_29/ # Date, Time, and Scheduling ├── chapter_30/ # XML, HTML, and Data Formats +├── chapter_31/ # String Methods and Formatting +├── chapter_32/ # Numeric Computing +├── chapter_33/ # OS and System Interaction +├── chapter_34/ # Email and Data Encoding +├── chapter_35/ # Advanced Python Patterns └── ... tests/ ├── conftest.py # Shared pytest fixtures @@ -136,6 +141,11 @@ tests/ | 28 | Command-Line Interfaces | argparse, subcommands, environment variables | Done | | 29 | Date, Time, and Scheduling | datetime, zoneinfo, timedelta, calendar, formatting | Done | | 30 | XML, HTML, and Data Formats | xml.etree, html.parser, configparser, struct | Done | +| 31 | String Methods and Formatting | f-strings, str methods, string module, difflib, textwrap | Done | +| 32 | Numeric Computing | math, decimal, fractions, random, statistics | Done | +| 33 | OS and System Interaction | os, shutil, tempfile, platform, subprocess | Done | +| 34 | Email and Data Encoding | email.message, base64, quopri, mimetypes, binascii | Done | +| 35 | Advanced Python Patterns | Descriptors, `__slots__`, weakrefs, ABCs, copy protocol | Done | ## Running Notebooks diff --git a/src/chapter_31/01_fstrings_and_formatting.ipynb b/src/chapter_31/01_fstrings_and_formatting.ipynb new file mode 100644 index 0000000..04c9e8b --- /dev/null +++ b/src/chapter_31/01_fstrings_and_formatting.ipynb @@ -0,0 +1,431 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 31: F-Strings and Formatting\n", + "\n", + "This notebook covers Python's string formatting tools, from modern f-strings to the `str.format()` method and the format specification mini-language. You will learn how to embed expressions, align text, pad output, and format numbers precisely.\n", + "\n", + "## Key Concepts\n", + "- **F-strings**: Inline expression evaluation with `f\"...\"`\n", + "- **Format spec mini-language**: Control alignment, padding, width, precision, and type\n", + "- **Number formatting**: Comma separators, fixed-point, binary, hex, percentage\n", + "- **`str.format()`**: Positional and keyword argument substitution\n", + "- **Debugging with `=`**: The `f\"{expr=}\"` shorthand for quick inspection" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: F-String Basics\n", + "\n", + "F-strings (formatted string literals) evaluate expressions inside curly braces at runtime. They were introduced in Python 3.6 and are the preferred way to embed values in strings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# Basic variable interpolation\n", + "name: str = \"Alice\"\n", + "age: int = 30\n", + "\n", + "greeting: str = f\"Hello, {name}!\"\n", + "info: str = f\"{name} is {age} years old.\"\n", + "\n", + "print(greeting)\n", + "print(info)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "# F-strings evaluate arbitrary expressions\n", + "print(f\"2 + 3 = {2 + 3}\")\n", + "print(f\"Uppercase: {'hello'.upper()}\")\n", + "print(f\"Length of 'python': {len('python')}\")\n", + "\n", + "# Conditional expressions work too\n", + "x: int = 42\n", + "print(f\"{x} is {'even' if x % 2 == 0 else 'odd'}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "# The = specifier shows both the expression and its value (Python 3.8+)\n", + "name: str = \"Alice\"\n", + "scores: list[int] = [85, 92, 78]\n", + "\n", + "print(f\"{name=}\")\n", + "print(f\"{len(scores)=}\")\n", + "print(f\"{sum(scores) / len(scores)=:.1f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 2: Format Spec Mini-Language -- Alignment and Padding\n", + "\n", + "After the colon inside an f-string brace, you can provide a format specification. The general form is:\n", + "\n", + "```\n", + "{value:[[fill]align][width][.precision][type]}\n", + "```\n", + "\n", + "Alignment characters:\n", + "- `<` -- left-align (default for strings)\n", + "- `>` -- right-align (default for numbers)\n", + "- `^` -- center-align" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "# Alignment with a fixed width\n", + "word: str = \"left\"\n", + "print(f\"'{word:<10}'\") # left-align in 10 chars\n", + "print(f\"'{word:>10}'\") # right-align in 10 chars\n", + "print(f\"'{word:^10}'\") # center in 10 chars" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "# Custom fill character (placed before the alignment character)\n", + "print(f\"{'fill':*^10}\") # center with * fill\n", + "print(f\"{'right':->10}\") # right-align with - fill\n", + "print(f\"{'left':.<10}\") # left-align with . fill" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "# Practical example: building a formatted table\n", + "headers: list[str] = [\"Name\", \"Score\", \"Grade\"]\n", + "rows: list[tuple[str, int, str]] = [\n", + " (\"Alice\", 95, \"A\"),\n", + " (\"Bob\", 82, \"B\"),\n", + " (\"Charlie\", 78, \"C+\"),\n", + "]\n", + "\n", + "# Print header\n", + "print(f\"{headers[0]:<10} {headers[1]:>5} {headers[2]:>5}\")\n", + "print(\"-\" * 22)\n", + "\n", + "# Print rows\n", + "for name, score, grade in rows:\n", + " print(f\"{name:<10} {score:>5} {grade:>5}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 3: Number Formatting\n", + "\n", + "Format specs provide fine control over numeric display: fixed-point decimals, thousands separators, percentages, and alternative bases." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "# Fixed-point and precision\n", + "pi: float = 3.14159265358979\n", + "\n", + "print(f\"Default: {pi}\")\n", + "print(f\"2 decimals: {pi:.2f}\")\n", + "print(f\"4 decimals: {pi:.4f}\")\n", + "print(f\"0 decimals: {pi:.0f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "# Thousands separators\n", + "big_number: int = 1000000\n", + "print(f\"Comma separator: {big_number:,}\")\n", + "print(f\"Underscore separator: {big_number:_}\")\n", + "\n", + "# Combining width, fill, separator, and precision\n", + "price: float = 1234.5\n", + "print(f\"\\nFormatted price: ${price:>12,.2f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "# Integer bases: binary, octal, hex\n", + "value: int = 255\n", + "\n", + "print(f\"Decimal: {value:d}\")\n", + "print(f\"Binary: {value:b}\")\n", + "print(f\"Octal: {value:o}\")\n", + "print(f\"Hex (lower): {value:x}\")\n", + "print(f\"Hex (upper): {value:X}\")\n", + "\n", + "# With prefix (# flag)\n", + "print(f\"\\nWith prefix:\")\n", + "print(f\"Binary: {value:#b}\")\n", + "print(f\"Octal: {value:#o}\")\n", + "print(f\"Hex: {value:#x}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "# Zero-padding integers\n", + "value: int = 255\n", + "print(f\"Zero-padded binary (8 bits): {value:08b}\")\n", + "print(f\"Zero-padded hex (4 digits): {value:04x}\")\n", + "print(f\"Zero-padded decimal (6 wide): {42:06d}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "# Percentage formatting (multiplies by 100 and adds %)\n", + "ratio: float = 0.856\n", + "print(f\"Default: {ratio:%}\")\n", + "print(f\"1 decimal: {ratio:.1%}\")\n", + "print(f\"0 decimal: {ratio:.0%}\")\n", + "\n", + "# Scientific notation\n", + "large: float = 6.022e23\n", + "print(f\"\\nScientific: {large:.3e}\")\n", + "print(f\"General: {large:.3g}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 4: The `str.format()` Method\n", + "\n", + "Before f-strings, `str.format()` was the standard way to format strings. It supports positional arguments, keyword arguments, and the same format spec mini-language." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "# Positional arguments\n", + "result: str = \"{} {}\".format(\"hello\", \"world\")\n", + "print(result)\n", + "\n", + "# Explicit positional indices\n", + "result = \"{1} {0}\".format(\"world\", \"hello\")\n", + "print(result)\n", + "\n", + "# Keyword arguments\n", + "result = \"{name} is {age}\".format(name=\"Alice\", age=30)\n", + "print(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "# Format specs work the same way in str.format()\n", + "print(\"{:<10} {:>10}\".format(\"left\", \"right\"))\n", + "print(\"{:,}\".format(1000000))\n", + "print(\"{:.2%}\".format(0.856))\n", + "\n", + "# Reusing a template\n", + "template: str = \"Item: {name:<15} Price: ${price:>8.2f}\"\n", + "print()\n", + "print(template.format(name=\"Widget\", price=9.99))\n", + "print(template.format(name=\"Gadget Pro\", price=149.50))\n", + "print(template.format(name=\"Thingamajig\", price=2499.00))" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 5: Nested and Dynamic Format Specs\n", + "\n", + "F-strings allow nesting braces to compute format specs dynamically at runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "# Dynamic width and precision\n", + "width: int = 15\n", + "precision: int = 3\n", + "value: float = 3.14159\n", + "\n", + "print(f\"{value:{width}.{precision}f}\")\n", + "\n", + "# Dynamic alignment\n", + "for align in [\"<\", \"^\", \">\"]:\n", + " print(f\"{'text':{align}10}|\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "# Practical example: formatting a report with dynamic column widths\n", + "data: list[dict[str, str | float]] = [\n", + " {\"city\": \"New York\", \"population\": 8336817, \"area_sq_mi\": 302.6},\n", + " {\"city\": \"Los Angeles\", \"population\": 3979576, \"area_sq_mi\": 468.7},\n", + " {\"city\": \"Chicago\", \"population\": 2693976, \"area_sq_mi\": 227.3},\n", + "]\n", + "\n", + "col_w: int = 14\n", + "print(f\"{'City':<{col_w}} {'Population':>{col_w}} {'Area (sq mi)':>{col_w}}\")\n", + "print(\"-\" * (col_w * 3 + 2))\n", + "for row in data:\n", + " print(\n", + " f\"{row['city']:<{col_w}} \"\n", + " f\"{row['population']:>{col_w},} \"\n", + " f\"{row['area_sq_mi']:>{col_w},.1f}\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 6: Custom `__format__` Support\n", + "\n", + "Any object can define a `__format__` method to control how it responds to format specs inside f-strings and `str.format()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "class Temperature:\n", + " \"\"\"A temperature that formats in Celsius or Fahrenheit.\"\"\"\n", + "\n", + " def __init__(self, celsius: float) -> None:\n", + " self.celsius: float = celsius\n", + "\n", + " def __format__(self, spec: str) -> str:\n", + " if spec == \"f\":\n", + " return f\"{self.celsius * 9 / 5 + 32:.1f}F\"\n", + " return f\"{self.celsius:.1f}C\"\n", + "\n", + "\n", + "temp: Temperature = Temperature(100)\n", + "print(f\"Celsius: {temp}\")\n", + "print(f\"Fahrenheit: {temp:f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### F-Strings\n", + "- **Basic**: `f\"{variable}\"` or `f\"{expression}\"`\n", + "- **Debug**: `f\"{expr=}\"` shows the expression and its value\n", + "- **Nested specs**: `f\"{value:{width}.{precision}f}\"` for dynamic formatting\n", + "\n", + "### Format Spec Mini-Language\n", + "- **Alignment**: `<` (left), `>` (right), `^` (center) with optional fill character\n", + "- **Width**: Minimum field width, e.g., `{val:10}` for 10-character field\n", + "- **Precision**: `.Nf` for N decimal places, `.N` for N significant digits\n", + "- **Type codes**: `d` (decimal), `f` (fixed-point), `e` (scientific), `%` (percentage), `b` (binary), `x` (hex)\n", + "- **Grouping**: `,` or `_` for thousands separators\n", + "- **Prefix**: `#` for base prefixes (`0b`, `0o`, `0x`)\n", + "\n", + "### `str.format()`\n", + "- **Positional**: `\"{} {}\".format(a, b)`\n", + "- **Keyword**: `\"{name}\".format(name=val)`\n", + "- **Reusable templates**: Store format strings for repeated use\n", + "\n", + "### Custom Formatting\n", + "- Define `__format__(self, spec)` to make your classes format-aware" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_31/02_string_methods.ipynb b/src/chapter_31/02_string_methods.ipynb new file mode 100644 index 0000000..f42623c --- /dev/null +++ b/src/chapter_31/02_string_methods.ipynb @@ -0,0 +1,503 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 31: String Methods\n", + "\n", + "This notebook covers the built-in `str` methods for splitting, joining, stripping, replacing, and transforming text. It also introduces the `string` module which provides useful character-set constants and the `Template` class for safe substitution.\n", + "\n", + "## Key Concepts\n", + "- **Splitting**: `split()`, `rsplit()`, `splitlines()`, `partition()`\n", + "- **Joining**: `str.join()` to reassemble sequences of strings\n", + "- **Stripping**: `strip()`, `lstrip()`, `rstrip()` for whitespace removal\n", + "- **Replacing**: `replace()` for substring substitution\n", + "- **Character mapping**: `maketrans()` and `translate()` for bulk character replacement\n", + "- **Testing**: `startswith()`, `endswith()`, `isdigit()`, `isalpha()`, and friends\n", + "- **`string` module**: `ascii_lowercase`, `digits`, `punctuation`, `Template`" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Splitting Strings\n", + "\n", + "`split()` breaks a string into a list of substrings. By default it splits on any whitespace and discards empty strings. You can provide a custom separator and a maximum number of splits." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# Default split on whitespace\n", + "text: str = \"hello world python\"\n", + "words: list[str] = text.split()\n", + "print(f\"Words: {words}\")\n", + "\n", + "# Split on a custom separator\n", + "csv_line: str = \"Alice,30,Engineer\"\n", + "fields: list[str] = csv_line.split(\",\")\n", + "print(f\"Fields: {fields}\")\n", + "\n", + "# Limit the number of splits\n", + "path: str = \"usr/local/bin/python\"\n", + "parts: list[str] = path.split(\"/\", maxsplit=2)\n", + "print(f\"Parts (maxsplit=2): {parts}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "# rsplit splits from the right\n", + "path: str = \"usr/local/bin/python\"\n", + "parts: list[str] = path.rsplit(\"/\", maxsplit=1)\n", + "print(f\"rsplit (maxsplit=1): {parts}\")\n", + "\n", + "# splitlines splits on line boundaries\n", + "multiline: str = \"line one\\nline two\\nline three\"\n", + "lines: list[str] = multiline.splitlines()\n", + "print(f\"Lines: {lines}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Joining Strings\n", + "\n", + "`str.join()` is the inverse of `split()`. It concatenates an iterable of strings with the separator string between each element." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "# Join words with different separators\n", + "words: list[str] = [\"hello\", \"world\", \"python\"]\n", + "\n", + "print(f\"Space join: {' '.join(words)}\")\n", + "print(f\"Dash join: {'-'.join(words)}\")\n", + "print(f\"Empty join: {''.join(words)}\")\n", + "print(f\"Newline join:\\n{chr(10).join(words)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "# Split and join are inverses\n", + "original: str = \"hello-world-python\"\n", + "parts: list[str] = original.split(\"-\")\n", + "reassembled: str = \"-\".join(parts)\n", + "\n", + "print(f\"Original: {original}\")\n", + "print(f\"Split: {parts}\")\n", + "print(f\"Reassembled: {reassembled}\")\n", + "print(f\"Equal: {original == reassembled}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Stripping Whitespace\n", + "\n", + "`strip()`, `lstrip()`, and `rstrip()` remove leading and/or trailing characters. By default they remove whitespace, but you can pass a string of characters to strip." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "# Strip whitespace from both edges\n", + "s: str = \" hello \"\n", + "\n", + "print(f\"Original: '{s}'\")\n", + "print(f\"strip(): '{s.strip()}'\")\n", + "print(f\"lstrip(): '{s.lstrip()}'\")\n", + "print(f\"rstrip(): '{s.rstrip()}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "# Strip specific characters (removes any combination of those characters)\n", + "url: str = \"https://example.com///\"\n", + "print(f\"Strip slashes: '{url.rstrip('/')}'\")\n", + "\n", + "tag: str = \"\"\n", + "print(f\"Strip angle brackets: '{tag.strip('<>')}')\")\n", + "\n", + "# Common pattern: cleaning user input\n", + "user_input: str = \" alice@example.com \\n\"\n", + "clean: str = user_input.strip()\n", + "print(f\"Clean email: '{clean}'\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Replacing and Removing Substrings\n", + "\n", + "`replace()` substitutes all (or a limited number of) occurrences of a substring. To remove a substring, replace it with an empty string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "# Basic replacement\n", + "text: str = \"hello world\"\n", + "print(f\"Replace 'world' with 'python': {text.replace('world', 'python')}\")\n", + "\n", + "# Limit the number of replacements\n", + "repeated: str = \"aaa\"\n", + "print(f\"Replace 2 of 3 a's: {repeated.replace('a', 'b', 2)}\")\n", + "\n", + "# Remove a substring by replacing with empty string\n", + "messy: str = \"H e l l o\"\n", + "print(f\"Remove spaces: {messy.replace(' ', '')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 5: Partition\n", + "\n", + "`partition()` splits a string on the first occurrence of a separator, returning a 3-tuple: `(before, separator, after)`. This is useful when you need to split on the first delimiter only." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "# Partition splits on first occurrence\n", + "config: str = \"key=value=extra\"\n", + "before, sep, after = config.partition(\"=\")\n", + "\n", + "print(f\"Before: '{before}'\")\n", + "print(f\"Separator: '{sep}'\")\n", + "print(f\"After: '{after}'\")\n", + "\n", + "# rpartition splits on the last occurrence\n", + "path: str = \"/home/user/documents/file.txt\"\n", + "directory, slash, filename = path.rpartition(\"/\")\n", + "print(f\"\\nDirectory: '{directory}'\")\n", + "print(f\"Filename: '{filename}'\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 6: Character Mapping with `translate` and `maketrans`\n", + "\n", + "`str.maketrans()` builds a translation table and `translate()` applies it. This is efficient for replacing or deleting many individual characters in a single pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "# Replace vowels with digits\n", + "table: dict[int, int] = str.maketrans(\"aeiou\", \"12345\")\n", + "result: str = \"hello\".translate(table)\n", + "print(f\"'hello' with vowel mapping: '{result}'\")\n", + "\n", + "# The third argument to maketrans specifies characters to delete\n", + "delete_vowels: dict[int, int | None] = str.maketrans(\"\", \"\", \"aeiou\")\n", + "result = \"hello world\".translate(delete_vowels)\n", + "print(f\"'hello world' without vowels: '{result}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "# Practical example: sanitize a filename\n", + "import string\n", + "\n", + "def sanitize_filename(name: str) -> str:\n", + " \"\"\"Remove characters that are unsafe in filenames.\"\"\"\n", + " unsafe: str = '<>:\"/\\\\|?*'\n", + " table: dict[int, int | None] = str.maketrans(\"\", \"\", unsafe)\n", + " return name.translate(table).strip()\n", + "\n", + "raw: str = 'My File: \"Report\" (2025).txt'\n", + "safe: str = sanitize_filename(raw)\n", + "print(f\"Raw: {raw}\")\n", + "print(f\"Safe: {safe}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 7: Testing String Content\n", + "\n", + "Python provides many `is*()` methods for checking the content of strings, plus `startswith()` and `endswith()` for prefix/suffix checks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "# Prefix and suffix checks\n", + "filename: str = \"hello.py\"\n", + "print(f\"Starts with 'hello': {filename.startswith('hello')}\")\n", + "print(f\"Ends with '.py': {filename.endswith('.py')}\")\n", + "\n", + "# Accept a tuple of options\n", + "test_file: str = \"test.txt\"\n", + "print(f\"Ends with .txt or .csv: {test_file.endswith(('.txt', '.csv'))}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "# Content testing methods\n", + "samples: list[str] = [\"hello\", \"123\", \"Hello World\", \"hello123\", \" \", \"\"]\n", + "\n", + "print(f\"{'Value':<15} {'alpha':>6} {'digit':>6} {'alnum':>6} {'space':>6}\")\n", + "print(\"-\" * 42)\n", + "for s in samples:\n", + " label: str = repr(s)\n", + " if s: # is* methods return False for empty strings\n", + " print(\n", + " f\"{label:<15} \"\n", + " f\"{str(s.isalpha()):>6} \"\n", + " f\"{str(s.isdigit()):>6} \"\n", + " f\"{str(s.isalnum()):>6} \"\n", + " f\"{str(s.isspace()):>6}\"\n", + " )\n", + " else:\n", + " print(f\"{label:<15} {'(empty string -- all return False)'}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 8: The `string` Module\n", + "\n", + "The `string` module provides useful constants for character sets and the `Template` class for safe string substitution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import string\n", + "\n", + "# Character set constants\n", + "print(f\"ascii_lowercase: {string.ascii_lowercase}\")\n", + "print(f\"ascii_uppercase: {string.ascii_uppercase}\")\n", + "print(f\"ascii_letters: {string.ascii_letters}\")\n", + "print(f\"digits: {string.digits}\")\n", + "print(f\"hexdigits: {string.hexdigits}\")\n", + "print(f\"punctuation: {string.punctuation}\")\n", + "print(f\"whitespace: {string.whitespace!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "# Membership checks against constants\n", + "print(f\"'a' in ascii_lowercase: {'a' in string.ascii_lowercase}\")\n", + "print(f\"'Z' in ascii_uppercase: {'Z' in string.ascii_uppercase}\")\n", + "print(f\"'5' in digits: {'5' in string.digits}\")\n", + "print(f\"'!' in punctuation: {'!' in string.punctuation}\")\n", + "\n", + "# Practical: check if string contains only hex characters\n", + "def is_hex_string(s: str) -> bool:\n", + " \"\"\"Check if s contains only valid hexadecimal characters.\"\"\"\n", + " return all(c in string.hexdigits for c in s)\n", + "\n", + "print(f\"\\nis_hex_string('1a2b'): {is_hex_string('1a2b')}\")\n", + "print(f\"is_hex_string('xyz'): {is_hex_string('xyz')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a5b6c7", + "metadata": {}, + "outputs": [], + "source": [ + "import string\n", + "\n", + "# string.Template provides safe substitution\n", + "tmpl: string.Template = string.Template(\"Hello, $name! You have $count messages.\")\n", + "\n", + "# substitute() raises KeyError for missing keys\n", + "result: str = tmpl.substitute(name=\"Alice\", count=5)\n", + "print(f\"substitute: {result}\")\n", + "\n", + "# safe_substitute() leaves missing keys as-is\n", + "partial: str = tmpl.safe_substitute(name=\"Alice\")\n", + "print(f\"safe_substitute (missing 'count'): {partial}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a5b6c7d8", + "metadata": {}, + "source": [ + "## Section 9: Case Conversion and Other Transforms\n", + "\n", + "Python strings provide several methods for case conversion and other common transformations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5c6d7e8", + "metadata": {}, + "outputs": [], + "source": [ + "text: str = \"hello WORLD python\"\n", + "\n", + "print(f\"upper(): {text.upper()}\")\n", + "print(f\"lower(): {text.lower()}\")\n", + "print(f\"title(): {text.title()}\")\n", + "print(f\"capitalize(): {text.capitalize()}\")\n", + "print(f\"swapcase(): {text.swapcase()}\")\n", + "\n", + "# casefold() is more aggressive than lower() for case-insensitive comparison\n", + "german: str = \"Stra\\u00dfe\" # Strasse with sharp-s\n", + "print(f\"\\n'{german}'.lower(): '{german.lower()}'\")\n", + "print(f\"'{german}'.casefold(): '{german.casefold()}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5d6e7f8", + "metadata": {}, + "outputs": [], + "source": [ + "# Other useful methods\n", + "\n", + "# center, ljust, rjust -- like format spec alignment\n", + "print(f\"center: '{'hello':^20}'\")\n", + "print(f\"ljust: '{'hello'.ljust(20)}'\")\n", + "print(f\"rjust: '{'hello'.rjust(20)}'\")\n", + "\n", + "# zfill pads with zeros (respects sign)\n", + "print(f\"\\nzfill: {'42'.zfill(6)}\")\n", + "print(f\"zfill: {'-42'.zfill(6)}\")\n", + "\n", + "# count occurrences\n", + "print(f\"\\n'banana'.count('a'): {'banana'.count('a')}\")\n", + "\n", + "# find / index\n", + "print(f\"'hello'.find('ll'): {'hello'.find('ll')}\")\n", + "print(f\"'hello'.find('xyz'): {'hello'.find('xyz')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d5e6f7a8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Splitting and Joining\n", + "- **`split(sep, maxsplit)`**: Break string into list; defaults to whitespace\n", + "- **`rsplit(sep, maxsplit)`**: Split from the right\n", + "- **`splitlines()`**: Split on line boundaries\n", + "- **`partition(sep)`** / **`rpartition(sep)`**: Split into 3-tuple on first/last occurrence\n", + "- **`sep.join(iterable)`**: Concatenate strings with separator\n", + "\n", + "### Stripping and Replacing\n", + "- **`strip()` / `lstrip()` / `rstrip()`**: Remove leading/trailing characters (default: whitespace)\n", + "- **`replace(old, new, count)`**: Substitute substrings\n", + "\n", + "### Character Mapping\n", + "- **`str.maketrans(from, to, delete)`**: Build a translation table\n", + "- **`str.translate(table)`**: Apply the table in one pass\n", + "\n", + "### Testing and Searching\n", + "- **`startswith()` / `endswith()`**: Check prefixes/suffixes (accept tuples)\n", + "- **`isalpha()`, `isdigit()`, `isalnum()`, `isspace()`**: Content checks\n", + "- **`find()` / `index()`**: Locate substrings (`find` returns -1; `index` raises `ValueError`)\n", + "\n", + "### `string` Module\n", + "- **Constants**: `ascii_lowercase`, `ascii_uppercase`, `digits`, `punctuation`, `whitespace`\n", + "- **`Template`**: `$name` substitution with `substitute()` and `safe_substitute()`" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_31/03_difflib_and_textwrap.ipynb b/src/chapter_31/03_difflib_and_textwrap.ipynb new file mode 100644 index 0000000..3a617c5 --- /dev/null +++ b/src/chapter_31/03_difflib_and_textwrap.ipynb @@ -0,0 +1,530 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 31: Difflib and Textwrap\n", + "\n", + "This notebook covers two standard-library modules for working with text. `difflib` compares sequences and finds similarities, while `textwrap` provides tools for wrapping, filling, dedenting, and shortening text.\n", + "\n", + "## Key Concepts\n", + "- **`difflib.SequenceMatcher`**: Compute similarity ratios between strings\n", + "- **`difflib.get_close_matches()`**: Find fuzzy matches from a list of candidates\n", + "- **`difflib.unified_diff()` / `ndiff()`**: Generate human-readable diffs\n", + "- **`textwrap.wrap()` / `fill()`**: Break long text into lines of a given width\n", + "- **`textwrap.dedent()`**: Remove common leading whitespace\n", + "- **`textwrap.shorten()`**: Truncate text with a placeholder\n", + "- **`textwrap.indent()`**: Add a prefix to selected lines" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Sequence Matching with `SequenceMatcher`\n", + "\n", + "`difflib.SequenceMatcher` compares two sequences and computes a similarity ratio between 0.0 (completely different) and 1.0 (identical). It works on any sequence type, but is most commonly used with strings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# Compare two similar strings\n", + "matcher: difflib.SequenceMatcher = difflib.SequenceMatcher(None, \"hello\", \"hallo\")\n", + "ratio: float = matcher.ratio()\n", + "print(f\"'hello' vs 'hallo': ratio = {ratio:.3f}\")\n", + "\n", + "# Compare identical strings\n", + "matcher = difflib.SequenceMatcher(None, \"python\", \"python\")\n", + "print(f\"'python' vs 'python': ratio = {matcher.ratio():.3f}\")\n", + "\n", + "# Compare very different strings\n", + "matcher = difflib.SequenceMatcher(None, \"hello\", \"world\")\n", + "print(f\"'hello' vs 'world': ratio = {matcher.ratio():.3f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# get_matching_blocks shows where the sequences agree\n", + "matcher: difflib.SequenceMatcher = difflib.SequenceMatcher(None, \"abcdef\", \"abcxef\")\n", + "\n", + "print(f\"Ratio: {matcher.ratio():.3f}\")\n", + "print(\"\\nMatching blocks (i, j, size):\")\n", + "for block in matcher.get_matching_blocks():\n", + " print(f\" a[{block.a}:{block.a + block.size}] == b[{block.b}:{block.b + block.size}]\"\n", + " f\" -> '{('abcdef')[block.a:block.a + block.size]}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# get_opcodes describes the edits needed to transform a into b\n", + "a: str = \"hello world\"\n", + "b: str = \"hello python\"\n", + "matcher: difflib.SequenceMatcher = difflib.SequenceMatcher(None, a, b)\n", + "\n", + "print(f\"Transform '{a}' -> '{b}'\")\n", + "print()\n", + "for tag, i1, i2, j1, j2 in matcher.get_opcodes():\n", + " print(f\" {tag:>7} a[{i1}:{i2}] '{a[i1:i2]}' -> b[{j1}:{j2}] '{b[j1:j2]}'\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 2: Finding Fuzzy Matches\n", + "\n", + "`difflib.get_close_matches()` returns the best matches for a word from a list of candidates. It uses `SequenceMatcher` internally and accepts a cutoff threshold." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# Find close matches for a misspelled word\n", + "words: list[str] = [\"apple\", \"application\", \"apply\", \"banana\", \"appetite\"]\n", + "\n", + "matches: list[str] = difflib.get_close_matches(\"appli\", words)\n", + "print(f\"Close matches for 'appli': {matches}\")\n", + "\n", + "# Adjust the cutoff (default is 0.6)\n", + "matches = difflib.get_close_matches(\"appli\", words, n=5, cutoff=0.4)\n", + "print(f\"With cutoff=0.4: {matches}\")\n", + "\n", + "# No matches above the threshold\n", + "matches = difflib.get_close_matches(\"zebra\", words)\n", + "print(f\"Close matches for 'zebra': {matches}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# Practical example: \"Did you mean?\" for command-line tools\n", + "valid_commands: list[str] = [\"status\", \"commit\", \"push\", \"pull\", \"branch\", \"checkout\", \"merge\"]\n", + "\n", + "def suggest_command(user_input: str) -> str:\n", + " \"\"\"Suggest a valid command if the input is not recognized.\"\"\"\n", + " if user_input in valid_commands:\n", + " return f\"Running: {user_input}\"\n", + " suggestions: list[str] = difflib.get_close_matches(user_input, valid_commands, n=3)\n", + " if suggestions:\n", + " return f\"Unknown command '{user_input}'. Did you mean: {', '.join(suggestions)}?\"\n", + " return f\"Unknown command '{user_input}'. No suggestions found.\"\n", + "\n", + "print(suggest_command(\"comit\"))\n", + "print(suggest_command(\"statis\"))\n", + "print(suggest_command(\"checkou\"))\n", + "print(suggest_command(\"xyzzy\"))" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 3: Generating Diffs\n", + "\n", + "`difflib` provides several functions to produce human-readable diffs between sequences of lines, similar to the Unix `diff` command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# ndiff produces a detailed character-level diff\n", + "text_a: list[str] = [\n", + " \"line one\\n\",\n", + " \"line two\\n\",\n", + " \"line three\\n\",\n", + "]\n", + "text_b: list[str] = [\n", + " \"line one\\n\",\n", + " \"line 2\\n\",\n", + " \"line three\\n\",\n", + " \"line four\\n\",\n", + "]\n", + "\n", + "print(\"ndiff output:\")\n", + "diff: list[str] = list(difflib.ndiff(text_a, text_b))\n", + "for line in diff:\n", + " print(f\" {line}\", end=\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# unified_diff produces output similar to `diff -u`\n", + "text_a: list[str] = [\n", + " \"def greet(name):\\n\",\n", + " \" print('Hello ' + name)\\n\",\n", + " \" return None\\n\",\n", + "]\n", + "text_b: list[str] = [\n", + " \"def greet(name: str) -> None:\\n\",\n", + " \" print(f'Hello {name}')\\n\",\n", + "]\n", + "\n", + "print(\"unified_diff output:\")\n", + "diff = difflib.unified_diff(\n", + " text_a, text_b,\n", + " fromfile=\"before.py\",\n", + " tofile=\"after.py\",\n", + ")\n", + "for line in diff:\n", + " print(line, end=\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib\n", + "\n", + "# context_diff produces output similar to `diff -c`\n", + "text_a: list[str] = [\"alpha\\n\", \"beta\\n\", \"gamma\\n\", \"delta\\n\"]\n", + "text_b: list[str] = [\"alpha\\n\", \"BETA\\n\", \"gamma\\n\", \"epsilon\\n\"]\n", + "\n", + "print(\"context_diff output:\")\n", + "diff = difflib.context_diff(\n", + " text_a, text_b,\n", + " fromfile=\"original.txt\",\n", + " tofile=\"modified.txt\",\n", + ")\n", + "for line in diff:\n", + " print(line, end=\"\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 4: Text Wrapping with `textwrap`\n", + "\n", + "`textwrap.wrap()` breaks a long string into a list of lines, each no longer than the specified width. `textwrap.fill()` does the same but returns a single string with newlines inserted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "text: str = (\n", + " \"This is a long sentence that should be wrapped at a certain width. \"\n", + " \"The textwrap module makes it easy to format text for display in \"\n", + " \"terminals, emails, or any fixed-width context.\"\n", + ")\n", + "\n", + "# wrap returns a list of lines\n", + "lines: list[str] = textwrap.wrap(text, width=40)\n", + "print(\"wrap(width=40):\")\n", + "for i, line in enumerate(lines):\n", + " print(f\" [{i}] '{line}'\")\n", + " assert len(line) <= 40\n", + "\n", + "print(f\"\\nAll lines <= 40 chars: True\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "text: str = (\n", + " \"This is a long sentence that should be wrapped at a certain width. \"\n", + " \"The textwrap module makes it easy to format text for display.\"\n", + ")\n", + "\n", + "# fill returns a single string with newlines\n", + "filled: str = textwrap.fill(text, width=50)\n", + "print(\"fill(width=50):\")\n", + "print(filled)\n", + "\n", + "# With indentation\n", + "print(\"\\nfill with initial_indent and subsequent_indent:\")\n", + "formatted: str = textwrap.fill(\n", + " text,\n", + " width=50,\n", + " initial_indent=\" * \",\n", + " subsequent_indent=\" \",\n", + ")\n", + "print(formatted)" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 5: Shortening Text\n", + "\n", + "`textwrap.shorten()` collapses whitespace and truncates text to fit within a given width, appending a placeholder (default `[...]`) when truncation occurs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "text: str = \"Hello World, this is a long string\"\n", + "\n", + "# Shorten to 20 characters\n", + "short: str = textwrap.shorten(text, width=20)\n", + "print(f\"Original ({len(text)} chars): {text}\")\n", + "print(f\"Shortened ({len(short)} chars): {short}\")\n", + "assert len(short) <= 20\n", + "assert short.endswith(\"[...]\")\n", + "\n", + "# Custom placeholder\n", + "short_custom: str = textwrap.shorten(text, width=25, placeholder=\"...\")\n", + "print(f\"Custom placeholder: {short_custom}\")\n", + "\n", + "# When text already fits, no truncation\n", + "no_change: str = textwrap.shorten(\"short\", width=20)\n", + "print(f\"No truncation needed: {no_change}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 6: Dedenting Text\n", + "\n", + "`textwrap.dedent()` removes common leading whitespace from all lines. This is useful for working with triple-quoted strings that are indented to match the surrounding code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "# A triple-quoted string indented to match surrounding code\n", + "indented: str = \"\"\"\\\n", + " Hello,\n", + " This is an indented block.\n", + " Each line has extra whitespace.\n", + " \"\"\"\n", + "\n", + "print(\"Before dedent:\")\n", + "print(indented)\n", + "\n", + "dedented: str = textwrap.dedent(indented)\n", + "print(\"After dedent:\")\n", + "print(dedented)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "# Common pattern: dedent then fill for clean multiline strings\n", + "def get_help_text() -> str:\n", + " \"\"\"Return formatted help text.\"\"\"\n", + " raw: str = \"\"\"\\\n", + " This is the help text for the application. It describes\n", + " what the program does and how to use it. The text is written\n", + " indented in the source code for readability, but dedented\n", + " before display.\n", + " \"\"\"\n", + " return textwrap.fill(textwrap.dedent(raw), width=50)\n", + "\n", + "print(get_help_text())" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 7: Indenting Text\n", + "\n", + "`textwrap.indent()` adds a prefix to the beginning of selected lines. By default, it only adds the prefix to non-empty lines." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "text: str = \"line one\\nline two\\n\\nline four\"\n", + "\n", + "# Add prefix to all non-empty lines\n", + "indented: str = textwrap.indent(text, prefix=\" \")\n", + "print(\"Default indent (4 spaces):\")\n", + "print(indented)\n", + "\n", + "# Add a custom prefix (e.g., line comments)\n", + "commented: str = textwrap.indent(text, prefix=\"# \")\n", + "print(\"\\nCommented:\")\n", + "print(commented)\n", + "\n", + "# Use predicate to control which lines get the prefix\n", + "selective: str = textwrap.indent(\n", + " text,\n", + " prefix=\"> \",\n", + " predicate=lambda line: \"two\" in line,\n", + ")\n", + "print(\"\\nSelective (only lines containing 'two'):\")\n", + "print(selective)" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Section 8: The `TextWrapper` Class\n", + "\n", + "For repeated wrapping with the same settings, use a `TextWrapper` object. This avoids passing the same parameters each time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a5b6c7", + "metadata": {}, + "outputs": [], + "source": [ + "import textwrap\n", + "\n", + "# Create a reusable wrapper\n", + "wrapper: textwrap.TextWrapper = textwrap.TextWrapper(\n", + " width=45,\n", + " initial_indent=\" \",\n", + " subsequent_indent=\" \",\n", + " break_long_words=False,\n", + " break_on_hyphens=False,\n", + ")\n", + "\n", + "paragraphs: list[str] = [\n", + " \"The textwrap module provides convenience functions and a TextWrapper class for formatting text.\",\n", + " \"TextWrapper instances are reusable, so you can configure wrapping once and apply it to multiple paragraphs.\",\n", + "]\n", + "\n", + "for i, para in enumerate(paragraphs):\n", + " print(f\"Paragraph {i + 1}:\")\n", + " print(wrapper.fill(para))\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "a5b6c7d8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `difflib` -- Sequence Comparison\n", + "- **`SequenceMatcher(None, a, b)`**: Compare two sequences\n", + " - `.ratio()` returns similarity from 0.0 to 1.0\n", + " - `.get_matching_blocks()` shows where sequences agree\n", + " - `.get_opcodes()` describes the edits needed\n", + "- **`get_close_matches(word, possibilities, n, cutoff)`**: Find fuzzy matches\n", + "- **`unified_diff(a, b)`**: Produce a unified diff (like `diff -u`)\n", + "- **`ndiff(a, b)`**: Character-level diff with indicators\n", + "- **`context_diff(a, b)`**: Context diff (like `diff -c`)\n", + "\n", + "### `textwrap` -- Text Formatting\n", + "- **`wrap(text, width)`**: Break text into a list of lines\n", + "- **`fill(text, width)`**: Wrap and return as a single string with newlines\n", + "- **`shorten(text, width, placeholder)`**: Truncate with placeholder (default `[...]`)\n", + "- **`dedent(text)`**: Remove common leading whitespace from all lines\n", + "- **`indent(text, prefix)`**: Add prefix to selected lines\n", + "- **`TextWrapper(...)`**: Reusable wrapper object with configurable settings\n", + "\n", + "### Common Patterns\n", + "- **\"Did you mean?\"**: Use `get_close_matches()` to suggest corrections for typos\n", + "- **Dedent + fill**: Clean up indented triple-quoted strings for display\n", + "- **Diff reports**: Use `unified_diff()` to show file changes" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_31/README.md b/src/chapter_31/README.md new file mode 100644 index 0000000..259583c --- /dev/null +++ b/src/chapter_31/README.md @@ -0,0 +1,9 @@ +# Chapter 31: String Methods and Formatting + +Advanced string manipulation — f-string expressions, str methods, the string module, difflib for sequence comparison, and textwrap for text formatting. + +## Notebooks + +1. **01_fstrings_and_formatting.ipynb** — f-string expressions, format spec mini-language, alignment, padding, number formatting +2. **02_string_methods.ipynb** — str methods (split, join, strip, replace, translate), string module constants +3. **03_difflib_and_textwrap.ipynb** — difflib for diffs and fuzzy matching, textwrap for wrapping and dedenting diff --git a/src/chapter_31/__init__.py b/src/chapter_31/__init__.py new file mode 100644 index 0000000..ae7d7e7 --- /dev/null +++ b/src/chapter_31/__init__.py @@ -0,0 +1 @@ +"""Chapter 31: String Methods and Formatting.""" diff --git a/src/chapter_32/01_math_and_cmath.ipynb b/src/chapter_32/01_math_and_cmath.ipynb new file mode 100644 index 0000000..e7c1d3a --- /dev/null +++ b/src/chapter_32/01_math_and_cmath.ipynb @@ -0,0 +1,476 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 32: Math and Complex Math\n", + "\n", + "This notebook covers Python's `math` module for standard mathematical operations and the `cmath` module for complex number arithmetic. You will learn how to use mathematical constants, common functions, logarithms, trigonometry, number theory utilities, and complex-plane operations.\n", + "\n", + "## Key Concepts\n", + "- **Constants**: `math.pi`, `math.e`, `math.inf`, `math.nan`\n", + "- **Basic functions**: `sqrt`, `ceil`, `floor`, `factorial`, `fabs`\n", + "- **Logarithms**: `log`, `log10`, `log2`\n", + "- **Trigonometry**: `sin`, `cos`, `tan`, `radians`, `degrees`\n", + "- **Number theory**: `gcd`, `lcm`, `isclose`, `comb`, `perm`\n", + "- **Complex math**: `cmath.phase`, `cmath.polar`, `cmath.rect`, `cmath.sqrt`" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Mathematical Constants\n", + "\n", + "The `math` module provides fundamental mathematical constants that are used across virtually every area of numeric computing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Fundamental constants\n", + "print(f\"pi: {math.pi}\")\n", + "print(f\"e: {math.e}\")\n", + "print(f\"tau: {math.tau}\") # tau == 2 * pi\n", + "\n", + "# Special float values\n", + "print(f\"\\ninf: {math.inf}\")\n", + "print(f\"inf > 1e308: {math.inf > 1e308}\")\n", + "print(f\"nan: {math.nan}\")\n", + "print(f\"isnan(nan): {math.isnan(math.nan)}\")\n", + "print(f\"isinf(inf): {math.isinf(math.inf)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e2f3a4", + "metadata": {}, + "source": [ + "## Section 2: Basic Mathematical Functions\n", + "\n", + "The `math` module provides common functions for rounding, roots, powers, and factorials." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Square root\n", + "result: float = math.sqrt(16)\n", + "print(f\"sqrt(16) = {result}\")\n", + "\n", + "# Ceiling and floor\n", + "print(f\"ceil(3.2) = {math.ceil(3.2)}\")\n", + "print(f\"floor(3.8) = {math.floor(3.8)}\")\n", + "\n", + "# Factorial\n", + "print(f\"\\n5! = {math.factorial(5)}\")\n", + "\n", + "# Absolute value (returns float)\n", + "print(f\"fabs(-7.5) = {math.fabs(-7.5)}\")\n", + "\n", + "# Power and exponent\n", + "print(f\"\\npow(2, 10) = {math.pow(2, 10)}\")\n", + "print(f\"exp(1) = {math.exp(1)}\")\n", + "print(f\"exp(1) == e: {math.isclose(math.exp(1), math.e)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Truncation vs floor for negative numbers\n", + "value: float = -3.7\n", + "print(f\"value: {value}\")\n", + "print(f\"floor(-3.7): {math.floor(value)}\")\n", + "print(f\"ceil(-3.7): {math.ceil(value)}\")\n", + "print(f\"trunc(-3.7): {math.trunc(value)}\")\n", + "\n", + "# fsum for accurate floating-point summation\n", + "values: list[float] = [0.1] * 10\n", + "print(f\"\\nsum([0.1]*10): {sum(values)}\")\n", + "print(f\"fsum([0.1]*10): {math.fsum(values)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: Logarithmic Functions\n", + "\n", + "Logarithms are the inverse of exponentiation. The `math` module supports natural, base-10, and base-2 logarithms, plus a general-base form." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Natural log (base e)\n", + "print(f\"log(e) = {math.log(math.e)}\")\n", + "print(f\"log(1) = {math.log(1)}\")\n", + "\n", + "# Base-10 log\n", + "print(f\"\\nlog10(100) = {math.log10(100)}\")\n", + "print(f\"log10(1000) = {math.log10(1000)}\")\n", + "\n", + "# Base-2 log\n", + "print(f\"\\nlog2(8) = {math.log2(8)}\")\n", + "print(f\"log2(256) = {math.log2(256)}\")\n", + "\n", + "# Arbitrary base using two-argument form\n", + "print(f\"\\nlog(27, 3) = {math.log(27, 3)}\")\n", + "print(f\"log(625, 5) = {math.log(625, 5)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 4: Trigonometric Functions\n", + "\n", + "Trigonometric functions operate in radians. Use `math.radians()` and `math.degrees()` to convert between degrees and radians." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Basic trig functions (input in radians)\n", + "angle: float = math.pi / 2 # 90 degrees\n", + "print(f\"sin(pi/2) = {math.sin(angle)}\")\n", + "print(f\"cos(0) = {math.cos(0)}\")\n", + "print(f\"tan(pi/4) = {math.tan(math.pi / 4)}\")\n", + "\n", + "# Inverse trig functions\n", + "print(f\"\\nasin(1) = {math.asin(1)}\")\n", + "print(f\"asin(1) == pi/2: {math.isclose(math.asin(1), math.pi / 2)}\")\n", + "\n", + "# Converting between degrees and radians\n", + "deg: float = 45.0\n", + "rad: float = math.radians(deg)\n", + "print(f\"\\n{deg} degrees = {rad} radians\")\n", + "print(f\"{rad} radians = {math.degrees(rad)} degrees\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Hyperbolic functions\n", + "x: float = 1.0\n", + "print(f\"sinh({x}) = {math.sinh(x)}\")\n", + "print(f\"cosh({x}) = {math.cosh(x)}\")\n", + "print(f\"tanh({x}) = {math.tanh(x)}\")\n", + "\n", + "# Pythagorean helper\n", + "a: float = 3.0\n", + "b: float = 4.0\n", + "hypotenuse: float = math.hypot(a, b)\n", + "print(f\"\\nhypot(3, 4) = {hypotenuse}\")\n", + "print(f\"Manual sqrt: {math.sqrt(a**2 + b**2)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 5: Number Theory Utilities\n", + "\n", + "Python 3.9+ includes `gcd`, `lcm`, `isclose`, `comb`, and `perm` for common number-theoretic and combinatoric operations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Greatest common divisor\n", + "print(f\"gcd(12, 8) = {math.gcd(12, 8)}\")\n", + "print(f\"gcd(54, 24) = {math.gcd(54, 24)}\")\n", + "\n", + "# Least common multiple (Python 3.9+)\n", + "print(f\"\\nlcm(4, 6) = {math.lcm(4, 6)}\")\n", + "print(f\"lcm(12, 18) = {math.lcm(12, 18)}\")\n", + "\n", + "# Floating-point comparison with isclose\n", + "print(f\"\\n0.1 + 0.2 == 0.3: {0.1 + 0.2 == 0.3}\")\n", + "print(f\"isclose(0.1+0.2, 0.3): {math.isclose(0.1 + 0.2, 0.3)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "# Combinations: C(n, k) = n! / (k! * (n-k)!)\n", + "print(f\"C(10, 3) = {math.comb(10, 3)}\")\n", + "print(f\"C(52, 5) = {math.comb(52, 5)}\") # poker hands\n", + "\n", + "# Permutations: P(n, k) = n! / (n-k)!\n", + "print(f\"\\nP(10, 3) = {math.perm(10, 3)}\")\n", + "print(f\"P(5, 5) = {math.perm(5, 5)}\") # same as 5!\n", + "\n", + "# isqrt -- integer square root (Python 3.8+)\n", + "print(f\"\\nisqrt(17) = {math.isqrt(17)}\")\n", + "print(f\"isqrt(100) = {math.isqrt(100)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 6: Complex Numbers and `cmath`\n", + "\n", + "Python has built-in support for complex numbers. The `cmath` module extends `math` to operate on complex values, including polar/rectangular conversions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "# Built-in complex number support\n", + "z1: complex = complex(3, 4)\n", + "z2: complex = 1 + 2j # literal syntax\n", + "\n", + "print(f\"z1 = {z1}\")\n", + "print(f\"z2 = {z2}\")\n", + "print(f\"z1.real = {z1.real}, z1.imag = {z1.imag}\")\n", + "\n", + "# Arithmetic\n", + "print(f\"\\nz1 + z2 = {z1 + z2}\")\n", + "print(f\"z1 * z2 = {z1 * z2}\")\n", + "print(f\"z1 / z2 = {z1 / z2}\")\n", + "\n", + "# Conjugate and magnitude\n", + "print(f\"\\nconjugate(z1) = {z1.conjugate()}\")\n", + "print(f\"|z1| = {abs(z1)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import cmath\n", + "import math\n", + "\n", + "z: complex = complex(3, 4)\n", + "\n", + "# Phase angle (argument) in radians\n", + "phase: float = cmath.phase(z)\n", + "print(f\"z = {z}\")\n", + "print(f\"phase(z) = {phase:.7f} radians\")\n", + "print(f\"phase(z) = {math.degrees(phase):.2f} degrees\")\n", + "\n", + "# Polar representation: (magnitude, phase)\n", + "r, phi = cmath.polar(z)\n", + "print(f\"\\npolar(z) = (r={r}, phi={phi:.7f})\")\n", + "\n", + "# Convert back to rectangular form\n", + "z_back: complex = cmath.rect(r, phi)\n", + "print(f\"rect(r, phi) = {z_back}\")\n", + "print(f\"Real part close to 3: {math.isclose(z_back.real, 3.0)}\")\n", + "print(f\"Imag part close to 4: {math.isclose(z_back.imag, 4.0)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import cmath\n", + "\n", + "# cmath can handle operations that math cannot\n", + "# Square root of a negative number\n", + "result: complex = cmath.sqrt(-1)\n", + "print(f\"cmath.sqrt(-1) = {result}\")\n", + "\n", + "# Euler's identity: e^(i*pi) + 1 = 0\n", + "euler: complex = cmath.exp(complex(0, cmath.pi)) + 1\n", + "print(f\"\\ne^(i*pi) + 1 = {euler}\")\n", + "print(f\"Close to zero: {abs(euler) < 1e-15}\")\n", + "\n", + "# Complex logarithm\n", + "z: complex = complex(1, 1)\n", + "print(f\"\\nlog({z}) = {cmath.log(z)}\")\n", + "print(f\"log10({z}) = {cmath.log10(z)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 7: Practical Patterns\n", + "\n", + "Common patterns that combine `math` and `cmath` for real-world numeric computations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "def distance(x1: float, y1: float, x2: float, y2: float) -> float:\n", + " \"\"\"Calculate Euclidean distance between two 2D points.\"\"\"\n", + " return math.hypot(x2 - x1, y2 - y1)\n", + "\n", + "def deg_to_rad(degrees: float) -> float:\n", + " \"\"\"Convert degrees to radians.\"\"\"\n", + " return math.radians(degrees)\n", + "\n", + "# Distance between two points\n", + "d: float = distance(0, 0, 3, 4)\n", + "print(f\"Distance (0,0) to (3,4) = {d}\")\n", + "\n", + "# Compound interest: A = P * (1 + r/n)^(n*t)\n", + "principal: float = 1000.0\n", + "rate: float = 0.05\n", + "compounds_per_year: int = 12\n", + "years: int = 10\n", + "\n", + "amount: float = principal * math.pow(\n", + " 1 + rate / compounds_per_year,\n", + " compounds_per_year * years,\n", + ")\n", + "print(f\"\\n${principal:.2f} at {rate:.0%} for {years}yr = ${amount:.2f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "def is_prime(n: int) -> bool:\n", + " \"\"\"Check if n is a prime number using trial division.\"\"\"\n", + " if n < 2:\n", + " return False\n", + " if n < 4:\n", + " return True\n", + " if n % 2 == 0 or n % 3 == 0:\n", + " return False\n", + " limit: int = math.isqrt(n)\n", + " i: int = 5\n", + " while i <= limit:\n", + " if n % i == 0 or n % (i + 2) == 0:\n", + " return False\n", + " i += 6\n", + " return True\n", + "\n", + "# Test some numbers\n", + "test_numbers: list[int] = [2, 7, 10, 13, 25, 29, 100, 97]\n", + "for n in test_numbers:\n", + " print(f\"is_prime({n:>3}) = {is_prime(n)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `math` Module Constants\n", + "- **`math.pi`**: Ratio of circumference to diameter (~3.14159)\n", + "- **`math.e`**: Base of natural logarithms (~2.71828)\n", + "- **`math.inf`**: Positive infinity (greater than any finite number)\n", + "- **`math.nan`**: Not a Number; use `math.isnan()` to detect\n", + "\n", + "### Core Functions\n", + "- **Rounding**: `ceil`, `floor`, `trunc`\n", + "- **Roots/powers**: `sqrt`, `pow`, `exp`, `isqrt`\n", + "- **Logarithms**: `log`, `log10`, `log2`, `log(x, base)`\n", + "- **Trigonometry**: `sin`, `cos`, `tan`, `asin`, `acos`, `atan`, `hypot`\n", + "- **Conversions**: `radians`, `degrees`\n", + "\n", + "### Number Theory\n", + "- **`gcd(a, b)`**: Greatest common divisor\n", + "- **`lcm(a, b)`**: Least common multiple (Python 3.9+)\n", + "- **`comb(n, k)`**: Combinations (n choose k)\n", + "- **`perm(n, k)`**: Permutations\n", + "- **`isclose(a, b)`**: Compare floats with tolerance\n", + "\n", + "### `cmath` Module for Complex Numbers\n", + "- **`phase(z)`**: Angle of complex number in radians\n", + "- **`polar(z)`**: Convert to (magnitude, phase) tuple\n", + "- **`rect(r, phi)`**: Convert polar back to rectangular\n", + "- **`cmath.sqrt(-1)`**: Handles operations that `math` cannot\n", + "- All `math` functions have `cmath` equivalents that accept complex numbers" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_32/02_decimal_and_fractions.ipynb b/src/chapter_32/02_decimal_and_fractions.ipynb new file mode 100644 index 0000000..7dabf3b --- /dev/null +++ b/src/chapter_32/02_decimal_and_fractions.ipynb @@ -0,0 +1,510 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 32: Decimal and Fractions\n", + "\n", + "This notebook covers Python's `decimal` module for precise fixed-point and floating-point arithmetic, and the `fractions` module for exact rational number representation. These tools solve the fundamental problem of binary floating-point inaccuracy in financial, scientific, and mathematical contexts.\n", + "\n", + "## Key Concepts\n", + "- **`Decimal`**: Arbitrary-precision decimal arithmetic, ideal for financial calculations\n", + "- **`Decimal` contexts**: Control precision, rounding, and error handling globally\n", + "- **`quantize`**: Round a Decimal to a specific number of decimal places\n", + "- **`Fraction`**: Exact representation of rational numbers as numerator/denominator\n", + "- **Auto-reduction**: Fractions are automatically reduced to lowest terms\n", + "- **Interoperability**: Converting between float, Decimal, and Fraction" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: The Floating-Point Problem\n", + "\n", + "Binary floating-point cannot represent all decimal fractions exactly. This leads to surprising rounding errors that matter in finance and precision-critical applications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# The classic floating-point surprise\n", + "result: float = 0.1 + 0.2\n", + "print(f\"0.1 + 0.2 = {result}\")\n", + "print(f\"0.1 + 0.2 == 0.3: {result == 0.3}\")\n", + "print(f\"Actual repr: {result!r}\")\n", + "\n", + "# This compounds over many operations\n", + "total: float = sum(0.1 for _ in range(10))\n", + "print(f\"\\nsum of 0.1 ten times = {total}\")\n", + "print(f\"Expected 1.0, equal: {total == 1.0}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e2f3a4", + "metadata": {}, + "source": [ + "## Section 2: Creating Decimal Objects\n", + "\n", + "The `Decimal` type provides exact decimal arithmetic. Always create Decimals from strings to avoid inheriting float inaccuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal\n", + "\n", + "# Create from string (preferred -- exact)\n", + "d1: Decimal = Decimal(\"0.1\")\n", + "d2: Decimal = Decimal(\"0.2\")\n", + "d3: Decimal = Decimal(\"0.3\")\n", + "\n", + "print(f\"Decimal('0.1') + Decimal('0.2') = {d1 + d2}\")\n", + "print(f\"Result == Decimal('0.3'): {d1 + d2 == d3}\")\n", + "\n", + "# Create from integer (also exact)\n", + "d_int: Decimal = Decimal(42)\n", + "print(f\"\\nDecimal(42) = {d_int}\")\n", + "\n", + "# Create from float (inherits inaccuracy -- avoid this)\n", + "d_float: Decimal = Decimal(0.1)\n", + "print(f\"\\nDecimal(0.1) = {d_float}\")\n", + "print(f\"Decimal('0.1') = {d1}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal\n", + "\n", + "# String representation is exact\n", + "pi: Decimal = Decimal(\"3.14159\")\n", + "print(f\"str(Decimal('3.14159')) = {str(pi)}\")\n", + "\n", + "# Decimal arithmetic preserves precision\n", + "price: Decimal = Decimal(\"19.99\")\n", + "tax_rate: Decimal = Decimal(\"0.0825\")\n", + "tax: Decimal = price * tax_rate\n", + "total: Decimal = price + tax\n", + "\n", + "print(f\"\\nPrice: ${price}\")\n", + "print(f\"Tax: ${tax}\")\n", + "print(f\"Total: ${total}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: Rounding with `quantize`\n", + "\n", + "The `quantize` method rounds a Decimal to match the exponent of another Decimal. This is essential for formatting currency and controlling output precision." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_CEILING\n", + "\n", + "d: Decimal = Decimal(\"3.14159\")\n", + "\n", + "# Quantize to different precisions\n", + "print(f\"Original: {d}\")\n", + "print(f\"To 0.01: {d.quantize(Decimal('0.01'))}\")\n", + "print(f\"To 0.1: {d.quantize(Decimal('0.1'))}\")\n", + "print(f\"To 1: {d.quantize(Decimal('1'))}\")\n", + "\n", + "# Different rounding modes\n", + "amount: Decimal = Decimal(\"2.675\")\n", + "print(f\"\\nRounding {amount} to 2 places:\")\n", + "print(f\" ROUND_HALF_UP: {amount.quantize(Decimal('0.01'), rounding=ROUND_HALF_UP)}\")\n", + "print(f\" ROUND_DOWN: {amount.quantize(Decimal('0.01'), rounding=ROUND_DOWN)}\")\n", + "print(f\" ROUND_CEILING: {amount.quantize(Decimal('0.01'), rounding=ROUND_CEILING)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal, ROUND_HALF_UP\n", + "\n", + "def format_currency(amount: Decimal) -> str:\n", + " \"\"\"Format a Decimal as a currency string with 2 decimal places.\"\"\"\n", + " rounded: Decimal = amount.quantize(Decimal(\"0.01\"), rounding=ROUND_HALF_UP)\n", + " return f\"${rounded:,}\"\n", + "\n", + "prices: list[Decimal] = [\n", + " Decimal(\"1234.567\"),\n", + " Decimal(\"0.005\"),\n", + " Decimal(\"99999.999\"),\n", + "]\n", + "\n", + "for price in prices:\n", + " print(f\"{str(price):>12} -> {format_currency(price)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 4: Decimal Contexts\n", + "\n", + "The decimal module uses a thread-local context that controls precision, rounding mode, and trap handling. You can modify the global context or create local contexts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal, getcontext, localcontext\n", + "\n", + "# View the current global context\n", + "ctx = getcontext()\n", + "print(f\"Default precision: {ctx.prec}\")\n", + "print(f\"Default rounding: {ctx.rounding}\")\n", + "\n", + "# Compute with default precision (28 digits)\n", + "result: Decimal = Decimal(\"1\") / Decimal(\"7\")\n", + "print(f\"\\n1/7 (prec=28): {result}\")\n", + "\n", + "# Use a local context to temporarily change precision\n", + "with localcontext() as local_ctx:\n", + " local_ctx.prec = 6\n", + " result_low: Decimal = Decimal(\"1\") / Decimal(\"7\")\n", + " print(f\"1/7 (prec=6): {result_low}\")\n", + "\n", + "# Outside the context, precision is restored\n", + "result_after: Decimal = Decimal(\"1\") / Decimal(\"7\")\n", + "print(f\"1/7 (prec=28): {result_after}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal, InvalidOperation\n", + "\n", + "# InvalidOperation is raised for invalid string inputs\n", + "try:\n", + " bad: Decimal = Decimal(\"not_a_number\")\n", + " print(f\"Result: {bad}\")\n", + "except InvalidOperation as e:\n", + " print(f\"InvalidOperation raised: {e}\")\n", + "\n", + "# Special Decimal values\n", + "print(f\"\\nDecimal('Infinity'): {Decimal('Infinity')}\")\n", + "print(f\"Decimal('-Infinity'): {Decimal('-Infinity')}\")\n", + "print(f\"Decimal('NaN'): {Decimal('NaN')}\")\n", + "print(f\"Is NaN: {Decimal('NaN').is_nan()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 5: Creating Fraction Objects\n", + "\n", + "The `Fraction` class represents rational numbers as an exact numerator/denominator pair. Fractions are automatically reduced to lowest terms." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "from fractions import Fraction\n", + "\n", + "# Create from numerator and denominator\n", + "f1: Fraction = Fraction(1, 3)\n", + "print(f\"Fraction(1, 3) = {f1}\")\n", + "print(f\"Numerator: {f1.numerator}\")\n", + "print(f\"Denominator: {f1.denominator}\")\n", + "\n", + "# Auto-reduction to lowest terms\n", + "f2: Fraction = Fraction(4, 8)\n", + "print(f\"\\nFraction(4, 8) = {f2}\")\n", + "print(f\"Fraction(4, 8) == Fraction(1, 2): {f2 == Fraction(1, 2)}\")\n", + "\n", + "# Create from string\n", + "f3: Fraction = Fraction(\"3/7\")\n", + "print(f\"\\nFraction('3/7') = {f3}\")\n", + "\n", + "# Create from decimal string\n", + "f4: Fraction = Fraction(\"0.125\")\n", + "print(f\"Fraction('0.125') = {f4}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "from fractions import Fraction\n", + "\n", + "# Create from float (exact representation of the float's value)\n", + "f_half: Fraction = Fraction(0.5)\n", + "print(f\"Fraction(0.5) = {f_half}\")\n", + "print(f\"Equal to 1/2: {f_half == Fraction(1, 2)}\")\n", + "\n", + "# Caution: floats may not be what you expect\n", + "f_tenth: Fraction = Fraction(0.1)\n", + "print(f\"\\nFraction(0.1) = {f_tenth}\")\n", + "print(f\"(Not exactly 1/10 because 0.1 cannot be represented exactly in binary)\")\n", + "\n", + "# Use limit_denominator for a cleaner approximation\n", + "f_approx: Fraction = Fraction(0.1).limit_denominator(1000)\n", + "print(f\"\\nFraction(0.1).limit_denominator(1000) = {f_approx}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 6: Fraction Arithmetic\n", + "\n", + "Fractions support all standard arithmetic operations and always return exact results as reduced fractions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "from fractions import Fraction\n", + "\n", + "a: Fraction = Fraction(1, 3)\n", + "b: Fraction = Fraction(1, 6)\n", + "\n", + "# Addition\n", + "print(f\"{a} + {b} = {a + b}\")\n", + "\n", + "# Subtraction\n", + "print(f\"{a} - {b} = {a - b}\")\n", + "\n", + "# Multiplication\n", + "c: Fraction = Fraction(2, 3)\n", + "d: Fraction = Fraction(3, 4)\n", + "print(f\"\\n{c} * {d} = {c * d}\")\n", + "\n", + "# Division\n", + "print(f\"{c} / {d} = {c / d}\")\n", + "\n", + "# Exponentiation\n", + "print(f\"\\n(1/2) ** 3 = {Fraction(1, 2) ** 3}\")\n", + "\n", + "# Comparison\n", + "print(f\"\\n1/3 > 1/4: {Fraction(1, 3) > Fraction(1, 4)}\")\n", + "print(f\"2/6 == 1/3: {Fraction(2, 6) == Fraction(1, 3)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "from fractions import Fraction\n", + "\n", + "# Practical example: recipe scaling\n", + "def scale_recipe(\n", + " ingredients: dict[str, Fraction],\n", + " factor: Fraction,\n", + ") -> dict[str, Fraction]:\n", + " \"\"\"Scale recipe ingredients by a given factor.\"\"\"\n", + " return {name: amount * factor for name, amount in ingredients.items()}\n", + "\n", + "recipe: dict[str, Fraction] = {\n", + " \"flour (cups)\": Fraction(2, 3),\n", + " \"sugar (cups)\": Fraction(1, 4),\n", + " \"butter (cups)\": Fraction(1, 3),\n", + " \"eggs\": Fraction(2),\n", + "}\n", + "\n", + "# Scale to 1.5x\n", + "scaled: dict[str, Fraction] = scale_recipe(recipe, Fraction(3, 2))\n", + "\n", + "print(\"Original recipe (1x):\")\n", + "for name, amount in recipe.items():\n", + " print(f\" {name}: {amount} ({float(amount):.3f})\")\n", + "\n", + "print(\"\\nScaled recipe (1.5x):\")\n", + "for name, amount in scaled.items():\n", + " print(f\" {name}: {amount} ({float(amount):.3f})\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 7: Converting Between Numeric Types\n", + "\n", + "Decimal and Fraction can interoperate with each other and with built-in numeric types, but some conversions require care." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal\n", + "from fractions import Fraction\n", + "\n", + "# Fraction to float\n", + "f: Fraction = Fraction(1, 3)\n", + "print(f\"float(1/3) = {float(f)}\")\n", + "\n", + "# Fraction to Decimal (via string for precision)\n", + "f2: Fraction = Fraction(1, 8)\n", + "d: Decimal = Decimal(f2.numerator) / Decimal(f2.denominator)\n", + "print(f\"Decimal(1/8) = {d}\")\n", + "\n", + "# Decimal to Fraction\n", + "d2: Decimal = Decimal(\"0.75\")\n", + "f3: Fraction = Fraction(d2)\n", + "print(f\"\\nFraction(Decimal('0.75')) = {f3}\")\n", + "\n", + "# Decimal to float\n", + "d3: Decimal = Decimal(\"3.14159\")\n", + "print(f\"float(Decimal('3.14159')) = {float(d3)}\")\n", + "\n", + "# Mixed arithmetic: Fraction + int works, but Fraction + float gives float\n", + "print(f\"\\nFraction(1,2) + 1 = {Fraction(1, 2) + 1}\")\n", + "print(f\"type: {type(Fraction(1, 2) + 1)}\")\n", + "print(f\"Fraction(1,2) + 0.5 = {Fraction(1, 2) + 0.5}\")\n", + "print(f\"type: {type(Fraction(1, 2) + 0.5)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 8: Practical Patterns\n", + "\n", + "Real-world examples showing when to choose Decimal vs Fraction vs float." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal, ROUND_HALF_UP\n", + "\n", + "def split_bill(\n", + " total: Decimal,\n", + " num_people: int,\n", + " tip_pct: Decimal,\n", + ") -> dict[str, Decimal]:\n", + " \"\"\"Calculate per-person cost with tip, using exact Decimal math.\"\"\"\n", + " tip: Decimal = (total * tip_pct / Decimal(\"100\")).quantize(\n", + " Decimal(\"0.01\"), rounding=ROUND_HALF_UP\n", + " )\n", + " grand_total: Decimal = total + tip\n", + " per_person: Decimal = (grand_total / num_people).quantize(\n", + " Decimal(\"0.01\"), rounding=ROUND_HALF_UP\n", + " )\n", + " return {\n", + " \"subtotal\": total,\n", + " \"tip\": tip,\n", + " \"grand_total\": grand_total,\n", + " \"per_person\": per_person,\n", + " }\n", + "\n", + "result = split_bill(Decimal(\"87.32\"), num_people=4, tip_pct=Decimal(\"18\"))\n", + "for key, value in result.items():\n", + " print(f\"{key:>12}: ${value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### When to Use Each Type\n", + "| Type | Use When | Example |\n", + "|------|----------|---------|\n", + "| `float` | Speed matters, small rounding is acceptable | Scientific computing, graphics |\n", + "| `Decimal` | Exact decimal precision is required | Money, financial calculations |\n", + "| `Fraction` | Exact rational arithmetic is needed | Math proofs, recipe scaling |\n", + "\n", + "### `Decimal` Key Points\n", + "- **Always create from strings**: `Decimal(\"0.1\")` not `Decimal(0.1)`\n", + "- **`quantize()`**: Round to a specific precision with a chosen rounding mode\n", + "- **Contexts**: `getcontext()` and `localcontext()` control precision and rounding\n", + "- **`InvalidOperation`**: Raised for invalid string conversions\n", + "\n", + "### `Fraction` Key Points\n", + "- **Auto-reduces**: `Fraction(4, 8)` becomes `Fraction(1, 2)`\n", + "- **Exact arithmetic**: `Fraction(1, 3) + Fraction(1, 6) == Fraction(1, 2)`\n", + "- **`limit_denominator()`**: Approximate a float with a cleaner fraction\n", + "- **Interoperability**: Works with `int` natively; mixing with `float` returns `float`\n", + "\n", + "### Conversions\n", + "- `float(fraction)` and `float(decimal)` convert to float (may lose precision)\n", + "- `Fraction(decimal)` converts Decimal to exact Fraction\n", + "- Create Decimal from Fraction via `Decimal(num) / Decimal(denom)`" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_32/03_random_and_statistics.ipynb b/src/chapter_32/03_random_and_statistics.ipynb new file mode 100644 index 0000000..0d379f6 --- /dev/null +++ b/src/chapter_32/03_random_and_statistics.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 32: Random and Statistics\n", + "\n", + "This notebook covers Python's `random` module for pseudorandom number generation and the `statistics` module for descriptive statistics. You will learn how to generate random values, sample from sequences, control reproducibility with seeds, and compute common statistical measures.\n", + "\n", + "## Key Concepts\n", + "- **Seeds**: `random.seed()` makes pseudorandom sequences reproducible\n", + "- **Random values**: `random()`, `randint()`, `uniform()`, `gauss()`\n", + "- **Sequence operations**: `choice()`, `choices()`, `sample()`, `shuffle()`\n", + "- **Distributions**: `gauss()`, `expovariate()`, `triangular()`\n", + "- **Descriptive statistics**: `mean()`, `median()`, `mode()`, `stdev()`, `variance()`\n", + "- **Advanced statistics**: `quantiles()`, `correlation()`, `linear_regression()`" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Pseudorandom Number Generation and Seeds\n", + "\n", + "The `random` module generates pseudorandom numbers using a deterministic algorithm. Setting a seed makes the sequence reproducible, which is essential for testing and debugging." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "# random() returns a float in [0.0, 1.0)\n", + "random.seed(42)\n", + "values: list[float] = [random.random() for _ in range(5)]\n", + "print(\"Five random floats [0, 1):\")\n", + "for i, v in enumerate(values):\n", + " print(f\" [{i}] {v:.6f}\")\n", + "\n", + "# Resetting the seed reproduces the same sequence\n", + "random.seed(42)\n", + "reproduced: list[float] = [random.random() for _ in range(5)]\n", + "print(f\"\\nReproducible: {values == reproduced}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "random.seed(42)\n", + "\n", + "# randint(a, b) returns an integer in [a, b] (inclusive)\n", + "dice_rolls: list[int] = [random.randint(1, 6) for _ in range(10)]\n", + "print(f\"Dice rolls: {dice_rolls}\")\n", + "\n", + "# randrange(start, stop, step) -- like range() but returns one random value\n", + "even: int = random.randrange(0, 100, 2)\n", + "print(f\"\\nRandom even [0, 100): {even}\")\n", + "\n", + "# uniform(a, b) returns a float in [a, b]\n", + "temperature: float = random.uniform(20.0, 35.0)\n", + "print(f\"Random temperature: {temperature:.2f} C\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Choosing and Sampling from Sequences\n", + "\n", + "The `random` module provides several ways to select elements from sequences: single selection, weighted selection, sampling without replacement, and in-place shuffling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "random.seed(42)\n", + "\n", + "colors: list[str] = [\"red\", \"green\", \"blue\", \"yellow\"]\n", + "\n", + "# choice() picks one element\n", + "picked: str = random.choice(colors)\n", + "print(f\"choice: {picked}\")\n", + "print(f\"picked is in colors: {picked in colors}\")\n", + "\n", + "# choices() picks k elements WITH replacement (may repeat)\n", + "picks: list[str] = random.choices(colors, k=6)\n", + "print(f\"\\nchoices(k=6): {picks}\")\n", + "\n", + "# Weighted choices\n", + "weights: list[int] = [10, 1, 1, 1] # red is 10x more likely\n", + "weighted: list[str] = random.choices(colors, weights=weights, k=10)\n", + "print(f\"\\nWeighted (red=10x): {weighted}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "random.seed(42)\n", + "\n", + "# sample() picks k UNIQUE elements (without replacement)\n", + "population: list[int] = list(range(100))\n", + "selected: list[int] = random.sample(population, k=5)\n", + "print(f\"sample(k=5): {selected}\")\n", + "print(f\"All unique: {len(selected) == len(set(selected))}\")\n", + "\n", + "# shuffle() randomizes a list IN PLACE\n", + "deck: list[str] = [\"A\", \"K\", \"Q\", \"J\", \"10\"]\n", + "print(f\"\\nBefore shuffle: {deck}\")\n", + "random.shuffle(deck)\n", + "print(f\"After shuffle: {deck}\")\n", + "\n", + "# The same elements are present, just reordered\n", + "print(f\"Same elements: {sorted(deck) == ['10', 'A', 'J', 'K', 'Q']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Random Distributions\n", + "\n", + "Beyond uniform random numbers, the `random` module can generate values from various probability distributions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "random.seed(42)\n", + "\n", + "# Gaussian (normal) distribution\n", + "gaussian_values: list[float] = [random.gauss(mu=0.0, sigma=1.0) for _ in range(10)]\n", + "print(\"Gaussian (mu=0, sigma=1):\")\n", + "for v in gaussian_values:\n", + " print(f\" {v:+.4f}\")\n", + "\n", + "# Triangular distribution (most values near the mode)\n", + "tri_values: list[float] = [\n", + " random.triangular(low=0.0, high=10.0, mode=7.0) for _ in range(5)\n", + "]\n", + "print(f\"\\nTriangular (low=0, high=10, mode=7):\")\n", + "for v in tri_values:\n", + " print(f\" {v:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "random.seed(42)\n", + "\n", + "# Exponential distribution (models time between events)\n", + "lambd: float = 1.0 / 5.0 # average 5 minutes between events\n", + "wait_times: list[float] = [random.expovariate(lambd) for _ in range(8)]\n", + "print(\"Exponential wait times (avg=5 min):\")\n", + "for t in wait_times:\n", + " print(f\" {t:.2f} min\")\n", + "\n", + "# Beta distribution (values between 0 and 1)\n", + "beta_values: list[float] = [random.betavariate(alpha=2, beta=5) for _ in range(5)]\n", + "print(f\"\\nBeta(alpha=2, beta=5):\")\n", + "for v in beta_values:\n", + " print(f\" {v:.4f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Basic Descriptive Statistics\n", + "\n", + "The `statistics` module (Python 3.4+) provides functions for calculating common statistical measures. It works with any iterable of numeric data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import statistics\n", + "\n", + "data: list[int] = [1, 2, 3, 4, 5]\n", + "\n", + "# Central tendency\n", + "print(f\"Data: {data}\")\n", + "print(f\"mean: {statistics.mean(data)}\")\n", + "print(f\"median: {statistics.median(data)}\")\n", + "\n", + "# Median with even-length data returns average of two middle values\n", + "even_data: list[int] = [1, 3, 5, 7]\n", + "print(f\"\\nData: {even_data}\")\n", + "print(f\"median: {statistics.median(even_data)}\")\n", + "\n", + "# Mode -- most frequent value\n", + "grades: list[str] = [\"A\", \"B\", \"A\", \"C\", \"A\", \"B\"]\n", + "print(f\"\\nGrades: {grades}\")\n", + "print(f\"mode: {statistics.mode(grades)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import statistics\n", + "\n", + "# Different types of mean\n", + "values: list[float] = [1.0, 2.0, 4.0, 8.0]\n", + "\n", + "print(f\"Data: {values}\")\n", + "print(f\"Arithmetic mean: {statistics.mean(values)}\")\n", + "print(f\"Geometric mean: {statistics.geometric_mean(values):.4f}\")\n", + "print(f\"Harmonic mean: {statistics.harmonic_mean(values):.4f}\")\n", + "\n", + "# Median variants\n", + "data: list[int] = [1, 3, 5, 7]\n", + "print(f\"\\nData: {data}\")\n", + "print(f\"median: {statistics.median(data)}\")\n", + "print(f\"median_low: {statistics.median_low(data)}\")\n", + "print(f\"median_high: {statistics.median_high(data)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: Spread and Variability\n", + "\n", + "Standard deviation and variance measure how spread out data is. The `statistics` module provides both sample and population variants." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import statistics\n", + "\n", + "data: list[int] = [2, 4, 4, 4, 5, 5, 7, 9]\n", + "\n", + "# Sample standard deviation and variance (N-1 denominator)\n", + "print(f\"Data: {data}\")\n", + "print(f\"stdev: {statistics.stdev(data):.4f}\")\n", + "print(f\"variance: {statistics.variance(data):.4f}\")\n", + "\n", + "# Population standard deviation and variance (N denominator)\n", + "print(f\"\\npstdev: {statistics.pstdev(data):.4f}\")\n", + "print(f\"pvariance: {statistics.pvariance(data):.4f}\")\n", + "\n", + "# Verify stdev is approximately 2.0\n", + "print(f\"\\nstdev close to 2.0: {abs(statistics.stdev(data) - 2.0) < 0.2}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 6: Quantiles and Advanced Statistics\n", + "\n", + "Python 3.8+ added `quantiles()` for splitting data into equal groups, and Python 3.10+ added `correlation()` and `linear_regression()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import statistics\n", + "\n", + "data: list[int] = list(range(1, 101)) # 1 to 100\n", + "\n", + "# Quartiles (split into 4 groups)\n", + "quartiles: list[float] = statistics.quantiles(data, n=4)\n", + "print(f\"Quartiles (Q1, Q2, Q3): {quartiles}\")\n", + "\n", + "# Deciles (split into 10 groups)\n", + "deciles: list[float] = statistics.quantiles(data, n=10)\n", + "print(f\"Deciles: {deciles}\")\n", + "\n", + "# Percentiles using quantiles\n", + "percentiles: list[float] = statistics.quantiles(data, n=100)\n", + "print(f\"\\n25th percentile: {percentiles[24]}\")\n", + "print(f\"50th percentile: {percentiles[49]}\")\n", + "print(f\"75th percentile: {percentiles[74]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import statistics\n", + "\n", + "# Correlation between two variables (Python 3.10+)\n", + "hours_studied: list[float] = [1, 2, 3, 4, 5, 6, 7, 8]\n", + "test_scores: list[float] = [52, 58, 65, 70, 74, 80, 85, 90]\n", + "\n", + "r: float = statistics.correlation(hours_studied, test_scores)\n", + "print(f\"Correlation (hours vs scores): {r:.4f}\")\n", + "print(f\"Strong positive correlation: {r > 0.9}\")\n", + "\n", + "# Linear regression (Python 3.10+)\n", + "slope, intercept = statistics.linear_regression(hours_studied, test_scores)\n", + "print(f\"\\nLinear regression: score = {slope:.2f} * hours + {intercept:.2f}\")\n", + "\n", + "# Predict score for 10 hours of study\n", + "predicted: float = slope * 10 + intercept\n", + "print(f\"Predicted score for 10 hours: {predicted:.1f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 7: Practical Patterns\n", + "\n", + "Combining `random` and `statistics` for common real-world tasks like simulations and data analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import statistics\n", + "\n", + "def simulate_dice_rolls(num_rolls: int, seed: int = 42) -> dict[str, float]:\n", + " \"\"\"Simulate dice rolls and return summary statistics.\"\"\"\n", + " random.seed(seed)\n", + " rolls: list[int] = [random.randint(1, 6) for _ in range(num_rolls)]\n", + " return {\n", + " \"count\": float(len(rolls)),\n", + " \"mean\": statistics.mean(rolls),\n", + " \"median\": statistics.median(rolls),\n", + " \"stdev\": statistics.stdev(rolls),\n", + " \"min\": float(min(rolls)),\n", + " \"max\": float(max(rolls)),\n", + " }\n", + "\n", + "# Small sample\n", + "small: dict[str, float] = simulate_dice_rolls(20)\n", + "print(\"20 rolls:\")\n", + "for key, value in small.items():\n", + " print(f\" {key:>7}: {value:.2f}\")\n", + "\n", + "# Large sample (law of large numbers: mean approaches 3.5)\n", + "large: dict[str, float] = simulate_dice_rolls(100_000)\n", + "print(\"\\n100,000 rolls:\")\n", + "for key, value in large.items():\n", + " print(f\" {key:>7}: {value:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "def generate_password(\n", + " length: int = 16,\n", + " seed: int | None = None,\n", + ") -> str:\n", + " \"\"\"Generate a random password from printable ASCII characters.\"\"\"\n", + " if seed is not None:\n", + " random.seed(seed)\n", + " chars: str = (\n", + " \"abcdefghijklmnopqrstuvwxyz\"\n", + " \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n", + " \"0123456789\"\n", + " \"!@#$%^&*\"\n", + " )\n", + " return \"\".join(random.choices(chars, k=length))\n", + "\n", + "# Generate a few passwords\n", + "random.seed(42)\n", + "for i in range(5):\n", + " print(f\"Password {i + 1}: {generate_password()}\")\n", + "\n", + "# Note: for real security, use the secrets module instead\n", + "import secrets\n", + "secure_token: str = secrets.token_urlsafe(16)\n", + "print(f\"\\nSecure token (secrets): {secure_token}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `random` Module\n", + "- **`seed(n)`**: Set the random seed for reproducibility\n", + "- **`random()`**: Float in [0.0, 1.0)\n", + "- **`randint(a, b)`**: Integer in [a, b] inclusive\n", + "- **`uniform(a, b)`**: Float in [a, b]\n", + "- **`choice(seq)`**: Pick one element from a sequence\n", + "- **`choices(seq, k=n)`**: Pick k elements with replacement\n", + "- **`sample(seq, k=n)`**: Pick k unique elements without replacement\n", + "- **`shuffle(list)`**: Randomize a list in place\n", + "- **`gauss(mu, sigma)`**: Gaussian/normal distribution\n", + "- **`expovariate(lambd)`**: Exponential distribution\n", + "\n", + "### `statistics` Module\n", + "- **Central tendency**: `mean()`, `median()`, `mode()`, `geometric_mean()`, `harmonic_mean()`\n", + "- **Spread**: `stdev()`, `variance()`, `pstdev()`, `pvariance()`\n", + "- **Median variants**: `median_low()`, `median_high()`\n", + "- **Quantiles**: `quantiles(data, n=4)` for quartiles, percentiles, etc.\n", + "- **Regression**: `correlation()`, `linear_regression()` (Python 3.10+)\n", + "\n", + "### Important Notes\n", + "- `random` is **not** cryptographically secure -- use `secrets` for security\n", + "- `statistics.stdev()` uses **sample** standard deviation (N-1); use `pstdev()` for population\n", + "- Always set a seed in tests and simulations for reproducibility\n", + "- `sample()` raises `ValueError` if k > population size; `choices()` does not" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_32/README.md b/src/chapter_32/README.md new file mode 100644 index 0000000..2ba6c91 --- /dev/null +++ b/src/chapter_32/README.md @@ -0,0 +1,9 @@ +# Chapter 32: Numeric Computing + +Numeric types and mathematical operations — math module, decimal for precision, fractions for exact rationals, random for pseudorandom numbers, and statistics for descriptive stats. + +## Notebooks + +1. **01_math_and_cmath.ipynb** — math functions, constants, cmath for complex numbers, number theory +2. **02_decimal_and_fractions.ipynb** — Decimal for financial precision, Fraction for exact rationals, contexts +3. **03_random_and_statistics.ipynb** — random module, distributions, statistics module, sampling diff --git a/src/chapter_32/__init__.py b/src/chapter_32/__init__.py new file mode 100644 index 0000000..dbd0de5 --- /dev/null +++ b/src/chapter_32/__init__.py @@ -0,0 +1 @@ +"""Chapter 32: Numeric Computing.""" diff --git a/src/chapter_33/01_os_and_platform.ipynb b/src/chapter_33/01_os_and_platform.ipynb new file mode 100644 index 0000000..0b767be --- /dev/null +++ b/src/chapter_33/01_os_and_platform.ipynb @@ -0,0 +1,501 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 33: OS and Platform\n", + "\n", + "This notebook covers Python's `os`, `os.path`, `platform`, and `sys` modules for interacting with the operating system. You will learn how to inspect environment variables, manipulate file paths, traverse directory trees, and query system information.\n", + "\n", + "## Key Concepts\n", + "- **`os.environ`**: A mapping of environment variables\n", + "- **`os.path`**: Portable path manipulation (join, split, basename, dirname, splitext)\n", + "- **`os.getcwd()`** / **`os.listdir()`**: Working directory and directory listing\n", + "- **`os.walk()`**: Recursive directory tree traversal\n", + "- **`platform`**: System identification (OS name, architecture, Python version)\n", + "- **`sys`**: Interpreter-level information (version, path, executable)" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Working Directory and Directory Listing\n", + "\n", + "`os.getcwd()` returns the current working directory as a string. `os.listdir()` returns the names of entries in a given directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Get the current working directory\n", + "cwd: str = os.getcwd()\n", + "print(f\"Current working directory: {cwd}\")\n", + "print(f\"Is a directory: {os.path.isdir(cwd)}\")\n", + "\n", + "# List entries in the current directory\n", + "entries: list[str] = os.listdir(cwd)\n", + "print(f\"\\nNumber of entries: {len(entries)}\")\n", + "print(f\"First 5 entries: {sorted(entries)[:5]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e2f3a4", + "metadata": {}, + "source": [ + "## Section 2: Environment Variables with `os.environ`\n", + "\n", + "`os.environ` is a mapping object representing the environment. You can read variables with bracket access or `.get()`, and set them by assignment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# os.environ behaves like a dictionary\n", + "print(f\"Type: {type(os.environ)}\")\n", + "print(f\"'PATH' in environ: {'PATH' in os.environ}\")\n", + "\n", + "# Read an environment variable safely with .get()\n", + "home: str | None = os.environ.get(\"HOME\")\n", + "print(f\"HOME: {home}\")\n", + "\n", + "# Read with a default for missing variables\n", + "editor: str = os.environ.get(\"EDITOR\", \"not set\")\n", + "print(f\"EDITOR: {editor}\")\n", + "\n", + "# List a few environment variable names\n", + "env_keys: list[str] = sorted(os.environ.keys())\n", + "print(f\"\\nSample env vars: {env_keys[:5]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set and read a custom environment variable\n", + "os.environ[\"MY_APP_MODE\"] = \"testing\"\n", + "print(f\"MY_APP_MODE: {os.environ['MY_APP_MODE']}\")\n", + "\n", + "# Remove it when done\n", + "del os.environ[\"MY_APP_MODE\"]\n", + "print(f\"After delete: {os.environ.get('MY_APP_MODE', 'not found')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: Path Manipulation with `os.path`\n", + "\n", + "`os.path` provides functions for portable path manipulation. These work on path strings without requiring the path to actually exist on disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Join path components portably\n", + "path: str = os.path.join(\"src\", \"chapter_33\", \"__init__.py\")\n", + "print(f\"Joined path: {path}\")\n", + "\n", + "# Extract the base name (last component)\n", + "base: str = os.path.basename(path)\n", + "print(f\"Basename: {base}\")\n", + "\n", + "# Extract the directory name (everything except last component)\n", + "directory: str = os.path.dirname(path)\n", + "print(f\"Dirname: {directory}\")\n", + "\n", + "# Split into (directory, basename)\n", + "head, tail = os.path.split(path)\n", + "print(f\"\\nsplit -> head: {head!r}, tail: {tail!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Split name and extension\n", + "name, ext = os.path.splitext(\"script.py\")\n", + "print(f\"Name: {name!r}, Extension: {ext!r}\")\n", + "\n", + "# Works with full paths too\n", + "name2, ext2 = os.path.splitext(\"/home/user/data.tar.gz\")\n", + "print(f\"Name: {name2!r}, Extension: {ext2!r}\")\n", + "\n", + "# Absolute path and normalization\n", + "rel: str = os.path.join(\"src\", \"..\", \"tests\")\n", + "normalized: str = os.path.normpath(rel)\n", + "absolute: str = os.path.abspath(rel)\n", + "print(f\"\\nRelative: {rel}\")\n", + "print(f\"Normalized: {normalized}\")\n", + "print(f\"Absolute: {absolute}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Check if paths exist and their types\n", + "cwd: str = os.getcwd()\n", + "\n", + "print(f\"exists('{cwd}'): {os.path.exists(cwd)}\")\n", + "print(f\"isdir('{cwd}'): {os.path.isdir(cwd)}\")\n", + "print(f\"isfile('{cwd}'): {os.path.isfile(cwd)}\")\n", + "\n", + "# Check a non-existent path\n", + "fake: str = os.path.join(cwd, \"does_not_exist.txt\")\n", + "print(f\"\\nexists(fake): {os.path.exists(fake)}\")\n", + "print(f\"isabs(fake): {os.path.isabs(fake)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Creating and Removing Directories\n", + "\n", + "`os.makedirs()` creates directories (including parents), and `os.rmdir()` removes empty directories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tempfile\n", + "\n", + "# Use a temporary directory so we don't litter the filesystem\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " # Create nested directories in one call\n", + " nested: str = os.path.join(tmpdir, \"level1\", \"level2\", \"level3\")\n", + " os.makedirs(nested)\n", + " print(f\"Created: {nested}\")\n", + " print(f\"Exists: {os.path.isdir(nested)}\")\n", + "\n", + " # exist_ok=True prevents errors if directory already exists\n", + " os.makedirs(nested, exist_ok=True)\n", + " print(\"makedirs with exist_ok=True succeeded\")\n", + "\n", + " # Remove the innermost empty directory\n", + " os.rmdir(nested)\n", + " print(f\"\\nAfter rmdir, exists: {os.path.isdir(nested)}\")\n", + "\n", + "print(f\"\\nTempdir cleaned up: {not os.path.exists(tmpdir)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 5: Walking Directory Trees with `os.walk()`\n", + "\n", + "`os.walk()` generates `(dirpath, dirnames, filenames)` tuples for every directory in a tree. It is the standard way to recursively process a file system hierarchy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tempfile\n", + "\n", + "# Build a small directory tree\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " os.makedirs(os.path.join(tmpdir, \"sub\"))\n", + " open(os.path.join(tmpdir, \"file.txt\"), \"w\").close()\n", + " open(os.path.join(tmpdir, \"sub\", \"nested.txt\"), \"w\").close()\n", + "\n", + " # Walk the tree\n", + " dirs_found: list[str] = []\n", + " files_found: list[str] = []\n", + "\n", + " for dirpath, dirnames, filenames in os.walk(tmpdir):\n", + " dirs_found.append(dirpath)\n", + " for fname in filenames:\n", + " full_path: str = os.path.join(dirpath, fname)\n", + " files_found.append(full_path)\n", + " print(f\"Directory: {os.path.relpath(dirpath, tmpdir)}\")\n", + " print(f\" Subdirs: {dirnames}\")\n", + " print(f\" Files: {filenames}\")\n", + "\n", + " print(f\"\\nTotal directories visited: {len(dirs_found)}\")\n", + " print(f\"Total files found: {len(files_found)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tempfile\n", + "\n", + "def find_files_by_extension(root: str, extension: str) -> list[str]:\n", + " \"\"\"Find all files with a given extension under root.\"\"\"\n", + " matches: list[str] = []\n", + " for dirpath, dirnames, filenames in os.walk(root):\n", + " for fname in filenames:\n", + " if fname.endswith(extension):\n", + " matches.append(os.path.join(dirpath, fname))\n", + " return matches\n", + "\n", + "# Demonstrate with a temporary tree\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " # Create test files\n", + " for name in [\"app.py\", \"utils.py\", \"readme.md\", \"data.csv\"]:\n", + " open(os.path.join(tmpdir, name), \"w\").close()\n", + " os.makedirs(os.path.join(tmpdir, \"pkg\"))\n", + " open(os.path.join(tmpdir, \"pkg\", \"models.py\"), \"w\").close()\n", + "\n", + " py_files: list[str] = find_files_by_extension(tmpdir, \".py\")\n", + " print(f\"Python files found: {len(py_files)}\")\n", + " for f in py_files:\n", + " print(f\" {os.path.relpath(f, tmpdir)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 6: Platform and System Information\n", + "\n", + "The `platform` module provides detailed information about the operating system, hardware, and Python interpreter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import platform\n", + "\n", + "# Operating system identification\n", + "print(f\"System: {platform.system()}\") # e.g., 'Darwin', 'Linux', 'Windows'\n", + "print(f\"Release: {platform.release()}\")\n", + "print(f\"Version: {platform.version()}\")\n", + "print(f\"Machine: {platform.machine()}\") # e.g., 'x86_64', 'arm64'\n", + "print(f\"Processor: {platform.processor()}\")\n", + "print(f\"Platform: {platform.platform()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import platform\n", + "import sys\n", + "\n", + "# Python version information\n", + "print(f\"Python version (platform): {platform.python_version()}\")\n", + "print(f\"Python version (sys): {sys.version}\")\n", + "\n", + "# sys.version_info gives structured access\n", + "vi = sys.version_info\n", + "print(f\"\\nVersion info: major={vi.major}, minor={vi.minor}, micro={vi.micro}\")\n", + "\n", + "# Verify they agree\n", + "expected: str = f\"{vi.major}.{vi.minor}.{vi.micro}\"\n", + "matches: bool = platform.python_version() == expected\n", + "print(f\"platform matches sys: {matches}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 7: The `sys` Module\n", + "\n", + "The `sys` module provides access to Python interpreter internals -- the module search path, the executable path, and more." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python executable and prefix\n", + "print(f\"Executable: {sys.executable}\")\n", + "print(f\"Prefix: {sys.prefix}\")\n", + "print(f\"Platform: {sys.platform}\") # e.g., 'darwin', 'linux', 'win32'\n", + "\n", + "# Module search path (first few entries)\n", + "print(f\"\\nModule search path (sys.path):\")\n", + "for i, p in enumerate(sys.path[:5]):\n", + " print(f\" [{i}] {p}\")\n", + "if len(sys.path) > 5:\n", + " print(f\" ... ({len(sys.path)} total entries)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Integer and recursion limits\n", + "print(f\"Max integer size (for indexing): {sys.maxsize}\")\n", + "print(f\"Recursion limit: {sys.getrecursionlimit()}\")\n", + "print(f\"Byte order: {sys.byteorder}\")\n", + "\n", + "# Size of objects in bytes\n", + "sample_int: int = 42\n", + "sample_str: str = \"hello\"\n", + "sample_list: list[int] = [1, 2, 3]\n", + "\n", + "print(f\"\\nSize of int 42: {sys.getsizeof(sample_int)} bytes\")\n", + "print(f\"Size of str 'hello': {sys.getsizeof(sample_str)} bytes\")\n", + "print(f\"Size of list [1,2,3]: {sys.getsizeof(sample_list)} bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 8: Practical Pattern -- System Info Report\n", + "\n", + "Combining `os`, `platform`, and `sys` to build a system information report that could be useful for debugging or logging." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import platform\n", + "import sys\n", + "\n", + "\n", + "def system_info_report() -> dict[str, str]:\n", + " \"\"\"Collect system information into a dictionary.\"\"\"\n", + " return {\n", + " \"os_name\": platform.system(),\n", + " \"os_release\": platform.release(),\n", + " \"machine\": platform.machine(),\n", + " \"python_version\": platform.python_version(),\n", + " \"python_executable\": sys.executable,\n", + " \"cwd\": os.getcwd(),\n", + " \"home\": os.environ.get(\"HOME\", os.environ.get(\"USERPROFILE\", \"unknown\")),\n", + " }\n", + "\n", + "\n", + "report: dict[str, str] = system_info_report()\n", + "print(\"System Information Report\")\n", + "print(\"=\" * 40)\n", + "for key, value in report.items():\n", + " print(f\"{key:>20}: {value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `os` Module\n", + "- **`os.getcwd()`**: Returns the current working directory\n", + "- **`os.listdir(path)`**: Lists entries in a directory\n", + "- **`os.environ`**: Dictionary-like mapping of environment variables\n", + "- **`os.makedirs(path, exist_ok=True)`**: Creates nested directories\n", + "- **`os.walk(top)`**: Recursively yields `(dirpath, dirnames, filenames)` tuples\n", + "\n", + "### `os.path` Module\n", + "- **`join()`**: Combines path components portably\n", + "- **`basename()` / `dirname()`**: Extracts the last component or everything before it\n", + "- **`splitext()`**: Splits filename and extension\n", + "- **`exists()` / `isdir()` / `isfile()`**: Tests for path existence and type\n", + "- **`abspath()` / `normpath()`**: Converts to absolute or normalized form\n", + "\n", + "### `platform` Module\n", + "- **`system()`**: OS name (`'Darwin'`, `'Linux'`, `'Windows'`)\n", + "- **`machine()`**: Hardware architecture (`'x86_64'`, `'arm64'`)\n", + "- **`python_version()`**: Python version string\n", + "\n", + "### `sys` Module\n", + "- **`sys.executable`**: Path to the Python interpreter\n", + "- **`sys.path`**: Module search path list\n", + "- **`sys.version_info`**: Structured version information\n", + "- **`sys.getsizeof()`**: Memory size of an object in bytes" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_33/02_shutil_and_tempfile.ipynb b/src/chapter_33/02_shutil_and_tempfile.ipynb new file mode 100644 index 0000000..9d2010a --- /dev/null +++ b/src/chapter_33/02_shutil_and_tempfile.ipynb @@ -0,0 +1,511 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 33: Shutil and Tempfile\n", + "\n", + "This notebook covers Python's `shutil` module for high-level file and directory operations and the `tempfile` module for creating temporary files and directories. Together they provide the tools needed for safe, portable file management.\n", + "\n", + "## Key Concepts\n", + "- **`shutil.copy()`** / **`shutil.copy2()`**: Copy files (with or without metadata)\n", + "- **`shutil.copytree()`**: Recursively copy entire directory trees\n", + "- **`shutil.move()`**: Move or rename files and directories\n", + "- **`shutil.rmtree()`**: Recursively delete a directory tree\n", + "- **`shutil.disk_usage()`**: Query disk space (total, used, free)\n", + "- **`tempfile.NamedTemporaryFile`**: Create a temporary file with a name on disk\n", + "- **`tempfile.TemporaryDirectory`**: Create a temporary directory that auto-cleans" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Copying Files with `shutil.copy()`\n", + "\n", + "`shutil.copy(src, dst)` copies a file's content and permissions. `shutil.copy2()` additionally preserves metadata (timestamps). Both return the path to the destination." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " # Create a source file\n", + " src: str = os.path.join(tmpdir, \"source.txt\")\n", + " with open(src, \"w\") as f:\n", + " f.write(\"hello\")\n", + "\n", + " # Copy to a new file\n", + " dst: str = os.path.join(tmpdir, \"dest.txt\")\n", + " result: str = shutil.copy(src, dst)\n", + "\n", + " print(f\"Source: {os.path.basename(src)}\")\n", + " print(f\"Destination: {os.path.basename(result)}\")\n", + "\n", + " # Verify contents match\n", + " with open(dst) as f:\n", + " content: str = f.read()\n", + " print(f\"Content: {content!r}\")\n", + " print(f\"Files match: {content == 'hello'}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "import time\n", + "\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " src: str = os.path.join(tmpdir, \"original.txt\")\n", + " with open(src, \"w\") as f:\n", + " f.write(\"metadata test\")\n", + "\n", + " # Small delay so timestamps differ if not preserved\n", + " time.sleep(0.1)\n", + "\n", + " # copy() does NOT preserve modification time\n", + " dst_copy: str = os.path.join(tmpdir, \"via_copy.txt\")\n", + " shutil.copy(src, dst_copy)\n", + "\n", + " # copy2() DOES preserve modification time\n", + " dst_copy2: str = os.path.join(tmpdir, \"via_copy2.txt\")\n", + " shutil.copy2(src, dst_copy2)\n", + "\n", + " src_mtime: float = os.path.getmtime(src)\n", + " copy_mtime: float = os.path.getmtime(dst_copy)\n", + " copy2_mtime: float = os.path.getmtime(dst_copy2)\n", + "\n", + " print(f\"Source mtime: {src_mtime:.4f}\")\n", + " print(f\"copy() mtime: {copy_mtime:.4f}\")\n", + " print(f\"copy2() mtime: {copy2_mtime:.4f}\")\n", + " print(f\"\\ncopy2 preserved: {abs(src_mtime - copy2_mtime) < 0.01}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Copying Directory Trees with `shutil.copytree()`\n", + "\n", + "`shutil.copytree(src, dst)` recursively copies an entire directory tree. The destination must not already exist (unless `dirs_exist_ok=True` is used)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " # Build a source tree\n", + " src_dir: str = os.path.join(tmpdir, \"project\")\n", + " os.makedirs(os.path.join(src_dir, \"pkg\"))\n", + " with open(os.path.join(src_dir, \"main.py\"), \"w\") as f:\n", + " f.write(\"print('hello')\")\n", + " with open(os.path.join(src_dir, \"pkg\", \"utils.py\"), \"w\") as f:\n", + " f.write(\"def helper(): pass\")\n", + "\n", + " # Copy the entire tree\n", + " dst_dir: str = os.path.join(tmpdir, \"project_backup\")\n", + " shutil.copytree(src_dir, dst_dir)\n", + "\n", + " # Verify the copy\n", + " print(\"Copied tree contents:\")\n", + " for dirpath, dirnames, filenames in os.walk(dst_dir):\n", + " level: int = dirpath.replace(dst_dir, \"\").count(os.sep)\n", + " indent: str = \" \" * level\n", + " print(f\"{indent}{os.path.basename(dirpath)}/\")\n", + " for fname in filenames:\n", + " print(f\"{indent} {fname}\")\n", + "\n", + " # Check a file in the copy\n", + " copied_file: str = os.path.join(dst_dir, \"pkg\", \"utils.py\")\n", + " print(f\"\\nCopied file exists: {os.path.exists(copied_file)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: Moving and Renaming with `shutil.move()`\n", + "\n", + "`shutil.move(src, dst)` moves a file or directory. It works across filesystems (unlike `os.rename()`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " # Create a file and a subdirectory\n", + " src: str = os.path.join(tmpdir, \"report.txt\")\n", + " with open(src, \"w\") as f:\n", + " f.write(\"quarterly results\")\n", + "\n", + " archive_dir: str = os.path.join(tmpdir, \"archive\")\n", + " os.makedirs(archive_dir)\n", + "\n", + " # Move the file into the archive directory\n", + " dst: str = os.path.join(archive_dir, \"report.txt\")\n", + " shutil.move(src, dst)\n", + "\n", + " print(f\"Source exists: {os.path.exists(src)}\")\n", + " print(f\"Destination exists: {os.path.exists(dst)}\")\n", + "\n", + " # Rename by moving to a new name\n", + " renamed: str = os.path.join(archive_dir, \"q1_report.txt\")\n", + " shutil.move(dst, renamed)\n", + " print(f\"\\nRenamed file exists: {os.path.exists(renamed)}\")\n", + " print(f\"Archive contents: {os.listdir(archive_dir)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 4: Removing Directory Trees with `shutil.rmtree()`\n", + "\n", + "`shutil.rmtree(path)` recursively deletes a directory and all its contents. Unlike `os.rmdir()`, it works on non-empty directories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " # Create a non-empty directory tree\n", + " target: str = os.path.join(tmpdir, \"build_output\")\n", + " os.makedirs(os.path.join(target, \"assets\", \"images\"))\n", + " for name in [\"index.html\", \"assets/style.css\", \"assets/images/logo.png\"]:\n", + " filepath: str = os.path.join(target, name)\n", + " open(filepath, \"w\").close()\n", + "\n", + " print(f\"Before rmtree: {os.path.isdir(target)}\")\n", + "\n", + " # Count files before removal\n", + " file_count: int = sum(len(files) for _, _, files in os.walk(target))\n", + " print(f\"Files in tree: {file_count}\")\n", + "\n", + " # Remove the entire tree\n", + " shutil.rmtree(target)\n", + " print(f\"After rmtree: {os.path.exists(target)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 5: Disk Usage with `shutil.disk_usage()`\n", + "\n", + "`shutil.disk_usage(path)` returns a named tuple with `total`, `used`, and `free` disk space in bytes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "\n", + "# Query disk usage for the root filesystem\n", + "usage = shutil.disk_usage(\"/\")\n", + "\n", + "print(f\"Total: {usage.total:>15,} bytes ({usage.total / (1024**3):.1f} GB)\")\n", + "print(f\"Used: {usage.used:>15,} bytes ({usage.used / (1024**3):.1f} GB)\")\n", + "print(f\"Free: {usage.free:>15,} bytes ({usage.free / (1024**3):.1f} GB)\")\n", + "\n", + "# Calculate percentage used\n", + "pct_used: float = (usage.used / usage.total) * 100\n", + "print(f\"\\nDisk usage: {pct_used:.1f}%\")\n", + "\n", + "# All fields are positive\n", + "print(f\"\\ntotal > 0: {usage.total > 0}\")\n", + "print(f\"used > 0: {usage.used > 0}\")\n", + "print(f\"free > 0: {usage.free > 0}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 6: Temporary Files with `tempfile.NamedTemporaryFile`\n", + "\n", + "`NamedTemporaryFile` creates a temporary file that has a visible name on the filesystem. By default it is deleted when closed; use `delete=False` to keep it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tempfile\n", + "\n", + "# NamedTemporaryFile with automatic cleanup (default)\n", + "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".txt\") as f:\n", + " f.write(\"temporary data\")\n", + " f.flush() # Ensure data is written to disk\n", + " temp_name: str = f.name\n", + " print(f\"Temp file name: {temp_name}\")\n", + " print(f\"Exists inside context: {os.path.exists(temp_name)}\")\n", + "\n", + "print(f\"Exists after context: {os.path.exists(temp_name)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tempfile\n", + "\n", + "# NamedTemporaryFile with delete=False for manual control\n", + "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".txt\", delete=False) as f:\n", + " f.write(\"persistent temp data\")\n", + " name: str = f.name\n", + "\n", + "# File persists after closing\n", + "print(f\"File: {name}\")\n", + "print(f\"Exists after close: {os.path.exists(name)}\")\n", + "\n", + "# Read it back\n", + "with open(name) as f:\n", + " content: str = f.read()\n", + "print(f\"Content: {content!r}\")\n", + "\n", + "# Clean up manually\n", + "os.unlink(name)\n", + "print(f\"After unlink: {os.path.exists(name)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 7: Temporary Directories with `tempfile.TemporaryDirectory`\n", + "\n", + "`TemporaryDirectory` creates a temporary directory that is automatically deleted (including all its contents) when the context manager exits." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tempfile\n", + "\n", + "# TemporaryDirectory auto-cleans everything inside it\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " print(f\"Temp dir: {tmpdir}\")\n", + " print(f\"Is a directory: {os.path.isdir(tmpdir)}\")\n", + "\n", + " # Create files and subdirectories inside it\n", + " os.makedirs(os.path.join(tmpdir, \"subdir\"))\n", + " with open(os.path.join(tmpdir, \"data.txt\"), \"w\") as f:\n", + " f.write(\"will be cleaned up\")\n", + " with open(os.path.join(tmpdir, \"subdir\", \"nested.txt\"), \"w\") as f:\n", + " f.write(\"also cleaned up\")\n", + "\n", + " contents: list[str] = os.listdir(tmpdir)\n", + " print(f\"Contents: {contents}\")\n", + " saved_path: str = tmpdir\n", + "\n", + "# Everything is gone after exiting the context\n", + "print(f\"\\nAfter exit, exists: {os.path.exists(saved_path)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "\n", + "# Customize the prefix and suffix of temp names\n", + "with tempfile.TemporaryDirectory(prefix=\"myapp_\", suffix=\"_work\") as tmpdir:\n", + " print(f\"Custom temp dir: {tmpdir}\")\n", + "\n", + "# tempfile.gettempdir() shows the default temp directory\n", + "print(f\"System temp dir: {tempfile.gettempdir()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 8: Practical Pattern -- Safe File Processing\n", + "\n", + "A common pattern is to write to a temporary file first, then move it into place. This prevents partial writes from corrupting the target file if something fails." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "\n", + "def safe_write(filepath: str, content: str) -> None:\n", + " \"\"\"Write content to filepath atomically using a temp file.\"\"\"\n", + " directory: str = os.path.dirname(filepath)\n", + " with tempfile.NamedTemporaryFile(\n", + " mode=\"w\", dir=directory, suffix=\".tmp\", delete=False\n", + " ) as tmp:\n", + " tmp.write(content)\n", + " tmp_name: str = tmp.name\n", + " # Move the temp file to the final destination\n", + " shutil.move(tmp_name, filepath)\n", + "\n", + "\n", + "# Demonstrate the pattern\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " target: str = os.path.join(tmpdir, \"config.json\")\n", + " safe_write(target, '{\"key\": \"value\"}')\n", + "\n", + " with open(target) as f:\n", + " print(f\"Written content: {f.read()}\")\n", + " print(f\"File exists: {os.path.exists(target)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "\n", + "def backup_and_update(filepath: str, new_content: str) -> str:\n", + " \"\"\"Create a backup of a file, then update it. Returns backup path.\"\"\"\n", + " backup_path: str = filepath + \".bak\"\n", + " if os.path.exists(filepath):\n", + " shutil.copy2(filepath, backup_path)\n", + " with open(filepath, \"w\") as f:\n", + " f.write(new_content)\n", + " return backup_path\n", + "\n", + "\n", + "with tempfile.TemporaryDirectory() as tmpdir:\n", + " config: str = os.path.join(tmpdir, \"settings.ini\")\n", + "\n", + " # Create original file\n", + " with open(config, \"w\") as f:\n", + " f.write(\"version=1\")\n", + "\n", + " # Update with backup\n", + " backup: str = backup_and_update(config, \"version=2\")\n", + "\n", + " with open(config) as f:\n", + " print(f\"Current: {f.read()}\")\n", + " with open(backup) as f:\n", + " print(f\"Backup: {f.read()}\")\n", + " print(f\"\\nFiles: {sorted(os.listdir(tmpdir))}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `shutil` Module\n", + "- **`shutil.copy(src, dst)`**: Copy file content and permissions\n", + "- **`shutil.copy2(src, dst)`**: Copy file content, permissions, and metadata (timestamps)\n", + "- **`shutil.copytree(src, dst)`**: Recursively copy an entire directory tree\n", + "- **`shutil.move(src, dst)`**: Move or rename a file or directory (works across filesystems)\n", + "- **`shutil.rmtree(path)`**: Recursively delete a directory and all its contents\n", + "- **`shutil.disk_usage(path)`**: Returns named tuple with `total`, `used`, `free` bytes\n", + "\n", + "### `tempfile` Module\n", + "- **`NamedTemporaryFile(mode, suffix, delete)`**: Temporary file with a visible name on disk\n", + " - `delete=True` (default): removed on close\n", + " - `delete=False`: persists until manually removed with `os.unlink()`\n", + "- **`TemporaryDirectory(prefix, suffix)`**: Temporary directory that auto-cleans on context exit\n", + "- **`tempfile.gettempdir()`**: Returns the system's default temporary directory\n", + "\n", + "### Key Patterns\n", + "- Write to a temp file, then `shutil.move()` into place for atomic writes\n", + "- Use `TemporaryDirectory` as a context manager for scratch space in tests\n", + "- Use `copy2()` when you need to preserve file timestamps" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_33/03_subprocess.ipynb b/src/chapter_33/03_subprocess.ipynb new file mode 100644 index 0000000..9843733 --- /dev/null +++ b/src/chapter_33/03_subprocess.ipynb @@ -0,0 +1,624 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 33: Subprocess\n", + "\n", + "This notebook covers Python's `subprocess` module for running external commands from within Python. You will learn how to execute commands, capture their output, handle errors, and work with pipes.\n", + "\n", + "## Key Concepts\n", + "- **`subprocess.run()`**: The recommended high-level function for running commands\n", + "- **`capture_output`** / **`text`**: Capturing stdout and stderr as strings\n", + "- **`check=True`** / **`check_returncode()`**: Raising exceptions on command failure\n", + "- **`subprocess.Popen`**: Low-level process management with direct pipe control\n", + "- **Piping**: Connecting the output of one process to the input of another\n", + "- **Timeouts**: Preventing commands from running indefinitely" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Running Commands with `subprocess.run()`\n", + "\n", + "`subprocess.run()` is the primary way to run external commands. It waits for the command to complete and returns a `CompletedProcess` object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "# Run a simple command\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [\"echo\", \"hello\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(f\"Return code: {result.returncode}\")\n", + "print(f\"Stdout: {result.stdout.strip()!r}\")\n", + "print(f\"Stderr: {result.stderr.strip()!r}\")\n", + "print(f\"Success: {result.returncode == 0}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "# The args parameter is the command as a list of strings\n", + "# Each element is a separate argument (no shell splitting needed)\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [\"python3\", \"-c\", \"print('Hello from Python subprocess!')\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(f\"Output: {result.stdout.strip()}\")\n", + "print(f\"Args used: {result.args}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Capturing Output\n", + "\n", + "Use `capture_output=True` (shorthand for `stdout=subprocess.PIPE, stderr=subprocess.PIPE`) and `text=True` to get string output instead of bytes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# capture_output=True captures both stdout and stderr\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", \"print('standard output')\\nimport sys; sys.stderr.write('standard error\\\\n')\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(f\"stdout: {result.stdout.strip()!r}\")\n", + "print(f\"stderr: {result.stderr.strip()!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# Without text=True, output is bytes\n", + "result_bytes: subprocess.CompletedProcess[bytes] = subprocess.run(\n", + " [sys.executable, \"-c\", \"print('bytes output')\"],\n", + " capture_output=True,\n", + ")\n", + "\n", + "print(f\"Type: {type(result_bytes.stdout)}\")\n", + "print(f\"Raw: {result_bytes.stdout}\")\n", + "\n", + "# With text=True, output is str\n", + "result_text: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", \"print('text output')\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(f\"\\nType: {type(result_text.stdout)}\")\n", + "print(f\"Text: {result_text.stdout.strip()!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 3: Error Handling and Return Codes\n", + "\n", + "Commands indicate success or failure via their return code. A return code of `0` means success; non-zero means failure. You can check this manually or let Python raise an exception." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# A command that exits with a non-zero code\n", + "result: subprocess.CompletedProcess[bytes] = subprocess.run(\n", + " [sys.executable, \"-c\", \"raise SystemExit(1)\"],\n", + " capture_output=True,\n", + ")\n", + "\n", + "print(f\"Return code: {result.returncode}\")\n", + "print(f\"Success: {result.returncode == 0}\")\n", + "\n", + "# check_returncode() raises CalledProcessError on non-zero exit\n", + "try:\n", + " result.check_returncode()\n", + " print(\"No error raised\")\n", + "except subprocess.CalledProcessError as e:\n", + " print(f\"\\nCalledProcessError raised!\")\n", + " print(f\" Command: {e.cmd}\")\n", + " print(f\" Return code: {e.returncode}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# check=True automatically raises CalledProcessError on non-zero exit\n", + "try:\n", + " subprocess.run(\n", + " [sys.executable, \"-c\", \"raise SystemExit(42)\"],\n", + " capture_output=True,\n", + " text=True,\n", + " check=True,\n", + " )\n", + "except subprocess.CalledProcessError as e:\n", + " print(f\"Command failed with return code {e.returncode}\")\n", + " print(f\"Stderr: {e.stderr.strip()!r}\")\n", + "\n", + "# Successful command with check=True works normally\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", \"print('all good')\"],\n", + " capture_output=True,\n", + " text=True,\n", + " check=True,\n", + ")\n", + "print(f\"\\nSuccess: {result.stdout.strip()!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 4: Sending Input to a Process\n", + "\n", + "The `input` parameter lets you send data to a command's stdin." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# Send input to a Python script via stdin\n", + "script: str = \"import sys; data = sys.stdin.read(); print(f'Received: {data.strip()}')\"\n", + "\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", script],\n", + " input=\"hello from parent process\",\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(f\"Output: {result.stdout.strip()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# Process multi-line input\n", + "script: str = \"\"\"\n", + "import sys\n", + "lines = sys.stdin.readlines()\n", + "for i, line in enumerate(lines, 1):\n", + " print(f\"Line {i}: {line.strip()}\")\n", + "\"\"\"\n", + "\n", + "input_data: str = \"apple\\nbanana\\ncherry\"\n", + "\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", script],\n", + " input=input_data,\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(result.stdout.strip())" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 5: Timeouts\n", + "\n", + "The `timeout` parameter (in seconds) prevents commands from running indefinitely. A `TimeoutExpired` exception is raised if the command takes too long." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# A command that completes quickly\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", \"print('fast')\"],\n", + " capture_output=True,\n", + " text=True,\n", + " timeout=5,\n", + ")\n", + "print(f\"Fast command: {result.stdout.strip()!r}\")\n", + "\n", + "# A command that takes too long\n", + "try:\n", + " subprocess.run(\n", + " [sys.executable, \"-c\", \"import time; time.sleep(10)\"],\n", + " capture_output=True,\n", + " timeout=1,\n", + " )\n", + "except subprocess.TimeoutExpired as e:\n", + " print(f\"\\nTimeoutExpired after {e.timeout}s\")\n", + " print(f\"Command: {e.cmd}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 6: Low-Level Process Control with `Popen`\n", + "\n", + "`subprocess.Popen` provides direct control over the process. Unlike `run()`, it does not wait for the process to finish -- you manage the lifecycle yourself." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# Start a process without waiting\n", + "proc: subprocess.Popen[str] = subprocess.Popen(\n", + " [sys.executable, \"-c\", \"print('from popen')\"],\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " text=True,\n", + ")\n", + "\n", + "# Wait for it and get output\n", + "stdout, stderr = proc.communicate()\n", + "\n", + "print(f\"PID: {proc.pid}\")\n", + "print(f\"Return code: {proc.returncode}\")\n", + "print(f\"Stdout: {stdout.strip()!r}\")\n", + "print(f\"Stderr: {stderr.strip()!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# Popen with input via communicate()\n", + "proc: subprocess.Popen[str] = subprocess.Popen(\n", + " [sys.executable, \"-c\", \"import sys; print(sys.stdin.read().upper())\"],\n", + " stdin=subprocess.PIPE,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " text=True,\n", + ")\n", + "\n", + "stdout, stderr = proc.communicate(input=\"hello world\")\n", + "\n", + "print(f\"Input: 'hello world'\")\n", + "print(f\"Output: {stdout.strip()!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c6d7", + "metadata": {}, + "source": [ + "## Section 7: Piping Between Processes\n", + "\n", + "You can connect the stdout of one process to the stdin of another, similar to shell pipes (`cmd1 | cmd2`). Use `Popen` for this pattern." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# Simulate: echo \"hello world\" | python -c \"upper()\"\n", + "# Process 1: generates output\n", + "producer: subprocess.Popen[str] = subprocess.Popen(\n", + " [sys.executable, \"-c\", \"print('hello world')\"],\n", + " stdout=subprocess.PIPE,\n", + " text=True,\n", + ")\n", + "\n", + "# Process 2: reads from process 1's stdout\n", + "consumer: subprocess.Popen[str] = subprocess.Popen(\n", + " [sys.executable, \"-c\", \"import sys; print(sys.stdin.read().strip().upper())\"],\n", + " stdin=producer.stdout,\n", + " stdout=subprocess.PIPE,\n", + " text=True,\n", + ")\n", + "\n", + "# Allow producer to receive SIGPIPE if consumer exits\n", + "if producer.stdout:\n", + " producer.stdout.close()\n", + "\n", + "output, _ = consumer.communicate()\n", + "print(f\"Piped result: {output.strip()!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "# For simple piping, subprocess.run with input is often cleaner\n", + "# Step 1: generate data\n", + "step1: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", \"print('line one\\\\nline two\\\\nline three')\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "# Step 2: process the data from step 1\n", + "step2: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", \"import sys; lines = sys.stdin.readlines(); print(f'Got {len(lines)} lines')\"],\n", + " input=step1.stdout,\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "\n", + "print(f\"Step 1 output: {step1.stdout.strip()!r}\")\n", + "print(f\"Step 2 output: {step2.stdout.strip()!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 8: Practical Patterns\n", + "\n", + "Common patterns for working with subprocess in real applications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "\n", + "def run_command(args: list[str], timeout: int = 30) -> tuple[bool, str, str]:\n", + " \"\"\"Run a command and return (success, stdout, stderr).\"\"\"\n", + " try:\n", + " result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " args,\n", + " capture_output=True,\n", + " text=True,\n", + " timeout=timeout,\n", + " )\n", + " return (result.returncode == 0, result.stdout, result.stderr)\n", + " except subprocess.TimeoutExpired:\n", + " return (False, \"\", f\"Command timed out after {timeout}s\")\n", + " except FileNotFoundError:\n", + " return (False, \"\", f\"Command not found: {args[0]}\")\n", + "\n", + "\n", + "# Test with a successful command\n", + "success, stdout, stderr = run_command([sys.executable, \"--version\"])\n", + "print(f\"Success: {success}\")\n", + "print(f\"Output: {stdout.strip() or stderr.strip()}\")\n", + "\n", + "# Test with a non-existent command\n", + "success, stdout, stderr = run_command([\"nonexistent_command\"])\n", + "print(f\"\\nSuccess: {success}\")\n", + "print(f\"Error: {stderr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a5b6c7", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "\n", + "def get_python_module_version(module_name: str) -> str | None:\n", + " \"\"\"Get the version of an installed Python module via subprocess.\"\"\"\n", + " script: str = f\"\"\"\n", + "try:\n", + " import {module_name}\n", + " version = getattr({module_name}, '__version__', 'unknown')\n", + " print(version)\n", + "except ImportError:\n", + " print('NOT_INSTALLED')\n", + "\"\"\"\n", + " result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [sys.executable, \"-c\", script],\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " output: str = result.stdout.strip()\n", + " if output == \"NOT_INSTALLED\":\n", + " return None\n", + " return output\n", + "\n", + "\n", + "# Check some modules\n", + "modules: list[str] = [\"os\", \"sys\", \"json\", \"subprocess\"]\n", + "for mod in modules:\n", + " version: str | None = get_python_module_version(mod)\n", + " status: str = version if version else \"not installed\"\n", + " print(f\"{mod:>12}: {status}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a5b6c7d8", + "metadata": {}, + "source": [ + "## Section 9: Shell Mode and Security\n", + "\n", + "Passing `shell=True` runs the command through the shell, enabling shell features like globbing and pipes. However, this introduces security risks when using untrusted input." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5c6d7e8", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "# shell=True allows shell syntax (use with caution!)\n", + "result: subprocess.CompletedProcess[str] = subprocess.run(\n", + " \"echo 'hello' | tr 'a-z' 'A-Z'\",\n", + " shell=True,\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "print(f\"Shell pipe result: {result.stdout.strip()!r}\")\n", + "\n", + "# PREFERRED: Avoid shell=True by using list args\n", + "# This is safer because arguments are not parsed by the shell\n", + "result_safe: subprocess.CompletedProcess[str] = subprocess.run(\n", + " [\"echo\", \"hello world\"],\n", + " capture_output=True,\n", + " text=True,\n", + ")\n", + "print(f\"List args result: {result_safe.stdout.strip()!r}\")\n", + "\n", + "print(\"\\nRule of thumb: avoid shell=True unless you need shell features\")\n", + "print(\"and you fully control the command string.\")" + ] + }, + { + "cell_type": "markdown", + "id": "c5d6e7f8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `subprocess.run()` -- High-Level API\n", + "- **`subprocess.run(args, ...)`**: Run a command, wait for completion, return `CompletedProcess`\n", + "- **`capture_output=True`**: Capture stdout and stderr (equivalent to `stdout=PIPE, stderr=PIPE`)\n", + "- **`text=True`**: Decode output as strings instead of bytes\n", + "- **`input=\"...\"`**: Send data to the command's stdin\n", + "- **`check=True`**: Raise `CalledProcessError` on non-zero exit code\n", + "- **`timeout=N`**: Raise `TimeoutExpired` after N seconds\n", + "\n", + "### `CompletedProcess` Attributes\n", + "- **`.returncode`**: Exit code (`0` = success)\n", + "- **`.stdout`** / **`.stderr`**: Captured output\n", + "- **`.args`**: The command that was run\n", + "- **`.check_returncode()`**: Raises `CalledProcessError` if returncode is non-zero\n", + "\n", + "### `subprocess.Popen` -- Low-Level API\n", + "- **`Popen(args, stdin, stdout, stderr)`**: Start a process without waiting\n", + "- **`.communicate(input=...)`**: Send input, wait for completion, return `(stdout, stderr)`\n", + "- **`.pid`**: Process ID\n", + "- Use `Popen` when you need to pipe between processes or manage process lifecycles\n", + "\n", + "### Best Practices\n", + "- Prefer `subprocess.run()` over `Popen` for simple use cases\n", + "- Always pass commands as **lists** (`[\"cmd\", \"arg\"]`) instead of strings\n", + "- Avoid `shell=True` unless you need shell features and fully control the input\n", + "- Use `check=True` or `check_returncode()` to catch failures early\n", + "- Set `timeout` to prevent runaway processes" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_33/README.md b/src/chapter_33/README.md new file mode 100644 index 0000000..38f1d3b --- /dev/null +++ b/src/chapter_33/README.md @@ -0,0 +1,9 @@ +# Chapter 33: OS and System Interaction + +Interacting with the operating system — os module, shutil for file operations, tempfile for temporary files, platform for system info, and subprocess for running external commands. + +## Notebooks + +1. **01_os_and_platform.ipynb** — os module (environ, path, walk), platform module, sys info +2. **02_shutil_and_tempfile.ipynb** — shutil (copy, move, rmtree, disk_usage), tempfile (NamedTemporaryFile, TemporaryDirectory) +3. **03_subprocess.ipynb** — subprocess.run, Popen, pipes, capturing output, error handling diff --git a/src/chapter_33/__init__.py b/src/chapter_33/__init__.py new file mode 100644 index 0000000..6374411 --- /dev/null +++ b/src/chapter_33/__init__.py @@ -0,0 +1 @@ +"""Chapter 33: OS and System Interaction.""" diff --git a/src/chapter_34/01_email_messages.ipynb b/src/chapter_34/01_email_messages.ipynb new file mode 100644 index 0000000..8b678b8 --- /dev/null +++ b/src/chapter_34/01_email_messages.ipynb @@ -0,0 +1,569 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 34: Email Messages\n", + "\n", + "This notebook covers Python's `email.message.EmailMessage` class for constructing, inspecting, and serializing email messages. You will learn how to set headers, add body content, work with attachments, and convert messages to strings.\n", + "\n", + "## Key Concepts\n", + "- **`EmailMessage`**: The modern API for building email messages\n", + "- **Headers**: Subject, From, To, and other metadata fields\n", + "- **Body content**: Setting plain text and HTML content via `set_content()`\n", + "- **Attachments**: Adding binary and text attachments to messages\n", + "- **Serialization**: Converting messages to strings with `as_string()`\n", + "- **Multipart**: Messages with both body and attachments" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Creating a Basic Email\n", + "\n", + "An `EmailMessage` object represents a complete email. You create it, then set headers and body content individually." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Create a new email message\n", + "msg: EmailMessage = EmailMessage()\n", + "\n", + "# Set standard headers\n", + "msg[\"Subject\"] = \"Test\"\n", + "msg[\"From\"] = \"alice@example.com\"\n", + "msg[\"To\"] = \"bob@example.com\"\n", + "\n", + "# Set the body content\n", + "msg.set_content(\"Hello, Bob!\")\n", + "\n", + "print(f\"Subject: {msg['Subject']}\")\n", + "print(f\"From: {msg['From']}\")\n", + "print(f\"To: {msg['To']}\")\n", + "print(f\"Body: {msg.get_content()!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Verify header values match what was set\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Test\"\n", + "msg[\"From\"] = \"alice@example.com\"\n", + "msg[\"To\"] = \"bob@example.com\"\n", + "msg.set_content(\"Hello, Bob!\")\n", + "\n", + "assert msg[\"Subject\"] == \"Test\"\n", + "assert msg[\"From\"] == \"alice@example.com\"\n", + "assert \"Hello, Bob!\" in msg.get_content()\n", + "\n", + "print(\"All assertions passed.\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Working with Headers\n", + "\n", + "Email headers are case-insensitive. You can access them using dictionary-style syntax, iterate over them, and check for existence." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Headers are case-insensitive\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Content-Type\"] = \"text/plain\"\n", + "\n", + "# Access with different casing\n", + "print(f\"msg['Content-Type']: {msg['Content-Type']}\")\n", + "print(f\"msg['content-type']: {msg['content-type']}\")\n", + "print(f\"msg['CONTENT-TYPE']: {msg['CONTENT-TYPE']}\")\n", + "\n", + "assert msg[\"content-type\"] == \"text/plain\"\n", + "print(\"\\nCase-insensitive access confirmed.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Exploring all headers on a message\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Weekly Report\"\n", + "msg[\"From\"] = \"reports@example.com\"\n", + "msg[\"To\"] = \"team@example.com\"\n", + "msg[\"CC\"] = \"manager@example.com\"\n", + "msg[\"Date\"] = \"Fri, 21 Feb 2026 10:00:00 +0000\"\n", + "msg.set_content(\"Please find the weekly report attached.\")\n", + "\n", + "# Iterate over all header keys\n", + "print(\"All headers:\")\n", + "for key in msg.keys():\n", + " print(f\" {key}: {msg[key]}\")\n", + "\n", + "# Check if a header exists\n", + "print(f\"\\nHas 'CC': {'CC' in msg}\")\n", + "print(f\"Has 'BCC': {'BCC' in msg}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Replacing and deleting headers\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Draft\"\n", + "print(f\"Before: {msg['Subject']}\")\n", + "\n", + "# To replace a header, delete it first, then set the new value\n", + "del msg[\"Subject\"]\n", + "msg[\"Subject\"] = \"Final Version\"\n", + "print(f\"After: {msg['Subject']}\")\n", + "\n", + "# replace_header() is a convenient alternative\n", + "msg.replace_header(\"Subject\", \"Published\")\n", + "print(f\"Replaced: {msg['Subject']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 3: Setting Body Content\n", + "\n", + "The `set_content()` method sets the message body. By default it creates a `text/plain` message, but you can also set HTML content or specify other subtypes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Plain text content (default)\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Plain Text\"\n", + "msg.set_content(\"This is a plain text email body.\")\n", + "\n", + "print(f\"Content type: {msg.get_content_type()}\")\n", + "print(f\"Body: {msg.get_content()!r}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# HTML content\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"HTML Email\"\n", + "msg.set_content(\n", + " \"<h1>Hello</h1><p>This is <b>HTML</b> content.</p>\",\n", + " subtype=\"html\",\n", + ")\n", + "\n", + "print(f\"Content type: {msg.get_content_type()}\")\n", + "print(f\"Body preview: {msg.get_content()[:60]}...\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 4: Attachments and Multipart Messages\n", + "\n", + "Adding an attachment to a message makes it multipart. The original body becomes one part, and each attachment becomes another." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Create a message with an attachment\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"With attachment\"\n", + "msg.set_content(\"Main body\")\n", + "\n", + "# Add a binary attachment\n", + "msg.add_attachment(\n", + " b\"file content\",\n", + " maintype=\"application\",\n", + " subtype=\"octet-stream\",\n", + " filename=\"data.bin\",\n", + ")\n", + "\n", + "print(f\"Is multipart: {msg.is_multipart()}\")\n", + "assert msg.is_multipart()\n", + "\n", + "# Iterate over the parts\n", + "for i, part in enumerate(msg.iter_parts()):\n", + " print(f\"\\nPart {i}:\")\n", + " print(f\" Content type: {part.get_content_type()}\")\n", + " filename: str | None = part.get_filename()\n", + " if filename:\n", + " print(f\" Filename: {filename}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Multiple attachments\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Multiple Attachments\"\n", + "msg.set_content(\"See attached files.\")\n", + "\n", + "# Attach a text file\n", + "msg.add_attachment(\n", + " \"This is a text file.\",\n", + " subtype=\"plain\",\n", + " filename=\"notes.txt\",\n", + ")\n", + "\n", + "# Attach binary data\n", + "msg.add_attachment(\n", + " b\"\\x89PNG\\r\\n\\x1a\\n\",\n", + " maintype=\"image\",\n", + " subtype=\"png\",\n", + " filename=\"logo.png\",\n", + ")\n", + "\n", + "parts: list[str] = []\n", + "for part in msg.iter_parts():\n", + " ct: str = part.get_content_type()\n", + " fn: str | None = part.get_filename()\n", + " parts.append(f\"{ct} ({fn or 'body'})\")\n", + "\n", + "print(\"Parts:\")\n", + "for p in parts:\n", + " print(f\" {p}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 5: Serializing Messages to Strings\n", + "\n", + "The `as_string()` method produces the full RFC 2822 representation of the message, including headers and encoded body. This is the format used when actually sending an email." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Serialize a simple message\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Hello\"\n", + "msg.set_content(\"Body text\")\n", + "\n", + "text: str = msg.as_string()\n", + "\n", + "# The serialized form contains headers and body\n", + "assert \"Subject: Hello\" in text\n", + "assert \"Body text\" in text\n", + "\n", + "print(\"Serialized message:\")\n", + "print(text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# as_bytes() produces bytes instead of str\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Binary form\"\n", + "msg.set_content(\"Hello in bytes.\")\n", + "\n", + "raw: bytes = msg.as_bytes()\n", + "print(f\"Type: {type(raw).__name__}\")\n", + "print(f\"First 80 bytes: {raw[:80]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 6: Parsing Email Strings\n", + "\n", + "The `email.message_from_string()` function parses a raw email string back into an `EmailMessage` (or `Message`) object. The `email.policy.default` policy returns the modern `EmailMessage` type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "from email import message_from_string\n", + "from email.message import EmailMessage\n", + "from email.policy import default\n", + "\n", + "# Build a raw email string\n", + "raw_email: str = (\n", + " \"Subject: Parsed Email\\n\"\n", + " \"From: sender@example.com\\n\"\n", + " \"To: receiver@example.com\\n\"\n", + " \"Content-Type: text/plain; charset=\\\"utf-8\\\"\\n\"\n", + " \"\\n\"\n", + " \"This is the body of a parsed email.\\n\"\n", + ")\n", + "\n", + "# Parse with the default policy for EmailMessage\n", + "parsed: EmailMessage = message_from_string(raw_email, policy=default)\n", + "\n", + "print(f\"Type: {type(parsed).__name__}\")\n", + "print(f\"Subject: {parsed['Subject']}\")\n", + "print(f\"From: {parsed['From']}\")\n", + "print(f\"Body: {parsed.get_content()!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 7: Content Type and Encoding Inspection\n", + "\n", + "`EmailMessage` provides methods to inspect the content type, character set, and transfer encoding of the message." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Inspect content metadata\n", + "msg: EmailMessage = EmailMessage()\n", + "msg.set_content(\"Unicode: \\u00e9\\u00e0\\u00fc\")\n", + "\n", + "print(f\"Content type: {msg.get_content_type()}\")\n", + "print(f\"Main type: {msg.get_content_maintype()}\")\n", + "print(f\"Sub type: {msg.get_content_subtype()}\")\n", + "print(f\"Charset: {msg.get_param('charset')}\")\n", + "print(f\"Content-Encoding: {msg['Content-Transfer-Encoding']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 8: Practical Patterns\n", + "\n", + "Common patterns for constructing email messages in real applications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "\n", + "def build_email(\n", + " subject: str,\n", + " from_addr: str,\n", + " to_addr: str,\n", + " body: str,\n", + " attachments: list[tuple[bytes, str, str, str]] | None = None,\n", + ") -> EmailMessage:\n", + " \"\"\"Build an email message with optional attachments.\n", + "\n", + " Each attachment is (data, maintype, subtype, filename).\n", + " \"\"\"\n", + " msg: EmailMessage = EmailMessage()\n", + " msg[\"Subject\"] = subject\n", + " msg[\"From\"] = from_addr\n", + " msg[\"To\"] = to_addr\n", + " msg.set_content(body)\n", + "\n", + " if attachments:\n", + " for data, maintype, subtype, filename in attachments:\n", + " msg.add_attachment(\n", + " data,\n", + " maintype=maintype,\n", + " subtype=subtype,\n", + " filename=filename,\n", + " )\n", + "\n", + " return msg\n", + "\n", + "\n", + "# Build a message with attachments\n", + "email: EmailMessage = build_email(\n", + " subject=\"Project Update\",\n", + " from_addr=\"dev@example.com\",\n", + " to_addr=\"pm@example.com\",\n", + " body=\"Here are the latest build artifacts.\",\n", + " attachments=[\n", + " (b\"build log data\", \"text\", \"plain\", \"build.log\"),\n", + " (b\"\\x00\\x01\\x02\", \"application\", \"octet-stream\", \"artifact.bin\"),\n", + " ],\n", + ")\n", + "\n", + "print(f\"Subject: {email['Subject']}\")\n", + "print(f\"Multipart: {email.is_multipart()}\")\n", + "print(f\"Part count: {len(list(email.iter_parts()))}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a5b6c7", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "\n", + "def extract_attachments(msg: EmailMessage) -> list[tuple[str, int]]:\n", + " \"\"\"Extract attachment filenames and sizes from a message.\"\"\"\n", + " results: list[tuple[str, int]] = []\n", + " if not msg.is_multipart():\n", + " return results\n", + "\n", + " for part in msg.iter_attachments():\n", + " filename: str = part.get_filename() or \"unnamed\"\n", + " content: bytes | str = part.get_content()\n", + " size: int = len(content) if isinstance(content, (bytes, str)) else 0\n", + " results.append((filename, size))\n", + " return results\n", + "\n", + "\n", + "# Create a message with attachments\n", + "msg: EmailMessage = EmailMessage()\n", + "msg.set_content(\"Body\")\n", + "msg.add_attachment(b\"A\" * 100, maintype=\"application\", subtype=\"octet-stream\", filename=\"big.bin\")\n", + "msg.add_attachment(b\"small\", maintype=\"application\", subtype=\"octet-stream\", filename=\"small.bin\")\n", + "\n", + "for name, size in extract_attachments(msg):\n", + " print(f\" {name}: {size} bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "a5b6c7d8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Core Class\n", + "- **`EmailMessage()`**: Create a new email message object\n", + "\n", + "### Setting Content\n", + "- **`msg['Header'] = value`**: Set a header (Subject, From, To, etc.)\n", + "- **`msg.set_content(text)`**: Set the body as plain text (default) or HTML (`subtype=\"html\"`)\n", + "- **`msg.add_attachment(data, maintype, subtype, filename)`**: Attach a file\n", + "\n", + "### Reading Content\n", + "- **`msg['Header']`**: Read a header (case-insensitive)\n", + "- **`msg.get_content()`**: Get the body content\n", + "- **`msg.get_content_type()`**: Get the MIME type (e.g., `text/plain`)\n", + "- **`msg.is_multipart()`**: Check if the message has multiple parts\n", + "- **`msg.iter_parts()`**: Iterate over all parts of a multipart message\n", + "- **`msg.iter_attachments()`**: Iterate over attachment parts only\n", + "\n", + "### Serialization\n", + "- **`msg.as_string()`**: Serialize to a string (RFC 2822 format)\n", + "- **`msg.as_bytes()`**: Serialize to bytes\n", + "- **`message_from_string(text, policy=default)`**: Parse a raw email string\n", + "\n", + "### Important Notes\n", + "- Headers are **case-insensitive** when accessed\n", + "- Adding an attachment automatically makes the message **multipart**\n", + "- Use `del msg['Header']` followed by assignment to **replace** a header, or use `replace_header()`\n", + "- Use `policy=email.policy.default` when parsing to get `EmailMessage` objects" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_34/02_mime_types.ipynb b/src/chapter_34/02_mime_types.ipynb new file mode 100644 index 0000000..c90c222 --- /dev/null +++ b/src/chapter_34/02_mime_types.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 34: MIME Types\n", + "\n", + "This notebook covers MIME (Multipurpose Internet Mail Extensions) types and the `mimetypes` module. You will learn how to detect file types from extensions, look up extensions from MIME types, understand MIME structure, and build multipart email messages with proper content types.\n", + "\n", + "## Key Concepts\n", + "- **MIME types**: Standardized identifiers for content types (e.g., `text/plain`, `image/png`)\n", + "- **`mimetypes.guess_type()`**: Detect a file's MIME type from its name or path\n", + "- **`mimetypes.guess_extension()`**: Find a file extension for a given MIME type\n", + "- **Multipart messages**: Emails composed of multiple parts with different content types\n", + "- **MIME structure**: The `maintype/subtype` format and how it organizes content" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Understanding MIME Type Structure\n", + "\n", + "A MIME type consists of a **maintype** and a **subtype** separated by a slash. Common maintypes include `text`, `image`, `application`, `audio`, and `video`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# MIME types follow the pattern: maintype/subtype\n", + "mime_examples: list[str] = [\n", + " \"text/plain\",\n", + " \"text/html\",\n", + " \"image/png\",\n", + " \"image/jpeg\",\n", + " \"application/json\",\n", + " \"application/pdf\",\n", + " \"audio/mpeg\",\n", + " \"video/mp4\",\n", + "]\n", + "\n", + "print(f\"{'MIME Type':<25} {'Main Type':<15} {'Sub Type'}\")\n", + "print(\"-\" * 55)\n", + "for mime in mime_examples:\n", + " maintype, subtype = mime.split(\"/\")\n", + " print(f\"{mime:<25} {maintype:<15} {subtype}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e2f3a4", + "metadata": {}, + "source": [ + "## Section 2: Guessing MIME Types from Filenames\n", + "\n", + "The `mimetypes.guess_type()` function returns a tuple of `(type, encoding)` based on a file's name or path. The encoding is typically `None` unless the file uses a known compression." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "# Guess type from filename\n", + "mime_pdf, enc_pdf = mimetypes.guess_type(\"document.pdf\")\n", + "print(f\"document.pdf -> type={mime_pdf}, encoding={enc_pdf}\")\n", + "assert mime_pdf == \"application/pdf\"\n", + "\n", + "mime_png, enc_png = mimetypes.guess_type(\"image.png\")\n", + "print(f\"image.png -> type={mime_png}, encoding={enc_png}\")\n", + "assert mime_png == \"image/png\"\n", + "\n", + "mime_py, enc_py = mimetypes.guess_type(\"script.py\")\n", + "print(f\"script.py -> type={mime_py}, encoding={enc_py}\")\n", + "assert mime_py == \"text/x-python\"\n", + "\n", + "print(\"\\nAll assertions passed.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "# More file types\n", + "filenames: list[str] = [\n", + " \"report.docx\",\n", + " \"data.csv\",\n", + " \"config.yaml\",\n", + " \"archive.tar.gz\",\n", + " \"song.mp3\",\n", + " \"movie.mp4\",\n", + " \"style.css\",\n", + " \"app.js\",\n", + "]\n", + "\n", + "print(f\"{'Filename':<20} {'MIME Type':<40} {'Encoding'}\")\n", + "print(\"-\" * 70)\n", + "for name in filenames:\n", + " mime, encoding = mimetypes.guess_type(name)\n", + " print(f\"{name:<20} {str(mime):<40} {encoding}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: Common MIME Types\n", + "\n", + "Certain MIME types appear frequently in web and email contexts. The `mimetypes` module recognizes all well-known types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "# Verify common MIME types\n", + "assert mimetypes.guess_type(\"file.json\")[0] == \"application/json\"\n", + "assert mimetypes.guess_type(\"file.csv\")[0] == \"text/csv\"\n", + "assert mimetypes.guess_type(\"file.txt\")[0] == \"text/plain\"\n", + "\n", + "# Additional common types\n", + "common: dict[str, str | None] = {\n", + " \"file.html\": mimetypes.guess_type(\"file.html\")[0],\n", + " \"file.xml\": mimetypes.guess_type(\"file.xml\")[0],\n", + " \"file.zip\": mimetypes.guess_type(\"file.zip\")[0],\n", + " \"file.gif\": mimetypes.guess_type(\"file.gif\")[0],\n", + " \"file.svg\": mimetypes.guess_type(\"file.svg\")[0],\n", + "}\n", + "\n", + "for filename, mime in common.items():\n", + " print(f\"{filename:<15} -> {mime}\")\n", + "\n", + "print(\"\\nAll assertions passed.\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 4: Guessing Extensions from MIME Types\n", + "\n", + "The reverse operation: given a MIME type string, find an appropriate file extension." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "# Guess file extension from MIME type\n", + "ext_html: str | None = mimetypes.guess_extension(\"text/html\")\n", + "print(f\"text/html -> {ext_html}\")\n", + "assert ext_html in (\".html\", \".htm\")\n", + "\n", + "ext_json: str | None = mimetypes.guess_extension(\"application/json\")\n", + "print(f\"application/json -> {ext_json}\")\n", + "\n", + "ext_png: str | None = mimetypes.guess_extension(\"image/png\")\n", + "print(f\"image/png -> {ext_png}\")\n", + "\n", + "ext_pdf: str | None = mimetypes.guess_extension(\"application/pdf\")\n", + "print(f\"application/pdf -> {ext_pdf}\")\n", + "\n", + "# Unknown MIME type returns None\n", + "ext_unknown: str | None = mimetypes.guess_extension(\"application/x-unknown-type\")\n", + "print(f\"\\napplication/x-unknown-type -> {ext_unknown}\")\n", + "\n", + "print(\"\\nAll assertions passed.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "# guess_all_extensions returns all known extensions for a type\n", + "all_html: list[str] = mimetypes.guess_all_extensions(\"text/html\")\n", + "print(f\"All extensions for text/html: {all_html}\")\n", + "\n", + "all_jpeg: list[str] = mimetypes.guess_all_extensions(\"image/jpeg\")\n", + "print(f\"All extensions for image/jpeg: {all_jpeg}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 5: The MimeTypes Class\n", + "\n", + "For more control, you can create a `mimetypes.MimeTypes` instance. This lets you add custom type mappings without modifying the global state." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "# Create a custom MimeTypes instance\n", + "mime_db: mimetypes.MimeTypes = mimetypes.MimeTypes()\n", + "\n", + "# Standard lookups work the same\n", + "result: tuple[str | None, str | None] = mime_db.guess_type(\"report.pdf\")\n", + "print(f\"report.pdf -> {result[0]}\")\n", + "\n", + "# Add a custom extension mapping\n", + "mime_db.add_type(\"application/x-custom\", \".custom\")\n", + "custom_result: tuple[str | None, str | None] = mime_db.guess_type(\"data.custom\")\n", + "print(f\"data.custom -> {custom_result[0]}\")\n", + "\n", + "# The global function does not know about our custom type\n", + "global_result: tuple[str | None, str | None] = mimetypes.guess_type(\"data.custom\")\n", + "print(f\"\\nGlobal guess for data.custom: {global_result[0]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 6: Multipart Email Messages\n", + "\n", + "Multipart MIME messages contain multiple sections, each with its own content type. This is how emails carry both a text body and attachments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Build a multipart message\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Report with Data\"\n", + "msg[\"From\"] = \"analyst@example.com\"\n", + "msg[\"To\"] = \"team@example.com\"\n", + "msg.set_content(\"Please find the data attached.\")\n", + "\n", + "# Before attachment: not multipart\n", + "print(f\"Before attachment:\")\n", + "print(f\" Content type: {msg.get_content_type()}\")\n", + "print(f\" Is multipart: {msg.is_multipart()}\")\n", + "\n", + "# Add a CSV attachment\n", + "csv_data: str = \"name,value\\nalpha,1\\nbeta,2\\n\"\n", + "msg.add_attachment(\n", + " csv_data,\n", + " subtype=\"csv\",\n", + " filename=\"data.csv\",\n", + ")\n", + "\n", + "# After attachment: multipart/mixed\n", + "print(f\"\\nAfter attachment:\")\n", + "print(f\" Content type: {msg.get_content_type()}\")\n", + "print(f\" Is multipart: {msg.is_multipart()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "from email.message import EmailMessage\n", + "\n", + "# Inspecting multipart structure\n", + "msg: EmailMessage = EmailMessage()\n", + "msg[\"Subject\"] = \"Multi-attachment\"\n", + "msg.set_content(\"Body text here.\")\n", + "\n", + "# Add various attachment types\n", + "msg.add_attachment(b\"binary data\", maintype=\"application\", subtype=\"octet-stream\", filename=\"file.bin\")\n", + "msg.add_attachment(\"log output here\", subtype=\"plain\", filename=\"output.log\")\n", + "msg.add_attachment(b\"\\x89PNG\\r\\n\", maintype=\"image\", subtype=\"png\", filename=\"chart.png\")\n", + "\n", + "print(f\"Top-level type: {msg.get_content_type()}\")\n", + "print(f\"Number of parts: {len(list(msg.iter_parts()))}\")\n", + "print()\n", + "\n", + "for i, part in enumerate(msg.iter_parts()):\n", + " ct: str = part.get_content_type()\n", + " fn: str | None = part.get_filename()\n", + " disp: str | None = part.get_content_disposition()\n", + " print(f\"Part {i}: type={ct}, filename={fn}, disposition={disp}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a5b6", + "metadata": {}, + "source": [ + "## Section 7: MIME Types in Practice\n", + "\n", + "Combining `mimetypes` detection with `EmailMessage` to automatically determine the correct MIME type for file attachments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3a4b5c6", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "from email.message import EmailMessage\n", + "\n", + "\n", + "def attach_file(\n", + " msg: EmailMessage,\n", + " filename: str,\n", + " data: bytes,\n", + ") -> None:\n", + " \"\"\"Attach data to a message, auto-detecting the MIME type from filename.\"\"\"\n", + " mime_type: str | None\n", + " encoding: str | None\n", + " mime_type, encoding = mimetypes.guess_type(filename)\n", + "\n", + " if mime_type is None:\n", + " maintype: str = \"application\"\n", + " subtype: str = \"octet-stream\"\n", + " else:\n", + " maintype, subtype = mime_type.split(\"/\")\n", + "\n", + " msg.add_attachment(\n", + " data,\n", + " maintype=maintype,\n", + " subtype=subtype,\n", + " filename=filename,\n", + " )\n", + "\n", + "\n", + "# Build a message with auto-detected types\n", + "msg: EmailMessage = EmailMessage()\n", + "msg.set_content(\"Files attached.\")\n", + "\n", + "attach_file(msg, \"photo.jpg\", b\"\\xff\\xd8\\xff\\xe0\")\n", + "attach_file(msg, \"data.json\", b'{\"key\": \"value\"}')\n", + "attach_file(msg, \"mystery.xyz\", b\"unknown format\")\n", + "\n", + "for part in msg.iter_attachments():\n", + " print(f\"{part.get_filename():<15} -> {part.get_content_type()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import mimetypes\n", + "\n", + "\n", + "def categorize_mime(filename: str) -> str:\n", + " \"\"\"Categorize a file by its MIME maintype.\"\"\"\n", + " mime: str | None = mimetypes.guess_type(filename)[0]\n", + " if mime is None:\n", + " return \"unknown\"\n", + " maintype: str = mime.split(\"/\")[0]\n", + " categories: dict[str, str] = {\n", + " \"text\": \"document\",\n", + " \"image\": \"media\",\n", + " \"audio\": \"media\",\n", + " \"video\": \"media\",\n", + " \"application\": \"binary\",\n", + " }\n", + " return categories.get(maintype, \"other\")\n", + "\n", + "\n", + "files: list[str] = [\"readme.txt\", \"logo.png\", \"song.mp3\", \"app.exe\", \"data.xyz\"]\n", + "for f in files:\n", + " print(f\"{f:<15} -> {categorize_mime(f)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### MIME Type Structure\n", + "- Format: **`maintype/subtype`** (e.g., `text/plain`, `image/png`, `application/json`)\n", + "- Common maintypes: `text`, `image`, `audio`, `video`, `application`, `multipart`\n", + "\n", + "### The `mimetypes` Module\n", + "- **`mimetypes.guess_type(filename)`**: Returns `(type, encoding)` tuple from a filename\n", + "- **`mimetypes.guess_extension(mime_type)`**: Returns a file extension for a MIME type\n", + "- **`mimetypes.guess_all_extensions(mime_type)`**: Returns all known extensions for a type\n", + "- **`mimetypes.MimeTypes()`**: Create a custom instance with isolated type mappings\n", + "\n", + "### Multipart Messages\n", + "- Adding an attachment converts a message to **`multipart/mixed`**\n", + "- **`msg.iter_parts()`**: Iterate over all parts (body + attachments)\n", + "- **`msg.iter_attachments()`**: Iterate over attachment parts only\n", + "- **`part.get_content_type()`**: Get the MIME type of a specific part\n", + "- **`part.get_content_disposition()`**: Returns `'attachment'` or `'inline'`\n", + "\n", + "### Important Notes\n", + "- `guess_type()` works on filenames and paths -- it does **not** read file contents\n", + "- Returns `(None, None)` for unrecognized extensions\n", + "- Use `guess_type()` with email attachment helpers to auto-detect content types\n", + "- The module reads from system MIME type databases on startup" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_34/03_encoding_and_decoding.ipynb b/src/chapter_34/03_encoding_and_decoding.ipynb new file mode 100644 index 0000000..1a22f1a --- /dev/null +++ b/src/chapter_34/03_encoding_and_decoding.ipynb @@ -0,0 +1,673 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 34: Encoding and Decoding\n", + "\n", + "This notebook covers Python's standard library modules for encoding binary data into text-safe formats and decoding them back. These encodings are essential for email transport, data URLs, checksums, and binary-to-text conversions.\n", + "\n", + "## Key Concepts\n", + "- **`base64`**: Encode/decode bytes using Base64 and URL-safe Base64\n", + "- **`quopri`**: Quoted-printable encoding for mostly-ASCII text with some non-ASCII characters\n", + "- **`binascii`**: Low-level binary-to-ASCII conversions, hexlify/unhexlify, CRC-32 checksums\n", + "- **`uu`**: Unix-to-Unix encoding (legacy format)\n", + "- **Hex and bytes**: Converting between bytes, hex strings, and integers" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Base64 Encoding Basics\n", + "\n", + "Base64 encodes arbitrary bytes into a printable ASCII string using 64 characters (A-Z, a-z, 0-9, +, /). It is widely used in email (MIME), data URLs, and API tokens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "# Encode bytes to Base64\n", + "original: bytes = b\"Hello, World!\"\n", + "encoded: bytes = base64.b64encode(original)\n", + "\n", + "print(f\"Original: {original}\")\n", + "print(f\"Encoded: {encoded}\")\n", + "print(f\"Type: {type(encoded).__name__}\")\n", + "\n", + "# The result is bytes, but contains only ASCII characters\n", + "assert isinstance(encoded, bytes)\n", + "\n", + "# Decode back to original\n", + "decoded: bytes = base64.b64decode(encoded)\n", + "assert decoded == original\n", + "\n", + "print(f\"\\nDecoded: {decoded}\")\n", + "print(\"Round-trip successful.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "# Base64 increases size by roughly 33%\n", + "# Every 3 bytes of input become 4 bytes of output\n", + "for size in [3, 6, 9, 12, 100]:\n", + " data: bytes = b\"A\" * size\n", + " encoded: bytes = base64.b64encode(data)\n", + " ratio: float = len(encoded) / len(data)\n", + " print(f\"Input: {size:>3} bytes -> Output: {len(encoded):>4} bytes (ratio: {ratio:.2f})\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Base64 with Strings\n", + "\n", + "Since `base64` works with bytes, you need to encode strings to bytes first and decode the result back to a string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "# Encoding a string: str -> bytes -> base64 bytes -> str\n", + "text: str = \"Python is great!\"\n", + "encoded: str = base64.b64encode(text.encode()).decode()\n", + "\n", + "print(f\"Text: {text}\")\n", + "print(f\"Encoded: {encoded}\")\n", + "assert isinstance(encoded, str)\n", + "\n", + "# Decoding: str -> bytes -> base64 decode -> bytes -> str\n", + "decoded: str = base64.b64decode(encoded).decode()\n", + "assert decoded == text\n", + "\n", + "print(f\"Decoded: {decoded}\")\n", + "print(\"\\nString round-trip successful.\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: URL-Safe Base64\n", + "\n", + "Standard Base64 uses `+` and `/` which are special characters in URLs. URL-safe Base64 replaces these with `-` and `_`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "# Generate data that produces + and / in standard base64\n", + "data: bytes = bytes(range(256))\n", + "\n", + "# Standard encoding may contain + and /\n", + "standard: bytes = base64.b64encode(data)\n", + "print(f\"Standard contains '+': {b'+' in standard}\")\n", + "print(f\"Standard contains '/': {b'/' in standard}\")\n", + "\n", + "# URL-safe encoding replaces + with - and / with _\n", + "url_safe: bytes = base64.urlsafe_b64encode(data)\n", + "print(f\"\\nURL-safe contains '+': {b'+' in url_safe}\")\n", + "print(f\"URL-safe contains '/': {b'/' in url_safe}\")\n", + "\n", + "assert b\"+\" not in url_safe\n", + "assert b\"/\" not in url_safe\n", + "\n", + "# Both decode back to the same data\n", + "assert base64.urlsafe_b64decode(url_safe) == data\n", + "print(\"\\nURL-safe round-trip successful.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "# Comparing standard vs URL-safe encodings side by side\n", + "# Use data that highlights the differences\n", + "sample: bytes = b\"\\xfb\\xff\\xfe\"\n", + "\n", + "std: str = base64.b64encode(sample).decode()\n", + "url: str = base64.urlsafe_b64encode(sample).decode()\n", + "\n", + "print(f\"Data: {sample.hex()}\")\n", + "print(f\"Standard: {std}\")\n", + "print(f\"URL-safe: {url}\")\n", + "print(f\"\\nDifferences: '+' -> '-' and '/' -> '_'\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 4: Base32 and Base16 Encoding\n", + "\n", + "The `base64` module also provides Base32 (32 uppercase characters) and Base16 (hexadecimal) encodings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "data: bytes = b\"Hello!\"\n", + "\n", + "# Base64: 6 bits per character\n", + "b64: bytes = base64.b64encode(data)\n", + "print(f\"Base64: {b64.decode():<20} ({len(b64)} chars)\")\n", + "\n", + "# Base32: 5 bits per character (uppercase A-Z and 2-7)\n", + "b32: bytes = base64.b32encode(data)\n", + "print(f\"Base32: {b32.decode():<20} ({len(b32)} chars)\")\n", + "\n", + "# Base16: 4 bits per character (hex digits)\n", + "b16: bytes = base64.b16encode(data)\n", + "print(f\"Base16: {b16.decode():<20} ({len(b16)} chars)\")\n", + "\n", + "# All decode back to the same data\n", + "assert base64.b64decode(b64) == data\n", + "assert base64.b32decode(b32) == data\n", + "assert base64.b16decode(b16) == data\n", + "print(\"\\nAll round-trips successful.\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 5: Quoted-Printable Encoding (quopri)\n", + "\n", + "Quoted-printable encoding is designed for data that is mostly ASCII text with occasional non-ASCII bytes. Non-printable bytes are encoded as `=XX` (hex). This is commonly used in email headers and bodies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import quopri\n", + "\n", + "# Encode text with non-ASCII characters\n", + "data: bytes = \"H\\u00e9llo W\\u00f6rld\".encode(\"utf-8\")\n", + "encoded: bytes = quopri.encodestring(data)\n", + "\n", + "print(f\"Original bytes: {data}\")\n", + "print(f\"Encoded: {encoded}\")\n", + "\n", + "# The accented characters are encoded as =XX sequences\n", + "assert b\"=C3\" in encoded # UTF-8 for accented chars uses C3 prefix\n", + "\n", + "# Decode back\n", + "decoded: bytes = quopri.decodestring(encoded)\n", + "assert decoded == data\n", + "print(f\"Decoded: {decoded}\")\n", + "print(f\"As string: {decoded.decode('utf-8')}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import quopri\n", + "\n", + "# Decoding =XX sequences\n", + "encoded: bytes = b\"Hello=20World\"\n", + "decoded: bytes = quopri.decodestring(encoded)\n", + "\n", + "print(f\"Encoded: {encoded}\")\n", + "print(f\"Decoded: {decoded}\")\n", + "assert decoded == b\"Hello World\" # =20 is the space character\n", + "\n", + "# More examples\n", + "examples: list[bytes] = [\n", + " b\"100=25 complete\", # =25 is '%'\n", + " b\"price=3D100\", # =3D is '='\n", + " b\"line1=0D=0Aline2\", # =0D=0A is CRLF\n", + "]\n", + "\n", + "print(\"\\nDecoding examples:\")\n", + "for enc in examples:\n", + " dec: bytes = quopri.decodestring(enc)\n", + " print(f\" {enc.decode():<25} -> {dec!r}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 6: Binary-ASCII Conversions (binascii)\n", + "\n", + "The `binascii` module provides low-level functions for converting between binary data and various ASCII representations, including hexadecimal." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import binascii\n", + "\n", + "# hexlify: bytes -> hex string (as bytes)\n", + "data: bytes = b\"\\xde\\xad\\xbe\\xef\"\n", + "hex_str: bytes = binascii.hexlify(data)\n", + "\n", + "print(f\"Data: {data!r}\")\n", + "print(f\"Hex: {hex_str}\")\n", + "print(f\"As text: {hex_str.decode()}\")\n", + "\n", + "assert hex_str == b\"deadbeef\"\n", + "\n", + "# unhexlify: hex string -> bytes\n", + "restored: bytes = binascii.unhexlify(hex_str)\n", + "assert restored == data\n", + "\n", + "print(f\"\\nRestored: {restored!r}\")\n", + "print(\"Round-trip successful.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import binascii\n", + "\n", + "# hexlify and unhexlify with various data\n", + "samples: list[bytes] = [\n", + " b\"\\x00\\x00\\x00\\x00\",\n", + " b\"\\xff\\xff\\xff\\xff\",\n", + " b\"Hello\",\n", + " b\"\\x01\\x02\\x03\",\n", + "]\n", + "\n", + "print(f\"{'Bytes':<25} {'Hex'}\")\n", + "print(\"-\" * 45)\n", + "for s in samples:\n", + " h: str = binascii.hexlify(s).decode()\n", + " print(f\"{str(s):<25} {h}\")\n", + " # Verify round-trip\n", + " assert binascii.unhexlify(h) == s" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 7: CRC-32 Checksums\n", + "\n", + "The `binascii.crc32()` function computes a CRC-32 checksum, which is a fast hash used for data integrity checks (not security)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import binascii\n", + "\n", + "# CRC-32 checksum\n", + "crc: int = binascii.crc32(b\"hello\")\n", + "print(f\"CRC-32 of b'hello': {crc}\")\n", + "print(f\"Hex: {crc:#010x}\")\n", + "\n", + "# The result is deterministic\n", + "assert isinstance(crc, int)\n", + "assert crc == binascii.crc32(b\"hello\")\n", + "\n", + "# Different data produces different checksums\n", + "crc2: int = binascii.crc32(b\"world\")\n", + "print(f\"\\nCRC-32 of b'world': {crc2}\")\n", + "print(f\"Different: {crc != crc2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4c5d6e7", + "metadata": {}, + "outputs": [], + "source": [ + "import binascii\n", + "\n", + "# Incremental CRC-32: pass previous result as second argument\n", + "chunk1: bytes = b\"Hello, \"\n", + "chunk2: bytes = b\"World!\"\n", + "\n", + "# Compute in one shot\n", + "full_crc: int = binascii.crc32(chunk1 + chunk2)\n", + "\n", + "# Compute incrementally\n", + "partial: int = binascii.crc32(chunk1)\n", + "incremental_crc: int = binascii.crc32(chunk2, partial)\n", + "\n", + "print(f\"Full CRC-32: {full_crc:#010x}\")\n", + "print(f\"Incremental CRC-32: {incremental_crc:#010x}\")\n", + "assert full_crc == incremental_crc\n", + "print(\"\\nIncremental computation matches.\")" + ] + }, + { + "cell_type": "markdown", + "id": "c4d5e6f7", + "metadata": {}, + "source": [ + "## Section 8: Hex and Bytes Conversions (Built-in Methods)\n", + "\n", + "Python's built-in `bytes` and `int` types also provide methods for hex conversions, which are often more convenient than `binascii`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "# bytes.hex() and bytes.fromhex()\n", + "data: bytes = b\"\\xca\\xfe\\xba\\xbe\"\n", + "\n", + "# Convert bytes to hex string\n", + "hex_str: str = data.hex()\n", + "print(f\"Bytes: {data!r}\")\n", + "print(f\"Hex: {hex_str}\")\n", + "print(f\"Type: {type(hex_str).__name__}\")\n", + "\n", + "# Convert hex string back to bytes\n", + "restored: bytes = bytes.fromhex(hex_str)\n", + "assert restored == data\n", + "print(f\"Restored: {restored!r}\")\n", + "\n", + "# hex() supports a separator argument\n", + "spaced: str = data.hex(\" \")\n", + "print(f\"\\nWith spaces: {spaced}\")\n", + "\n", + "coloned: str = data.hex(\":\")\n", + "print(f\"With colons: {coloned}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "# int.to_bytes() and int.from_bytes()\n", + "value: int = 305419896 # 0x12345678\n", + "\n", + "# Convert int to bytes (big-endian)\n", + "big: bytes = value.to_bytes(4, byteorder=\"big\")\n", + "print(f\"Value: {value} (0x{value:08x})\")\n", + "print(f\"Big-endian: {big.hex()}\")\n", + "\n", + "# Convert int to bytes (little-endian)\n", + "little: bytes = value.to_bytes(4, byteorder=\"little\")\n", + "print(f\"Little-end: {little.hex()}\")\n", + "\n", + "# Convert back\n", + "from_big: int = int.from_bytes(big, byteorder=\"big\")\n", + "from_little: int = int.from_bytes(little, byteorder=\"little\")\n", + "assert from_big == value\n", + "assert from_little == value\n", + "print(f\"\\nBoth decode back to: {from_big}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Section 9: UU Encoding (Legacy)\n", + "\n", + "UU (Unix-to-Unix) encoding is a legacy format for transferring binary files over text-only channels. While rarely used today, it is still available in Python's `uu` module." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5b6c7d8", + "metadata": {}, + "outputs": [], + "source": [ + "import uu\n", + "import io\n", + "\n", + "# UU encode: binary data -> text representation\n", + "data: bytes = b\"Hello, UU encoding!\"\n", + "input_buf: io.BytesIO = io.BytesIO(data)\n", + "output_buf: io.BytesIO = io.BytesIO()\n", + "\n", + "uu.encode(input_buf, output_buf, name=\"hello.txt\")\n", + "\n", + "encoded: bytes = output_buf.getvalue()\n", + "print(\"UU Encoded:\")\n", + "print(encoded.decode())\n", + "\n", + "# UU decode: text -> binary data\n", + "decode_input: io.BytesIO = io.BytesIO(encoded)\n", + "decode_output: io.BytesIO = io.BytesIO()\n", + "\n", + "uu.decode(decode_input, decode_output)\n", + "\n", + "decoded: bytes = decode_output.getvalue()\n", + "assert decoded == data\n", + "print(f\"Decoded: {decoded}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b5c6d7e8", + "metadata": {}, + "source": [ + "## Section 10: Practical Patterns\n", + "\n", + "Common encoding patterns used in real applications." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5d6e7f8", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "import binascii\n", + "\n", + "\n", + "def create_data_uri(data: bytes, mime_type: str) -> str:\n", + " \"\"\"Create a data URI from bytes and a MIME type.\"\"\"\n", + " encoded: str = base64.b64encode(data).decode(\"ascii\")\n", + " return f\"data:{mime_type};base64,{encoded}\"\n", + "\n", + "\n", + "def parse_data_uri(uri: str) -> tuple[str, bytes]:\n", + " \"\"\"Parse a data URI and return (mime_type, decoded_data).\"\"\"\n", + " header, encoded = uri.split(\",\", 1)\n", + " mime_type: str = header.split(\":\")[1].split(\";\")[0]\n", + " data: bytes = base64.b64decode(encoded)\n", + " return mime_type, data\n", + "\n", + "\n", + "# Create a data URI for a small text snippet\n", + "content: bytes = b\"<h1>Hello</h1>\"\n", + "uri: str = create_data_uri(content, \"text/html\")\n", + "print(f\"Data URI: {uri}\")\n", + "\n", + "# Parse it back\n", + "mime, decoded = parse_data_uri(uri)\n", + "print(f\"\\nMIME type: {mime}\")\n", + "print(f\"Data: {decoded}\")\n", + "assert decoded == content" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5e6f7a8", + "metadata": {}, + "outputs": [], + "source": [ + "import binascii\n", + "\n", + "\n", + "def verify_integrity(data: bytes, expected_crc: int) -> bool:\n", + " \"\"\"Verify data integrity using CRC-32.\"\"\"\n", + " actual_crc: int = binascii.crc32(data)\n", + " return actual_crc == expected_crc\n", + "\n", + "\n", + "# Simulate sending data with a checksum\n", + "message: bytes = b\"Important data payload\"\n", + "checksum: int = binascii.crc32(message)\n", + "print(f\"Data: {message}\")\n", + "print(f\"Checksum: {checksum:#010x}\")\n", + "\n", + "# Verify intact data\n", + "print(f\"\\nIntact: {verify_integrity(message, checksum)}\")\n", + "\n", + "# Verify corrupted data\n", + "corrupted: bytes = b\"Important data payloak\" # last char changed\n", + "print(f\"Corrupted: {verify_integrity(corrupted, checksum)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5f6a7b8", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "\n", + "def encode_token(user_id: int, timestamp: int) -> str:\n", + " \"\"\"Create a URL-safe token from user_id and timestamp.\"\"\"\n", + " payload: bytes = f\"{user_id}:{timestamp}\".encode()\n", + " return base64.urlsafe_b64encode(payload).decode()\n", + "\n", + "\n", + "def decode_token(token: str) -> tuple[int, int]:\n", + " \"\"\"Decode a token back to (user_id, timestamp).\"\"\"\n", + " payload: str = base64.urlsafe_b64decode(token).decode()\n", + " user_id_str, ts_str = payload.split(\":\")\n", + " return int(user_id_str), int(ts_str)\n", + "\n", + "\n", + "# Create and decode a token\n", + "token: str = encode_token(user_id=42, timestamp=1740000000)\n", + "print(f\"Token: {token}\")\n", + "\n", + "uid, ts = decode_token(token)\n", + "print(f\"User ID: {uid}\")\n", + "print(f\"Timestamp: {ts}\")\n", + "\n", + "assert uid == 42\n", + "assert ts == 1740000000" + ] + }, + { + "cell_type": "markdown", + "id": "f5a6b7c8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Base64 (`base64` module)\n", + "- **`b64encode(data)`** / **`b64decode(data)`**: Standard Base64 encoding\n", + "- **`urlsafe_b64encode(data)`** / **`urlsafe_b64decode(data)`**: URL-safe variant (uses `-` and `_`)\n", + "- **`b32encode(data)`** / **`b32decode(data)`**: Base32 encoding\n", + "- **`b16encode(data)`** / **`b16decode(data)`**: Base16 (hex) encoding\n", + "- Base64 increases data size by approximately **33%** (4 output bytes per 3 input bytes)\n", + "\n", + "### Quoted-Printable (`quopri` module)\n", + "- **`encodestring(data)`**: Encode bytes with non-ASCII as `=XX` sequences\n", + "- **`decodestring(data)`**: Decode `=XX` sequences back to bytes\n", + "- Best for data that is **mostly ASCII** with occasional non-ASCII bytes\n", + "\n", + "### Binary-ASCII (`binascii` module)\n", + "- **`hexlify(data)`**: Convert bytes to hex representation (as bytes)\n", + "- **`unhexlify(hex_str)`**: Convert hex back to bytes\n", + "- **`crc32(data)`**: Compute a CRC-32 checksum (returns `int`)\n", + "- CRC-32 supports **incremental** computation via a second argument\n", + "\n", + "### Built-in Hex/Bytes Conversions\n", + "- **`bytes.hex(sep)`**: Convert bytes to hex string with optional separator\n", + "- **`bytes.fromhex(s)`**: Create bytes from a hex string\n", + "- **`int.to_bytes(length, byteorder)`**: Convert int to bytes\n", + "- **`int.from_bytes(data, byteorder)`**: Convert bytes to int\n", + "\n", + "### UU Encoding (`uu` module)\n", + "- **`uu.encode(input, output)`**: Legacy Unix-to-Unix encoding\n", + "- **`uu.decode(input, output)`**: Decode UU-encoded data\n", + "- Rarely used today; prefer Base64 for new applications" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_34/README.md b/src/chapter_34/README.md new file mode 100644 index 0000000..d85d6a2 --- /dev/null +++ b/src/chapter_34/README.md @@ -0,0 +1,9 @@ +# Chapter 34: Email and Data Encoding + +Email message construction and data encoding — email.message, MIME types, base64/quopri encoding, mimetypes detection, and binascii conversions. + +## Notebooks + +1. **01_email_messages.ipynb** — email.message.EmailMessage, headers, body, attachments +2. **02_mime_types.ipynb** — MIME structure, multipart messages, mimetypes module +3. **03_encoding_and_decoding.ipynb** — base64, quopri, binascii, uu encoding, hex/bytes conversions diff --git a/src/chapter_34/__init__.py b/src/chapter_34/__init__.py new file mode 100644 index 0000000..7947d36 --- /dev/null +++ b/src/chapter_34/__init__.py @@ -0,0 +1 @@ +"""Chapter 34: Email and Data Encoding.""" diff --git a/src/chapter_35/01_descriptors_deep_dive.ipynb b/src/chapter_35/01_descriptors_deep_dive.ipynb new file mode 100644 index 0000000..9a63c96 --- /dev/null +++ b/src/chapter_35/01_descriptors_deep_dive.ipynb @@ -0,0 +1,587 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 35: Descriptors Deep Dive\n", + "\n", + "This notebook explores Python's descriptor protocol -- the mechanism that powers `property`, `classmethod`, `staticmethod`, and many ORM frameworks. You will learn the difference between data and non-data descriptors, how `__set_name__` works, and how to build reusable validation descriptors.\n", + "\n", + "## Key Concepts\n", + "- **Descriptor protocol**: Objects that define `__get__`, `__set__`, or `__delete__`\n", + "- **Data descriptor**: Defines both `__get__` and `__set__` (or `__delete__`) -- takes priority over instance `__dict__`\n", + "- **Non-data descriptor**: Defines only `__get__` -- instance `__dict__` takes priority\n", + "- **`__set_name__`**: Called automatically when a descriptor is assigned to a class attribute\n", + "- **Validation descriptors**: Reusable attribute validators using the descriptor protocol" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: The Descriptor Protocol\n", + "\n", + "A descriptor is any object that defines at least one of `__get__`, `__set__`, or `__delete__`. When such an object is stored as a class attribute, Python invokes these methods during attribute access instead of returning the descriptor itself." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "# A minimal descriptor that logs access\n", + "class VerboseDescriptor:\n", + " \"\"\"A descriptor that prints when its __get__ is called.\"\"\"\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> str:\n", + " print(f\"__get__ called: obj={obj!r}, objtype={objtype!r}\")\n", + " return \"descriptor value\"\n", + "\n", + "\n", + "class Demo:\n", + " attr = VerboseDescriptor()\n", + "\n", + "\n", + "# Accessing via instance triggers __get__ with obj set\n", + "d = Demo()\n", + "print(f\"Instance access: {d.attr}\")\n", + "\n", + "print()\n", + "\n", + "# Accessing via class triggers __get__ with obj=None\n", + "print(f\"Class access: {Demo.attr}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e2f3a4", + "metadata": {}, + "source": [ + "## Section 2: Data Descriptors\n", + "\n", + "A **data descriptor** defines both `__get__` and `__set__` (or `__delete__`). Data descriptors take precedence over instance `__dict__` entries. This is how `property` works under the hood." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "class DataDescriptor:\n", + " \"\"\"A data descriptor that stores values in the instance __dict__.\"\"\"\n", + "\n", + " def __set_name__(self, owner: type, name: str) -> None:\n", + " self.name = name\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> object:\n", + " if obj is None:\n", + " return self\n", + " print(f\" DataDescriptor.__get__ for '{self.name}'\")\n", + " return obj.__dict__.get(self.name, \"<not set>\")\n", + "\n", + " def __set__(self, obj: object, value: object) -> None:\n", + " print(f\" DataDescriptor.__set__ for '{self.name}' = {value!r}\")\n", + " obj.__dict__[self.name] = value\n", + "\n", + "\n", + "class Product:\n", + " name = DataDescriptor()\n", + " price = DataDescriptor()\n", + "\n", + "\n", + "p = Product()\n", + "p.name = \"Widget\" # Goes through __set__\n", + "p.price = 9.99\n", + "\n", + "print()\n", + "print(f\"Name: {p.name}\") # Goes through __get__\n", + "print(f\"Price: {p.price}\")\n", + "\n", + "print()\n", + "\n", + "# Even though 'name' exists in instance __dict__, the data descriptor\n", + "# __get__ is still called (data descriptors have priority)\n", + "print(f\"Instance __dict__: {p.__dict__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 3: Non-Data Descriptors\n", + "\n", + "A **non-data descriptor** only defines `__get__` (no `__set__` or `__delete__`). The instance `__dict__` takes priority over non-data descriptors. This is the mechanism behind `functools.cached_property`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Any, Callable\n", + "\n", + "\n", + "class CachedProperty:\n", + " \"\"\"Non-data descriptor that caches the result in the instance __dict__.\"\"\"\n", + "\n", + " def __init__(self, func: Callable[..., Any]) -> None:\n", + " self.func = func\n", + " self.name = func.__name__\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> Any:\n", + " if obj is None:\n", + " return self\n", + " print(f\" Computing '{self.name}' for the first time...\")\n", + " value = self.func(obj)\n", + " # Store in instance __dict__ so next access bypasses descriptor\n", + " obj.__dict__[self.name] = value\n", + " return value\n", + "\n", + "\n", + "class Circle:\n", + " def __init__(self, radius: float) -> None:\n", + " self.radius = radius\n", + "\n", + " @CachedProperty\n", + " def area(self) -> float:\n", + " return 3.14159 * self.radius ** 2\n", + "\n", + "\n", + "c = Circle(5.0)\n", + "\n", + "# First access goes through __get__ and computes\n", + "print(f\"Area: {c.area}\")\n", + "\n", + "# Second access reads from instance __dict__ (no descriptor call)\n", + "print(f\"Area: {c.area}\")\n", + "\n", + "# Verify the value is cached in instance __dict__\n", + "print(f\"\\n'area' in __dict__: {'area' in c.__dict__}\")\n", + "print(f\"Cached value: {c.__dict__['area']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 4: Data vs Non-Data Descriptor Lookup Order\n", + "\n", + "Python's attribute lookup follows this order:\n", + "1. **Data descriptors** on the class (and its MRO)\n", + "2. **Instance `__dict__`**\n", + "3. **Non-data descriptors** on the class (and its MRO)\n", + "\n", + "This is why data descriptors always intercept access, while non-data descriptors can be shadowed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "class DataDesc:\n", + " \"\"\"Data descriptor (has __get__ and __set__).\"\"\"\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> str:\n", + " return \"from data descriptor\"\n", + "\n", + " def __set__(self, obj: object, value: object) -> None:\n", + " pass # Intentionally does nothing for demo\n", + "\n", + "\n", + "class NonDataDesc:\n", + " \"\"\"Non-data descriptor (has only __get__).\"\"\"\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> str:\n", + " return \"from non-data descriptor\"\n", + "\n", + "\n", + "class Example:\n", + " data_attr = DataDesc()\n", + " non_data_attr = NonDataDesc()\n", + "\n", + "\n", + "e = Example()\n", + "\n", + "# Put values directly in instance __dict__\n", + "e.__dict__[\"data_attr\"] = \"from instance dict\"\n", + "e.__dict__[\"non_data_attr\"] = \"from instance dict\"\n", + "\n", + "# Data descriptor wins over instance __dict__\n", + "print(f\"data_attr: {e.data_attr}\")\n", + "\n", + "# Instance __dict__ wins over non-data descriptor\n", + "print(f\"non_data_attr: {e.non_data_attr}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 5: `__set_name__` -- Automatic Name Discovery\n", + "\n", + "When a descriptor is assigned as a class attribute, Python automatically calls `__set_name__(self, owner, name)` during class creation. This lets the descriptor know its attribute name without requiring it to be passed explicitly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "class Tracker:\n", + " \"\"\"Descriptor that tracks which names it was assigned to.\"\"\"\n", + "\n", + " def __set_name__(self, owner: type, name: str) -> None:\n", + " self.owner_name = owner.__name__\n", + " self.attr_name = name\n", + " print(f\"__set_name__ called: owner={owner.__name__}, name={name!r}\")\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> object:\n", + " if obj is None:\n", + " return self\n", + " return f\"{self.owner_name}.{self.attr_name}\"\n", + "\n", + "\n", + "# __set_name__ is called for each descriptor during class creation\n", + "class MyModel:\n", + " field_a = Tracker()\n", + " field_b = Tracker()\n", + " field_c = Tracker()\n", + "\n", + "\n", + "print()\n", + "m = MyModel()\n", + "print(f\"field_a: {m.field_a}\")\n", + "print(f\"field_b: {m.field_b}\")\n", + "print(f\"field_c: {m.field_c}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 6: Validation Descriptors\n", + "\n", + "One of the most powerful uses of descriptors is building reusable validators. A validation descriptor checks values in `__set__` and raises errors for invalid data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "class Validated:\n", + " \"\"\"Data descriptor that validates values are non-negative integers.\"\"\"\n", + "\n", + " def __set_name__(self, owner: type, name: str) -> None:\n", + " self.name = name\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> object:\n", + " if obj is None:\n", + " return self\n", + " return obj.__dict__.get(self.name)\n", + "\n", + " def __set__(self, obj: object, value: object) -> None:\n", + " if not isinstance(value, int) or value < 0:\n", + " raise ValueError(f\"{self.name} must be a non-negative int\")\n", + " obj.__dict__[self.name] = value\n", + "\n", + "\n", + "class Order:\n", + " quantity = Validated()\n", + " item_count = Validated()\n", + "\n", + "\n", + "order = Order()\n", + "order.quantity = 10\n", + "order.item_count = 5\n", + "print(f\"Quantity: {order.quantity}\")\n", + "print(f\"Item count: {order.item_count}\")\n", + "\n", + "# Invalid values are rejected\n", + "print()\n", + "for bad_value in [-1, \"ten\", 3.5]:\n", + " try:\n", + " order.quantity = bad_value\n", + " except ValueError as e:\n", + " print(f\"Rejected {bad_value!r}: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 7: Composable Validation Descriptors\n", + "\n", + "We can build a family of validators by using a base class with a `validate` method that subclasses override." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "class ValidatorBase:\n", + " \"\"\"Base class for validation descriptors.\"\"\"\n", + "\n", + " def __set_name__(self, owner: type, name: str) -> None:\n", + " self.name = name\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> object:\n", + " if obj is None:\n", + " return self\n", + " return obj.__dict__.get(self.name)\n", + "\n", + " def __set__(self, obj: object, value: object) -> None:\n", + " self.validate(value)\n", + " obj.__dict__[self.name] = value\n", + "\n", + " def validate(self, value: object) -> None:\n", + " raise NotImplementedError\n", + "\n", + "\n", + "class PositiveInt(ValidatorBase):\n", + " def validate(self, value: object) -> None:\n", + " if not isinstance(value, int) or value <= 0:\n", + " raise ValueError(f\"{self.name} must be a positive integer, got {value!r}\")\n", + "\n", + "\n", + "class NonEmptyString(ValidatorBase):\n", + " def validate(self, value: object) -> None:\n", + " if not isinstance(value, str) or len(value) == 0:\n", + " raise ValueError(f\"{self.name} must be a non-empty string, got {value!r}\")\n", + "\n", + "\n", + "class Employee:\n", + " name = NonEmptyString()\n", + " age = PositiveInt()\n", + " employee_id = PositiveInt()\n", + "\n", + "\n", + "emp = Employee()\n", + "emp.name = \"Alice\"\n", + "emp.age = 30\n", + "emp.employee_id = 1001\n", + "\n", + "print(f\"Name: {emp.name}, Age: {emp.age}, ID: {emp.employee_id}\")\n", + "\n", + "# Test validation\n", + "print()\n", + "for attr, bad_val in [(\"name\", \"\"), (\"age\", -5), (\"employee_id\", \"ABC\")]:\n", + " try:\n", + " setattr(emp, attr, bad_val)\n", + " except ValueError as e:\n", + " print(f\"Rejected {attr}={bad_val!r}: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 8: How `property` Is a Descriptor\n", + "\n", + "Python's built-in `property` is itself a data descriptor. Understanding this helps demystify how properties work." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "# property is a data descriptor because it has __get__, __set__, and __delete__\n", + "print(f\"property has __get__: {hasattr(property, '__get__')}\")\n", + "print(f\"property has __set__: {hasattr(property, '__set__')}\")\n", + "print(f\"property has __delete__: {hasattr(property, '__delete__')}\")\n", + "\n", + "\n", + "# A property-based class\n", + "class Temperature:\n", + " def __init__(self, celsius: float) -> None:\n", + " self._celsius = celsius\n", + "\n", + " @property\n", + " def celsius(self) -> float:\n", + " return self._celsius\n", + "\n", + " @celsius.setter\n", + " def celsius(self, value: float) -> None:\n", + " if value < -273.15:\n", + " raise ValueError(\"Temperature below absolute zero\")\n", + " self._celsius = value\n", + "\n", + " @property\n", + " def fahrenheit(self) -> float:\n", + " return self._celsius * 9 / 5 + 32\n", + "\n", + "\n", + "t = Temperature(100.0)\n", + "print(f\"\\n{t.celsius}C = {t.fahrenheit}F\")\n", + "\n", + "t.celsius = 0.0\n", + "print(f\"{t.celsius}C = {t.fahrenheit}F\")\n", + "\n", + "# Temperature.celsius is a property descriptor on the class\n", + "print(f\"\\nType of Temperature.celsius: {type(Temperature.__dict__['celsius'])}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 9: Functions Are Non-Data Descriptors\n", + "\n", + "Regular functions are non-data descriptors. Their `__get__` method is what creates bound methods when accessed via an instance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "class Greeter:\n", + " def greet(self, name: str) -> str:\n", + " return f\"Hello, {name}!\"\n", + "\n", + "\n", + "g = Greeter()\n", + "\n", + "# Functions have __get__ but not __set__ -- they are non-data descriptors\n", + "greet_func = Greeter.__dict__[\"greet\"]\n", + "print(f\"greet is a: {type(greet_func)}\")\n", + "print(f\"Has __get__: {hasattr(greet_func, '__get__')}\")\n", + "print(f\"Has __set__: {hasattr(greet_func, '__set__')}\")\n", + "\n", + "# Accessing via instance invokes __get__, producing a bound method\n", + "bound = g.greet\n", + "print(f\"\\nBound method: {bound}\")\n", + "print(f\"Bound to: {bound.__self__}\")\n", + "print(f\"Result: {bound('World')}\")\n", + "\n", + "# Manually calling __get__ does the same thing\n", + "manual_bound = greet_func.__get__(g, Greeter)\n", + "print(f\"\\nManual bind: {manual_bound('Python')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 10: Descriptor with `__delete__`\n", + "\n", + "A descriptor can also define `__delete__` to control what happens when `del obj.attr` is called." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "class ManagedAttribute:\n", + " \"\"\"Descriptor that manages the full lifecycle of an attribute.\"\"\"\n", + "\n", + " def __set_name__(self, owner: type, name: str) -> None:\n", + " self.name = name\n", + "\n", + " def __get__(self, obj: object, objtype: type | None = None) -> object:\n", + " if obj is None:\n", + " return self\n", + " return obj.__dict__.get(self.name, \"<undefined>\")\n", + "\n", + " def __set__(self, obj: object, value: object) -> None:\n", + " print(f\" Setting {self.name} = {value!r}\")\n", + " obj.__dict__[self.name] = value\n", + "\n", + " def __delete__(self, obj: object) -> None:\n", + " print(f\" Deleting {self.name}\")\n", + " obj.__dict__.pop(self.name, None)\n", + "\n", + "\n", + "class Config:\n", + " setting = ManagedAttribute()\n", + "\n", + "\n", + "cfg = Config()\n", + "cfg.setting = \"enabled\"\n", + "print(f\"Value: {cfg.setting}\")\n", + "\n", + "del cfg.setting\n", + "print(f\"After delete: {cfg.setting}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Descriptor Protocol\n", + "- A **descriptor** is any object with `__get__`, `__set__`, or `__delete__`\n", + "- **Data descriptors** define `__get__` and `__set__` (or `__delete__`) -- they have priority over instance `__dict__`\n", + "- **Non-data descriptors** define only `__get__` -- instance `__dict__` entries shadow them\n", + "\n", + "### Attribute Lookup Order\n", + "1. Data descriptors on the class (via MRO)\n", + "2. Instance `__dict__`\n", + "3. Non-data descriptors on the class (via MRO)\n", + "\n", + "### `__set_name__`\n", + "- Called automatically during class creation: `descriptor.__set_name__(owner_class, attr_name)`\n", + "- Lets descriptors discover their attribute name without explicit configuration\n", + "\n", + "### Validation Descriptors\n", + "- Enforce constraints in `__set__` before storing values\n", + "- Use a base class with a `validate` method for composable validators\n", + "- Store validated values in `obj.__dict__[self.name]` to avoid infinite recursion\n", + "\n", + "### Built-in Descriptors\n", + "- `property` is a data descriptor (`__get__` + `__set__` + `__delete__`)\n", + "- Functions are non-data descriptors (`__get__` only) -- this creates bound methods" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_35/02_slots_and_weakrefs.ipynb b/src/chapter_35/02_slots_and_weakrefs.ipynb new file mode 100644 index 0000000..ea70832 --- /dev/null +++ b/src/chapter_35/02_slots_and_weakrefs.ipynb @@ -0,0 +1,584 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 35: Slots and Weak References\n", + "\n", + "This notebook explores two advanced memory-optimization techniques in Python: `__slots__` for reducing per-instance memory overhead, and `weakref` for creating references that do not prevent garbage collection. Together, these patterns are essential for building high-performance, memory-efficient systems.\n", + "\n", + "## Key Concepts\n", + "- **`__slots__`**: Replaces per-instance `__dict__` with a fixed set of attribute slots\n", + "- **Memory savings**: Slotted objects use significantly less memory than regular objects\n", + "- **`weakref.ref`**: Creates a reference that does not keep the object alive\n", + "- **`WeakValueDictionary`**: A dictionary whose values are weak references, auto-cleaned on garbage collection\n", + "- **Caching patterns**: Using weak references for memory-friendly caches" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: `__slots__` Basics\n", + "\n", + "By default, Python stores instance attributes in a per-instance `__dict__`. Defining `__slots__` in a class replaces the dictionary with a fixed-size structure, restricting which attributes can be set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "class PointRegular:\n", + " \"\"\"A regular class with __dict__.\"\"\"\n", + "\n", + " def __init__(self, x: float, y: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + "\n", + "\n", + "class PointSlotted:\n", + " \"\"\"A slotted class with fixed attributes.\"\"\"\n", + "\n", + " __slots__ = (\"x\", \"y\")\n", + "\n", + " def __init__(self, x: float, y: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + "\n", + "\n", + "# Both work the same for basic access\n", + "r = PointRegular(1.0, 2.0)\n", + "s = PointSlotted(3.0, 4.0)\n", + "\n", + "print(f\"Regular: ({r.x}, {r.y})\")\n", + "print(f\"Slotted: ({s.x}, {s.y})\")\n", + "\n", + "# Regular has __dict__, slotted does not\n", + "print(f\"\\nRegular has __dict__: {hasattr(r, '__dict__')}\")\n", + "print(f\"Slotted has __dict__: {hasattr(s, '__dict__')}\")\n", + "\n", + "print(f\"\\nRegular __dict__: {r.__dict__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1e2f3a4", + "metadata": {}, + "source": [ + "## Section 2: `__slots__` Restricts Attributes\n", + "\n", + "With `__slots__`, you can only set attributes that are listed in the slots tuple. Attempting to set an unlisted attribute raises `AttributeError`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f2a3b4", + "metadata": {}, + "outputs": [], + "source": [ + "class Point:\n", + " __slots__ = (\"x\", \"y\")\n", + "\n", + "\n", + "p = Point()\n", + "p.x = 1\n", + "p.y = 2\n", + "print(f\"Point: ({p.x}, {p.y})\")\n", + "\n", + "# Trying to set an unlisted attribute fails\n", + "try:\n", + " p.z = 3 # type: ignore[attr-defined]\n", + " print(\"This should not print\")\n", + "except AttributeError as e:\n", + " print(f\"\\nAttributeError: {e}\")\n", + "\n", + "# Listing the available slots\n", + "print(f\"\\nSlots: {Point.__slots__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f1a2b3c4", + "metadata": {}, + "source": [ + "## Section 3: Memory Savings with `__slots__`\n", + "\n", + "The primary benefit of `__slots__` is memory reduction. Each instance no longer carries a `__dict__` (which is a hash table). The savings are significant when creating many instances." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b3c4d5", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "\n", + "class RegularPoint:\n", + " def __init__(self, x: float, y: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + "\n", + "\n", + "class SlottedPoint:\n", + " __slots__ = (\"x\", \"y\")\n", + "\n", + " def __init__(self, x: float, y: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + "\n", + "\n", + "r = RegularPoint(1.0, 2.0)\n", + "s = SlottedPoint(1.0, 2.0)\n", + "\n", + "# sys.getsizeof measures the object itself (not its __dict__)\n", + "r_size: int = sys.getsizeof(r) + sys.getsizeof(r.__dict__)\n", + "s_size: int = sys.getsizeof(s)\n", + "\n", + "print(f\"Regular point size: {r_size} bytes (object + __dict__)\")\n", + "print(f\"Slotted point size: {s_size} bytes\")\n", + "print(f\"Savings per object: {r_size - s_size} bytes ({(1 - s_size / r_size) * 100:.0f}%)\")\n", + "\n", + "# With 100,000 objects the savings add up\n", + "n: int = 100_000\n", + "print(f\"\\nWith {n:,} objects:\")\n", + "print(f\" Regular: ~{r_size * n / 1024 / 1024:.1f} MB\")\n", + "print(f\" Slotted: ~{s_size * n / 1024 / 1024:.1f} MB\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2c3d4e5", + "metadata": {}, + "source": [ + "## Section 4: `__slots__` with Inheritance\n", + "\n", + "When using `__slots__` with inheritance, each class in the hierarchy should declare only its own new slots. If a parent does not use `__slots__`, the subclass will still have a `__dict__`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [], + "source": [ + "class Base:\n", + " __slots__ = (\"x\",)\n", + "\n", + "\n", + "class Child(Base):\n", + " __slots__ = (\"y\",) # Only new slots, not 'x' again\n", + "\n", + "\n", + "c = Child()\n", + "c.x = 1\n", + "c.y = 2\n", + "print(f\"Child: x={c.x}, y={c.y}\")\n", + "print(f\"Has __dict__: {hasattr(c, '__dict__')}\")\n", + "\n", + "\n", + "# If a subclass does NOT define __slots__, it gets a __dict__\n", + "class ChildWithDict(Base):\n", + " pass # No __slots__ defined\n", + "\n", + "\n", + "d = ChildWithDict()\n", + "d.x = 1\n", + "d.z = 99 # This works because __dict__ is available\n", + "print(f\"\\nChildWithDict: x={d.x}, z={d.z}\")\n", + "print(f\"Has __dict__: {hasattr(d, '__dict__')}\")\n", + "print(f\"__dict__: {d.__dict__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d2e3f4a5", + "metadata": {}, + "source": [ + "## Section 5: Combining `__slots__` with `__dict__`\n", + "\n", + "You can include `'__dict__'` in `__slots__` to get both fixed slots and a dynamic dictionary. This gives you the best of both worlds: fast access for common attributes and flexibility for extras." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [], + "source": [ + "class Hybrid:\n", + " __slots__ = (\"x\", \"y\", \"__dict__\")\n", + "\n", + " def __init__(self, x: float, y: float) -> None:\n", + " self.x = x\n", + " self.y = y\n", + "\n", + "\n", + "h = Hybrid(1.0, 2.0)\n", + "\n", + "# Slotted attributes\n", + "print(f\"x={h.x}, y={h.y}\")\n", + "\n", + "# Dynamic attributes via __dict__\n", + "h.color = \"red\" # type: ignore[attr-defined]\n", + "h.label = \"origin\" # type: ignore[attr-defined]\n", + "print(f\"color={h.color}, label={h.label}\")\n", + "\n", + "# x and y are NOT in __dict__ (they use slots)\n", + "print(f\"\\n__dict__: {h.__dict__}\")\n", + "print(f\"'x' in __dict__: {'x' in h.__dict__}\")\n", + "print(f\"'color' in __dict__: {'color' in h.__dict__}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## Section 6: Weak References with `weakref.ref`\n", + "\n", + "A weak reference does not increase an object's reference count, so it does not prevent garbage collection. When the referent is collected, the weak reference returns `None`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class MyObj:\n", + " \"\"\"A simple class that supports weak references.\"\"\"\n", + "\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"MyObj({self.name!r})\"\n", + "\n", + "\n", + "# Create an object and a weak reference to it\n", + "obj = MyObj(\"example\")\n", + "ref: weakref.ref[MyObj] = weakref.ref(obj)\n", + "\n", + "# The weak reference is callable -- calling it returns the referent\n", + "print(f\"ref() = {ref()}\")\n", + "print(f\"ref() is obj: {ref() is obj}\")\n", + "\n", + "# Delete the strong reference\n", + "del obj\n", + "\n", + "# Now the weak reference returns None\n", + "print(f\"\\nAfter del obj:\")\n", + "print(f\"ref() = {ref()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## Section 7: Weak Reference Callbacks\n", + "\n", + "You can pass a callback to `weakref.ref` that is called when the referent is about to be garbage-collected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class Resource:\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + "\n", + "\n", + "def cleanup_callback(ref: weakref.ref) -> None:\n", + " \"\"\"Called when the referent is about to be collected.\"\"\"\n", + " print(f\" Callback: object was garbage collected (ref={ref})\")\n", + "\n", + "\n", + "# Create object with callback on weak reference\n", + "r = Resource(\"database_connection\")\n", + "ref: weakref.ref[Resource] = weakref.ref(r, cleanup_callback)\n", + "\n", + "print(f\"Before del: ref() = {ref()!r}\")\n", + "print(f\"Name: {ref().name}\")\n", + "\n", + "# Deleting the strong reference triggers garbage collection\n", + "print(\"\\nDeleting strong reference...\")\n", + "del r\n", + "print(f\"After del: ref() = {ref()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## Section 8: `WeakValueDictionary`\n", + "\n", + "A `WeakValueDictionary` holds weak references as its values. When a value object is garbage collected, its entry is automatically removed from the dictionary. This is ideal for caches." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class Item:\n", + " def __init__(self, name: str) -> None:\n", + " self.name = name\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Item({self.name!r})\"\n", + "\n", + "\n", + "# Create a WeakValueDictionary\n", + "cache: weakref.WeakValueDictionary[str, Item] = weakref.WeakValueDictionary()\n", + "\n", + "# Add items to the cache\n", + "item_a = Item(\"alpha\")\n", + "item_b = Item(\"beta\")\n", + "item_c = Item(\"gamma\")\n", + "\n", + "cache[\"a\"] = item_a\n", + "cache[\"b\"] = item_b\n", + "cache[\"c\"] = item_c\n", + "\n", + "print(f\"Cache keys: {list(cache.keys())}\")\n", + "print(f\"Cache['a']: {cache['a']}\")\n", + "\n", + "# Delete one strong reference\n", + "del item_b\n", + "print(f\"\\nAfter del item_b:\")\n", + "print(f\"Cache keys: {list(cache.keys())}\")\n", + "print(f\"'b' in cache: {'b' in cache}\")\n", + "\n", + "# Delete another\n", + "del item_a\n", + "print(f\"\\nAfter del item_a:\")\n", + "print(f\"Cache keys: {list(cache.keys())}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 9: Caching Pattern with Weak References\n", + "\n", + "A common pattern uses `WeakValueDictionary` to cache expensive objects so they are reused while alive, but do not prevent garbage collection when no longer needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class ExpensiveObject:\n", + " \"\"\"Simulates an object that is expensive to create.\"\"\"\n", + "\n", + " _cache: weakref.WeakValueDictionary[str, \"ExpensiveObject\"] = (\n", + " weakref.WeakValueDictionary()\n", + " )\n", + " _creation_count: int = 0\n", + "\n", + " def __init__(self, key: str) -> None:\n", + " self.key = key\n", + " ExpensiveObject._creation_count += 1\n", + " print(f\" Created ExpensiveObject({key!r}) [creation #{self._creation_count}]\")\n", + "\n", + " @classmethod\n", + " def get_or_create(cls, key: str) -> \"ExpensiveObject\":\n", + " \"\"\"Return cached instance if available, otherwise create new.\"\"\"\n", + " obj = cls._cache.get(key)\n", + " if obj is not None:\n", + " print(f\" Cache hit for {key!r}\")\n", + " return obj\n", + " print(f\" Cache miss for {key!r}\")\n", + " obj = cls(key)\n", + " cls._cache[key] = obj\n", + " return obj\n", + "\n", + "\n", + "# First access creates the object\n", + "obj1 = ExpensiveObject.get_or_create(\"config\")\n", + "\n", + "# Second access returns the cached object\n", + "obj2 = ExpensiveObject.get_or_create(\"config\")\n", + "print(f\"\\nSame object: {obj1 is obj2}\")\n", + "\n", + "# Delete strong references -- object can be collected\n", + "del obj1, obj2\n", + "\n", + "# Next access must create a new one\n", + "print()\n", + "obj3 = ExpensiveObject.get_or_create(\"config\")\n", + "print(f\"Total creations: {ExpensiveObject._creation_count}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 10: `__slots__` and `__weakref__`\n", + "\n", + "By default, slotted classes do not support weak references. To enable them, include `'__weakref__'` in `__slots__`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class SlottedNoWeakref:\n", + " __slots__ = (\"value\",)\n", + "\n", + "\n", + "class SlottedWithWeakref:\n", + " __slots__ = (\"value\", \"__weakref__\")\n", + "\n", + "\n", + "# Cannot create weak reference to SlottedNoWeakref\n", + "obj1 = SlottedNoWeakref()\n", + "try:\n", + " ref1 = weakref.ref(obj1)\n", + "except TypeError as e:\n", + " print(f\"Cannot weakref SlottedNoWeakref: {e}\")\n", + "\n", + "# Can create weak reference to SlottedWithWeakref\n", + "obj2 = SlottedWithWeakref()\n", + "obj2.value = 42\n", + "ref2: weakref.ref[SlottedWithWeakref] = weakref.ref(obj2)\n", + "print(f\"\\nWeakref works: ref2().value = {ref2().value}\")\n", + "\n", + "del obj2\n", + "print(f\"After del: ref2() = {ref2()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 11: `WeakKeyDictionary`\n", + "\n", + "While `WeakValueDictionary` has weak values, `WeakKeyDictionary` has weak keys. Entries are removed when the key object is garbage collected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "import weakref\n", + "\n", + "\n", + "class Session:\n", + " def __init__(self, user: str) -> None:\n", + " self.user = user\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Session({self.user!r})\"\n", + "\n", + "\n", + "# Track metadata about sessions without preventing their cleanup\n", + "session_metadata: weakref.WeakKeyDictionary[Session, dict[str, str]] = (\n", + " weakref.WeakKeyDictionary()\n", + ")\n", + "\n", + "s1 = Session(\"alice\")\n", + "s2 = Session(\"bob\")\n", + "\n", + "session_metadata[s1] = {\"ip\": \"10.0.0.1\", \"browser\": \"Firefox\"}\n", + "session_metadata[s2] = {\"ip\": \"10.0.0.2\", \"browser\": \"Chrome\"}\n", + "\n", + "print(f\"Tracked sessions: {len(session_metadata)}\")\n", + "print(f\"Alice's metadata: {session_metadata[s1]}\")\n", + "\n", + "# When a session is deleted, its metadata is automatically cleaned up\n", + "del s1\n", + "print(f\"\\nAfter del s1: tracked sessions = {len(session_metadata)}\")\n", + "print(f\"Remaining keys: {list(session_metadata.keys())}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### `__slots__`\n", + "- **`__slots__ = (\"attr1\", \"attr2\")`**: Replaces per-instance `__dict__` with fixed attribute slots\n", + "- **Memory savings**: Significant when creating many instances (no per-object hash table)\n", + "- **Attribute restriction**: Only listed attributes can be set; others raise `AttributeError`\n", + "- **Inheritance**: Each class should declare only its own new slots\n", + "- **`__dict__` in slots**: Include `\"__dict__\"` in `__slots__` for hybrid behavior\n", + "- **`__weakref__` in slots**: Include `\"__weakref__\"` to support weak references\n", + "\n", + "### `weakref`\n", + "- **`weakref.ref(obj)`**: Creates a weak reference; call it to get the referent or `None`\n", + "- **Callbacks**: `weakref.ref(obj, callback)` calls `callback` when the referent is collected\n", + "- **`WeakValueDictionary`**: Values are weak references; entries auto-removed on collection\n", + "- **`WeakKeyDictionary`**: Keys are weak references; entries auto-removed on collection\n", + "\n", + "### Caching Patterns\n", + "- Use `WeakValueDictionary` for caches that do not prevent garbage collection\n", + "- `get_or_create` pattern: check cache first, create and cache on miss\n", + "- Combine `__slots__` with `__weakref__` for memory-efficient weakly-referenceable objects" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_35/03_abc_and_copy_protocol.ipynb b/src/chapter_35/03_abc_and_copy_protocol.ipynb new file mode 100644 index 0000000..ca59898 --- /dev/null +++ b/src/chapter_35/03_abc_and_copy_protocol.ipynb @@ -0,0 +1,725 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# Chapter 35: ABCs and the Copy Protocol\n", + "\n", + "This notebook explores Abstract Base Classes (ABCs) for defining interfaces, virtual subclass registration, and Python's copy and pickle protocols for controlling how objects are duplicated and serialized.\n", + "\n", + "## Key Concepts\n", + "- **`ABC` and `@abstractmethod`**: Define interfaces that enforce method implementation\n", + "- **Virtual subclasses**: Register classes as subclasses without inheritance via `ABC.register`\n", + "- **`__subclasshook__`**: Customize `isinstance` and `issubclass` checks\n", + "- **`copy.copy` / `copy.deepcopy`**: Shallow and deep copying of objects\n", + "- **`__copy__` / `__deepcopy__`**: Customize copy behavior\n", + "- **Pickle protocol**: `__getstate__` / `__setstate__` for serialization control" + ] + }, + { + "cell_type": "markdown", + "id": "b1c2d3e4", + "metadata": {}, + "source": [ + "## Section 1: Abstract Base Classes (ABCs)\n", + "\n", + "An ABC defines an interface by declaring abstract methods. Subclasses must implement all abstract methods before they can be instantiated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1d2e3f4", + "metadata": {}, + "outputs": [], + "source": [ + "from abc import ABC, abstractmethod\n", + "\n", + "\n", + "class Shape(ABC):\n", + " \"\"\"Abstract base class for geometric shapes.\"\"\"\n", + "\n", + " @abstractmethod\n", + " def area(self) -> float:\n", + " \"\"\"Return the area of the shape.\"\"\"\n", + " ...\n", + "\n", + " @abstractmethod\n", + " def perimeter(self) -> float:\n", + " \"\"\"Return the perimeter of the shape.\"\"\"\n", + " ...\n", + "\n", + " def describe(self) -> str:\n", + " \"\"\"Concrete method available to all subclasses.\"\"\"\n", + " return f\"{type(self).__name__}: area={self.area():.2f}, perimeter={self.perimeter():.2f}\"\n", + "\n", + "\n", + "# Cannot instantiate an ABC directly\n", + "try:\n", + " Shape()\n", + "except TypeError as e:\n", + " print(f\"TypeError: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "\n", + "class Square(Shape):\n", + " \"\"\"Concrete subclass implementing all abstract methods.\"\"\"\n", + "\n", + " def __init__(self, side: float) -> None:\n", + " self.side = side\n", + "\n", + " def area(self) -> float:\n", + " return self.side ** 2\n", + "\n", + " def perimeter(self) -> float:\n", + " return 4 * self.side\n", + "\n", + "\n", + "class CircleShape(Shape):\n", + " \"\"\"Another concrete subclass.\"\"\"\n", + "\n", + " def __init__(self, radius: float) -> None:\n", + " self.radius = radius\n", + "\n", + " def area(self) -> float:\n", + " return math.pi * self.radius ** 2\n", + "\n", + " def perimeter(self) -> float:\n", + " return 2 * math.pi * self.radius\n", + "\n", + "\n", + "# Both concrete subclasses can be instantiated\n", + "s = Square(4.0)\n", + "c = CircleShape(3.0)\n", + "\n", + "print(s.describe())\n", + "print(c.describe())\n", + "\n", + "# isinstance checks work\n", + "print(f\"\\nisinstance(s, Shape): {isinstance(s, Shape)}\")\n", + "print(f\"isinstance(c, Shape): {isinstance(c, Shape)}\")\n", + "print(f\"s.area() == 16.0: {s.area() == 16.0}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## Section 2: Incomplete Implementations\n", + "\n", + "If a subclass does not implement all abstract methods, it is still abstract and cannot be instantiated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [], + "source": [ + "from abc import ABC, abstractmethod\n", + "\n", + "\n", + "class Animal(ABC):\n", + " @abstractmethod\n", + " def speak(self) -> str: ...\n", + "\n", + " @abstractmethod\n", + " def move(self) -> str: ...\n", + "\n", + "\n", + "# Incomplete: only implements speak, not move\n", + "class PartialAnimal(Animal):\n", + " def speak(self) -> str:\n", + " return \"...\"\n", + "\n", + "\n", + "try:\n", + " PartialAnimal()\n", + "except TypeError as e:\n", + " print(f\"TypeError: {e}\")\n", + "\n", + "\n", + "# Complete implementation\n", + "class Dog(Animal):\n", + " def speak(self) -> str:\n", + " return \"Woof!\"\n", + "\n", + " def move(self) -> str:\n", + " return \"runs\"\n", + "\n", + "\n", + "dog = Dog()\n", + "print(f\"\\nDog says: {dog.speak()}, {dog.move()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## Section 3: Virtual Subclasses with `register`\n", + "\n", + "ABCs support virtual subclass registration. A class can be registered as a subclass of an ABC without actually inheriting from it. This makes `isinstance` and `issubclass` return `True`, but the class does not appear in the MRO." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [], + "source": [ + "from abc import ABC, abstractmethod\n", + "\n", + "\n", + "class Drawable(ABC):\n", + " @abstractmethod\n", + " def draw(self) -> str: ...\n", + "\n", + "\n", + "# A regular class (no inheritance from Drawable)\n", + "class Circle:\n", + " def draw(self) -> str:\n", + " return \"O\"\n", + "\n", + "\n", + "# Register Circle as a virtual subclass of Drawable\n", + "Drawable.register(Circle)\n", + "\n", + "c = Circle()\n", + "print(f\"isinstance(c, Drawable): {isinstance(c, Drawable)}\")\n", + "print(f\"issubclass(Circle, Drawable): {issubclass(Circle, Drawable)}\")\n", + "\n", + "# But Drawable is NOT in Circle's MRO\n", + "print(f\"\\nCircle.__mro__: {Circle.__mro__}\")\n", + "print(f\"Drawable in Circle.__mro__: {Drawable in Circle.__mro__}\")\n", + "\n", + "# Virtual subclasses are NOT checked for abstract method implementation\n", + "class EmptyVirtual:\n", + " pass\n", + "\n", + "Drawable.register(EmptyVirtual)\n", + "ev = EmptyVirtual() # No error even though draw() is not implemented\n", + "print(f\"\\nisinstance(ev, Drawable): {isinstance(ev, Drawable)}\")\n", + "print(f\"hasattr(ev, 'draw'): {hasattr(ev, 'draw')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e4f5", + "metadata": {}, + "source": [ + "## Section 4: `__subclasshook__` for Structural Typing\n", + "\n", + "The `__subclasshook__` classmethod allows an ABC to customize `isinstance` and `issubclass` checks. This enables structural (duck) typing -- if a class has the right methods, it is considered a subclass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [], + "source": [ + "from abc import ABC, abstractmethod\n", + "\n", + "\n", + "class Renderable(ABC):\n", + " @abstractmethod\n", + " def render(self) -> str: ...\n", + "\n", + " @classmethod\n", + " def __subclasshook__(cls, C: type) -> bool:\n", + " \"\"\"Any class with a render() method is considered Renderable.\"\"\"\n", + " if cls is Renderable:\n", + " if hasattr(C, \"render\") and callable(getattr(C, \"render\")):\n", + " return True\n", + " return NotImplemented\n", + "\n", + "\n", + "# This class has render() but does NOT inherit from Renderable\n", + "class HTMLWidget:\n", + " def render(self) -> str:\n", + " return \"<div>Widget</div>\"\n", + "\n", + "\n", + "# This class does NOT have render()\n", + "class DataStore:\n", + " def save(self) -> None:\n", + " pass\n", + "\n", + "\n", + "widget = HTMLWidget()\n", + "store = DataStore()\n", + "\n", + "print(f\"isinstance(widget, Renderable): {isinstance(widget, Renderable)}\")\n", + "print(f\"isinstance(store, Renderable): {isinstance(store, Renderable)}\")\n", + "\n", + "# No registration needed -- structural check is automatic\n", + "print(f\"\\nissubclass(HTMLWidget, Renderable): {issubclass(HTMLWidget, Renderable)}\")\n", + "print(f\"issubclass(DataStore, Renderable): {issubclass(DataStore, Renderable)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a4b5", + "metadata": {}, + "source": [ + "## Section 5: Shallow Copy with `copy.copy`\n", + "\n", + "A shallow copy creates a new object but does not recursively copy nested objects. The new object shares references to the same inner objects." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2a3b4c5", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "\n", + "# Shallow copy of a nested list\n", + "original: list[list[int]] = [[1, 2], [3, 4], [5, 6]]\n", + "shallow: list[list[int]] = copy.copy(original)\n", + "\n", + "print(f\"original: {original}\")\n", + "print(f\"shallow: {shallow}\")\n", + "\n", + "# The outer list is a new object\n", + "print(f\"\\noriginal is shallow: {original is shallow}\")\n", + "print(f\"original == shallow: {original == shallow}\")\n", + "\n", + "# But inner lists are shared (same references)\n", + "print(f\"\\noriginal[0] is shallow[0]: {original[0] is shallow[0]}\")\n", + "\n", + "# Modifying a shared inner list affects both\n", + "shallow[0].append(99)\n", + "print(f\"\\nAfter shallow[0].append(99):\")\n", + "print(f\"original: {original}\")\n", + "print(f\"shallow: {shallow}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c5d6", + "metadata": {}, + "source": [ + "## Section 6: Deep Copy with `copy.deepcopy`\n", + "\n", + "A deep copy recursively copies all nested objects, creating a fully independent clone." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3c4d5e6", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "\n", + "original: list[list[int]] = [[1, 2], [3, 4]]\n", + "deep: list[list[int]] = copy.deepcopy(original)\n", + "\n", + "print(f\"original: {original}\")\n", + "print(f\"deep: {deep}\")\n", + "\n", + "# Both the outer and inner lists are different objects\n", + "print(f\"\\noriginal is deep: {original is deep}\")\n", + "print(f\"original[0] is deep[0]: {original[0] is deep[0]}\")\n", + "\n", + "# Modifying the deep copy does NOT affect the original\n", + "deep[0].append(99)\n", + "print(f\"\\nAfter deep[0].append(99):\")\n", + "print(f\"original: {original}\")\n", + "print(f\"deep: {deep}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e5f6", + "metadata": {}, + "source": [ + "## Section 7: Custom `__copy__` and `__deepcopy__`\n", + "\n", + "Classes can customize shallow and deep copy behavior by implementing `__copy__` and `__deepcopy__`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3e4f5a6", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "\n", + "\n", + "class Config:\n", + " \"\"\"A configuration object that tracks how many times it was copied.\"\"\"\n", + "\n", + " def __init__(self, settings: dict[str, object]) -> None:\n", + " self.settings = settings\n", + " self.copy_count: int = 0\n", + "\n", + " def __copy__(self) -> \"Config\":\n", + " \"\"\"Shallow copy: copies the settings dict but shares nested values.\"\"\"\n", + " new = Config(self.settings.copy())\n", + " new.copy_count = self.copy_count + 1\n", + " return new\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Config(settings={self.settings}, copy_count={self.copy_count})\"\n", + "\n", + "\n", + "c1 = Config({\"debug\": True, \"log_level\": \"INFO\"})\n", + "c2 = copy.copy(c1)\n", + "c3 = copy.copy(c2)\n", + "\n", + "print(f\"c1: {c1}\")\n", + "print(f\"c2: {c2}\")\n", + "print(f\"c3: {c3}\")\n", + "\n", + "# Settings are independent (shallow-copied dict)\n", + "print(f\"\\nc1.settings is c2.settings: {c1.settings is c2.settings}\")\n", + "\n", + "c2.settings[\"debug\"] = False\n", + "print(f\"c1.settings['debug']: {c1.settings['debug']}\")\n", + "print(f\"c2.settings['debug']: {c2.settings['debug']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "\n", + "\n", + "class TreeNode:\n", + " \"\"\"A tree node that customizes deep copy.\"\"\"\n", + "\n", + " def __init__(self, value: str, children: list[\"TreeNode\"] | None = None) -> None:\n", + " self.value = value\n", + " self.children: list[TreeNode] = children or []\n", + "\n", + " def __deepcopy__(self, memo: dict) -> \"TreeNode\":\n", + " \"\"\"Deep copy that tracks visited nodes via memo dict.\"\"\"\n", + " # Check memo to handle circular references\n", + " if id(self) in memo:\n", + " return memo[id(self)]\n", + "\n", + " new_node = TreeNode(self.value)\n", + " memo[id(self)] = new_node\n", + " new_node.children = copy.deepcopy(self.children, memo)\n", + " return new_node\n", + "\n", + " def __repr__(self) -> str:\n", + " child_vals: list[str] = [c.value for c in self.children]\n", + " return f\"TreeNode({self.value!r}, children={child_vals})\"\n", + "\n", + "\n", + "# Build a tree\n", + "root = TreeNode(\"root\", [\n", + " TreeNode(\"child1\", [TreeNode(\"grandchild1\")]),\n", + " TreeNode(\"child2\"),\n", + "])\n", + "\n", + "# Deep copy the entire tree\n", + "root_copy = copy.deepcopy(root)\n", + "\n", + "print(f\"Original: {root}\")\n", + "print(f\"Copy: {root_copy}\")\n", + "\n", + "# Completely independent\n", + "print(f\"\\nroot is root_copy: {root is root_copy}\")\n", + "print(f\"root.children[0] is root_copy.children[0]: {root.children[0] is root_copy.children[0]}\")\n", + "\n", + "# Modify copy without affecting original\n", + "root_copy.children[0].value = \"MODIFIED\"\n", + "print(f\"\\nOriginal child1: {root.children[0].value}\")\n", + "print(f\"Copy child1: {root_copy.children[0].value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## Section 8: Copy with Class Instances\n", + "\n", + "By default, `copy.copy` and `copy.deepcopy` work with most objects. Understanding the default behavior helps you decide when custom `__copy__` or `__deepcopy__` methods are needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "\n", + "\n", + "class Team:\n", + " def __init__(self, name: str, members: list[str]) -> None:\n", + " self.name = name\n", + " self.members = members\n", + "\n", + "\n", + "original = Team(\"Engineering\", [\"Alice\", \"Bob\"])\n", + "\n", + "# Shallow copy: new Team object but shares the members list\n", + "shallow = copy.copy(original)\n", + "print(f\"Shallow copy name: {shallow.name}\")\n", + "print(f\"Members shared: {original.members is shallow.members}\")\n", + "\n", + "# Deep copy: new Team object AND new members list\n", + "deep = copy.deepcopy(original)\n", + "print(f\"\\nDeep copy name: {deep.name}\")\n", + "print(f\"Members shared: {original.members is deep.members}\")\n", + "\n", + "# Verify independence of deep copy\n", + "deep.members.append(\"Charlie\")\n", + "print(f\"\\nOriginal members: {original.members}\")\n", + "print(f\"Deep copy members: {deep.members}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## Section 9: Pickle Protocol -- `__getstate__` and `__setstate__`\n", + "\n", + "The `pickle` module serializes Python objects to bytes. Classes can customize serialization with `__getstate__` (what to save) and `__setstate__` (how to restore)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "\n", + "\n", + "class Connection:\n", + " \"\"\"A class with non-serializable state that customizes pickling.\"\"\"\n", + "\n", + " def __init__(self, host: str, port: int) -> None:\n", + " self.host = host\n", + " self.port = port\n", + " self._socket = f\"<socket to {host}:{port}>\" # Simulate non-serializable\n", + " self.connected: bool = True\n", + "\n", + " def __getstate__(self) -> dict[str, object]:\n", + " \"\"\"Return state for pickling (exclude _socket).\"\"\"\n", + " state: dict[str, object] = self.__dict__.copy()\n", + " del state[\"_socket\"] # Don't pickle the socket\n", + " state[\"connected\"] = False # Mark as disconnected\n", + " return state\n", + "\n", + " def __setstate__(self, state: dict[str, object]) -> None:\n", + " \"\"\"Restore state from pickle and reconnect.\"\"\"\n", + " self.__dict__.update(state)\n", + " # Recreate the socket\n", + " self._socket = f\"<socket to {self.host}:{self.port}>\"\n", + " self.connected = True\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Connection({self.host}:{self.port}, connected={self.connected})\"\n", + "\n", + "\n", + "# Pickle and unpickle\n", + "conn = Connection(\"localhost\", 5432)\n", + "print(f\"Before pickle: {conn}\")\n", + "print(f\"Socket: {conn._socket}\")\n", + "\n", + "data: bytes = pickle.dumps(conn)\n", + "print(f\"\\nPickled size: {len(data)} bytes\")\n", + "\n", + "restored: Connection = pickle.loads(data)\n", + "print(f\"\\nAfter unpickle: {restored}\")\n", + "print(f\"Socket: {restored._socket}\")\n", + "print(f\"Connected: {restored.connected}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f6a7", + "metadata": {}, + "source": [ + "## Section 10: `__reduce__` for Advanced Pickle Control\n", + "\n", + "For more control over pickling, implement `__reduce__`. It returns a tuple that tells pickle how to reconstruct the object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f5a6b7", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "\n", + "\n", + "class Color:\n", + " \"\"\"A color class that uses __reduce__ for custom pickling.\"\"\"\n", + "\n", + " def __init__(self, r: int, g: int, b: int) -> None:\n", + " self.r = r\n", + " self.g = g\n", + " self.b = b\n", + "\n", + " def __reduce__(self) -> tuple:\n", + " \"\"\"Return (callable, args) for reconstruction.\"\"\"\n", + " return (Color, (self.r, self.g, self.b))\n", + "\n", + " def __repr__(self) -> str:\n", + " return f\"Color({self.r}, {self.g}, {self.b})\"\n", + "\n", + " def hex_code(self) -> str:\n", + " return f\"#{self.r:02x}{self.g:02x}{self.b:02x}\"\n", + "\n", + "\n", + "original = Color(255, 128, 0)\n", + "print(f\"Original: {original} -> {original.hex_code()}\")\n", + "\n", + "# Pickle round-trip\n", + "data: bytes = pickle.dumps(original)\n", + "restored: Color = pickle.loads(data)\n", + "print(f\"Restored: {restored} -> {restored.hex_code()}\")\n", + "\n", + "# Verify equality of attributes\n", + "print(f\"\\nSame r: {original.r == restored.r}\")\n", + "print(f\"Same g: {original.g == restored.g}\")\n", + "print(f\"Same b: {original.b == restored.b}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f4a5b6c7", + "metadata": {}, + "source": [ + "## Section 11: Combining ABCs with Copy\n", + "\n", + "A practical pattern: define an ABC with a `clone` method that uses `copy.deepcopy` to create independent copies of any concrete subclass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5b6c7d8", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "from abc import ABC, abstractmethod\n", + "\n", + "\n", + "class Prototype(ABC):\n", + " \"\"\"ABC that provides a clone method via copy.deepcopy.\"\"\"\n", + "\n", + " @abstractmethod\n", + " def describe(self) -> str: ...\n", + "\n", + " def clone(self) -> \"Prototype\":\n", + " \"\"\"Create a deep copy of this object.\"\"\"\n", + " return copy.deepcopy(self)\n", + "\n", + "\n", + "class Document(Prototype):\n", + " def __init__(self, title: str, pages: list[str]) -> None:\n", + " self.title = title\n", + " self.pages = pages\n", + "\n", + " def describe(self) -> str:\n", + " return f\"Document({self.title!r}, {len(self.pages)} pages)\"\n", + "\n", + "\n", + "# Create and clone a document\n", + "doc = Document(\"Report\", [\"Introduction\", \"Analysis\", \"Conclusion\"])\n", + "doc_clone = doc.clone()\n", + "\n", + "print(f\"Original: {doc.describe()}\")\n", + "print(f\"Clone: {doc_clone.describe()}\")\n", + "\n", + "# Clone is fully independent\n", + "doc_clone.title = \"Report v2\"\n", + "doc_clone.pages.append(\"Appendix\")\n", + "\n", + "print(f\"\\nAfter modifying clone:\")\n", + "print(f\"Original: {doc.describe()}, pages={doc.pages}\")\n", + "print(f\"Clone: {doc_clone.describe()}, pages={doc_clone.pages}\")\n", + "\n", + "# isinstance checks\n", + "print(f\"\\nisinstance(doc_clone, Prototype): {isinstance(doc_clone, Prototype)}\")\n", + "print(f\"isinstance(doc_clone, Document): {isinstance(doc_clone, Document)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b5c6d7e8", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### Abstract Base Classes\n", + "- **`class MyABC(ABC)`**: Create an abstract base class\n", + "- **`@abstractmethod`**: Declare methods that subclasses must implement\n", + "- **Instantiation check**: ABCs with unimplemented abstract methods raise `TypeError`\n", + "- **Concrete methods**: ABCs can include non-abstract methods that subclasses inherit\n", + "\n", + "### Virtual Subclasses\n", + "- **`MyABC.register(SomeClass)`**: Register a class as a virtual subclass\n", + "- **`isinstance` / `issubclass`**: Return `True` for registered classes\n", + "- **No MRO change**: Virtual subclasses do not inherit from the ABC\n", + "- **`__subclasshook__`**: Customize structural type checking (duck typing)\n", + "\n", + "### Copy Protocol\n", + "- **`copy.copy(obj)`**: Shallow copy -- new object, shared nested references\n", + "- **`copy.deepcopy(obj)`**: Deep copy -- recursively copies all nested objects\n", + "- **`__copy__(self)`**: Customize shallow copy behavior\n", + "- **`__deepcopy__(self, memo)`**: Customize deep copy with circular reference tracking\n", + "\n", + "### Pickle Protocol\n", + "- **`__getstate__`**: Return the state to be serialized (exclude non-serializable data)\n", + "- **`__setstate__`**: Restore object from deserialized state\n", + "- **`__reduce__`**: Return `(callable, args)` for advanced reconstruction control\n", + "- Use pickle for persistence; use copy for in-memory duplication" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/chapter_35/README.md b/src/chapter_35/README.md new file mode 100644 index 0000000..de8b83c --- /dev/null +++ b/src/chapter_35/README.md @@ -0,0 +1,9 @@ +# Chapter 35: Advanced Python Patterns + +Deep dive into advanced object model patterns — descriptors, __slots__ optimization, weakrefs for caching, ABCs for interfaces, and the copy protocol. + +## Notebooks + +1. **01_descriptors_deep_dive.ipynb** — Data vs non-data descriptors, descriptor protocol, __set_name__, validation descriptors +2. **02_slots_and_weakrefs.ipynb** — __slots__ memory optimization, weakref.ref, WeakValueDictionary, caching patterns +3. **03_abc_and_copy_protocol.ipynb** — ABCs, virtual subclasses, __copy__/__deepcopy__, pickle protocol diff --git a/src/chapter_35/__init__.py b/src/chapter_35/__init__.py new file mode 100644 index 0000000..992f5f2 --- /dev/null +++ b/src/chapter_35/__init__.py @@ -0,0 +1 @@ +"""Chapter 35: Advanced Python Patterns.""" diff --git a/tests/test_chapter_31.py b/tests/test_chapter_31.py new file mode 100644 index 0000000..c150cfe --- /dev/null +++ b/tests/test_chapter_31.py @@ -0,0 +1,124 @@ +"""Tests for Chapter 31: String Methods and Formatting.""" + +import difflib +import string +import textwrap + + +class TestFStringsAndFormatting: + """Test f-string and format spec features.""" + + def test_fstring_expression(self) -> None: + """f-strings evaluate expressions inline.""" + name = "Alice" + assert f"Hello, {name}!" == "Hello, Alice!" + assert f"{2 + 3}" == "5" + assert f"{'hello'.upper()}" == "HELLO" + + def test_format_spec_alignment(self) -> None: + """Format specs control alignment and padding.""" + assert f"{'left':<10}" == "left " + assert f"{'right':>10}" == " right" + assert f"{'center':^10}" == " center " + assert f"{'fill':*^10}" == "***fill***" + + def test_format_spec_numbers(self) -> None: + """Format specs control number display.""" + assert f"{3.14159:.2f}" == "3.14" + assert f"{1000000:,}" == "1,000,000" + assert f"{255:08b}" == "11111111" + assert f"{255:#x}" == "0xff" + + def test_format_spec_percentage(self) -> None: + """Percentage format multiplies by 100.""" + assert f"{0.856:.1%}" == "85.6%" + + def test_format_method(self) -> None: + """str.format() supports positional and keyword args.""" + assert "{} {}".format("hello", "world") == "hello world" + assert "{name} is {age}".format(name="Alice", age=30) == "Alice is 30" + + +class TestStringMethods: + """Test str methods.""" + + def test_split_and_join(self) -> None: + """split and join are inverses.""" + words = "hello world python".split() + assert words == ["hello", "world", "python"] + assert "-".join(words) == "hello-world-python" + + def test_strip_variants(self) -> None: + """strip removes whitespace from edges.""" + s = " hello " + assert s.strip() == "hello" + assert s.lstrip() == "hello " + assert s.rstrip() == " hello" + + def test_replace(self) -> None: + """replace substitutes substrings.""" + assert "hello world".replace("world", "python") == "hello python" + assert "aaa".replace("a", "b", 2) == "bba" + + def test_startswith_endswith(self) -> None: + """startswith and endswith check prefixes/suffixes.""" + assert "hello.py".endswith(".py") + assert "hello.py".startswith("hello") + assert "test.txt".endswith((".txt", ".csv")) + + def test_translate_table(self) -> None: + """maketrans and translate perform character mapping.""" + table = str.maketrans("aeiou", "12345") + assert "hello".translate(table) == "h2ll4" + + def test_partition(self) -> None: + """partition splits on first occurrence.""" + before, sep, after = "key=value=extra".partition("=") + assert before == "key" + assert sep == "=" + assert after == "value=extra" + + +class TestStringModule: + """Test string module constants and templates.""" + + def test_string_constants(self) -> None: + """string module provides character sets.""" + assert "a" in string.ascii_lowercase + assert "Z" in string.ascii_uppercase + assert "5" in string.digits + assert "!" in string.punctuation + + def test_string_template(self) -> None: + """Template provides safe substitution.""" + tmpl = string.Template("Hello, $name!") + assert tmpl.substitute(name="Alice") == "Hello, Alice!" + assert tmpl.safe_substitute() == "Hello, $name!" + + +class TestDifflibAndTextwrap: + """Test difflib and textwrap utilities.""" + + def test_difflib_get_close_matches(self) -> None: + """get_close_matches finds similar strings.""" + words = ["apple", "application", "apply", "banana"] + matches = difflib.get_close_matches("appli", words) + assert "application" in matches or "apply" in matches + + def test_difflib_sequence_matcher(self) -> None: + """SequenceMatcher computes similarity ratio.""" + ratio = difflib.SequenceMatcher(None, "hello", "hallo").ratio() + assert 0.7 < ratio < 1.0 + + def test_textwrap_wrap(self) -> None: + """textwrap.wrap breaks text into lines.""" + text = "This is a long sentence that should be wrapped at a certain width." + lines = textwrap.wrap(text, width=30) + assert all(len(line) <= 30 for line in lines) + + def test_textwrap_shorten(self) -> None: + """textwrap.shorten truncates with placeholder.""" + text = "Hello World, this is a long string" + short = textwrap.shorten(text, width=20) + assert len(short) <= 20 + assert short.endswith("[...]") diff --git a/tests/test_chapter_32.py b/tests/test_chapter_32.py new file mode 100644 index 0000000..3578175 --- /dev/null +++ b/tests/test_chapter_32.py @@ -0,0 +1,147 @@ +"""Tests for Chapter 32: Numeric Computing.""" + +import cmath +import math +import random +import statistics +from decimal import Decimal, InvalidOperation +from fractions import Fraction + + +class TestMathModule: + """Test math module functions.""" + + def test_constants(self) -> None: + """math provides fundamental constants.""" + assert abs(math.pi - 3.14159265) < 1e-6 + assert abs(math.e - 2.71828182) < 1e-6 + assert math.inf > 1e308 + assert math.isnan(math.nan) + + def test_basic_functions(self) -> None: + """math provides common mathematical functions.""" + assert math.sqrt(16) == 4.0 + assert math.ceil(3.2) == 4 + assert math.floor(3.8) == 3 + assert math.factorial(5) == 120 + + def test_logarithms(self) -> None: + """math provides logarithmic functions.""" + assert math.log(math.e) == 1.0 + assert math.log10(100) == 2.0 + assert math.log2(8) == 3.0 + + def test_trigonometry(self) -> None: + """math provides trigonometric functions.""" + assert abs(math.sin(math.pi / 2) - 1.0) < 1e-10 + assert abs(math.cos(0) - 1.0) < 1e-10 + + def test_number_theory(self) -> None: + """math provides number theory functions.""" + assert math.gcd(12, 8) == 4 + assert math.lcm(4, 6) == 12 + assert math.isclose(0.1 + 0.2, 0.3) + + def test_cmath_complex(self) -> None: + """cmath handles complex number operations.""" + z = complex(3, 4) + assert abs(cmath.phase(z) - 0.9272952) < 1e-5 + assert abs(abs(z) - 5.0) < 1e-10 + + +class TestDecimal: + """Test Decimal for precise arithmetic.""" + + def test_decimal_precision(self) -> None: + """Decimal avoids floating-point rounding errors.""" + assert Decimal("0.1") + Decimal("0.2") == Decimal("0.3") + assert 0.1 + 0.2 != 0.3 # float fails + + def test_decimal_from_string(self) -> None: + """Decimal should be created from strings for precision.""" + d = Decimal("3.14159") + assert str(d) == "3.14159" + + def test_decimal_quantize(self) -> None: + """quantize rounds to a specific number of places.""" + d = Decimal("3.14159") + assert d.quantize(Decimal("0.01")) == Decimal("3.14") + + def test_decimal_invalid(self) -> None: + """Invalid operations raise InvalidOperation.""" + try: + Decimal("not_a_number") + assert False, "Should have raised" + except InvalidOperation: + pass + + +class TestFractions: + """Test Fraction for exact rationals.""" + + def test_fraction_creation(self) -> None: + """Fractions represent exact rational numbers.""" + f = Fraction(1, 3) + assert f.numerator == 1 + assert f.denominator == 3 + + def test_fraction_auto_reduce(self) -> None: + """Fractions auto-reduce to lowest terms.""" + assert Fraction(4, 8) == Fraction(1, 2) + + def test_fraction_arithmetic(self) -> None: + """Fractions support exact arithmetic.""" + assert Fraction(1, 3) + Fraction(1, 6) == Fraction(1, 2) + assert Fraction(2, 3) * Fraction(3, 4) == Fraction(1, 2) + + def test_fraction_from_float(self) -> None: + """Fractions can approximate floats.""" + f = Fraction(0.5) + assert f == Fraction(1, 2) + + +class TestRandomAndStatistics: + """Test random and statistics modules.""" + + def test_random_seed_reproducible(self) -> None: + """Setting seed makes random reproducible.""" + random.seed(42) + a = random.random() + random.seed(42) + b = random.random() + assert a == b + + def test_random_choice(self) -> None: + """choice picks from a sequence.""" + random.seed(42) + items = ["a", "b", "c", "d"] + result = random.choice(items) + assert result in items + + def test_random_shuffle(self) -> None: + """shuffle randomizes a list in place.""" + random.seed(42) + items = [1, 2, 3, 4, 5] + random.shuffle(items) + assert sorted(items) == [1, 2, 3, 4, 5] + + def test_random_sample(self) -> None: + """sample returns unique elements.""" + random.seed(42) + result = random.sample(range(100), k=5) + assert len(result) == 5 + assert len(set(result)) == 5 + + def test_statistics_mean(self) -> None: + """statistics.mean computes arithmetic mean.""" + assert statistics.mean([1, 2, 3, 4, 5]) == 3 + + def test_statistics_median(self) -> None: + """statistics.median returns the middle value.""" + assert statistics.median([1, 3, 5]) == 3 + assert statistics.median([1, 3, 5, 7]) == 4 + + def test_statistics_stdev(self) -> None: + """statistics.stdev computes standard deviation.""" + data = [2, 4, 4, 4, 5, 5, 7, 9] + assert abs(statistics.stdev(data) - 2.0) < 0.2 diff --git a/tests/test_chapter_33.py b/tests/test_chapter_33.py new file mode 100644 index 0000000..237d086 --- /dev/null +++ b/tests/test_chapter_33.py @@ -0,0 +1,142 @@ +"""Tests for Chapter 33: OS and System Interaction.""" + +import os +import platform +import shutil +import subprocess +import sys +import tempfile + + +class TestOSAndPlatform: + """Test os and platform modules.""" + + def test_os_getcwd(self) -> None: + """os.getcwd returns current working directory.""" + cwd = os.getcwd() + assert isinstance(cwd, str) + assert os.path.isdir(cwd) + + def test_os_path_operations(self) -> None: + """os.path provides path manipulation.""" + path = os.path.join("src", "chapter_33", "__init__.py") + assert os.path.basename(path) == "__init__.py" + assert os.path.dirname(path) == os.path.join("src", "chapter_33") + name, ext = os.path.splitext("script.py") + assert name == "script" + assert ext == ".py" + + def test_os_environ_is_mapping(self) -> None: + """os.environ is a mapping of environment variables.""" + assert isinstance(os.environ, os.environ.__class__) + assert "PATH" in os.environ + + def test_platform_info(self) -> None: + """platform module provides system information.""" + assert platform.system() in ("Darwin", "Linux", "Windows") + assert isinstance(platform.python_version(), str) + assert ( + platform.python_version() + == f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + ) + + def test_os_walk(self) -> None: + """os.walk traverses directory trees.""" + with tempfile.TemporaryDirectory() as tmpdir: + os.makedirs(os.path.join(tmpdir, "sub")) + open(os.path.join(tmpdir, "file.txt"), "w").close() + open(os.path.join(tmpdir, "sub", "nested.txt"), "w").close() + + dirs_found: list[str] = [] + for dirpath, dirnames, filenames in os.walk(tmpdir): + dirs_found.append(dirpath) + assert len(dirs_found) == 2 + + +class TestShutilAndTempfile: + """Test shutil and tempfile operations.""" + + def test_shutil_copy(self) -> None: + """shutil.copy copies files.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = os.path.join(tmpdir, "source.txt") + dst = os.path.join(tmpdir, "dest.txt") + with open(src, "w") as f: + f.write("hello") + shutil.copy(src, dst) + with open(dst) as f: + assert f.read() == "hello" + + def test_shutil_copytree(self) -> None: + """shutil.copytree copies directory trees.""" + with tempfile.TemporaryDirectory() as tmpdir: + src_dir = os.path.join(tmpdir, "src") + dst_dir = os.path.join(tmpdir, "dst") + os.makedirs(src_dir) + with open(os.path.join(src_dir, "file.txt"), "w") as f: + f.write("content") + shutil.copytree(src_dir, dst_dir) + assert os.path.exists(os.path.join(dst_dir, "file.txt")) + + def test_shutil_disk_usage(self) -> None: + """shutil.disk_usage returns disk space info.""" + usage = shutil.disk_usage("/") + assert usage.total > 0 + assert usage.used > 0 + assert usage.free > 0 + + def test_tempfile_named(self) -> None: + """NamedTemporaryFile creates a temporary file with a name.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write("temp data") + name = f.name + assert os.path.exists(name) + os.unlink(name) + + def test_tempfile_directory(self) -> None: + """TemporaryDirectory auto-cleans on exit.""" + with tempfile.TemporaryDirectory() as tmpdir: + assert os.path.isdir(tmpdir) + path = tmpdir + assert not os.path.exists(path) + + +class TestSubprocess: + """Test subprocess module.""" + + def test_subprocess_run(self) -> None: + """subprocess.run executes a command.""" + result = subprocess.run(["echo", "hello"], capture_output=True, text=True) + assert result.returncode == 0 + assert "hello" in result.stdout + + def test_subprocess_capture_stderr(self) -> None: + """subprocess captures stderr separately.""" + result = subprocess.run( + [sys.executable, "-c", "import sys; sys.stderr.write('error\\n')"], + capture_output=True, + text=True, + ) + assert "error" in result.stderr + + def test_subprocess_check_returncode(self) -> None: + """check_returncode raises on non-zero exit.""" + result = subprocess.run( + [sys.executable, "-c", "raise SystemExit(1)"], + capture_output=True, + ) + assert result.returncode == 1 + try: + result.check_returncode() + assert False, "Should have raised" + except subprocess.CalledProcessError: + pass + + def test_subprocess_pipe(self) -> None: + """subprocess can pipe between commands.""" + result = subprocess.run( + [sys.executable, "-c", "print('hello world')"], + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "hello world" diff --git a/tests/test_chapter_34.py b/tests/test_chapter_34.py new file mode 100644 index 0000000..bb7f569 --- /dev/null +++ b/tests/test_chapter_34.py @@ -0,0 +1,134 @@ +"""Tests for Chapter 34: Email and Data Encoding.""" + +import base64 +import binascii +import mimetypes +import quopri +from email.message import EmailMessage + + +class TestEmailMessages: + """Test email message construction.""" + + def test_email_basic(self) -> None: + """EmailMessage creates a basic email.""" + msg = EmailMessage() + msg["Subject"] = "Test" + msg["From"] = "alice@example.com" + msg["To"] = "bob@example.com" + msg.set_content("Hello, Bob!") + + assert msg["Subject"] == "Test" + assert msg["From"] == "alice@example.com" + assert "Hello, Bob!" in msg.get_content() + + def test_email_headers(self) -> None: + """Email headers are case-insensitive.""" + msg = EmailMessage() + msg["Content-Type"] = "text/plain" + assert msg["content-type"] == "text/plain" + + def test_email_multipart(self) -> None: + """EmailMessage supports multipart messages.""" + msg = EmailMessage() + msg["Subject"] = "With attachment" + msg.set_content("Main body") + msg.add_attachment( + b"file content", + maintype="application", + subtype="octet-stream", + filename="data.bin", + ) + assert msg.is_multipart() + + def test_email_as_string(self) -> None: + """EmailMessage serializes to string.""" + msg = EmailMessage() + msg["Subject"] = "Hello" + msg.set_content("Body text") + text = msg.as_string() + assert "Subject: Hello" in text + assert "Body text" in text + + +class TestMimeTypes: + """Test MIME type detection.""" + + def test_guess_type(self) -> None: + """mimetypes.guess_type identifies file types.""" + mime, _ = mimetypes.guess_type("document.pdf") + assert mime == "application/pdf" + + mime, _ = mimetypes.guess_type("image.png") + assert mime == "image/png" + + mime, _ = mimetypes.guess_type("script.py") + assert mime == "text/x-python" + + def test_guess_extension(self) -> None: + """mimetypes.guess_extension finds extensions.""" + ext = mimetypes.guess_extension("text/html") + assert ext in (".html", ".htm") + + def test_common_types(self) -> None: + """Common MIME types are well-known.""" + assert mimetypes.guess_type("file.json")[0] == "application/json" + assert mimetypes.guess_type("file.csv")[0] == "text/csv" + assert mimetypes.guess_type("file.txt")[0] == "text/plain" + + +class TestBase64Encoding: + """Test base64 encoding/decoding.""" + + def test_base64_encode_decode(self) -> None: + """base64 encodes bytes to ASCII and back.""" + original = b"Hello, World!" + encoded = base64.b64encode(original) + assert isinstance(encoded, bytes) + decoded = base64.b64decode(encoded) + assert decoded == original + + def test_base64_urlsafe(self) -> None: + """URL-safe base64 uses - and _ instead of + and /.""" + data = bytes(range(256)) + encoded = base64.urlsafe_b64encode(data) + assert b"+" not in encoded + assert b"/" not in encoded + assert base64.urlsafe_b64decode(encoded) == data + + def test_base64_string(self) -> None: + """base64 works with string data via encode/decode.""" + text = "Python is great!" + encoded = base64.b64encode(text.encode()).decode() + assert isinstance(encoded, str) + decoded = base64.b64decode(encoded).decode() + assert decoded == text + + +class TestQuopriAndBinascii: + """Test quoted-printable and binary-ASCII conversions.""" + + def test_quopri_encode(self) -> None: + """quopri encodes non-ASCII as =XX sequences.""" + data = "Héllo Wörld".encode("utf-8") + encoded = quopri.encodestring(data) + assert b"=C3" in encoded # UTF-8 for accented chars + + def test_quopri_decode(self) -> None: + """quopri decodes =XX sequences back to bytes.""" + encoded = b"Hello=20World" + decoded = quopri.decodestring(encoded) + assert decoded == b"Hello World" + + def test_binascii_hexlify(self) -> None: + """binascii converts between binary and hex.""" + data = b"\xde\xad\xbe\xef" + hex_str = binascii.hexlify(data) + assert hex_str == b"deadbeef" + assert binascii.unhexlify(hex_str) == data + + def test_binascii_crc32(self) -> None: + """binascii.crc32 computes CRC-32 checksums.""" + crc = binascii.crc32(b"hello") + assert isinstance(crc, int) + assert crc == binascii.crc32(b"hello") # deterministic diff --git a/tests/test_chapter_35.py b/tests/test_chapter_35.py new file mode 100644 index 0000000..f802281 --- /dev/null +++ b/tests/test_chapter_35.py @@ -0,0 +1,225 @@ +"""Tests for Chapter 35: Advanced Python Patterns.""" + +import copy +import weakref +from abc import ABC, abstractmethod + + +class TestDescriptorsDeepDive: + """Test descriptor protocol patterns.""" + + def test_data_descriptor(self) -> None: + """Data descriptors have __set__ and control attribute access.""" + + class Validated: + def __set_name__(self, owner: type, name: str) -> None: + self.name = name + + def __get__(self, obj: object, objtype: type | None = None) -> object: + if obj is None: + return self + return obj.__dict__.get(self.name) + + def __set__(self, obj: object, value: object) -> None: + if not isinstance(value, int) or value < 0: + raise ValueError(f"{self.name} must be a non-negative int") + obj.__dict__[self.name] = value + + class Order: + quantity = Validated() + + order = Order() + order.quantity = 10 + assert order.quantity == 10 + + try: + order.quantity = -1 + assert False, "Should have raised" + except ValueError: + pass + + def test_non_data_descriptor(self) -> None: + """Non-data descriptors only have __get__.""" + + class CachedProperty: + def __init__(self, func): + self.func = func + self.name = func.__name__ + + def __get__(self, obj: object, objtype: type | None = None): + if obj is None: + return self + value = self.func(obj) + obj.__dict__[self.name] = value # Cache in instance + return value + + class Circle: + def __init__(self, radius: float) -> None: + self.radius = radius + + @CachedProperty + def area(self) -> float: + return 3.14159 * self.radius**2 + + c = Circle(5.0) + assert abs(c.area - 78.53975) < 0.001 + assert "area" in c.__dict__ # Cached in instance + + def test_descriptor_set_name(self) -> None: + """__set_name__ is called when descriptor is assigned to class.""" + names: list[str] = [] + + class Tracker: + def __set_name__(self, owner: type, name: str) -> None: + names.append(name) + + def __get__(self, obj: object, objtype: type | None = None) -> object: + return None + + class MyClass: + x = Tracker() + y = Tracker() + + assert names == ["x", "y"] + + +class TestSlotsAndWeakrefs: + """Test __slots__ and weakref patterns.""" + + def test_slots_restricts_attributes(self) -> None: + """__slots__ restricts which attributes can be set.""" + + class Point: + __slots__ = ("x", "y") + + p = Point() + p.x = 1 + p.y = 2 + assert p.x == 1 + + try: + p.z = 3 # type: ignore[attr-defined] + assert False, "Should have raised" + except AttributeError: + pass + + def test_slots_no_dict(self) -> None: + """Slotted classes have no __dict__ by default.""" + + class Slotted: + __slots__ = ("value",) + + obj = Slotted() + assert not hasattr(obj, "__dict__") + + def test_weakref_basic(self) -> None: + """weakref.ref creates a weak reference.""" + + class MyObj: + pass + + obj = MyObj() + ref = weakref.ref(obj) + assert ref() is obj + del obj + assert ref() is None + + def test_weak_value_dictionary(self) -> None: + """WeakValueDictionary auto-removes garbage-collected values.""" + + class Item: + def __init__(self, name: str) -> None: + self.name = name + + cache: weakref.WeakValueDictionary = weakref.WeakValueDictionary() + item = Item("test") + cache["key"] = item + assert cache["key"].name == "test" + del item + assert "key" not in cache + + +class TestABCAndCopyProtocol: + """Test ABCs, virtual subclasses, and copy protocol.""" + + def test_abc_cannot_instantiate(self) -> None: + """ABCs with abstract methods can't be instantiated.""" + + class Shape(ABC): + @abstractmethod + def area(self) -> float: ... + + try: + Shape() # type: ignore[abstract] + assert False, "Should have raised" + except TypeError: + pass + + def test_abc_concrete_subclass(self) -> None: + """Concrete subclasses implement all abstract methods.""" + + class Shape(ABC): + @abstractmethod + def area(self) -> float: ... + + class Square(Shape): + def __init__(self, side: float) -> None: + self.side = side + + def area(self) -> float: + return self.side**2 + + s = Square(4.0) + assert s.area() == 16.0 + assert isinstance(s, Shape) + + def test_abc_register_virtual(self) -> None: + """register creates virtual subclasses without inheritance.""" + + class Drawable(ABC): + @abstractmethod + def draw(self) -> str: ... + + class Circle: + def draw(self) -> str: + return "O" + + Drawable.register(Circle) + assert isinstance(Circle(), Drawable) + assert issubclass(Circle, Drawable) + assert Circle.__mro__ == (Circle, object) # No Drawable in MRO + + def test_copy_shallow(self) -> None: + """copy.copy creates a shallow copy.""" + original = [[1, 2], [3, 4]] + shallow = copy.copy(original) + assert shallow == original + assert shallow is not original + assert shallow[0] is original[0] # Same inner list + + def test_copy_deep(self) -> None: + """copy.deepcopy creates a fully independent copy.""" + original = [[1, 2], [3, 4]] + deep = copy.deepcopy(original) + assert deep == original + assert deep is not original + assert deep[0] is not original[0] # Different inner list + + def test_custom_copy(self) -> None: + """__copy__ customizes shallow copy behavior.""" + + class Config: + def __init__(self, settings: dict) -> None: + self.settings = settings + self.copy_count = 0 + + def __copy__(self): + new = Config(self.settings.copy()) + new.copy_count = self.copy_count + 1 + return new + + c1 = Config({"debug": True}) + c2 = copy.copy(c1) + assert c2.settings == {"debug": True} + assert c2.copy_count == 1 + assert c2.settings is not c1.settings