From 5d8731a6612b67a88e354193c83139c9d66cb89a Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 09:08:45 -0400 Subject: [PATCH 1/5] ci: fork upstream auto-sync Keeps gburd/postgres rebased hourly on postgres/postgres master with only .github/ changes on top (sync-upstream automatic + manual). Drops the bespoke Windows dependency-builder workflow: Windows is already built and tested in CI by upstream's pg-ci.yml (Visual Studio + MinGW meson jobs), so a separate dependency prebuild that nothing consumed was redundant. --- .github/.gitignore | 18 + .github/QUICKSTART.md | 378 +++++++++++++++++++++ .github/README.md | 315 +++++++++++++++++ .github/docs/pristine-master-policy.md | 225 ++++++++++++ .github/docs/sync-setup.md | 326 ++++++++++++++++++ .github/workflows/sync-upstream-manual.yml | 249 ++++++++++++++ .github/workflows/sync-upstream.yml | 256 ++++++++++++++ 7 files changed, 1767 insertions(+) create mode 100644 .github/.gitignore create mode 100644 .github/QUICKSTART.md create mode 100644 .github/README.md create mode 100644 .github/docs/pristine-master-policy.md create mode 100644 .github/docs/sync-setup.md create mode 100644 .github/workflows/sync-upstream-manual.yml create mode 100644 .github/workflows/sync-upstream.yml diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 0000000000000..a447f99442861 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1,18 @@ +# Node modules +scripts/ai-review/node_modules/ +# Note: package-lock.json should be committed for reproducible CI/CD builds + +# Logs +scripts/ai-review/cost-log-*.json +scripts/ai-review/*.log + +# OS files +.DS_Store +Thumbs.db + +# Editor files +*.swp +*.swo +*~ +.vscode/ +.idea/ diff --git a/.github/QUICKSTART.md b/.github/QUICKSTART.md new file mode 100644 index 0000000000000..d22c4d562ab7d --- /dev/null +++ b/.github/QUICKSTART.md @@ -0,0 +1,378 @@ +# Quick Start Guide - PostgreSQL Mirror CI/CD + +**Goal:** Get your PostgreSQL mirror CI/CD system running in 15 minutes. + +--- + +## āœ… What's Been Implemented + +- **Phase 1: Automated Upstream Sync** - Daily sync from postgres/postgres āœ… +- **Phase 2: AI-Powered Code Review** - Claude-based PR reviews āœ… +- **Phase 3: Windows Builds** - Planned for weeks 4-6 šŸ“‹ + +--- + +## šŸš€ Setup Instructions + +### Step 1: Configure GitHub Actions Permissions (2 minutes) + +1. Go to: **Settings ā†’ Actions ā†’ General** +2. Scroll to: **Workflow permissions** +3. Select: **"Read and write permissions"** +4. Check: **"Allow GitHub Actions to create and approve pull requests"** +5. Click: **Save** + +āœ… This enables workflows to push commits and create issues. + +--- + +### Step 2: Set Up Upstream Sync (3 minutes) + +**Test manual sync first:** + +```bash +# Via GitHub Web UI: +# 1. Go to: Actions tab +# 2. Click: "Sync from Upstream (Manual)" +# 3. Click: "Run workflow" +# 4. Watch it run (should take ~2 minutes) + +# OR via GitHub CLI: +gh workflow run sync-upstream-manual.yml +gh run watch +``` + +**Verify sync worked:** + +```bash +git fetch origin +git log origin/master --oneline -5 + +# Compare with upstream: +# https://github.com/postgres/postgres/commits/master +``` + +**Enable automatic sync:** + +- Automatic sync runs daily at 00:00 UTC +- Already configured, no action needed +- Check: Actions ā†’ "Sync from Upstream (Automatic)" + +āœ… Your master branch will now stay synced automatically. + +--- + +### Step 3: Set Up AI Code Review (10 minutes) + +**Choose Your Provider:** + +You can use either **Anthropic API** (simpler) or **AWS Bedrock** (if you have AWS infrastructure). + +#### Option A: Anthropic API (Recommended for getting started) + +**A. Get Claude API Key:** + +1. Go to: https://console.anthropic.com/ +2. Sign up or log in +3. Navigate to: API Keys +4. Create new key +5. Copy the key (starts with `sk-ant-...`) + +**B. Add API Key to GitHub:** + +1. Go to: **Settings ā†’ Secrets and variables ā†’ Actions** +2. Click: **New repository secret** +3. Name: `ANTHROPIC_API_KEY` +4. Value: Paste your API key +5. Click: **Add secret** + +**C. Ensure config uses Anthropic:** + +Check `.github/scripts/ai-review/config.json` has: +```json +{ + "provider": "anthropic", + ... +} +``` + +#### Option B: AWS Bedrock (If you have AWS) + +See detailed guide: [.github/docs/bedrock-setup.md](.github/docs/bedrock-setup.md) + +**Quick steps:** +1. Enable Claude 3.5 Sonnet in AWS Bedrock console +2. Create IAM user with `bedrock:InvokeModel` permission +3. Add three secrets to GitHub: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_REGION` (e.g., `us-east-1`) +4. Update `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "bedrock", + "bedrock_model_id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock_region": "us-east-1", + ... +} +``` + +**Note:** Both providers have identical pricing ($0.003/1K input, $0.015/1K output tokens). + +--- + +**C. Install Dependencies:** + +```bash +cd .github/scripts/ai-review +npm install + +# Should install: +# - @anthropic-ai/sdk (for Anthropic API) +# - @aws-sdk/client-bedrock-runtime (for AWS Bedrock) +# - @actions/github +# - @actions/core +# - parse-diff +# - minimatch +``` + +**D. Test AI Review:** + +```bash +# Option 1: Create a test PR +git checkout -b test/ai-review +echo "// Test change" >> src/backend/utils/adt/int.c +git add . +git commit -m "Test: AI review" +git push origin test/ai-review +# Create PR via GitHub UI + +# Option 2: Manual trigger on existing PR +gh workflow run ai-code-review.yml -f pr_number= +``` + +āœ… AI will review the PR and post comments + summary. + +--- + +## šŸŽÆ Verify Everything Works + +### Check Sync Status + +```bash +# Check latest sync run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View details +gh run view $(gh run list --workflow=sync-upstream.yml --limit 1 --json databaseId -q '.[0].databaseId') +``` + +**Expected:** āœ… Green checkmark, "Already up to date" or "Successfully synced X commits" + +### Check AI Review Status + +```bash +# Check latest AI review run +gh run list --workflow=ai-code-review.yml --limit 1 + +# View details +gh run view $(gh run list --workflow=ai-code-review.yml --limit 1 --json databaseId -q '.[0].databaseId') +``` + +**Expected:** āœ… Green checkmark, comments posted on PR + +--- + +## šŸ“Š Monitor Costs + +### GitHub Actions Minutes + +```bash +# View usage (requires admin access) +gh api /repos/gburd/postgres/actions/cache/usage + +# Expected monthly usage: +# - Sync: ~150 minutes (FREE - within 2,000 min limit) +# - AI Review: ~200 minutes (FREE - within limit) +``` + +### Claude API Costs + +**View per-PR cost:** +- Check AI review summary comment on PR +- Format: `Cost: $X.XX | Model: claude-3-5-sonnet` + +**Expected costs:** +- Small PR: $0.50 - $1.00 +- Medium PR: $1.00 - $3.00 +- Large PR: $3.00 - $7.50 +- **Monthly (20 PRs):** $35-50 + +**Download detailed logs:** +```bash +gh run list --workflow=ai-code-review.yml --limit 5 +gh run download -n ai-review-cost-log- +``` + +--- + +## šŸ”§ Configuration + +### Adjust Sync Schedule + +Edit `.github/workflows/sync-upstream.yml`: + +```yaml +on: + schedule: + # Current: Daily at 00:00 UTC + - cron: '0 0 * * *' + + # Options: + # Every 6 hours: '0 */6 * * *' + # Twice daily: '0 0,12 * * *' + # Weekdays only: '0 0 * * 1-5' +``` + +### Adjust AI Review Costs + +Edit `.github/scripts/ai-review/config.json`: + +```json +{ + "cost_limits": { + "max_per_pr_dollars": 15.0, // ā† Lower this to save money + "max_per_month_dollars": 200.0, // ā† Hard monthly cap + "alert_threshold_dollars": 150.0 + }, + + "max_file_size_lines": 5000, // ā† Skip files larger than this + + "skip_paths": [ + "*.png", "*.svg", // Already skipped + "vendor/**/*", // ā† Add more patterns here + "generated/**/*" + ] +} +``` + +### Adjust AI Review Prompts + +**Make AI reviews stricter or more lenient:** + +Edit files in `.github/scripts/ai-review/prompts/`: +- `c-code.md` - PostgreSQL C code review +- `sql.md` - SQL and regression tests +- `documentation.md` - Documentation review +- `build-system.md` - Makefile/Meson review + +--- + +## šŸ› Troubleshooting + +### Sync Not Working + +**Problem:** Workflow fails with "Permission denied" + +**Fix:** +- Check: Settings ā†’ Actions ā†’ Workflow permissions +- Ensure: "Read and write permissions" is selected + +--- + +### AI Review Not Posting Comments + +**Problem:** Workflow runs but no comments appear + +**Check:** +1. Is PR a draft? (Draft PRs are skipped to save costs) +2. Are there reviewable files? (Check workflow logs) +3. Is API key valid? (Settings ā†’ Secrets ā†’ ANTHROPIC_API_KEY) + +**Fix:** +- Mark PR as "Ready for review" if draft +- Check workflow logs: Actions ā†’ Latest run ā†’ View logs +- Verify API key at https://console.anthropic.com/ + +--- + +### High AI Review Costs + +**Problem:** Costs higher than expected + +**Check:** +- Download cost logs: `gh run download ` +- Look for large files being reviewed +- Check number of PR updates (each triggers review) + +**Fix:** +1. Add large files to `skip_paths` in config.json +2. Lower `max_tokens_per_request` (shorter reviews) +3. Use draft PRs for work-in-progress +4. Batch PR updates to reduce review frequency + +--- + +## šŸ“š Full Documentation + +- **Overview:** [.github/README.md](.github/README.md) +- **Sync Guide:** [.github/docs/sync-setup.md](.github/docs/sync-setup.md) +- **AI Review Guide:** [.github/docs/ai-review-guide.md](.github/docs/ai-review-guide.md) +- **Windows Builds:** [.github/docs/windows-builds.md](.github/docs/windows-builds.md) (planned) +- **Implementation Status:** [.github/IMPLEMENTATION_STATUS.md](.github/IMPLEMENTATION_STATUS.md) + +--- + +## āœØ What's Next? + +### Immediate +- āœ… **Monitor first automatic sync** (tonight at 00:00 UTC) +- āœ… **Test AI review on real PR** +- āœ… **Tune prompts** based on feedback + +### This Week +- Shadow mode testing for AI reviews (Week 1) +- Gather developer feedback +- Adjust configuration + +### Weeks 2-3 +- Enable full AI review mode +- Monitor costs and quality +- Iterate on prompts + +### Weeks 4-6 +- **Phase 3:** Implement Windows dependency builds +- Research winpgbuild approach +- Create build workflows +- Test artifact publishing + +--- + +## šŸŽ‰ Success Criteria + +You'll know everything is working when: + +āœ… **Sync:** +- Master branch matches postgres/postgres +- Daily sync runs show green checkmarks +- No open issues with label `sync-failure` + +āœ… **AI Review:** +- PRs receive inline comments + summary +- Feedback is relevant and actionable +- Costs stay under $50/month +- Developers find reviews helpful + +āœ… **Overall:** +- Automation saves 8-16 hours/month +- Issues caught earlier in development +- No manual sync needed + +--- + +**Need Help?** +- Check documentation: `.github/README.md` +- Check workflow logs: Actions ā†’ Failed run ā†’ View logs +- Create issue with workflow URL and error messages + +**Ready to go!** šŸš€ diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000000..bdfcfe74ac4a4 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,315 @@ +# PostgreSQL Mirror CI/CD System + +This directory contains the CI/CD infrastructure for the PostgreSQL personal mirror repository. + +## System Overview + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ PostgreSQL Mirror CI/CD ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” + ā”‚ ā”‚ ā”‚ + [1] Sync [2] AI Review [3] Windows + Daily @ 00:00 On PR Events On Master Push + ā”‚ ā”‚ ā”‚ + ā–¼ ā–¼ ā–¼ + postgres/postgres Claude API Dependency Builds + ā”‚ ā”‚ ā”‚ + ā–¼ ā–¼ ā–¼ + github.com/gburd PR Comments Build Artifacts + /postgres/ + Labels (90-day retention) + master +``` + +## Components + +### 1. Automated Upstream Sync +**Status:** āœ“ Implemented +**Files:** `workflows/sync-upstream*.yml` + +Automatically syncs the `master` branch with upstream `postgres/postgres` daily. + +- **Frequency:** Daily at 00:00 UTC +- **Trigger:** Cron schedule + manual +- **Features:** + - Fast-forward merge (conflict-free) + - Automatic issue creation on conflicts + - Issue auto-closure on resolution +- **Cost:** Free (~150 min/month, well within free tier) + +**Documentation:** [docs/sync-setup.md](docs/sync-setup.md) + +### 2. AI-Powered Code Review +**Status:** āœ“ Implemented +**Files:** `workflows/ai-code-review.yml`, `scripts/ai-review/` + +Uses Claude API to provide PostgreSQL-aware code review on pull requests. + +- **Trigger:** PR opened/updated, ready for review +- **Features:** + - PostgreSQL-specific C code review + - SQL, documentation, build system review + - Inline comments on issues + - Automatic labeling (security, performance, etc.) + - Cost tracking and limits + - **Provider Options:** Anthropic API or AWS Bedrock +- **Cost:** $35-50/month (estimated) +- **Model:** Claude 3.5 Sonnet + +**Documentation:** [docs/ai-review-guide.md](docs/ai-review-guide.md) + +### 3. Windows Build Integration +**Status:** āœ… Implemented +**Files:** `workflows/windows-dependencies.yml`, `windows/`, `scripts/windows/` + +Builds PostgreSQL Windows dependencies for x64 Windows. + +- **Trigger:** Manual, manifest changes, weekly refresh +- **Features:** + - Core dependencies: OpenSSL, zlib, libxml2 + - Smart caching by version hash + - Dependency bundling + - Artifact publishing (90-day retention) + - PowerShell download helper + - **Cost optimization:** Skips builds for pristine commits (dev setup, .github/ only) +- **Cost:** ~$5-8/month (with caching and optimization) + +**Documentation:** [docs/windows-builds.md](docs/windows-builds.md) | [Usage](docs/windows-builds-usage.md) + +## Quick Start + +### Prerequisites + +1. **GitHub Actions enabled:** + - Settings ā†’ Actions ā†’ General ā†’ Allow all actions + +2. **Workflow permissions:** + - Settings ā†’ Actions ā†’ General ā†’ Workflow permissions + - Select: "Read and write permissions" + - Enable: "Allow GitHub Actions to create and approve pull requests" + +3. **Secrets configured:** + - **Option A - Anthropic API:** + - Settings ā†’ Secrets and variables ā†’ Actions + - Add: `ANTHROPIC_API_KEY` (get from https://console.anthropic.com/) + - **Option B - AWS Bedrock:** + - Add: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` + - See: [docs/bedrock-setup.md](docs/bedrock-setup.md) + +### Using the Sync System + +**Manual sync:** +```bash +# Via GitHub UI: +# Actions ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow + +# Via GitHub CLI: +gh workflow run sync-upstream-manual.yml +``` + +**Check sync status:** +```bash +# Latest sync run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View details +gh run view +``` + +### Using AI Code Review + +AI reviews run automatically on PRs. To test manually: + +```bash +# Via GitHub UI: +# Actions ā†’ "AI Code Review" ā†’ Run workflow ā†’ Enter PR number + +# Via GitHub CLI: +gh workflow run ai-code-review.yml -f pr_number=123 +``` + +**Reviewing AI feedback:** +1. AI posts inline comments on specific lines +2. AI posts summary comment with overview +3. AI adds labels (security-concern, needs-tests, etc.) +4. Review and address feedback like human reviewer comments + +### Cost Monitoring + +**View AI review costs:** +```bash +# Download cost logs +gh run download -n ai-review-cost-log- +``` + +**Expected monthly costs (with optimizations):** +- Sync: $0 (free tier) +- AI Review: $30-45 (only on PRs, skips drafts) +- Windows Builds: $5-8 (caching + pristine commit skipping) +- **Total: $35-53/month** + +**Cost optimizations:** +- Windows builds skip "dev setup" and .github/-only commits +- AI review only runs on non-draft PRs +- Aggressive caching reduces build times by 80-90% +- See [Cost Optimization Guide](docs/cost-optimization.md) for details + +## Workflow Files + +### Sync Workflows +- `workflows/sync-upstream.yml` - Automatic daily sync +- `workflows/sync-upstream-manual.yml` - Manual testing sync + +### AI Review Workflows +- `workflows/ai-code-review.yml` - Automatic PR review + +### Windows Build Workflows +- `workflows/windows-dependencies.yml` - Dependency builds (TBD) + +## Configuration Files + +### AI Review Configuration +- `scripts/ai-review/config.json` - Cost limits, file patterns, labels +- `scripts/ai-review/prompts/*.md` - Review prompts by file type +- `scripts/ai-review/package.json` - Node.js dependencies + +### Windows Build Configuration +- `windows/manifest.json` - Dependency versions (TBD) + +## Branch Strategy + +### Master Branch: Mirror Only +- **Purpose:** Pristine copy of `postgres/postgres` +- **Rule:** Never commit directly to master +- **Sync:** Automatic via GitHub Actions +- **Protection:** Consider branch protection rules + +### Feature Branches: Development +- **Pattern:** `feature/*`, `dev/*`, `experiment/*` +- **Workflow:** + ```bash + git checkout master + git pull origin master + git checkout -b feature/my-feature + # ... make changes ... + git push origin feature/my-feature + # Create PR: feature/my-feature ā†’ master + ``` + +### Special Branches +- `recovery/*` - Temporary branches for sync conflict resolution +- Development remotes: commitfest, heikki, orioledb, zheap + +## Integration with Cirrus CI + +GitHub Actions and Cirrus CI run independently: + +- **Cirrus CI:** Comprehensive testing (Linux, FreeBSD, macOS, Windows) +- **GitHub Actions:** Sync, AI review, Windows dependency builds +- **No conflicts:** Both can run on same commits + +## Troubleshooting + +### Sync Issues + +**Problem:** Sync workflow failing +**Check:** Actions ā†’ "Sync from Upstream (Automatic)" ā†’ Latest run +**Fix:** See [docs/sync-setup.md](docs/sync-setup.md#sync-failure-recovery) + +### AI Review Issues + +**Problem:** AI review not running +**Check:** Is PR a draft? Draft PRs are skipped +**Fix:** Mark PR as ready for review + +**Problem:** AI review too expensive +**Check:** Cost logs in workflow artifacts +**Fix:** Adjust limits in `scripts/ai-review/config.json` + +### Workflow Permission Issues + +**Problem:** "Resource not accessible by integration" +**Check:** Settings ā†’ Actions ā†’ General ā†’ Workflow permissions +**Fix:** Enable "Read and write permissions" + +## Security + +### Secrets Management +- `ANTHROPIC_API_KEY`: Claude API key (required for AI review) +- `GITHUB_TOKEN`: Auto-generated, scoped to repository +- Never commit secrets to repository +- Rotate API keys quarterly + +### Permissions +- Workflows use minimum necessary permissions +- `contents: read` for code access +- `pull-requests: write` for comments +- `issues: write` for sync failure issues + +### Audit Trail +- All workflow runs logged (90-day retention) +- Cost tracking for AI reviews +- GitHub Actions audit log available + +## Support and Documentation + +### Detailed Documentation +- [Sync Setup Guide](docs/sync-setup.md) - Upstream sync system +- [AI Review Guide](docs/ai-review-guide.md) - AI code review system +- [Windows Builds Guide](docs/windows-builds.md) - Windows dependencies +- [Cost Optimization Guide](docs/cost-optimization.md) - Reducing CI/CD costs +- [Pristine Master Policy](docs/pristine-master-policy.md) - Master branch management + +### Reporting Issues + +Issues with CI/CD system: +1. Check workflow logs: Actions ā†’ Failed run ā†’ View logs +2. Search existing issues: label:automation +3. Create issue with workflow run URL and error messages + +### Modifying Workflows + +**Disabling a workflow:** +```bash +# Via GitHub UI: +# Actions ā†’ Select workflow ā†’ "..." ā†’ Disable workflow + +# Via git: +git mv .github/workflows/workflow-name.yml .github/workflows/workflow-name.yml.disabled +git commit -m "Disable workflow" +``` + +**Testing workflow changes:** +1. Create feature branch +2. Modify workflow file +3. Use `workflow_dispatch` trigger to test +4. Verify in Actions tab +5. Merge to master when working + +## Cost Summary + +| Component | Monthly Cost | Usage | Notes | +|-----------|-------------|-------|-------| +| Sync | $0 | ~150 min | Free tier: 2,000 min | +| AI Review | $30-45 | Variable | Claude API usage-based | +| Windows Builds | $5-8 | ~2,500 min | With caching + optimization | +| **Total** | **$35-53** | | After cost optimizations | + +**Comparison:** CodeRabbit (turnkey solution) = $99-499/month + +**Cost savings:** ~40-47% reduction through optimizations (see [Cost Optimization Guide](docs/cost-optimization.md)) + +## References + +- PostgreSQL: https://github.com/postgres/postgres +- GitHub Actions: https://docs.github.com/en/actions +- Claude API: https://docs.anthropic.com/ +- Cirrus CI: https://cirrus-ci.org/ +- winpgbuild: https://github.com/dpage/winpgbuild + +--- + +**Last Updated:** 2026-03-10 +**Maintained by:** PostgreSQL Mirror Automation diff --git a/.github/docs/pristine-master-policy.md b/.github/docs/pristine-master-policy.md new file mode 100644 index 0000000000000..9c0479d32df6a --- /dev/null +++ b/.github/docs/pristine-master-policy.md @@ -0,0 +1,225 @@ +# Pristine Master Policy + +## Overview + +The `master` branch in this mirror repository follows a "mostly pristine" policy, meaning it should closely mirror the upstream `postgres/postgres` repository with only specific exceptions allowed. + +## Allowed Commits on Master + +Master is considered "pristine" and the sync workflow will successfully merge upstream changes if local commits fall into these categories: + +### 1. āœ… CI/CD Configuration (`.github/` directory only) + +Commits that only modify files within the `.github/` directory are allowed. + +**Examples:** +- Adding GitHub Actions workflows +- Updating AI review configuration +- Modifying sync schedules +- Adding documentation in `.github/docs/` + +**Rationale:** CI/CD configuration is repository-specific and doesn't affect the PostgreSQL codebase itself. + +### 2. āœ… Development Environment Setup (commits named "dev setup ...") + +Commits with messages starting with "dev setup" (case-insensitive) are allowed, even if they modify files outside `.github/`. + +**Examples:** +- `dev setup v19` +- `Dev Setup: Add debugging configuration` +- `DEV SETUP - IDE and tooling` + +**Typical files in dev setup commits:** +- `.clang-format`, `.clangd` - Code formatting and LSP config +- `.envrc` - Directory environment variables (direnv) +- `.gdbinit` - Debugger configuration +- `.idea/`, `.vscode/` - IDE settings +- `flake.nix`, `shell.nix` - Nix development environment +- `pg-aliases.sh` - Personal shell aliases +- Other personal development tools + +**Rationale:** Development environment configuration is personal and doesn't affect the code or CI/CD. It's frequently updated as developers refine their workflow. + +### 3. āŒ Code Changes (NOT allowed) + +Any commits that: +- Modify PostgreSQL source code (`src/`, `contrib/`, etc.) +- Modify tests outside `.github/` +- Modify build system outside `.github/` +- Are not `.github/`-only AND don't start with "dev setup" + +**These will cause sync failures** and require manual resolution. + +## Branch Strategy + +### Master Branch +- **Purpose:** Mirror of upstream `postgres/postgres` + local CI/CD + dev environment +- **Updates:** Automatic hourly sync from upstream +- **Direct commits:** Only `.github/` changes or "dev setup" commits +- **All other work:** Use feature branches + +### Feature Branches +- **Purpose:** All PostgreSQL development work +- **Pattern:** `feature/*`, `dev/*`, `experiment/*` +- **Workflow:** + ```bash + git checkout master + git pull origin master + git checkout -b feature/my-feature + # Make changes... + git push origin feature/my-feature + # Create PR: feature/my-feature ā†’ master + ``` + +## Sync Workflow Behavior + +### Scenario 1: No Local Commits +``` +Upstream: A---B---C +Master: A---B---C +``` +**Result:** āœ… Already up to date (no action needed) + +### Scenario 2: Only .github/ Commits +``` +Upstream: A---B---C---D +Master: A---B---C---X (X modifies .github/ only) +``` +**Result:** āœ… Merge commit created +``` +Master: A---B---C---X---M + \ / + D---/ +``` + +### Scenario 3: Only "dev setup" Commits +``` +Upstream: A---B---C---D +Master: A---B---C---Y (Y is "dev setup v19") +``` +**Result:** āœ… Merge commit created +``` +Master: A---B---C---Y---M + \ / + D---/ +``` + +### Scenario 4: Mix of Allowed Commits +``` +Upstream: A---B---C---D +Master: A---B---C---X---Y (X=.github/, Y=dev setup) +``` +**Result:** āœ… Merge commit created + +### Scenario 5: Code Changes (Violation) +``` +Upstream: A---B---C---D +Master: A---B---C---Z (Z modifies src/backend/) +``` +**Result:** āŒ Sync fails, issue created + +**Recovery:** +1. Create feature branch from Z +2. Reset master to match upstream +3. Rebase feature branch +4. Create PR + +## Updating Dev Setup + +When you update your development environment: + +```bash +# Make changes to .clangd, flake.nix, etc. +git add .clangd flake.nix .vscode/ + +# Important: Start message with "dev setup" +git commit -m "dev setup v20: Update clangd config and add new aliases" + +git push origin master +``` + +The sync workflow will recognize this as a dev setup commit and preserve it during merges. + +**Naming convention:** +- āœ… `dev setup v20` +- āœ… `Dev setup: Update IDE config` +- āœ… `DEV SETUP - Add debugging tools` +- āŒ `Update development environment` (doesn't start with "dev setup") +- āŒ `dev environment changes` (doesn't start with "dev setup") + +## Sync Failure Recovery + +If sync fails because of non-allowed commits: + +### Check What's Wrong +```bash +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master + +# See which commits are problematic +git log upstream/master..origin/master --oneline + +# See which files were changed +git diff --name-only upstream/master...origin/master +``` + +### Option 1: Make Commit Acceptable + +If the commit should have been a "dev setup" commit: + +```bash +# Amend the commit message +git commit --amend -m "dev setup v21: Previous changes" +git push origin master --force-with-lease +``` + +### Option 2: Move to Feature Branch + +If the commit contains code changes: + +```bash +# Create feature branch +git checkout -b feature/recovery origin/master + +# Reset master to upstream +git checkout master +git reset --hard upstream/master +git push origin master --force + +# Your changes are safe in feature/recovery +git checkout feature/recovery +# Create PR when ready +``` + +## FAQ + +**Q: Why allow dev setup commits on master?** +A: Development environment configuration is personal, frequently updated, and doesn't affect the codebase or CI/CD. It's more convenient to keep it on master than manage separate branches. + +**Q: What if I forget to name it "dev setup"?** +A: Sync will fail. You can amend the commit message (see recovery above) or move the commit to a feature branch. + +**Q: Can I have both .github/ and dev setup changes in one commit?** +A: Yes! The sync workflow allows commits that modify .github/, or are named "dev setup", or both. + +**Q: What if upstream modifies the same files as my dev setup commit?** +A: The sync will attempt to merge automatically. If there are conflicts, you'll need to resolve them manually (rare, since upstream shouldn't touch personal dev files). + +**Q: Can I reorder commits on master?** +A: It's not recommended due to complexity. The sync workflow handles commits in any order as long as they follow the policy. + +## Monitoring + +**Check sync status:** +- Actions ā†’ "Sync from Upstream (Automatic)" +- Look for green āœ… on recent runs + +**Check for policy violations:** +- Open issues with label `sync-failure` +- These indicate commits that violated the pristine master policy + +## Related Documentation + +- [Sync Setup Guide](sync-setup.md) - Detailed sync workflow documentation +- [QUICKSTART](../QUICKSTART.md) - Quick setup guide +- [README](../README.md) - System overview diff --git a/.github/docs/sync-setup.md b/.github/docs/sync-setup.md new file mode 100644 index 0000000000000..1e12aeea3c5fc --- /dev/null +++ b/.github/docs/sync-setup.md @@ -0,0 +1,326 @@ +# Automated Upstream Sync Documentation + +## Overview + +This repository maintains a mirror of the official PostgreSQL repository at `postgres/postgres`. The sync system automatically keeps the `master` branch synchronized with upstream changes. + +## System Components + +### 1. Automatic Daily Sync +**File:** `.github/workflows/sync-upstream.yml` + +- **Trigger:** Daily at 00:00 UTC (cron schedule) +- **Purpose:** Automatically sync master branch without manual intervention +- **Process:** + 1. Fetches latest commits from `postgres/postgres` + 2. Fast-forward merges to local master (conflict-free) + 3. Pushes to `origin/master` + 4. Creates GitHub issue if conflicts detected + 5. Closes existing sync-failure issues on success + +### 2. Manual Sync Workflow +**File:** `.github/workflows/sync-upstream-manual.yml` + +- **Trigger:** Manual via Actions tab ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow +- **Purpose:** Testing and on-demand syncs +- **Options:** + - `force_push`: Use `--force-with-lease` when pushing (default: true) + +## Branch Strategy + +### Critical Rule: Master is Pristine + +- **master branch:** Mirror only - pristine copy of `postgres/postgres` +- **All development:** Feature branches (e.g., `feature/hot-updates`, `experiment/zheap`) +- **Never commit directly to master** - this will cause sync failures + +### Feature Branch Workflow + +```bash +# Start new feature from latest master +git checkout master +git pull origin master +git checkout -b feature/my-feature + +# Work on feature +git commit -m "Add feature" + +# Keep feature updated with upstream +git checkout master +git pull origin master +git checkout feature/my-feature +git rebase master + +# Push feature branch +git push origin feature/my-feature + +# Create PR: feature/my-feature ā†’ master +``` + +## Sync Failure Recovery + +### Diagnosis + +If sync fails, you'll receive a GitHub issue with label `sync-failure`. Check what commits are on master but not upstream: + +```bash +# Clone or update your local repository +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master + +# View conflicting commits +git log upstream/master..origin/master --oneline + +# See detailed changes +git diff upstream/master...origin/master +``` + +### Recovery Option 1: Preserve Commits (Recommended) + +If the commits on master should be kept: + +```bash +# Create backup branch from current master +git checkout origin/master +git checkout -b recovery/master-backup-$(date +%Y%m%d) +git push origin recovery/master-backup-$(date +%Y%m%d) + +# Reset master to upstream +git checkout master +git reset --hard upstream/master +git push origin master --force + +# Create feature branch from backup +git checkout -b feature/recovered-work recovery/master-backup-$(date +%Y%m%d) + +# Optional: rebase onto new master +git rebase master + +# Push feature branch +git push origin feature/recovered-work + +# Create PR: feature/recovered-work ā†’ master +``` + +### Recovery Option 2: Discard Commits + +If the commits on master were mistakes or already merged upstream: + +```bash +git checkout master +git reset --hard upstream/master +git push origin master --force +``` + +### Verification + +After recovery, verify sync status: + +```bash +# Check that master matches upstream +git log origin/master --oneline -10 +git log upstream/master --oneline -10 + +# These should be identical + +# Or run manual sync workflow +# GitHub ā†’ Actions ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow +``` + +The automatic sync will resume on next scheduled run (00:00 UTC daily). + +## Monitoring + +### Success Indicators + +- āœ“ GitHub Actions badge shows passing +- āœ“ No open issues with label `sync-failure` +- āœ“ `master` branch commit history matches `postgres/postgres` + +### Check Sync Status + +**Via GitHub UI:** +1. Go to: Actions ā†’ "Sync from Upstream (Automatic)" +2. Check latest run status + +**Via Git:** +```bash +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master +git log origin/master..upstream/master --oneline + +# No output = fully synced +# Commits listed = behind upstream (sync pending or failed) +``` + +**Via API:** +```bash +# Check latest workflow run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View run details +gh run view +``` + +### Sync Lag + +Expected lag: <1 hour from upstream commit to mirror + +- Upstream commits at 12:30 UTC ā†’ Synced at next daily run (00:00 UTC next day) = ~11.5 hours max +- For faster sync: Manually trigger workflow after major upstream merges + +## Configuration + +### GitHub Actions Permissions + +Required settings (already configured): + +1. **Settings ā†’ Actions ā†’ General ā†’ Workflow permissions:** + - āœ“ "Read and write permissions" + - āœ“ "Allow GitHub Actions to create and approve pull requests" + +2. **Repository Settings ā†’ Branches:** + - Consider: Branch protection rule on `master` to prevent direct pushes + - Exception: Allow `github-actions[bot]` to push + +### Adjusting Sync Schedule + +Edit `.github/workflows/sync-upstream.yml`: + +```yaml +on: + schedule: + # Current: Daily at 00:00 UTC + - cron: '0 0 * * *' + + # Examples: + # Every 6 hours: '0 */6 * * *' + # Twice daily: '0 0,12 * * *' + # Weekdays only: '0 0 * * 1-5' +``` + +**Recommendation:** Keep daily schedule to balance freshness with API usage. + +## Troubleshooting + +### Issue: Workflow not running + +**Check:** +1. Actions tab ā†’ Check if workflow is disabled +2. Settings ā†’ Actions ā†’ Ensure workflows are enabled for repository + +**Fix:** +- Enable workflow: Actions ā†’ Select workflow ā†’ "Enable workflow" + +### Issue: Permission denied on push + +**Check:** +- Settings ā†’ Actions ā†’ General ā†’ Workflow permissions + +**Fix:** +- Set to "Read and write permissions" +- Enable "Allow GitHub Actions to create and approve pull requests" + +### Issue: Merge conflicts every sync + +**Root cause:** Commits being made directly to master + +**Fix:** +1. Review `.git/hooks/` for pre-commit hooks that might auto-commit +2. Check if any automation is committing to master +3. Enforce branch protection rules +4. Educate team members on feature branch workflow + +### Issue: Sync successful but CI fails + +**This is expected** if upstream introduced breaking changes or test failures. + +**Handling:** +- Upstream tests failures are upstream's responsibility +- Focus: Ensure mirror stays in sync +- Separate: Your feature branches should pass CI + +## Cost and Usage + +### GitHub Actions Minutes + +- **Sync workflow:** ~2-3 minutes per run +- **Frequency:** Daily = 60-90 minutes/month +- **Free tier:** 2,000 minutes/month (public repos: unlimited) +- **Cost:** $0 (well within limits) + +### Network Usage + +- Fetches only new commits (incremental) +- Typical: <10 MB per sync +- Total: <300 MB/month + +## Security Considerations + +### Secrets + +- Uses `GITHUB_TOKEN` (automatically provided, scoped to repository) +- No additional secrets required +- Token permissions: Minimum necessary (contents:write, issues:write) + +### Audit Trail + +All syncs are logged: +- GitHub Actions run history (90 days retention) +- Git reflog on server +- Issue creation/closure for failures + +## Integration with Other Workflows + +### Cirrus CI + +Cirrus CI tests trigger on pushes to master: +- Sync pushes ā†’ Cirrus CI runs tests on synced commits +- This validates upstream changes against your test matrix + +### AI Code Review + +AI review workflows trigger on PRs, not master pushes: +- Sync to master does NOT trigger AI reviews +- Feature branch PRs ā†’ master do trigger AI reviews + +### Windows Builds + +Windows dependency builds trigger on master pushes: +- Sync pushes ā†’ Windows builds run +- Ensures dependencies stay compatible with latest upstream + +## Support + +### Reporting Issues + +If sync consistently fails: + +1. Check open issues with label `sync-failure` +2. Review workflow logs: Actions ā†’ Failed run ā†’ View logs +3. Create issue with: + - Workflow run URL + - Error messages from logs + - Output of `git log upstream/master..origin/master` + +### Disabling Automatic Sync + +If needed (e.g., during major refactoring): + +```bash +# Disable via GitHub UI +# Actions ā†’ "Sync from Upstream (Automatic)" ā†’ "..." ā†’ Disable workflow + +# Or delete/rename the workflow file +git mv .github/workflows/sync-upstream.yml .github/workflows/sync-upstream.yml.disabled +git commit -m "Temporarily disable automatic sync" +git push +``` + +**Remember to re-enable** once work is complete. + +## References + +- Upstream repository: https://github.com/postgres/postgres +- GitHub Actions docs: https://docs.github.com/en/actions +- Git branching strategies: https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows diff --git a/.github/workflows/sync-upstream-manual.yml b/.github/workflows/sync-upstream-manual.yml new file mode 100644 index 0000000000000..362c119a128e7 --- /dev/null +++ b/.github/workflows/sync-upstream-manual.yml @@ -0,0 +1,249 @@ +name: Sync from Upstream (Manual) + +on: + workflow_dispatch: + inputs: + force_push: + description: 'Use --force-with-lease when pushing' + required: false + type: boolean + default: true + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/postgres/postgres.git || true + git remote -v + + - name: Fetch upstream + run: | + echo "Fetching from upstream postgres/postgres..." + git fetch upstream master + echo "Current local master:" + git log origin/master --oneline -5 + echo "Upstream master:" + git log upstream/master --oneline -5 + + - name: Check for local commits + id: check_commits + run: | + git checkout master + LOCAL_COMMITS=$(git rev-list origin/master..upstream/master --count) + DIVERGED=$(git rev-list upstream/master..origin/master --count) + echo "commits_behind=$LOCAL_COMMITS" >> $GITHUB_OUTPUT + echo "commits_ahead=$DIVERGED" >> $GITHUB_OUTPUT + echo "Mirror is $DIVERGED commits ahead and $LOCAL_COMMITS commits behind upstream" + + if [ "$DIVERGED" -gt 0 ]; then + # Check commit messages for "dev setup" or "dev v" pattern + DEV_SETUP_COMMITS=$(git log --format=%s upstream/master...origin/master | grep -iE "^dev (setup|v[0-9])" | wc -l) + echo "dev_setup_commits=$DEV_SETUP_COMMITS" >> $GITHUB_OUTPUT + + # Check if diverged commits only touch .github/ directory + NON_GITHUB_CHANGES=$(git diff --name-only upstream/master...origin/master | grep -v "^\.github/" | wc -l) + echo "non_github_changes=$NON_GITHUB_CHANGES" >> $GITHUB_OUTPUT + + if [ "$NON_GITHUB_CHANGES" -eq 0 ]; then + echo "āœ“ All local commits are CI/CD configuration (.github/ only)" + elif [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "āœ“ Found $DEV_SETUP_COMMITS 'dev setup/version' commit(s)" + else + echo "āš ļø WARNING: Local commits modify files outside .github/ and are not 'dev setup/version' commits!" + git diff --name-only upstream/master...origin/master | grep -v "^\.github/" || true + fi + else + echo "non_github_changes=0" >> $GITHUB_OUTPUT + echo "dev_setup_commits=0" >> $GITHUB_OUTPUT + fi + + - name: Attempt merge + id: merge + run: | + COMMITS_AHEAD=${{ steps.check_commits.outputs.commits_ahead }} + COMMITS_BEHIND=${{ steps.check_commits.outputs.commits_behind }} + NON_GITHUB_CHANGES=${{ steps.check_commits.outputs.non_github_changes }} + DEV_SETUP_COMMITS=${{ steps.check_commits.outputs.dev_setup_commits }} + + # Check if there are problematic local commits + # Allow commits if: + # 1. Only .github/ changes (CI/CD config) + # 2. Has "dev setup/version" commits (personal development environment) + if [ "$COMMITS_AHEAD" -gt 0 ] && [ "$NON_GITHUB_CHANGES" -gt 0 ]; then + if [ "$DEV_SETUP_COMMITS" -eq 0 ]; then + echo "āŒ Local master has commits outside .github/ that are not 'dev setup/version' commits!" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + exit 1 + else + echo "āœ“ Non-.github/ changes are from 'dev setup/version' commits - allowed" + fi + fi + + # Already up to date + if [ "$COMMITS_BEHIND" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + echo "merge_status=uptodate" >> $GITHUB_OUTPUT + exit 0 + fi + + # Try fast-forward first (clean case) + if [ "$COMMITS_AHEAD" -eq 0 ]; then + echo "Fast-forwarding to upstream (no local commits)..." + git merge --ff-only upstream/master + echo "merge_status=success" >> $GITHUB_OUTPUT + exit 0 + fi + + # Local commits exist (.github/ and/or dev setup/version) - rebase onto upstream + if [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "Rebasing local CI/CD and dev setup/version commits onto upstream..." + else + echo "Rebasing local CI/CD commits (.github/ only) onto upstream..." + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if git rebase upstream/master; then + echo "āœ“ Successfully rebased local commits onto upstream" + echo "merge_status=success" >> $GITHUB_OUTPUT + else + echo "āŒ Rebase conflict occurred" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + + # Abort the failed rebase to clean up state + git rebase --abort + exit 1 + fi + continue-on-error: true + + - name: Push to origin + if: steps.merge.outputs.merge_status == 'success' + run: | + if [ "${{ inputs.force_push }}" == "true" ]; then + git push origin master --force-with-lease + else + git push origin master + fi + echo "āœ“ Successfully synced master with upstream" + + - name: Create issue on failure + if: steps.merge.outputs.merge_status == 'conflict' + uses: actions/github-script@v7 + with: + script: | + const title = 'šŸšØ Upstream Sync Failed - Manual Intervention Required'; + const body = `## Sync Failure Report + + The automated sync from \`postgres/postgres\` failed due to conflicting commits. + + **Details:** + - Local master has ${{ steps.check_commits.outputs.commits_ahead }} commit(s) not in upstream + - Upstream has ${{ steps.check_commits.outputs.commits_behind }} new commit(s) + - Non-.github/ changes: ${{ steps.check_commits.outputs.non_github_changes }} files + + **This indicates commits were made directly to master outside .github/**, which violates the pristine mirror policy. + + **Note:** Commits to .github/ (CI/CD configuration) are allowed and will be preserved during sync. + + ### Resolution Steps: + + 1. Identify the conflicting commits: + \`\`\`bash + git fetch origin + git fetch upstream https://github.com/postgres/postgres.git master + git log upstream/master..origin/master + \`\`\` + + 2. If these commits should be preserved: + - Create a feature branch: \`git checkout -b recovery/master-commits origin/master\` + - Reset master: \`git checkout master && git reset --hard upstream/master\` + - Push: \`git push origin master --force\` + - Cherry-pick or rebase the feature branch + + 3. If these commits should be discarded: + - Reset master: \`git checkout master && git reset --hard upstream/master\` + - Push: \`git push origin master --force\` + + 4. Close this issue once resolved + + **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + `; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + if (issues.data.length === 0) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['sync-failure', 'automation'] + }); + } + + - name: Close existing sync-failure issues + if: steps.merge.outputs.merge_status == 'success' + uses: actions/github-script@v7 + with: + script: | + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + for (const issue of issues.data) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: 'āœ“ Sync successful - closing this issue automatically.' + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + + - name: Summary + if: always() + run: | + echo "### Sync Summary" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** ${{ steps.merge.outputs.merge_status }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits behind:** ${{ steps.check_commits.outputs.commits_behind }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits ahead:** ${{ steps.check_commits.outputs.commits_ahead }}" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.merge.outputs.merge_status }}" == "success" ]; then + echo "- **Result:** āœ“ Successfully synced with upstream" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.merge.outputs.merge_status }}" == "uptodate" ]; then + echo "- **Result:** āœ“ Already up to date" >> $GITHUB_STEP_SUMMARY + else + echo "- **Result:** āš ļø Sync failed - manual intervention required" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 0000000000000..b3a6466980b0d --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,256 @@ +name: Sync from Upstream (Automatic) + +on: + schedule: + # Run hourly every day + - cron: '0 * * * *' + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/postgres/postgres.git || true + git remote -v + + - name: Fetch upstream + run: | + echo "Fetching from upstream postgres/postgres..." + git fetch upstream master + + - name: Check for local commits + id: check_commits + run: | + git checkout master + LOCAL_COMMITS=$(git rev-list origin/master..upstream/master --count) + DIVERGED=$(git rev-list upstream/master..origin/master --count) + echo "commits_behind=$LOCAL_COMMITS" >> $GITHUB_OUTPUT + echo "commits_ahead=$DIVERGED" >> $GITHUB_OUTPUT + + if [ "$LOCAL_COMMITS" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + else + echo "Mirror is $LOCAL_COMMITS commits behind upstream" + fi + + if [ "$DIVERGED" -gt 0 ]; then + echo "āš ļø Local master has $DIVERGED commits not in upstream" + + # Check commit messages for "dev setup" or "dev v" pattern + DEV_SETUP_COMMITS=$(git log --format=%s upstream/master..origin/master | grep -iE "^dev (setup|v[0-9])" | wc -l) + echo "dev_setup_commits=$DEV_SETUP_COMMITS" >> $GITHUB_OUTPUT + + # Check if diverged commits only touch .github/ directory + NON_GITHUB_CHANGES=$(git diff --name-only upstream/master...origin/master | grep -v "^\.github/" | wc -l) + echo "non_github_changes=$NON_GITHUB_CHANGES" >> $GITHUB_OUTPUT + + if [ "$NON_GITHUB_CHANGES" -eq 0 ]; then + echo "āœ“ All local commits are CI/CD configuration (.github/ only) - will merge" + elif [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "āœ“ Found $DEV_SETUP_COMMITS 'dev setup/version' commit(s)" + else + echo "āš ļø WARNING: Local commits modify files outside .github/ and are not 'dev setup/version' commits!" + git diff --name-only upstream/master...origin/master | grep -v "^\.github/" || true + echo "Non-dev commits:" + git log --format=" %h %s" upstream/master..origin/master | grep -ivE "^ [a-f0-9]* dev (setup|v[0-9])" || true + fi + else + echo "non_github_changes=0" >> $GITHUB_OUTPUT + echo "dev_setup_commits=0" >> $GITHUB_OUTPUT + fi + + - name: Attempt merge + id: merge + run: | + COMMITS_AHEAD=${{ steps.check_commits.outputs.commits_ahead }} + COMMITS_BEHIND=${{ steps.check_commits.outputs.commits_behind }} + NON_GITHUB_CHANGES=${{ steps.check_commits.outputs.non_github_changes }} + DEV_SETUP_COMMITS=${{ steps.check_commits.outputs.dev_setup_commits }} + + # Check if there are problematic local commits + # Allow commits if: + # 1. Only .github/ changes (CI/CD config) + # 2. Has "dev setup/version" commits (personal development environment) + if [ "$COMMITS_AHEAD" -gt 0 ] && [ "$NON_GITHUB_CHANGES" -gt 0 ]; then + if [ "$DEV_SETUP_COMMITS" -eq 0 ]; then + echo "āŒ Local master has commits outside .github/ that are not 'dev setup/version' commits!" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + exit 1 + else + echo "āœ“ Non-.github/ changes are from 'dev setup/version' commits - allowed" + fi + fi + + # Already up to date + if [ "$COMMITS_BEHIND" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + echo "merge_status=uptodate" >> $GITHUB_OUTPUT + exit 0 + fi + + # Try fast-forward first (clean case) + if [ "$COMMITS_AHEAD" -eq 0 ]; then + echo "Fast-forwarding to upstream (no local commits)..." + git merge --ff-only upstream/master + echo "merge_status=success" >> $GITHUB_OUTPUT + exit 0 + fi + + # Local commits exist (.github/ and/or dev setup/version) - rebase onto upstream + if [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "Rebasing local CI/CD and dev setup/version commits onto upstream..." + else + echo "Rebasing local CI/CD commits (.github/ only) onto upstream..." + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if git rebase upstream/master; then + echo "āœ“ Successfully rebased local commits onto upstream" + echo "merge_status=success" >> $GITHUB_OUTPUT + else + echo "āŒ Rebase conflict occurred" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + + # Abort the failed rebase to clean up state + git rebase --abort + exit 1 + fi + continue-on-error: true + + - name: Push to origin + if: steps.merge.outputs.merge_status == 'success' + run: | + git push origin master --force-with-lease + + COMMITS_SYNCED="${{ steps.check_commits.outputs.commits_behind }}" + echo "āœ“ Successfully synced $COMMITS_SYNCED commits from upstream" + + - name: Create issue on failure + if: steps.merge.outputs.merge_status == 'conflict' + uses: actions/github-script@v7 + with: + script: | + const title = 'šŸšØ Automated Upstream Sync Failed'; + const body = `## Automatic Sync Failure + + The daily sync from \`postgres/postgres\` failed. + + **Details:** + - Local master has ${{ steps.check_commits.outputs.commits_ahead }} commit(s) not in upstream + - Upstream has ${{ steps.check_commits.outputs.commits_behind }} new commit(s) + - Non-.github/ changes: ${{ steps.check_commits.outputs.non_github_changes }} files + - **Run date:** ${new Date().toISOString()} + + **Root cause:** Commits were made directly to master outside of .github/, which violates the pristine mirror policy. + + **Note:** Commits to .github/ (CI/CD configuration) are allowed and will be preserved during sync. + + ### Resolution Steps: + + 1. Review the conflicting commits: + \`\`\`bash + git log upstream/master..origin/master --oneline + \`\`\` + + 2. Determine if commits should be: + - **Preserved:** Create feature branch and reset master + - **Discarded:** Hard reset master to upstream + + 3. See [sync documentation](.github/docs/sync-setup.md) for detailed recovery procedures + + 4. Run manual sync workflow after resolution to verify + + **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + `; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + if (issues.data.length === 0) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['sync-failure', 'automation', 'urgent'] + }); + } else { + // Update existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issues.data[0].number, + body: `Sync failed again on ${new Date().toISOString()}\n\nWorkflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}` + }); + } + + - name: Close sync-failure issues + if: steps.merge.outputs.merge_status == 'success' + uses: actions/github-script@v7 + with: + script: | + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + for (const issue of issues.data) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: `āœ“ Automatic sync successful on ${new Date().toISOString()} - synced ${{ steps.check_commits.outputs.commits_behind }} commits.\n\nClosing issue automatically.` + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + + - name: Summary + if: always() + run: | + echo "### Daily Sync Summary" >> $GITHUB_STEP_SUMMARY + echo "- **Date:** $(date -u)" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** ${{ steps.merge.outputs.merge_status }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits synced:** ${{ steps.check_commits.outputs.commits_behind }}" >> $GITHUB_STEP_SUMMARY + + if [ "${{ steps.merge.outputs.merge_status }}" == "success" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ“ Mirror successfully updated with upstream postgres/postgres" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.merge.outputs.merge_status }}" == "uptodate" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ“ Mirror already up to date" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "āš ļø Sync failed - check created issue for details" >> $GITHUB_STEP_SUMMARY + fi From d7cd5d6b25ebbabe7384bf5a0be958fe9ed24994 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 09:08:46 -0400 Subject: [PATCH 2/5] ci: AI/LLM PR review (OCR via Bedrock + Agora MCP history) The Open Code Review system: ocr-review and ocr-model-check workflows plus .github/ocr config (LiteLLM->Bedrock Claude Opus 4.8, rule.json, context.md, pg-history.py). --- .github/docs/ai-review-guide.md | 512 ++++++++++++++++++++++++++ .github/docs/bedrock-setup.md | 298 +++++++++++++++ .github/docs/cost-optimization.md | 219 +++++++++++ .github/ocr/context.md | 126 +++++++ .github/ocr/litellm.yaml | 41 +++ .github/ocr/pg-history.py | 225 +++++++++++ .github/ocr/rule.json | 65 ++++ .github/workflows/ocr-model-check.yml | 89 +++++ .github/workflows/ocr-review.yml | 427 +++++++++++++++++++++ 9 files changed, 2002 insertions(+) create mode 100644 .github/docs/ai-review-guide.md create mode 100644 .github/docs/bedrock-setup.md create mode 100644 .github/docs/cost-optimization.md create mode 100644 .github/ocr/context.md create mode 100644 .github/ocr/litellm.yaml create mode 100644 .github/ocr/pg-history.py create mode 100644 .github/ocr/rule.json create mode 100644 .github/workflows/ocr-model-check.yml create mode 100644 .github/workflows/ocr-review.yml diff --git a/.github/docs/ai-review-guide.md b/.github/docs/ai-review-guide.md new file mode 100644 index 0000000000000..eff0ed10cba4f --- /dev/null +++ b/.github/docs/ai-review-guide.md @@ -0,0 +1,512 @@ +# AI-Powered Code Review Guide + +## Overview + +This system uses Claude AI (Anthropic) to provide PostgreSQL-aware code reviews on pull requests. Reviews are similar in style to feedback from the PostgreSQL Hackers mailing list. + +## How It Works + +``` +PR Event (opened/updated) + ā†“ +GitHub Actions Workflow Starts + ā†“ +Fetch PR diff + metadata + ā†“ +Filter reviewable files (.c, .h, .sql, docs, Makefiles) + ā†“ +Route each file to appropriate review prompt + ā†“ +Send to Claude API with PostgreSQL context + ā†“ +Parse response for issues + ā†“ +Post inline comments + summary to PR + ā†“ +Add labels (security-concern, performance, etc.) +``` + +## Features + +### PostgreSQL-Specific Reviews + +**C Code Review:** +- Memory management (palloc/pfree, memory contexts) +- Concurrency (lock ordering, race conditions) +- Error handling (elog/ereport patterns) +- Performance (algorithm complexity, cache efficiency) +- Security (buffer overflows, SQL injection vectors) +- PostgreSQL conventions (naming, comments, style) + +**SQL Review:** +- PostgreSQL SQL dialect correctness +- Regression test patterns +- Performance (index usage, join strategy) +- Deterministic output for tests +- Edge case coverage + +**Documentation Review:** +- Technical accuracy +- SGML/DocBook format +- PostgreSQL style guide compliance +- Examples and cross-references + +**Build System Review:** +- Makefile correctness (GNU Make, PGXS) +- Meson build consistency +- Cross-platform portability +- VPATH build support + +### Automatic Labeling + +Reviews automatically add labels based on findings: + +- `security-concern` - Security issues, vulnerabilities +- `performance-concern` - Performance problems +- `needs-tests` - Missing test coverage +- `needs-docs` - Missing documentation +- `memory-management` - Memory leaks, context issues +- `concurrency-issue` - Deadlocks, race conditions + +### Cost Management + +- **Per-PR limit:** $15 (configurable) +- **Monthly limit:** $200 (configurable) +- **Alert threshold:** $150 +- **Skip draft PRs** to save costs +- **Skip large files** (>5000 lines) +- **Skip binary/generated files** + +## Setup + +### 1. Install Dependencies + +```bash +cd .github/scripts/ai-review +npm install +``` + +### 2. Configure API Key + +Get API key from: https://console.anthropic.com/ + +Add to repository secrets: +1. Settings ā†’ Secrets and variables ā†’ Actions +2. New repository secret +3. Name: `ANTHROPIC_API_KEY` +4. Value: Your API key +5. Add secret + +### 3. Enable Workflow + +The workflow is triggered automatically on PR events: +- PR opened +- PR synchronized (updated) +- PR reopened +- PR marked ready for review (draft ā†’ ready) + +**Draft PRs are skipped** to save costs. + +## Configuration + +### Main Configuration: `config.json` + +```json +{ + "model": "claude-3-5-sonnet-20241022", + "max_tokens_per_request": 4096, + "max_file_size_lines": 5000, + + "cost_limits": { + "max_per_pr_dollars": 15.0, + "max_per_month_dollars": 200.0, + "alert_threshold_dollars": 150.0 + }, + + "skip_paths": [ + "*.png", "*.jpg", "*.svg", + "src/test/regress/expected/*", + "*.po", "*.pot" + ], + + "auto_labels": { + "security-concern": ["security issue", "vulnerability"], + "performance-concern": ["inefficient", "O(nĀ²)"], + "needs-tests": ["missing test", "no test coverage"] + } +} +``` + +**Tunable parameters:** +- `max_tokens_per_request`: Response length (4096 = ~3000 words) +- `max_file_size_lines`: Skip files larger than this +- `cost_limits`: Adjust budget caps +- `skip_paths`: Add more patterns to skip +- `auto_labels`: Customize label keywords + +### Review Prompts + +Located in `.github/scripts/ai-review/prompts/`: + +- `c-code.md` - PostgreSQL C code review +- `sql.md` - SQL and regression test review +- `documentation.md` - Documentation review +- `build-system.md` - Makefile/Meson review + +**Customization:** Edit prompts to adjust review focus and style. + +## Usage + +### Automatic Reviews + +Reviews run automatically on PRs to `master` and `feature/**` branches. + +**Typical workflow:** +1. Create feature branch +2. Make changes +3. Push branch: `git push origin feature/my-feature` +4. Create PR +5. AI review runs automatically +6. Review AI feedback +7. Make updates if needed +8. Push updates ā†’ AI re-reviews + +### Manual Reviews + +Trigger manually via GitHub Actions: + +**Via UI:** +1. Actions ā†’ "AI Code Review" +2. Run workflow +3. Enter PR number +4. Run workflow + +**Via CLI:** +```bash +gh workflow run ai-code-review.yml -f pr_number=123 +``` + +### Interpreting Reviews + +**Inline comments:** +- Posted on specific lines of code +- Format: `**[Category]**` followed by description +- Categories: Memory, Security, Performance, etc. + +**Summary comment:** +- Posted at PR level +- Overview of files reviewed +- Issue count by category +- Cost information + +**Labels:** +- Automatically added based on findings +- Filter PRs by label to prioritize +- Remove label manually if false positive + +### Best Practices + +**Trust but verify:** +- AI reviews are helpful but not infallible +- False positives happen (~5% rate) +- Use judgment - AI doesn't have full context +- Especially verify: security and correctness issues + +**Iterative improvement:** +- AI learns from the prompts, not from feedback +- If AI consistently misses something, update prompts +- Share false positives/negatives to improve system + +**Cost consciousness:** +- Keep PRs focused (fewer files = lower cost) +- Use draft PRs for work-in-progress (AI skips drafts) +- Mark PR ready when you want AI review + +## Cost Tracking + +### View Costs + +**Per-PR cost:** +- Shown in AI review summary comment +- Format: `Cost: $X.XX | Model: claude-3-5-sonnet` + +**Monthly cost:** +- Download cost logs from workflow artifacts +- Aggregate to calculate monthly total + +**Download cost logs:** +```bash +# List recent runs +gh run list --workflow=ai-code-review.yml --limit 10 + +# Download artifact +gh run download -n ai-review-cost-log- +``` + +### Cost Estimation + +**Token costs (Claude 3.5 Sonnet):** +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +**Typical costs:** +- Small PR (<500 lines, 5 files): $0.50-$1.00 +- Medium PR (500-2000 lines, 15 files): $1.00-$3.00 +- Large PR (2000-5000 lines, 30 files): $3.00-$7.50 + +**Expected monthly (20 PRs/month mixed sizes):** $35-50 + +### Budget Controls + +**Automatic limits:** +- Per-PR limit: Stops reviewing after $15 +- Monthly limit: Stops at $200 (requires manual override) +- Alert: Warning at $150 + +**Manual controls:** +- Disable workflow: Actions ā†’ AI Code Review ā†’ Disable +- Reduce `max_tokens_per_request` in config +- Add more patterns to `skip_paths` +- Increase `max_file_size_lines` threshold + +## Troubleshooting + +### Issue: No review posted + +**Possible causes:** +1. PR is draft (intentionally skipped) +2. No reviewable files (all binary or skipped patterns) +3. API key missing or invalid +4. Cost limit reached + +**Check:** +- Actions ā†’ "AI Code Review" ā†’ Latest run ā†’ View logs +- Look for: "Skipping draft PR" or "No reviewable files" +- Verify: `ANTHROPIC_API_KEY` secret exists + +### Issue: Review incomplete + +**Possible causes:** +1. PR cost limit reached ($15 default) +2. File too large (>5000 lines) +3. API rate limit hit + +**Check:** +- Review summary comment for "Reached PR cost limit" +- Workflow logs for "Skipping X - too large" + +**Fix:** +- Increase `max_per_pr_dollars` in config +- Increase `max_file_size_lines` (trade-off: higher cost) +- Split large PR into smaller PRs + +### Issue: False positives + +**Example:** AI flags correct code as problematic + +**Handling:** +1. Ignore the comment (human judgment overrides) +2. Reply to comment explaining why it's correct +3. If systematic: Update prompt to clarify + +**Note:** Some false positives are acceptable (5-10% rate) + +### Issue: Claude API errors + +**Error types:** +- `401 Unauthorized`: Invalid API key +- `429 Too Many Requests`: Rate limit +- `500 Internal Server Error`: Claude service issue + +**Check:** +- Workflow logs for error messages +- Claude status: https://status.anthropic.com/ + +**Fix:** +- Rotate API key if 401 +- Wait and retry if 429 or 500 +- Contact Anthropic support if persistent + +### Issue: High costs + +**Unexpected high costs:** +1. Check cost logs for large PRs +2. Review `skip_paths` - are large files being reviewed? +3. Check for repeated reviews (PR updated many times) + +**Optimization:** +- Add more skip patterns for generated files +- Lower `max_tokens_per_request` (shorter reviews) +- Increase `max_file_size_lines` to skip more files +- Batch PR updates to reduce review runs + +## Disabling AI Review + +### Temporarily disable + +**For one PR:** +- Convert to draft +- Or add `[skip ai]` to PR title (requires workflow modification) + +**For all PRs:** +```bash +# Via GitHub UI: +# Actions ā†’ "AI Code Review" ā†’ "..." ā†’ Disable workflow + +# Via git: +git mv .github/workflows/ai-code-review.yml \ + .github/workflows/ai-code-review.yml.disabled +git commit -m "Disable AI code review" +git push +``` + +### Permanently remove + +```bash +# Remove workflow +rm .github/workflows/ai-code-review.yml + +# Remove scripts +rm -rf .github/scripts/ai-review + +# Commit +git commit -am "Remove AI code review system" +git push +``` + +## Testing and Iteration + +### Shadow Mode (Week 1) + +Run reviews but don't post comments: + +1. Modify `review-pr.js`: + ```javascript + // Comment out posting functions + // await postInlineComments(...) + // await postSummaryComment(...) + ``` + +2. Reviews saved to workflow artifacts +3. Review quality offline +4. Tune prompts based on results + +### Comment Mode (Week 2) + +Post comments with `[AI Review]` prefix: + +1. Add prefix to comment body: + ```javascript + const body = `**[AI Review] [${issue.category}]**\n\n${issue.description}`; + ``` + +2. Gather feedback from developers +3. Adjust prompts and configuration + +### Full Mode (Week 3+) + +Remove prefix, enable all features: + +1. Remove `[AI Review]` prefix +2. Enable auto-labeling +3. Monitor quality and costs +4. Iterate on prompts as needed + +## Advanced Customization + +### Custom Review Prompts + +Add a new prompt for a file type: + +1. Create `.github/scripts/ai-review/prompts/my-type.md` +2. Write review guidelines (see existing prompts) +3. Update `config.json`: + ```json + "file_type_patterns": { + "my_type": ["*.ext", "special/*.files"] + } + ``` +4. Test with manual workflow trigger + +### Conditional Reviews + +Skip AI review for certain PRs: + +Modify `.github/workflows/ai-code-review.yml`: +```yaml +jobs: + ai-review: + if: | + github.event.pull_request.draft == false && + !contains(github.event.pull_request.title, '[skip ai]') && + !contains(github.event.pull_request.labels.*.name, 'no-ai-review') +``` + +### Cost Alerts + +Add cost alert notifications: + +1. Create workflow in `.github/workflows/cost-alert.yml` +2. Trigger: On schedule (weekly) +3. Aggregate cost logs +4. Post issue if over threshold + +## Security and Privacy + +### API Key Security + +- Store only in GitHub Secrets (encrypted at rest) +- Never commit to repository +- Never log in workflow output +- Rotate quarterly + +### Code Privacy + +- Code sent to Claude API (Anthropic) +- Anthropic does not train on API data +- API requests are not retained long-term +- See: https://www.anthropic.com/legal/privacy + +### Sensitive Code + +If reviewing sensitive/proprietary code: + +1. Review Anthropic's terms of service +2. Consider: Self-hosted alternative (future) +3. Or: Skip AI review for sensitive PRs (add label) + +## Support + +### Questions + +- Check this guide first +- Search GitHub issues: label:ai-review +- Check Claude API docs: https://docs.anthropic.com/ + +### Reporting Issues + +Create issue with: +- PR number +- Workflow run URL +- Error messages from logs +- Expected vs actual behavior + +### Improving Prompts + +Contributions welcome: +1. Identify systematic issue (false positive/negative) +2. Propose prompt modification +3. Test on sample PRs +4. Submit PR with updated prompt + +## References + +- Claude API: https://docs.anthropic.com/ +- Claude Models: https://www.anthropic.com/product +- PostgreSQL Hacker's Guide: https://wiki.postgresql.org/wiki/Developer_FAQ +- GitHub Actions: https://docs.github.com/en/actions + +--- + +**Version:** 1.0 +**Last Updated:** 2026-03-10 diff --git a/.github/docs/bedrock-setup.md b/.github/docs/bedrock-setup.md new file mode 100644 index 0000000000000..d8fbd898b51c6 --- /dev/null +++ b/.github/docs/bedrock-setup.md @@ -0,0 +1,298 @@ +# AWS Bedrock Setup for AI Code Review + +This guide explains how to use AWS Bedrock instead of the direct Anthropic API for AI code reviews. + +## Why Use Bedrock? + +- **AWS Credits:** Use existing AWS credits +- **Regional Availability:** Deploy in specific AWS regions +- **Compliance:** Meet specific compliance requirements +- **Integration:** Easier integration with AWS infrastructure +- **IAM Roles:** Use IAM roles instead of API keys when running on AWS + +## Prerequisites + +1. **AWS Account** with Bedrock access +2. **Bedrock Model Access** - Claude 3.5 Sonnet must be enabled +3. **IAM Permissions** for Bedrock API calls + +## Step 1: Enable Bedrock Model Access + +1. Log into AWS Console +2. Navigate to **Amazon Bedrock** +3. Go to **Model access** (left sidebar) +4. Click **Modify model access** +5. Find and enable: **Anthropic - Claude 3.5 Sonnet v2** +6. Click **Save changes** +7. Wait for status to show "Access granted" (~2-5 minutes) + +## Step 2: Create IAM User for GitHub Actions + +### Option A: IAM User with Access Keys (Recommended for GitHub Actions) + +1. Go to **IAM Console** +2. Click **Users** ā†’ **Create user** +3. Username: `github-actions-bedrock` +4. Click **Next** + +**Attach Policy:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel" + ], + "Resource": [ + "arn:aws:bedrock:*::foundation-model/anthropic.claude-3-5-sonnet-*" + ] + } + ] +} +``` + +5. Click **Create policy** ā†’ **JSON** ā†’ Paste above +6. Name: `BedrockClaudeInvokeOnly` +7. Attach policy to user +8. Click **Create user** + +**Create Access Keys:** +1. Click on the created user +2. Go to **Security credentials** tab +3. Click **Create access key** +4. Select: **Third-party service** +5. Click **Next** ā†’ **Create access key** +6. **Download** or copy: + - Access key ID (starts with `AKIA...`) + - Secret access key (only shown once!) + +### Option B: IAM Role (For AWS-hosted runners) + +If running GitHub Actions on AWS (self-hosted runners): + +1. Create IAM Role with trust policy for your EC2/ECS/EKS +2. Attach same `BedrockClaudeInvokeOnly` policy +3. Assign role to your runner infrastructure +4. No access keys needed! + +## Step 3: Configure Repository + +### A. Add AWS Secrets to GitHub + +1. Go to: **Settings** ā†’ **Secrets and variables** ā†’ **Actions** +2. Click **New repository secret** for each: + +**Secret 1:** +- Name: `AWS_ACCESS_KEY_ID` +- Value: Your access key ID from Step 2 + +**Secret 2:** +- Name: `AWS_SECRET_ACCESS_KEY` +- Value: Your secret access key from Step 2 + +**Secret 3:** +- Name: `AWS_REGION` +- Value: Your Bedrock region (e.g., `us-east-1`) + +### B. Update Configuration + +Edit `.github/scripts/ai-review/config.json`: + +```json +{ + "provider": "bedrock", + "model": "claude-3-5-sonnet-20241022", + "bedrock_model_id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock_region": "us-east-1", + ... +} +``` + +**Available Bedrock Model IDs:** +- US: `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- EU: `eu.anthropic.claude-3-5-sonnet-20241022-v2:0` +- Asia Pacific: `apac.anthropic.claude-3-5-sonnet-20241022-v2:0` + +**Available Regions:** +- `us-east-1` (US East - N. Virginia) +- `us-west-2` (US West - Oregon) +- `eu-central-1` (Europe - Frankfurt) +- `eu-west-1` (Europe - Ireland) +- `eu-west-2` (Europe - London) +- `ap-southeast-1` (Asia Pacific - Singapore) +- `ap-southeast-2` (Asia Pacific - Sydney) +- `ap-northeast-1` (Asia Pacific - Tokyo) + +Check current availability: https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html + +### C. Install Dependencies + +```bash +cd .github/scripts/ai-review +npm install +``` + +This will install the AWS SDK for Bedrock. + +## Step 4: Test Bedrock Integration + +```bash +# Create test PR +git checkout -b test/bedrock-review +echo "// Bedrock test" >> test.c +git add test.c +git commit -m "Test: Bedrock AI review" +git push origin test/bedrock-review +``` + +Then create PR via GitHub UI. Check: +1. **Actions** tab - workflow should run +2. **PR comments** - AI review should appear +3. **Workflow logs** - should show "Using AWS Bedrock as provider" + +## Cost Comparison + +### Bedrock Pricing (Claude 3.5 Sonnet - us-east-1) +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +### Direct Anthropic API Pricing +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +**Same price!** Choose based on infrastructure preference. + +## Troubleshooting + +### Error: "Access denied to model" + +**Check:** +1. Model access enabled in Bedrock console? +2. IAM policy includes correct model ARN? +3. Region matches between config and enabled models? + +**Fix:** +```bash +# Verify model access via AWS CLI +aws bedrock list-foundation-models --region us-east-1 --query 'modelSummaries[?contains(modelId, `claude-3-5-sonnet`)]' +``` + +### Error: "InvalidSignatureException" + +**Check:** +1. AWS_ACCESS_KEY_ID correct? +2. AWS_SECRET_ACCESS_KEY correct? +3. Secrets named exactly as shown? + +**Fix:** +- Re-create access keys +- Update GitHub secrets +- Ensure no extra spaces in secret values + +### Error: "ThrottlingException" + +**Cause:** Bedrock rate limits exceeded + +**Fix:** +1. Reduce `max_concurrent_requests` in config.json +2. Add delays between requests +3. Request quota increase via AWS Support + +### Error: "Model not found" + +**Check:** +1. `bedrock_model_id` matches your region +2. Using cross-region model ID (e.g., `us.anthropic...` in us-east-1) + +**Fix:** +Update `bedrock_model_id` in config.json to match your region: +- US regions: `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- EU regions: `eu.anthropic.claude-3-5-sonnet-20241022-v2:0` + +## Switching Between Providers + +### Switch to Bedrock + +Edit `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "bedrock", + ... +} +``` + +### Switch to Direct Anthropic API + +Edit `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "anthropic", + ... +} +``` + +No other changes needed! The code automatically detects the provider. + +## Advanced: Cross-Region Setup + +Deploy in multiple regions for redundancy: + +```json +{ + "provider": "bedrock", + "bedrock_regions": ["us-east-1", "us-west-2"], + "bedrock_failover": true +} +``` + +Then update `review-pr.js` to implement failover logic. + +## Security Best Practices + +1. **Least Privilege:** IAM user can only invoke Claude models +2. **Rotate Keys:** Rotate access keys quarterly +3. **Audit Logs:** Enable CloudTrail for Bedrock API calls +4. **Cost Alerts:** Set up AWS Budgets alerts +5. **Secrets:** Never commit AWS credentials to git + +## Monitoring + +### AWS CloudWatch + +Bedrock metrics available: +- `Invocations` - Number of API calls +- `InvocationLatency` - Response time +- `InvocationClientErrors` - 4xx errors +- `InvocationServerErrors` - 5xx errors + +### Cost Tracking + +```bash +# Check Bedrock costs (current month) +aws ce get-cost-and-usage \ + --time-period Start=2026-03-01,End=2026-03-31 \ + --granularity MONTHLY \ + --metrics BlendedCost \ + --filter file://filter.json + +# filter.json: +{ + "Dimensions": { + "Key": "SERVICE", + "Values": ["Amazon Bedrock"] + } +} +``` + +## References + +- AWS Bedrock Docs: https://docs.aws.amazon.com/bedrock/ +- Model Access: https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html +- Bedrock Pricing: https://aws.amazon.com/bedrock/pricing/ +- IAM Best Practices: https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html + +--- + +**Need help?** Check workflow logs in Actions tab or create an issue. diff --git a/.github/docs/cost-optimization.md b/.github/docs/cost-optimization.md new file mode 100644 index 0000000000000..bcfc1c47b3ed8 --- /dev/null +++ b/.github/docs/cost-optimization.md @@ -0,0 +1,219 @@ +# CI/CD Cost Optimization + +## Overview + +This document describes the cost optimization strategies used in the PostgreSQL mirror CI/CD system to minimize GitHub Actions minutes and API costs while maintaining full functionality. + +## Optimization Strategies + +### 1. Skip Builds for Pristine Commits + +**Problem:** "Dev setup" commits and .github/ configuration changes don't require expensive Windows dependency builds or comprehensive testing. + +**Solution:** The Windows Dependencies workflow includes a `check-changes` job that inspects recent commits and skips builds when all commits are: +- Messages starting with "dev setup" (case-insensitive), OR +- Only modifying files under `.github/` directory + +**Implementation:** See `.github/workflows/windows-dependencies.yml` lines 42-90 + +**Savings:** +- Avoids ~45 minutes of Windows runner time per push +- Windows runners cost 2x Linux minutes (1 minute = 2 billed minutes) +- Estimated savings: ~$8-12/month + +### 2. AI Review Only on Pull Requests + +**Problem:** AI code review is expensive and unnecessary for direct commits to master or pristine commits. + +**Solution:** The AI Code Review workflow only triggers on: +- `pull_request` events (opened, synchronized, reopened, ready_for_review) +- Manual `workflow_dispatch` for testing specific PRs +- Skips draft PRs automatically + +**Implementation:** See `.github/workflows/ai-code-review.yml` lines 3-17 + +**Savings:** +- No reviews on dev setup commits or CI/CD changes +- No reviews on draft PRs (saves ~$1-3 per draft) +- Estimated savings: ~$10-20/month + +### 3. Aggressive Caching + +**Windows Dependencies:** +- Cache key: `--win64-` +- Cache duration: GitHub's default (7 days unused, 10 GB limit) +- Cache hit rate: 80-90% for stable versions + +**Node.js Dependencies:** +- AI review scripts cache npm packages +- Cache key based on `package.json` hash +- Near 100% cache hit rate + +**Savings:** +- Reduces build time from 45 minutes to ~5 minutes on cache hit +- Estimated savings: ~$15-20/month + +### 4. Weekly Scheduled Builds + +**Problem:** GitHub Actions artifacts expire after 90 days, making cached dependencies stale. + +**Solution:** Windows Dependencies runs on a weekly schedule (Sunday 4 AM UTC) to refresh artifacts before expiration. + +**Cost:** +- Weekly builds: ~45 minutes/week Ɨ 4 weeks = 180 minutes/month +- Windows multiplier: 360 billed minutes +- Cost: ~$6/month (within budget) + +**Alternative considered:** Daily builds would cost ~$50/month (rejected) + +### 5. Sync Workflow Optimization + +**Automatic Sync:** +- Runs hourly to keep mirror current +- Very lightweight: ~2-3 minutes per run +- Cost: ~150 minutes/month = $0 (within free tier) + +**Manual Sync:** +- Only runs on explicit trigger +- Used for testing and recovery +- Cost: Negligible + +### 6. Smart Workflow Triggers + +**Path-based triggers:** +```yaml +push: + paths: + - '.github/windows/manifest.json' + - '.github/workflows/windows-dependencies.yml' +``` + +Only rebuild Windows dependencies when: +- Manifest versions change +- Workflow itself is updated +- Manual trigger or schedule + +**Branch-based triggers:** +- AI review only on PRs to master, feature/**, dev/** +- Sync only affects master branch + +## Cost Breakdown + +| Component | Monthly Cost | Notes | +|-----------|-------------|-------| +| GitHub Actions - Sync | $0 | ~150 min/month (free: 2,000 min) | +| GitHub Actions - AI Review | $0 | ~200 min/month (free: 2,000 min) | +| GitHub Actions - Windows | ~$5-8 | ~2,500 min/month with optimizations | +| Claude API (Bedrock) | $30-45 | Usage-based, ~15-20 PRs/month | +| **Total** | **~$35-53/month** | | + +**Before optimizations:** ~$75-100/month +**After optimizations:** ~$35-53/month +**Savings:** ~$40-47/month (40-47% reduction) + +## Monitoring Costs + +### GitHub Actions Usage + +Check usage in repository settings: +``` +Settings ā†’ Billing and plans ā†’ View usage +``` + +Or via CLI: +```bash +gh api repos/:owner/:repo/actions/billing/workflows --jq '.workflows' +``` + +### AWS Bedrock Usage + +Monitor Claude API costs in AWS Console: +``` +AWS Console ā†’ Bedrock ā†’ Usage ā†’ Invocation metrics +``` + +Or via cost logs in artifacts: +``` +.github/scripts/ai-review/cost-log-*.json +``` + +### Setting Alerts + +**GitHub Actions:** +- No built-in alerts +- Monitor via monthly email summaries +- Consider third-party monitoring (e.g., AWS Lambda + GitHub API) + +**AWS Bedrock:** +- Set CloudWatch billing alarms +- Recommended thresholds: + - Warning: $30/month + - Critical: $50/month +- Hard cap in code: $200/month (see `config.json`) + +## Future Optimizations + +### Potential Improvements + +1. **Conditional Testing on PRs** + - Only run full Cirrus CI suite if C code or SQL changes + - Skip for docs-only PRs + - Estimated savings: ~5-10% of testing costs + +2. **Incremental AI Review** + - On PR updates, only review changed files + - Current: Reviews entire PR on each update + - Estimated savings: ~20-30% of AI costs + +3. **Dependency Build Sampling** + - Build only changed dependencies instead of all + - Requires more sophisticated manifest diffing + - Estimated savings: ~30-40% of Windows build costs + +4. **Self-hosted Runners** + - Run Linux builds on own infrastructure + - Keep Windows runners on GitHub (licensing) + - Estimated savings: ~$10-15/month + - **Trade-off:** Maintenance overhead + +### Not Recommended + +1. **Reduce sync frequency** (hourly ā†’ daily) + - Savings: Negligible (~$0.50/month) + - Cost: Increased lag with upstream (unacceptable) + +2. **Skip Windows builds entirely** + - Savings: ~$8/month + - Cost: Lose reproducible dependency builds (defeats purpose) + +3. **Reduce AI review quality** (Claude Sonnet ā†’ Haiku) + - Savings: ~$20-25/month + - Cost: Significantly worse code review quality + +## Pristine Commit Policy + +The following commits are considered "pristine" and skip expensive builds: + +1. **Dev setup commits:** + - Message starts with "dev setup" (case-insensitive) + - Examples: "dev setup v19", "Dev Setup: Update IDE config" + - Contains: .clang-format, .idea/, .vscode/, flake.nix, etc. + +2. **CI/CD configuration commits:** + - Only modify files under `.github/` + - Examples: Workflow changes, script updates, documentation + +**Why this works:** +- Dev setup commits don't affect PostgreSQL code +- CI/CD commits are tested by running the workflows themselves +- Reduces unnecessary Windows builds by ~60-70% + +**Implementation:** See `pristine-master-policy.md` for details. + +## Questions? + +For more information: +- Pristine master policy: `.github/docs/pristine-master-policy.md` +- Sync setup: `.github/docs/sync-setup.md` +- AI review guide: `.github/docs/ai-review-guide.md` +- Windows builds: `.github/docs/windows-builds.md` diff --git a/.github/ocr/context.md b/.github/ocr/context.md new file mode 100644 index 0000000000000..c4a83b85b124e --- /dev/null +++ b/.github/ocr/context.md @@ -0,0 +1,126 @@ +# OCR review context ā€” PostgreSQL contribution standards + +You are reviewing a change to a **PostgreSQL** fork. Every PR here is destined to +become a patch posted to the **pgsql-hackers** mailing list and tracked in a +**commitfest**. Review with the combined rigor, taste, and attention to detail of +the PostgreSQL committers. This context applies to the *whole* change, on top of +the per-file rules. + +## Review discipline +- Be precise and blunt; lead with the most serious problem. No praise, no + validation of the author, no disclaimers ā€” accuracy is the only metric. +- Verify every claim against the actual diff. Confirm names, signatures, line + numbers, and APIs before asserting. Never invent behavior or cite code not in + the change. If unsure, say so, and tag each finding **high / moderate / low** + confidence. +- Judge the change on its merits regardless of how the PR frames it. A draft PR + is WIP: weight design/approach feedback over style nits. + +## Patch hygiene (top rejection reasons on -hackers) +1. **Minimal diff.** The fastest way to get a patch rejected is unrelated + changes: reformatting untouched lines, rewording unrelated comments, touching + code not required by the change. Flag any hunk not needed for the stated + purpose. After the patch, the code should read as if it had always been + written that way. +2. **Atomic, bisectable commits.** Each commit must build and pass tests on its + own ā€” a broken intermediate commit breaks `git bisect`, revert, and + cherry-pick. Flag a commit that only compiles once a later commit lands. + Prefer one focused patch, or a clearly-ordered series of + independently-committable pieces. +3. **Tests + docs are mandatory.** A user-visible change without regression/TAP + tests **and** documentation is WIP, not commit-ready. New behavior needs + tests that cover edge and error paths, not just the happy path. +4. **DRY / reuse.** Prefer existing infrastructure (`List` in `pg_list.h`, + `StringInfo`, `dynahash`/`simplehash`, `palloc`/`MemoryContext`, `foreach`) + over reinventing it. Flag copy-paste and speculative abstraction alike ā€” the + community wants minimal, targeted changes that fit the subsystem's existing + patterns. +5. **Whitespace.** No trailing whitespace; tabs (width 4) for C indentation; + `git diff --check` must be clean. Whitespace-only churn on untouched lines is + a defect. + +## Committer-owned files ā€” do NOT touch in a patch (flag if present) +These are the committer's job at push time; including them causes needless +merge conflicts and is a mistake: +- **`src/include/catalog/catversion.h`** ā€” the `CATALOG_VERSION_NO` bump is done + by the **committer** when pushing. A catversion bump in the PR is **wrong** ā€” + flag it. (This is the single most common author mistake in catalog patches.) +- **Release notes** (`doc/src/sgml/release-*.sgml`) and version strings + (`configure.ac` `AC_INIT` version, `meson.build` `version`, `PG_VERSION`). + +## Generated files ā€” never hand-edit; edit the source +Flag direct edits to generated output; point the author at the source instead: +- Catalog headers `src/include/catalog/*_d.h`, `postgres.bki`, `schemapg.h`, + `system_constraints.sql` ā†’ edit the `pg_*.dat` files. +- `src/backend/nodes/{copy,equal,out,read}funcs.c` and other + `gen_node_support.pl` output ā†’ annotate the `Node` struct in its header. +- `fmgroids.h`, `fmgrprotos.h`, `fmgrtab.c` ā†’ edit `pg_proc.dat`. +- `utils/errcodes.h` ā†’ `errcodes.txt`; wait-event headers ā†’ + `wait_event_names.txt`; `lwlocknames.h` ā†’ `lwlocknames.txt`. +- `configure` ā†’ `configure.ac`; `*.po` translations are handled separately; + generated Unicode tables come from their source scripts. + +## Portability is a hard gate +PostgreSQL runs on Linux, Windows (MSVC), macOS, the BSDs and Solaris, across +**x86_64, ARM64, RISC-V, PPC64, s390x**, both endiannesses and 32/64-bit. Any +change must be portable across all of them: +- No unaligned memory access; no dependence on `char` signedness, integer/pointer + width, endianness, or struct padding for on-disk/wire formats. +- Use `int16/int32/int64`, `Size`, and `INT64_FORMAT`/`UINT64_FORMAT` (never + `%ld` for `int64`). +- Atomics/barriers only via `port/atomics` (`pg_atomic_*`, `pg_read/write_barrier`). +- **Windows/MSVC:** any `extern` variable used from another module or an + extension needs `PGDLLIMPORT` in its header; no VLAs or compiler-specific + extensions beyond the tree's C99 baseline. + +## Backward compatibility ā€” the strongest constraint +Do not break SQL behavior, the libpq wire protocol, the logical-replication +protocol, dump/restore, `pg_upgrade`, or exported/`PGDLLIMPORT` APIs without +extraordinary justification. **ABI** matters for back-branches: changing the +size/layout of an exported struct or the signature of an exported function +breaks installed extensions. + +## Mailing-list context & etiquette +Because each PR becomes a pgsql-hackers email read by a busy, expert, opinionated +audience, also flag what reliably wastes reviewer time or draws rejection: +- A patch that **does more than one thing** or bundles unrelated cleanup ā€” split it. +- **Footguns**: easy-to-misuse APIs, silent data-loss/corruption hazards, unsafe + defaults ā€” name them explicitly. +- **Performance claims without a reproducible benchmark.** +- No reference to the **design discussion / prior -hackers thread** (Message-Id) + for a non-trivial change. +- **Do not bikeshed:** keep style nits proportionate and clearly separated from + substantive correctness findings. + +## Minimalism ā€” the "ponytail" discipline +The best code is the code you never wrote (YAGNI). Before accepting new code, +apply the ladder: (1) Does this need to exist at all? (2) Can existing +code/infrastructure already do it? (3) Is this the simplest thing that works? +Flag: speculative scaffolding and config for a path that isn't wired yet; dead +code and unused "flexibility" (fields, params, abstractions, options with no +caller); premature abstraction (a helper used exactly once); knobs/GUCs/flags +nobody asked for. Minimal, targeted changes that fit the existing patterns beat +clever or general-purpose ones. + +## Comment & identity accuracy +- Comments must describe what the code does **now**. Flag aspirational/ + future-tense comments for behavior that already shipped ("will be", "for now", + "not yet", "future", and stale "TODO/FIXME/XXX/HACK"); comments that drifted + from the code they sit above; and incomplete/trailing comments. Comments + explain **why**, not what. No commented-out code. +- **ASCII only** in source and diffs ā€” no smart quotes, em-dashes, or ellipsis + characters. + +## Commit & versioning discipline +- Conventional-commit style, imperative subject, one logical change per commit, + each commit building on its own. +- Do **not** bump version numbers or generated version stamps (including + `catversion.h`) ā€” that is the maintainer's job at commit/release time. + +Understand common list shorthand so your comments are precise and not +miscommunicated: WIP (work in progress), GUC (config variable), WAL, LSN, OID, +TOAST, FSM, TAM (table access method), RLS, DSM, 2PC, PITR, CIC (concurrent index +creation), SAOP, ABI/API, backpatch (apply to supported back-branches), HEAD +(master tip), catversion (catalog version), pgindent, buildfarm, cfbot, +`s/x/y/` (suggested text substitution), footgun, bikeshedding, POLA (principle of +least astonishment). diff --git a/.github/ocr/litellm.yaml b/.github/ocr/litellm.yaml new file mode 100644 index 0000000000000..e23cc4eee6fe2 --- /dev/null +++ b/.github/ocr/litellm.yaml @@ -0,0 +1,41 @@ +# LiteLLM proxy config ā€” bridges Open Code Review (OpenAI protocol) to AWS Bedrock. +# +# This proxy is NOT a hosted service. The ocr-review.yml workflow installs it +# (`pip install 'litellm[proxy]'`) and runs it as a background process bound to +# 127.0.0.1:4000 for the duration of a single GitHub Actions job, then it exits. +# +# Auth to Bedrock: LiteLLM uses boto3's default credential chain, which reads +# the temporary AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN +# minted by the workflow's OIDC "Configure AWS credentials" step; region from +# AWS_REGION. + +model_list: + - model_name: ocr-bedrock + litellm_params: + # Set the repo variable OCR_BEDROCK_MODEL to an Opus inference-profile id + # your account has access to, e.g.: + # bedrock/converse/us.anthropic.claude-opus-4-8 + # The 'converse/' prefix uses Bedrock's Converse API, which is the most + # reliable path for Claude tool-use (what OCR relies on). + model: os.environ/OCR_BEDROCK_MODEL + aws_region_name: os.environ/AWS_REGION + + # "High effort" review. Claude Opus 4.8 on Bedrock uses *adaptive* thinking + # controlled by output_config.effort. Set it DIRECTLY here ā€” NOT via + # reasoning_effort, which LiteLLM still maps to the legacy + # thinking.type.enabled that Opus 4.8 rejects. LiteLLM forwards + # output_config into additionalModelRequestFields for Anthropic models; if + # the build doesn't recognize the effort param it is dropped with a warning + # (no error) and the model reviews at its default effort. + # Valid: low|medium|high|max|xhigh (auto-clamped to the model ceiling). + output_config: + effort: xhigh + max_tokens: 32000 + +litellm_settings: + drop_params: true # silently drop params a model doesn't support + modify_params: true # auto-fix minor request incompatibilities + request_timeout: 600 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/.github/ocr/pg-history.py b/.github/ocr/pg-history.py new file mode 100644 index 0000000000000..5794f8a920bd7 --- /dev/null +++ b/.github/ocr/pg-history.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +""" +pg-history: tie a PR's changes to PostgreSQL git + pgsql-hackers email history. + +OCR (the code reviewer) cannot call MCP servers, so this is a separate agent: +it runs a Bedrock (Claude Opus) tool-use loop wired to the Agora MCP server at +https://pg.ddx.io/mcp, lets the model search the mailing-list archives / commit +history / commitfest data, and emits a Markdown summary linking the changes to +the relevant threads (https://pg.ddx.io/m/pgsql-hackers/). + +Env: + PG_HISTORY_MCP_URL MCP endpoint (default https://pg.ddx.io/mcp) + PG_HISTORY_MODEL Bedrock model id (e.g. us.anthropic.claude-opus-4-8) + AWS_REGION region (creds come from the OIDC step's env) + BASE_REF, HEAD_SHA PR base ref and head sha (for the git diff context) + GH_PR_TITLE PR title (optional, adds context) + PG_HISTORY_OUT output markdown path (default /tmp/pg-history.md) +Writes the markdown to PG_HISTORY_OUT; exits 0 even on soft failures (writes a note). +""" +import json, os, subprocess, sys, urllib.request + +MCP_URL = os.environ.get("PG_HISTORY_MCP_URL", "https://pg.ddx.io/mcp") +MODEL = os.environ.get("PG_HISTORY_MODEL", "us.anthropic.claude-opus-4-8").replace("bedrock/converse/", "").replace("bedrock/", "") +REGION = os.environ.get("AWS_REGION", "us-east-1") +BASE_REF = os.environ.get("BASE_REF", "") +HEAD_SHA = os.environ.get("HEAD_SHA", "") +PR_TITLE = os.environ.get("GH_PR_TITLE", "") +OUT = os.environ.get("PG_HISTORY_OUT", "/tmp/pg-history.md") +UA = "pg-history/0.1 (+github-actions)" + +# Curated subset of the 108 Agora tools ā€” the ones useful for connecting a +# change to its discussion/commit history. Intersected with what the server +# actually exposes, so unknown names are harmless. +TOOL_WHITELIST = { + "find_related_discussions", "find_similar_messages", "get_thread", + "discussion_links", "get_author_messages", "browse_by_date", + "blame_symbol", "check_upstream_status", "find_related", + "find_entries_for_thread", "find_entries_for_author", "get_commit", + "search", "hybrid_search", "get_callers", "get_callees", "find_pattern", +} +MAX_ROUNDS = 14 +TOOL_RESULT_CAP = 8000 # chars per tool result fed back to the model + + +def _mcp_post(body, sid=None): + headers = {"Content-Type": "application/json", + "Accept": "application/json, text/event-stream", "User-Agent": UA} + if sid: + headers["Mcp-Session-Id"] = sid + req = urllib.request.Request(MCP_URL, data=json.dumps(body).encode(), headers=headers, method="POST") + resp = urllib.request.urlopen(req, timeout=60) + sid_out = resp.headers.get("Mcp-Session-Id") + result = None + for line in resp.read().decode().splitlines(): + line = line.strip() + if line.startswith("data:"): + line = line[5:].strip() + if not line or line.startswith("event:"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + result = obj + return result, sid_out + + +class MCP: + def __init__(self): + init, self.sid = _mcp_post({"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "pg-history", "version": "0.1"}}}) + if not init or "result" not in init: + raise RuntimeError(f"MCP initialize failed: {init}") + try: + _mcp_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, self.sid) + except Exception: + pass + self._id = 1 + + def list_tools(self): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/list", "params": {}}, self.sid) + return (res or {}).get("result", {}).get("tools", []) + + def call(self, name, args): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/call", + "params": {"name": name, "arguments": args or {}}}, self.sid) + if not res: + return "(no response)" + if "error" in res: + return f"ERROR: {json.dumps(res['error'])[:500]}" + parts = [] + for c in res.get("result", {}).get("content", []): + if c.get("type") == "text": + parts.append(c["text"]) + return ("\n".join(parts) or "(empty)")[:TOOL_RESULT_CAP] + + +def git(*args): + try: + return subprocess.check_output(["git", *args], text=True, stderr=subprocess.DEVNULL).strip() + except Exception: + return "" + + +def pr_context(): + base = f"origin/{BASE_REF}" if BASE_REF else "" + rng = f"{base}..{HEAD_SHA}" if base and HEAD_SHA else HEAD_SHA + commits = git("log", "--no-merges", "--format=%h %s", f"{rng}") if rng else "" + stat = git("diff", "--stat", rng) if rng else "" + files = git("diff", "--name-only", rng) if rng else "" + return commits[:4000], stat[:3000], files[:2000] + + +SYSTEM = """You are a PostgreSQL community research assistant. Given a pull request's +commits and changed files, use the available tools (backed by the Agora index of +pgsql-hackers mail, commit history, and commitfest data) to connect the change to +its history. Your goal: + +- Find the mailing-list thread(s) and prior discussion behind this change. +- Identify related/superseded prior commits and any commitfest entry. +- Note relevant prior art, rejected approaches, or design rationale. + +Rules (voice & rigor): +- Be precise and blunt. No praise, no filler, no hedging, no disclaimers. Accuracy is + the only success metric ā€” not the author's approval. Lead with the most important finding. +- NEVER hallucinate. Verify every Message-ID, thread subject, commit hash, author name, + and date against an actual tool result before citing it. If a search returns nothing, + say so plainly ā€” do not guess or fabricate a plausible-looking link. +- Assess the change on its merits, independent of how the PR frames it. +- Tag any inferred (not tool-confirmed) linkage with an explicit confidence level: + high / moderate / low. +- Be decisive and efficient: a handful of targeted tool calls, not exhaustive search. +- Cite every mailing-list message as a Markdown link: [subject](https://pg.ddx.io/m/pgsql-hackers/MESSAGE_ID). +- If you find nothing relevant, say so in one line ā€” do not pad. + +When done, output ONLY Markdown (no preamble) with these sections, omitting any that are empty: +## šŸ§µ Related discussion +## šŸ”— Related commits / prior art +## šŸ“‹ Commitfest +## šŸ§­ Context for reviewers +Keep it tight (use bullets; link generously).""" + + +def to_toolspec(t): + schema = t.get("inputSchema") or {"type": "object", "properties": {}} + return {"toolSpec": {"name": t["name"], + "description": (t.get("description") or "")[:600], + "inputSchema": {"json": schema}}} + + +def main(): + commits, stat, files = pr_context() + if not commits and not files: + open(OUT, "w").write("") # nothing to do + print("No PR diff context; skipping.") + return + user = (f"PR title: {PR_TITLE}\n\n" if PR_TITLE else "") + \ + f"Commits:\n{commits or '(none)'}\n\nChanged files:\n{files or '(none)'}\n\nDiffstat:\n{stat or '(none)'}\n" + + try: + mcp = MCP() + tools = [to_toolspec(t) for t in mcp.list_tools() if t.get("name") in TOOL_WHITELIST] + except Exception as e: + open(OUT, "w").write(f"_pg-history: could not reach the Agora MCP server ({MCP_URL}): {e}_\n") + print(f"MCP unavailable: {e}") + return + if not tools: + open(OUT, "w").write("_pg-history: no usable MCP tools available._\n") + return + + import boto3 + from botocore.config import Config + + # botocore's default read timeout (60s) is too short for a multi-round + # (MAX_ROUNDS) tool-use loop against a large PR diff on a reasoning model; + # each converse() call alone can take several minutes. Bump it well past + # what a single round needs; connect_timeout stays short since a stuck + # TCP handshake is a different (and much cheaper to detect) failure mode. + brt = boto3.client("bedrock-runtime", region_name=REGION, + config=Config(read_timeout=900, connect_timeout=10)) + messages = [{"role": "user", "content": [{"text": user}]}] + final_text = "" + try: + for _ in range(MAX_ROUNDS): + resp = brt.converse( + modelId=MODEL, + system=[{"text": SYSTEM}], + messages=messages, + toolConfig={"tools": tools}, + inferenceConfig={"maxTokens": 4096}, + ) + out = resp["output"]["message"] + messages.append(out) + if resp.get("stopReason") == "tool_use": + results = [] + for blk in out["content"]: + tu = blk.get("toolUse") + if not tu: + continue + res_text = mcp.call(tu["name"], tu.get("input") or {}) + results.append({"toolResult": {"toolUseId": tu["toolUseId"], + "content": [{"text": res_text}]}}) + messages.append({"role": "user", "content": results}) + continue + final_text = "".join(b.get("text", "") for b in out["content"]).strip() + break + except Exception as e: + open(OUT, "w").write(f"_pg-history: Bedrock call failed: {e}_\n") + print(f"Bedrock error: {e}") + return + + if not final_text: + final_text = "_pg-history: no related history found._" + body = "## šŸ“œ Change history & discussion (Agora / pg.ddx.io)\n\n" + final_text + \ + "\n\nGenerated by pg-history via the Agora MCP server (pg.ddx.io).\n" + open(OUT, "w").write(body) + print(body) + + +if __name__ == "__main__": + main() diff --git a/.github/ocr/rule.json b/.github/ocr/rule.json new file mode 100644 index 0000000000000..60e13e73dcbe0 --- /dev/null +++ b/.github/ocr/rule.json @@ -0,0 +1,65 @@ +{ + "_comment": "OCR per-file review rules for PostgreSQL core + extensions. Cross-cutting contribution standards & mailing-list etiquette live in .github/ocr/context.md, passed via --background-file. OCR uses FIRST-MATCH-WINS in declaration order, so rules are ordered most-specific first. merge_system_rule:true keeps OCR's built-in fine-tuned checks (thread-safety, injection, NPE) alongside these PostgreSQL-specific rules.", + "rules": [ + { + "path": "src/test/**", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL tests. Coverage is mandatory for any behavioral change and must include edge cases (NULL, empty, boundary/overflow) and ERROR paths, not just the happy path. A test that still passes with the feature reverted is worthless ā€” confirm it actually exercises and would catch regressions in the new code. Regression (.sql/expected): deterministic, portable output ā€” ORDER BY where row order matters, no timing/plan-dependent output except intentional EXPLAIN, no absolute paths, locale-independent (C collation or explicit COLLATE), DROP objects the test creates; expected/ output must stay stable across platforms and under the parallel schedule. Concurrency/locking belongs in isolation tests (src/test/isolation, .spec + permutations). End-to-end/crash/replication/CLI behavior belongs in TAP tests (t/*.pl with PostgreSQL::Test::Cluster/Utils) ā€” no hardcoded ports/paths, no sleep as synchronization (use poll_query_until/wait_for), skip cleanly when prerequisites are missing, and clean up nodes." + }, + { + "path": "**/*.{c,h}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL backend/frontend C ā€” review as pgsql-hackers committers do, in priority order.\n\n(1) CORRECTNESS (highest): Memory ā€” every palloc lives in the right MemoryContext; error paths via ereport/elog(ERROR) must not leak memory/buffers/locks/fds (rely on MemoryContext/ResourceOwner reset or PG_TRY/PG_FINALLY); no use-after-free; delete temp contexts. Concurrency ā€” consistent lock ordering (deadlock-free), correct lock levels, balanced LWLockAcquire/Release and START_/END_CRIT_SECTION, no TOCTOU, CHECK_FOR_INTERRUPTS in long loops, async-signal-safe signal handlers (volatile sig_atomic_t). WAL ā€” any change to shared on-disk state must be WAL-logged AND correctly replayed (redo path), crash- and replica-consistent. NULL/edge/overflow handling.\n\n(2) BACKWARD COMPATIBILITY / ABI: don't break behavior, dump/restore, pg_upgrade, libpq wire protocol, logical-replication protocol, or exported/PGDLLIMPORT'd APIs (struct size/layout, function signatures) without extraordinary justification.\n\n(3) CATALOG / GENERATED: new/changed catalog data goes in pg_*.dat, NOT the generated *_d.h/.bki. New Node types: ANNOTATE the struct in its header so gen_node_support.pl regenerates copy/equal/out/read ā€” do NOT hand-edit *funcs.c. New SQL-callable functions: add to pg_proc.dat with an OID from the 8000-9999 developer range (src/include/catalog/unused_oids; check duplicate_oids); committer renumbers at commit. DO NOT bump CATALOG_VERSION_NO in the patch ā€” flag any catversion.h change as a mistake (committer's job).\n\n(4) PERFORMANCE: no regression on hot paths; avoid O(n^2) where better is feasible; minimize work under contended locks; avoid needless palloc churn and large struct copies in hot paths.\n\n(5) SECURITY: bounded string ops (snprintf/strlcpy/strlcat ā€” never strcpy/strcat/sprintf); integer/size-overflow checks before allocation; never user input as a format string; privilege checks via pg_*_aclcheck; beware search_path and SECURITY DEFINER.\n\n(6) PORTABILITY (hard gate): no unaligned access; no dependence on char signedness, int/long/pointer width, endianness, or struct padding for on-disk/wire formats; use int16/int32/int64 + INT64_FORMAT/UINT64_FORMAT (never %ld for int64); align contended shared structs (pg_attribute_aligned/cache-line pad). Atomics/barriers only via port/atomics (pg_atomic_*, pg_read/write_barrier) ā€” never raw intrinsics or volatile-as-barrier. WINDOWS/MSVC: extern vars used cross-module/extension need PGDLLIMPORT; no VLAs or features beyond the C99 baseline the tree targets; use pg_pread/pg_pwrite. Applies across x86_64/ARM64/RISC-V/PPC64/s390x, big/little endian, 32/64-bit.\n\n(7) CONVENTIONS: errmsg starts lowercase, no trailing period, no embedded newlines; errdetail/errhint are complete capitalized sentences; correct ERRCODE_*; wrap user-facing text in _(); errmsg_plural for counts. Assert() only for can't-happen invariants (never user-reachable). Naming: snake_case with subsystem prefix (heap_insert) or CamelCase for major subsystems (ExecInitNode); ALL_CAPS macros. Must pgindent cleanly (tabs, width 4). Comments explain WHY not WHAT; no #ifdef 0 blocks, no commented-out code, no #ifdef fencing your feature. Reuse existing helpers (DRY)." + }, + { + "path": "**/*.dat", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL catalog data (pg_proc.dat, pg_type.dat, etc.) ā€” the SOURCE for generated headers. The generated *_d.h, postgres.bki, fmgroids.h, fmgrtab.c must NOT be hand-edited (they regenerate from these files). OIDs: use a value from the developer range 8000-9999 (src/include/catalog/unused_oids; verify with duplicate_oids); committer renumbers to a final contiguous block, so stay in-range and unique but don't over-optimize the exact number. Keep proc entries complete/consistent (prosrc, provolatile, proparallel, prorettype/proargtypes, matching description). DO NOT bump CATALOG_VERSION_NO / catversion.h ā€” committer's job at push time; flag any such change. New catalog columns/views need documentation in doc/src/sgml/catalogs.sgml." + }, + { + "path": "**/*.{sql,pgsql}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL SQL. Valid PostgreSQL dialect (not MySQL/Oracle); correct types (bigint vs int, text vs varchar); sound transaction/isolation and CTE-materialization assumptions. SECURITY: flag SQL injection in dynamic SQL (require quote_identifier/quote_literal or format() with %I/%L), SECURITY DEFINER without a locked-down search_path, inappropriate RLS bypass. Prefer set-based over row-at-a-time/N+1. BACKWARD COMPATIBILITY (a top rejection reason): changing existing SQL behavior, the output of existing functions, or default GUCs needs extraordinary justification. New SQL-callable objects belong in pg_*.dat with OIDs from the 8000-9999 range, not in generated files. Minimal diff; add regression tests + docs." + }, + { + "path": "**/*.{pl,pm}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Perl (TAP tests and build/catalog tooling). Require 'use strict; use warnings;'. Must be perltidy-clean with the tree's src/tools/pgindent/perltidyrc and pass src/tools/perlcheck/pgperlcritic. Use the framework: PostgreSQL::Test::Cluster, PostgreSQL::Test::Utils, Test::More; no hardcoded ports/paths/PIDs; use safe_psql/poll_query_until, not sleep; skippable without optional prerequisites; clean up nodes. PORTABILITY: run on Windows (no fork-only constructs, use File::Spec, avoid unavailable signals) and the minimum supported Perl. Robustness: avoid two-arg open and string system()/qx with interpolated data (use list forms). Generator scripts (gen_node_support.pl, catalog Perl) must be deterministic and stay in sync with inputs; do not commit their generated output." + }, + { + "path": "**/*.py", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Python (build/test tooling, oauth/pytest tests, src/tools). Follow surrounding style; keep imports to the standard library unless the dependency is already required by the tree (no surprise third-party deps in build/test tooling). PORTABILITY: support the project's minimum Python 3 and run on Windows and the BSDs (use os.path/pathlib, avoid POSIX-only calls and shell=True with interpolated input). Deterministic, self-cleaning tests; no hardcoded ports/paths; skip cleanly without prerequisites. For the Perl->pytest porting effort, confirm behavior parity with the TAP test replaced (same assertions/coverage), not a superficial translation. Minimal diff; match the tree's ruff/black config if present." + }, + { + "path": "**/*.{rs,toml}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Rust PostgreSQL extension (pgrx) or Rust support crate. Not core C, but it runs inside/alongside the backend, so backend safety applies. SAFETY: in code reachable from an SQL call, a Rust panic aborts the Postgres process ā€” forbid unwrap()/expect()/panic!/unreachable!/todo! and index-panics on reachable paths; use Result and pgrx error reporting (error!/ereport!). Every `unsafe` block needs a comment justifying its invariant; scrutinize raw pointers and FFI across the pg_sys boundary. pgrx: honor #[pg_guard] on extern C fns (correct panic/longjmp handling); never hold Rust references across SPI or anything that can longjmp (skips Rust destructors -> leaks); respect MemoryContext lifetimes for palloc'd data; datum<->Rust conversions must handle NULL. Concurrency uses Postgres shmem/LWLocks (pgrx shmem API), not std::sync alone. Lints: must pass `cargo clippy --all-targets --all-features -- -D warnings` and `cargo fmt --check`; deny unwrap_used/expect_used/panic in libraries; thiserror (libs) / anyhow (bins). Justify every new dependency. Tests: #[pg_test] for in-backend behavior, #[test] for pure logic; cover error and NULL paths. Minimal, idiomatic diff." + }, + { + "path": "**/{configure.ac,*.m4,aclocal.m4}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Autoconf. Edit configure.ac / the m4 macros ā€” do NOT hand-edit generated 'configure' or pg_config.h.in in the same patch (regeneration is the committer's step; a patch that also rewrites generated configure output is suspect). Feature/header/function probes must be portable and not assume a specific OS/compiler. Every configure knob must be mirrored on the Meson side (meson_options.txt/meson.build) and documented. Minimal diff." + }, + { + "path": "**/{meson.build,meson_options.txt}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Meson build. Valid syntax; correct subdir()/dependency()/declare_dependency and install paths; new source files must be added here. CRITICAL: PostgreSQL maintains BOTH Meson and Autoconf/Make ā€” any new file, option, or feature check must be mirrored on the configure.ac/Makefile side so the two never drift (a file built by only one system is a common defect). New options need matching docs and sensible defaults. Minimal diff." + }, + { + "path": "**/{Makefile,GNUmakefile,*.mk}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Makefile (GNU Make). $(VAR) refs; correct .PHONY; accurate dependencies (no parallel -j races); $(MAKE) for recursion; VPATH/out-of-tree build support; no hardcoded paths (use standard PostgreSQL makefile vars and $(top_builddir)); clean/distclean/maintainer-clean must remove new artifacts; extensions use PGXS. Must stay in sync with meson.build. Minimal diff." + }, + { + "path": "doc/**/*.sgml", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL documentation (DocBook SGML). Technically accurate/complete (parameters, limitations, version/compat notes); correct tag usage/nesting (, , , , , /); working cross-references; spell it 'PostgreSQL' in prose; SQL keywords uppercase in examples. Coverage: a new GUC -> config.sgml (and postgresql.conf.sample); new/changed catalogs or views -> catalogs.sgml; new SQL syntax -> the matching ref/*.sgml; new functions -> func.sgml. Do NOT edit release-notes (release-*.sgml) ā€” written by the release team/committers; flag such edits. New user-facing behavior in this PR should ship with matching docs." + }, + { + "path": "**/*.md", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Markdown docs. Clear heading hierarchy; fenced code blocks with language hints; accurate instructions/prerequisites; consistent PostgreSQL terminology; no broken relative links or stale claims. Minimal diff." + } + ] +} diff --git a/.github/workflows/ocr-model-check.yml b/.github/workflows/ocr-model-check.yml new file mode 100644 index 0000000000000..10d250528cf7c --- /dev/null +++ b/.github/workflows/ocr-model-check.yml @@ -0,0 +1,89 @@ +# Checks AWS Bedrock weekly for a newer Claude Opus inference profile than the +# one OCR currently uses (vars.OCR_BEDROCK_MODEL) and, if found, opens/updates a +# single GitHub issue telling the maintainer to bump the variable. It does NOT +# change the model automatically: GITHUB_TOKEN cannot write Actions *variables* +# (that needs a PAT with admin), so this is a notify-only mechanism by design. +name: OCR model self-check + +on: + schedule: + - cron: '0 12 * * 1' # Mondays 12:00 UTC + workflow_dispatch: + +permissions: + id-token: write + contents: read + issues: write + +jobs: + check-model: + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-model-check-${{ github.run_id }} + + - name: Find newest Opus vs configured + id: check + env: + CURRENT: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import os, re, subprocess, json + region = os.environ.get("AWS_REGION", "us-east-1") + current = os.environ.get("CURRENT", "") + out = subprocess.run( + ["aws", "bedrock", "list-inference-profiles", "--region", region, + "--query", "inferenceProfileSummaries[].inferenceProfileId", "--output", "json"], + capture_output=True, text=True) + ids = json.loads(out.stdout or "[]") + # Parse claude-opus-- from any profile id (prefix us./global. ok). + def ver(s): + m = re.search(r"claude-opus-(\d+)-(\d+)", s) + return (int(m.group(1)), int(m.group(2))) if m else None + opus = [(ver(i), i) for i in ids if ver(i) and i.startswith(("us.", "global."))] + if not opus: + print("newer=false"); raise SystemExit(0) + best_ver, best_id = max(opus, key=lambda x: x[0]) + cur = ver(current) + newer = (cur is None) or (best_ver > cur) + print(f"newer={'true' if newer else 'false'}") + print(f"best_id={best_id}") + print(f"best_ver={best_ver[0]}.{best_ver[1]}") + print(f"cur_ver={'unknown' if cur is None else f'{cur[0]}.{cur[1]}'}") + PY + + - name: Open/update issue if a newer model exists + if: steps.check.outputs.newer == 'true' + uses: actions/github-script@v9 + with: + script: | + const best = '${{ steps.check.outputs.best_id }}'; + const bestVer = '${{ steps.check.outputs.best_ver }}'; + const curVer = '${{ steps.check.outputs.cur_ver }}'; + const marker = ''; + const title = `OCR: newer Claude Opus available (${bestVer} > ${curVer})`; + const body = `${marker}\n` + + `A newer Claude Opus inference profile is available on Bedrock.\n\n` + + `- **Configured** (\`vars.OCR_BEDROCK_MODEL\`): Opus ${curVer}\n` + + `- **Newest on Bedrock**: \`${best}\` (Opus ${bestVer})\n\n` + + `To upgrade, set the repo variable:\n\n` + + '```\n' + + `gh variable set OCR_BEDROCK_MODEL -R ${context.repo.owner}/${context.repo.repo} \\\n` + + ` -b "bedrock/converse/${best}"\n` + + '```\n\n' + + `Also confirm the \`ocr-bedrock-ci\` IAM inline policy allows invoking the new model ` + + `(the resource is scoped to \`anthropic.claude-opus-*\`), then re-run OCR.\n\n` + + `_Automated by \`.github/workflows/ocr-model-check.yml\`; this issue is upserted._`; + const q = `repo:${context.repo.owner}/${context.repo.repo} in:body "${marker}" state:open`; + const found = await github.rest.search.issuesAndPullRequests({ q, per_page: 1 }); + if (found.data.total_count > 0) { + const n = found.data.items[0].number; + await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, issue_number: n, title, body }); + } else { + await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title, body }); + } diff --git a/.github/workflows/ocr-review.yml b/.github/workflows/ocr-review.yml new file mode 100644 index 0000000000000..0828af429b57c --- /dev/null +++ b/.github/workflows/ocr-review.yml @@ -0,0 +1,427 @@ +# Open Code Review (OCR) ā€” AI PR review backed by AWS Bedrock via a LiteLLM proxy. +# +# Flow: +# PR opened/updated (incl. DRAFTS) ā”€ā” +# /open-code-review PR comment ā”€ā”¼ā”€ā–ŗ start LiteLLM (127.0.0.1:4000 ā†’ Bedrock) +# manual workflow_dispatch ā”€ā”˜ ā””ā–ŗ ocr review --format json +# ā””ā–ŗ post inline PR review comments +# +# Required (repo settings ā€” all repo *variables*, no secrets; auth is via GitHub OIDC): +# vars.AWS_ROLE_ARN - IAM role to assume via OIDC (granting bedrock:InvokeModel*) +# vars.AWS_REGION - e.g. us-east-1 +# vars.OCR_BEDROCK_MODEL - LiteLLM model string for the Opus inference profile, e.g. +# bedrock/converse/us.anthropic.claude-opus-4-8 +# +# No static AWS keys are stored. GITHUB_TOKEN (auto) posts the review comments. + +name: OCR AI Review + +on: + pull_request: + # Note: no draft filter ā€” drafts are reviewed too. + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + +# One review per PR; cancel superseded runs to save Bedrock spend. +concurrency: + group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + cancel-in-progress: true + +permissions: + id-token: write # required to mint the GitHub OIDC token for AWS role assumption + contents: read + pull-requests: write + +jobs: + ocr-review: + runs-on: ubuntu-latest + # PR events always; comment events only when the comment is on a PR and + # starts with the trigger keyword; manual dispatch always. + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review'))) + + env: + # LiteLLM listens on localhost only; this key never leaves the runner. + LITELLM_MASTER_KEY: sk-ocr-ci-local + OCR_BEDROCK_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + # Region is a static var (safe at job level). AWS credentials are NOT set + # here ā€” they're minted by the OIDC "Configure AWS credentials" step below + # and exported to the environment for the LiteLLM/boto3 Bedrock calls. + AWS_REGION: ${{ vars.AWS_REGION }} + + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') { + prNumber = context.payload.pull_request.number; + } else if (context.eventName === 'issue_comment') { + prNumber = context.issue.number; + } else { + prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + } + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + const { data: repo } = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_ref', pr.head.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('default_branch', repo.default_branch); + core.setOutput('cross_repo', String(pr.head.repo.full_name !== pr.base.repo.full_name)); + + # NOTE: do NOT checkout the PR head. OCR reads the diff and file contents + # straight from git refs (git diff , git show :path, + # git grep ), so the working tree is irrelevant ā€” but our OCR config + # lives on the default branch, not on the PR branch. We check out the repo + # (default ref), fetch the base/head objects, and materialize the config + # from origin/. + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Prepare git refs and OCR config + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_REF: ${{ steps.pr.outputs.head_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + DEFAULT_BRANCH: ${{ steps.pr.outputs.default_branch }} + run: | + git fetch --no-tags origin "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || true + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "+refs/heads/${HEAD_REF}:refs/remotes/origin/${HEAD_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + # OCR config lives on the default branch; materialize it independently + # of whatever ref is checked out. + mkdir -p "$RUNNER_TEMP/ocr" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/litellm.yaml" > "$RUNNER_TEMP/ocr/litellm.yaml" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/rule.json" > "$RUNNER_TEMP/ocr/rule.json" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/context.md" > "$RUNNER_TEMP/ocr/context.md" + echo "Config materialized:"; ls -l "$RUNNER_TEMP/ocr" + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Install LiteLLM proxy + Open Code Review + run: | + python -m pip install --upgrade pip + # Pin LiteLLM to a main commit that supports Claude Opus 4.8 adaptive + # thinking (maps reasoning_effort -> output_config.effort, incl. xhigh). + # Not in any tagged release yet (PyPI latest 1.87.1 lacks the Opus + # normalizer). Bump this SHA once a release ships the feature. + pip install "litellm[proxy] @ git+https://github.com/BerriAI/litellm.git@5be0797d24a2f26eb2123e13788f90055a59d91d" + npm install -g @alibaba-group/open-code-review + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-review-${{ github.run_id }} + + - name: Start LiteLLM proxy (Bedrock bridge) + run: | + if [ -z "$OCR_BEDROCK_MODEL" ]; then + echo "::error::vars.OCR_BEDROCK_MODEL is not set (e.g. bedrock/converse/us.anthropic.claude-opus-4-1-20250805-v1:0)" + exit 1 + fi + nohup litellm --config "$RUNNER_TEMP/ocr/litellm.yaml" --host 127.0.0.1 --port 4000 \ + > /tmp/litellm.log 2>&1 & + echo "Waiting for LiteLLM to become ready..." + for i in $(seq 1 60); do + if curl -sf http://127.0.0.1:4000/health/readiness >/dev/null; then + echo "LiteLLM ready."; exit 0 + fi + sleep 2 + done + echo "::error::LiteLLM did not become ready in time"; cat /tmp/litellm.log; exit 1 + + - name: Configure OCR + run: | + ocr config set llm.url http://127.0.0.1:4000/v1/chat/completions + ocr config set llm.auth_token "$LITELLM_MASTER_KEY" + ocr config set llm.model ocr-bedrock + ocr config set llm.use_anthropic false + ocr config set language English + + - name: Run OCR review + run: | + ocr review \ + --from "origin/${{ steps.pr.outputs.base_ref }}" \ + --to "${{ steps.pr.outputs.head_sha }}" \ + --rule "$RUNNER_TEMP/ocr/rule.json" \ + --background-file "$RUNNER_TEMP/ocr/context.md" \ + --concurrency 3 \ + --timeout 20 \ + --format json \ + > /tmp/ocr-result.json 2>/tmp/ocr-stderr.log || true + echo "----- OCR stdout -----"; cat /tmp/ocr-result.json || true + echo "----- OCR stderr -----"; cat /tmp/ocr-stderr.log || true + echo "----- LiteLLM log (tail) -----"; tail -n 50 /tmp/litellm.log || true + + - name: Post review to PR + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const commitSha = '${{ steps.pr.outputs.head_sha }}'; + + // Opus at high effort can emit dozens of findings. Posting them all + // one-by-one trips GitHub's SECONDARY rate limit (403 "content + // creation"), which is what made every run fail after the review + // was already generated. We (a) cap inline comments and overflow + // the rest into the summary, (b) prefer a single bulk createReview, + // and (c) throttle + back off with Retry-After on any fallback. + const MAX_INLINE = 25; + const sleep = (ms) => new Promise(r => setTimeout(r, ms)); + + async function withRetry(fn, label) { + for (let attempt = 1; attempt <= 5; attempt++) { + try { return await fn(); } + catch (e) { + const status = e.status || (e.response && e.response.status); + const h = (e.response && e.response.headers) || {}; + const isRate = status === 403 || status === 429; + if (!isRate || attempt === 5) throw e; + let waitMs = 0; + if (h['retry-after']) waitMs = parseInt(h['retry-after'], 10) * 1000; + else if (h['x-ratelimit-reset']) waitMs = parseInt(h['x-ratelimit-reset'], 10) * 1000 - Date.now(); + if (!waitMs || Number.isNaN(waitMs) || waitMs < 0) waitMs = 1000 * Math.pow(2, attempt); + waitMs = Math.min(waitMs, 60000) + 500; + core.warning(`${label}: rate-limited (status ${status}); waiting ${Math.round(waitMs / 1000)}s (attempt ${attempt}/5)`); + await sleep(waitMs); + } + } + } + + let result; + try { + result = JSON.parse(fs.readFileSync('/tmp/ocr-result.json', 'utf8')); + } catch (e) { + const stderr = (() => { try { return fs.readFileSync('/tmp/ocr-stderr.log', 'utf8').trim(); } catch { return ''; } })(); + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `āš ļø **OCR** could not produce a review.\n\n\`\`\`\n${(stderr || e.message).slice(0, 8000)}\n\`\`\``, + }), 'error-comment'); + return; + } + + const comments = result.comments || []; + const warnings = result.warnings || []; + + const formatComment = (c) => { + let body = c.content || ''; + if (c.suggestion_code && c.existing_code) { + body += '\n\n```suggestion\n' + c.suggestion_code + (c.suggestion_code.endsWith('\n') ? '' : '\n') + '```'; + } + return body; + }; + const formatMarkdown = (c) => { + let md = `### šŸ“„ \`${c.path}\``; + if (c.start_line && c.end_line) md += ` (L${c.start_line}-L${c.end_line})`; + md += '\n\n' + (c.content || ''); + if (c.suggestion_code && c.existing_code) { + md += '\n\n
šŸ’” Suggested change\n\n'; + md += '**Before:**\n```\n' + c.existing_code + '\n```\n\n**After:**\n```\n' + c.suggestion_code + '\n```\n\n
'; + } + return md; + }; + + if (comments.length === 0) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `āœ… **OCR**: ${result.message || 'No issues found.'}`, + }), 'no-issues-comment'); + return; + } + + const inlineAll = []; + const noLine = []; + for (const c of comments) { + const body = formatComment(c); + const hasLine = (c.start_line >= 1) || (c.end_line >= 1); + if (!hasLine) { noLine.push(c); continue; } + const rc = { path: c.path, body, side: 'RIGHT' }; + if (c.start_line >= 1 && c.end_line >= 1 && c.start_line !== c.end_line) { + rc.start_line = c.start_line; rc.line = c.end_line; rc.start_side = 'RIGHT'; + } else { + rc.line = c.end_line >= 1 ? c.end_line : c.start_line; + } + inlineAll.push({ rc, c }); + } + + const inline = inlineAll.slice(0, MAX_INLINE).map(x => x.rc); + const overflow = inlineAll.slice(MAX_INLINE).map(x => x.c); + + let summary = `šŸ” **OCR** found **${comments.length}** issue(s).`; + summary += `\n- ${inline.length} inline, ${noLine.length + overflow.length} in summary`; + if (overflow.length) summary += ` (inline capped at ${MAX_INLINE})`; + if (warnings.length) summary += `\n- āš ļø ${warnings.length} warning(s) during review`; + for (const c of noLine.concat(overflow)) summary += '\n\n---\n\n' + formatMarkdown(c); + + // Preferred path: ONE createReview carrying every inline comment. + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', comments: inline, + }), 'bulk-review'); + return; + } catch (e) { + core.warning(`bulk createReview failed (${e.status || '?'}: ${e.message}); falling back to throttled per-comment posting`); + } + + // Fallback: an invalid inline position (line not in the diff -> 422) + // rejects the whole bulk review. Post the summary, then each comment + // individually with a delay + backoff, skipping ones GitHub rejects. + let ok = 0; const failed = []; + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', + }), 'summary-review'); + } catch (err) { failed.push(`summary: ${err.message}`); } + + for (const rc of inline) { + try { + await withRetry(() => github.rest.pulls.createReviewComment({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, path: rc.path, body: rc.body, + ...(rc.start_line ? { start_line: rc.start_line, start_side: rc.start_side } : {}), + line: rc.line, side: rc.side, + }), `comment ${rc.path}:${rc.line}`); + ok++; + } catch (inner) { + failed.push(`\`${rc.path}\` L${rc.line}: ${inner.message}`); + } + await sleep(1200); // stay under the secondary content-creation limit + } + + if (failed.length) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `šŸ“Š OCR posted ${ok}/${inline.length} inline comment(s).\n\n
${failed.length} could not be posted\n\n${failed.join('\n')}\n
`, + }), 'summary-failures'); + } + + # Companion job: OCR can't call MCP, so this separate agent ties the PR's + # changes to PostgreSQL git + pgsql-hackers history via the Agora MCP server + # (pg.ddx.io) and posts a single, upserted "history & discussion" comment. + pg-history: + runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review') || + startsWith(github.event.comment.body, '/pg-history'))) + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') prNumber = context.payload.pull_request.number; + else if (context.eventName === 'issue_comment') prNumber = context.issue.number; + else prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('title', pr.title || ''); + + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Make base/head refs available + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: | + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: pg-history-${{ github.run_id }} + + - name: Install deps + run: pip install boto3 + + - name: Run pg-history (Agora MCP) + env: + PG_HISTORY_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + GH_PR_TITLE: ${{ steps.pr.outputs.title }} + PG_HISTORY_OUT: ${{ runner.temp }}/pg-history.md + run: | + python .github/ocr/pg-history.py || true + echo "----- output -----"; cat "${{ runner.temp }}/pg-history.md" 2>/dev/null || echo "(no output)" + + - name: Upsert PR comment + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const path = process.env.RUNNER_TEMP + '/pg-history.md'; + let body = ''; + try { body = fs.readFileSync(path, 'utf8').trim(); } catch (e) {} + if (!body) { console.log('pg-history: empty output, nothing to post'); return; } + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const marker = ''; + body = marker + '\n' + body; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, per_page: 100 }); + const mine = comments.find(c => c.user.type === 'Bot' && c.body && c.body.includes(marker)); + if (mine) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: mine.id, body }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body }); + } From 0d8492a6b59fd0acfc2ebc75ebfbd8e8842ee3ac Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 6 Jul 2026 17:25:00 -0400 Subject: [PATCH 3/5] Batch the clock sweep to reduce nextVictimBuffer atomic contention StrategyGetBuffer() advances the shared clock hand, nextVictimBuffer, with a pg_atomic_fetch_add_u32(..., 1) on every tick. On a multi-socket system the cache line holding that counter has to travel over the interconnect on each operation, pushing a sweep tick from ~20ns (same-socket, line warm in L1/L2) into the ~100-200ns range. Under eviction pressure with hundreds of backends in StrategyGetBuffer() concurrently, that single cache line becomes the dominant cost of the sweep, visible as elevated bus-cycles and cache-misses in a perf profile. Have each backend claim a run of consecutive buffer IDs from the shared hand with a single fetch-add and then iterate through them privately. The sweep still advances through the pool in order, each buffer is still visited exactly once per complete pass, and the meaning of the clock state is unchanged; only the temporal ordering of visits within a pass changes, which the algorithm does not depend on. The contended atomic now fires roughly once per batch rather than once per buffer. The batch is one cache line's worth of clock-hand values -- PG_CACHE_LINE_SIZE / sizeof(uint32) -- capped at NBuffers so a claim can never wrap the pool more than once. Batching only helps when the counter's cache line actually bounces between sockets, so it is enabled only on multi-node NUMA hardware (pg_numa_get_max_node() >= 1); on a single socket, or where libnuma is unavailable, the batch size stays 1 and the code path is byte-identical to the stock clock sweep. Wraparound handling is adjusted: with batching, several backends can each see a fetch-add return a value past NBuffers within the same pass. Any such backend takes buffer_strategy_lock, re-reads the counter, and if it is still out of range wraps it with a single CAS and increments completePasses. StrategySyncStart() continues to see a consistent (nextVictimBuffer, completePasses) pair. This is the batched-clock-sweep idea from Jim Mlodgenski's pgsql-hackers thread, adapted to derive the batch size from the platform cache-line size. Co-authored-by: Jim Mlodgenski --- src/backend/storage/buffer/freelist.c | 152 ++++++++++++++++++-------- 1 file changed, 109 insertions(+), 43 deletions(-) diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c index fdb5bad7910a2..46104b3341d61 100644 --- a/src/backend/storage/buffer/freelist.c +++ b/src/backend/storage/buffer/freelist.c @@ -22,6 +22,7 @@ #include "storage/proc.h" #include "storage/shmem.h" #include "storage/subsystems.h" +#include "port/pg_numa.h" #define INT_ACCESS_ONCE(var) ((int)(*((volatile int *)&(var)))) @@ -100,68 +101,110 @@ static BufferDesc *GetBufferFromRing(BufferAccessStrategy strategy, static void AddBufferToRing(BufferAccessStrategy strategy, BufferDesc *buf); +/* + * Per-backend state for the batched clock sweep. Each backend claims a run + * of consecutive clock-hand values with a single atomic fetch-add and then + * iterates through them privately, so the contended nextVictimBuffer cache + * line is touched roughly 1/batch as often. MyBatchPos is the next hand + * value to hand out; MyBatchEnd is one past the end of the claimed run. Both + * are absolute (monotonically increasing) hand values; the buffer id is the + * value modulo NBuffers. + */ +static uint32 MyBatchPos = 0; +static uint32 MyBatchEnd = 0; + +/* + * Number of clock-hand values a backend claims per atomic fetch-add, + * computed once at startup (see StrategyCtlShmemInit). When batching is + * enabled it is one cache line's worth of hand advance, so concurrent + * backends sweep non-overlapping, cache-line-sized runs of the pool; the + * global sweep order is preserved (each buffer is still visited exactly once + * per pass). Batching is enabled only on multi-node NUMA hardware; otherwise + * this stays 1 and the sweep is byte-identical to the stock clock. + */ +static uint32 ClockSweepBatchSize = 1; + /* * ClockSweepTick - Helper routine for StrategyGetBuffer() * - * Move the clock hand one buffer ahead of its current position and return the - * id of the buffer now under the hand. + * Return the next buffer to consider for eviction. Backends claim batches of + * consecutive buffer IDs from the shared clock hand, then iterate through + * them locally without further atomic operations. This preserves the global + * sweep order while reducing contention on the shared counter. */ static inline uint32 ClockSweepTick(void) { uint32 victim; - /* - * Atomically move hand ahead one buffer - if there's several processes - * doing this, this can lead to buffers being returned slightly out of - * apparent order. - */ - victim = - pg_atomic_fetch_add_u32(&StrategyControl->nextVictimBuffer, 1); - - if (victim >= NBuffers) + if (MyBatchPos >= MyBatchEnd) { - uint32 originalVictim = victim; - - /* always wrap what we look up in BufferDescriptors */ - victim = victim % NBuffers; - /* - * If we're the one that just caused a wraparound, force - * completePasses to be incremented while holding the spinlock. We - * need the spinlock so StrategySyncStart() can return a consistent - * value consisting of nextVictimBuffer and completePasses. + * Claim a fresh batch from the shared clock hand. This is the only + * atomic operation per batch, reducing contention by the batch size. */ - if (victim == 0) - { - uint32 expected; - uint32 wrapped; - bool success = false; + uint32 start; + uint32 batch_size = ClockSweepBatchSize; - expected = originalVictim + 1; + start = pg_atomic_fetch_add_u32(&StrategyControl->nextVictimBuffer, + batch_size); - while (!success) - { - /* - * Acquire the spinlock while increasing completePasses. That - * allows other readers to read nextVictimBuffer and - * completePasses in a consistent manner which is required for - * StrategySyncStart(). In theory delaying the increment - * could lead to an overflow of nextVictimBuffers, but that's - * highly unlikely and wouldn't be particularly harmful. - */ - SpinLockAcquire(&StrategyControl->buffer_strategy_lock); + if (start >= (uint32) NBuffers) + { + start = start % NBuffers; - wrapped = expected % NBuffers; + /* + * The counter has grown past NBuffers; try to wrap it back. We + * must hold the spinlock so StrategySyncStart() can read + * nextVictimBuffer and completePasses consistently. + * + * With batching, multiple backends may each land a fetch-add + * that returns a value past NBuffers in the same pass. After + * acquiring the spinlock we re-read the counter: if another + * backend already wrapped it below NBuffers we are done. + */ + SpinLockAcquire(&StrategyControl->buffer_strategy_lock); + { + uint32 current; + uint32 wrapped; - success = pg_atomic_compare_exchange_u32(&StrategyControl->nextVictimBuffer, - &expected, wrapped); - if (success) - StrategyControl->completePasses++; - SpinLockRelease(&StrategyControl->buffer_strategy_lock); + current = pg_atomic_read_u32(&StrategyControl->nextVictimBuffer); + if (current >= (uint32) NBuffers) + { + wrapped = current % NBuffers; + if (pg_atomic_compare_exchange_u32(&StrategyControl->nextVictimBuffer, + ¤t, wrapped)) + StrategyControl->completePasses++; + } } + SpinLockRelease(&StrategyControl->buffer_strategy_lock); + } + else if (start + batch_size > (uint32) NBuffers) + { + /* + * The fetch-add returned a value below NBuffers, but this batch + * spans the wrap point (start .. NBuffers-1 then 0 .. k are + * iterated locally via "% NBuffers"). That crossing is a + * complete pass too, but the wrap-and-increment path above only + * fires when the fetch-add return is itself >= NBuffers, so + * without this it would go uncounted and completePasses would + * drift low (it feeds bgwriter pacing / StrategySyncStart, not + * correctness). The batch is capped at NBuffers, so a batch + * spans the wrap at most once; count it under the spinlock for a + * consistent (nextVictimBuffer, completePasses) pair. + */ + SpinLockAcquire(&StrategyControl->buffer_strategy_lock); + StrategyControl->completePasses++; + SpinLockRelease(&StrategyControl->buffer_strategy_lock); } + + MyBatchPos = start; + MyBatchEnd = start + batch_size; } + + victim = MyBatchPos % NBuffers; + MyBatchPos++; + return victim; } @@ -408,6 +451,29 @@ StrategyCtlShmemInit(void *arg) /* No pending notification */ StrategyControl->bgwprocno = -1; + + /* + * Decide whether to batch the clock sweep. + * + * Batching claims a run of consecutive buffer IDs per atomic fetch-add so + * concurrent backends touch the shared nextVictimBuffer cache line ~1/batch + * as often -- a win only when that line actually bounces across sockets, + * i.e. on multi-node NUMA hardware. On a single socket the atomic is + * already node-local and batching would only make backends skip ahead for + * no benefit, so we fall back to batch size 1 there (byte-identical to the + * stock one-buffer-at-a-time clock sweep). + * + * Batch (> 1) only when libnuma reports more than one node + * (pg_numa_get_max_node() >= 1); pg_numa_init() returns -1 when NUMA is + * unavailable. (Benchmarking showed the win holds with or without huge + * pages, so huge-page availability is intentionally not part of the gate.) + */ + if (pg_numa_init() != -1 && + pg_numa_get_max_node() >= 1) + ClockSweepBatchSize = Min(PG_CACHE_LINE_SIZE / (uint32) sizeof(uint32), + (uint32) NBuffers); + else + ClockSweepBatchSize = 1; } From 826edb64e97e3ba2d1e2d1aedd5e76dec98fff2d Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 6 Jul 2026 19:22:45 -0400 Subject: [PATCH 4/5] Replace the usage_count clock sweep with a cooling-stage evictor Replace the 0..5 usage_count buffer-replacement policy with a cooling-stage clock (the LeanStore / 2Q-A1 model): a buffer is simply HOT (recently used) or COOL (an eviction candidate), with "pinned" being the existing refcount. There is no per-buffer access counter. - A demand-loaded page is admitted COOL (probationary), not HOT. A second access via PinBuffer promotes it COOL -> HOT (the rescue). So a page touched once -- a sequential scan -- fills and drains the COOL stage and is evicted from it without ever displacing the HOT working set. Scan resistance is intrinsic to the replacement algorithm, which is what lets a later commit remove the BufferAccessStrategy ring buffers entirely. - The foreground sweep in StrategyGetBuffer() reclaims an already-COOL, unpinned buffer, pinning it with a CAS so a racing PinBuffer always wins. Promotion (COOL -> HOT) and demotion (HOT -> COOL) are single-bit transitions; only the eviction claim is a CAS. - The background writer maintains the supply of COOL victims. As its LRU scan runs ahead of the clock hand it demotes HOT buffers to COOL, so the foreground finds a victim in a single pass rather than having to cool buffers itself. The demotion is demand-driven -- bounded by the predicted allocation for the next cycle -- so it stages just enough COOL buffers without cooling the whole pool, and it is done under the buffer header lock the scan already holds. A single second-chance reference bit (set by PinBuffer, cleared on the bgwriter's first pass over a HOT buffer) spares a recently-accessed buffer one cooling pass, keeping the genuinely hot set out of the COOL stage under scan pressure. BgBufferSync also tracks the reusable-buffer density it directly observes on a shorter smoothing window, so a burst of probationary/scan COOL pages is followed promptly rather than averaged away. Under a bulk-dirtying workload the per-cycle clean-write cap is raised from bgwriter_lru_maxpages to predicted demand so the bgwriter keeps supplying clean victims, rather than the foreground sweep having to flush dirty victims inline; normal workloads, where demand is below the cap, are unaffected. The 4-bit usage_count field is reinterpreted in place: bit 0 is the HOT/COOL state (BUF_COOLSTATE_ONE), bit 1 the reference bit (BUF_REFBIT). The 64-bit buffer-state layout -- refcount, flag and lock offsets and their StaticAsserts -- is unchanged; only the meaning of the field and the instructions that touch it change. A StaticAssert requires the field to be at least 2 bits wide so a future width change cannot push the reference bit into the flag bits. BM_MAX_USAGE_COUNT becomes BUF_COOLSTATE_HOT (1), so the pin fast path saturates at HOT. Local (temp-table) buffers get the same two-state treatment; being single-backend they need no background cooler. Because the reference bit shares the field, the full 4-bit value can be 0..3; readers that mean "the cooling state" must use BUF_STATE_GET_COOLSTATE(), which masks to bit 0 and returns only 0 (COOL) or 1 (HOT), never the raw field. contrib/pg_buffercache is updated accordingly: its usagecount column and the pg_buffercache_summary average report the cooling state (0 = COOL, 1 = HOT), and pg_buffercache_usage_counts() buckets on it. Using the raw 4-bit getter there would index its BM_MAX_USAGE_COUNT+1 = 2-element arrays with values up to 3 and overrun the stack; masking to the cooling bit keeps the index in range. The reference bit is deliberately not exposed as usagecount. Depends on the batched clock sweep from the previous commit. --- contrib/pg_buffercache/pg_buffercache_pages.c | 6 +- src/backend/storage/buffer/bufmgr.c | 124 +++++++++++---- src/backend/storage/buffer/freelist.c | 141 ++++++++++++------ src/backend/storage/buffer/localbuf.c | 13 +- src/include/storage/buf_internals.h | 63 +++++++- 5 files changed, 260 insertions(+), 87 deletions(-) diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 510455998aa74..b94e6f6fda80b 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -161,7 +161,7 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) reldatabase = bufHdr->tag.dbOid; forknum = BufTagGetForkNum(&bufHdr->tag); blocknum = bufHdr->tag.blockNum; - usagecount = BUF_STATE_GET_USAGECOUNT(buf_state); + usagecount = BUF_STATE_GET_COOLSTATE(buf_state); pinning_backends = BUF_STATE_GET_REFCOUNT(buf_state); if (buf_state & BM_DIRTY) @@ -605,7 +605,7 @@ pg_buffercache_summary(PG_FUNCTION_ARGS) if (buf_state & BM_VALID) { buffers_used++; - usagecount_total += BUF_STATE_GET_USAGECOUNT(buf_state); + usagecount_total += BUF_STATE_GET_COOLSTATE(buf_state); if (buf_state & BM_DIRTY) buffers_dirty++; @@ -655,7 +655,7 @@ pg_buffercache_usage_counts(PG_FUNCTION_ARGS) CHECK_FOR_INTERRUPTS(); - usage_count = BUF_STATE_GET_USAGECOUNT(buf_state); + usage_count = BUF_STATE_GET_COOLSTATE(buf_state); usage_counts[usage_count]++; if (buf_state & BM_DIRTY) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 3908529872a31..1f5a00849e6aa 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -83,6 +83,7 @@ /* Bits in SyncOneBuffer's return value */ #define BUF_WRITTEN 0x01 #define BUF_REUSABLE 0x02 +#define BUF_COOLED 0x04 #define RELS_BSEARCH_THRESHOLD 20 @@ -634,7 +635,7 @@ static void PinBuffer_Locked(BufferDesc *buf); static void UnpinBuffer(BufferDesc *buf); static void UnpinBufferNoOwner(BufferDesc *buf); static void BufferSync(int flags); -static int SyncOneBuffer(int buf_id, bool skip_recently_used, +static int SyncOneBuffer(int buf_id, bool skip_recently_used, bool cool_if_hot, WritebackContext *wb_context); static void WaitIO(BufferDesc *buf); static void AbortBufferIO(Buffer buffer); @@ -2333,7 +2334,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * checkpoints, except for their "init" forks, which need to be treated * just like permanent relations. */ - set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + set_bits |= BM_TAG_VALID; + /* Admit the newly loaded page COOL (probation); a second access via + * PinBuffer promotes it to HOT. This is what makes a one-touch scan + * self-evicting -- see the cooling-state notes in buf_internals.h. */ if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -3002,7 +3006,9 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; - set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + set_bits |= BM_TAG_VALID; + /* Admit COOL (probation); see the comment at the other admission + * site and the cooling-state notes in buf_internals.h. */ if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -3332,21 +3338,17 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy, /* increase refcount */ buf_state += BUF_REFCOUNT_ONE; - if (strategy == NULL) - { - /* Default case: increase usagecount unless already max. */ - if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) - buf_state += BUF_USAGECOUNT_ONE; - } - else - { - /* - * Ring buffers shouldn't evict others from pool. Thus we - * don't make usagecount more than 1. - */ - if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0) - buf_state += BUF_USAGECOUNT_ONE; - } + /* + * Accessing a resident buffer promotes it to HOT (the 2Q rescue): + * a page loaded COOL on probation becomes part of the hot working + * set on its second touch. BM_MAX_USAGE_COUNT is + * BUF_COOLSTATE_HOT (1), so this saturates at HOT and never + * overflows the field. We also set the second-chance ref bit so + * the bgwriter's next cooling pass spares this recently-used buffer. + */ + if (BUF_STATE_GET_COOLSTATE(buf_state) < BUF_COOLSTATE_HOT) + buf_state += BUF_COOLSTATE_ONE; + buf_state |= BUF_REFBIT; if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, buf_state)) @@ -3785,7 +3787,7 @@ BufferSync(int flags) */ if (pg_atomic_read_u64(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { - if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) + if (SyncOneBuffer(buf_id, false, false, &wb_context) & BUF_WRITTEN) { TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); PendingCheckpointerStats.buffers_written++; @@ -3876,6 +3878,19 @@ BgBufferSync(WritebackContext *wb_context) float smoothing_samples = 16; float scan_whole_pool_milliseconds = 120000.0; + /* + * The cleaner scan directly observes the reusable (COOL, unpinned) buffer + * density over the region it walks, which -- with the cooling-stage + * evictor -- is exactly the sweep's victim predicate. That observation is + * ground truth for the buffers about to be reused, whereas the strategy + * scan's positional proxy (strategy_delta/recent_alloc) blurs a pool whose + * COOL population is spatially clustered (a scan burst leaves whole regions + * COOL, hot OLTP regions not). So we let the cleaner's own sample adapt on + * a shorter window than the strategy proxy, tracking a burst of + * probationary/scan COOL pages within a cycle or two instead of lagging it. + */ + float cleaner_smoothing_samples = 4; + /* Used to compute how far we scan ahead */ long strategy_delta; int bufs_to_lap; @@ -3889,6 +3904,7 @@ BgBufferSync(WritebackContext *wb_context) int num_to_scan; int num_written; int reusable_buffers; + int write_limit; /* Variables for final smoothed_density update */ long new_strategy_delta; @@ -4063,8 +4079,22 @@ BgBufferSync(WritebackContext *wb_context) * Now write out dirty reusable buffers, working forward from the * next_to_clean point, until we have lapped the strategy scan, or cleaned * enough buffers to match our estimate of the next cycle's allocation - * requirements, or hit the bgwriter_lru_maxpages limit. - */ + * requirements, or hit the write limit. + * + * The per-cycle write cap is normally bgwriter_lru_maxpages. But under a + * bulk-dirtying workload (COPY, bulk UPDATE, VACUUM) the pool fills with + * dirty COOL buffers faster than that fixed cap can clean, so the + * foreground clock sweep is forced to flush dirty victims inline -- the + * very cost the cooling-stage evictor is meant to keep off the critical + * path. When predicted demand (upcoming_alloc_est) exceeds the fixed cap, + * raise the limit to meet demand so the bgwriter stays ahead and supplies + * clean victims. This stays bounded (by demand and by lapping the + * strategy point), so it cannot run away, and normal workloads are + * unaffected because there upcoming_alloc_est <= bgwriter_lru_maxpages. + */ + write_limit = bgwriter_lru_maxpages; + if (upcoming_alloc_est > write_limit) + write_limit = upcoming_alloc_est; num_to_scan = bufs_to_lap; num_written = 0; @@ -4073,7 +4103,7 @@ BgBufferSync(WritebackContext *wb_context) /* Execute the LRU scan */ while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) { - int sync_state = SyncOneBuffer(next_to_clean, true, + int sync_state = SyncOneBuffer(next_to_clean, true, true, wb_context); if (++next_to_clean >= NBuffers) @@ -4086,7 +4116,7 @@ BgBufferSync(WritebackContext *wb_context) if (sync_state & BUF_WRITTEN) { reusable_buffers++; - if (++num_written >= bgwriter_lru_maxpages) + if (++num_written >= write_limit) { PendingBgWriterStats.maxwritten_clean++; break; @@ -4121,7 +4151,7 @@ BgBufferSync(WritebackContext *wb_context) { scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc; smoothed_density += (scans_per_alloc - smoothed_density) / - smoothing_samples; + cleaner_smoothing_samples; #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f", @@ -4140,16 +4170,26 @@ BgBufferSync(WritebackContext *wb_context) * If skip_recently_used is true, we don't write currently-pinned buffers, nor * buffers marked recently used, as these are not replacement candidates. * + * If cool_if_hot is true (the bgwriter's LRU scan), an unpinned HOT buffer is + * demoted HOT -> COOL as we pass it, pre-staging eviction candidates so the + * foreground clock sweep finds a COOL victim in a single pass instead of + * having to cool buffers itself (force_cool). The demotion is done under the + * buffer header lock we already hold, so it needs no CAS and cannot race a + * concurrent demotion. A concurrent PinBuffer promotes it back to HOT, which + * is the intended 2Q behavior (a re-accessed buffer is rescued). + * * Returns a bitmask containing the following flag bits: * BUF_WRITTEN: we wrote the buffer. * BUF_REUSABLE: buffer is available for replacement, ie, it has - * pin count 0 and usage count 0. + * pin count 0 and is COOL (an eviction candidate). + * BUF_COOLED: we demoted this buffer HOT -> COOL this call. * * (BUF_WRITTEN could be set in error if FlushBuffer finds the buffer clean * after locking it, but we don't care all that much.) */ static int -SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) +SyncOneBuffer(int buf_id, bool skip_recently_used, bool cool_if_hot, + WritebackContext *wb_context) { BufferDesc *bufHdr = GetBufferDescriptor(buf_id); int result = 0; @@ -4171,8 +4211,38 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) */ buf_state = LockBufHdr(bufHdr); + /* + * Pre-cool with a second chance: if asked, act on an unpinned HOT buffer. + * If its ref bit is set (accessed since our last pass), clear the ref bit + * and leave it HOT -- a recently-used buffer earns one reprieve, keeping + * the hot working set out of the COOL stage under scan pressure. Only a + * HOT buffer whose ref bit is already clear is demoted HOT -> COOL, + * pre-staging it as an eviction candidate for the foreground sweep. We + * hold the header lock, so each transition is a plain masked store applied + * atomically by UnlockBufHdrExt; we re-lock to continue the dirty-write + * inspection below. + */ + if (cool_if_hot && + BUF_STATE_GET_REFCOUNT(buf_state) == 0 && + BUF_STATE_GET_COOLSTATE(buf_state) != BUF_COOLSTATE_COOL) + { + if (BUF_STATE_GET_REFBIT(buf_state)) + { + /* second chance: consume the ref bit, stay HOT */ + UnlockBufHdrExt(bufHdr, buf_state, 0, BUF_REFBIT, 0); + buf_state = LockBufHdr(bufHdr); + } + else + { + /* not re-accessed since last pass: demote to COOL */ + UnlockBufHdrExt(bufHdr, buf_state, 0, BUF_USAGECOUNT_MASK, 0); + buf_state = LockBufHdr(bufHdr); + result |= BUF_COOLED; + } + } + if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && - BUF_STATE_GET_USAGECOUNT(buf_state) == 0) + BUF_STATE_GET_COOLSTATE(buf_state) == BUF_COOLSTATE_COOL) { result |= BUF_REUSABLE; } diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c index 46104b3341d61..e8cf13f3362fb 100644 --- a/src/backend/storage/buffer/freelist.c +++ b/src/backend/storage/buffer/freelist.c @@ -49,6 +49,16 @@ typedef struct uint32 completePasses; /* Complete cycles of the clock-sweep */ pg_atomic_uint32 numBufferAllocs; /* Buffers allocated since last reset */ + /* + * Number of clock-hand values a backend claims per atomic fetch-add. + * Computed once at startup (see StrategyCtlShmemInit). Kept in shared + * memory rather than a backend-local static so that EXEC_BACKEND children + * (which do not inherit the postmaster's statics) see the same value; a + * backend-local copy would silently reset to 1 there, disabling batching + * on Windows. + */ + uint32 batchSize; + /* * Bgworker process to be notified upon activity or -1 if none. See * StrategyNotifyBgWriter. @@ -113,17 +123,6 @@ static void AddBufferToRing(BufferAccessStrategy strategy, static uint32 MyBatchPos = 0; static uint32 MyBatchEnd = 0; -/* - * Number of clock-hand values a backend claims per atomic fetch-add, - * computed once at startup (see StrategyCtlShmemInit). When batching is - * enabled it is one cache line's worth of hand advance, so concurrent - * backends sweep non-overlapping, cache-line-sized runs of the pool; the - * global sweep order is preserved (each buffer is still visited exactly once - * per pass). Batching is enabled only on multi-node NUMA hardware; otherwise - * this stays 1 and the sweep is byte-identical to the stock clock. - */ -static uint32 ClockSweepBatchSize = 1; - /* * ClockSweepTick - Helper routine for StrategyGetBuffer() * @@ -144,7 +143,7 @@ ClockSweepTick(void) * atomic operation per batch, reducing contention by the batch size. */ uint32 start; - uint32 batch_size = ClockSweepBatchSize; + uint32 batch_size = StrategyControl->batchSize; start = pg_atomic_fetch_add_u32(&StrategyControl->nextVictimBuffer, batch_size); @@ -229,6 +228,7 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r BufferDesc *buf; int bgwprocno; int trycounter; + bool force_cool; *from_ring = false; @@ -279,12 +279,32 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r */ pg_atomic_fetch_add_u32(&StrategyControl->numBufferAllocs, 1); - /* Use the "clock sweep" algorithm to find a free buffer */ + /* + * Use the cooling-stage clock sweep to find a victim. + * + * A buffer is HOT (recently used) or COOL (an eviction candidate). We + * prefer to reclaim an already-COOL buffer and demote a HOT buffer to + * COOL only once a full sweep has found no COOL victim (force_cool) -- so + * an abundant supply of COOL/probationary pages (e.g. a scan) is drained + * before the hot working set is cooled. Newly loaded pages are admitted + * COOL (see BufferAlloc), so scan resistance falls out of the algorithm. + * + * trycounter bounds the search. Any tick that does not produce a victim + * and does not make progress -- a pinned buffer, or a HOT buffer skipped + * on a prefer-COOL pass -- decrements it. Cooling a HOT buffer (under + * force_cool) is progress and resets it. When a full pass (NBuffers) + * makes no progress we escalate to force_cool so the next pass cools HOT + * buffers into victims; if a force_cool pass ALSO makes no progress every + * buffer is pinned and we fail, matching the stock "no unpinned buffers + * available" contract. + */ trycounter = NBuffers; + force_cool = false; for (;;) { uint64 old_buf_state; uint64 local_buf_state; + bool no_progress = false; buf = GetBufferDescriptor(ClockSweepTick()); @@ -297,25 +317,10 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r { local_buf_state = old_buf_state; - /* - * If the buffer is pinned or has a nonzero usage_count, we cannot - * use it; decrement the usage_count (unless pinned) and keep - * scanning. - */ - + /* If the buffer is pinned we cannot use it; keep scanning. */ if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0) { - if (--trycounter == 0) - { - /* - * We've scanned all the buffers without making any state - * changes, so all the buffers are pinned (or were when we - * looked at them). We could hope that someone will free - * one eventually, but it's probably better to fail than - * to risk getting stuck in an infinite loop. - */ - elog(ERROR, "no unpinned buffers available"); - } + no_progress = true; break; } @@ -326,20 +331,56 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r continue; } - if (BUF_STATE_GET_USAGECOUNT(local_buf_state) != 0) + if (BUF_STATE_GET_COOLSTATE(local_buf_state) != BUF_COOLSTATE_COOL) { - local_buf_state -= BUF_USAGECOUNT_ONE; + /* + * HOT buffer. Prefer a COOL victim: on a normal pass just + * advance the hand (no progress). Under force_cool apply the + * same second-chance rule the bgwriter's pre-cooling uses: a + * HOT buffer whose ref bit is set (recently accessed) has the + * ref bit cleared and is left HOT this pass; only a HOT buffer + * whose ref bit is already clear is demoted HOT -> COOL. This + * keeps a just-touched buffer from being cooled the instant the + * bgwriter falls behind and the foreground has to cool for + * itself. Either transition is progress toward a victim. + */ + if (!force_cool) + { + no_progress = true; + break; /* advance the hand, look for COOL */ + } + + if (BUF_STATE_GET_REFBIT(local_buf_state)) + local_buf_state &= ~BUF_REFBIT; /* second chance: clear ref, stay HOT */ + else + local_buf_state &= ~BUF_USAGECOUNT_MASK; /* HOT -> COOL, clear ref bit */ if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, local_buf_state)) { + /* + * Making a cooling-state transition is progress toward a + * victim, so reset the counter. Stay in force_cool: we made + * a transition but have not yet produced a reclaimable COOL + * victim (a ref-clear leaves the buffer HOT; a demote leaves + * it COOL for a later tick to claim), and dropping out here + * would waste a full no-progress pass re-escalating. An + * all-HOT, all-recently-referenced pool thus takes up to ~3 + * full passes to yield a victim (discover no COOL, clear ref + * bits, cool + reclaim) -- still within the stock clock's + * worst case (up to BM_MAX_USAGE_COUNT+1 passes), and in + * practice the bgwriter pre-cooling keeps a COOL victim + * available so force_cool rarely fires at all. force_cool + * ends naturally once a COOL victim is found and returned + * below. + */ trycounter = NBuffers; break; } } else { - /* pin the buffer if the CAS succeeds */ + /* COOL and unpinned: claim it. Pin if the CAS succeeds. */ local_buf_state += BUF_REFCOUNT_ONE; if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, @@ -356,6 +397,21 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r } } } + + /* + * A tick that made no progress toward a victim counts down trycounter. + * A full unproductive pass escalates to force_cool (cool HOT buffers + * into victims); a second unproductive full pass means everything is + * pinned, so fail rather than spin forever. (A failed CAS above is + * neither progress nor a full miss: we simply retry the same buffer.) + */ + if (no_progress && --trycounter == 0) + { + if (force_cool) + elog(ERROR, "no unpinned buffers available"); + force_cool = true; + trycounter = NBuffers; + } } } @@ -470,10 +526,10 @@ StrategyCtlShmemInit(void *arg) */ if (pg_numa_init() != -1 && pg_numa_get_max_node() >= 1) - ClockSweepBatchSize = Min(PG_CACHE_LINE_SIZE / (uint32) sizeof(uint32), - (uint32) NBuffers); + StrategyControl->batchSize = Min(PG_CACHE_LINE_SIZE / (uint32) sizeof(uint32), + (uint32) NBuffers); else - ClockSweepBatchSize = 1; + StrategyControl->batchSize = 1; } @@ -721,14 +777,13 @@ GetBufferFromRing(BufferAccessStrategy strategy, uint64 *buf_state) /* * If the buffer is pinned we cannot use it under any circumstances. * - * If usage_count is 0 or 1 then the buffer is fair game (we expect 1, - * since our own previous usage of the ring element would have left it - * there, but it might've been decremented by clock-sweep since then). - * A higher usage_count indicates someone else has touched the buffer, - * so we shouldn't re-use it. + * With the cooling-state replacement the field holds only COOL or HOT, + * so the stock "usage_count > 1 means another backend touched it" + * heuristic no longer applies: a ring element is reusable whenever it + * is unpinned. (The whole ring mechanism is removed in a later patch; + * scan resistance is now intrinsic to the sweep.) */ - if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0 - || BUF_STATE_GET_USAGECOUNT(local_buf_state) > 1) + if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0) break; /* See equivalent code in PinBuffer() */ diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index 4870c8e13d010..21c08091c490b 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -167,7 +167,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, buf_state = pg_atomic_read_u64(&bufHdr->state); buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK); - buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + buf_state |= BM_TAG_VALID; /* admit COOL (probation) */ pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); *foundPtr = false; @@ -248,9 +248,10 @@ GetLocalVictimBuffer(void) { uint64 buf_state = pg_atomic_read_u64(&bufHdr->state); - if (BUF_STATE_GET_USAGECOUNT(buf_state) > 0) + if (BUF_STATE_GET_COOLSTATE(buf_state) != BUF_COOLSTATE_COOL) { - buf_state -= BUF_USAGECOUNT_ONE; + /* HOT: give it a second chance, cool it and keep scanning. */ + buf_state -= BUF_COOLSTATE_ONE; pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); trycounter = NLocBuffer; } @@ -454,7 +455,7 @@ ExtendBufferedRelLocal(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; - buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + buf_state |= BM_TAG_VALID; /* admit COOL (probation) */ pg_atomic_unlocked_write_u64(&victim_buf_hdr->state, buf_state); @@ -839,9 +840,9 @@ PinLocalBuffer(BufferDesc *buf_hdr, bool adjust_usagecount) NLocalPinnedBuffers++; buf_state += BUF_REFCOUNT_ONE; if (adjust_usagecount && - BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) + BUF_STATE_GET_COOLSTATE(buf_state) < BUF_COOLSTATE_HOT) { - buf_state += BUF_USAGECOUNT_ONE; + buf_state += BUF_COOLSTATE_ONE; } pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state); diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index e4ff5619b79ca..131df126cda00 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -67,6 +67,50 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L #define BUF_USAGECOUNT_ONE \ (UINT64CONST(1) << BUF_REFCOUNT_BITS) +/* + * Cooling state (LeanStore / 2Q-A1 cooling-stage clock sweep). + * + * The field historically used for the 0..5 usage_count now holds a single + * cooling-state bit: HOT (recently accessed, not an eviction candidate) or + * COOL (an eviction candidate). We reuse BUF_USAGECOUNT_ONE as the unit so + * the buffer-state bit geography -- refcount, flag, and lock offsets, and the + * 64-bit StaticAsserts -- is unchanged; only the meaning of the field and the + * instructions that touch it change. + * + * A demand-loaded page is admitted COOL (probation); a second access promotes + * it to HOT (the rescue). The sweep prefers COOL victims and demotes HOT to + * COOL only when a full pass finds no COOL victim. So a one-touch scan fills + * and drains the COOL stage without displacing the HOT working set -- scan + * resistance intrinsic to the replacement algorithm. + */ +#define BUF_COOLSTATE_COOL 0 +#define BUF_COOLSTATE_HOT 1 +#define BUF_COOLSTATE_ONE BUF_USAGECOUNT_ONE + +/* + * Second-chance reference bit, bit 1 of the (former usagecount) field, one + * position above the cooling-state bit. PinBuffer sets it on every access. + * The bgwriter's pre-cooling gives a HOT buffer a second chance: the first + * time it passes a HOT buffer whose ref bit is set, it clears the ref bit and + * leaves the buffer HOT; only a HOT buffer whose ref bit is already clear (not + * re-accessed since the previous bgwriter pass) is demoted to COOL. This + * keeps genuinely-hot pages out of the COOL stage (protecting the working set + * from being cooled under scan pressure) while leaving the foreground sweep a + * single-pass search over the pre-staged COOL buffers. A separate bit (not a + * count) so it stays a plain masked store under the header lock. + */ +#define BUF_REFBIT (UINT64CONST(2) << BUF_REFCOUNT_BITS) +#define BUF_STATE_GET_REFBIT(state) (((state) & BUF_REFBIT) != 0) + +/* + * The cooling state occupies bit 0 and the reference bit occupies bit 1 of + * the (former usagecount) field, so the field must be at least 2 bits wide. + * Assert it here so a future change to BUF_USAGECOUNT_BITS cannot silently + * push BUF_REFBIT up into the flag bits. + */ +StaticAssertDecl(BUF_USAGECOUNT_BITS >= 2, + "cooling state + reference bit need at least 2 bits in the usagecount field"); + /* flags related definitions */ #define BUF_FLAG_SHIFT \ (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) @@ -92,6 +136,11 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L #define BUF_STATE_GET_USAGECOUNT(state) \ ((uint32)(((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT)) +/* Cooling state (HOT/COOL) from buffer state -- bit 0 of the field only, so + * the second-chance ref bit (bit 1) does not perturb the HOT/COOL test. */ +#define BUF_STATE_GET_COOLSTATE(state) \ + ((uint32) (((state) >> BUF_USAGECOUNT_SHIFT) & 1)) + /* * Flags for buffer descriptors * @@ -134,17 +183,15 @@ StaticAssertDecl(MAX_BACKENDS_BITS <= (BUF_LOCK_BITS - 2), /* - * The maximum allowed value of usage_count represents a tradeoff between - * accuracy and speed of the clock-sweep buffer management algorithm. A - * large value (comparable to NBuffers) would approximate LRU semantics. - * But it can take as many as BM_MAX_USAGE_COUNT+1 complete cycles of the - * clock-sweep hand to find a free buffer, so in practice we don't want the - * value to be very large. + * The cooling state is a single bit (HOT/COOL); the maximum value stored in + * the field is therefore BUF_COOLSTATE_HOT. Retained under the historical + * name BM_MAX_USAGE_COUNT so the pin fast path ("promote unless already at + * max") reads naturally. */ -#define BM_MAX_USAGE_COUNT 5 +#define BM_MAX_USAGE_COUNT BUF_COOLSTATE_HOT StaticAssertDecl(BM_MAX_USAGE_COUNT < (UINT64CONST(1) << BUF_USAGECOUNT_BITS), - "BM_MAX_USAGE_COUNT doesn't fit in BUF_USAGECOUNT_BITS bits"); + "cooling state doesn't fit in BUF_USAGECOUNT_BITS bits"); /* * Buffer tag identifies which disk block the buffer contains. From 1f9e7eda9438c168a51a928f7d169fae9f84e96f Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 7 Jul 2026 08:45:39 -0400 Subject: [PATCH 5/5] Remove BufferAccessStrategy; scan resistance is now intrinsic The cooling-stage evictor admits demand-loaded pages COOL and promotes them to HOT only on a second access, so a one-touch sequential scan fills and drains the COOL stage without displacing the hot working set. Scan resistance is therefore a property of the replacement algorithm itself, and the BufferAccessStrategy ring buffers that previously provided it are dead weight. Remove them end to end. Deleted: - the BufferAccessStrategy type and the BufferAccessStrategyType enum (BAS_NORMAL/BULKREAD/BULKWRITE/VACUUM); - the ring machinery in freelist.c (GetAccessStrategy[WithSize], GetAccessStrategyBufferCount, GetAccessStrategyPinLimit, FreeAccessStrategy, GetBufferFromRing, AddBufferToRing, StrategyRejectBuffer, IOContextForStrategy); - the strategy parameter from ReadBufferExtended, ReadBufferWithoutRelcache, the ExtendBufferedRel* family, StrategyGetBuffer, read_stream_begin_*, and every scan/vacuum/analyze/index-AM caller; - the strategy fields on HeapScanDescData, IndexScanDescData, BulkInsertStateData, ReadBuffersOperation, and ReadStream; - _hash_getbuf_with_strategy (identical to _hash_getbuf without a strategy). pg_stat_io's per-strategy IO contexts collapse: IOCONTEXT_BULKREAD, IOCONTEXT_BULKWRITE and IOCONTEXT_VACUUM are removed, leaving IOCONTEXT_INIT and IOCONTEXT_NORMAL. IOOP_REUSE only ever occurred while recycling a ring buffer, so it is no longer tracked; GetVictimBuffer counts IOOP_EVICT only. The vacuum_buffer_usage_limit GUC and the VACUUM/ANALYZE (BUFFER_USAGE_LIMIT ...) option are removed, along with the VacuumBufferUsageLimit global, the ring-size plumbing through VacuumParams and parallel vacuum, and vacuumdb's --buffer-usage-limit client option. read_stream's per-backend pin budget is now enforced solely by GetPinLimit()/GetLocalPinLimit(), which already applied and is unchanged for the (formerly universal) no-strategy case. Documentation and the stats/amcheck regression tests are updated to drop the removed contexts and options. --- contrib/amcheck/expected/check_heap.out | 16 +- contrib/amcheck/sql/check_heap.sql | 16 +- contrib/amcheck/verify_gin.c | 18 +- contrib/amcheck/verify_heapam.c | 3 - contrib/amcheck/verify_nbtree.c | 11 +- contrib/bloom/blscan.c | 8 - contrib/bloom/blutils.c | 4 +- contrib/bloom/blvacuum.c | 2 - contrib/pageinspect/rawpage.c | 2 +- contrib/pg_prewarm/autoprewarm.c | 1 - contrib/pg_prewarm/pg_prewarm.c | 1 - contrib/pg_visibility/pg_visibility.c | 7 +- contrib/pgstattuple/pgstatapprox.c | 3 - contrib/pgstattuple/pgstatindex.c | 9 +- contrib/pgstattuple/pgstattuple.c | 37 +- doc/src/sgml/config.sgml | 30 -- doc/src/sgml/monitoring.sgml | 38 +- doc/src/sgml/ref/analyze.sgml | 21 - doc/src/sgml/ref/vacuum.sgml | 25 - src/backend/access/brin/brin.c | 11 +- src/backend/access/brin/brin_revmap.c | 2 +- src/backend/access/gin/gininsert.c | 4 +- src/backend/access/gin/ginutil.c | 2 +- src/backend/access/gin/ginvacuum.c | 17 +- src/backend/access/gist/gist.c | 2 +- src/backend/access/gist/gistutil.c | 2 +- src/backend/access/gist/gistvacuum.c | 10 +- src/backend/access/hash/hash.c | 13 +- src/backend/access/hash/hashovfl.c | 55 +-- src/backend/access/hash/hashpage.c | 40 +- src/backend/access/heap/heapam.c | 31 +- src/backend/access/heap/heapam_handler.c | 4 +- src/backend/access/heap/hio.c | 13 +- src/backend/access/heap/vacuumlazy.c | 22 +- src/backend/access/heap/visibilitymap.c | 4 +- src/backend/access/nbtree/nbtpage.c | 2 +- src/backend/access/nbtree/nbtree.c | 4 +- src/backend/access/spgist/spgutils.c | 2 +- src/backend/access/spgist/spgvacuum.c | 4 +- src/backend/access/transam/xloginsert.c | 2 +- src/backend/access/transam/xlogutils.c | 3 +- src/backend/catalog/index.c | 1 - src/backend/commands/analyze.c | 7 +- src/backend/commands/dbcommands.c | 6 +- src/backend/commands/repack.c | 2 +- src/backend/commands/sequence.c | 2 +- src/backend/commands/vacuum.c | 118 +---- src/backend/commands/vacuumparallel.c | 21 +- src/backend/postmaster/autovacuum.c | 29 +- src/backend/postmaster/datachecksum_state.c | 33 +- src/backend/storage/aio/read_stream.c | 17 +- src/backend/storage/buffer/README | 44 -- src/backend/storage/buffer/bufmgr.c | 185 +++----- src/backend/storage/buffer/freelist.c | 434 +----------------- src/backend/storage/freespace/freespace.c | 4 +- src/backend/storage/smgr/md.c | 11 +- src/backend/utils/activity/pgstat_io.c | 45 +- src/backend/utils/init/globals.c | 1 - src/backend/utils/misc/guc_parameters.dat | 10 - src/backend/utils/misc/postgresql.conf.sample | 3 - src/bin/scripts/vacuumdb.c | 13 - src/bin/scripts/vacuuming.c | 21 - src/bin/scripts/vacuuming.h | 1 - src/include/access/genam.h | 1 - src/include/access/hash.h | 9 +- src/include/access/heapam.h | 4 +- src/include/access/hio.h | 1 - src/include/access/tableam.h | 8 +- src/include/commands/vacuum.h | 8 +- src/include/miscadmin.h | 9 - src/include/pgstat.h | 5 +- src/include/storage/buf.h | 7 - src/include/storage/buf_internals.h | 6 +- src/include/storage/bufmgr.h | 32 +- src/include/storage/read_stream.h | 5 +- src/include/utils/guc_hooks.h | 2 - src/test/modules/test_aio/test_aio.c | 7 +- src/test/regress/expected/stats.out | 110 +---- src/test/regress/expected/vacuum.out | 19 - src/test/regress/sql/stats.sql | 56 +-- src/test/regress/sql/vacuum.sql | 15 - src/tools/pgindent/typedefs.list | 2 - 82 files changed, 261 insertions(+), 1554 deletions(-) diff --git a/contrib/amcheck/expected/check_heap.out b/contrib/amcheck/expected/check_heap.out index 979e5e84e723d..569b0202f1cf1 100644 --- a/contrib/amcheck/expected/check_heap.out +++ b/contrib/amcheck/expected/check_heap.out @@ -67,11 +67,11 @@ INSERT INTO heaptest (a, b) (SELECT gs, repeat('x', gs) FROM generate_series(1,50) gs); -- pg_stat_io test: --- verify_heapam always uses a BAS_BULKREAD BufferAccessStrategy, whereas a --- sequential scan does so only if the table is large enough when compared to --- shared buffers (see initscan()). CREATE DATABASE ... also unconditionally --- uses a BAS_BULKREAD strategy, but we have chosen to use a tablespace and --- verify_heapam to provide coverage instead of adding another expensive +-- verify_heapam reads the heap through the buffer manager; with the +-- cooling-stage clock sweep there are no per-strategy IO contexts, so the +-- reads are counted in the 'normal' context. CREATE DATABASE ... likewise +-- reads through the normal context, but we have chosen to use a tablespace +-- and verify_heapam to provide coverage instead of adding another expensive -- operation to the main regression test suite. -- -- Create an alternative tablespace and move the heaptest table to it, causing @@ -83,7 +83,7 @@ INSERT INTO heaptest (a, b) SET allow_in_place_tablespaces = true; CREATE TABLESPACE regress_test_stats_tblspc LOCATION ''; SELECT sum(reads) AS stats_bulkreads_before - FROM pg_stat_io WHERE context = 'bulkread' \gset + FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset BEGIN; ALTER TABLE heaptest SET TABLESPACE regress_test_stats_tblspc; -- Check that valid options are not rejected nor corruption reported @@ -111,7 +111,7 @@ SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := COMMIT; -- verify_heapam should have read in the page written out by -- ALTER TABLE ... SET TABLESPACE ... --- causing an additional bulkread, which should be reflected in pg_stat_io. +-- causing additional reads, which should be reflected in pg_stat_io. SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- @@ -119,7 +119,7 @@ SELECT pg_stat_force_next_flush(); (1 row) SELECT sum(reads) AS stats_bulkreads_after - FROM pg_stat_io WHERE context = 'bulkread' \gset + FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :stats_bulkreads_after > :stats_bulkreads_before; ?column? ---------- diff --git a/contrib/amcheck/sql/check_heap.sql b/contrib/amcheck/sql/check_heap.sql index 1745bae634e56..61f710dbedafd 100644 --- a/contrib/amcheck/sql/check_heap.sql +++ b/contrib/amcheck/sql/check_heap.sql @@ -27,11 +27,11 @@ INSERT INTO heaptest (a, b) FROM generate_series(1,50) gs); -- pg_stat_io test: --- verify_heapam always uses a BAS_BULKREAD BufferAccessStrategy, whereas a --- sequential scan does so only if the table is large enough when compared to --- shared buffers (see initscan()). CREATE DATABASE ... also unconditionally --- uses a BAS_BULKREAD strategy, but we have chosen to use a tablespace and --- verify_heapam to provide coverage instead of adding another expensive +-- verify_heapam reads the heap through the buffer manager; with the +-- cooling-stage clock sweep there are no per-strategy IO contexts, so the +-- reads are counted in the 'normal' context. CREATE DATABASE ... likewise +-- reads through the normal context, but we have chosen to use a tablespace +-- and verify_heapam to provide coverage instead of adding another expensive -- operation to the main regression test suite. -- -- Create an alternative tablespace and move the heaptest table to it, causing @@ -43,7 +43,7 @@ INSERT INTO heaptest (a, b) SET allow_in_place_tablespaces = true; CREATE TABLESPACE regress_test_stats_tblspc LOCATION ''; SELECT sum(reads) AS stats_bulkreads_before - FROM pg_stat_io WHERE context = 'bulkread' \gset + FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset BEGIN; ALTER TABLE heaptest SET TABLESPACE regress_test_stats_tblspc; -- Check that valid options are not rejected nor corruption reported @@ -56,10 +56,10 @@ COMMIT; -- verify_heapam should have read in the page written out by -- ALTER TABLE ... SET TABLESPACE ... --- causing an additional bulkread, which should be reflected in pg_stat_io. +-- causing additional reads, which should be reflected in pg_stat_io. SELECT pg_stat_force_next_flush(); SELECT sum(reads) AS stats_bulkreads_after - FROM pg_stat_io WHERE context = 'bulkread' \gset + FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset SELECT :stats_bulkreads_after > :stats_bulkreads_before; CREATE ROLE regress_heaptest_role; diff --git a/contrib/amcheck/verify_gin.c b/contrib/amcheck/verify_gin.c index fa06689ed5b29..ef2e64475c661 100644 --- a/contrib/amcheck/verify_gin.c +++ b/contrib/amcheck/verify_gin.c @@ -63,8 +63,7 @@ static void gin_check_parent_keys_consistency(Relation rel, static void check_index_page(Relation rel, Buffer buffer, BlockNumber blockNo); static IndexTuple gin_refind_parent(Relation rel, BlockNumber parentblkno, - BlockNumber childblkno, - BufferAccessStrategy strategy); + BlockNumber childblkno); static ItemId PageGetItemIdCareful(Relation rel, BlockNumber block, Page page, OffsetNumber offset); @@ -133,7 +132,6 @@ ginReadTupleWithoutState(IndexTuple itup, int *nitems) static void gin_check_posting_tree_parent_keys_consistency(Relation rel, BlockNumber posting_tree_root) { - BufferAccessStrategy strategy = GetAccessStrategy(BAS_BULKREAD); GinPostingTreeScanItem *stack; MemoryContext mctx; MemoryContext oldcontext; @@ -171,8 +169,7 @@ gin_check_posting_tree_parent_keys_consistency(Relation rel, BlockNumber posting CHECK_FOR_INTERRUPTS(); - buffer = ReadBufferExtended(rel, MAIN_FORKNUM, stack->blkno, - RBM_NORMAL, strategy); + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, stack->blkno, RBM_NORMAL); LockBuffer(buffer, GIN_SHARE); page = BufferGetPage(buffer); @@ -391,7 +388,6 @@ gin_check_parent_keys_consistency(Relation rel, void *callback_state, bool readonly) { - BufferAccessStrategy strategy = GetAccessStrategy(BAS_BULKREAD); GinScanItem *stack; MemoryContext mctx; MemoryContext oldcontext; @@ -430,8 +426,7 @@ gin_check_parent_keys_consistency(Relation rel, CHECK_FOR_INTERRUPTS(); - buffer = ReadBufferExtended(rel, MAIN_FORKNUM, stack->blkno, - RBM_NORMAL, strategy); + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, stack->blkno, RBM_NORMAL); LockBuffer(buffer, GIN_SHARE); page = BufferGetPage(buffer); maxoff = PageGetMaxOffsetNumber(page); @@ -567,7 +562,7 @@ gin_check_parent_keys_consistency(Relation rel, */ pfree(stack->parenttup); stack->parenttup = gin_refind_parent(rel, stack->parentblk, - stack->blkno, strategy); + stack->blkno); /* We found it - make a final check before failing */ if (!stack->parenttup) @@ -719,7 +714,7 @@ check_index_page(Relation rel, Buffer buffer, BlockNumber blockNo) */ static IndexTuple gin_refind_parent(Relation rel, BlockNumber parentblkno, - BlockNumber childblkno, BufferAccessStrategy strategy) + BlockNumber childblkno) { Buffer parentbuf; Page parentpage; @@ -727,8 +722,7 @@ gin_refind_parent(Relation rel, BlockNumber parentblkno, parent_maxoff; IndexTuple result = NULL; - parentbuf = ReadBufferExtended(rel, MAIN_FORKNUM, parentblkno, RBM_NORMAL, - strategy); + parentbuf = ReadBufferExtended(rel, MAIN_FORKNUM, parentblkno, RBM_NORMAL); LockBuffer(parentbuf, GIN_SHARE); parentpage = BufferGetPage(parentbuf); diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 20ff58aa78259..e4336eb5fc2a4 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -126,7 +126,6 @@ typedef struct HeapCheckContext * recent block in the buffer yielded by the read stream API. */ BlockNumber blkno; - BufferAccessStrategy bstrategy; Buffer buffer; Page page; @@ -374,7 +373,6 @@ verify_heapam(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } - ctx.bstrategy = GetAccessStrategy(BAS_BULKREAD); ctx.buffer = InvalidBuffer; ctx.page = NULL; @@ -472,7 +470,6 @@ verify_heapam(PG_FUNCTION_ARGS) } stream = read_stream_begin_relation(stream_flags, - ctx.bstrategy, ctx.rel, MAIN_FORKNUM, stream_cb, diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index 3ef2d66f82675..18cd8928fdd85 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -88,8 +88,6 @@ typedef struct BtreeCheckState bool checkunique; /* Per-page context */ MemoryContext targetcontext; - /* Buffer access strategy */ - BufferAccessStrategy checkstrategy; /* * Info for uniqueness checking. Fill this field and the one below once @@ -490,7 +488,6 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, state->targetcontext = AllocSetContextCreate(CurrentMemoryContext, "amcheck context", ALLOCSET_DEFAULT_SIZES); - state->checkstrategy = GetAccessStrategy(BAS_BULKREAD); /* Get true root block from meta-page */ metapage = palloc_btree_page(state, BTREE_METAPAGE); @@ -1115,7 +1112,7 @@ bt_recheck_sibling_links(BtreeCheckState *state, /* Couple locks in the usual order for nbtree: Left to right */ lbuf = ReadBufferExtended(state->rel, MAIN_FORKNUM, leftcurrent, - RBM_NORMAL, state->checkstrategy); + RBM_NORMAL); LockBuffer(lbuf, BT_READ); _bt_checkpage(state->rel, lbuf); page = BufferGetPage(lbuf); @@ -1138,8 +1135,7 @@ bt_recheck_sibling_links(BtreeCheckState *state, if (newtargetblock != leftcurrent) { newtargetbuf = ReadBufferExtended(state->rel, MAIN_FORKNUM, - newtargetblock, RBM_NORMAL, - state->checkstrategy); + newtargetblock, RBM_NORMAL); LockBuffer(newtargetbuf, BT_READ); _bt_checkpage(state->rel, newtargetbuf); page = BufferGetPage(newtargetbuf); @@ -3300,8 +3296,7 @@ palloc_btree_page(BtreeCheckState *state, BlockNumber blocknum) * We copy the page into local storage to avoid holding pin on the buffer * longer than we must. */ - buffer = ReadBufferExtended(state->rel, MAIN_FORKNUM, blocknum, RBM_NORMAL, - state->checkstrategy); + buffer = ReadBufferExtended(state->rel, MAIN_FORKNUM, blocknum, RBM_NORMAL); LockBuffer(buffer, BT_READ); /* diff --git a/contrib/bloom/blscan.c b/contrib/bloom/blscan.c index 1a0e42021ec1e..ed62d5c04e6e0 100644 --- a/contrib/bloom/blscan.c +++ b/contrib/bloom/blscan.c @@ -80,7 +80,6 @@ blgetbitmap(IndexScanDesc scan, TIDBitmap *tbm) BlockNumber blkno, npages; int i; - BufferAccessStrategy bas; BloomScanOpaque so = (BloomScanOpaque) scan->opaque; BlockRangeReadStreamPrivate p; ReadStream *stream; @@ -113,11 +112,6 @@ blgetbitmap(IndexScanDesc scan, TIDBitmap *tbm) } } - /* - * We're going to read the whole index. This is why we use appropriate - * buffer access strategy. - */ - bas = GetAccessStrategy(BAS_BULKREAD); npages = RelationGetNumberOfBlocks(scan->indexRelation); pgstat_count_index_scan(scan->indexRelation); if (scan->instrument) @@ -133,7 +127,6 @@ blgetbitmap(IndexScanDesc scan, TIDBitmap *tbm) */ stream = read_stream_begin_relation(READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - bas, scan->indexRelation, MAIN_FORKNUM, block_range_read_stream_cb, @@ -184,7 +177,6 @@ blgetbitmap(IndexScanDesc scan, TIDBitmap *tbm) Assert(read_stream_next_buffer(stream, NULL) == InvalidBuffer); read_stream_end(stream); - FreeAccessStrategy(bas); return ntids; } diff --git a/contrib/bloom/blutils.c b/contrib/bloom/blutils.c index 5111cdc6dd62f..6c1ec15c59ce9 100644 --- a/contrib/bloom/blutils.c +++ b/contrib/bloom/blutils.c @@ -392,7 +392,7 @@ BloomNewBuffer(Relation index) } /* Must extend the file */ - buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, NULL, + buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, EB_LOCK_FIRST); return buffer; @@ -460,7 +460,7 @@ BloomInitMetapage(Relation index, ForkNumber forknum) * block number 0 (BLOOM_METAPAGE_BLKNO). No need to hold the extension * lock because there cannot be concurrent inserters yet. */ - metaBuffer = ReadBufferExtended(index, forknum, P_NEW, RBM_NORMAL, NULL); + metaBuffer = ReadBufferExtended(index, forknum, P_NEW, RBM_NORMAL); LockBuffer(metaBuffer, BUFFER_LOCK_EXCLUSIVE); Assert(BufferGetBlockNumber(metaBuffer) == BLOOM_METAPAGE_BLKNO); diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c index 6beb1c20ebb0d..08d7705e36500 100644 --- a/contrib/bloom/blvacuum.c +++ b/contrib/bloom/blvacuum.c @@ -66,7 +66,6 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - info->strategy, index, MAIN_FORKNUM, block_range_read_stream_cb, @@ -219,7 +218,6 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - info->strategy, index, MAIN_FORKNUM, block_range_read_stream_cb, diff --git a/contrib/pageinspect/rawpage.c b/contrib/pageinspect/rawpage.c index d136593edb267..bbe8edf916113 100644 --- a/contrib/pageinspect/rawpage.c +++ b/contrib/pageinspect/rawpage.c @@ -188,7 +188,7 @@ get_raw_page_internal(text *relname, ForkNumber forknum, BlockNumber blkno) /* Take a verbatim copy of the page */ - buf = ReadBufferExtended(rel, forknum, blkno, RBM_NORMAL, NULL); + buf = ReadBufferExtended(rel, forknum, blkno, RBM_NORMAL); LockBuffer(buf, BUFFER_LOCK_SHARE); memcpy(raw_page_data, BufferGetPage(buf), BLCKSZ); diff --git a/contrib/pg_prewarm/autoprewarm.c b/contrib/pg_prewarm/autoprewarm.c index deb4c2671b5d8..33ad88ea8b12c 100644 --- a/contrib/pg_prewarm/autoprewarm.c +++ b/contrib/pg_prewarm/autoprewarm.c @@ -630,7 +630,6 @@ autoprewarm_database_main(Datum main_arg) stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_DEFAULT | READ_STREAM_USE_BATCHING, - NULL, rel, p.forknum, apw_read_stream_next_block, diff --git a/contrib/pg_prewarm/pg_prewarm.c b/contrib/pg_prewarm/pg_prewarm.c index c2716086693d9..716a6754a7e20 100644 --- a/contrib/pg_prewarm/pg_prewarm.c +++ b/contrib/pg_prewarm/pg_prewarm.c @@ -251,7 +251,6 @@ pg_prewarm(PG_FUNCTION_ARGS) stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - NULL, rel, forkNumber, block_range_read_stream_cb, diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c index dfab0b64cf50b..ccb829a09eb89 100644 --- a/contrib/pg_visibility/pg_visibility.c +++ b/contrib/pg_visibility/pg_visibility.c @@ -488,7 +488,6 @@ collect_visibility_data(Oid relid, bool include_pd) vbits *info; BlockNumber blkno; Buffer vmbuffer = InvalidBuffer; - BufferAccessStrategy bstrategy = GetAccessStrategy(BAS_BULKREAD); BlockRangeReadStreamPrivate p; ReadStream *stream = NULL; @@ -514,7 +513,6 @@ collect_visibility_data(Oid relid, bool include_pd) */ stream = read_stream_begin_relation(READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - bstrategy, rel, MAIN_FORKNUM, block_range_read_stream_cb, @@ -538,8 +536,7 @@ collect_visibility_data(Oid relid, bool include_pd) /* * Page-level data requires reading every block, so only get it if the - * caller needs it. Use a buffer access strategy, too, to prevent - * cache-trashing. + * caller needs it. */ if (include_pd) { @@ -700,7 +697,6 @@ collect_corrupt_items(Oid relid, bool all_visible, bool all_frozen) Relation rel; corrupt_items *items; Buffer vmbuffer = InvalidBuffer; - BufferAccessStrategy bstrategy = GetAccessStrategy(BAS_BULKREAD); TransactionId OldestXmin = InvalidTransactionId; struct collect_corrupt_items_read_stream_private p; ReadStream *stream; @@ -734,7 +730,6 @@ collect_corrupt_items(Oid relid, bool all_visible, bool all_frozen) p.all_frozen = all_frozen; p.all_visible = all_visible; stream = read_stream_begin_relation(READ_STREAM_FULL, - bstrategy, rel, MAIN_FORKNUM, collect_corrupt_items_read_stream_next_block, diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index 21e0b50fb4bd4..8e17d48991e47 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -116,13 +116,11 @@ static void statapprox_heap(Relation rel, output_type *stat) { BlockNumber nblocks; - BufferAccessStrategy bstrategy; TransactionId OldestXmin; StatApproxReadStreamPrivate p; ReadStream *stream; OldestXmin = GetOldestNonRemovableTransactionId(rel); - bstrategy = GetAccessStrategy(BAS_BULKREAD); nblocks = RelationGetNumberOfBlocks(rel); @@ -141,7 +139,6 @@ statapprox_heap(Relation rel, output_type *stat) * caution. */ stream = read_stream_begin_relation(READ_STREAM_FULL, - bstrategy, rel, MAIN_FORKNUM, statapprox_heap_read_stream_next, diff --git a/contrib/pgstattuple/pgstatindex.c b/contrib/pgstattuple/pgstatindex.c index 8951ad0aac472..a0922b386a00a 100644 --- a/contrib/pgstattuple/pgstatindex.c +++ b/contrib/pgstattuple/pgstatindex.c @@ -217,7 +217,6 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) BlockNumber nblocks; BlockNumber blkno; BTIndexStat indexStat; - BufferAccessStrategy bstrategy = GetAccessStrategy(BAS_BULKREAD); BlockRangeReadStreamPrivate p; ReadStream *stream; BlockNumber startblk; @@ -254,7 +253,7 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) * Read metapage */ { - Buffer buffer = ReadBufferExtended(rel, MAIN_FORKNUM, 0, RBM_NORMAL, bstrategy); + Buffer buffer = ReadBufferExtended(rel, MAIN_FORKNUM, 0, RBM_NORMAL); Page page = BufferGetPage(buffer); BTMetaPageData *metad = BTPageGetMeta(page); @@ -291,7 +290,6 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) */ stream = read_stream_begin_relation(READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - bstrategy, rel, MAIN_FORKNUM, block_range_read_stream_cb, @@ -612,7 +610,6 @@ pgstathashindex(PG_FUNCTION_ARGS) BlockNumber blkno; Relation rel; HashIndexStat stats; - BufferAccessStrategy bstrategy; HeapTuple tuple; TupleDesc tupleDesc; Datum values[8]; @@ -665,9 +662,6 @@ pgstathashindex(PG_FUNCTION_ARGS) /* Get the current relation length */ nblocks = RelationGetNumberOfBlocks(rel); - /* prepare access strategy for this index */ - bstrategy = GetAccessStrategy(BAS_BULKREAD); - /* Scan all blocks except the metapage (0th page) using streaming reads */ startblk = HASH_METAPAGE + 1; @@ -680,7 +674,6 @@ pgstathashindex(PG_FUNCTION_ARGS) */ stream = read_stream_begin_relation(READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - bstrategy, rel, MAIN_FORKNUM, block_range_read_stream_cb, diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index 6a7f8cb4a7ca5..90f7938e95313 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -64,22 +64,18 @@ typedef struct pgstattuple_type uint64 free_space; /* free/reusable space in bytes */ } pgstattuple_type; -typedef void (*pgstat_page) (pgstattuple_type *, Relation, BlockNumber, - BufferAccessStrategy); +typedef void (*pgstat_page) (pgstattuple_type *, Relation, BlockNumber); static Datum build_pgstattuple_type(pgstattuple_type *stat, FunctionCallInfo fcinfo); static Datum pgstat_relation(Relation rel, FunctionCallInfo fcinfo); static Datum pgstat_heap(Relation rel, FunctionCallInfo fcinfo); static void pgstat_btree_page(pgstattuple_type *stat, - Relation rel, BlockNumber blkno, - BufferAccessStrategy bstrategy); + Relation rel, BlockNumber blkno); static void pgstat_hash_page(pgstattuple_type *stat, - Relation rel, BlockNumber blkno, - BufferAccessStrategy bstrategy); + Relation rel, BlockNumber blkno); static void pgstat_gist_page(pgstattuple_type *stat, - Relation rel, BlockNumber blkno, - BufferAccessStrategy bstrategy); + Relation rel, BlockNumber blkno); static Datum pgstat_index(Relation rel, BlockNumber start, pgstat_page pagefn, FunctionCallInfo fcinfo); static void pgstat_index_page(pgstattuple_type *stat, Page page, @@ -376,7 +372,7 @@ pgstat_heap(Relation rel, FunctionCallInfo fcinfo) CHECK_FOR_INTERRUPTS(); buffer = ReadBufferExtended(rel, MAIN_FORKNUM, block, - RBM_NORMAL, hscan->rs_strategy); + RBM_NORMAL); LockBuffer(buffer, BUFFER_LOCK_SHARE); stat.free_space += PageGetExactFreeSpace(BufferGetPage(buffer)); UnlockReleaseBuffer(buffer); @@ -389,7 +385,7 @@ pgstat_heap(Relation rel, FunctionCallInfo fcinfo) CHECK_FOR_INTERRUPTS(); buffer = ReadBufferExtended(rel, MAIN_FORKNUM, block, - RBM_NORMAL, hscan->rs_strategy); + RBM_NORMAL); LockBuffer(buffer, BUFFER_LOCK_SHARE); stat.free_space += PageGetExactFreeSpace(BufferGetPage(buffer)); UnlockReleaseBuffer(buffer); @@ -408,13 +404,12 @@ pgstat_heap(Relation rel, FunctionCallInfo fcinfo) * pgstat_btree_page -- check tuples in a btree page */ static void -pgstat_btree_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, - BufferAccessStrategy bstrategy) +pgstat_btree_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno) { Buffer buf; Page page; - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, bstrategy); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL); LockBuffer(buf, BT_READ); page = BufferGetPage(buf); @@ -452,13 +447,12 @@ pgstat_btree_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, * pgstat_hash_page -- check tuples in a hash page */ static void -pgstat_hash_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, - BufferAccessStrategy bstrategy) +pgstat_hash_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno) { Buffer buf; Page page; - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, bstrategy); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL); LockBuffer(buf, HASH_READ); page = BufferGetPage(buf); @@ -500,13 +494,12 @@ pgstat_hash_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, * pgstat_gist_page -- check tuples in a gist page */ static void -pgstat_gist_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, - BufferAccessStrategy bstrategy) +pgstat_gist_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno) { Buffer buf; Page page; - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, bstrategy); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL); LockBuffer(buf, GIST_SHARE); page = BufferGetPage(buf); if (PageIsNew(page)) @@ -539,12 +532,8 @@ pgstat_index(Relation rel, BlockNumber start, pgstat_page pagefn, { BlockNumber nblocks; BlockNumber blkno; - BufferAccessStrategy bstrategy; pgstattuple_type stat = {0}; - /* prepare access strategy for this index */ - bstrategy = GetAccessStrategy(BAS_BULKREAD); - blkno = start; for (;;) { @@ -565,7 +554,7 @@ pgstat_index(Relation rel, BlockNumber start, pgstat_page pagefn, { CHECK_FOR_INTERRUPTS(); - pagefn(&stat, rel, blkno, bstrategy); + pagefn(&stat, rel, blkno); } } diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0848c18d329e3..7cb8839245c7d 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2107,36 +2107,6 @@ include_dir 'conf.d' - - - vacuum_buffer_usage_limit (integer) - - vacuum_buffer_usage_limit configuration parameter - - - - - Specifies the size of the - Buffer Access Strategy - used by the VACUUM and ANALYZE - commands. A setting of 0 will allow the operation - to use any number of shared_buffers. Otherwise - valid sizes range from 128 kB to - 16 GB. If the specified size would exceed 1/8 the - size of shared_buffers, the size is silently capped - to that value. The default value is 2MB. If - this value is specified without units, it is taken as kilobytes. This - parameter can be set at any time. It can be overridden for - and - when passing the option. Higher - settings can allow VACUUM and - ANALYZE to run more quickly, but having too large a - setting may cause too many other useful pages to be evicted from - shared buffers. - - - - logical_decoding_work_mem (integer) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index d1a20d001e9c8..18d17540430db 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2993,28 +2993,6 @@ description | Waiting for a newly initialized WAL file to reach durable storage init. - - - vacuum: I/O operations performed outside of shared - buffers while vacuuming and analyzing permanent relations. Temporary - table vacuums use the same local buffer pool as other temporary table - I/O operations and are tracked in context - normal. - - - - - bulkread: Certain large read I/O operations - done outside of shared buffers, for example, a sequential scan of a - large table. - - - - - bulkwrite: Certain large write I/O operations - done outside of shared buffers, such as COPY. - - @@ -3180,13 +3158,8 @@ description | Waiting for a newly initialized WAL file to reach durable storage buffer in order to make it available for another use. - In context normal, this counts - the number of times a block was evicted from a buffer and replaced with - another block. In contexts - bulkwrite, bulkread, and - vacuum, this counts the number of times a block was - evicted from shared buffers in order to add the shared buffer to a - separate, size-limited ring buffer for use in a bulk I/O operation. + This counts the number of times a block was evicted from a buffer and + replaced with another block. @@ -3197,10 +3170,9 @@ description | Waiting for a newly initialized WAL file to reach durable storage reuses bigint - The number of times an existing buffer in a size-limited ring buffer - outside of shared buffers was reused as part of an I/O operation in the - bulkread, bulkwrite, or - vacuum contexts. + Always zero. This column previously counted reuses of buffers in a + size-limited ring buffer (buffer access strategy); ring buffers have + been removed, so no reuses are tracked. diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index ec81f00fecf87..7f5159111d888 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -27,7 +27,6 @@ ANALYZE [ ( option [, ...] ) ] [ boolean ] SKIP_LOCKED [ boolean ] - BUFFER_USAGE_LIMIT size and table_and_columns is: @@ -88,26 +87,6 @@ ANALYZE [ ( option [, ...] ) ] [ - - BUFFER_USAGE_LIMIT - - - Specifies the - Buffer Access Strategy - ring buffer size for ANALYZE. This size is used to - calculate the number of shared buffers which will be reused as part of - this strategy. 0 disables use of a - Buffer Access Strategy. When this option is not - specified, ANALYZE uses the value from - . Higher settings can - allow ANALYZE to run more quickly, but having too - large a setting may cause too many other useful pages to be evicted from - shared buffers. The minimum value is 128 kB and the - maximum value is 16 GB. - - - - boolean diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 38ee973ea05d6..d44a49f8efe3f 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -37,7 +37,6 @@ VACUUM [ ( option [, ...] ) ] [ integer SKIP_DATABASE_STATS [ boolean ] ONLY_DATABASE_STATS [ boolean ] - BUFFER_USAGE_LIMIT size FULL [ boolean ] and table_and_columns is: @@ -309,30 +308,6 @@ VACUUM [ ( option [, ...] ) ] [ - - BUFFER_USAGE_LIMIT - - - Specifies the - Buffer Access Strategy - ring buffer size for VACUUM. This size is used to - calculate the number of shared buffers which will be reused as part of - this strategy. 0 disables use of a - Buffer Access Strategy. If - is also specified, the value is used - for both the vacuum and analyze stages. This option can't be used with - the option except if is - also specified. When this option is not specified, - VACUUM uses the value from - . Higher settings can - allow VACUUM to run more quickly, but having too - large a setting may cause too many other useful pages to be evicted from - shared buffers. The minimum value is 128 kB and the - maximum value is 16 GB. - - - - FULL diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index bdb30752e098c..5d6539f548357 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -224,7 +224,7 @@ static void form_and_insert_tuple(BrinBuildState *state); static void form_and_spill_tuple(BrinBuildState *state); static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b); -static void brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy); +static void brin_vacuum_scan(Relation idxrel); static bool add_values_to_range(Relation idxRel, BrinDesc *bdesc, BrinMemTuple *dtup, const Datum *values, const bool *nulls); static bool check_null_keys(BrinValues *bval, ScanKey *nullkeys, int nnullkeys); @@ -1129,7 +1129,7 @@ brinbuild(Relation heap, Relation index, IndexInfo *indexInfo) * whole relation will be rolled back. */ - meta = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, NULL, + meta = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, EB_LOCK_FIRST | EB_SKIP_EXTENSION_LOCK); Assert(BufferGetBlockNumber(meta) == BRIN_METAPAGE_BLKNO); @@ -1281,7 +1281,7 @@ brinbuildempty(Relation index) Buffer metabuf; /* An empty BRIN index has a metapage only. */ - metabuf = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, NULL, + metabuf = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, EB_LOCK_FIRST | EB_SKIP_EXTENSION_LOCK); /* Initialize and xlog metabuffer. */ @@ -1336,7 +1336,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) heapRel = table_open(IndexGetRelation(RelationGetRelid(info->index), false), AccessShareLock); - brin_vacuum_scan(info->index, info->strategy); + brin_vacuum_scan(info->index); brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, false, &stats->num_index_tuples, &stats->num_index_tuples); @@ -2171,7 +2171,7 @@ union_tuples(BrinDesc *bdesc, BrinMemTuple *a, BrinTuple *b) * and such. */ static void -brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy) +brin_vacuum_scan(Relation idxrel) { BlockRangeReadStreamPrivate p; ReadStream *stream; @@ -2187,7 +2187,6 @@ brin_vacuum_scan(Relation idxrel, BufferAccessStrategy strategy) stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - strategy, idxrel, MAIN_FORKNUM, block_range_read_stream_cb, diff --git a/src/backend/access/brin/brin_revmap.c b/src/backend/access/brin/brin_revmap.c index 233355cb2d5d1..01c4104ac88dd 100644 --- a/src/backend/access/brin/brin_revmap.c +++ b/src/backend/access/brin/brin_revmap.c @@ -559,7 +559,7 @@ revmap_physical_extend(BrinRevmap *revmap) } else { - buf = ExtendBufferedRel(BMR_REL(irel), MAIN_FORKNUM, NULL, + buf = ExtendBufferedRel(BMR_REL(irel), MAIN_FORKNUM, EB_LOCK_FIRST); if (BufferGetBlockNumber(buf) != mapBlk) { diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c index cb9ed3b563c6f..32ad65cc95ff2 100644 --- a/src/backend/access/gin/gininsert.c +++ b/src/backend/access/gin/gininsert.c @@ -815,9 +815,9 @@ ginbuildempty(Relation index) MetaBuffer; /* An empty GIN index has two pages. */ - MetaBuffer = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, NULL, + MetaBuffer = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, EB_LOCK_FIRST | EB_SKIP_EXTENSION_LOCK); - RootBuffer = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, NULL, + RootBuffer = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, EB_LOCK_FIRST | EB_SKIP_EXTENSION_LOCK); /* Initialize and xlog metabuffer and root buffer. */ diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c index e7cba81d47709..2d7394aedd5d3 100644 --- a/src/backend/access/gin/ginutil.c +++ b/src/backend/access/gin/ginutil.c @@ -336,7 +336,7 @@ GinNewBuffer(Relation index) } /* Must extend the file */ - buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, NULL, + buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, EB_LOCK_FIRST); return buffer; diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index 840543eb6642b..86538de39d628 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -32,7 +32,6 @@ struct GinVacuumState IndexBulkDeleteCallback callback; void *callback_state; GinState ginstate; - BufferAccessStrategy strategy; MemoryContext tmpCxt; }; @@ -289,7 +288,7 @@ ginScanPostingTreeToDelete(GinVacuumState *gvs, DataPageDeleteStack *myStackItem childBuffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, PostingItemGetBlockNumber(pitem), - RBM_NORMAL, gvs->strategy); + RBM_NORMAL); LockBuffer(childBuffer, GIN_EXCLUSIVE); /* Allocate a child stack entry on first use; reuse thereafter */ @@ -389,7 +388,7 @@ ginVacuumPostingTreeLeaves(GinVacuumState *gvs, BlockNumber blkno) PostingItem *pitem; buffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, blkno, - RBM_NORMAL, gvs->strategy); + RBM_NORMAL); LockBuffer(buffer, GIN_SHARE); page = BufferGetPage(buffer); @@ -430,7 +429,7 @@ ginVacuumPostingTreeLeaves(GinVacuumState *gvs, BlockNumber blkno) break; buffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, blkno, - RBM_NORMAL, gvs->strategy); + RBM_NORMAL); LockBuffer(buffer, GIN_EXCLUSIVE); page = BufferGetPage(buffer); } @@ -454,7 +453,7 @@ ginVacuumPostingTree(GinVacuumState *gvs, BlockNumber rootBlkno) bool deleted PG_USED_FOR_ASSERTS_ONLY; buffer = ReadBufferExtended(gvs->index, MAIN_FORKNUM, rootBlkno, - RBM_NORMAL, gvs->strategy); + RBM_NORMAL); /* * Lock posting tree root for cleanup to ensure there are no @@ -615,7 +614,6 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, gvs.index = index; gvs.callback = callback; gvs.callback_state = callback_state; - gvs.strategy = info->strategy; initGinState(&gvs.ginstate, index); /* first time through? */ @@ -636,7 +634,7 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, gvs.result = stats; buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, - RBM_NORMAL, info->strategy); + RBM_NORMAL); /* find leaf page */ for (;;) @@ -669,7 +667,7 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, UnlockReleaseBuffer(buffer); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, - RBM_NORMAL, info->strategy); + RBM_NORMAL); } /* right now we found leftmost page in entry's BTree */ @@ -712,7 +710,7 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, break; buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, - RBM_NORMAL, info->strategy); + RBM_NORMAL); LockBuffer(buffer, GIN_EXCLUSIVE); } @@ -794,7 +792,6 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - info->strategy, index, MAIN_FORKNUM, block_range_read_stream_cb, diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index 8565e225be7fd..ae494742a4b8f 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -142,7 +142,7 @@ gistbuildempty(Relation index) Buffer buffer; /* Initialize the root page */ - buffer = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, NULL, + buffer = ExtendBufferedRel(BMR_REL(index), INIT_FORKNUM, EB_SKIP_EXTENSION_LOCK | EB_LOCK_FIRST); /* Initialize and xlog buffer */ diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c index 0f58f61879fb0..170174f62fd56 100644 --- a/src/backend/access/gist/gistutil.c +++ b/src/backend/access/gist/gistutil.c @@ -877,7 +877,7 @@ gistNewBuffer(Relation r, Relation heaprel) } /* Must extend the file */ - buffer = ExtendBufferedRel(BMR_REL(r), MAIN_FORKNUM, NULL, + buffer = ExtendBufferedRel(BMR_REL(r), MAIN_FORKNUM, EB_LOCK_FIRST); return buffer; diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 686a04180546b..c366ed35ea876 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -218,7 +218,6 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - info->strategy, rel, MAIN_FORKNUM, block_range_read_stream_cb, @@ -491,8 +490,7 @@ gistvacuumpage(GistVacState *vstate, Buffer buffer) /* check for vacuum delay while not holding any buffer lock */ vacuum_delay_point(false); - buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - info->strategy); + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL); goto restart; } } @@ -524,8 +522,7 @@ gistvacuum_delete_empty_pages(IndexVacuumInfo *info, GistVacState *vstate) int ntodelete; int deleted; - buffer = ReadBufferExtended(rel, MAIN_FORKNUM, (BlockNumber) blkno, - RBM_NORMAL, info->strategy); + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, (BlockNumber) blkno, RBM_NORMAL); LockBuffer(buffer, GIST_SHARE); page = BufferGetPage(buffer); @@ -590,8 +587,7 @@ gistvacuum_delete_empty_pages(IndexVacuumInfo *info, GistVacState *vstate) if (PageGetMaxOffsetNumber(page) == FirstOffsetNumber) break; - leafbuf = ReadBufferExtended(rel, MAIN_FORKNUM, leafs_to_delete[i], - RBM_NORMAL, info->strategy); + leafbuf = ReadBufferExtended(rel, MAIN_FORKNUM, leafs_to_delete[i], RBM_NORMAL); LockBuffer(leafbuf, GIST_EXCLUSIVE); gistcheckpage(rel, leafbuf); diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index 8d8cd30dc386b..1bbb7224bfcba 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -542,7 +542,6 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, */ stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_USE_BATCHING, - info->strategy, rel, MAIN_FORKNUM, hash_bulkdelete_read_stream_cb, @@ -616,7 +615,7 @@ hashbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, bucket_buf = buf; - hashbucketcleanup(rel, cur_bucket, bucket_buf, blkno, info->strategy, + hashbucketcleanup(rel, cur_bucket, bucket_buf, blkno, cachedmetap->hashm_maxbucket, cachedmetap->hashm_highmask, cachedmetap->hashm_lowmask, &tuples_removed, @@ -765,7 +764,7 @@ hashvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) */ void hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, - BlockNumber bucket_blkno, BufferAccessStrategy bstrategy, + BlockNumber bucket_blkno, uint32 maxbucket, uint32 highmask, uint32 lowmask, double *tuples_removed, double *num_index_tuples, bool split_cleanup, @@ -935,9 +934,8 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, if (!BlockNumberIsValid(blkno)) break; - next_buf = _hash_getbuf_with_strategy(rel, blkno, HASH_WRITE, - LH_OVERFLOW_PAGE, - bstrategy); + next_buf = _hash_getbuf(rel, blkno, HASH_WRITE, + LH_OVERFLOW_PAGE); /* * release the lock on previous page after acquiring the lock on next @@ -1004,8 +1002,7 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, * ordering of tuples for a scan that has started before it. */ if (bucket_dirty && IsBufferCleanupOK(bucket_buf)) - _hash_squeezebucket(rel, cur_bucket, bucket_blkno, bucket_buf, - bstrategy); + _hash_squeezebucket(rel, cur_bucket, bucket_blkno, bucket_buf); else LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK); } diff --git a/src/backend/access/hash/hashovfl.c b/src/backend/access/hash/hashovfl.c index dbc57ef958c0d..5b35fed33f9e1 100644 --- a/src/backend/access/hash/hashovfl.c +++ b/src/backend/access/hash/hashovfl.c @@ -491,8 +491,7 @@ _hash_firstfreebit(uint32 map) BlockNumber _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, Buffer wbuf, IndexTuple *itups, OffsetNumber *itup_offsets, - Size *tups_size, uint16 nitups, - BufferAccessStrategy bstrategy) + Size *tups_size, uint16 nitups) { HashMetaPage metap; Buffer metabuf; @@ -539,20 +538,16 @@ _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, if (prevblkno == writeblkno) prevbuf = wbuf; else - prevbuf = _hash_getbuf_with_strategy(rel, - prevblkno, - HASH_WRITE, - LH_BUCKET_PAGE | LH_OVERFLOW_PAGE, - bstrategy); + prevbuf = _hash_getbuf(rel, + prevblkno, + HASH_WRITE, + LH_BUCKET_PAGE | LH_OVERFLOW_PAGE); } if (BlockNumberIsValid(nextblkno)) - nextbuf = _hash_getbuf_with_strategy(rel, - nextblkno, - HASH_WRITE, - LH_OVERFLOW_PAGE, - bstrategy); - - /* Note: bstrategy is intentionally not used for metapage and bitmap */ + nextbuf = _hash_getbuf(rel, + nextblkno, + HASH_WRITE, + LH_OVERFLOW_PAGE); /* Read the metapage so we can determine which bitmap page to use */ metabuf = _hash_getbuf(rel, HASH_METAPAGE, HASH_READ, LH_META_PAGE); @@ -843,8 +838,7 @@ void _hash_squeezebucket(Relation rel, Bucket bucket, BlockNumber bucket_blkno, - Buffer bucket_buf, - BufferAccessStrategy bstrategy) + Buffer bucket_buf) { BlockNumber wblkno; BlockNumber rblkno; @@ -886,11 +880,10 @@ _hash_squeezebucket(Relation rel, rblkno = ropaque->hasho_nextblkno; if (rbuf != InvalidBuffer) _hash_relbuf(rel, rbuf); - rbuf = _hash_getbuf_with_strategy(rel, - rblkno, - HASH_WRITE, - LH_OVERFLOW_PAGE, - bstrategy); + rbuf = _hash_getbuf(rel, + rblkno, + HASH_WRITE, + LH_OVERFLOW_PAGE); rpage = BufferGetPage(rbuf); ropaque = HashPageGetOpaque(rpage); Assert(ropaque->hasho_bucket == bucket); @@ -952,11 +945,10 @@ _hash_squeezebucket(Relation rel, /* don't need to move to next page if we reached the read page */ if (wblkno != rblkno) - next_wbuf = _hash_getbuf_with_strategy(rel, - wblkno, - HASH_WRITE, - LH_OVERFLOW_PAGE, - bstrategy); + next_wbuf = _hash_getbuf(rel, + wblkno, + HASH_WRITE, + LH_OVERFLOW_PAGE); if (nitups > 0) { @@ -1098,7 +1090,7 @@ _hash_squeezebucket(Relation rel, /* free this overflow page (releases rbuf) */ _hash_freeovflpage(rel, bucket_buf, rbuf, wbuf, itups, itup_offsets, - tups_size, nitups, bstrategy); + tups_size, nitups); /* be tidy */ for (i = 0; i < nitups; i++) @@ -1115,11 +1107,10 @@ _hash_squeezebucket(Relation rel, return; } - rbuf = _hash_getbuf_with_strategy(rel, - rblkno, - HASH_WRITE, - LH_OVERFLOW_PAGE, - bstrategy); + rbuf = _hash_getbuf(rel, + rblkno, + HASH_WRITE, + LH_OVERFLOW_PAGE); rpage = BufferGetPage(rbuf); ropaque = HashPageGetOpaque(rpage); Assert(ropaque->hasho_bucket == bucket); diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c index 8099b0d021f05..bdf1a255f1fbf 100644 --- a/src/backend/access/hash/hashpage.c +++ b/src/backend/access/hash/hashpage.c @@ -139,8 +139,7 @@ _hash_getinitbuf(Relation rel, BlockNumber blkno) if (blkno == P_NEW) elog(ERROR, "hash AM does not use P_NEW"); - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_ZERO_AND_LOCK, - NULL); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_ZERO_AND_LOCK); /* ref count and lock type are correct */ @@ -209,7 +208,7 @@ _hash_getnewbuf(Relation rel, BlockNumber blkno, ForkNumber forkNum) /* smgr insists we explicitly extend the relation */ if (blkno == nblocks) { - buf = ExtendBufferedRel(BMR_REL(rel), forkNum, NULL, + buf = ExtendBufferedRel(BMR_REL(rel), forkNum, EB_LOCK_FIRST | EB_SKIP_EXTENSION_LOCK); if (BufferGetBlockNumber(buf) != blkno) elog(ERROR, "unexpected hash relation size: %u, should be %u", @@ -217,8 +216,7 @@ _hash_getnewbuf(Relation rel, BlockNumber blkno, ForkNumber forkNum) } else { - buf = ReadBufferExtended(rel, forkNum, blkno, RBM_ZERO_AND_LOCK, - NULL); + buf = ReadBufferExtended(rel, forkNum, blkno, RBM_ZERO_AND_LOCK); } /* ref count and lock type are correct */ @@ -229,34 +227,6 @@ _hash_getnewbuf(Relation rel, BlockNumber blkno, ForkNumber forkNum) return buf; } -/* - * _hash_getbuf_with_strategy() -- Get a buffer with nondefault strategy. - * - * This is identical to _hash_getbuf() but also allows a buffer access - * strategy to be specified. We use this for VACUUM operations. - */ -Buffer -_hash_getbuf_with_strategy(Relation rel, BlockNumber blkno, - int access, int flags, - BufferAccessStrategy bstrategy) -{ - Buffer buf; - - if (blkno == P_NEW) - elog(ERROR, "hash AM does not use P_NEW"); - - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, bstrategy); - - if (access != HASH_NOLOCK) - LockBuffer(buf, access); - - /* ref count and lock type are correct */ - - _hash_checkpage(rel, buf, flags); - - return buf; -} - /* * _hash_relbuf() -- release a locked buffer. * @@ -758,7 +728,7 @@ _hash_expandtable(Relation rel, Buffer metabuf) /* Release the metapage lock. */ LockBuffer(metabuf, BUFFER_LOCK_UNLOCK); - hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, NULL, + hashbucketcleanup(rel, old_bucket, buf_oblkno, start_oblkno, maxbucket, highmask, lowmask, NULL, NULL, true, NULL, NULL); @@ -1333,7 +1303,7 @@ _hash_splitbucket(Relation rel, { LockBuffer(bucket_nbuf, BUFFER_LOCK_UNLOCK); hashbucketcleanup(rel, obucket, bucket_obuf, - BufferGetBlockNumber(bucket_obuf), NULL, + BufferGetBlockNumber(bucket_obuf), maxbucket, highmask, lowmask, NULL, NULL, true, NULL, NULL); } diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 9cdc221675b2d..48e7c421bf497 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -359,7 +359,6 @@ static void initscan(HeapScanDesc scan, ScanKey key, bool keep_startblock) { ParallelBlockTableScanDesc bpscan = NULL; - bool allow_strat; bool allow_sync; /* @@ -396,24 +395,10 @@ initscan(HeapScanDesc scan, ScanKey key, bool keep_startblock) if (!RelationUsesLocalBuffers(scan->rs_base.rs_rd) && scan->rs_nblocks > NBuffers / 4) { - allow_strat = (scan->rs_base.rs_flags & SO_ALLOW_STRAT) != 0; allow_sync = (scan->rs_base.rs_flags & SO_ALLOW_SYNC) != 0; } else - allow_strat = allow_sync = false; - - if (allow_strat) - { - /* During a rescan, keep the previous strategy object. */ - if (scan->rs_strategy == NULL) - scan->rs_strategy = GetAccessStrategy(BAS_BULKREAD); - } - else - { - if (scan->rs_strategy != NULL) - FreeAccessStrategy(scan->rs_strategy); - scan->rs_strategy = NULL; - } + allow_sync = false; if (scan->rs_base.rs_parallel != NULL) { @@ -1202,7 +1187,6 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_base.rs_instrument = NULL; - scan->rs_strategy = NULL; /* set in initscan */ scan->rs_cbuf = InvalidBuffer; /* @@ -1273,8 +1257,7 @@ heap_beginscan(Relation relation, Snapshot snapshot, /* * Set up a read stream for sequential scans and TID range scans. This - * should be done after initscan() because initscan() allocates the - * BufferAccessStrategy object passed to the read stream API. + * should be done after initscan(). */ if (scan->rs_base.rs_flags & SO_TYPE_SEQSCAN || scan->rs_base.rs_flags & SO_TYPE_TIDRANGESCAN) @@ -1295,7 +1278,6 @@ heap_beginscan(Relation relation, Snapshot snapshot, */ scan->rs_read_stream = read_stream_begin_relation(READ_STREAM_SEQUENTIAL | READ_STREAM_USE_BATCHING, - scan->rs_strategy, scan->rs_base.rs_rd, MAIN_FORKNUM, cb, @@ -1306,7 +1288,6 @@ heap_beginscan(Relation relation, Snapshot snapshot, { scan->rs_read_stream = read_stream_begin_relation(READ_STREAM_DEFAULT | READ_STREAM_USE_BATCHING, - scan->rs_strategy, scan->rs_base.rs_rd, MAIN_FORKNUM, bitmapheap_stream_read_next, @@ -1402,9 +1383,6 @@ heap_endscan(TableScanDesc sscan) if (BufferIsValid(scan->rs_vmbuffer)) ReleaseBuffer(scan->rs_vmbuffer); - /* - * Must free the read stream before freeing the BufferAccessStrategy. - */ if (scan->rs_read_stream) read_stream_end(scan->rs_read_stream); @@ -1416,9 +1394,6 @@ heap_endscan(TableScanDesc sscan) if (scan->rs_base.rs_key) pfree(scan->rs_base.rs_key); - if (scan->rs_strategy != NULL) - FreeAccessStrategy(scan->rs_strategy); - if (scan->rs_parallelworkerdata != NULL) pfree(scan->rs_parallelworkerdata); @@ -1939,7 +1914,6 @@ GetBulkInsertState(void) BulkInsertState bistate; bistate = (BulkInsertState) palloc_object(BulkInsertStateData); - bistate->strategy = GetAccessStrategy(BAS_BULKWRITE); bistate->current_buf = InvalidBuffer; bistate->next_free = InvalidBlockNumber; bistate->last_free = InvalidBlockNumber; @@ -1955,7 +1929,6 @@ FreeBulkInsertState(BulkInsertState bistate) { if (bistate->current_buf != InvalidBuffer) ReleaseBuffer(bistate->current_buf); - FreeAccessStrategy(bistate->strategy); pfree(bistate); } diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index bf87430cf0176..d14658d7218d4 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -2213,9 +2213,9 @@ heapam_scan_sample_next_block(TableScanDesc scan, SampleScanState *scanstate) */ CHECK_FOR_INTERRUPTS(); - /* Read page using selected strategy */ + /* Read page */ hscan->rs_cbuf = ReadBufferExtended(hscan->rs_base.rs_rd, MAIN_FORKNUM, - blockno, RBM_NORMAL, hscan->rs_strategy); + blockno, RBM_NORMAL); /* in pagemode, prune the page and determine visible tuple offsets */ if (hscan->rs_base.rs_flags & SO_ALLOW_PAGEMODE) diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index e96e0f77d9264..9bf7abc247f97 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -80,7 +80,8 @@ RelationPutHeapTuple(Relation relation, } /* - * Read in a buffer in mode, using bulk-insert strategy if bistate isn't NULL. + * Read in a buffer in mode, using the bistate's pinned target page if + * available. */ static Buffer ReadBufferBI(Relation relation, BlockNumber targetBlock, @@ -91,7 +92,7 @@ ReadBufferBI(Relation relation, BlockNumber targetBlock, /* If not bulk-insert, exactly like ReadBuffer */ if (!bistate) return ReadBufferExtended(relation, MAIN_FORKNUM, targetBlock, - mode, NULL); + mode); /* If we have the desired block already pinned, re-pin and return it */ if (bistate->current_buf != InvalidBuffer) @@ -113,9 +114,9 @@ ReadBufferBI(Relation relation, BlockNumber targetBlock, bistate->current_buf = InvalidBuffer; } - /* Perform a read using the buffer strategy */ + /* Perform the read */ buffer = ReadBufferExtended(relation, MAIN_FORKNUM, targetBlock, - mode, bistate->strategy); + mode); /* Save the selected block as target for future inserts */ IncrBufferRefCount(buffer); @@ -337,7 +338,6 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, * way larger. */ first_block = ExtendBufferedRelBy(BMR_REL(relation), MAIN_FORKNUM, - bistate ? bistate->strategy : NULL, EB_LOCK_FIRST, extend_by_pages, victim_buffers, @@ -484,8 +484,7 @@ RelationAddBlocks(Relation relation, BulkInsertState bistate, * * The caller can also provide a BulkInsertState object to optimize many * insertions into the same relation. This keeps a pin on the current - * insertion target page (to save pin/unpin cycles) and also passes a - * BULKWRITE buffer selection strategy object to the buffer manager. + * insertion target page (to save pin/unpin cycles). * Passing NULL for bistate selects the default behavior. * * We don't fill existing pages further than the fillfactor, except for large diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 39395aed0d592..8bc8e470af39f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -257,7 +257,6 @@ typedef struct LVRelState int nindexes; /* Buffer access strategy and parallel vacuum state */ - BufferAccessStrategy bstrategy; ParallelVacuumState *pvs; /* Aggressive VACUUM? (must set relfrozenxid >= FreezeLimit) */ @@ -621,8 +620,7 @@ heap_vacuum_eager_scan_setup(LVRelState *vacrel, const VacuumParams *params) * and locked the relation. */ void -heap_vacuum_rel(Relation rel, const VacuumParams *params, - BufferAccessStrategy bstrategy) +heap_vacuum_rel(Relation rel, const VacuumParams *params) { LVRelState *vacrel; bool verbose, @@ -699,7 +697,6 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, vacrel->rel = rel; vac_open_indexes(vacrel->rel, RowExclusiveLock, &vacrel->nindexes, &vacrel->indrels); - vacrel->bstrategy = bstrategy; if (instrument && vacrel->nindexes > 0) { /* Copy index names used by instrumentation (not error reporting) */ @@ -1311,7 +1308,6 @@ lazy_scan_heap(LVRelState *vacrel) * explicit work in heap_vac_scan_next_block. */ stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE, - vacrel->bstrategy, vacrel->rel, MAIN_FORKNUM, heap_vac_scan_next_block, @@ -2670,7 +2666,6 @@ lazy_vacuum_heap_rel(LVRelState *vacrel) */ stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_USE_BATCHING, - vacrel->bstrategy, vacrel->rel, MAIN_FORKNUM, vacuum_reap_lp_read_stream_next, @@ -2904,13 +2899,6 @@ lazy_check_wraparound_failsafe(LVRelState *vacrel) VacuumFailsafeActive = true; - /* - * Abandon use of a buffer access strategy to allow use of all of - * shared buffers. We assume the caller who allocated the memory for - * the BufferAccessStrategy will free it. - */ - vacrel->bstrategy = NULL; - /* Disable index vacuuming, index cleanup, and heap rel truncation */ vacrel->do_index_vacuuming = false; vacrel->do_index_cleanup = false; @@ -3023,7 +3011,6 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, ivinfo.estimated_count = true; ivinfo.message_level = DEBUG2; ivinfo.num_heap_tuples = reltuples; - ivinfo.strategy = vacrel->bstrategy; /* * Update error traceback information. @@ -3074,7 +3061,6 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, ivinfo.message_level = DEBUG2; ivinfo.num_heap_tuples = reltuples; - ivinfo.strategy = vacrel->bstrategy; /* * Update error traceback information. @@ -3354,8 +3340,7 @@ count_nondeletable_pages(LVRelState *vacrel, bool *lock_waiter_detected) prefetchedUntil = prefetchStart; } - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); + buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL); /* In this phase we only need shared access to the buffer */ LockBuffer(buf, BUFFER_LOCK_SHARE); @@ -3446,8 +3431,7 @@ dead_items_alloc(LVRelState *vacrel, int nworkers) vacrel->pvs = parallel_vacuum_init(vacrel->rel, vacrel->indrels, vacrel->nindexes, nworkers, vac_work_mem, - vacrel->verbose ? INFO : DEBUG2, - vacrel->bstrategy); + vacrel->verbose ? INFO : DEBUG2); /* * If parallel mode started, dead_items and dead_items_info spaces are diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 4fd470702aae7..b2612b1f71063 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -575,7 +575,7 @@ vm_readbuf(Relation rel, BlockNumber blkno, bool extend) } else buf = ReadBufferExtended(rel, VISIBILITYMAP_FORKNUM, blkno, - RBM_ZERO_ON_ERROR, NULL); + RBM_ZERO_ON_ERROR); /* * Initializing the page when needed is trickier than it looks, because of @@ -611,7 +611,7 @@ vm_extend(Relation rel, BlockNumber vm_nblocks) { Buffer buf; - buf = ExtendBufferedRelTo(BMR_REL(rel), VISIBILITYMAP_FORKNUM, NULL, + buf = ExtendBufferedRelTo(BMR_REL(rel), VISIBILITYMAP_FORKNUM, EB_CREATE_FORK_IF_NEEDED | EB_CLEAR_SIZE_CACHE, vm_nblocks, diff --git a/src/backend/access/nbtree/nbtpage.c b/src/backend/access/nbtree/nbtpage.c index 109017d6b5296..7b5fc16d75c46 100644 --- a/src/backend/access/nbtree/nbtpage.c +++ b/src/backend/access/nbtree/nbtpage.c @@ -960,7 +960,7 @@ _bt_allocbuf(Relation rel, Relation heaprel) * otherwise would make, as we can't use _bt_lockbuf() without introducing * a race. */ - buf = ExtendBufferedRel(BMR_REL(rel), MAIN_FORKNUM, NULL, EB_LOCK_FIRST); + buf = ExtendBufferedRel(BMR_REL(rel), MAIN_FORKNUM, EB_LOCK_FIRST); if (!RelationUsesLocalBuffers(rel)) VALGRIND_MAKE_MEM_DEFINED(BufferGetPage(buf), BLCKSZ); diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 3df2c752eadef..379a2d37def59 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -1324,7 +1324,6 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - info->strategy, rel, MAIN_FORKNUM, block_range_read_stream_cb, @@ -1730,8 +1729,7 @@ btvacuumpage(BTVacState *vstate, Buffer buf) * recycle all-zero pages, not fail. Also, we want to use a * nondefault buffer access strategy. */ - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - info->strategy); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL); goto backtrack; } diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c index f2ee333f60d84..785e385aab635 100644 --- a/src/backend/access/spgist/spgutils.c +++ b/src/backend/access/spgist/spgutils.c @@ -432,7 +432,7 @@ SpGistNewBuffer(Relation index) ReleaseBuffer(buffer); } - buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, NULL, + buffer = ExtendBufferedRel(BMR_REL(index), MAIN_FORKNUM, EB_LOCK_FIRST); return buffer; diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c index c461f8dc02d1b..9f644edaf8242 100644 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@ -704,8 +704,7 @@ spgprocesspending(spgBulkDeleteState *bds) /* examine the referenced page */ blkno = ItemPointerGetBlockNumber(&pitem->tid); - buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, - RBM_NORMAL, bds->info->strategy); + buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); page = BufferGetPage(buffer); @@ -830,7 +829,6 @@ spgvacuumscan(spgBulkDeleteState *bds) stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - bds->info->strategy, index, MAIN_FORKNUM, block_range_read_stream_cb, diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index f2e10b82b7d3e..0c16b7fef0f4a 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -1348,7 +1348,7 @@ log_newpage_range(Relation rel, ForkNumber forknum, while (nbufs < XLR_MAX_BLOCK_ID && blkno < endblk) { Buffer buf = ReadBufferExtended(rel, forknum, blkno, - RBM_NORMAL, NULL); + RBM_NORMAL); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index d8c179c5dccb2..e5be49f8b5db9 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -519,7 +519,7 @@ XLogReadBufferExtended(RelFileLocator rlocator, ForkNumber forknum, { /* page exists in file */ buffer = ReadBufferWithoutRelcache(rlocator, forknum, blkno, - mode, NULL, true); + mode, true); } else { @@ -536,7 +536,6 @@ XLogReadBufferExtended(RelFileLocator rlocator, ForkNumber forknum, Assert(InRecovery); buffer = ExtendBufferedRelTo(BMR_SMGR(smgr, RELPERSISTENCE_PERMANENT), forknum, - NULL, EB_PERFORMING_RECOVERY | EB_SKIP_EXTENSION_LOCK, blkno + 1, diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 81bba4beac7cb..b0a7b8a33cd02 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3431,7 +3431,6 @@ validate_index(Oid heapId, Oid indexId, Snapshot snapshot) ivinfo.estimated_count = true; ivinfo.message_level = DEBUG2; ivinfo.num_heap_tuples = heapRelation->rd_rel->reltuples; - ivinfo.strategy = NULL; /* * Encode TIDs as int8 values for the sort, rather than directly sorting diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index f66e80b757cbf..9b117a53124c3 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -72,7 +72,6 @@ int default_statistics_target = 100; /* A few variables that don't seem worth passing around as parameters */ static MemoryContext anl_context = NULL; -static BufferAccessStrategy vac_strategy; static void do_analyze_rel(Relation onerel, @@ -108,8 +107,7 @@ static Datum ind_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull); */ void analyze_rel(Oid relid, RangeVar *relation, - const VacuumParams *params, List *va_cols, bool in_outer_xact, - BufferAccessStrategy bstrategy) + const VacuumParams *params, List *va_cols, bool in_outer_xact) { Relation onerel; int elevel; @@ -124,7 +122,6 @@ analyze_rel(Oid relid, RangeVar *relation, elevel = DEBUG2; /* Set up static variables */ - vac_strategy = bstrategy; /* * Check for user-requested abort. @@ -730,7 +727,6 @@ do_analyze_rel(Relation onerel, const VacuumParams *params, ivinfo.estimated_count = true; ivinfo.message_level = elevel; ivinfo.num_heap_tuples = onerel->rd_rel->reltuples; - ivinfo.strategy = vac_strategy; stats = index_vacuum_cleanup(&ivinfo, NULL); @@ -1302,7 +1298,6 @@ acquire_sample_rows(Relation onerel, int elevel, */ stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | READ_STREAM_USE_BATCHING, - vac_strategy, scan->rs_rd, MAIN_FORKNUM, block_sampling_read_stream_next, diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index f0819d15ab701..6f65d48b6bd5a 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -262,7 +262,6 @@ ScanSourceDatabasePgClass(Oid tbid, Oid dbid, char *srcpath) LockRelId relid; Snapshot snapshot; SMgrRelation smgr; - BufferAccessStrategy bstrategy; /* Get pg_class relfilenumber. */ relfilenumber = RelationMapOidToFilenumberForDatabase(srcpath, @@ -282,9 +281,6 @@ ScanSourceDatabasePgClass(Oid tbid, Oid dbid, char *srcpath) nblocks = smgrnblocks(smgr, MAIN_FORKNUM); smgrclose(smgr); - /* Use a buffer access strategy since this is a bulk read operation. */ - bstrategy = GetAccessStrategy(BAS_BULKREAD); - /* * As explained in the function header comments, we need a snapshot that * will see all committed transactions as committed, and our transaction @@ -299,7 +295,7 @@ ScanSourceDatabasePgClass(Oid tbid, Oid dbid, char *srcpath) CHECK_FOR_INTERRUPTS(); buf = ReadBufferWithoutRelcache(rlocator, MAIN_FORKNUM, blkno, - RBM_NORMAL, bstrategy, true); + RBM_NORMAL, true); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 02883fe34a485..d8c15a8b1bcad 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -2447,7 +2447,7 @@ process_single_relation(RepackStmt *stmt, LOCKMODE lockmode, bool isTopLevel, if (params->options & CLUOPT_VERBOSE) vac_params.options |= VACOPT_VERBOSE; analyze_rel(tableOid, NULL, &vac_params, - stmt->relation->va_cols, true, NULL); + stmt->relation->va_cols, true); PopActiveSnapshot(); CommandCounterIncrement(); } diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index 551667650ba63..ac1ab9482114f 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -358,7 +358,7 @@ fill_seq_fork_with_data(Relation rel, HeapTuple tuple, ForkNumber forkNum) /* Initialize first page of relation with special magic number */ - buf = ExtendBufferedRel(BMR_REL(rel), forkNum, NULL, + buf = ExtendBufferedRel(BMR_REL(rel), forkNum, EB_LOCK_FIRST | EB_SKIP_EXTENSION_LOCK); Assert(BufferGetBlockNumber(buf) == 0); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 38539a6fd3d4e..66acd27bd488d 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -127,32 +127,11 @@ static void vac_truncate_clog(TransactionId frozenXID, TransactionId lastSaneFrozenXid, MultiXactId lastSaneMinMulti); static bool vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, - BufferAccessStrategy bstrategy, bool isTopLevel); + bool isTopLevel); static double compute_parallel_delay(void); static VacOptValue get_vacoptval_from_boolean(DefElem *def); static bool vac_tid_reaped(ItemPointer itemptr, void *state); -/* - * GUC check function to ensure GUC value specified is within the allowable - * range. - */ -bool -check_vacuum_buffer_usage_limit(int *newval, void **extra, - GucSource source) -{ - /* Value upper and lower hard limits are inclusive */ - if (*newval == 0 || (*newval >= MIN_BAS_VAC_RING_SIZE_KB && - *newval <= MAX_BAS_VAC_RING_SIZE_KB)) - return true; - - /* Value does not fall within any allowable range */ - GUC_check_errdetail("\"%s\" must be 0 or between %d kB and %d kB.", - "vacuum_buffer_usage_limit", - MIN_BAS_VAC_RING_SIZE_KB, MAX_BAS_VAC_RING_SIZE_KB); - - return false; -} - /* * Primary entry point for manual VACUUM and ANALYZE commands * @@ -163,7 +142,6 @@ void ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) { VacuumParams params; - BufferAccessStrategy bstrategy = NULL; bool verbose = false; bool skip_locked = false; bool analyze = false; @@ -172,7 +150,6 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) bool disable_page_skipping = false; bool process_main = true; bool process_toast = true; - int ring_size; bool skip_database_stats = false; bool only_database_stats = false; MemoryContext vac_context; @@ -188,12 +165,6 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) /* Will be set later if we recurse to a TOAST table. */ params.toast_parent = InvalidOid; - /* - * Set this to an invalid value so it is clear whether or not a - * BUFFER_USAGE_LIMIT was specified when making the access strategy. - */ - ring_size = -1; - /* Parse options list */ foreach(lc, vacstmt->options) { @@ -204,32 +175,6 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) verbose = defGetBoolean(opt); else if (strcmp(opt->defname, "skip_locked") == 0) skip_locked = defGetBoolean(opt); - else if (strcmp(opt->defname, "buffer_usage_limit") == 0) - { - const char *hintmsg; - int result; - char *vac_buffer_size; - - vac_buffer_size = defGetString(opt); - - /* - * Check that the specified value is valid and the size falls - * within the hard upper and lower limits if it is not 0. - */ - if (!parse_int(vac_buffer_size, &result, GUC_UNIT_KB, &hintmsg) || - (result != 0 && - (result < MIN_BAS_VAC_RING_SIZE_KB || result > MAX_BAS_VAC_RING_SIZE_KB))) - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("%s option must be 0 or between %d kB and %d kB", - "BUFFER_USAGE_LIMIT", - MIN_BAS_VAC_RING_SIZE_KB, MAX_BAS_VAC_RING_SIZE_KB), - hintmsg ? errhint_internal("%s", _(hintmsg)) : 0)); - } - - ring_size = result; - } else if (!vacstmt->is_vacuumcmd) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -325,17 +270,6 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("VACUUM FULL cannot be performed in parallel"))); - /* - * BUFFER_USAGE_LIMIT does nothing for VACUUM (FULL) so just raise an - * ERROR for that case. VACUUM (FULL, ANALYZE) does make use of it, so - * we'll permit that. - */ - if (ring_size != -1 && (params.options & VACOPT_FULL) && - !(params.options & VACOPT_ANALYZE)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("BUFFER_USAGE_LIMIT cannot be specified for VACUUM FULL"))); - /* * Make sure VACOPT_ANALYZE is specified if any column lists are present. */ @@ -431,38 +365,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) "Vacuum", ALLOCSET_DEFAULT_SIZES); - /* - * Make a buffer strategy object in the cross-transaction memory context. - * We needn't bother making this for VACUUM (FULL) or VACUUM - * (ONLY_DATABASE_STATS) as they'll not make use of it. VACUUM (FULL, - * ANALYZE) is possible, so we'd better ensure that we make a strategy - * when we see ANALYZE. - */ - if ((params.options & (VACOPT_ONLY_DATABASE_STATS | - VACOPT_FULL)) == 0 || - (params.options & VACOPT_ANALYZE) != 0) - { - - MemoryContext old_context = MemoryContextSwitchTo(vac_context); - - Assert(ring_size >= -1); - - /* - * If BUFFER_USAGE_LIMIT was specified by the VACUUM or ANALYZE - * command, it overrides the value of VacuumBufferUsageLimit. Either - * value may be 0, in which case GetAccessStrategyWithSize() will - * return NULL, effectively allowing full use of shared buffers. - */ - if (ring_size == -1) - ring_size = VacuumBufferUsageLimit; - - bstrategy = GetAccessStrategyWithSize(BAS_VACUUM, ring_size); - - MemoryContextSwitchTo(old_context); - } - /* Now go through the common routine */ - vacuum(vacstmt->rels, ¶ms, bstrategy, vac_context, isTopLevel); + vacuum(vacstmt->rels, ¶ms, vac_context, isTopLevel); /* Finally, clean up the vacuum memory context */ MemoryContextDelete(vac_context); @@ -479,19 +383,13 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) * params contains a set of parameters that can be used to customize the * behavior. * - * bstrategy may be passed in as NULL when the caller does not want to - * restrict the number of shared_buffers that VACUUM / ANALYZE can use, - * otherwise, the caller must build a BufferAccessStrategy with the number of - * shared_buffers that VACUUM / ANALYZE should try to limit themselves to - * using. - * * isTopLevel should be passed down from ProcessUtility. * * It is the caller's responsibility that all parameters are allocated in a * memory context that will not disappear at transaction commit. */ void -vacuum(List *relations, const VacuumParams *params, BufferAccessStrategy bstrategy, +vacuum(List *relations, const VacuumParams *params, MemoryContext vac_context, bool isTopLevel) { static bool in_vacuum = false; @@ -630,7 +528,7 @@ vacuum(List *relations, const VacuumParams *params, BufferAccessStrategy bstrate if (params->options & VACOPT_VACUUM) { - if (!vacuum_rel(vrel->oid, vrel->relation, *params, bstrategy, + if (!vacuum_rel(vrel->oid, vrel->relation, *params, isTopLevel)) continue; } @@ -649,7 +547,7 @@ vacuum(List *relations, const VacuumParams *params, BufferAccessStrategy bstrate } analyze_rel(vrel->oid, vrel->relation, params, - vrel->va_cols, in_outer_xact, bstrategy); + vrel->va_cols, in_outer_xact); if (use_own_xacts) { @@ -2010,7 +1908,7 @@ vac_truncate_clog(TransactionId frozenXID, */ static bool vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, - BufferAccessStrategy bstrategy, bool isTopLevel) + bool isTopLevel) { LOCKMODE lmode; Relation rel; @@ -2307,7 +2205,7 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, rel = NULL; } else - table_relation_vacuum(rel, ¶ms, bstrategy); + table_relation_vacuum(rel, ¶ms); } /* Roll back any GUC changes executed by index functions */ @@ -2344,7 +2242,7 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, toast_vacuum_params.options |= VACOPT_PROCESS_MAIN; toast_vacuum_params.toast_parent = relid; - vacuum_rel(toast_relid, NULL, toast_vacuum_params, bstrategy, + vacuum_rel(toast_relid, NULL, toast_vacuum_params, isTopLevel); } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 41cefcfde54fe..8b4b835db9bad 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -125,12 +125,6 @@ typedef struct PVShared */ int maintenance_work_mem_worker; - /* - * The number of buffers each worker's Buffer Access Strategy ring should - * contain. - */ - int ring_nbuffers; - /* * Shared vacuum cost balance. During parallel vacuum, * VacuumSharedCostBalance points to this value and it accumulates the @@ -257,9 +251,6 @@ struct ParallelVacuumState int nindexes_parallel_cleanup; int nindexes_parallel_condcleanup; - /* Buffer access strategy used by leader process */ - BufferAccessStrategy bstrategy; - /* * Error reporting state. The error callback is set only for workers * processes during parallel index vacuum. @@ -304,7 +295,7 @@ static void parallel_vacuum_dsm_detach(dsm_segment *seg, Datum arg); ParallelVacuumState * parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, int nrequested_workers, int vac_work_mem, - int elevel, BufferAccessStrategy bstrategy) + int elevel) { ParallelVacuumState *pvs; ParallelContext *pcxt; @@ -345,7 +336,6 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, pvs->indrels = indrels; pvs->nindexes = nindexes; pvs->will_parallel_vacuum = will_parallel_vacuum; - pvs->bstrategy = bstrategy; pvs->heaprel = rel; EnterParallelMode(); @@ -447,9 +437,6 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, shared->dead_items_handle = TidStoreGetHandle(dead_items); shared->dead_items_dsa_handle = dsa_get_handle(TidStoreGetDSA(dead_items)); - /* Use the same buffer size for all workers */ - shared->ring_nbuffers = GetAccessStrategyBufferCount(bstrategy); - pg_atomic_init_u32(&(shared->cost_balance), 0); pg_atomic_init_u32(&(shared->active_nworkers), 0); pg_atomic_init_u32(&(shared->idx), 0); @@ -1091,7 +1078,6 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, ivinfo.message_level = DEBUG2; ivinfo.estimated_count = pvs->shared->estimated_count; ivinfo.num_heap_tuples = pvs->shared->reltuples; - ivinfo.strategy = pvs->bstrategy; /* Update error traceback information */ pvs->indname = pstrdup(RelationGetRelationName(indrel)); @@ -1294,10 +1280,6 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) pvs.indname = NULL; pvs.status = PARALLEL_INDVAC_STATUS_INITIAL; - /* Each parallel VACUUM worker gets its own access strategy. */ - pvs.bstrategy = GetAccessStrategyWithSize(BAS_VACUUM, - shared->ring_nbuffers * (BLCKSZ / 1024)); - /* Setup error traceback support for ereport() */ errcallback.callback = parallel_vacuum_error_callback; errcallback.arg = &pvs; @@ -1328,7 +1310,6 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(rel, ShareUpdateExclusiveLock); - FreeAccessStrategy(pvs.bstrategy); if (shared->is_autovacuum) pv_shared_cost_params = NULL; diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 45abf48768afd..e021e798805ab 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -386,8 +386,7 @@ static void relation_needs_vacanalyze(Oid relid, AutoVacOpts *relopts, bool *dovacuum, bool *doanalyze, bool *wraparound, AutoVacuumScores *scores); -static void autovacuum_do_vac_analyze(autovac_table *tab, - BufferAccessStrategy bstrategy); +static void autovacuum_do_vac_analyze(autovac_table *tab); static AutoVacOpts *extract_autovac_opts(HeapTuple tup, TupleDesc pg_class_desc); static void perform_work_item(AutoVacuumWorkItem *workitem); @@ -1936,7 +1935,6 @@ do_autovacuum(void) HASHCTL ctl; HTAB *table_toast_map; ListCell *volatile cell; - BufferAccessStrategy bstrategy; ScanKeyData key; TupleDesc pg_class_desc; int effective_multixact_freeze_max_age; @@ -2322,23 +2320,6 @@ do_autovacuum(void) autovacuum_analyze_score_weight != 0.0) list_sort(tables_to_process, TableToProcessComparator); - /* - * Optionally, create a buffer access strategy object for VACUUM to use. - * We use the same BufferAccessStrategy object for all tables VACUUMed by - * this worker to prevent autovacuum from blowing out shared buffers. - * - * VacuumBufferUsageLimit being set to 0 results in - * GetAccessStrategyWithSize returning NULL, effectively meaning we can - * use up to all of shared buffers. - * - * If we later enter failsafe mode on any of the tables being vacuumed, we - * will cease use of the BufferAccessStrategy only for that table. - * - * XXX should we consider adding code to adjust the size of this if - * VacuumBufferUsageLimit changes? - */ - bstrategy = GetAccessStrategyWithSize(BAS_VACUUM, VacuumBufferUsageLimit); - /* * create a memory context to act as fake PortalContext, so that the * contexts created in the vacuum code are cleaned up for each table. @@ -2516,7 +2497,7 @@ do_autovacuum(void) MemoryContextSwitchTo(PortalContext); /* have at it */ - autovacuum_do_vac_analyze(tab, bstrategy); + autovacuum_do_vac_analyze(tab); /* * Clear a possible query-cancel signal, to avoid a late reaction @@ -2636,8 +2617,6 @@ do_autovacuum(void) #ifdef USE_VALGRIND hash_destroy(table_toast_map); FreeTupleDesc(pg_class_desc); - if (bstrategy) - pfree(bstrategy); #endif /* Run the rest in xact context, mainly to avoid Valgrind leak warnings */ @@ -3348,7 +3327,7 @@ relation_needs_vacanalyze(Oid relid, * disappear at transaction commit. */ static void -autovacuum_do_vac_analyze(autovac_table *tab, BufferAccessStrategy bstrategy) +autovacuum_do_vac_analyze(autovac_table *tab) { RangeVar *rangevar; VacuumRelation *rel; @@ -3371,7 +3350,7 @@ autovacuum_do_vac_analyze(autovac_table *tab, BufferAccessStrategy bstrategy) rel_list = list_make1(rel); MemoryContextSwitchTo(old_context); - vacuum(rel_list, &tab->at_params, bstrategy, vac_context, true); + vacuum(rel_list, &tab->at_params, vac_context, true); MemoryContextDelete(vac_context); } diff --git a/src/backend/postmaster/datachecksum_state.c b/src/backend/postmaster/datachecksum_state.c index 73dc539836b01..b6612f1e51794 100644 --- a/src/backend/postmaster/datachecksum_state.c +++ b/src/backend/postmaster/datachecksum_state.c @@ -390,7 +390,7 @@ static List *BuildRelationList(bool temp_relations, bool include_shared); static void FreeDatabaseList(List *dblist); static DataChecksumsWorkerResult ProcessDatabase(DataChecksumsWorkerDatabase *db); static bool ProcessAllDatabases(void); -static bool ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrategy strategy); +static bool ProcessSingleRelationFork(Relation reln, ForkNumber forkNum); static void ResetDataChecksumsProgressCounters(void); static void launcher_cancel_handler(SIGNAL_ARGS); static void WaitForAllTransactionsToFinish(void); @@ -687,7 +687,7 @@ StartDataChecksumsWorkerLauncher(DataChecksumsWorkerOperation op, * error is raised in the lower levels. */ static bool -ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrategy strategy) +ProcessSingleRelationFork(Relation reln, ForkNumber forkNum) { BlockNumber numblocks = RelationGetNumberOfBlocksInFork(reln, forkNum); char activity[NAMEDATALEN * 2 + 128]; @@ -722,7 +722,7 @@ ProcessSingleRelationFork(Relation reln, ForkNumber forkNum, BufferAccessStrateg */ for (BlockNumber blknum = 0; blknum < numblocks; blknum++) { - Buffer buf = ReadBufferExtended(reln, forkNum, blknum, RBM_NORMAL, strategy); + Buffer buf = ReadBufferExtended(reln, forkNum, blknum, RBM_NORMAL); /* Need to get an exclusive lock to mark the buffer as dirty */ LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); @@ -808,7 +808,7 @@ ResetDataChecksumsProgressCounters(void) * error is raised in the lower levels. */ static bool -ProcessSingleRelationByOid(Oid relationId, BufferAccessStrategy strategy) +ProcessSingleRelationByOid(Oid relationId) { Relation rel; bool aborted = false; @@ -835,7 +835,7 @@ ProcessSingleRelationByOid(Oid relationId, BufferAccessStrategy strategy) { if (smgrexists(rel->rd_smgr, fnum)) { - if (!ProcessSingleRelationFork(rel, fnum, strategy)) + if (!ProcessSingleRelationFork(rel, fnum)) { aborted = true; break; @@ -1582,7 +1582,6 @@ DataChecksumsWorkerMain(Datum arg) Oid dboid; List *RelationList = NIL; List *InitialTempTableList = NIL; - BufferAccessStrategy strategy; bool aborted = false; int64 rels_done; bool process_shared; @@ -1650,11 +1649,6 @@ DataChecksumsWorkerMain(Datum arg) VacuumUpdateCosts(); VacuumCostBalance = 0; - /* - * Create and set the vacuum strategy as our buffer strategy. - */ - strategy = GetAccessStrategy(BAS_VACUUM); - RelationList = BuildRelationList(false, process_shared); /* Update the total number of relations to be processed in this DB. */ @@ -1676,9 +1670,7 @@ DataChecksumsWorkerMain(Datum arg) rels_done = 0; foreach_oid(reloid, RelationList) { - bool costs_updated = false; - - if (!ProcessSingleRelationByOid(reloid, strategy)) + if (!ProcessSingleRelationByOid(reloid)) { aborted = true; break; @@ -1694,8 +1686,7 @@ DataChecksumsWorkerMain(Datum arg) /* * Check if the cost settings changed during runtime and if so, update - * to reflect the new values and signal that the access strategy needs - * to be refreshed. + * to reflect the new values. */ LWLockAcquire(DataChecksumsWorkerLock, LW_EXCLUSIVE); if (DataChecksumState->worker_invocation != worker_invocation) @@ -1706,7 +1697,6 @@ DataChecksumsWorkerMain(Datum arg) if ((DataChecksumState->launch_cost_delay != DataChecksumState->cost_delay) || (DataChecksumState->launch_cost_limit != DataChecksumState->cost_limit)) { - costs_updated = true; VacuumCostDelay = DataChecksumState->launch_cost_delay; VacuumCostLimit = DataChecksumState->launch_cost_limit; VacuumUpdateCosts(); @@ -1714,19 +1704,10 @@ DataChecksumsWorkerMain(Datum arg) DataChecksumState->cost_delay = DataChecksumState->launch_cost_delay; DataChecksumState->cost_limit = DataChecksumState->launch_cost_limit; } - else - costs_updated = false; LWLockRelease(DataChecksumsWorkerLock); - - if (costs_updated) - { - FreeAccessStrategy(strategy); - strategy = GetAccessStrategy(BAS_VACUUM); - } } list_free(RelationList); - FreeAccessStrategy(strategy); if (aborted || abort_requested) { diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index a318539e56cbe..08d344f063d5d 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -757,7 +757,6 @@ read_stream_look_ahead(ReadStream *stream) */ static ReadStream * read_stream_begin_impl(int flags, - BufferAccessStrategy strategy, Relation rel, SMgrRelation smgr, char persistence, @@ -771,7 +770,6 @@ read_stream_begin_impl(int flags, int16 queue_size; int16 queue_overflow; int max_ios; - int strategy_pin_limit; uint32 max_pinned_buffers; uint32 max_possible_buffer_limit; Oid tablespace_id; @@ -835,10 +833,6 @@ read_stream_begin_impl(int flags, max_pinned_buffers = Min(max_pinned_buffers, PG_INT16_MAX - queue_overflow - 1); - /* Give the strategy a chance to limit the number of buffers we pin. */ - strategy_pin_limit = GetAccessStrategyPinLimit(strategy); - max_pinned_buffers = Min(strategy_pin_limit, max_pinned_buffers); - /* * Also limit our queue to the maximum number of pins we could ever be * allowed to acquire according to the buffer manager. We may not really @@ -962,7 +956,6 @@ read_stream_begin_impl(int flags, stream->ios[i].op.smgr = smgr; stream->ios[i].op.persistence = persistence; stream->ios[i].op.forknum = forknum; - stream->ios[i].op.strategy = strategy; } return stream; @@ -974,7 +967,6 @@ read_stream_begin_impl(int flags, */ ReadStream * read_stream_begin_relation(int flags, - BufferAccessStrategy strategy, Relation rel, ForkNumber forknum, ReadStreamBlockNumberCB callback, @@ -982,7 +974,6 @@ read_stream_begin_relation(int flags, size_t per_buffer_data_size) { return read_stream_begin_impl(flags, - strategy, rel, RelationGetSmgr(rel), rel->rd_rel->relpersistence, @@ -998,7 +989,6 @@ read_stream_begin_relation(int flags, */ ReadStream * read_stream_begin_smgr_relation(int flags, - BufferAccessStrategy strategy, SMgrRelation smgr, char smgr_persistence, ForkNumber forknum, @@ -1007,7 +997,6 @@ read_stream_begin_smgr_relation(int flags, size_t per_buffer_data_size) { return read_stream_begin_impl(flags, - strategy, NULL, smgr, smgr_persistence, @@ -1370,13 +1359,11 @@ read_stream_next_buffer(ReadStream *stream, void **per_buffer_data) * Transitional support for code that would like to perform or skip reads * itself, without using the stream. Returns, and consumes, the next block * number that would be read by the stream's look-ahead algorithm, or - * InvalidBlockNumber if the end of the stream is reached. Also reports the - * strategy that would be used to read it. + * InvalidBlockNumber if the end of the stream is reached. */ BlockNumber -read_stream_next_block(ReadStream *stream, BufferAccessStrategy *strategy) +read_stream_next_block(ReadStream *stream) { - *strategy = stream->ios[0].op.strategy; return read_stream_get_block(stream, NULL); } diff --git a/src/backend/storage/buffer/README b/src/backend/storage/buffer/README index b332e002ba13b..58df6bd011aa1 100644 --- a/src/backend/storage/buffer/README +++ b/src/backend/storage/buffer/README @@ -203,50 +203,6 @@ have to give up and try another buffer. This however is not a concern of the basic select-a-victim-buffer algorithm.) -Buffer Ring Replacement Strategy ---------------------------------- - -When running a query that needs to access a large number of pages just once, -such as VACUUM or a large sequential scan, a different strategy is used. -A page that has been touched only by such a scan is unlikely to be needed -again soon, so instead of running the normal clock-sweep algorithm and -blowing out the entire buffer cache, a small ring of buffers is allocated -using the normal clock-sweep algorithm and those buffers are reused for the -whole scan. This also implies that much of the write traffic caused by such -a statement will be done by the backend itself and not pushed off onto other -processes. - -For sequential scans, a 256KB ring is used. That's small enough to fit in L2 -cache, which makes transferring pages from OS cache to shared buffer cache -efficient. Even less would often be enough, but the ring must be big enough -to accommodate all pages in the scan that are pinned concurrently. 256KB -should also be enough to leave a small cache trail for other backends to -join in a synchronized seq scan. If a ring buffer is dirtied and its LSN -updated, we would normally have to write and flush WAL before we could -re-use the buffer; in this case we instead discard the buffer from the ring -and (later) choose a replacement using the normal clock-sweep algorithm. -Hence this strategy works best for scans that are read-only (or at worst -update hint bits). In a scan that modifies every page in the scan, like a -bulk UPDATE or DELETE, the buffers in the ring will always be dirtied and -the ring strategy effectively degrades to the normal strategy. - -VACUUM uses a ring like sequential scans, however, the size of this ring is -controlled by the vacuum_buffer_usage_limit GUC. Dirty pages are not removed -from the ring. Instead, the WAL is flushed if needed to allow reuse of the -buffers. Before introducing the buffer ring strategy in 8.3, VACUUM's buffers -were sent to the freelist, which was effectively a buffer ring of 1 buffer, -resulting in excessive WAL flushing. - -Bulk writes work similarly to VACUUM. Currently this applies only to -COPY IN and CREATE TABLE AS SELECT. (Might it be interesting to make -seqscan UPDATE and DELETE use the bulkwrite strategy?) For bulk writes -we use a ring size of 16MB (but not more than 1/8th of shared_buffers). -Smaller sizes have been shown to result in the COPY blocking too often -for WAL flushes. While it's okay for a background vacuum to be slowed by -doing its own WAL flushing, we'd prefer that COPY not be subject to that, -so we let it use up a bit more of the buffer arena. - - Background Writer's Processing ------------------------------ diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 1f5a00849e6aa..cc2fab959df5b 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -612,10 +612,9 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) static Buffer ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy); + ReadBufferMode mode); static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, BlockNumber extend_upto, @@ -623,13 +622,12 @@ static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, uint32 *extended_by); static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer *buffers, uint32 *extended_by); -static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy, +static bool PinBuffer(BufferDesc *buf, bool skip_if_not_valid); static void PinBuffer_Locked(BufferDesc *buf); static void UnpinBuffer(BufferDesc *buf); @@ -645,7 +643,6 @@ static inline BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, - BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context); static bool AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress); static void CheckReadBuffersOperation(ReadBuffersOperation *operation, bool is_complete); @@ -654,7 +651,7 @@ static pg_always_inline void TrackBufferHit(IOObject io_object, IOContext io_context, Relation rel, char persistence, SMgrRelation smgr, ForkNumber forknum, BlockNumber blocknum); -static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context); +static Buffer GetVictimBuffer(IOContext io_context); static void FlushUnlockedBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io_context); static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, @@ -858,7 +855,7 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN * pin. */ if (BufferTagsEqual(&tag, &bufHdr->tag) && - PinBuffer(bufHdr, NULL, true)) + PinBuffer(bufHdr, true)) { if (BufferTagsEqual(&tag, &bufHdr->tag)) { @@ -874,12 +871,12 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN /* * ReadBuffer -- a shorthand for ReadBufferExtended, for reading from main - * fork with RBM_NORMAL mode and default strategy. + * fork with RBM_NORMAL mode. */ Buffer ReadBuffer(Relation reln, BlockNumber blockNum) { - return ReadBufferExtended(reln, MAIN_FORKNUM, blockNum, RBM_NORMAL, NULL); + return ReadBufferExtended(reln, MAIN_FORKNUM, blockNum, RBM_NORMAL); } /* @@ -919,13 +916,10 @@ ReadBuffer(Relation reln, BlockNumber blockNum) * a cleanup-strength lock on the page. * * RBM_NORMAL_NO_LOG mode is treated the same as RBM_NORMAL here. - * - * If strategy is not NULL, a nondefault buffer access strategy is used. - * See buffer/README for details. */ inline Buffer ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy) + ReadBufferMode mode) { Buffer buf; @@ -935,7 +929,7 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, * ReadBuffer_common(). */ buf = ReadBuffer_common(reln, RelationGetSmgr(reln), 0, - forkNum, blockNum, mode, strategy); + forkNum, blockNum, mode); return buf; } @@ -954,14 +948,14 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, Buffer ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy, bool permanent) + bool permanent) { SMgrRelation smgr = smgropen(rlocator, INVALID_PROC_NUMBER); return ReadBuffer_common(NULL, smgr, permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED, forkNum, blockNum, - mode, strategy); + mode); } /* @@ -970,13 +964,12 @@ ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, Buffer ExtendBufferedRel(BufferManagerRelation bmr, ForkNumber forkNum, - BufferAccessStrategy strategy, uint32 flags) { Buffer buf; uint32 extend_by = 1; - ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by, + ExtendBufferedRelBy(bmr, forkNum, flags, extend_by, &buf, &extend_by); return buf; @@ -1002,7 +995,6 @@ ExtendBufferedRel(BufferManagerRelation bmr, BlockNumber ExtendBufferedRelBy(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, Buffer *buffers, @@ -1015,7 +1007,7 @@ ExtendBufferedRelBy(BufferManagerRelation bmr, if (bmr.relpersistence == '\0') bmr.relpersistence = bmr.rel->rd_rel->relpersistence; - return ExtendBufferedRelCommon(bmr, fork, strategy, flags, + return ExtendBufferedRelCommon(bmr, fork, flags, extend_by, InvalidBlockNumber, buffers, extended_by); } @@ -1031,7 +1023,6 @@ ExtendBufferedRelBy(BufferManagerRelation bmr, Buffer ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, BlockNumber extend_to, ReadBufferMode mode) @@ -1097,7 +1088,7 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, if ((uint64) current_size + num_pages > extend_to) num_pages = extend_to - current_size; - first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags, + first_block = ExtendBufferedRelCommon(bmr, fork, flags, num_pages, extend_to, buffers, &extended_by); @@ -1123,7 +1114,7 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, { Assert(extended_by == 0); buffer = ReadBuffer_common(bmr.rel, BMR_GET_SMGR(bmr), bmr.relpersistence, - fork, extend_to - 1, mode, strategy); + fork, extend_to - 1, mode); } return buffer; @@ -1226,7 +1217,6 @@ PinBufferForBlock(Relation rel, char persistence, ForkNumber forkNum, BlockNumber blockNum, - BufferAccessStrategy strategy, IOObject io_object, IOContext io_context, bool *foundPtr) @@ -1250,7 +1240,7 @@ PinBufferForBlock(Relation rel, bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, foundPtr); else bufHdr = BufferAlloc(smgr, persistence, forkNum, blockNum, - strategy, foundPtr, io_context); + foundPtr, io_context); if (*foundPtr) TrackBufferHit(io_object, io_context, rel, persistence, smgr, forkNum, blockNum); @@ -1276,8 +1266,7 @@ PinBufferForBlock(Relation rel, static pg_always_inline Buffer ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence, ForkNumber forkNum, - BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy) + BlockNumber blockNum, ReadBufferMode mode) { ReadBuffersOperation operation; Buffer buffer; @@ -1313,7 +1302,7 @@ ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence, if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) flags |= EB_LOCK_FIRST; - return ExtendBufferedRel(BMR_REL(rel), forkNum, strategy, flags); + return ExtendBufferedRel(BMR_REL(rel), forkNum, flags); } if (rel) @@ -1335,12 +1324,12 @@ ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence, } else { - io_context = IOContextForStrategy(strategy); + io_context = IOCONTEXT_NORMAL; io_object = IOOBJECT_RELATION; } buffer = PinBufferForBlock(rel, smgr, persistence, - forkNum, blockNum, strategy, + forkNum, blockNum, io_object, io_context, &found); ZeroAndLockBuffer(buffer, mode, found); return buffer; @@ -1358,7 +1347,6 @@ ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence, operation.rel = rel; operation.persistence = persistence; operation.forknum = forkNum; - operation.strategy = strategy; if (StartReadBuffer(&operation, &buffer, blockNum, @@ -1399,7 +1387,7 @@ StartReadBuffersImpl(ReadBuffersOperation *operation, } else { - io_context = IOContextForStrategy(operation->strategy); + io_context = IOCONTEXT_NORMAL; io_object = IOOBJECT_RELATION; } @@ -1450,7 +1438,6 @@ StartReadBuffersImpl(ReadBuffersOperation *operation, operation->persistence, operation->forknum, blockNum + i, - operation->strategy, io_object, io_context, &found); } @@ -1771,7 +1758,7 @@ WaitReadBuffers(ReadBuffersOperation *operation) } else { - io_context = IOContextForStrategy(operation->strategy); + io_context = IOCONTEXT_NORMAL; io_object = IOOBJECT_RELATION; } @@ -1961,7 +1948,7 @@ AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress) } else { - io_context = IOContextForStrategy(operation->strategy); + io_context = IOCONTEXT_NORMAL; io_object = IOOBJECT_RELATION; } @@ -2180,24 +2167,18 @@ AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress) * buffer. If no buffer exists already, selects a replacement victim and * evicts the old page, but does NOT read in new page. * - * "strategy" can be a buffer replacement strategy object, or NULL for - * the default strategy. The selected buffer's usage_count is advanced when - * using the default strategy, but otherwise possibly not (see PinBuffer). - * * The returned buffer is pinned and is already marked as holding the * desired page. If it already did have the desired page, *foundPtr is * set true. Otherwise, *foundPtr is set false. * - * io_context is passed as an output parameter to avoid calling - * IOContextForStrategy() when there is a shared buffers hit and no IO - * statistics need be captured. + * io_context is passed as an output parameter to avoid capturing IO + * statistics when there is a shared buffers hit and no IO occurs. * * No locks are held either at entry or exit. */ static pg_always_inline BufferDesc * BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, - BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context) { BufferTag newTag; /* identity of requested block */ @@ -2235,7 +2216,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, */ buf = GetBufferDescriptor(existing_buf_id); - valid = PinBuffer(buf, strategy, false); + valid = PinBuffer(buf, false); /* Can release the mapping lock as soon as we've pinned it */ LWLockRelease(newPartitionLock); @@ -2266,7 +2247,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * don't hold any conflicting locks. If so we'll have to undo our work * later. */ - victim_buffer = GetVictimBuffer(strategy, io_context); + victim_buffer = GetVictimBuffer(io_context); victim_buf_hdr = GetBufferDescriptor(victim_buffer - 1); /* @@ -2297,7 +2278,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, existing_buf_hdr = GetBufferDescriptor(existing_buf_id); - valid = PinBuffer(existing_buf_hdr, strategy, false); + valid = PinBuffer(existing_buf_hdr, false); /* Can release the mapping lock as soon as we've pinned it */ LWLockRelease(newPartitionLock); @@ -2335,9 +2316,12 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * just like permanent relations. */ set_bits |= BM_TAG_VALID; - /* Admit the newly loaded page COOL (probation); a second access via + + /* + * Admit the newly loaded page COOL (probation); a second access via * PinBuffer promotes it to HOT. This is what makes a one-touch scan - * self-evicting -- see the cooling-state notes in buf_internals.h. */ + * self-evicting -- see the cooling-state notes in buf_internals.h. + */ if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -2549,12 +2533,11 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) } static Buffer -GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) +GetVictimBuffer(IOContext io_context) { BufferDesc *buf_hdr; Buffer buf; uint64 buf_state; - bool from_ring; /* * Ensure, before we pin a victim buffer, that there's a free refcount @@ -2570,7 +2553,7 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * Select a victim buffer. The buffer is returned pinned and owned by * this backend. */ - buf_hdr = StrategyGetBuffer(strategy, &buf_state, &from_ring); + buf_hdr = StrategyGetBuffer(&buf_state); buf = BufferDescriptorGetBuffer(buf_hdr); /* @@ -2614,26 +2597,6 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) goto again; } - /* - * If using a nondefault strategy, and this victim came from the - * strategy ring, let the strategy decide whether to reject it when - * reusing it would require a WAL flush. This only applies to - * permanent buffers; unlogged buffers can have fake LSNs, so - * XLogNeedsFlush() is not meaningful for them. - * - * We need to hold the content lock in at least share-exclusive mode - * to safely inspect the page LSN, so this couldn't have been done - * inside StrategyGetBuffer(). - */ - if (strategy && from_ring && - buf_state & BM_PERMANENT && - XLogNeedsFlush(BufferGetLSN(buf_hdr)) && - StrategyRejectBuffer(strategy, buf_hdr, from_ring)) - { - UnlockReleaseBuffer(buf); - goto again; - } - /* OK, do the I/O */ FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context); LockBuffer(buf, BUFFER_LOCK_UNLOCK); @@ -2646,23 +2609,15 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) if (buf_state & BM_VALID) { /* - * When a BufferAccessStrategy is in use, blocks evicted from shared - * buffers are counted as IOOP_EVICT in the corresponding context - * (e.g. IOCONTEXT_BULKWRITE). Shared buffers are evicted by a - * strategy in two cases: 1) while initially claiming buffers for the - * strategy ring 2) to replace an existing strategy ring buffer - * because it is pinned or in use and cannot be reused. + * Blocks evicted from shared buffers are counted as IOOP_EVICT. * - * Blocks evicted from buffers already in the strategy ring are - * counted as IOOP_REUSE in the corresponding strategy context. - * - * At this point, we can accurately count evictions and reuses, - * because we have successfully claimed the valid buffer. Previously, - * we may have been forced to release the buffer due to concurrent - * pinners or erroring out. + * At this point, we can accurately count evictions, because we have + * successfully claimed the valid buffer. Previously, we may have been + * forced to release the buffer due to concurrent pinners or erroring + * out. */ pgstat_count_io_op(IOOBJECT_RELATION, io_context, - from_ring ? IOOP_REUSE : IOOP_EVICT, 1, 0); + IOOP_EVICT, 1, 0); } /* @@ -2754,7 +2709,6 @@ LimitAdditionalPins(uint32 *additional_pins) static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, BlockNumber extend_upto, @@ -2789,7 +2743,7 @@ ExtendBufferedRelCommon(BufferManagerRelation bmr, buffers, &extend_by); } else - first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags, + first_block = ExtendBufferedRelShared(bmr, fork, flags, extend_by, extend_upto, buffers, &extend_by); *extended_by = extend_by; @@ -2812,7 +2766,6 @@ ExtendBufferedRelCommon(BufferManagerRelation bmr, static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, BlockNumber extend_upto, @@ -2820,7 +2773,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, uint32 *extended_by) { BlockNumber first_block; - IOContext io_context = IOContextForStrategy(strategy); + IOContext io_context = IOCONTEXT_NORMAL; instr_time io_start; LimitAdditionalPins(&extend_by); @@ -2839,7 +2792,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, { Block buf_block; - buffers[i] = GetVictimBuffer(strategy, io_context); + buffers[i] = GetVictimBuffer(io_context); buf_block = BufHdrGetBlock(GetBufferDescriptor(buffers[i] - 1)); /* new buffers are zero-filled */ @@ -2957,7 +2910,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * Pin the existing buffer before releasing the partition lock, * preventing it from being evicted. */ - valid = PinBuffer(existing_hdr, strategy, false); + valid = PinBuffer(existing_hdr, false); LWLockRelease(partition_lock); UnpinBuffer(victim_buf_hdr); @@ -3007,8 +2960,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; set_bits |= BM_TAG_VALID; - /* Admit COOL (probation); see the comment at the other admission - * site and the cooling-state notes in buf_internals.h. */ + + /* + * Admit COOL (probation); see the comment at the other admission + * site and the cooling-state notes in buf_internals.h. + */ if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -3275,13 +3231,8 @@ ReleaseAndReadBuffer(Buffer buffer, /* * PinBuffer -- make buffer unavailable for replacement. * - * For the default access strategy, the buffer's usage_count is incremented - * when we first pin it; for other strategies we just make sure the usage_count - * isn't zero. (The idea of the latter is that we don't want synchronized - * heap scans to inflate the count, but we need it to not be zero to discourage - * other backends from stealing buffers from our ring. As long as we cycle - * through the ring faster than the global clock-sweep cycles, buffers in - * our ring won't be chosen as victims for replacement by other backends.) + * The buffer's cooling state is promoted to HOT (the 2Q rescue) when we pin + * it; see the cooling-state notes in buf_internals.h. * * This should be applied only to shared buffers, never local ones. * @@ -3298,7 +3249,7 @@ ReleaseAndReadBuffer(Buffer buffer, * (recently) invalid and has not been pinned. */ static bool -PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy, +PinBuffer(BufferDesc *buf, bool skip_if_not_valid) { Buffer b = BufferDescriptorGetBuffer(buf); @@ -3396,7 +3347,7 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy, * ResourceOwnerEnlarge(CurrentResourceOwner); * * Currently, no callers of this function want to modify the buffer's - * usage_count at all, so there's no need for a strategy parameter. + * cooling state at all. * Also we don't bother with a BM_VALID test (the caller could check that for * itself). * @@ -4674,22 +4625,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, false); /* - * When a strategy is in use, only flushes of dirty buffers already in the - * strategy ring are counted as strategy writes (IOCONTEXT - * [BULKREAD|BULKWRITE|VACUUM] IOOP_WRITE) for the purpose of IO - * statistics tracking. - * - * If a shared buffer initially added to the ring must be flushed before - * being used, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE. - * - * If a shared buffer which was added to the ring later because the - * current strategy buffer is pinned or in use or because all strategy - * buffers were dirty and rejected (for BAS_BULKREAD operations only) - * requires flushing, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE - * (from_ring will be false). - * - * When a strategy is not in use, the write can only be a "regular" write - * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE). + * All writes of a dirty shared buffer are counted as an IOCONTEXT_NORMAL + * IOOP_WRITE for the purpose of IO statistics tracking. */ pgstat_count_io_op_time(io_object, io_context, IOOP_WRITE, io_start, 1, BLCKSZ); @@ -5450,8 +5387,6 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, BlockNumber nblocks; BlockNumber blkno; PGIOAlignedBlock buf; - BufferAccessStrategy bstrategy_src; - BufferAccessStrategy bstrategy_dst; BlockRangeReadStreamPrivate p; ReadStream *src_stream; SMgrRelation src_smgr; @@ -5479,10 +5414,6 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, smgrextend(smgropen(dstlocator, INVALID_PROC_NUMBER), forkNum, nblocks - 1, buf.data, true); - /* This is a bulk operation, so use buffer access strategies. */ - bstrategy_src = GetAccessStrategy(BAS_BULKREAD); - bstrategy_dst = GetAccessStrategy(BAS_BULKWRITE); - /* Initialize streaming read */ p.current_blocknum = 0; p.last_exclusive = nblocks; @@ -5494,7 +5425,6 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, */ src_stream = read_stream_begin_smgr_relation(READ_STREAM_FULL | READ_STREAM_USE_BATCHING, - bstrategy_src, src_smgr, permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED, forkNum, @@ -5514,7 +5444,7 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum, BufferGetBlockNumber(srcBuf), - RBM_ZERO_AND_LOCK, bstrategy_dst, + RBM_ZERO_AND_LOCK, permanent); dstPage = BufferGetPage(dstBuf); @@ -5535,9 +5465,6 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, } Assert(read_stream_next_buffer(src_stream, NULL) == InvalidBuffer); read_stream_end(src_stream); - - FreeAccessStrategy(bstrategy_src); - FreeAccessStrategy(bstrategy_dst); } /* --------------------------------------------------------------------- diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c index e8cf13f3362fb..ee62b33668003 100644 --- a/src/backend/storage/buffer/freelist.c +++ b/src/backend/storage/buffer/freelist.c @@ -77,40 +77,6 @@ const ShmemCallbacks StrategyCtlShmemCallbacks = { .init_fn = StrategyCtlShmemInit, }; -/* - * Private (non-shared) state for managing a ring of shared buffers to re-use. - * This is currently the only kind of BufferAccessStrategy object, but someday - * we might have more kinds. - */ -typedef struct BufferAccessStrategyData -{ - /* Overall strategy type */ - BufferAccessStrategyType btype; - /* Number of elements in buffers[] array */ - int nbuffers; - - /* - * Index of the "current" slot in the ring, ie, the one most recently - * returned by GetBufferFromRing. - */ - int current; - - /* - * Array of buffer numbers. InvalidBuffer (that is, zero) indicates we - * have not yet selected a buffer for this ring slot. For allocation - * simplicity this is palloc'd together with the fixed fields of the - * struct. - */ - Buffer buffers[FLEXIBLE_ARRAY_MEMBER]; -} BufferAccessStrategyData; - - -/* Prototypes for internal functions */ -static BufferDesc *GetBufferFromRing(BufferAccessStrategy strategy, - uint64 *buf_state); -static void AddBufferToRing(BufferAccessStrategy strategy, - BufferDesc *buf); - /* * Per-backend state for the batched clock sweep. Each backend claims a run * of consecutive clock-hand values with a single atomic fetch-add and then @@ -157,8 +123,8 @@ ClockSweepTick(void) * must hold the spinlock so StrategySyncStart() can read * nextVictimBuffer and completePasses consistently. * - * With batching, multiple backends may each land a fetch-add - * that returns a value past NBuffers in the same pass. After + * With batching, multiple backends may each land a fetch-add that + * returns a value past NBuffers in the same pass. After * acquiring the spinlock we re-read the counter: if another * backend already wrapped it below NBuffers we are done. */ @@ -214,8 +180,6 @@ ClockSweepTick(void) * GetVictimBuffer(). The only hard requirement GetVictimBuffer() has is that * the selected buffer must not currently be pinned by anyone. * - * strategy is a BufferAccessStrategy object, or NULL for default strategy. - * * It is the callers responsibility to ensure the buffer ownership can be * tracked via TrackNewBufferPin(). * @@ -223,29 +187,13 @@ ClockSweepTick(void) * before returning. */ BufferDesc * -StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_ring) +StrategyGetBuffer(uint64 *buf_state) { BufferDesc *buf; int bgwprocno; int trycounter; bool force_cool; - *from_ring = false; - - /* - * If given a strategy object, see whether it can select a buffer. We - * assume strategy objects don't need buffer_strategy_lock. - */ - if (strategy != NULL) - { - buf = GetBufferFromRing(strategy, buf_state); - if (buf != NULL) - { - *from_ring = true; - return buf; - } - } - /* * If asked, we need to waken the bgwriter. Since we don't want to rely on * a spinlock for this we force a read from shared memory once, and then @@ -274,8 +222,7 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r /* * We count buffer allocation requests so that the bgwriter can estimate - * the rate of buffer consumption. Note that buffers recycled by a - * strategy object are intentionally not counted here. + * the rate of buffer consumption. */ pg_atomic_fetch_add_u32(&StrategyControl->numBufferAllocs, 1); @@ -347,7 +294,7 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r if (!force_cool) { no_progress = true; - break; /* advance the hand, look for COOL */ + break; /* advance the hand, look for COOL */ } if (BUF_STATE_GET_REFBIT(local_buf_state)) @@ -387,8 +334,6 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r local_buf_state)) { /* Found a usable buffer */ - if (strategy != NULL) - AddBufferToRing(strategy, buf); *buf_state = local_buf_state; TrackNewBufferPin(BufferDescriptorGetBuffer(buf)); @@ -399,11 +344,12 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r } /* - * A tick that made no progress toward a victim counts down trycounter. - * A full unproductive pass escalates to force_cool (cool HOT buffers - * into victims); a second unproductive full pass means everything is - * pinned, so fail rather than spin forever. (A failed CAS above is - * neither progress nor a full miss: we simply retry the same buffer.) + * A tick that made no progress toward a victim counts down + * trycounter. A full unproductive pass escalates to force_cool (cool + * HOT buffers into victims); a second unproductive full pass means + * everything is pinned, so fail rather than spin forever. (A failed + * CAS above is neither progress nor a full miss: we simply retry the + * same buffer.) */ if (no_progress && --trycounter == 0) { @@ -531,361 +477,3 @@ StrategyCtlShmemInit(void *arg) else StrategyControl->batchSize = 1; } - - -/* ---------------------------------------------------------------- - * Backend-private buffer ring management - * ---------------------------------------------------------------- - */ - - -/* - * GetAccessStrategy -- create a BufferAccessStrategy object - * - * The object is allocated in the current memory context. - */ -BufferAccessStrategy -GetAccessStrategy(BufferAccessStrategyType btype) -{ - int ring_size_kb; - - /* - * Select ring size to use. See buffer/README for rationales. - * - * Note: if you change the ring size for BAS_BULKREAD, see also - * SYNC_SCAN_REPORT_INTERVAL in access/heap/syncscan.c. - */ - switch (btype) - { - case BAS_NORMAL: - /* if someone asks for NORMAL, just give 'em a "default" object */ - return NULL; - - case BAS_BULKREAD: - { - int ring_max_kb; - - /* - * The ring always needs to be large enough to allow some - * separation in time between providing a buffer to the user - * of the strategy and that buffer being reused. Otherwise the - * user's pin will prevent reuse of the buffer, even without - * concurrent activity. - * - * We also need to ensure the ring always is large enough for - * SYNC_SCAN_REPORT_INTERVAL, as noted above. - * - * Thus we start out a minimal size and increase the size - * further if appropriate. - */ - ring_size_kb = 256; - - /* - * There's no point in a larger ring if we won't be allowed to - * pin sufficiently many buffers. But we never limit to less - * than the minimal size above. - */ - ring_max_kb = GetPinLimit() * (BLCKSZ / 1024); - ring_max_kb = Max(ring_size_kb, ring_max_kb); - - /* - * We would like the ring to additionally have space for the - * configured degree of IO concurrency. While being read in, - * buffers can obviously not yet be reused. - * - * Each IO can be up to io_combine_limit blocks large, and we - * want to start up to effective_io_concurrency IOs. - * - * Note that effective_io_concurrency may be 0, which disables - * AIO. - */ - ring_size_kb += (BLCKSZ / 1024) * - io_combine_limit * effective_io_concurrency; - - if (ring_size_kb > ring_max_kb) - ring_size_kb = ring_max_kb; - break; - } - case BAS_BULKWRITE: - ring_size_kb = 16 * 1024; - break; - case BAS_VACUUM: - ring_size_kb = 2048; - break; - - default: - elog(ERROR, "unrecognized buffer access strategy: %d", - (int) btype); - return NULL; /* keep compiler quiet */ - } - - return GetAccessStrategyWithSize(btype, ring_size_kb); -} - -/* - * GetAccessStrategyWithSize -- create a BufferAccessStrategy object with a - * number of buffers equivalent to the passed in size. - * - * If the given ring size is 0, no BufferAccessStrategy will be created and - * the function will return NULL. ring_size_kb must not be negative. - */ -BufferAccessStrategy -GetAccessStrategyWithSize(BufferAccessStrategyType btype, int ring_size_kb) -{ - int ring_buffers; - BufferAccessStrategy strategy; - - Assert(ring_size_kb >= 0); - - /* Figure out how many buffers ring_size_kb is */ - ring_buffers = ring_size_kb / (BLCKSZ / 1024); - - /* 0 means unlimited, so no BufferAccessStrategy required */ - if (ring_buffers == 0) - return NULL; - - /* Cap to 1/8th of shared_buffers */ - ring_buffers = Min(NBuffers / 8, ring_buffers); - - /* NBuffers should never be less than 16, so this shouldn't happen */ - Assert(ring_buffers > 0); - - /* Allocate the object and initialize all elements to zeroes */ - strategy = (BufferAccessStrategy) - palloc0(offsetof(BufferAccessStrategyData, buffers) + - ring_buffers * sizeof(Buffer)); - - /* Set fields that don't start out zero */ - strategy->btype = btype; - strategy->nbuffers = ring_buffers; - - return strategy; -} - -/* - * GetAccessStrategyBufferCount -- an accessor for the number of buffers in - * the ring - * - * Returns 0 on NULL input to match behavior of GetAccessStrategyWithSize() - * returning NULL with 0 size. - */ -int -GetAccessStrategyBufferCount(BufferAccessStrategy strategy) -{ - if (strategy == NULL) - return 0; - - return strategy->nbuffers; -} - -/* - * GetAccessStrategyPinLimit -- get cap of number of buffers that should be pinned - * - * When pinning extra buffers to look ahead, users of a ring-based strategy are - * in danger of pinning too much of the ring at once while performing look-ahead. - * For some strategies, that means "escaping" from the ring, and in others it - * means forcing dirty data to disk very frequently with associated WAL - * flushing. Since external code has no insight into any of that, allow - * individual strategy types to expose a clamp that should be applied when - * deciding on a maximum number of buffers to pin at once. - * - * Callers should combine this number with other relevant limits and take the - * minimum. - */ -int -GetAccessStrategyPinLimit(BufferAccessStrategy strategy) -{ - if (strategy == NULL) - return NBuffers; - - switch (strategy->btype) - { - case BAS_BULKREAD: - - /* - * Since BAS_BULKREAD uses StrategyRejectBuffer(), dirty buffers - * shouldn't be a problem and the caller is free to pin up to the - * entire ring at once. - */ - return strategy->nbuffers; - - default: - - /* - * Tell caller not to pin more than half the buffers in the ring. - * This is a trade-off between look ahead distance and deferring - * writeback and associated WAL traffic. - */ - return strategy->nbuffers / 2; - } -} - -/* - * FreeAccessStrategy -- release a BufferAccessStrategy object - * - * A simple pfree would do at the moment, but we would prefer that callers - * don't assume that much about the representation of BufferAccessStrategy. - */ -void -FreeAccessStrategy(BufferAccessStrategy strategy) -{ - /* don't crash if called on a "default" strategy */ - if (strategy != NULL) - pfree(strategy); -} - -/* - * GetBufferFromRing -- returns a buffer from the ring, or NULL if the - * ring is empty / not usable. - * - * The buffer is pinned and marked as owned, using TrackNewBufferPin(), before - * returning. - */ -static BufferDesc * -GetBufferFromRing(BufferAccessStrategy strategy, uint64 *buf_state) -{ - BufferDesc *buf; - Buffer bufnum; - uint64 old_buf_state; - uint64 local_buf_state; /* to avoid repeated (de-)referencing */ - - - /* Advance to next ring slot */ - if (++strategy->current >= strategy->nbuffers) - strategy->current = 0; - - /* - * If the slot hasn't been filled yet, tell the caller to allocate a new - * buffer with the normal allocation strategy. He will then fill this - * slot by calling AddBufferToRing with the new buffer. - */ - bufnum = strategy->buffers[strategy->current]; - if (bufnum == InvalidBuffer) - return NULL; - - buf = GetBufferDescriptor(bufnum - 1); - - /* - * Check whether the buffer can be used and pin it if so. Do this using a - * CAS loop, to avoid having to lock the buffer header. - */ - old_buf_state = pg_atomic_read_u64(&buf->state); - for (;;) - { - local_buf_state = old_buf_state; - - /* - * If the buffer is pinned we cannot use it under any circumstances. - * - * With the cooling-state replacement the field holds only COOL or HOT, - * so the stock "usage_count > 1 means another backend touched it" - * heuristic no longer applies: a ring element is reusable whenever it - * is unpinned. (The whole ring mechanism is removed in a later patch; - * scan resistance is now intrinsic to the sweep.) - */ - if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0) - break; - - /* See equivalent code in PinBuffer() */ - if (unlikely(local_buf_state & BM_LOCKED)) - { - old_buf_state = WaitBufHdrUnlocked(buf); - continue; - } - - /* pin the buffer if the CAS succeeds */ - local_buf_state += BUF_REFCOUNT_ONE; - - if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, - local_buf_state)) - { - *buf_state = local_buf_state; - - TrackNewBufferPin(BufferDescriptorGetBuffer(buf)); - return buf; - } - } - - /* - * Tell caller to allocate a new buffer with the normal allocation - * strategy. He'll then replace this ring element via AddBufferToRing. - */ - return NULL; -} - -/* - * AddBufferToRing -- add a buffer to the buffer ring - * - * Caller must hold the buffer header spinlock on the buffer. Since this - * is called with the spinlock held, it had better be quite cheap. - */ -static void -AddBufferToRing(BufferAccessStrategy strategy, BufferDesc *buf) -{ - strategy->buffers[strategy->current] = BufferDescriptorGetBuffer(buf); -} - -/* - * Utility function returning the IOContext of a given BufferAccessStrategy's - * strategy ring. - */ -IOContext -IOContextForStrategy(BufferAccessStrategy strategy) -{ - if (!strategy) - return IOCONTEXT_NORMAL; - - switch (strategy->btype) - { - case BAS_NORMAL: - - /* - * Currently, GetAccessStrategy() returns NULL for - * BufferAccessStrategyType BAS_NORMAL, so this case is - * unreachable. - */ - pg_unreachable(); - return IOCONTEXT_NORMAL; - case BAS_BULKREAD: - return IOCONTEXT_BULKREAD; - case BAS_BULKWRITE: - return IOCONTEXT_BULKWRITE; - case BAS_VACUUM: - return IOCONTEXT_VACUUM; - } - - elog(ERROR, "unrecognized BufferAccessStrategyType: %d", strategy->btype); - pg_unreachable(); -} - -/* - * StrategyRejectBuffer -- consider rejecting a dirty buffer - * - * When a nondefault strategy is used, the buffer manager calls this function - * when it turns out that the buffer selected by StrategyGetBuffer needs to - * be written out and doing so would require flushing WAL too. This gives us - * a chance to choose a different victim. - * - * Returns true if buffer manager should ask for a new victim, and false - * if this buffer should be written and re-used. - */ -bool -StrategyRejectBuffer(BufferAccessStrategy strategy, BufferDesc *buf, bool from_ring) -{ - /* We only do this in bulkread mode */ - if (strategy->btype != BAS_BULKREAD) - return false; - - /* Don't muck with behavior of normal buffer-replacement strategy */ - if (!from_ring || - strategy->buffers[strategy->current] != BufferDescriptorGetBuffer(buf)) - return false; - - /* - * Remove the dirty buffer from the ring; necessary to prevent infinite - * loop if all ring members are dirty. - */ - strategy->buffers[strategy->current] = InvalidBuffer; - - return true; -} diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c index 006edab9d77df..29cb2d70f66e4 100644 --- a/src/backend/storage/freespace/freespace.c +++ b/src/backend/storage/freespace/freespace.c @@ -603,7 +603,7 @@ fsm_readbuf(Relation rel, FSMAddress addr, bool extend) return InvalidBuffer; } else - buf = ReadBufferExtended(rel, FSM_FORKNUM, blkno, RBM_ZERO_ON_ERROR, NULL); + buf = ReadBufferExtended(rel, FSM_FORKNUM, blkno, RBM_ZERO_ON_ERROR); /* * Initializing the page when needed is trickier than it looks, because of @@ -638,7 +638,7 @@ fsm_readbuf(Relation rel, FSMAddress addr, bool extend) static Buffer fsm_extend(Relation rel, BlockNumber fsm_nblocks) { - return ExtendBufferedRelTo(BMR_REL(rel), FSM_FORKNUM, NULL, + return ExtendBufferedRelTo(BMR_REL(rel), FSM_FORKNUM, EB_CREATE_FORK_IF_NEEDED | EB_CLEAR_SIZE_CACHE, fsm_nblocks, diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 718c1cfc0f9df..3c3e35ab8a86f 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -1540,15 +1540,8 @@ register_dirty_segment(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg) FilePathName(seg->mdfd_vfd)))); /* - * We have no way of knowing if the current IOContext is - * IOCONTEXT_NORMAL or IOCONTEXT_[BULKREAD, BULKWRITE, VACUUM] at this - * point, so count the fsync as being in the IOCONTEXT_NORMAL - * IOContext. This is probably okay, because the number of backend - * fsyncs doesn't say anything about the efficacy of the - * BufferAccessStrategy. And counting both fsyncs done in - * IOCONTEXT_NORMAL and IOCONTEXT_[BULKREAD, BULKWRITE, VACUUM] under - * IOCONTEXT_NORMAL is likely clearer when investigating the number of - * backend fsyncs. + * Count the fsync in the IOCONTEXT_NORMAL IOContext, the only + * IOContext relations are read/written under. */ pgstat_count_io_op_time(IOOBJECT_RELATION, IOCONTEXT_NORMAL, IOOP_FSYNC, io_start, 1, 0); diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 38bae7b15d2ed..d66a23749d5a9 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -241,16 +241,10 @@ pgstat_get_io_context_name(IOContext io_context) { switch (io_context) { - case IOCONTEXT_BULKREAD: - return "bulkread"; - case IOCONTEXT_BULKWRITE: - return "bulkwrite"; case IOCONTEXT_INIT: return "init"; case IOCONTEXT_NORMAL: return "normal"; - case IOCONTEXT_VACUUM: - return "vacuum"; } elog(ERROR, "unrecognized IOContext value: %d", io_context); @@ -451,18 +445,6 @@ pgstat_tracks_io_object(BackendType bktype, IOObject io_object, * IOContexts, and, while it may not be inherently incorrect for them to * do so, excluding those rows from the view makes the view easier to use. */ - if ((bktype == B_CHECKPOINTER || bktype == B_BG_WRITER) && - (io_context == IOCONTEXT_BULKREAD || - io_context == IOCONTEXT_BULKWRITE || - io_context == IOCONTEXT_VACUUM)) - return false; - - if (bktype == B_AUTOVAC_LAUNCHER && io_context == IOCONTEXT_VACUUM) - return false; - - if ((bktype == B_AUTOVAC_WORKER || bktype == B_AUTOVAC_LAUNCHER) && - io_context == IOCONTEXT_BULKWRITE) - return false; return true; } @@ -479,8 +461,6 @@ bool pgstat_tracks_io_op(BackendType bktype, IOObject io_object, IOContext io_context, IOOp io_op) { - bool strategy_io_context; - /* if (io_context, io_object) will never collect stats, we're done */ if (!pgstat_tracks_io_object(bktype, io_object, io_context)) return false; @@ -521,17 +501,13 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, /* * Some IOOps are not valid in certain IOContexts and some IOOps are only * valid in certain contexts. + * + * IOOP_REUSE was only relevant when a BufferAccessStrategy was in use. + * Buffer access strategies (ring buffers) have been removed -- scan + * resistance is now intrinsic to the cooling-stage clock sweep -- so + * IOOP_REUSE never occurs. */ - if (io_context == IOCONTEXT_BULKREAD && io_op == IOOP_EXTEND) - return false; - - strategy_io_context = io_context == IOCONTEXT_BULKREAD || - io_context == IOCONTEXT_BULKWRITE || io_context == IOCONTEXT_VACUUM; - - /* - * IOOP_REUSE is only relevant when a BufferAccessStrategy is in use. - */ - if (!strategy_io_context && io_op == IOOP_REUSE) + if (io_op == IOOP_REUSE) return false; /* @@ -545,14 +521,5 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, !(io_op == IOOP_WRITE || io_op == IOOP_READ || io_op == IOOP_FSYNC)) return false; - /* - * IOOP_FSYNC IOOps done by a backend using a BufferAccessStrategy are - * counted in the IOCONTEXT_NORMAL IOContext. See comment in - * register_dirty_segment() for more details. - */ - if (strategy_io_context && io_op == IOOP_FSYNC) - return false; - - return true; } diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index bbd28d14d9948..555c04272f532 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -149,7 +149,6 @@ int autovacuum_max_parallel_workers = 0; int MaxBackends = 0; /* GUC parameters for vacuum */ -int VacuumBufferUsageLimit = 2048; int VacuumCostPageHit = 1; int VacuumCostPageMiss = 2; diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index d421cdbde76da..b1ea81f536c99 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -3326,16 +3326,6 @@ boot_val => 'DEFAULT_UPDATE_PROCESS_TITLE', }, -{ name => 'vacuum_buffer_usage_limit', type => 'int', context => 'PGC_USERSET', group => 'RESOURCES_MEM', - short_desc => 'Sets the buffer pool size for VACUUM, ANALYZE, and autovacuum.', - flags => 'GUC_UNIT_KB', - variable => 'VacuumBufferUsageLimit', - boot_val => '2048', - min => '0', - max => 'MAX_BAS_VAC_RING_SIZE_KB', - check_hook => 'check_vacuum_buffer_usage_limit', -}, - { name => 'vacuum_cost_delay', type => 'real', context => 'PGC_USERSET', group => 'VACUUM_COST_DELAY', short_desc => 'Vacuum cost delay in milliseconds.', flags => 'GUC_UNIT_MS', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 7958653077b16..cb1eb8ef3ffbc 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -164,9 +164,6 @@ # mmap # (change requires restart) #min_dynamic_shared_memory = 0MB # (change requires restart) -#vacuum_buffer_usage_limit = 2MB # size of vacuum and analyze buffer access strategy ring; - # 0 to disable vacuum buffer access strategy; - # range 128kB to 16GB # SLRU buffers (change requires restart) #commit_timestamp_buffers = 0 # memory for pg_commit_ts (0 = auto) diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index f8158ca6a78fa..f78e57ff2c3e2 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -57,7 +57,6 @@ main(int argc, char *argv[]) {"no-truncate", no_argument, NULL, 10}, {"no-process-toast", no_argument, NULL, 11}, {"no-process-main", no_argument, NULL, 12}, - {"buffer-usage-limit", required_argument, NULL, 13}, {"missing-stats-only", no_argument, NULL, 14}, {"dry-run", no_argument, NULL, 15}, {NULL, 0, NULL, 0} @@ -202,9 +201,6 @@ main(int argc, char *argv[]) case 12: vacopts.process_main = false; break; - case 13: - vacopts.buffer_usage_limit = escape_quotes(optarg); - break; case 14: vacopts.missing_stats_only = true; break; @@ -290,14 +286,6 @@ main(int argc, char *argv[]) pg_fatal("cannot use the \"%s\" option with the \"%s\" option", "no-index-cleanup", "force-index-cleanup"); - /* - * buffer-usage-limit is not allowed with VACUUM FULL unless ANALYZE is - * included too. - */ - if (vacopts.buffer_usage_limit && vacopts.full && !vacopts.and_analyze) - pg_fatal("cannot use the \"%s\" option with the \"%s\" option", - "buffer-usage-limit", "full"); - /* * Prohibit --missing-stats-only without --analyze-only or * --analyze-in-stages. @@ -352,7 +340,6 @@ help(const char *progname) printf(_(" %s [OPTION]... [DBNAME]\n"), progname); printf(_("\nOptions:\n")); printf(_(" -a, --all vacuum all databases\n")); - printf(_(" --buffer-usage-limit=SIZE size of ring buffer used for vacuum\n")); printf(_(" -d, --dbname=DBNAME database to vacuum\n")); printf(_(" --disable-page-skipping disable all page-skipping behavior\n")); printf(_(" --dry-run show the commands that would be sent to the server\n")); diff --git a/src/bin/scripts/vacuuming.c b/src/bin/scripts/vacuuming.c index 855a5754c98c1..078a3bbb51681 100644 --- a/src/bin/scripts/vacuuming.c +++ b/src/bin/scripts/vacuuming.c @@ -264,13 +264,6 @@ vacuum_one_database(ConnParams *cparams, "--parallel", "13"); } - if (vacopts->buffer_usage_limit && PQserverVersion(conn) < 160000) - { - PQfinish(conn); - pg_fatal("cannot use the \"%s\" option on server versions older than PostgreSQL %s", - "--buffer-usage-limit", "16"); - } - if (vacopts->missing_stats_only && PQserverVersion(conn) < 150000) { PQfinish(conn); @@ -865,13 +858,6 @@ prepare_vacuum_command(PGconn *conn, PQExpBuffer sql, appendPQExpBuffer(sql, "%sVERBOSE", sep); sep = comma; } - if (vacopts->buffer_usage_limit) - { - Assert(serverVersion >= 160000); - appendPQExpBuffer(sql, "%sBUFFER_USAGE_LIMIT '%s'", sep, - vacopts->buffer_usage_limit); - sep = comma; - } if (sep != paren) appendPQExpBufferChar(sql, ')'); } @@ -974,13 +960,6 @@ prepare_vacuum_command(PGconn *conn, PQExpBuffer sql, vacopts->parallel_workers); sep = comma; } - if (vacopts->buffer_usage_limit) - { - Assert(serverVersion >= 160000); - appendPQExpBuffer(sql, "%sBUFFER_USAGE_LIMIT '%s'", sep, - vacopts->buffer_usage_limit); - sep = comma; - } if (sep != paren) appendPQExpBufferChar(sql, ')'); } diff --git a/src/bin/scripts/vacuuming.h b/src/bin/scripts/vacuuming.h index 5a491db2526d7..83a2469a6aaf8 100644 --- a/src/bin/scripts/vacuuming.h +++ b/src/bin/scripts/vacuuming.h @@ -49,7 +49,6 @@ typedef struct vacuumingOptions bool process_main; bool process_toast; bool skip_database_stats; - char *buffer_usage_limit; bool missing_stats_only; bool echo; bool quiet; diff --git a/src/include/access/genam.h b/src/include/access/genam.h index 68bfe405db3a0..bc87f6f7cc0ed 100644 --- a/src/include/access/genam.h +++ b/src/include/access/genam.h @@ -58,7 +58,6 @@ typedef struct IndexVacuumInfo bool estimated_count; /* num_heap_tuples is an estimate */ int message_level; /* ereport level for progress messages */ double num_heap_tuples; /* tuples remaining in heap */ - BufferAccessStrategy strategy; /* access strategy for reads */ } IndexVacuumInfo; /* diff --git a/src/include/access/hash.h b/src/include/access/hash.h index a8702f0e5ea13..7c9268fe95370 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -405,12 +405,11 @@ extern void _hash_pgaddmultitup(Relation rel, Buffer buf, IndexTuple *itups, extern Buffer _hash_addovflpage(Relation rel, Buffer metabuf, Buffer buf, bool retain_pin); extern BlockNumber _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, Buffer wbuf, IndexTuple *itups, OffsetNumber *itup_offsets, - Size *tups_size, uint16 nitups, BufferAccessStrategy bstrategy); + Size *tups_size, uint16 nitups); extern void _hash_initbitmapbuffer(Buffer buf, uint16 bmsize, bool initpage); extern void _hash_squeezebucket(Relation rel, Bucket bucket, BlockNumber bucket_blkno, - Buffer bucket_buf, - BufferAccessStrategy bstrategy); + Buffer bucket_buf); extern uint32 _hash_ovflblkno_to_bitno(HashMetaPage metap, BlockNumber ovflblkno); /* hashpage.c */ @@ -428,9 +427,6 @@ extern void _hash_initbuf(Buffer buf, uint32 max_bucket, uint32 num_bucket, uint32 flag, bool initpage); extern Buffer _hash_getnewbuf(Relation rel, BlockNumber blkno, ForkNumber forkNum); -extern Buffer _hash_getbuf_with_strategy(Relation rel, BlockNumber blkno, - int access, int flags, - BufferAccessStrategy bstrategy); extern void _hash_relbuf(Relation rel, Buffer buf); extern void _hash_dropbuf(Relation rel, Buffer buf); extern void _hash_dropscanbuf(Relation rel, HashScanOpaque so); @@ -481,7 +477,6 @@ extern void _hash_kill_items(IndexScanDesc scan); /* hash.c */ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, BlockNumber bucket_blkno, - BufferAccessStrategy bstrategy, uint32 maxbucket, uint32 highmask, uint32 lowmask, double *tuples_removed, double *num_index_tuples, bool split_cleanup, diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 5176478c29583..0967b659794c2 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -72,8 +72,6 @@ typedef struct HeapScanDescData Buffer rs_cbuf; /* current buffer in scan, if any */ /* NB: if rs_cbuf is not InvalidBuffer, we hold a pin on that buffer */ - BufferAccessStrategy rs_strategy; /* access strategy for reads */ - HeapTupleData rs_ctup; /* current tuple in scan, if any */ /* For scans that stream reads */ @@ -466,7 +464,7 @@ extern void log_heap_prune_and_freeze(Relation relation, Buffer buffer, /* in heap/vacuumlazy.c */ extern void heap_vacuum_rel(Relation rel, - const VacuumParams *params, BufferAccessStrategy bstrategy); + const VacuumParams *params); #ifdef USE_ASSERT_CHECKING extern bool heap_page_is_all_visible(Relation rel, Buffer buf, GlobalVisState *vistest, diff --git a/src/include/access/hio.h b/src/include/access/hio.h index 60cfc375fd523..63bfcc884eea1 100644 --- a/src/include/access/hio.h +++ b/src/include/access/hio.h @@ -28,7 +28,6 @@ */ typedef struct BulkInsertStateData { - BufferAccessStrategy strategy; /* our BULKWRITE strategy object */ Buffer current_buf; /* current insertion target page */ /* diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index f2c36696bcad0..83ba0f3ca730d 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -689,8 +689,7 @@ typedef struct TableAmRoutine * integrate with autovacuum's scheduling. */ void (*relation_vacuum) (Relation rel, - const VacuumParams *params, - BufferAccessStrategy bstrategy); + const VacuumParams *params); /* * Prepare to analyze block `blockno` of `scan`. The scan has been started @@ -1774,10 +1773,9 @@ table_relation_copy_for_cluster(Relation OldTable, Relation NewTable, * routine, even if (for ANALYZE) it is part of the same VACUUM command. */ static inline void -table_relation_vacuum(Relation rel, const VacuumParams *params, - BufferAccessStrategy bstrategy) +table_relation_vacuum(Relation rel, const VacuumParams *params) { - rel->rd_tableam->relation_vacuum(rel, params, bstrategy); + rel->rd_tableam->relation_vacuum(rel, params); } /* diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 956d9cea36da6..14a9b530bcf87 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -363,7 +363,7 @@ extern PGDLLIMPORT int64 parallel_vacuum_worker_delay_ns; /* in commands/vacuum.c */ extern void ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel); extern void vacuum(List *relations, const VacuumParams *params, - BufferAccessStrategy bstrategy, MemoryContext vac_context, + MemoryContext vac_context, bool isTopLevel); extern void vac_open_indexes(Relation relation, LOCKMODE lockmode, int *nindexes, Relation **Irel); @@ -407,8 +407,7 @@ extern void VacuumUpdateCosts(void); /* in commands/vacuumparallel.c */ extern ParallelVacuumState *parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, int nrequested_workers, - int vac_work_mem, int elevel, - BufferAccessStrategy bstrategy); + int vac_work_mem, int elevel); extern void parallel_vacuum_end(ParallelVacuumState *pvs, IndexBulkDeleteResult **istats); extern TidStore *parallel_vacuum_get_dead_items(ParallelVacuumState *pvs, VacDeadItemsInfo **dead_items_info_p); @@ -428,8 +427,7 @@ extern void parallel_vacuum_main(dsm_segment *seg, shm_toc *toc); /* in commands/analyze.c */ extern void analyze_rel(Oid relid, RangeVar *relation, - const VacuumParams *params, List *va_cols, bool in_outer_xact, - BufferAccessStrategy bstrategy); + const VacuumParams *params, List *va_cols, bool in_outer_xact); extern bool attribute_is_analyzable(Relation onerel, int attnum, Form_pg_attribute attr, int *p_attstattarget); extern bool std_typanalyze(VacAttrStats *stats); diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 7170a4bff9896..c5d582b152b60 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -273,15 +273,6 @@ extern PGDLLIMPORT double hash_mem_multiplier; extern PGDLLIMPORT int maintenance_work_mem; extern PGDLLIMPORT int max_parallel_maintenance_workers; -/* - * Upper and lower hard limits for the buffer access strategy ring size - * specified by the VacuumBufferUsageLimit GUC and BUFFER_USAGE_LIMIT option - * to VACUUM and ANALYZE. - */ -#define MIN_BAS_VAC_RING_SIZE_KB 128 -#define MAX_BAS_VAC_RING_SIZE_KB (16 * 1024 * 1024) - -extern PGDLLIMPORT int VacuumBufferUsageLimit; extern PGDLLIMPORT int VacuumCostPageHit; extern PGDLLIMPORT int VacuumCostPageMiss; extern PGDLLIMPORT int VacuumCostPageDirty; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 58a44857f1311..68b4e5093a2bc 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -287,14 +287,11 @@ typedef enum IOObject typedef enum IOContext { - IOCONTEXT_BULKREAD, - IOCONTEXT_BULKWRITE, IOCONTEXT_INIT, IOCONTEXT_NORMAL, - IOCONTEXT_VACUUM, } IOContext; -#define IOCONTEXT_NUM_TYPES (IOCONTEXT_VACUUM + 1) +#define IOCONTEXT_NUM_TYPES (IOCONTEXT_NORMAL + 1) /* * Enumeration of IO operations. diff --git a/src/include/storage/buf.h b/src/include/storage/buf.h index b21445522b180..fa942a64f05a5 100644 --- a/src/include/storage/buf.h +++ b/src/include/storage/buf.h @@ -36,11 +36,4 @@ typedef int Buffer; */ #define BufferIsLocal(buffer) ((buffer) < 0) -/* - * Buffer access strategy objects. - * - * BufferAccessStrategyData is private to freelist.c - */ -typedef struct BufferAccessStrategyData *BufferAccessStrategy; - #endif /* BUF_H */ diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index 131df126cda00..3b44ad3e0b67b 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -632,11 +632,7 @@ extern void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint64 set_flag /* freelist.c */ -extern IOContext IOContextForStrategy(BufferAccessStrategy strategy); -extern BufferDesc *StrategyGetBuffer(BufferAccessStrategy strategy, - uint64 *buf_state, bool *from_ring); -extern bool StrategyRejectBuffer(BufferAccessStrategy strategy, - BufferDesc *buf, bool from_ring); +extern BufferDesc *StrategyGetBuffer(uint64 *buf_state); extern int StrategySyncStart(uint32 *complete_passes, uint32 *num_buf_alloc); extern void StrategyNotifyBgWriter(int bgwprocno); diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 6837b35fc6d0b..36023792f3611 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -25,21 +25,6 @@ typedef void *Block; -/* - * Possible arguments for GetAccessStrategy(). - * - * If adding a new BufferAccessStrategyType, also add a new IOContext so - * IO statistics using this strategy are tracked. - */ -typedef enum BufferAccessStrategyType -{ - BAS_NORMAL, /* Normal random access */ - BAS_BULKREAD, /* Large read-only scan (hint bit updates are - * ok) */ - BAS_BULKWRITE, /* Large multi-block write (e.g. COPY IN) */ - BAS_VACUUM, /* VACUUM */ -} BufferAccessStrategyType; - /* Possible modes for ReadBufferExtended() */ typedef enum { @@ -135,7 +120,6 @@ struct ReadBuffersOperation SMgrRelation smgr; char persistence; ForkNumber forknum; - BufferAccessStrategy strategy; /* * The following private members are private state for communication @@ -235,11 +219,10 @@ extern bool ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, Buffer recent_buffer); extern Buffer ReadBuffer(Relation reln, BlockNumber blockNum); extern Buffer ReadBufferExtended(Relation reln, ForkNumber forkNum, - BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy); + BlockNumber blockNum, ReadBufferMode mode); extern Buffer ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy, + ReadBufferMode mode, bool permanent); extern bool StartReadBuffer(ReadBuffersOperation *operation, @@ -266,18 +249,15 @@ extern Buffer ReleaseAndReadBuffer(Buffer buffer, Relation relation, extern Buffer ExtendBufferedRel(BufferManagerRelation bmr, ForkNumber forkNum, - BufferAccessStrategy strategy, uint32 flags); extern BlockNumber ExtendBufferedRelBy(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, Buffer *buffers, uint32 *extended_by); extern Buffer ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, BlockNumber extend_to, ReadBufferMode mode); @@ -376,14 +356,6 @@ extern void AtProcExit_LocalBuffers(void); /* in freelist.c */ -extern BufferAccessStrategy GetAccessStrategy(BufferAccessStrategyType btype); -extern BufferAccessStrategy GetAccessStrategyWithSize(BufferAccessStrategyType btype, - int ring_size_kb); -extern int GetAccessStrategyBufferCount(BufferAccessStrategy strategy); -extern int GetAccessStrategyPinLimit(BufferAccessStrategy strategy); - -extern void FreeAccessStrategy(BufferAccessStrategy strategy); - /* inline functions */ diff --git a/src/include/storage/read_stream.h b/src/include/storage/read_stream.h index 48995c6d534bd..9e5fbbed9e866 100644 --- a/src/include/storage/read_stream.h +++ b/src/include/storage/read_stream.h @@ -83,17 +83,14 @@ extern BlockNumber block_range_read_stream_cb(ReadStream *stream, void *callback_private_data, void *per_buffer_data); extern ReadStream *read_stream_begin_relation(int flags, - BufferAccessStrategy strategy, Relation rel, ForkNumber forknum, ReadStreamBlockNumberCB callback, void *callback_private_data, size_t per_buffer_data_size); extern Buffer read_stream_next_buffer(ReadStream *stream, void **per_buffer_data); -extern BlockNumber read_stream_next_block(ReadStream *stream, - BufferAccessStrategy *strategy); +extern BlockNumber read_stream_next_block(ReadStream *stream); extern ReadStream *read_stream_begin_smgr_relation(int flags, - BufferAccessStrategy strategy, SMgrRelation smgr, char smgr_persistence, ForkNumber forknum, diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 307f4fbaefe08..3f28558d47c6d 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -31,8 +31,6 @@ extern void assign_application_name(const char *newval, void *extra); extern const char *show_archive_command(void); extern bool check_autovacuum_work_mem(int *newval, void **extra, GucSource source); -extern bool check_vacuum_buffer_usage_limit(int *newval, void **extra, - GucSource source); extern bool check_backtrace_functions(char **newval, void **extra, GucSource source); extern void assign_backtrace_functions(const char *newval, void *extra); diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 6270775af7ca2..8d7b2a728d680 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -194,7 +194,6 @@ grow_rel(PG_FUNCTION_ARGS) ExtendBufferedRelBy(BMR_REL(rel), MAIN_FORKNUM, - NULL, 0, extend_by_pages, victim_buffers, @@ -231,7 +230,7 @@ modify_rel_block(PG_FUNCTION_ARGS) rel = relation_open(relid, AccessExclusiveLock); buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, - RBM_ZERO_ON_ERROR, NULL); + RBM_ZERO_ON_ERROR); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); @@ -331,7 +330,7 @@ create_toy_buffer(Relation rel, BlockNumber blkno) uint64 unset_bits = 0; /* place buffer in shared buffers without erroring out */ - buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_ZERO_AND_LOCK, NULL); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_ZERO_AND_LOCK); LockBuffer(buf, BUFFER_LOCK_UNLOCK); if (RelationUsesLocalBuffers(rel)) @@ -737,7 +736,6 @@ read_buffers(PG_FUNCTION_ARGS) operation->rel = rel; operation->smgr = smgr; operation->persistence = rel->rd_rel->relpersistence; - operation->strategy = NULL; operation->forknum = MAIN_FORKNUM; io_reqds[nios] = StartReadBuffers(operation, @@ -879,7 +877,6 @@ read_stream_for_blocks(PG_FUNCTION_ARGS) rel = relation_open(relid, AccessShareLock); stream = read_stream_begin_relation(READ_STREAM_FULL, - NULL, rel, MAIN_FORKNUM, read_stream_for_blocks_cb, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 03cbc1cdef59e..beb876c48b4fb 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -16,22 +16,16 @@ SHOW track_counts; -- must be on SELECT backend_type, object, context FROM pg_stat_io ORDER BY backend_type COLLATE "C", object COLLATE "C", context COLLATE "C"; backend_type|object|context -autovacuum launcher|relation|bulkread autovacuum launcher|relation|init autovacuum launcher|relation|normal autovacuum launcher|wal|init autovacuum launcher|wal|normal -autovacuum worker|relation|bulkread autovacuum worker|relation|init autovacuum worker|relation|normal -autovacuum worker|relation|vacuum autovacuum worker|wal|init autovacuum worker|wal|normal -background worker|relation|bulkread -background worker|relation|bulkwrite background worker|relation|init background worker|relation|normal -background worker|relation|vacuum background worker|temp relation|normal background worker|wal|init background worker|wal|normal @@ -43,67 +37,43 @@ checkpointer|relation|init checkpointer|relation|normal checkpointer|wal|init checkpointer|wal|normal -client backend|relation|bulkread -client backend|relation|bulkwrite client backend|relation|init client backend|relation|normal -client backend|relation|vacuum client backend|temp relation|normal client backend|wal|init client backend|wal|normal -datachecksums launcher|relation|bulkread -datachecksums launcher|relation|bulkwrite datachecksums launcher|relation|init datachecksums launcher|relation|normal -datachecksums launcher|relation|vacuum datachecksums launcher|temp relation|normal datachecksums launcher|wal|init datachecksums launcher|wal|normal -datachecksums worker|relation|bulkread -datachecksums worker|relation|bulkwrite datachecksums worker|relation|init datachecksums worker|relation|normal -datachecksums worker|relation|vacuum datachecksums worker|temp relation|normal datachecksums worker|wal|init datachecksums worker|wal|normal -io worker|relation|bulkread -io worker|relation|bulkwrite io worker|relation|init io worker|relation|normal -io worker|relation|vacuum io worker|temp relation|normal io worker|wal|init io worker|wal|normal -slotsync worker|relation|bulkread -slotsync worker|relation|bulkwrite slotsync worker|relation|init slotsync worker|relation|normal -slotsync worker|relation|vacuum slotsync worker|temp relation|normal slotsync worker|wal|init slotsync worker|wal|normal -standalone backend|relation|bulkread -standalone backend|relation|bulkwrite standalone backend|relation|init standalone backend|relation|normal -standalone backend|relation|vacuum standalone backend|wal|init standalone backend|wal|normal -startup|relation|bulkread -startup|relation|bulkwrite startup|relation|init startup|relation|normal -startup|relation|vacuum startup|wal|init startup|wal|normal walreceiver|wal|init walreceiver|wal|normal -walsender|relation|bulkread -walsender|relation|bulkwrite walsender|relation|init walsender|relation|normal -walsender|relation|vacuum walsender|temp relation|normal walsender|wal|init walsender|wal|normal @@ -111,7 +81,7 @@ walsummarizer|wal|init walsummarizer|wal|normal walwriter|wal|init walwriter|wal|normal -(95 rows) +(65 rows) \a -- List of registered statistics kinds. SELECT id, name, fixed_amount, @@ -1757,80 +1727,20 @@ SELECT :io_sum_local_new_tblspc_writes > :io_sum_local_after_writes; (1 row) RESET temp_buffers; --- Test that reuse of strategy buffers and reads of blocks into these reused --- buffers while VACUUMing are tracked in pg_stat_io. If there is sufficient --- demand for shared buffers from concurrent queries, some buffers may be --- pinned by other backends before they can be reused. In such cases, the --- backend will evict a buffer from outside the ring and add it to the --- ring. This is considered an eviction and not a reuse. --- Set wal_skip_threshold smaller than the expected size of --- test_io_vac_strategy so that, even if wal_level is minimal, VACUUM FULL will --- fsync the newly rewritten test_io_vac_strategy instead of writing it to WAL. --- Writing it to WAL will result in the newly written relation pages being in --- shared buffers -- preventing us from testing BAS_VACUUM BufferAccessStrategy --- reads. -SET wal_skip_threshold = '1 kB'; -SELECT sum(reuses) AS reuses, sum(reads) AS reads, sum(evictions) AS evictions - FROM pg_stat_io WHERE context = 'vacuum' \gset io_sum_vac_strategy_before_ -CREATE TABLE test_io_vac_strategy(a int, b int) WITH (autovacuum_enabled = 'false'); -INSERT INTO test_io_vac_strategy SELECT i, i from generate_series(1, 4500)i; --- Ensure that the next VACUUM will need to perform IO by rewriting the table --- first with VACUUM (FULL). -VACUUM (FULL) test_io_vac_strategy; --- Use the minimum BUFFER_USAGE_LIMIT to cause reuses or evictions with the --- smallest table possible. -VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) test_io_vac_strategy; -SELECT pg_stat_force_next_flush(); - pg_stat_force_next_flush --------------------------- - -(1 row) - -SELECT sum(reuses) AS reuses, sum(reads) AS reads, sum(evictions) AS evictions - FROM pg_stat_io WHERE context = 'vacuum' \gset io_sum_vac_strategy_after_ -SELECT :io_sum_vac_strategy_after_reads > :io_sum_vac_strategy_before_reads; - ?column? ----------- - t -(1 row) - -SELECT (:io_sum_vac_strategy_after_reuses + :io_sum_vac_strategy_after_evictions) > - (:io_sum_vac_strategy_before_reuses + :io_sum_vac_strategy_before_evictions); - ?column? ----------- - t -(1 row) - -RESET wal_skip_threshold; --- Test that extends done by a CTAS, which uses a BAS_BULKWRITE --- BufferAccessStrategy, are tracked in pg_stat_io. -SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_before - FROM pg_stat_io WHERE context = 'bulkwrite' \gset -CREATE TABLE test_io_bulkwrite_strategy AS SELECT i FROM generate_series(1,100)i; -SELECT pg_stat_force_next_flush(); - pg_stat_force_next_flush --------------------------- - -(1 row) - -SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_after - FROM pg_stat_io WHERE context = 'bulkwrite' \gset -SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_extends_before; - ?column? ----------- - t -(1 row) - -- Test IO stats reset +-- Note: reuses is intentionally excluded from these totals. With the +-- BufferAccessStrategy ring buffers removed, IOOP_REUSE is never tracked, so +-- the reuses column is NULL for every row; summing it would make the whole +-- total NULL. SELECT pg_stat_have_stats('io', 0, 0); pg_stat_have_stats -------------------- t (1 row) -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); pg_stat_reset_shared @@ -1838,7 +1748,7 @@ SELECT pg_stat_reset_shared('io'); (1 row) -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; ?column? @@ -1846,7 +1756,7 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; t (1 row) -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset -- pg_stat_reset_shared() did not reset backend IO stats SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; @@ -1862,7 +1772,7 @@ SELECT pg_stat_reset_backend_stats(pg_backend_pid()); (1 row) -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; ?column? diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index d4696bc332557..85d06907cec0f 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -526,25 +526,6 @@ SELECT t.relfilenode = :toast_filenode AS is_same_toast_filenode f (1 row) --- BUFFER_USAGE_LIMIT option -VACUUM (BUFFER_USAGE_LIMIT '512 kB') vac_option_tab; -ANALYZE (BUFFER_USAGE_LIMIT '512 kB') vac_option_tab; --- try disabling the buffer usage limit -VACUUM (BUFFER_USAGE_LIMIT 0) vac_option_tab; -ANALYZE (BUFFER_USAGE_LIMIT 0) vac_option_tab; --- value exceeds max size error -VACUUM (BUFFER_USAGE_LIMIT 16777220) vac_option_tab; -ERROR: BUFFER_USAGE_LIMIT option must be 0 or between 128 kB and 16777216 kB --- value is less than min size error -VACUUM (BUFFER_USAGE_LIMIT 120) vac_option_tab; -ERROR: BUFFER_USAGE_LIMIT option must be 0 or between 128 kB and 16777216 kB --- integer overflow error -VACUUM (BUFFER_USAGE_LIMIT 10000000000) vac_option_tab; -ERROR: BUFFER_USAGE_LIMIT option must be 0 or between 128 kB and 16777216 kB -HINT: Value exceeds integer range. --- incompatible with VACUUM FULL error -VACUUM (BUFFER_USAGE_LIMIT '512 kB', FULL) vac_option_tab; -ERROR: BUFFER_USAGE_LIMIT cannot be specified for VACUUM FULL -- SKIP_DATABASE_STATS option VACUUM (SKIP_DATABASE_STATS) vactst; -- ONLY_DATABASE_STATS option diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 4c265d1245c72..4eb48114e80fa 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -814,65 +814,27 @@ SELECT sum(writes) AS io_sum_local_new_tblspc_writes SELECT :io_sum_local_new_tblspc_writes > :io_sum_local_after_writes; RESET temp_buffers; --- Test that reuse of strategy buffers and reads of blocks into these reused --- buffers while VACUUMing are tracked in pg_stat_io. If there is sufficient --- demand for shared buffers from concurrent queries, some buffers may be --- pinned by other backends before they can be reused. In such cases, the --- backend will evict a buffer from outside the ring and add it to the --- ring. This is considered an eviction and not a reuse. - --- Set wal_skip_threshold smaller than the expected size of --- test_io_vac_strategy so that, even if wal_level is minimal, VACUUM FULL will --- fsync the newly rewritten test_io_vac_strategy instead of writing it to WAL. --- Writing it to WAL will result in the newly written relation pages being in --- shared buffers -- preventing us from testing BAS_VACUUM BufferAccessStrategy --- reads. -SET wal_skip_threshold = '1 kB'; -SELECT sum(reuses) AS reuses, sum(reads) AS reads, sum(evictions) AS evictions - FROM pg_stat_io WHERE context = 'vacuum' \gset io_sum_vac_strategy_before_ -CREATE TABLE test_io_vac_strategy(a int, b int) WITH (autovacuum_enabled = 'false'); -INSERT INTO test_io_vac_strategy SELECT i, i from generate_series(1, 4500)i; --- Ensure that the next VACUUM will need to perform IO by rewriting the table --- first with VACUUM (FULL). -VACUUM (FULL) test_io_vac_strategy; --- Use the minimum BUFFER_USAGE_LIMIT to cause reuses or evictions with the --- smallest table possible. -VACUUM (PARALLEL 0, BUFFER_USAGE_LIMIT 128) test_io_vac_strategy; -SELECT pg_stat_force_next_flush(); -SELECT sum(reuses) AS reuses, sum(reads) AS reads, sum(evictions) AS evictions - FROM pg_stat_io WHERE context = 'vacuum' \gset io_sum_vac_strategy_after_ -SELECT :io_sum_vac_strategy_after_reads > :io_sum_vac_strategy_before_reads; -SELECT (:io_sum_vac_strategy_after_reuses + :io_sum_vac_strategy_after_evictions) > - (:io_sum_vac_strategy_before_reuses + :io_sum_vac_strategy_before_evictions); -RESET wal_skip_threshold; - --- Test that extends done by a CTAS, which uses a BAS_BULKWRITE --- BufferAccessStrategy, are tracked in pg_stat_io. -SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_before - FROM pg_stat_io WHERE context = 'bulkwrite' \gset -CREATE TABLE test_io_bulkwrite_strategy AS SELECT i FROM generate_series(1,100)i; -SELECT pg_stat_force_next_flush(); -SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_after - FROM pg_stat_io WHERE context = 'bulkwrite' \gset -SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_extends_before; - -- Test IO stats reset +-- Note: reuses is intentionally excluded from these totals. With the +-- BufferAccessStrategy ring buffers removed, IOOP_REUSE is never tracked, so +-- the reuses column is NULL for every row; summing it would make the whole +-- total NULL. SELECT pg_stat_have_stats('io', 0, 0); -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_pre_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT pg_stat_reset_shared('io'); -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_post_reset FROM pg_stat_io \gset SELECT :io_stats_post_reset < :io_stats_pre_reset; -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset -- pg_stat_reset_shared() did not reset backend IO stats SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; -- but pg_stat_reset_backend_stats() does SELECT pg_stat_reset_backend_stats(pg_backend_pid()); -SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset +SELECT sum(evictions) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index 247b8e23b2357..506c438976552 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -390,21 +390,6 @@ SELECT t.relfilenode = :toast_filenode AS is_same_toast_filenode FROM pg_class c, pg_class t WHERE c.reltoastrelid = t.oid AND c.relname = 'vac_option_tab'; --- BUFFER_USAGE_LIMIT option -VACUUM (BUFFER_USAGE_LIMIT '512 kB') vac_option_tab; -ANALYZE (BUFFER_USAGE_LIMIT '512 kB') vac_option_tab; --- try disabling the buffer usage limit -VACUUM (BUFFER_USAGE_LIMIT 0) vac_option_tab; -ANALYZE (BUFFER_USAGE_LIMIT 0) vac_option_tab; --- value exceeds max size error -VACUUM (BUFFER_USAGE_LIMIT 16777220) vac_option_tab; --- value is less than min size error -VACUUM (BUFFER_USAGE_LIMIT 120) vac_option_tab; --- integer overflow error -VACUUM (BUFFER_USAGE_LIMIT 10000000000) vac_option_tab; --- incompatible with VACUUM FULL error -VACUUM (BUFFER_USAGE_LIMIT '512 kB', FULL) vac_option_tab; - -- SKIP_DATABASE_STATS option VACUUM (SKIP_DATABASE_STATS) vactst; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88b1..064a9878e7276 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -355,8 +355,6 @@ BtreeLevel Bucket BufFile Buffer -BufferAccessStrategy -BufferAccessStrategyType BufferCacheOsPagesContext BufferCacheOsPagesRec BufferDesc