From 4646ad9cb1f3ad4c3ae9ddf9b90e4c122ddfbc28 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 09:08:45 -0400 Subject: [PATCH 1/6] ci: fork upstream auto-sync Keeps gburd/postgres rebased hourly on postgres/postgres master with only .github/ changes on top (sync-upstream automatic + manual). Drops the bespoke Windows dependency-builder workflow: Windows is already built and tested in CI by upstream's pg-ci.yml (Visual Studio + MinGW meson jobs), so a separate dependency prebuild that nothing consumed was redundant. --- .github/.gitignore | 18 + .github/QUICKSTART.md | 378 +++++++++++++++++++++ .github/README.md | 315 +++++++++++++++++ .github/docs/pristine-master-policy.md | 225 ++++++++++++ .github/docs/sync-setup.md | 326 ++++++++++++++++++ .github/workflows/sync-upstream-manual.yml | 249 ++++++++++++++ .github/workflows/sync-upstream.yml | 256 ++++++++++++++ 7 files changed, 1767 insertions(+) create mode 100644 .github/.gitignore create mode 100644 .github/QUICKSTART.md create mode 100644 .github/README.md create mode 100644 .github/docs/pristine-master-policy.md create mode 100644 .github/docs/sync-setup.md create mode 100644 .github/workflows/sync-upstream-manual.yml create mode 100644 .github/workflows/sync-upstream.yml diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 0000000000000..a447f99442861 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1,18 @@ +# Node modules +scripts/ai-review/node_modules/ +# Note: package-lock.json should be committed for reproducible CI/CD builds + +# Logs +scripts/ai-review/cost-log-*.json +scripts/ai-review/*.log + +# OS files +.DS_Store +Thumbs.db + +# Editor files +*.swp +*.swo +*~ +.vscode/ +.idea/ diff --git a/.github/QUICKSTART.md b/.github/QUICKSTART.md new file mode 100644 index 0000000000000..d22c4d562ab7d --- /dev/null +++ b/.github/QUICKSTART.md @@ -0,0 +1,378 @@ +# Quick Start Guide - PostgreSQL Mirror CI/CD + +**Goal:** Get your PostgreSQL mirror CI/CD system running in 15 minutes. + +--- + +## āœ… What's Been Implemented + +- **Phase 1: Automated Upstream Sync** - Daily sync from postgres/postgres āœ… +- **Phase 2: AI-Powered Code Review** - Claude-based PR reviews āœ… +- **Phase 3: Windows Builds** - Planned for weeks 4-6 šŸ“‹ + +--- + +## šŸš€ Setup Instructions + +### Step 1: Configure GitHub Actions Permissions (2 minutes) + +1. Go to: **Settings ā†’ Actions ā†’ General** +2. Scroll to: **Workflow permissions** +3. Select: **"Read and write permissions"** +4. Check: **"Allow GitHub Actions to create and approve pull requests"** +5. Click: **Save** + +āœ… This enables workflows to push commits and create issues. + +--- + +### Step 2: Set Up Upstream Sync (3 minutes) + +**Test manual sync first:** + +```bash +# Via GitHub Web UI: +# 1. Go to: Actions tab +# 2. Click: "Sync from Upstream (Manual)" +# 3. Click: "Run workflow" +# 4. Watch it run (should take ~2 minutes) + +# OR via GitHub CLI: +gh workflow run sync-upstream-manual.yml +gh run watch +``` + +**Verify sync worked:** + +```bash +git fetch origin +git log origin/master --oneline -5 + +# Compare with upstream: +# https://github.com/postgres/postgres/commits/master +``` + +**Enable automatic sync:** + +- Automatic sync runs daily at 00:00 UTC +- Already configured, no action needed +- Check: Actions ā†’ "Sync from Upstream (Automatic)" + +āœ… Your master branch will now stay synced automatically. + +--- + +### Step 3: Set Up AI Code Review (10 minutes) + +**Choose Your Provider:** + +You can use either **Anthropic API** (simpler) or **AWS Bedrock** (if you have AWS infrastructure). + +#### Option A: Anthropic API (Recommended for getting started) + +**A. Get Claude API Key:** + +1. Go to: https://console.anthropic.com/ +2. Sign up or log in +3. Navigate to: API Keys +4. Create new key +5. Copy the key (starts with `sk-ant-...`) + +**B. Add API Key to GitHub:** + +1. Go to: **Settings ā†’ Secrets and variables ā†’ Actions** +2. Click: **New repository secret** +3. Name: `ANTHROPIC_API_KEY` +4. Value: Paste your API key +5. Click: **Add secret** + +**C. Ensure config uses Anthropic:** + +Check `.github/scripts/ai-review/config.json` has: +```json +{ + "provider": "anthropic", + ... +} +``` + +#### Option B: AWS Bedrock (If you have AWS) + +See detailed guide: [.github/docs/bedrock-setup.md](.github/docs/bedrock-setup.md) + +**Quick steps:** +1. Enable Claude 3.5 Sonnet in AWS Bedrock console +2. Create IAM user with `bedrock:InvokeModel` permission +3. Add three secrets to GitHub: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_REGION` (e.g., `us-east-1`) +4. Update `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "bedrock", + "bedrock_model_id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock_region": "us-east-1", + ... +} +``` + +**Note:** Both providers have identical pricing ($0.003/1K input, $0.015/1K output tokens). + +--- + +**C. Install Dependencies:** + +```bash +cd .github/scripts/ai-review +npm install + +# Should install: +# - @anthropic-ai/sdk (for Anthropic API) +# - @aws-sdk/client-bedrock-runtime (for AWS Bedrock) +# - @actions/github +# - @actions/core +# - parse-diff +# - minimatch +``` + +**D. Test AI Review:** + +```bash +# Option 1: Create a test PR +git checkout -b test/ai-review +echo "// Test change" >> src/backend/utils/adt/int.c +git add . +git commit -m "Test: AI review" +git push origin test/ai-review +# Create PR via GitHub UI + +# Option 2: Manual trigger on existing PR +gh workflow run ai-code-review.yml -f pr_number= +``` + +āœ… AI will review the PR and post comments + summary. + +--- + +## šŸŽÆ Verify Everything Works + +### Check Sync Status + +```bash +# Check latest sync run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View details +gh run view $(gh run list --workflow=sync-upstream.yml --limit 1 --json databaseId -q '.[0].databaseId') +``` + +**Expected:** āœ… Green checkmark, "Already up to date" or "Successfully synced X commits" + +### Check AI Review Status + +```bash +# Check latest AI review run +gh run list --workflow=ai-code-review.yml --limit 1 + +# View details +gh run view $(gh run list --workflow=ai-code-review.yml --limit 1 --json databaseId -q '.[0].databaseId') +``` + +**Expected:** āœ… Green checkmark, comments posted on PR + +--- + +## šŸ“Š Monitor Costs + +### GitHub Actions Minutes + +```bash +# View usage (requires admin access) +gh api /repos/gburd/postgres/actions/cache/usage + +# Expected monthly usage: +# - Sync: ~150 minutes (FREE - within 2,000 min limit) +# - AI Review: ~200 minutes (FREE - within limit) +``` + +### Claude API Costs + +**View per-PR cost:** +- Check AI review summary comment on PR +- Format: `Cost: $X.XX | Model: claude-3-5-sonnet` + +**Expected costs:** +- Small PR: $0.50 - $1.00 +- Medium PR: $1.00 - $3.00 +- Large PR: $3.00 - $7.50 +- **Monthly (20 PRs):** $35-50 + +**Download detailed logs:** +```bash +gh run list --workflow=ai-code-review.yml --limit 5 +gh run download -n ai-review-cost-log- +``` + +--- + +## šŸ”§ Configuration + +### Adjust Sync Schedule + +Edit `.github/workflows/sync-upstream.yml`: + +```yaml +on: + schedule: + # Current: Daily at 00:00 UTC + - cron: '0 0 * * *' + + # Options: + # Every 6 hours: '0 */6 * * *' + # Twice daily: '0 0,12 * * *' + # Weekdays only: '0 0 * * 1-5' +``` + +### Adjust AI Review Costs + +Edit `.github/scripts/ai-review/config.json`: + +```json +{ + "cost_limits": { + "max_per_pr_dollars": 15.0, // ā† Lower this to save money + "max_per_month_dollars": 200.0, // ā† Hard monthly cap + "alert_threshold_dollars": 150.0 + }, + + "max_file_size_lines": 5000, // ā† Skip files larger than this + + "skip_paths": [ + "*.png", "*.svg", // Already skipped + "vendor/**/*", // ā† Add more patterns here + "generated/**/*" + ] +} +``` + +### Adjust AI Review Prompts + +**Make AI reviews stricter or more lenient:** + +Edit files in `.github/scripts/ai-review/prompts/`: +- `c-code.md` - PostgreSQL C code review +- `sql.md` - SQL and regression tests +- `documentation.md` - Documentation review +- `build-system.md` - Makefile/Meson review + +--- + +## šŸ› Troubleshooting + +### Sync Not Working + +**Problem:** Workflow fails with "Permission denied" + +**Fix:** +- Check: Settings ā†’ Actions ā†’ Workflow permissions +- Ensure: "Read and write permissions" is selected + +--- + +### AI Review Not Posting Comments + +**Problem:** Workflow runs but no comments appear + +**Check:** +1. Is PR a draft? (Draft PRs are skipped to save costs) +2. Are there reviewable files? (Check workflow logs) +3. Is API key valid? (Settings ā†’ Secrets ā†’ ANTHROPIC_API_KEY) + +**Fix:** +- Mark PR as "Ready for review" if draft +- Check workflow logs: Actions ā†’ Latest run ā†’ View logs +- Verify API key at https://console.anthropic.com/ + +--- + +### High AI Review Costs + +**Problem:** Costs higher than expected + +**Check:** +- Download cost logs: `gh run download ` +- Look for large files being reviewed +- Check number of PR updates (each triggers review) + +**Fix:** +1. Add large files to `skip_paths` in config.json +2. Lower `max_tokens_per_request` (shorter reviews) +3. Use draft PRs for work-in-progress +4. Batch PR updates to reduce review frequency + +--- + +## šŸ“š Full Documentation + +- **Overview:** [.github/README.md](.github/README.md) +- **Sync Guide:** [.github/docs/sync-setup.md](.github/docs/sync-setup.md) +- **AI Review Guide:** [.github/docs/ai-review-guide.md](.github/docs/ai-review-guide.md) +- **Windows Builds:** [.github/docs/windows-builds.md](.github/docs/windows-builds.md) (planned) +- **Implementation Status:** [.github/IMPLEMENTATION_STATUS.md](.github/IMPLEMENTATION_STATUS.md) + +--- + +## āœØ What's Next? + +### Immediate +- āœ… **Monitor first automatic sync** (tonight at 00:00 UTC) +- āœ… **Test AI review on real PR** +- āœ… **Tune prompts** based on feedback + +### This Week +- Shadow mode testing for AI reviews (Week 1) +- Gather developer feedback +- Adjust configuration + +### Weeks 2-3 +- Enable full AI review mode +- Monitor costs and quality +- Iterate on prompts + +### Weeks 4-6 +- **Phase 3:** Implement Windows dependency builds +- Research winpgbuild approach +- Create build workflows +- Test artifact publishing + +--- + +## šŸŽ‰ Success Criteria + +You'll know everything is working when: + +āœ… **Sync:** +- Master branch matches postgres/postgres +- Daily sync runs show green checkmarks +- No open issues with label `sync-failure` + +āœ… **AI Review:** +- PRs receive inline comments + summary +- Feedback is relevant and actionable +- Costs stay under $50/month +- Developers find reviews helpful + +āœ… **Overall:** +- Automation saves 8-16 hours/month +- Issues caught earlier in development +- No manual sync needed + +--- + +**Need Help?** +- Check documentation: `.github/README.md` +- Check workflow logs: Actions ā†’ Failed run ā†’ View logs +- Create issue with workflow URL and error messages + +**Ready to go!** šŸš€ diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000000..bdfcfe74ac4a4 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,315 @@ +# PostgreSQL Mirror CI/CD System + +This directory contains the CI/CD infrastructure for the PostgreSQL personal mirror repository. + +## System Overview + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ PostgreSQL Mirror CI/CD ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” + ā”‚ ā”‚ ā”‚ + [1] Sync [2] AI Review [3] Windows + Daily @ 00:00 On PR Events On Master Push + ā”‚ ā”‚ ā”‚ + ā–¼ ā–¼ ā–¼ + postgres/postgres Claude API Dependency Builds + ā”‚ ā”‚ ā”‚ + ā–¼ ā–¼ ā–¼ + github.com/gburd PR Comments Build Artifacts + /postgres/ + Labels (90-day retention) + master +``` + +## Components + +### 1. Automated Upstream Sync +**Status:** āœ“ Implemented +**Files:** `workflows/sync-upstream*.yml` + +Automatically syncs the `master` branch with upstream `postgres/postgres` daily. + +- **Frequency:** Daily at 00:00 UTC +- **Trigger:** Cron schedule + manual +- **Features:** + - Fast-forward merge (conflict-free) + - Automatic issue creation on conflicts + - Issue auto-closure on resolution +- **Cost:** Free (~150 min/month, well within free tier) + +**Documentation:** [docs/sync-setup.md](docs/sync-setup.md) + +### 2. AI-Powered Code Review +**Status:** āœ“ Implemented +**Files:** `workflows/ai-code-review.yml`, `scripts/ai-review/` + +Uses Claude API to provide PostgreSQL-aware code review on pull requests. + +- **Trigger:** PR opened/updated, ready for review +- **Features:** + - PostgreSQL-specific C code review + - SQL, documentation, build system review + - Inline comments on issues + - Automatic labeling (security, performance, etc.) + - Cost tracking and limits + - **Provider Options:** Anthropic API or AWS Bedrock +- **Cost:** $35-50/month (estimated) +- **Model:** Claude 3.5 Sonnet + +**Documentation:** [docs/ai-review-guide.md](docs/ai-review-guide.md) + +### 3. Windows Build Integration +**Status:** āœ… Implemented +**Files:** `workflows/windows-dependencies.yml`, `windows/`, `scripts/windows/` + +Builds PostgreSQL Windows dependencies for x64 Windows. + +- **Trigger:** Manual, manifest changes, weekly refresh +- **Features:** + - Core dependencies: OpenSSL, zlib, libxml2 + - Smart caching by version hash + - Dependency bundling + - Artifact publishing (90-day retention) + - PowerShell download helper + - **Cost optimization:** Skips builds for pristine commits (dev setup, .github/ only) +- **Cost:** ~$5-8/month (with caching and optimization) + +**Documentation:** [docs/windows-builds.md](docs/windows-builds.md) | [Usage](docs/windows-builds-usage.md) + +## Quick Start + +### Prerequisites + +1. **GitHub Actions enabled:** + - Settings ā†’ Actions ā†’ General ā†’ Allow all actions + +2. **Workflow permissions:** + - Settings ā†’ Actions ā†’ General ā†’ Workflow permissions + - Select: "Read and write permissions" + - Enable: "Allow GitHub Actions to create and approve pull requests" + +3. **Secrets configured:** + - **Option A - Anthropic API:** + - Settings ā†’ Secrets and variables ā†’ Actions + - Add: `ANTHROPIC_API_KEY` (get from https://console.anthropic.com/) + - **Option B - AWS Bedrock:** + - Add: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` + - See: [docs/bedrock-setup.md](docs/bedrock-setup.md) + +### Using the Sync System + +**Manual sync:** +```bash +# Via GitHub UI: +# Actions ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow + +# Via GitHub CLI: +gh workflow run sync-upstream-manual.yml +``` + +**Check sync status:** +```bash +# Latest sync run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View details +gh run view +``` + +### Using AI Code Review + +AI reviews run automatically on PRs. To test manually: + +```bash +# Via GitHub UI: +# Actions ā†’ "AI Code Review" ā†’ Run workflow ā†’ Enter PR number + +# Via GitHub CLI: +gh workflow run ai-code-review.yml -f pr_number=123 +``` + +**Reviewing AI feedback:** +1. AI posts inline comments on specific lines +2. AI posts summary comment with overview +3. AI adds labels (security-concern, needs-tests, etc.) +4. Review and address feedback like human reviewer comments + +### Cost Monitoring + +**View AI review costs:** +```bash +# Download cost logs +gh run download -n ai-review-cost-log- +``` + +**Expected monthly costs (with optimizations):** +- Sync: $0 (free tier) +- AI Review: $30-45 (only on PRs, skips drafts) +- Windows Builds: $5-8 (caching + pristine commit skipping) +- **Total: $35-53/month** + +**Cost optimizations:** +- Windows builds skip "dev setup" and .github/-only commits +- AI review only runs on non-draft PRs +- Aggressive caching reduces build times by 80-90% +- See [Cost Optimization Guide](docs/cost-optimization.md) for details + +## Workflow Files + +### Sync Workflows +- `workflows/sync-upstream.yml` - Automatic daily sync +- `workflows/sync-upstream-manual.yml` - Manual testing sync + +### AI Review Workflows +- `workflows/ai-code-review.yml` - Automatic PR review + +### Windows Build Workflows +- `workflows/windows-dependencies.yml` - Dependency builds (TBD) + +## Configuration Files + +### AI Review Configuration +- `scripts/ai-review/config.json` - Cost limits, file patterns, labels +- `scripts/ai-review/prompts/*.md` - Review prompts by file type +- `scripts/ai-review/package.json` - Node.js dependencies + +### Windows Build Configuration +- `windows/manifest.json` - Dependency versions (TBD) + +## Branch Strategy + +### Master Branch: Mirror Only +- **Purpose:** Pristine copy of `postgres/postgres` +- **Rule:** Never commit directly to master +- **Sync:** Automatic via GitHub Actions +- **Protection:** Consider branch protection rules + +### Feature Branches: Development +- **Pattern:** `feature/*`, `dev/*`, `experiment/*` +- **Workflow:** + ```bash + git checkout master + git pull origin master + git checkout -b feature/my-feature + # ... make changes ... + git push origin feature/my-feature + # Create PR: feature/my-feature ā†’ master + ``` + +### Special Branches +- `recovery/*` - Temporary branches for sync conflict resolution +- Development remotes: commitfest, heikki, orioledb, zheap + +## Integration with Cirrus CI + +GitHub Actions and Cirrus CI run independently: + +- **Cirrus CI:** Comprehensive testing (Linux, FreeBSD, macOS, Windows) +- **GitHub Actions:** Sync, AI review, Windows dependency builds +- **No conflicts:** Both can run on same commits + +## Troubleshooting + +### Sync Issues + +**Problem:** Sync workflow failing +**Check:** Actions ā†’ "Sync from Upstream (Automatic)" ā†’ Latest run +**Fix:** See [docs/sync-setup.md](docs/sync-setup.md#sync-failure-recovery) + +### AI Review Issues + +**Problem:** AI review not running +**Check:** Is PR a draft? Draft PRs are skipped +**Fix:** Mark PR as ready for review + +**Problem:** AI review too expensive +**Check:** Cost logs in workflow artifacts +**Fix:** Adjust limits in `scripts/ai-review/config.json` + +### Workflow Permission Issues + +**Problem:** "Resource not accessible by integration" +**Check:** Settings ā†’ Actions ā†’ General ā†’ Workflow permissions +**Fix:** Enable "Read and write permissions" + +## Security + +### Secrets Management +- `ANTHROPIC_API_KEY`: Claude API key (required for AI review) +- `GITHUB_TOKEN`: Auto-generated, scoped to repository +- Never commit secrets to repository +- Rotate API keys quarterly + +### Permissions +- Workflows use minimum necessary permissions +- `contents: read` for code access +- `pull-requests: write` for comments +- `issues: write` for sync failure issues + +### Audit Trail +- All workflow runs logged (90-day retention) +- Cost tracking for AI reviews +- GitHub Actions audit log available + +## Support and Documentation + +### Detailed Documentation +- [Sync Setup Guide](docs/sync-setup.md) - Upstream sync system +- [AI Review Guide](docs/ai-review-guide.md) - AI code review system +- [Windows Builds Guide](docs/windows-builds.md) - Windows dependencies +- [Cost Optimization Guide](docs/cost-optimization.md) - Reducing CI/CD costs +- [Pristine Master Policy](docs/pristine-master-policy.md) - Master branch management + +### Reporting Issues + +Issues with CI/CD system: +1. Check workflow logs: Actions ā†’ Failed run ā†’ View logs +2. Search existing issues: label:automation +3. Create issue with workflow run URL and error messages + +### Modifying Workflows + +**Disabling a workflow:** +```bash +# Via GitHub UI: +# Actions ā†’ Select workflow ā†’ "..." ā†’ Disable workflow + +# Via git: +git mv .github/workflows/workflow-name.yml .github/workflows/workflow-name.yml.disabled +git commit -m "Disable workflow" +``` + +**Testing workflow changes:** +1. Create feature branch +2. Modify workflow file +3. Use `workflow_dispatch` trigger to test +4. Verify in Actions tab +5. Merge to master when working + +## Cost Summary + +| Component | Monthly Cost | Usage | Notes | +|-----------|-------------|-------|-------| +| Sync | $0 | ~150 min | Free tier: 2,000 min | +| AI Review | $30-45 | Variable | Claude API usage-based | +| Windows Builds | $5-8 | ~2,500 min | With caching + optimization | +| **Total** | **$35-53** | | After cost optimizations | + +**Comparison:** CodeRabbit (turnkey solution) = $99-499/month + +**Cost savings:** ~40-47% reduction through optimizations (see [Cost Optimization Guide](docs/cost-optimization.md)) + +## References + +- PostgreSQL: https://github.com/postgres/postgres +- GitHub Actions: https://docs.github.com/en/actions +- Claude API: https://docs.anthropic.com/ +- Cirrus CI: https://cirrus-ci.org/ +- winpgbuild: https://github.com/dpage/winpgbuild + +--- + +**Last Updated:** 2026-03-10 +**Maintained by:** PostgreSQL Mirror Automation diff --git a/.github/docs/pristine-master-policy.md b/.github/docs/pristine-master-policy.md new file mode 100644 index 0000000000000..9c0479d32df6a --- /dev/null +++ b/.github/docs/pristine-master-policy.md @@ -0,0 +1,225 @@ +# Pristine Master Policy + +## Overview + +The `master` branch in this mirror repository follows a "mostly pristine" policy, meaning it should closely mirror the upstream `postgres/postgres` repository with only specific exceptions allowed. + +## Allowed Commits on Master + +Master is considered "pristine" and the sync workflow will successfully merge upstream changes if local commits fall into these categories: + +### 1. āœ… CI/CD Configuration (`.github/` directory only) + +Commits that only modify files within the `.github/` directory are allowed. + +**Examples:** +- Adding GitHub Actions workflows +- Updating AI review configuration +- Modifying sync schedules +- Adding documentation in `.github/docs/` + +**Rationale:** CI/CD configuration is repository-specific and doesn't affect the PostgreSQL codebase itself. + +### 2. āœ… Development Environment Setup (commits named "dev setup ...") + +Commits with messages starting with "dev setup" (case-insensitive) are allowed, even if they modify files outside `.github/`. + +**Examples:** +- `dev setup v19` +- `Dev Setup: Add debugging configuration` +- `DEV SETUP - IDE and tooling` + +**Typical files in dev setup commits:** +- `.clang-format`, `.clangd` - Code formatting and LSP config +- `.envrc` - Directory environment variables (direnv) +- `.gdbinit` - Debugger configuration +- `.idea/`, `.vscode/` - IDE settings +- `flake.nix`, `shell.nix` - Nix development environment +- `pg-aliases.sh` - Personal shell aliases +- Other personal development tools + +**Rationale:** Development environment configuration is personal and doesn't affect the code or CI/CD. It's frequently updated as developers refine their workflow. + +### 3. āŒ Code Changes (NOT allowed) + +Any commits that: +- Modify PostgreSQL source code (`src/`, `contrib/`, etc.) +- Modify tests outside `.github/` +- Modify build system outside `.github/` +- Are not `.github/`-only AND don't start with "dev setup" + +**These will cause sync failures** and require manual resolution. + +## Branch Strategy + +### Master Branch +- **Purpose:** Mirror of upstream `postgres/postgres` + local CI/CD + dev environment +- **Updates:** Automatic hourly sync from upstream +- **Direct commits:** Only `.github/` changes or "dev setup" commits +- **All other work:** Use feature branches + +### Feature Branches +- **Purpose:** All PostgreSQL development work +- **Pattern:** `feature/*`, `dev/*`, `experiment/*` +- **Workflow:** + ```bash + git checkout master + git pull origin master + git checkout -b feature/my-feature + # Make changes... + git push origin feature/my-feature + # Create PR: feature/my-feature ā†’ master + ``` + +## Sync Workflow Behavior + +### Scenario 1: No Local Commits +``` +Upstream: A---B---C +Master: A---B---C +``` +**Result:** āœ… Already up to date (no action needed) + +### Scenario 2: Only .github/ Commits +``` +Upstream: A---B---C---D +Master: A---B---C---X (X modifies .github/ only) +``` +**Result:** āœ… Merge commit created +``` +Master: A---B---C---X---M + \ / + D---/ +``` + +### Scenario 3: Only "dev setup" Commits +``` +Upstream: A---B---C---D +Master: A---B---C---Y (Y is "dev setup v19") +``` +**Result:** āœ… Merge commit created +``` +Master: A---B---C---Y---M + \ / + D---/ +``` + +### Scenario 4: Mix of Allowed Commits +``` +Upstream: A---B---C---D +Master: A---B---C---X---Y (X=.github/, Y=dev setup) +``` +**Result:** āœ… Merge commit created + +### Scenario 5: Code Changes (Violation) +``` +Upstream: A---B---C---D +Master: A---B---C---Z (Z modifies src/backend/) +``` +**Result:** āŒ Sync fails, issue created + +**Recovery:** +1. Create feature branch from Z +2. Reset master to match upstream +3. Rebase feature branch +4. Create PR + +## Updating Dev Setup + +When you update your development environment: + +```bash +# Make changes to .clangd, flake.nix, etc. +git add .clangd flake.nix .vscode/ + +# Important: Start message with "dev setup" +git commit -m "dev setup v20: Update clangd config and add new aliases" + +git push origin master +``` + +The sync workflow will recognize this as a dev setup commit and preserve it during merges. + +**Naming convention:** +- āœ… `dev setup v20` +- āœ… `Dev setup: Update IDE config` +- āœ… `DEV SETUP - Add debugging tools` +- āŒ `Update development environment` (doesn't start with "dev setup") +- āŒ `dev environment changes` (doesn't start with "dev setup") + +## Sync Failure Recovery + +If sync fails because of non-allowed commits: + +### Check What's Wrong +```bash +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master + +# See which commits are problematic +git log upstream/master..origin/master --oneline + +# See which files were changed +git diff --name-only upstream/master...origin/master +``` + +### Option 1: Make Commit Acceptable + +If the commit should have been a "dev setup" commit: + +```bash +# Amend the commit message +git commit --amend -m "dev setup v21: Previous changes" +git push origin master --force-with-lease +``` + +### Option 2: Move to Feature Branch + +If the commit contains code changes: + +```bash +# Create feature branch +git checkout -b feature/recovery origin/master + +# Reset master to upstream +git checkout master +git reset --hard upstream/master +git push origin master --force + +# Your changes are safe in feature/recovery +git checkout feature/recovery +# Create PR when ready +``` + +## FAQ + +**Q: Why allow dev setup commits on master?** +A: Development environment configuration is personal, frequently updated, and doesn't affect the codebase or CI/CD. It's more convenient to keep it on master than manage separate branches. + +**Q: What if I forget to name it "dev setup"?** +A: Sync will fail. You can amend the commit message (see recovery above) or move the commit to a feature branch. + +**Q: Can I have both .github/ and dev setup changes in one commit?** +A: Yes! The sync workflow allows commits that modify .github/, or are named "dev setup", or both. + +**Q: What if upstream modifies the same files as my dev setup commit?** +A: The sync will attempt to merge automatically. If there are conflicts, you'll need to resolve them manually (rare, since upstream shouldn't touch personal dev files). + +**Q: Can I reorder commits on master?** +A: It's not recommended due to complexity. The sync workflow handles commits in any order as long as they follow the policy. + +## Monitoring + +**Check sync status:** +- Actions ā†’ "Sync from Upstream (Automatic)" +- Look for green āœ… on recent runs + +**Check for policy violations:** +- Open issues with label `sync-failure` +- These indicate commits that violated the pristine master policy + +## Related Documentation + +- [Sync Setup Guide](sync-setup.md) - Detailed sync workflow documentation +- [QUICKSTART](../QUICKSTART.md) - Quick setup guide +- [README](../README.md) - System overview diff --git a/.github/docs/sync-setup.md b/.github/docs/sync-setup.md new file mode 100644 index 0000000000000..1e12aeea3c5fc --- /dev/null +++ b/.github/docs/sync-setup.md @@ -0,0 +1,326 @@ +# Automated Upstream Sync Documentation + +## Overview + +This repository maintains a mirror of the official PostgreSQL repository at `postgres/postgres`. The sync system automatically keeps the `master` branch synchronized with upstream changes. + +## System Components + +### 1. Automatic Daily Sync +**File:** `.github/workflows/sync-upstream.yml` + +- **Trigger:** Daily at 00:00 UTC (cron schedule) +- **Purpose:** Automatically sync master branch without manual intervention +- **Process:** + 1. Fetches latest commits from `postgres/postgres` + 2. Fast-forward merges to local master (conflict-free) + 3. Pushes to `origin/master` + 4. Creates GitHub issue if conflicts detected + 5. Closes existing sync-failure issues on success + +### 2. Manual Sync Workflow +**File:** `.github/workflows/sync-upstream-manual.yml` + +- **Trigger:** Manual via Actions tab ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow +- **Purpose:** Testing and on-demand syncs +- **Options:** + - `force_push`: Use `--force-with-lease` when pushing (default: true) + +## Branch Strategy + +### Critical Rule: Master is Pristine + +- **master branch:** Mirror only - pristine copy of `postgres/postgres` +- **All development:** Feature branches (e.g., `feature/hot-updates`, `experiment/zheap`) +- **Never commit directly to master** - this will cause sync failures + +### Feature Branch Workflow + +```bash +# Start new feature from latest master +git checkout master +git pull origin master +git checkout -b feature/my-feature + +# Work on feature +git commit -m "Add feature" + +# Keep feature updated with upstream +git checkout master +git pull origin master +git checkout feature/my-feature +git rebase master + +# Push feature branch +git push origin feature/my-feature + +# Create PR: feature/my-feature ā†’ master +``` + +## Sync Failure Recovery + +### Diagnosis + +If sync fails, you'll receive a GitHub issue with label `sync-failure`. Check what commits are on master but not upstream: + +```bash +# Clone or update your local repository +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master + +# View conflicting commits +git log upstream/master..origin/master --oneline + +# See detailed changes +git diff upstream/master...origin/master +``` + +### Recovery Option 1: Preserve Commits (Recommended) + +If the commits on master should be kept: + +```bash +# Create backup branch from current master +git checkout origin/master +git checkout -b recovery/master-backup-$(date +%Y%m%d) +git push origin recovery/master-backup-$(date +%Y%m%d) + +# Reset master to upstream +git checkout master +git reset --hard upstream/master +git push origin master --force + +# Create feature branch from backup +git checkout -b feature/recovered-work recovery/master-backup-$(date +%Y%m%d) + +# Optional: rebase onto new master +git rebase master + +# Push feature branch +git push origin feature/recovered-work + +# Create PR: feature/recovered-work ā†’ master +``` + +### Recovery Option 2: Discard Commits + +If the commits on master were mistakes or already merged upstream: + +```bash +git checkout master +git reset --hard upstream/master +git push origin master --force +``` + +### Verification + +After recovery, verify sync status: + +```bash +# Check that master matches upstream +git log origin/master --oneline -10 +git log upstream/master --oneline -10 + +# These should be identical + +# Or run manual sync workflow +# GitHub ā†’ Actions ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow +``` + +The automatic sync will resume on next scheduled run (00:00 UTC daily). + +## Monitoring + +### Success Indicators + +- āœ“ GitHub Actions badge shows passing +- āœ“ No open issues with label `sync-failure` +- āœ“ `master` branch commit history matches `postgres/postgres` + +### Check Sync Status + +**Via GitHub UI:** +1. Go to: Actions ā†’ "Sync from Upstream (Automatic)" +2. Check latest run status + +**Via Git:** +```bash +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master +git log origin/master..upstream/master --oneline + +# No output = fully synced +# Commits listed = behind upstream (sync pending or failed) +``` + +**Via API:** +```bash +# Check latest workflow run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View run details +gh run view +``` + +### Sync Lag + +Expected lag: <1 hour from upstream commit to mirror + +- Upstream commits at 12:30 UTC ā†’ Synced at next daily run (00:00 UTC next day) = ~11.5 hours max +- For faster sync: Manually trigger workflow after major upstream merges + +## Configuration + +### GitHub Actions Permissions + +Required settings (already configured): + +1. **Settings ā†’ Actions ā†’ General ā†’ Workflow permissions:** + - āœ“ "Read and write permissions" + - āœ“ "Allow GitHub Actions to create and approve pull requests" + +2. **Repository Settings ā†’ Branches:** + - Consider: Branch protection rule on `master` to prevent direct pushes + - Exception: Allow `github-actions[bot]` to push + +### Adjusting Sync Schedule + +Edit `.github/workflows/sync-upstream.yml`: + +```yaml +on: + schedule: + # Current: Daily at 00:00 UTC + - cron: '0 0 * * *' + + # Examples: + # Every 6 hours: '0 */6 * * *' + # Twice daily: '0 0,12 * * *' + # Weekdays only: '0 0 * * 1-5' +``` + +**Recommendation:** Keep daily schedule to balance freshness with API usage. + +## Troubleshooting + +### Issue: Workflow not running + +**Check:** +1. Actions tab ā†’ Check if workflow is disabled +2. Settings ā†’ Actions ā†’ Ensure workflows are enabled for repository + +**Fix:** +- Enable workflow: Actions ā†’ Select workflow ā†’ "Enable workflow" + +### Issue: Permission denied on push + +**Check:** +- Settings ā†’ Actions ā†’ General ā†’ Workflow permissions + +**Fix:** +- Set to "Read and write permissions" +- Enable "Allow GitHub Actions to create and approve pull requests" + +### Issue: Merge conflicts every sync + +**Root cause:** Commits being made directly to master + +**Fix:** +1. Review `.git/hooks/` for pre-commit hooks that might auto-commit +2. Check if any automation is committing to master +3. Enforce branch protection rules +4. Educate team members on feature branch workflow + +### Issue: Sync successful but CI fails + +**This is expected** if upstream introduced breaking changes or test failures. + +**Handling:** +- Upstream tests failures are upstream's responsibility +- Focus: Ensure mirror stays in sync +- Separate: Your feature branches should pass CI + +## Cost and Usage + +### GitHub Actions Minutes + +- **Sync workflow:** ~2-3 minutes per run +- **Frequency:** Daily = 60-90 minutes/month +- **Free tier:** 2,000 minutes/month (public repos: unlimited) +- **Cost:** $0 (well within limits) + +### Network Usage + +- Fetches only new commits (incremental) +- Typical: <10 MB per sync +- Total: <300 MB/month + +## Security Considerations + +### Secrets + +- Uses `GITHUB_TOKEN` (automatically provided, scoped to repository) +- No additional secrets required +- Token permissions: Minimum necessary (contents:write, issues:write) + +### Audit Trail + +All syncs are logged: +- GitHub Actions run history (90 days retention) +- Git reflog on server +- Issue creation/closure for failures + +## Integration with Other Workflows + +### Cirrus CI + +Cirrus CI tests trigger on pushes to master: +- Sync pushes ā†’ Cirrus CI runs tests on synced commits +- This validates upstream changes against your test matrix + +### AI Code Review + +AI review workflows trigger on PRs, not master pushes: +- Sync to master does NOT trigger AI reviews +- Feature branch PRs ā†’ master do trigger AI reviews + +### Windows Builds + +Windows dependency builds trigger on master pushes: +- Sync pushes ā†’ Windows builds run +- Ensures dependencies stay compatible with latest upstream + +## Support + +### Reporting Issues + +If sync consistently fails: + +1. Check open issues with label `sync-failure` +2. Review workflow logs: Actions ā†’ Failed run ā†’ View logs +3. Create issue with: + - Workflow run URL + - Error messages from logs + - Output of `git log upstream/master..origin/master` + +### Disabling Automatic Sync + +If needed (e.g., during major refactoring): + +```bash +# Disable via GitHub UI +# Actions ā†’ "Sync from Upstream (Automatic)" ā†’ "..." ā†’ Disable workflow + +# Or delete/rename the workflow file +git mv .github/workflows/sync-upstream.yml .github/workflows/sync-upstream.yml.disabled +git commit -m "Temporarily disable automatic sync" +git push +``` + +**Remember to re-enable** once work is complete. + +## References + +- Upstream repository: https://github.com/postgres/postgres +- GitHub Actions docs: https://docs.github.com/en/actions +- Git branching strategies: https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows diff --git a/.github/workflows/sync-upstream-manual.yml b/.github/workflows/sync-upstream-manual.yml new file mode 100644 index 0000000000000..362c119a128e7 --- /dev/null +++ b/.github/workflows/sync-upstream-manual.yml @@ -0,0 +1,249 @@ +name: Sync from Upstream (Manual) + +on: + workflow_dispatch: + inputs: + force_push: + description: 'Use --force-with-lease when pushing' + required: false + type: boolean + default: true + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/postgres/postgres.git || true + git remote -v + + - name: Fetch upstream + run: | + echo "Fetching from upstream postgres/postgres..." + git fetch upstream master + echo "Current local master:" + git log origin/master --oneline -5 + echo "Upstream master:" + git log upstream/master --oneline -5 + + - name: Check for local commits + id: check_commits + run: | + git checkout master + LOCAL_COMMITS=$(git rev-list origin/master..upstream/master --count) + DIVERGED=$(git rev-list upstream/master..origin/master --count) + echo "commits_behind=$LOCAL_COMMITS" >> $GITHUB_OUTPUT + echo "commits_ahead=$DIVERGED" >> $GITHUB_OUTPUT + echo "Mirror is $DIVERGED commits ahead and $LOCAL_COMMITS commits behind upstream" + + if [ "$DIVERGED" -gt 0 ]; then + # Check commit messages for "dev setup" or "dev v" pattern + DEV_SETUP_COMMITS=$(git log --format=%s upstream/master...origin/master | grep -iE "^dev (setup|v[0-9])" | wc -l) + echo "dev_setup_commits=$DEV_SETUP_COMMITS" >> $GITHUB_OUTPUT + + # Check if diverged commits only touch .github/ directory + NON_GITHUB_CHANGES=$(git diff --name-only upstream/master...origin/master | grep -v "^\.github/" | wc -l) + echo "non_github_changes=$NON_GITHUB_CHANGES" >> $GITHUB_OUTPUT + + if [ "$NON_GITHUB_CHANGES" -eq 0 ]; then + echo "āœ“ All local commits are CI/CD configuration (.github/ only)" + elif [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "āœ“ Found $DEV_SETUP_COMMITS 'dev setup/version' commit(s)" + else + echo "āš ļø WARNING: Local commits modify files outside .github/ and are not 'dev setup/version' commits!" + git diff --name-only upstream/master...origin/master | grep -v "^\.github/" || true + fi + else + echo "non_github_changes=0" >> $GITHUB_OUTPUT + echo "dev_setup_commits=0" >> $GITHUB_OUTPUT + fi + + - name: Attempt merge + id: merge + run: | + COMMITS_AHEAD=${{ steps.check_commits.outputs.commits_ahead }} + COMMITS_BEHIND=${{ steps.check_commits.outputs.commits_behind }} + NON_GITHUB_CHANGES=${{ steps.check_commits.outputs.non_github_changes }} + DEV_SETUP_COMMITS=${{ steps.check_commits.outputs.dev_setup_commits }} + + # Check if there are problematic local commits + # Allow commits if: + # 1. Only .github/ changes (CI/CD config) + # 2. Has "dev setup/version" commits (personal development environment) + if [ "$COMMITS_AHEAD" -gt 0 ] && [ "$NON_GITHUB_CHANGES" -gt 0 ]; then + if [ "$DEV_SETUP_COMMITS" -eq 0 ]; then + echo "āŒ Local master has commits outside .github/ that are not 'dev setup/version' commits!" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + exit 1 + else + echo "āœ“ Non-.github/ changes are from 'dev setup/version' commits - allowed" + fi + fi + + # Already up to date + if [ "$COMMITS_BEHIND" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + echo "merge_status=uptodate" >> $GITHUB_OUTPUT + exit 0 + fi + + # Try fast-forward first (clean case) + if [ "$COMMITS_AHEAD" -eq 0 ]; then + echo "Fast-forwarding to upstream (no local commits)..." + git merge --ff-only upstream/master + echo "merge_status=success" >> $GITHUB_OUTPUT + exit 0 + fi + + # Local commits exist (.github/ and/or dev setup/version) - rebase onto upstream + if [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "Rebasing local CI/CD and dev setup/version commits onto upstream..." + else + echo "Rebasing local CI/CD commits (.github/ only) onto upstream..." + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if git rebase upstream/master; then + echo "āœ“ Successfully rebased local commits onto upstream" + echo "merge_status=success" >> $GITHUB_OUTPUT + else + echo "āŒ Rebase conflict occurred" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + + # Abort the failed rebase to clean up state + git rebase --abort + exit 1 + fi + continue-on-error: true + + - name: Push to origin + if: steps.merge.outputs.merge_status == 'success' + run: | + if [ "${{ inputs.force_push }}" == "true" ]; then + git push origin master --force-with-lease + else + git push origin master + fi + echo "āœ“ Successfully synced master with upstream" + + - name: Create issue on failure + if: steps.merge.outputs.merge_status == 'conflict' + uses: actions/github-script@v7 + with: + script: | + const title = 'šŸšØ Upstream Sync Failed - Manual Intervention Required'; + const body = `## Sync Failure Report + + The automated sync from \`postgres/postgres\` failed due to conflicting commits. + + **Details:** + - Local master has ${{ steps.check_commits.outputs.commits_ahead }} commit(s) not in upstream + - Upstream has ${{ steps.check_commits.outputs.commits_behind }} new commit(s) + - Non-.github/ changes: ${{ steps.check_commits.outputs.non_github_changes }} files + + **This indicates commits were made directly to master outside .github/**, which violates the pristine mirror policy. + + **Note:** Commits to .github/ (CI/CD configuration) are allowed and will be preserved during sync. + + ### Resolution Steps: + + 1. Identify the conflicting commits: + \`\`\`bash + git fetch origin + git fetch upstream https://github.com/postgres/postgres.git master + git log upstream/master..origin/master + \`\`\` + + 2. If these commits should be preserved: + - Create a feature branch: \`git checkout -b recovery/master-commits origin/master\` + - Reset master: \`git checkout master && git reset --hard upstream/master\` + - Push: \`git push origin master --force\` + - Cherry-pick or rebase the feature branch + + 3. If these commits should be discarded: + - Reset master: \`git checkout master && git reset --hard upstream/master\` + - Push: \`git push origin master --force\` + + 4. Close this issue once resolved + + **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + `; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + if (issues.data.length === 0) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['sync-failure', 'automation'] + }); + } + + - name: Close existing sync-failure issues + if: steps.merge.outputs.merge_status == 'success' + uses: actions/github-script@v7 + with: + script: | + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + for (const issue of issues.data) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: 'āœ“ Sync successful - closing this issue automatically.' + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + + - name: Summary + if: always() + run: | + echo "### Sync Summary" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** ${{ steps.merge.outputs.merge_status }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits behind:** ${{ steps.check_commits.outputs.commits_behind }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits ahead:** ${{ steps.check_commits.outputs.commits_ahead }}" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.merge.outputs.merge_status }}" == "success" ]; then + echo "- **Result:** āœ“ Successfully synced with upstream" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.merge.outputs.merge_status }}" == "uptodate" ]; then + echo "- **Result:** āœ“ Already up to date" >> $GITHUB_STEP_SUMMARY + else + echo "- **Result:** āš ļø Sync failed - manual intervention required" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 0000000000000..b3a6466980b0d --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,256 @@ +name: Sync from Upstream (Automatic) + +on: + schedule: + # Run hourly every day + - cron: '0 * * * *' + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/postgres/postgres.git || true + git remote -v + + - name: Fetch upstream + run: | + echo "Fetching from upstream postgres/postgres..." + git fetch upstream master + + - name: Check for local commits + id: check_commits + run: | + git checkout master + LOCAL_COMMITS=$(git rev-list origin/master..upstream/master --count) + DIVERGED=$(git rev-list upstream/master..origin/master --count) + echo "commits_behind=$LOCAL_COMMITS" >> $GITHUB_OUTPUT + echo "commits_ahead=$DIVERGED" >> $GITHUB_OUTPUT + + if [ "$LOCAL_COMMITS" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + else + echo "Mirror is $LOCAL_COMMITS commits behind upstream" + fi + + if [ "$DIVERGED" -gt 0 ]; then + echo "āš ļø Local master has $DIVERGED commits not in upstream" + + # Check commit messages for "dev setup" or "dev v" pattern + DEV_SETUP_COMMITS=$(git log --format=%s upstream/master..origin/master | grep -iE "^dev (setup|v[0-9])" | wc -l) + echo "dev_setup_commits=$DEV_SETUP_COMMITS" >> $GITHUB_OUTPUT + + # Check if diverged commits only touch .github/ directory + NON_GITHUB_CHANGES=$(git diff --name-only upstream/master...origin/master | grep -v "^\.github/" | wc -l) + echo "non_github_changes=$NON_GITHUB_CHANGES" >> $GITHUB_OUTPUT + + if [ "$NON_GITHUB_CHANGES" -eq 0 ]; then + echo "āœ“ All local commits are CI/CD configuration (.github/ only) - will merge" + elif [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "āœ“ Found $DEV_SETUP_COMMITS 'dev setup/version' commit(s)" + else + echo "āš ļø WARNING: Local commits modify files outside .github/ and are not 'dev setup/version' commits!" + git diff --name-only upstream/master...origin/master | grep -v "^\.github/" || true + echo "Non-dev commits:" + git log --format=" %h %s" upstream/master..origin/master | grep -ivE "^ [a-f0-9]* dev (setup|v[0-9])" || true + fi + else + echo "non_github_changes=0" >> $GITHUB_OUTPUT + echo "dev_setup_commits=0" >> $GITHUB_OUTPUT + fi + + - name: Attempt merge + id: merge + run: | + COMMITS_AHEAD=${{ steps.check_commits.outputs.commits_ahead }} + COMMITS_BEHIND=${{ steps.check_commits.outputs.commits_behind }} + NON_GITHUB_CHANGES=${{ steps.check_commits.outputs.non_github_changes }} + DEV_SETUP_COMMITS=${{ steps.check_commits.outputs.dev_setup_commits }} + + # Check if there are problematic local commits + # Allow commits if: + # 1. Only .github/ changes (CI/CD config) + # 2. Has "dev setup/version" commits (personal development environment) + if [ "$COMMITS_AHEAD" -gt 0 ] && [ "$NON_GITHUB_CHANGES" -gt 0 ]; then + if [ "$DEV_SETUP_COMMITS" -eq 0 ]; then + echo "āŒ Local master has commits outside .github/ that are not 'dev setup/version' commits!" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + exit 1 + else + echo "āœ“ Non-.github/ changes are from 'dev setup/version' commits - allowed" + fi + fi + + # Already up to date + if [ "$COMMITS_BEHIND" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + echo "merge_status=uptodate" >> $GITHUB_OUTPUT + exit 0 + fi + + # Try fast-forward first (clean case) + if [ "$COMMITS_AHEAD" -eq 0 ]; then + echo "Fast-forwarding to upstream (no local commits)..." + git merge --ff-only upstream/master + echo "merge_status=success" >> $GITHUB_OUTPUT + exit 0 + fi + + # Local commits exist (.github/ and/or dev setup/version) - rebase onto upstream + if [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "Rebasing local CI/CD and dev setup/version commits onto upstream..." + else + echo "Rebasing local CI/CD commits (.github/ only) onto upstream..." + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if git rebase upstream/master; then + echo "āœ“ Successfully rebased local commits onto upstream" + echo "merge_status=success" >> $GITHUB_OUTPUT + else + echo "āŒ Rebase conflict occurred" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + + # Abort the failed rebase to clean up state + git rebase --abort + exit 1 + fi + continue-on-error: true + + - name: Push to origin + if: steps.merge.outputs.merge_status == 'success' + run: | + git push origin master --force-with-lease + + COMMITS_SYNCED="${{ steps.check_commits.outputs.commits_behind }}" + echo "āœ“ Successfully synced $COMMITS_SYNCED commits from upstream" + + - name: Create issue on failure + if: steps.merge.outputs.merge_status == 'conflict' + uses: actions/github-script@v7 + with: + script: | + const title = 'šŸšØ Automated Upstream Sync Failed'; + const body = `## Automatic Sync Failure + + The daily sync from \`postgres/postgres\` failed. + + **Details:** + - Local master has ${{ steps.check_commits.outputs.commits_ahead }} commit(s) not in upstream + - Upstream has ${{ steps.check_commits.outputs.commits_behind }} new commit(s) + - Non-.github/ changes: ${{ steps.check_commits.outputs.non_github_changes }} files + - **Run date:** ${new Date().toISOString()} + + **Root cause:** Commits were made directly to master outside of .github/, which violates the pristine mirror policy. + + **Note:** Commits to .github/ (CI/CD configuration) are allowed and will be preserved during sync. + + ### Resolution Steps: + + 1. Review the conflicting commits: + \`\`\`bash + git log upstream/master..origin/master --oneline + \`\`\` + + 2. Determine if commits should be: + - **Preserved:** Create feature branch and reset master + - **Discarded:** Hard reset master to upstream + + 3. See [sync documentation](.github/docs/sync-setup.md) for detailed recovery procedures + + 4. Run manual sync workflow after resolution to verify + + **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + `; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + if (issues.data.length === 0) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['sync-failure', 'automation', 'urgent'] + }); + } else { + // Update existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issues.data[0].number, + body: `Sync failed again on ${new Date().toISOString()}\n\nWorkflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}` + }); + } + + - name: Close sync-failure issues + if: steps.merge.outputs.merge_status == 'success' + uses: actions/github-script@v7 + with: + script: | + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + for (const issue of issues.data) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: `āœ“ Automatic sync successful on ${new Date().toISOString()} - synced ${{ steps.check_commits.outputs.commits_behind }} commits.\n\nClosing issue automatically.` + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + + - name: Summary + if: always() + run: | + echo "### Daily Sync Summary" >> $GITHUB_STEP_SUMMARY + echo "- **Date:** $(date -u)" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** ${{ steps.merge.outputs.merge_status }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits synced:** ${{ steps.check_commits.outputs.commits_behind }}" >> $GITHUB_STEP_SUMMARY + + if [ "${{ steps.merge.outputs.merge_status }}" == "success" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ“ Mirror successfully updated with upstream postgres/postgres" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.merge.outputs.merge_status }}" == "uptodate" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ“ Mirror already up to date" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "āš ļø Sync failed - check created issue for details" >> $GITHUB_STEP_SUMMARY + fi From 3ed29c42a4007366e9c50e03b5cd1be6e501ea4a Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 09:08:46 -0400 Subject: [PATCH 2/6] ci: AI/LLM PR review (OCR via Bedrock + Agora MCP history) The Open Code Review system: ocr-review and ocr-model-check workflows plus .github/ocr config (LiteLLM->Bedrock Claude Opus 4.8, rule.json, context.md, pg-history.py). --- .github/docs/ai-review-guide.md | 512 ++++++++++++++++++++++++++ .github/docs/bedrock-setup.md | 298 +++++++++++++++ .github/docs/cost-optimization.md | 219 +++++++++++ .github/ocr/context.md | 126 +++++++ .github/ocr/litellm.yaml | 41 +++ .github/ocr/pg-history.py | 225 +++++++++++ .github/ocr/rule.json | 65 ++++ .github/workflows/ocr-model-check.yml | 89 +++++ .github/workflows/ocr-review.yml | 427 +++++++++++++++++++++ 9 files changed, 2002 insertions(+) create mode 100644 .github/docs/ai-review-guide.md create mode 100644 .github/docs/bedrock-setup.md create mode 100644 .github/docs/cost-optimization.md create mode 100644 .github/ocr/context.md create mode 100644 .github/ocr/litellm.yaml create mode 100644 .github/ocr/pg-history.py create mode 100644 .github/ocr/rule.json create mode 100644 .github/workflows/ocr-model-check.yml create mode 100644 .github/workflows/ocr-review.yml diff --git a/.github/docs/ai-review-guide.md b/.github/docs/ai-review-guide.md new file mode 100644 index 0000000000000..eff0ed10cba4f --- /dev/null +++ b/.github/docs/ai-review-guide.md @@ -0,0 +1,512 @@ +# AI-Powered Code Review Guide + +## Overview + +This system uses Claude AI (Anthropic) to provide PostgreSQL-aware code reviews on pull requests. Reviews are similar in style to feedback from the PostgreSQL Hackers mailing list. + +## How It Works + +``` +PR Event (opened/updated) + ā†“ +GitHub Actions Workflow Starts + ā†“ +Fetch PR diff + metadata + ā†“ +Filter reviewable files (.c, .h, .sql, docs, Makefiles) + ā†“ +Route each file to appropriate review prompt + ā†“ +Send to Claude API with PostgreSQL context + ā†“ +Parse response for issues + ā†“ +Post inline comments + summary to PR + ā†“ +Add labels (security-concern, performance, etc.) +``` + +## Features + +### PostgreSQL-Specific Reviews + +**C Code Review:** +- Memory management (palloc/pfree, memory contexts) +- Concurrency (lock ordering, race conditions) +- Error handling (elog/ereport patterns) +- Performance (algorithm complexity, cache efficiency) +- Security (buffer overflows, SQL injection vectors) +- PostgreSQL conventions (naming, comments, style) + +**SQL Review:** +- PostgreSQL SQL dialect correctness +- Regression test patterns +- Performance (index usage, join strategy) +- Deterministic output for tests +- Edge case coverage + +**Documentation Review:** +- Technical accuracy +- SGML/DocBook format +- PostgreSQL style guide compliance +- Examples and cross-references + +**Build System Review:** +- Makefile correctness (GNU Make, PGXS) +- Meson build consistency +- Cross-platform portability +- VPATH build support + +### Automatic Labeling + +Reviews automatically add labels based on findings: + +- `security-concern` - Security issues, vulnerabilities +- `performance-concern` - Performance problems +- `needs-tests` - Missing test coverage +- `needs-docs` - Missing documentation +- `memory-management` - Memory leaks, context issues +- `concurrency-issue` - Deadlocks, race conditions + +### Cost Management + +- **Per-PR limit:** $15 (configurable) +- **Monthly limit:** $200 (configurable) +- **Alert threshold:** $150 +- **Skip draft PRs** to save costs +- **Skip large files** (>5000 lines) +- **Skip binary/generated files** + +## Setup + +### 1. Install Dependencies + +```bash +cd .github/scripts/ai-review +npm install +``` + +### 2. Configure API Key + +Get API key from: https://console.anthropic.com/ + +Add to repository secrets: +1. Settings ā†’ Secrets and variables ā†’ Actions +2. New repository secret +3. Name: `ANTHROPIC_API_KEY` +4. Value: Your API key +5. Add secret + +### 3. Enable Workflow + +The workflow is triggered automatically on PR events: +- PR opened +- PR synchronized (updated) +- PR reopened +- PR marked ready for review (draft ā†’ ready) + +**Draft PRs are skipped** to save costs. + +## Configuration + +### Main Configuration: `config.json` + +```json +{ + "model": "claude-3-5-sonnet-20241022", + "max_tokens_per_request": 4096, + "max_file_size_lines": 5000, + + "cost_limits": { + "max_per_pr_dollars": 15.0, + "max_per_month_dollars": 200.0, + "alert_threshold_dollars": 150.0 + }, + + "skip_paths": [ + "*.png", "*.jpg", "*.svg", + "src/test/regress/expected/*", + "*.po", "*.pot" + ], + + "auto_labels": { + "security-concern": ["security issue", "vulnerability"], + "performance-concern": ["inefficient", "O(nĀ²)"], + "needs-tests": ["missing test", "no test coverage"] + } +} +``` + +**Tunable parameters:** +- `max_tokens_per_request`: Response length (4096 = ~3000 words) +- `max_file_size_lines`: Skip files larger than this +- `cost_limits`: Adjust budget caps +- `skip_paths`: Add more patterns to skip +- `auto_labels`: Customize label keywords + +### Review Prompts + +Located in `.github/scripts/ai-review/prompts/`: + +- `c-code.md` - PostgreSQL C code review +- `sql.md` - SQL and regression test review +- `documentation.md` - Documentation review +- `build-system.md` - Makefile/Meson review + +**Customization:** Edit prompts to adjust review focus and style. + +## Usage + +### Automatic Reviews + +Reviews run automatically on PRs to `master` and `feature/**` branches. + +**Typical workflow:** +1. Create feature branch +2. Make changes +3. Push branch: `git push origin feature/my-feature` +4. Create PR +5. AI review runs automatically +6. Review AI feedback +7. Make updates if needed +8. Push updates ā†’ AI re-reviews + +### Manual Reviews + +Trigger manually via GitHub Actions: + +**Via UI:** +1. Actions ā†’ "AI Code Review" +2. Run workflow +3. Enter PR number +4. Run workflow + +**Via CLI:** +```bash +gh workflow run ai-code-review.yml -f pr_number=123 +``` + +### Interpreting Reviews + +**Inline comments:** +- Posted on specific lines of code +- Format: `**[Category]**` followed by description +- Categories: Memory, Security, Performance, etc. + +**Summary comment:** +- Posted at PR level +- Overview of files reviewed +- Issue count by category +- Cost information + +**Labels:** +- Automatically added based on findings +- Filter PRs by label to prioritize +- Remove label manually if false positive + +### Best Practices + +**Trust but verify:** +- AI reviews are helpful but not infallible +- False positives happen (~5% rate) +- Use judgment - AI doesn't have full context +- Especially verify: security and correctness issues + +**Iterative improvement:** +- AI learns from the prompts, not from feedback +- If AI consistently misses something, update prompts +- Share false positives/negatives to improve system + +**Cost consciousness:** +- Keep PRs focused (fewer files = lower cost) +- Use draft PRs for work-in-progress (AI skips drafts) +- Mark PR ready when you want AI review + +## Cost Tracking + +### View Costs + +**Per-PR cost:** +- Shown in AI review summary comment +- Format: `Cost: $X.XX | Model: claude-3-5-sonnet` + +**Monthly cost:** +- Download cost logs from workflow artifacts +- Aggregate to calculate monthly total + +**Download cost logs:** +```bash +# List recent runs +gh run list --workflow=ai-code-review.yml --limit 10 + +# Download artifact +gh run download -n ai-review-cost-log- +``` + +### Cost Estimation + +**Token costs (Claude 3.5 Sonnet):** +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +**Typical costs:** +- Small PR (<500 lines, 5 files): $0.50-$1.00 +- Medium PR (500-2000 lines, 15 files): $1.00-$3.00 +- Large PR (2000-5000 lines, 30 files): $3.00-$7.50 + +**Expected monthly (20 PRs/month mixed sizes):** $35-50 + +### Budget Controls + +**Automatic limits:** +- Per-PR limit: Stops reviewing after $15 +- Monthly limit: Stops at $200 (requires manual override) +- Alert: Warning at $150 + +**Manual controls:** +- Disable workflow: Actions ā†’ AI Code Review ā†’ Disable +- Reduce `max_tokens_per_request` in config +- Add more patterns to `skip_paths` +- Increase `max_file_size_lines` threshold + +## Troubleshooting + +### Issue: No review posted + +**Possible causes:** +1. PR is draft (intentionally skipped) +2. No reviewable files (all binary or skipped patterns) +3. API key missing or invalid +4. Cost limit reached + +**Check:** +- Actions ā†’ "AI Code Review" ā†’ Latest run ā†’ View logs +- Look for: "Skipping draft PR" or "No reviewable files" +- Verify: `ANTHROPIC_API_KEY` secret exists + +### Issue: Review incomplete + +**Possible causes:** +1. PR cost limit reached ($15 default) +2. File too large (>5000 lines) +3. API rate limit hit + +**Check:** +- Review summary comment for "Reached PR cost limit" +- Workflow logs for "Skipping X - too large" + +**Fix:** +- Increase `max_per_pr_dollars` in config +- Increase `max_file_size_lines` (trade-off: higher cost) +- Split large PR into smaller PRs + +### Issue: False positives + +**Example:** AI flags correct code as problematic + +**Handling:** +1. Ignore the comment (human judgment overrides) +2. Reply to comment explaining why it's correct +3. If systematic: Update prompt to clarify + +**Note:** Some false positives are acceptable (5-10% rate) + +### Issue: Claude API errors + +**Error types:** +- `401 Unauthorized`: Invalid API key +- `429 Too Many Requests`: Rate limit +- `500 Internal Server Error`: Claude service issue + +**Check:** +- Workflow logs for error messages +- Claude status: https://status.anthropic.com/ + +**Fix:** +- Rotate API key if 401 +- Wait and retry if 429 or 500 +- Contact Anthropic support if persistent + +### Issue: High costs + +**Unexpected high costs:** +1. Check cost logs for large PRs +2. Review `skip_paths` - are large files being reviewed? +3. Check for repeated reviews (PR updated many times) + +**Optimization:** +- Add more skip patterns for generated files +- Lower `max_tokens_per_request` (shorter reviews) +- Increase `max_file_size_lines` to skip more files +- Batch PR updates to reduce review runs + +## Disabling AI Review + +### Temporarily disable + +**For one PR:** +- Convert to draft +- Or add `[skip ai]` to PR title (requires workflow modification) + +**For all PRs:** +```bash +# Via GitHub UI: +# Actions ā†’ "AI Code Review" ā†’ "..." ā†’ Disable workflow + +# Via git: +git mv .github/workflows/ai-code-review.yml \ + .github/workflows/ai-code-review.yml.disabled +git commit -m "Disable AI code review" +git push +``` + +### Permanently remove + +```bash +# Remove workflow +rm .github/workflows/ai-code-review.yml + +# Remove scripts +rm -rf .github/scripts/ai-review + +# Commit +git commit -am "Remove AI code review system" +git push +``` + +## Testing and Iteration + +### Shadow Mode (Week 1) + +Run reviews but don't post comments: + +1. Modify `review-pr.js`: + ```javascript + // Comment out posting functions + // await postInlineComments(...) + // await postSummaryComment(...) + ``` + +2. Reviews saved to workflow artifacts +3. Review quality offline +4. Tune prompts based on results + +### Comment Mode (Week 2) + +Post comments with `[AI Review]` prefix: + +1. Add prefix to comment body: + ```javascript + const body = `**[AI Review] [${issue.category}]**\n\n${issue.description}`; + ``` + +2. Gather feedback from developers +3. Adjust prompts and configuration + +### Full Mode (Week 3+) + +Remove prefix, enable all features: + +1. Remove `[AI Review]` prefix +2. Enable auto-labeling +3. Monitor quality and costs +4. Iterate on prompts as needed + +## Advanced Customization + +### Custom Review Prompts + +Add a new prompt for a file type: + +1. Create `.github/scripts/ai-review/prompts/my-type.md` +2. Write review guidelines (see existing prompts) +3. Update `config.json`: + ```json + "file_type_patterns": { + "my_type": ["*.ext", "special/*.files"] + } + ``` +4. Test with manual workflow trigger + +### Conditional Reviews + +Skip AI review for certain PRs: + +Modify `.github/workflows/ai-code-review.yml`: +```yaml +jobs: + ai-review: + if: | + github.event.pull_request.draft == false && + !contains(github.event.pull_request.title, '[skip ai]') && + !contains(github.event.pull_request.labels.*.name, 'no-ai-review') +``` + +### Cost Alerts + +Add cost alert notifications: + +1. Create workflow in `.github/workflows/cost-alert.yml` +2. Trigger: On schedule (weekly) +3. Aggregate cost logs +4. Post issue if over threshold + +## Security and Privacy + +### API Key Security + +- Store only in GitHub Secrets (encrypted at rest) +- Never commit to repository +- Never log in workflow output +- Rotate quarterly + +### Code Privacy + +- Code sent to Claude API (Anthropic) +- Anthropic does not train on API data +- API requests are not retained long-term +- See: https://www.anthropic.com/legal/privacy + +### Sensitive Code + +If reviewing sensitive/proprietary code: + +1. Review Anthropic's terms of service +2. Consider: Self-hosted alternative (future) +3. Or: Skip AI review for sensitive PRs (add label) + +## Support + +### Questions + +- Check this guide first +- Search GitHub issues: label:ai-review +- Check Claude API docs: https://docs.anthropic.com/ + +### Reporting Issues + +Create issue with: +- PR number +- Workflow run URL +- Error messages from logs +- Expected vs actual behavior + +### Improving Prompts + +Contributions welcome: +1. Identify systematic issue (false positive/negative) +2. Propose prompt modification +3. Test on sample PRs +4. Submit PR with updated prompt + +## References + +- Claude API: https://docs.anthropic.com/ +- Claude Models: https://www.anthropic.com/product +- PostgreSQL Hacker's Guide: https://wiki.postgresql.org/wiki/Developer_FAQ +- GitHub Actions: https://docs.github.com/en/actions + +--- + +**Version:** 1.0 +**Last Updated:** 2026-03-10 diff --git a/.github/docs/bedrock-setup.md b/.github/docs/bedrock-setup.md new file mode 100644 index 0000000000000..d8fbd898b51c6 --- /dev/null +++ b/.github/docs/bedrock-setup.md @@ -0,0 +1,298 @@ +# AWS Bedrock Setup for AI Code Review + +This guide explains how to use AWS Bedrock instead of the direct Anthropic API for AI code reviews. + +## Why Use Bedrock? + +- **AWS Credits:** Use existing AWS credits +- **Regional Availability:** Deploy in specific AWS regions +- **Compliance:** Meet specific compliance requirements +- **Integration:** Easier integration with AWS infrastructure +- **IAM Roles:** Use IAM roles instead of API keys when running on AWS + +## Prerequisites + +1. **AWS Account** with Bedrock access +2. **Bedrock Model Access** - Claude 3.5 Sonnet must be enabled +3. **IAM Permissions** for Bedrock API calls + +## Step 1: Enable Bedrock Model Access + +1. Log into AWS Console +2. Navigate to **Amazon Bedrock** +3. Go to **Model access** (left sidebar) +4. Click **Modify model access** +5. Find and enable: **Anthropic - Claude 3.5 Sonnet v2** +6. Click **Save changes** +7. Wait for status to show "Access granted" (~2-5 minutes) + +## Step 2: Create IAM User for GitHub Actions + +### Option A: IAM User with Access Keys (Recommended for GitHub Actions) + +1. Go to **IAM Console** +2. Click **Users** ā†’ **Create user** +3. Username: `github-actions-bedrock` +4. Click **Next** + +**Attach Policy:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel" + ], + "Resource": [ + "arn:aws:bedrock:*::foundation-model/anthropic.claude-3-5-sonnet-*" + ] + } + ] +} +``` + +5. Click **Create policy** ā†’ **JSON** ā†’ Paste above +6. Name: `BedrockClaudeInvokeOnly` +7. Attach policy to user +8. Click **Create user** + +**Create Access Keys:** +1. Click on the created user +2. Go to **Security credentials** tab +3. Click **Create access key** +4. Select: **Third-party service** +5. Click **Next** ā†’ **Create access key** +6. **Download** or copy: + - Access key ID (starts with `AKIA...`) + - Secret access key (only shown once!) + +### Option B: IAM Role (For AWS-hosted runners) + +If running GitHub Actions on AWS (self-hosted runners): + +1. Create IAM Role with trust policy for your EC2/ECS/EKS +2. Attach same `BedrockClaudeInvokeOnly` policy +3. Assign role to your runner infrastructure +4. No access keys needed! + +## Step 3: Configure Repository + +### A. Add AWS Secrets to GitHub + +1. Go to: **Settings** ā†’ **Secrets and variables** ā†’ **Actions** +2. Click **New repository secret** for each: + +**Secret 1:** +- Name: `AWS_ACCESS_KEY_ID` +- Value: Your access key ID from Step 2 + +**Secret 2:** +- Name: `AWS_SECRET_ACCESS_KEY` +- Value: Your secret access key from Step 2 + +**Secret 3:** +- Name: `AWS_REGION` +- Value: Your Bedrock region (e.g., `us-east-1`) + +### B. Update Configuration + +Edit `.github/scripts/ai-review/config.json`: + +```json +{ + "provider": "bedrock", + "model": "claude-3-5-sonnet-20241022", + "bedrock_model_id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock_region": "us-east-1", + ... +} +``` + +**Available Bedrock Model IDs:** +- US: `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- EU: `eu.anthropic.claude-3-5-sonnet-20241022-v2:0` +- Asia Pacific: `apac.anthropic.claude-3-5-sonnet-20241022-v2:0` + +**Available Regions:** +- `us-east-1` (US East - N. Virginia) +- `us-west-2` (US West - Oregon) +- `eu-central-1` (Europe - Frankfurt) +- `eu-west-1` (Europe - Ireland) +- `eu-west-2` (Europe - London) +- `ap-southeast-1` (Asia Pacific - Singapore) +- `ap-southeast-2` (Asia Pacific - Sydney) +- `ap-northeast-1` (Asia Pacific - Tokyo) + +Check current availability: https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html + +### C. Install Dependencies + +```bash +cd .github/scripts/ai-review +npm install +``` + +This will install the AWS SDK for Bedrock. + +## Step 4: Test Bedrock Integration + +```bash +# Create test PR +git checkout -b test/bedrock-review +echo "// Bedrock test" >> test.c +git add test.c +git commit -m "Test: Bedrock AI review" +git push origin test/bedrock-review +``` + +Then create PR via GitHub UI. Check: +1. **Actions** tab - workflow should run +2. **PR comments** - AI review should appear +3. **Workflow logs** - should show "Using AWS Bedrock as provider" + +## Cost Comparison + +### Bedrock Pricing (Claude 3.5 Sonnet - us-east-1) +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +### Direct Anthropic API Pricing +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +**Same price!** Choose based on infrastructure preference. + +## Troubleshooting + +### Error: "Access denied to model" + +**Check:** +1. Model access enabled in Bedrock console? +2. IAM policy includes correct model ARN? +3. Region matches between config and enabled models? + +**Fix:** +```bash +# Verify model access via AWS CLI +aws bedrock list-foundation-models --region us-east-1 --query 'modelSummaries[?contains(modelId, `claude-3-5-sonnet`)]' +``` + +### Error: "InvalidSignatureException" + +**Check:** +1. AWS_ACCESS_KEY_ID correct? +2. AWS_SECRET_ACCESS_KEY correct? +3. Secrets named exactly as shown? + +**Fix:** +- Re-create access keys +- Update GitHub secrets +- Ensure no extra spaces in secret values + +### Error: "ThrottlingException" + +**Cause:** Bedrock rate limits exceeded + +**Fix:** +1. Reduce `max_concurrent_requests` in config.json +2. Add delays between requests +3. Request quota increase via AWS Support + +### Error: "Model not found" + +**Check:** +1. `bedrock_model_id` matches your region +2. Using cross-region model ID (e.g., `us.anthropic...` in us-east-1) + +**Fix:** +Update `bedrock_model_id` in config.json to match your region: +- US regions: `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- EU regions: `eu.anthropic.claude-3-5-sonnet-20241022-v2:0` + +## Switching Between Providers + +### Switch to Bedrock + +Edit `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "bedrock", + ... +} +``` + +### Switch to Direct Anthropic API + +Edit `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "anthropic", + ... +} +``` + +No other changes needed! The code automatically detects the provider. + +## Advanced: Cross-Region Setup + +Deploy in multiple regions for redundancy: + +```json +{ + "provider": "bedrock", + "bedrock_regions": ["us-east-1", "us-west-2"], + "bedrock_failover": true +} +``` + +Then update `review-pr.js` to implement failover logic. + +## Security Best Practices + +1. **Least Privilege:** IAM user can only invoke Claude models +2. **Rotate Keys:** Rotate access keys quarterly +3. **Audit Logs:** Enable CloudTrail for Bedrock API calls +4. **Cost Alerts:** Set up AWS Budgets alerts +5. **Secrets:** Never commit AWS credentials to git + +## Monitoring + +### AWS CloudWatch + +Bedrock metrics available: +- `Invocations` - Number of API calls +- `InvocationLatency` - Response time +- `InvocationClientErrors` - 4xx errors +- `InvocationServerErrors` - 5xx errors + +### Cost Tracking + +```bash +# Check Bedrock costs (current month) +aws ce get-cost-and-usage \ + --time-period Start=2026-03-01,End=2026-03-31 \ + --granularity MONTHLY \ + --metrics BlendedCost \ + --filter file://filter.json + +# filter.json: +{ + "Dimensions": { + "Key": "SERVICE", + "Values": ["Amazon Bedrock"] + } +} +``` + +## References + +- AWS Bedrock Docs: https://docs.aws.amazon.com/bedrock/ +- Model Access: https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html +- Bedrock Pricing: https://aws.amazon.com/bedrock/pricing/ +- IAM Best Practices: https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html + +--- + +**Need help?** Check workflow logs in Actions tab or create an issue. diff --git a/.github/docs/cost-optimization.md b/.github/docs/cost-optimization.md new file mode 100644 index 0000000000000..bcfc1c47b3ed8 --- /dev/null +++ b/.github/docs/cost-optimization.md @@ -0,0 +1,219 @@ +# CI/CD Cost Optimization + +## Overview + +This document describes the cost optimization strategies used in the PostgreSQL mirror CI/CD system to minimize GitHub Actions minutes and API costs while maintaining full functionality. + +## Optimization Strategies + +### 1. Skip Builds for Pristine Commits + +**Problem:** "Dev setup" commits and .github/ configuration changes don't require expensive Windows dependency builds or comprehensive testing. + +**Solution:** The Windows Dependencies workflow includes a `check-changes` job that inspects recent commits and skips builds when all commits are: +- Messages starting with "dev setup" (case-insensitive), OR +- Only modifying files under `.github/` directory + +**Implementation:** See `.github/workflows/windows-dependencies.yml` lines 42-90 + +**Savings:** +- Avoids ~45 minutes of Windows runner time per push +- Windows runners cost 2x Linux minutes (1 minute = 2 billed minutes) +- Estimated savings: ~$8-12/month + +### 2. AI Review Only on Pull Requests + +**Problem:** AI code review is expensive and unnecessary for direct commits to master or pristine commits. + +**Solution:** The AI Code Review workflow only triggers on: +- `pull_request` events (opened, synchronized, reopened, ready_for_review) +- Manual `workflow_dispatch` for testing specific PRs +- Skips draft PRs automatically + +**Implementation:** See `.github/workflows/ai-code-review.yml` lines 3-17 + +**Savings:** +- No reviews on dev setup commits or CI/CD changes +- No reviews on draft PRs (saves ~$1-3 per draft) +- Estimated savings: ~$10-20/month + +### 3. Aggressive Caching + +**Windows Dependencies:** +- Cache key: `--win64-` +- Cache duration: GitHub's default (7 days unused, 10 GB limit) +- Cache hit rate: 80-90% for stable versions + +**Node.js Dependencies:** +- AI review scripts cache npm packages +- Cache key based on `package.json` hash +- Near 100% cache hit rate + +**Savings:** +- Reduces build time from 45 minutes to ~5 minutes on cache hit +- Estimated savings: ~$15-20/month + +### 4. Weekly Scheduled Builds + +**Problem:** GitHub Actions artifacts expire after 90 days, making cached dependencies stale. + +**Solution:** Windows Dependencies runs on a weekly schedule (Sunday 4 AM UTC) to refresh artifacts before expiration. + +**Cost:** +- Weekly builds: ~45 minutes/week Ɨ 4 weeks = 180 minutes/month +- Windows multiplier: 360 billed minutes +- Cost: ~$6/month (within budget) + +**Alternative considered:** Daily builds would cost ~$50/month (rejected) + +### 5. Sync Workflow Optimization + +**Automatic Sync:** +- Runs hourly to keep mirror current +- Very lightweight: ~2-3 minutes per run +- Cost: ~150 minutes/month = $0 (within free tier) + +**Manual Sync:** +- Only runs on explicit trigger +- Used for testing and recovery +- Cost: Negligible + +### 6. Smart Workflow Triggers + +**Path-based triggers:** +```yaml +push: + paths: + - '.github/windows/manifest.json' + - '.github/workflows/windows-dependencies.yml' +``` + +Only rebuild Windows dependencies when: +- Manifest versions change +- Workflow itself is updated +- Manual trigger or schedule + +**Branch-based triggers:** +- AI review only on PRs to master, feature/**, dev/** +- Sync only affects master branch + +## Cost Breakdown + +| Component | Monthly Cost | Notes | +|-----------|-------------|-------| +| GitHub Actions - Sync | $0 | ~150 min/month (free: 2,000 min) | +| GitHub Actions - AI Review | $0 | ~200 min/month (free: 2,000 min) | +| GitHub Actions - Windows | ~$5-8 | ~2,500 min/month with optimizations | +| Claude API (Bedrock) | $30-45 | Usage-based, ~15-20 PRs/month | +| **Total** | **~$35-53/month** | | + +**Before optimizations:** ~$75-100/month +**After optimizations:** ~$35-53/month +**Savings:** ~$40-47/month (40-47% reduction) + +## Monitoring Costs + +### GitHub Actions Usage + +Check usage in repository settings: +``` +Settings ā†’ Billing and plans ā†’ View usage +``` + +Or via CLI: +```bash +gh api repos/:owner/:repo/actions/billing/workflows --jq '.workflows' +``` + +### AWS Bedrock Usage + +Monitor Claude API costs in AWS Console: +``` +AWS Console ā†’ Bedrock ā†’ Usage ā†’ Invocation metrics +``` + +Or via cost logs in artifacts: +``` +.github/scripts/ai-review/cost-log-*.json +``` + +### Setting Alerts + +**GitHub Actions:** +- No built-in alerts +- Monitor via monthly email summaries +- Consider third-party monitoring (e.g., AWS Lambda + GitHub API) + +**AWS Bedrock:** +- Set CloudWatch billing alarms +- Recommended thresholds: + - Warning: $30/month + - Critical: $50/month +- Hard cap in code: $200/month (see `config.json`) + +## Future Optimizations + +### Potential Improvements + +1. **Conditional Testing on PRs** + - Only run full Cirrus CI suite if C code or SQL changes + - Skip for docs-only PRs + - Estimated savings: ~5-10% of testing costs + +2. **Incremental AI Review** + - On PR updates, only review changed files + - Current: Reviews entire PR on each update + - Estimated savings: ~20-30% of AI costs + +3. **Dependency Build Sampling** + - Build only changed dependencies instead of all + - Requires more sophisticated manifest diffing + - Estimated savings: ~30-40% of Windows build costs + +4. **Self-hosted Runners** + - Run Linux builds on own infrastructure + - Keep Windows runners on GitHub (licensing) + - Estimated savings: ~$10-15/month + - **Trade-off:** Maintenance overhead + +### Not Recommended + +1. **Reduce sync frequency** (hourly ā†’ daily) + - Savings: Negligible (~$0.50/month) + - Cost: Increased lag with upstream (unacceptable) + +2. **Skip Windows builds entirely** + - Savings: ~$8/month + - Cost: Lose reproducible dependency builds (defeats purpose) + +3. **Reduce AI review quality** (Claude Sonnet ā†’ Haiku) + - Savings: ~$20-25/month + - Cost: Significantly worse code review quality + +## Pristine Commit Policy + +The following commits are considered "pristine" and skip expensive builds: + +1. **Dev setup commits:** + - Message starts with "dev setup" (case-insensitive) + - Examples: "dev setup v19", "Dev Setup: Update IDE config" + - Contains: .clang-format, .idea/, .vscode/, flake.nix, etc. + +2. **CI/CD configuration commits:** + - Only modify files under `.github/` + - Examples: Workflow changes, script updates, documentation + +**Why this works:** +- Dev setup commits don't affect PostgreSQL code +- CI/CD commits are tested by running the workflows themselves +- Reduces unnecessary Windows builds by ~60-70% + +**Implementation:** See `pristine-master-policy.md` for details. + +## Questions? + +For more information: +- Pristine master policy: `.github/docs/pristine-master-policy.md` +- Sync setup: `.github/docs/sync-setup.md` +- AI review guide: `.github/docs/ai-review-guide.md` +- Windows builds: `.github/docs/windows-builds.md` diff --git a/.github/ocr/context.md b/.github/ocr/context.md new file mode 100644 index 0000000000000..c4a83b85b124e --- /dev/null +++ b/.github/ocr/context.md @@ -0,0 +1,126 @@ +# OCR review context ā€” PostgreSQL contribution standards + +You are reviewing a change to a **PostgreSQL** fork. Every PR here is destined to +become a patch posted to the **pgsql-hackers** mailing list and tracked in a +**commitfest**. Review with the combined rigor, taste, and attention to detail of +the PostgreSQL committers. This context applies to the *whole* change, on top of +the per-file rules. + +## Review discipline +- Be precise and blunt; lead with the most serious problem. No praise, no + validation of the author, no disclaimers ā€” accuracy is the only metric. +- Verify every claim against the actual diff. Confirm names, signatures, line + numbers, and APIs before asserting. Never invent behavior or cite code not in + the change. If unsure, say so, and tag each finding **high / moderate / low** + confidence. +- Judge the change on its merits regardless of how the PR frames it. A draft PR + is WIP: weight design/approach feedback over style nits. + +## Patch hygiene (top rejection reasons on -hackers) +1. **Minimal diff.** The fastest way to get a patch rejected is unrelated + changes: reformatting untouched lines, rewording unrelated comments, touching + code not required by the change. Flag any hunk not needed for the stated + purpose. After the patch, the code should read as if it had always been + written that way. +2. **Atomic, bisectable commits.** Each commit must build and pass tests on its + own ā€” a broken intermediate commit breaks `git bisect`, revert, and + cherry-pick. Flag a commit that only compiles once a later commit lands. + Prefer one focused patch, or a clearly-ordered series of + independently-committable pieces. +3. **Tests + docs are mandatory.** A user-visible change without regression/TAP + tests **and** documentation is WIP, not commit-ready. New behavior needs + tests that cover edge and error paths, not just the happy path. +4. **DRY / reuse.** Prefer existing infrastructure (`List` in `pg_list.h`, + `StringInfo`, `dynahash`/`simplehash`, `palloc`/`MemoryContext`, `foreach`) + over reinventing it. Flag copy-paste and speculative abstraction alike ā€” the + community wants minimal, targeted changes that fit the subsystem's existing + patterns. +5. **Whitespace.** No trailing whitespace; tabs (width 4) for C indentation; + `git diff --check` must be clean. Whitespace-only churn on untouched lines is + a defect. + +## Committer-owned files ā€” do NOT touch in a patch (flag if present) +These are the committer's job at push time; including them causes needless +merge conflicts and is a mistake: +- **`src/include/catalog/catversion.h`** ā€” the `CATALOG_VERSION_NO` bump is done + by the **committer** when pushing. A catversion bump in the PR is **wrong** ā€” + flag it. (This is the single most common author mistake in catalog patches.) +- **Release notes** (`doc/src/sgml/release-*.sgml`) and version strings + (`configure.ac` `AC_INIT` version, `meson.build` `version`, `PG_VERSION`). + +## Generated files ā€” never hand-edit; edit the source +Flag direct edits to generated output; point the author at the source instead: +- Catalog headers `src/include/catalog/*_d.h`, `postgres.bki`, `schemapg.h`, + `system_constraints.sql` ā†’ edit the `pg_*.dat` files. +- `src/backend/nodes/{copy,equal,out,read}funcs.c` and other + `gen_node_support.pl` output ā†’ annotate the `Node` struct in its header. +- `fmgroids.h`, `fmgrprotos.h`, `fmgrtab.c` ā†’ edit `pg_proc.dat`. +- `utils/errcodes.h` ā†’ `errcodes.txt`; wait-event headers ā†’ + `wait_event_names.txt`; `lwlocknames.h` ā†’ `lwlocknames.txt`. +- `configure` ā†’ `configure.ac`; `*.po` translations are handled separately; + generated Unicode tables come from their source scripts. + +## Portability is a hard gate +PostgreSQL runs on Linux, Windows (MSVC), macOS, the BSDs and Solaris, across +**x86_64, ARM64, RISC-V, PPC64, s390x**, both endiannesses and 32/64-bit. Any +change must be portable across all of them: +- No unaligned memory access; no dependence on `char` signedness, integer/pointer + width, endianness, or struct padding for on-disk/wire formats. +- Use `int16/int32/int64`, `Size`, and `INT64_FORMAT`/`UINT64_FORMAT` (never + `%ld` for `int64`). +- Atomics/barriers only via `port/atomics` (`pg_atomic_*`, `pg_read/write_barrier`). +- **Windows/MSVC:** any `extern` variable used from another module or an + extension needs `PGDLLIMPORT` in its header; no VLAs or compiler-specific + extensions beyond the tree's C99 baseline. + +## Backward compatibility ā€” the strongest constraint +Do not break SQL behavior, the libpq wire protocol, the logical-replication +protocol, dump/restore, `pg_upgrade`, or exported/`PGDLLIMPORT` APIs without +extraordinary justification. **ABI** matters for back-branches: changing the +size/layout of an exported struct or the signature of an exported function +breaks installed extensions. + +## Mailing-list context & etiquette +Because each PR becomes a pgsql-hackers email read by a busy, expert, opinionated +audience, also flag what reliably wastes reviewer time or draws rejection: +- A patch that **does more than one thing** or bundles unrelated cleanup ā€” split it. +- **Footguns**: easy-to-misuse APIs, silent data-loss/corruption hazards, unsafe + defaults ā€” name them explicitly. +- **Performance claims without a reproducible benchmark.** +- No reference to the **design discussion / prior -hackers thread** (Message-Id) + for a non-trivial change. +- **Do not bikeshed:** keep style nits proportionate and clearly separated from + substantive correctness findings. + +## Minimalism ā€” the "ponytail" discipline +The best code is the code you never wrote (YAGNI). Before accepting new code, +apply the ladder: (1) Does this need to exist at all? (2) Can existing +code/infrastructure already do it? (3) Is this the simplest thing that works? +Flag: speculative scaffolding and config for a path that isn't wired yet; dead +code and unused "flexibility" (fields, params, abstractions, options with no +caller); premature abstraction (a helper used exactly once); knobs/GUCs/flags +nobody asked for. Minimal, targeted changes that fit the existing patterns beat +clever or general-purpose ones. + +## Comment & identity accuracy +- Comments must describe what the code does **now**. Flag aspirational/ + future-tense comments for behavior that already shipped ("will be", "for now", + "not yet", "future", and stale "TODO/FIXME/XXX/HACK"); comments that drifted + from the code they sit above; and incomplete/trailing comments. Comments + explain **why**, not what. No commented-out code. +- **ASCII only** in source and diffs ā€” no smart quotes, em-dashes, or ellipsis + characters. + +## Commit & versioning discipline +- Conventional-commit style, imperative subject, one logical change per commit, + each commit building on its own. +- Do **not** bump version numbers or generated version stamps (including + `catversion.h`) ā€” that is the maintainer's job at commit/release time. + +Understand common list shorthand so your comments are precise and not +miscommunicated: WIP (work in progress), GUC (config variable), WAL, LSN, OID, +TOAST, FSM, TAM (table access method), RLS, DSM, 2PC, PITR, CIC (concurrent index +creation), SAOP, ABI/API, backpatch (apply to supported back-branches), HEAD +(master tip), catversion (catalog version), pgindent, buildfarm, cfbot, +`s/x/y/` (suggested text substitution), footgun, bikeshedding, POLA (principle of +least astonishment). diff --git a/.github/ocr/litellm.yaml b/.github/ocr/litellm.yaml new file mode 100644 index 0000000000000..e23cc4eee6fe2 --- /dev/null +++ b/.github/ocr/litellm.yaml @@ -0,0 +1,41 @@ +# LiteLLM proxy config ā€” bridges Open Code Review (OpenAI protocol) to AWS Bedrock. +# +# This proxy is NOT a hosted service. The ocr-review.yml workflow installs it +# (`pip install 'litellm[proxy]'`) and runs it as a background process bound to +# 127.0.0.1:4000 for the duration of a single GitHub Actions job, then it exits. +# +# Auth to Bedrock: LiteLLM uses boto3's default credential chain, which reads +# the temporary AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN +# minted by the workflow's OIDC "Configure AWS credentials" step; region from +# AWS_REGION. + +model_list: + - model_name: ocr-bedrock + litellm_params: + # Set the repo variable OCR_BEDROCK_MODEL to an Opus inference-profile id + # your account has access to, e.g.: + # bedrock/converse/us.anthropic.claude-opus-4-8 + # The 'converse/' prefix uses Bedrock's Converse API, which is the most + # reliable path for Claude tool-use (what OCR relies on). + model: os.environ/OCR_BEDROCK_MODEL + aws_region_name: os.environ/AWS_REGION + + # "High effort" review. Claude Opus 4.8 on Bedrock uses *adaptive* thinking + # controlled by output_config.effort. Set it DIRECTLY here ā€” NOT via + # reasoning_effort, which LiteLLM still maps to the legacy + # thinking.type.enabled that Opus 4.8 rejects. LiteLLM forwards + # output_config into additionalModelRequestFields for Anthropic models; if + # the build doesn't recognize the effort param it is dropped with a warning + # (no error) and the model reviews at its default effort. + # Valid: low|medium|high|max|xhigh (auto-clamped to the model ceiling). + output_config: + effort: xhigh + max_tokens: 32000 + +litellm_settings: + drop_params: true # silently drop params a model doesn't support + modify_params: true # auto-fix minor request incompatibilities + request_timeout: 600 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/.github/ocr/pg-history.py b/.github/ocr/pg-history.py new file mode 100644 index 0000000000000..5794f8a920bd7 --- /dev/null +++ b/.github/ocr/pg-history.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +""" +pg-history: tie a PR's changes to PostgreSQL git + pgsql-hackers email history. + +OCR (the code reviewer) cannot call MCP servers, so this is a separate agent: +it runs a Bedrock (Claude Opus) tool-use loop wired to the Agora MCP server at +https://pg.ddx.io/mcp, lets the model search the mailing-list archives / commit +history / commitfest data, and emits a Markdown summary linking the changes to +the relevant threads (https://pg.ddx.io/m/pgsql-hackers/). + +Env: + PG_HISTORY_MCP_URL MCP endpoint (default https://pg.ddx.io/mcp) + PG_HISTORY_MODEL Bedrock model id (e.g. us.anthropic.claude-opus-4-8) + AWS_REGION region (creds come from the OIDC step's env) + BASE_REF, HEAD_SHA PR base ref and head sha (for the git diff context) + GH_PR_TITLE PR title (optional, adds context) + PG_HISTORY_OUT output markdown path (default /tmp/pg-history.md) +Writes the markdown to PG_HISTORY_OUT; exits 0 even on soft failures (writes a note). +""" +import json, os, subprocess, sys, urllib.request + +MCP_URL = os.environ.get("PG_HISTORY_MCP_URL", "https://pg.ddx.io/mcp") +MODEL = os.environ.get("PG_HISTORY_MODEL", "us.anthropic.claude-opus-4-8").replace("bedrock/converse/", "").replace("bedrock/", "") +REGION = os.environ.get("AWS_REGION", "us-east-1") +BASE_REF = os.environ.get("BASE_REF", "") +HEAD_SHA = os.environ.get("HEAD_SHA", "") +PR_TITLE = os.environ.get("GH_PR_TITLE", "") +OUT = os.environ.get("PG_HISTORY_OUT", "/tmp/pg-history.md") +UA = "pg-history/0.1 (+github-actions)" + +# Curated subset of the 108 Agora tools ā€” the ones useful for connecting a +# change to its discussion/commit history. Intersected with what the server +# actually exposes, so unknown names are harmless. +TOOL_WHITELIST = { + "find_related_discussions", "find_similar_messages", "get_thread", + "discussion_links", "get_author_messages", "browse_by_date", + "blame_symbol", "check_upstream_status", "find_related", + "find_entries_for_thread", "find_entries_for_author", "get_commit", + "search", "hybrid_search", "get_callers", "get_callees", "find_pattern", +} +MAX_ROUNDS = 14 +TOOL_RESULT_CAP = 8000 # chars per tool result fed back to the model + + +def _mcp_post(body, sid=None): + headers = {"Content-Type": "application/json", + "Accept": "application/json, text/event-stream", "User-Agent": UA} + if sid: + headers["Mcp-Session-Id"] = sid + req = urllib.request.Request(MCP_URL, data=json.dumps(body).encode(), headers=headers, method="POST") + resp = urllib.request.urlopen(req, timeout=60) + sid_out = resp.headers.get("Mcp-Session-Id") + result = None + for line in resp.read().decode().splitlines(): + line = line.strip() + if line.startswith("data:"): + line = line[5:].strip() + if not line or line.startswith("event:"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + result = obj + return result, sid_out + + +class MCP: + def __init__(self): + init, self.sid = _mcp_post({"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "pg-history", "version": "0.1"}}}) + if not init or "result" not in init: + raise RuntimeError(f"MCP initialize failed: {init}") + try: + _mcp_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, self.sid) + except Exception: + pass + self._id = 1 + + def list_tools(self): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/list", "params": {}}, self.sid) + return (res or {}).get("result", {}).get("tools", []) + + def call(self, name, args): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/call", + "params": {"name": name, "arguments": args or {}}}, self.sid) + if not res: + return "(no response)" + if "error" in res: + return f"ERROR: {json.dumps(res['error'])[:500]}" + parts = [] + for c in res.get("result", {}).get("content", []): + if c.get("type") == "text": + parts.append(c["text"]) + return ("\n".join(parts) or "(empty)")[:TOOL_RESULT_CAP] + + +def git(*args): + try: + return subprocess.check_output(["git", *args], text=True, stderr=subprocess.DEVNULL).strip() + except Exception: + return "" + + +def pr_context(): + base = f"origin/{BASE_REF}" if BASE_REF else "" + rng = f"{base}..{HEAD_SHA}" if base and HEAD_SHA else HEAD_SHA + commits = git("log", "--no-merges", "--format=%h %s", f"{rng}") if rng else "" + stat = git("diff", "--stat", rng) if rng else "" + files = git("diff", "--name-only", rng) if rng else "" + return commits[:4000], stat[:3000], files[:2000] + + +SYSTEM = """You are a PostgreSQL community research assistant. Given a pull request's +commits and changed files, use the available tools (backed by the Agora index of +pgsql-hackers mail, commit history, and commitfest data) to connect the change to +its history. Your goal: + +- Find the mailing-list thread(s) and prior discussion behind this change. +- Identify related/superseded prior commits and any commitfest entry. +- Note relevant prior art, rejected approaches, or design rationale. + +Rules (voice & rigor): +- Be precise and blunt. No praise, no filler, no hedging, no disclaimers. Accuracy is + the only success metric ā€” not the author's approval. Lead with the most important finding. +- NEVER hallucinate. Verify every Message-ID, thread subject, commit hash, author name, + and date against an actual tool result before citing it. If a search returns nothing, + say so plainly ā€” do not guess or fabricate a plausible-looking link. +- Assess the change on its merits, independent of how the PR frames it. +- Tag any inferred (not tool-confirmed) linkage with an explicit confidence level: + high / moderate / low. +- Be decisive and efficient: a handful of targeted tool calls, not exhaustive search. +- Cite every mailing-list message as a Markdown link: [subject](https://pg.ddx.io/m/pgsql-hackers/MESSAGE_ID). +- If you find nothing relevant, say so in one line ā€” do not pad. + +When done, output ONLY Markdown (no preamble) with these sections, omitting any that are empty: +## šŸ§µ Related discussion +## šŸ”— Related commits / prior art +## šŸ“‹ Commitfest +## šŸ§­ Context for reviewers +Keep it tight (use bullets; link generously).""" + + +def to_toolspec(t): + schema = t.get("inputSchema") or {"type": "object", "properties": {}} + return {"toolSpec": {"name": t["name"], + "description": (t.get("description") or "")[:600], + "inputSchema": {"json": schema}}} + + +def main(): + commits, stat, files = pr_context() + if not commits and not files: + open(OUT, "w").write("") # nothing to do + print("No PR diff context; skipping.") + return + user = (f"PR title: {PR_TITLE}\n\n" if PR_TITLE else "") + \ + f"Commits:\n{commits or '(none)'}\n\nChanged files:\n{files or '(none)'}\n\nDiffstat:\n{stat or '(none)'}\n" + + try: + mcp = MCP() + tools = [to_toolspec(t) for t in mcp.list_tools() if t.get("name") in TOOL_WHITELIST] + except Exception as e: + open(OUT, "w").write(f"_pg-history: could not reach the Agora MCP server ({MCP_URL}): {e}_\n") + print(f"MCP unavailable: {e}") + return + if not tools: + open(OUT, "w").write("_pg-history: no usable MCP tools available._\n") + return + + import boto3 + from botocore.config import Config + + # botocore's default read timeout (60s) is too short for a multi-round + # (MAX_ROUNDS) tool-use loop against a large PR diff on a reasoning model; + # each converse() call alone can take several minutes. Bump it well past + # what a single round needs; connect_timeout stays short since a stuck + # TCP handshake is a different (and much cheaper to detect) failure mode. + brt = boto3.client("bedrock-runtime", region_name=REGION, + config=Config(read_timeout=900, connect_timeout=10)) + messages = [{"role": "user", "content": [{"text": user}]}] + final_text = "" + try: + for _ in range(MAX_ROUNDS): + resp = brt.converse( + modelId=MODEL, + system=[{"text": SYSTEM}], + messages=messages, + toolConfig={"tools": tools}, + inferenceConfig={"maxTokens": 4096}, + ) + out = resp["output"]["message"] + messages.append(out) + if resp.get("stopReason") == "tool_use": + results = [] + for blk in out["content"]: + tu = blk.get("toolUse") + if not tu: + continue + res_text = mcp.call(tu["name"], tu.get("input") or {}) + results.append({"toolResult": {"toolUseId": tu["toolUseId"], + "content": [{"text": res_text}]}}) + messages.append({"role": "user", "content": results}) + continue + final_text = "".join(b.get("text", "") for b in out["content"]).strip() + break + except Exception as e: + open(OUT, "w").write(f"_pg-history: Bedrock call failed: {e}_\n") + print(f"Bedrock error: {e}") + return + + if not final_text: + final_text = "_pg-history: no related history found._" + body = "## šŸ“œ Change history & discussion (Agora / pg.ddx.io)\n\n" + final_text + \ + "\n\nGenerated by pg-history via the Agora MCP server (pg.ddx.io).\n" + open(OUT, "w").write(body) + print(body) + + +if __name__ == "__main__": + main() diff --git a/.github/ocr/rule.json b/.github/ocr/rule.json new file mode 100644 index 0000000000000..60e13e73dcbe0 --- /dev/null +++ b/.github/ocr/rule.json @@ -0,0 +1,65 @@ +{ + "_comment": "OCR per-file review rules for PostgreSQL core + extensions. Cross-cutting contribution standards & mailing-list etiquette live in .github/ocr/context.md, passed via --background-file. OCR uses FIRST-MATCH-WINS in declaration order, so rules are ordered most-specific first. merge_system_rule:true keeps OCR's built-in fine-tuned checks (thread-safety, injection, NPE) alongside these PostgreSQL-specific rules.", + "rules": [ + { + "path": "src/test/**", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL tests. Coverage is mandatory for any behavioral change and must include edge cases (NULL, empty, boundary/overflow) and ERROR paths, not just the happy path. A test that still passes with the feature reverted is worthless ā€” confirm it actually exercises and would catch regressions in the new code. Regression (.sql/expected): deterministic, portable output ā€” ORDER BY where row order matters, no timing/plan-dependent output except intentional EXPLAIN, no absolute paths, locale-independent (C collation or explicit COLLATE), DROP objects the test creates; expected/ output must stay stable across platforms and under the parallel schedule. Concurrency/locking belongs in isolation tests (src/test/isolation, .spec + permutations). End-to-end/crash/replication/CLI behavior belongs in TAP tests (t/*.pl with PostgreSQL::Test::Cluster/Utils) ā€” no hardcoded ports/paths, no sleep as synchronization (use poll_query_until/wait_for), skip cleanly when prerequisites are missing, and clean up nodes." + }, + { + "path": "**/*.{c,h}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL backend/frontend C ā€” review as pgsql-hackers committers do, in priority order.\n\n(1) CORRECTNESS (highest): Memory ā€” every palloc lives in the right MemoryContext; error paths via ereport/elog(ERROR) must not leak memory/buffers/locks/fds (rely on MemoryContext/ResourceOwner reset or PG_TRY/PG_FINALLY); no use-after-free; delete temp contexts. Concurrency ā€” consistent lock ordering (deadlock-free), correct lock levels, balanced LWLockAcquire/Release and START_/END_CRIT_SECTION, no TOCTOU, CHECK_FOR_INTERRUPTS in long loops, async-signal-safe signal handlers (volatile sig_atomic_t). WAL ā€” any change to shared on-disk state must be WAL-logged AND correctly replayed (redo path), crash- and replica-consistent. NULL/edge/overflow handling.\n\n(2) BACKWARD COMPATIBILITY / ABI: don't break behavior, dump/restore, pg_upgrade, libpq wire protocol, logical-replication protocol, or exported/PGDLLIMPORT'd APIs (struct size/layout, function signatures) without extraordinary justification.\n\n(3) CATALOG / GENERATED: new/changed catalog data goes in pg_*.dat, NOT the generated *_d.h/.bki. New Node types: ANNOTATE the struct in its header so gen_node_support.pl regenerates copy/equal/out/read ā€” do NOT hand-edit *funcs.c. New SQL-callable functions: add to pg_proc.dat with an OID from the 8000-9999 developer range (src/include/catalog/unused_oids; check duplicate_oids); committer renumbers at commit. DO NOT bump CATALOG_VERSION_NO in the patch ā€” flag any catversion.h change as a mistake (committer's job).\n\n(4) PERFORMANCE: no regression on hot paths; avoid O(n^2) where better is feasible; minimize work under contended locks; avoid needless palloc churn and large struct copies in hot paths.\n\n(5) SECURITY: bounded string ops (snprintf/strlcpy/strlcat ā€” never strcpy/strcat/sprintf); integer/size-overflow checks before allocation; never user input as a format string; privilege checks via pg_*_aclcheck; beware search_path and SECURITY DEFINER.\n\n(6) PORTABILITY (hard gate): no unaligned access; no dependence on char signedness, int/long/pointer width, endianness, or struct padding for on-disk/wire formats; use int16/int32/int64 + INT64_FORMAT/UINT64_FORMAT (never %ld for int64); align contended shared structs (pg_attribute_aligned/cache-line pad). Atomics/barriers only via port/atomics (pg_atomic_*, pg_read/write_barrier) ā€” never raw intrinsics or volatile-as-barrier. WINDOWS/MSVC: extern vars used cross-module/extension need PGDLLIMPORT; no VLAs or features beyond the C99 baseline the tree targets; use pg_pread/pg_pwrite. Applies across x86_64/ARM64/RISC-V/PPC64/s390x, big/little endian, 32/64-bit.\n\n(7) CONVENTIONS: errmsg starts lowercase, no trailing period, no embedded newlines; errdetail/errhint are complete capitalized sentences; correct ERRCODE_*; wrap user-facing text in _(); errmsg_plural for counts. Assert() only for can't-happen invariants (never user-reachable). Naming: snake_case with subsystem prefix (heap_insert) or CamelCase for major subsystems (ExecInitNode); ALL_CAPS macros. Must pgindent cleanly (tabs, width 4). Comments explain WHY not WHAT; no #ifdef 0 blocks, no commented-out code, no #ifdef fencing your feature. Reuse existing helpers (DRY)." + }, + { + "path": "**/*.dat", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL catalog data (pg_proc.dat, pg_type.dat, etc.) ā€” the SOURCE for generated headers. The generated *_d.h, postgres.bki, fmgroids.h, fmgrtab.c must NOT be hand-edited (they regenerate from these files). OIDs: use a value from the developer range 8000-9999 (src/include/catalog/unused_oids; verify with duplicate_oids); committer renumbers to a final contiguous block, so stay in-range and unique but don't over-optimize the exact number. Keep proc entries complete/consistent (prosrc, provolatile, proparallel, prorettype/proargtypes, matching description). DO NOT bump CATALOG_VERSION_NO / catversion.h ā€” committer's job at push time; flag any such change. New catalog columns/views need documentation in doc/src/sgml/catalogs.sgml." + }, + { + "path": "**/*.{sql,pgsql}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL SQL. Valid PostgreSQL dialect (not MySQL/Oracle); correct types (bigint vs int, text vs varchar); sound transaction/isolation and CTE-materialization assumptions. SECURITY: flag SQL injection in dynamic SQL (require quote_identifier/quote_literal or format() with %I/%L), SECURITY DEFINER without a locked-down search_path, inappropriate RLS bypass. Prefer set-based over row-at-a-time/N+1. BACKWARD COMPATIBILITY (a top rejection reason): changing existing SQL behavior, the output of existing functions, or default GUCs needs extraordinary justification. New SQL-callable objects belong in pg_*.dat with OIDs from the 8000-9999 range, not in generated files. Minimal diff; add regression tests + docs." + }, + { + "path": "**/*.{pl,pm}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Perl (TAP tests and build/catalog tooling). Require 'use strict; use warnings;'. Must be perltidy-clean with the tree's src/tools/pgindent/perltidyrc and pass src/tools/perlcheck/pgperlcritic. Use the framework: PostgreSQL::Test::Cluster, PostgreSQL::Test::Utils, Test::More; no hardcoded ports/paths/PIDs; use safe_psql/poll_query_until, not sleep; skippable without optional prerequisites; clean up nodes. PORTABILITY: run on Windows (no fork-only constructs, use File::Spec, avoid unavailable signals) and the minimum supported Perl. Robustness: avoid two-arg open and string system()/qx with interpolated data (use list forms). Generator scripts (gen_node_support.pl, catalog Perl) must be deterministic and stay in sync with inputs; do not commit their generated output." + }, + { + "path": "**/*.py", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Python (build/test tooling, oauth/pytest tests, src/tools). Follow surrounding style; keep imports to the standard library unless the dependency is already required by the tree (no surprise third-party deps in build/test tooling). PORTABILITY: support the project's minimum Python 3 and run on Windows and the BSDs (use os.path/pathlib, avoid POSIX-only calls and shell=True with interpolated input). Deterministic, self-cleaning tests; no hardcoded ports/paths; skip cleanly without prerequisites. For the Perl->pytest porting effort, confirm behavior parity with the TAP test replaced (same assertions/coverage), not a superficial translation. Minimal diff; match the tree's ruff/black config if present." + }, + { + "path": "**/*.{rs,toml}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Rust PostgreSQL extension (pgrx) or Rust support crate. Not core C, but it runs inside/alongside the backend, so backend safety applies. SAFETY: in code reachable from an SQL call, a Rust panic aborts the Postgres process ā€” forbid unwrap()/expect()/panic!/unreachable!/todo! and index-panics on reachable paths; use Result and pgrx error reporting (error!/ereport!). Every `unsafe` block needs a comment justifying its invariant; scrutinize raw pointers and FFI across the pg_sys boundary. pgrx: honor #[pg_guard] on extern C fns (correct panic/longjmp handling); never hold Rust references across SPI or anything that can longjmp (skips Rust destructors -> leaks); respect MemoryContext lifetimes for palloc'd data; datum<->Rust conversions must handle NULL. Concurrency uses Postgres shmem/LWLocks (pgrx shmem API), not std::sync alone. Lints: must pass `cargo clippy --all-targets --all-features -- -D warnings` and `cargo fmt --check`; deny unwrap_used/expect_used/panic in libraries; thiserror (libs) / anyhow (bins). Justify every new dependency. Tests: #[pg_test] for in-backend behavior, #[test] for pure logic; cover error and NULL paths. Minimal, idiomatic diff." + }, + { + "path": "**/{configure.ac,*.m4,aclocal.m4}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Autoconf. Edit configure.ac / the m4 macros ā€” do NOT hand-edit generated 'configure' or pg_config.h.in in the same patch (regeneration is the committer's step; a patch that also rewrites generated configure output is suspect). Feature/header/function probes must be portable and not assume a specific OS/compiler. Every configure knob must be mirrored on the Meson side (meson_options.txt/meson.build) and documented. Minimal diff." + }, + { + "path": "**/{meson.build,meson_options.txt}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Meson build. Valid syntax; correct subdir()/dependency()/declare_dependency and install paths; new source files must be added here. CRITICAL: PostgreSQL maintains BOTH Meson and Autoconf/Make ā€” any new file, option, or feature check must be mirrored on the configure.ac/Makefile side so the two never drift (a file built by only one system is a common defect). New options need matching docs and sensible defaults. Minimal diff." + }, + { + "path": "**/{Makefile,GNUmakefile,*.mk}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Makefile (GNU Make). $(VAR) refs; correct .PHONY; accurate dependencies (no parallel -j races); $(MAKE) for recursion; VPATH/out-of-tree build support; no hardcoded paths (use standard PostgreSQL makefile vars and $(top_builddir)); clean/distclean/maintainer-clean must remove new artifacts; extensions use PGXS. Must stay in sync with meson.build. Minimal diff." + }, + { + "path": "doc/**/*.sgml", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL documentation (DocBook SGML). Technically accurate/complete (parameters, limitations, version/compat notes); correct tag usage/nesting (, , , , , /); working cross-references; spell it 'PostgreSQL' in prose; SQL keywords uppercase in examples. Coverage: a new GUC -> config.sgml (and postgresql.conf.sample); new/changed catalogs or views -> catalogs.sgml; new SQL syntax -> the matching ref/*.sgml; new functions -> func.sgml. Do NOT edit release-notes (release-*.sgml) ā€” written by the release team/committers; flag such edits. New user-facing behavior in this PR should ship with matching docs." + }, + { + "path": "**/*.md", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Markdown docs. Clear heading hierarchy; fenced code blocks with language hints; accurate instructions/prerequisites; consistent PostgreSQL terminology; no broken relative links or stale claims. Minimal diff." + } + ] +} diff --git a/.github/workflows/ocr-model-check.yml b/.github/workflows/ocr-model-check.yml new file mode 100644 index 0000000000000..10d250528cf7c --- /dev/null +++ b/.github/workflows/ocr-model-check.yml @@ -0,0 +1,89 @@ +# Checks AWS Bedrock weekly for a newer Claude Opus inference profile than the +# one OCR currently uses (vars.OCR_BEDROCK_MODEL) and, if found, opens/updates a +# single GitHub issue telling the maintainer to bump the variable. It does NOT +# change the model automatically: GITHUB_TOKEN cannot write Actions *variables* +# (that needs a PAT with admin), so this is a notify-only mechanism by design. +name: OCR model self-check + +on: + schedule: + - cron: '0 12 * * 1' # Mondays 12:00 UTC + workflow_dispatch: + +permissions: + id-token: write + contents: read + issues: write + +jobs: + check-model: + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-model-check-${{ github.run_id }} + + - name: Find newest Opus vs configured + id: check + env: + CURRENT: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import os, re, subprocess, json + region = os.environ.get("AWS_REGION", "us-east-1") + current = os.environ.get("CURRENT", "") + out = subprocess.run( + ["aws", "bedrock", "list-inference-profiles", "--region", region, + "--query", "inferenceProfileSummaries[].inferenceProfileId", "--output", "json"], + capture_output=True, text=True) + ids = json.loads(out.stdout or "[]") + # Parse claude-opus-- from any profile id (prefix us./global. ok). + def ver(s): + m = re.search(r"claude-opus-(\d+)-(\d+)", s) + return (int(m.group(1)), int(m.group(2))) if m else None + opus = [(ver(i), i) for i in ids if ver(i) and i.startswith(("us.", "global."))] + if not opus: + print("newer=false"); raise SystemExit(0) + best_ver, best_id = max(opus, key=lambda x: x[0]) + cur = ver(current) + newer = (cur is None) or (best_ver > cur) + print(f"newer={'true' if newer else 'false'}") + print(f"best_id={best_id}") + print(f"best_ver={best_ver[0]}.{best_ver[1]}") + print(f"cur_ver={'unknown' if cur is None else f'{cur[0]}.{cur[1]}'}") + PY + + - name: Open/update issue if a newer model exists + if: steps.check.outputs.newer == 'true' + uses: actions/github-script@v9 + with: + script: | + const best = '${{ steps.check.outputs.best_id }}'; + const bestVer = '${{ steps.check.outputs.best_ver }}'; + const curVer = '${{ steps.check.outputs.cur_ver }}'; + const marker = ''; + const title = `OCR: newer Claude Opus available (${bestVer} > ${curVer})`; + const body = `${marker}\n` + + `A newer Claude Opus inference profile is available on Bedrock.\n\n` + + `- **Configured** (\`vars.OCR_BEDROCK_MODEL\`): Opus ${curVer}\n` + + `- **Newest on Bedrock**: \`${best}\` (Opus ${bestVer})\n\n` + + `To upgrade, set the repo variable:\n\n` + + '```\n' + + `gh variable set OCR_BEDROCK_MODEL -R ${context.repo.owner}/${context.repo.repo} \\\n` + + ` -b "bedrock/converse/${best}"\n` + + '```\n\n' + + `Also confirm the \`ocr-bedrock-ci\` IAM inline policy allows invoking the new model ` + + `(the resource is scoped to \`anthropic.claude-opus-*\`), then re-run OCR.\n\n` + + `_Automated by \`.github/workflows/ocr-model-check.yml\`; this issue is upserted._`; + const q = `repo:${context.repo.owner}/${context.repo.repo} in:body "${marker}" state:open`; + const found = await github.rest.search.issuesAndPullRequests({ q, per_page: 1 }); + if (found.data.total_count > 0) { + const n = found.data.items[0].number; + await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, issue_number: n, title, body }); + } else { + await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title, body }); + } diff --git a/.github/workflows/ocr-review.yml b/.github/workflows/ocr-review.yml new file mode 100644 index 0000000000000..0828af429b57c --- /dev/null +++ b/.github/workflows/ocr-review.yml @@ -0,0 +1,427 @@ +# Open Code Review (OCR) ā€” AI PR review backed by AWS Bedrock via a LiteLLM proxy. +# +# Flow: +# PR opened/updated (incl. DRAFTS) ā”€ā” +# /open-code-review PR comment ā”€ā”¼ā”€ā–ŗ start LiteLLM (127.0.0.1:4000 ā†’ Bedrock) +# manual workflow_dispatch ā”€ā”˜ ā””ā–ŗ ocr review --format json +# ā””ā–ŗ post inline PR review comments +# +# Required (repo settings ā€” all repo *variables*, no secrets; auth is via GitHub OIDC): +# vars.AWS_ROLE_ARN - IAM role to assume via OIDC (granting bedrock:InvokeModel*) +# vars.AWS_REGION - e.g. us-east-1 +# vars.OCR_BEDROCK_MODEL - LiteLLM model string for the Opus inference profile, e.g. +# bedrock/converse/us.anthropic.claude-opus-4-8 +# +# No static AWS keys are stored. GITHUB_TOKEN (auto) posts the review comments. + +name: OCR AI Review + +on: + pull_request: + # Note: no draft filter ā€” drafts are reviewed too. + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + +# One review per PR; cancel superseded runs to save Bedrock spend. +concurrency: + group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + cancel-in-progress: true + +permissions: + id-token: write # required to mint the GitHub OIDC token for AWS role assumption + contents: read + pull-requests: write + +jobs: + ocr-review: + runs-on: ubuntu-latest + # PR events always; comment events only when the comment is on a PR and + # starts with the trigger keyword; manual dispatch always. + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review'))) + + env: + # LiteLLM listens on localhost only; this key never leaves the runner. + LITELLM_MASTER_KEY: sk-ocr-ci-local + OCR_BEDROCK_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + # Region is a static var (safe at job level). AWS credentials are NOT set + # here ā€” they're minted by the OIDC "Configure AWS credentials" step below + # and exported to the environment for the LiteLLM/boto3 Bedrock calls. + AWS_REGION: ${{ vars.AWS_REGION }} + + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') { + prNumber = context.payload.pull_request.number; + } else if (context.eventName === 'issue_comment') { + prNumber = context.issue.number; + } else { + prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + } + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + const { data: repo } = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_ref', pr.head.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('default_branch', repo.default_branch); + core.setOutput('cross_repo', String(pr.head.repo.full_name !== pr.base.repo.full_name)); + + # NOTE: do NOT checkout the PR head. OCR reads the diff and file contents + # straight from git refs (git diff , git show :path, + # git grep ), so the working tree is irrelevant ā€” but our OCR config + # lives on the default branch, not on the PR branch. We check out the repo + # (default ref), fetch the base/head objects, and materialize the config + # from origin/. + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Prepare git refs and OCR config + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_REF: ${{ steps.pr.outputs.head_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + DEFAULT_BRANCH: ${{ steps.pr.outputs.default_branch }} + run: | + git fetch --no-tags origin "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || true + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "+refs/heads/${HEAD_REF}:refs/remotes/origin/${HEAD_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + # OCR config lives on the default branch; materialize it independently + # of whatever ref is checked out. + mkdir -p "$RUNNER_TEMP/ocr" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/litellm.yaml" > "$RUNNER_TEMP/ocr/litellm.yaml" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/rule.json" > "$RUNNER_TEMP/ocr/rule.json" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/context.md" > "$RUNNER_TEMP/ocr/context.md" + echo "Config materialized:"; ls -l "$RUNNER_TEMP/ocr" + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Install LiteLLM proxy + Open Code Review + run: | + python -m pip install --upgrade pip + # Pin LiteLLM to a main commit that supports Claude Opus 4.8 adaptive + # thinking (maps reasoning_effort -> output_config.effort, incl. xhigh). + # Not in any tagged release yet (PyPI latest 1.87.1 lacks the Opus + # normalizer). Bump this SHA once a release ships the feature. + pip install "litellm[proxy] @ git+https://github.com/BerriAI/litellm.git@5be0797d24a2f26eb2123e13788f90055a59d91d" + npm install -g @alibaba-group/open-code-review + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-review-${{ github.run_id }} + + - name: Start LiteLLM proxy (Bedrock bridge) + run: | + if [ -z "$OCR_BEDROCK_MODEL" ]; then + echo "::error::vars.OCR_BEDROCK_MODEL is not set (e.g. bedrock/converse/us.anthropic.claude-opus-4-1-20250805-v1:0)" + exit 1 + fi + nohup litellm --config "$RUNNER_TEMP/ocr/litellm.yaml" --host 127.0.0.1 --port 4000 \ + > /tmp/litellm.log 2>&1 & + echo "Waiting for LiteLLM to become ready..." + for i in $(seq 1 60); do + if curl -sf http://127.0.0.1:4000/health/readiness >/dev/null; then + echo "LiteLLM ready."; exit 0 + fi + sleep 2 + done + echo "::error::LiteLLM did not become ready in time"; cat /tmp/litellm.log; exit 1 + + - name: Configure OCR + run: | + ocr config set llm.url http://127.0.0.1:4000/v1/chat/completions + ocr config set llm.auth_token "$LITELLM_MASTER_KEY" + ocr config set llm.model ocr-bedrock + ocr config set llm.use_anthropic false + ocr config set language English + + - name: Run OCR review + run: | + ocr review \ + --from "origin/${{ steps.pr.outputs.base_ref }}" \ + --to "${{ steps.pr.outputs.head_sha }}" \ + --rule "$RUNNER_TEMP/ocr/rule.json" \ + --background-file "$RUNNER_TEMP/ocr/context.md" \ + --concurrency 3 \ + --timeout 20 \ + --format json \ + > /tmp/ocr-result.json 2>/tmp/ocr-stderr.log || true + echo "----- OCR stdout -----"; cat /tmp/ocr-result.json || true + echo "----- OCR stderr -----"; cat /tmp/ocr-stderr.log || true + echo "----- LiteLLM log (tail) -----"; tail -n 50 /tmp/litellm.log || true + + - name: Post review to PR + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const commitSha = '${{ steps.pr.outputs.head_sha }}'; + + // Opus at high effort can emit dozens of findings. Posting them all + // one-by-one trips GitHub's SECONDARY rate limit (403 "content + // creation"), which is what made every run fail after the review + // was already generated. We (a) cap inline comments and overflow + // the rest into the summary, (b) prefer a single bulk createReview, + // and (c) throttle + back off with Retry-After on any fallback. + const MAX_INLINE = 25; + const sleep = (ms) => new Promise(r => setTimeout(r, ms)); + + async function withRetry(fn, label) { + for (let attempt = 1; attempt <= 5; attempt++) { + try { return await fn(); } + catch (e) { + const status = e.status || (e.response && e.response.status); + const h = (e.response && e.response.headers) || {}; + const isRate = status === 403 || status === 429; + if (!isRate || attempt === 5) throw e; + let waitMs = 0; + if (h['retry-after']) waitMs = parseInt(h['retry-after'], 10) * 1000; + else if (h['x-ratelimit-reset']) waitMs = parseInt(h['x-ratelimit-reset'], 10) * 1000 - Date.now(); + if (!waitMs || Number.isNaN(waitMs) || waitMs < 0) waitMs = 1000 * Math.pow(2, attempt); + waitMs = Math.min(waitMs, 60000) + 500; + core.warning(`${label}: rate-limited (status ${status}); waiting ${Math.round(waitMs / 1000)}s (attempt ${attempt}/5)`); + await sleep(waitMs); + } + } + } + + let result; + try { + result = JSON.parse(fs.readFileSync('/tmp/ocr-result.json', 'utf8')); + } catch (e) { + const stderr = (() => { try { return fs.readFileSync('/tmp/ocr-stderr.log', 'utf8').trim(); } catch { return ''; } })(); + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `āš ļø **OCR** could not produce a review.\n\n\`\`\`\n${(stderr || e.message).slice(0, 8000)}\n\`\`\``, + }), 'error-comment'); + return; + } + + const comments = result.comments || []; + const warnings = result.warnings || []; + + const formatComment = (c) => { + let body = c.content || ''; + if (c.suggestion_code && c.existing_code) { + body += '\n\n```suggestion\n' + c.suggestion_code + (c.suggestion_code.endsWith('\n') ? '' : '\n') + '```'; + } + return body; + }; + const formatMarkdown = (c) => { + let md = `### šŸ“„ \`${c.path}\``; + if (c.start_line && c.end_line) md += ` (L${c.start_line}-L${c.end_line})`; + md += '\n\n' + (c.content || ''); + if (c.suggestion_code && c.existing_code) { + md += '\n\n
šŸ’” Suggested change\n\n'; + md += '**Before:**\n```\n' + c.existing_code + '\n```\n\n**After:**\n```\n' + c.suggestion_code + '\n```\n\n
'; + } + return md; + }; + + if (comments.length === 0) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `āœ… **OCR**: ${result.message || 'No issues found.'}`, + }), 'no-issues-comment'); + return; + } + + const inlineAll = []; + const noLine = []; + for (const c of comments) { + const body = formatComment(c); + const hasLine = (c.start_line >= 1) || (c.end_line >= 1); + if (!hasLine) { noLine.push(c); continue; } + const rc = { path: c.path, body, side: 'RIGHT' }; + if (c.start_line >= 1 && c.end_line >= 1 && c.start_line !== c.end_line) { + rc.start_line = c.start_line; rc.line = c.end_line; rc.start_side = 'RIGHT'; + } else { + rc.line = c.end_line >= 1 ? c.end_line : c.start_line; + } + inlineAll.push({ rc, c }); + } + + const inline = inlineAll.slice(0, MAX_INLINE).map(x => x.rc); + const overflow = inlineAll.slice(MAX_INLINE).map(x => x.c); + + let summary = `šŸ” **OCR** found **${comments.length}** issue(s).`; + summary += `\n- ${inline.length} inline, ${noLine.length + overflow.length} in summary`; + if (overflow.length) summary += ` (inline capped at ${MAX_INLINE})`; + if (warnings.length) summary += `\n- āš ļø ${warnings.length} warning(s) during review`; + for (const c of noLine.concat(overflow)) summary += '\n\n---\n\n' + formatMarkdown(c); + + // Preferred path: ONE createReview carrying every inline comment. + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', comments: inline, + }), 'bulk-review'); + return; + } catch (e) { + core.warning(`bulk createReview failed (${e.status || '?'}: ${e.message}); falling back to throttled per-comment posting`); + } + + // Fallback: an invalid inline position (line not in the diff -> 422) + // rejects the whole bulk review. Post the summary, then each comment + // individually with a delay + backoff, skipping ones GitHub rejects. + let ok = 0; const failed = []; + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', + }), 'summary-review'); + } catch (err) { failed.push(`summary: ${err.message}`); } + + for (const rc of inline) { + try { + await withRetry(() => github.rest.pulls.createReviewComment({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, path: rc.path, body: rc.body, + ...(rc.start_line ? { start_line: rc.start_line, start_side: rc.start_side } : {}), + line: rc.line, side: rc.side, + }), `comment ${rc.path}:${rc.line}`); + ok++; + } catch (inner) { + failed.push(`\`${rc.path}\` L${rc.line}: ${inner.message}`); + } + await sleep(1200); // stay under the secondary content-creation limit + } + + if (failed.length) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `šŸ“Š OCR posted ${ok}/${inline.length} inline comment(s).\n\n
${failed.length} could not be posted\n\n${failed.join('\n')}\n
`, + }), 'summary-failures'); + } + + # Companion job: OCR can't call MCP, so this separate agent ties the PR's + # changes to PostgreSQL git + pgsql-hackers history via the Agora MCP server + # (pg.ddx.io) and posts a single, upserted "history & discussion" comment. + pg-history: + runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review') || + startsWith(github.event.comment.body, '/pg-history'))) + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') prNumber = context.payload.pull_request.number; + else if (context.eventName === 'issue_comment') prNumber = context.issue.number; + else prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('title', pr.title || ''); + + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Make base/head refs available + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: | + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: pg-history-${{ github.run_id }} + + - name: Install deps + run: pip install boto3 + + - name: Run pg-history (Agora MCP) + env: + PG_HISTORY_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + GH_PR_TITLE: ${{ steps.pr.outputs.title }} + PG_HISTORY_OUT: ${{ runner.temp }}/pg-history.md + run: | + python .github/ocr/pg-history.py || true + echo "----- output -----"; cat "${{ runner.temp }}/pg-history.md" 2>/dev/null || echo "(no output)" + + - name: Upsert PR comment + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const path = process.env.RUNNER_TEMP + '/pg-history.md'; + let body = ''; + try { body = fs.readFileSync(path, 'utf8').trim(); } catch (e) {} + if (!body) { console.log('pg-history: empty output, nothing to post'); return; } + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const marker = ''; + body = marker + '\n' + body; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, per_page: 100 }); + const mine = comments.find(c => c.user.type === 'Bot' && c.body && c.body.includes(marker)); + if (mine) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: mine.id, body }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body }); + } From c918e8541e51448545856d97bc07a1db40c2d1c7 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Wed, 29 Apr 2026 11:40:42 -0400 Subject: [PATCH 3/6] dev setup v36 Nix-based development environment for PostgreSQL hacking. Not for merge; staged here so per-commit build/test runs can share a single toolchain. --- .clangd | 43 ++ .envrc | 23 + .gdbinit | 156 +++++++ .local-gitignore | 21 + flake.lock | 78 ++++ flake.nix | 45 ++ glibc-no-fortify-warning.patch | 24 ++ pg-aliases.sh | 658 +++++++++++++++++++++++++++++ shell.nix | 745 +++++++++++++++++++++++++++++++++ src/test/regress/pg_regress.c | 2 +- src/tools/pgindent/pgindent | 2 +- 11 files changed, 1795 insertions(+), 2 deletions(-) create mode 100644 .clangd create mode 100644 .envrc create mode 100644 .gdbinit create mode 100644 .local-gitignore create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 glibc-no-fortify-warning.patch create mode 100644 pg-aliases.sh create mode 100644 shell.nix diff --git a/.clangd b/.clangd new file mode 100644 index 0000000000000..490220f56a7c0 --- /dev/null +++ b/.clangd @@ -0,0 +1,43 @@ +Diagnostics: + MissingIncludes: None +InlayHints: + Enabled: true + ParameterNames: true + DeducedTypes: true +CompileFlags: + CompilationDatabase: build/ # Search build/ directory for compile_commands.json + Remove: [ -Werror ] + Add: + - -DDEBUG + - -DLOCAL + - -DPGDLLIMPORT= + - -DPIC + - -O2 + - -Wall + - -Wcast-function-type + - -Wconversion + - -Wdeclaration-after-statement + - -Wendif-labels + - -Werror=vla + - -Wextra + - -Wfloat-equal + - -Wformat-security + - -Wimplicit-fallthrough=3 + - -Wmissing-format-attribute + - -Wmissing-prototypes + - -Wno-format-truncation + - -Wno-sign-conversion + - -Wno-stringop-truncation + - -Wno-unused-const-variable + - -Wpointer-arith + - -Wshadow + - -Wshadow=compatible-local + - -fPIC + - -fexcess-precision=standard + - -fno-strict-aliasing + - -fvisibility=hidden + - -fwrapv + - -g + - -std=c11 + - -I. + - -I../../../../src/include diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000000..c05ca9a78f9f5 --- /dev/null +++ b/.envrc @@ -0,0 +1,23 @@ +source_up +watch_file flake.nix +use flake +use project_steering postgresql +use project_mcp postgresq +use aws + +# libumem is automatically loaded via LD_PRELOAD in the shell environment +# All programs in this shell will use libumem instead of glibc malloc +# Environment variables: +# UMEM_DEBUG=audit,default +# UMEM_LOGGING=transaction,contents,fail + +# Configure libumem with AUDIT mode (more aggressive checking) +#export UMEM_DEBUG="audit,contents,guards" # audit=catch more bugs, guards=detect overruns +#export UMEM_LOGGING="transaction,contents,fail" + +#export MESON_EXTRA_SETUP="-Db_coverage=true" +#export GENINFO_OPTIONS="--ignore-errors inconsistent,gcov" +#export LCOV_OPTIONS="--ignore-errors inconsistent,gcov" + +#export CFLAGS="-Wall -Wextra -Wconversion -Wdouble-promotion -Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion -fsanitize-trap --werror" +# -fsanitize=undefined,address,undefined,thread diff --git a/.gdbinit b/.gdbinit new file mode 100644 index 0000000000000..854e5ecbaf69c --- /dev/null +++ b/.gdbinit @@ -0,0 +1,156 @@ +# HOT Indexed Updates ā€” GDB breakpoints for code review +# +# Usage: gdb -x .gdbinit +# Or from gdb: source .gdbinit +# +# These breakpoints cover the major code paths introduced or modified by +# the HOT indexed updates patch series. They are organized by subsystem +# to make it easy to enable/disable groups during debugging. +# +# Tip: to skip to a specific subsystem, disable all then enable selectively: +# disable breakpoints +# enable 1 2 3 # just the update-decision group + +# ========================================================================= +# 1. UPDATE DECISION ā€” heap_update() HOT/HOT-indexed/non-HOT choice +# src/backend/access/heap/heapam.c +# ========================================================================= + +# Main entry: heap_update +break heapam.c:3210 + +# HOT decision block: pure HOT vs HOT indexed vs non-HOT +# Line 4019: pure HOT (no indexed columns changed) +# Line 4024: HOT indexed path (non-catalog, some indexed columns changed) +# Line 4031: predict augmented tuple size +# Line 4033: size+space check before creating augmented tuple +break heapam.c:4019 +break heapam.c:4024 +break heapam.c:4033 + +# Set HEAP_INDEXED_UPDATED flag on new tuple before page insertion +break heapam.c:4101 + +# Restore HEAP_INDEXED_UPDATED on old tuple (only if it previously had it) +break heapam.c:4147 + +# ========================================================================= +# 2. TUPLE CREATION ā€” building the augmented tuple with embedded bitmap +# src/backend/access/heap/heapam.c +# ========================================================================= + +# Predict augmented tuple size (returns 0 if t_hoff would overflow) +break heap_hot_indexed_tuple_size + +# Create augmented tuple with embedded modified-column bitmap +break heap_hot_indexed_create_tuple + +# Serialize Bitmapset into raw bytes in tuple header +break heap_hot_indexed_serialize_bitmap + +# ========================================================================= +# 3. BITMAP UTILITIES ā€” raw bitmap operations for chain following +# src/backend/access/heap/heapam.c +# ========================================================================= + +# Compute raw bitmap byte size from natts +break heap_hot_indexed_bitmap_raw_size + +# Check if tuple header has room for bitmap between null bitmap and data +break heap_hot_indexed_has_bitmap_space + +# Read HOT indexed bitmap from tuple header (returns Bitmapset) +break heap_hot_indexed_read_bitmap + +# Fast overlap check: does tuple's raw bitmap overlap with indexed_attrs? +break heap_hot_indexed_bitmap_overlaps_raw + +# OR a tuple's raw bitmap into an accumulator buffer +break heap_hot_indexed_bitmap_or_raw + +# Check if accumulated raw bitmap overlaps with indexed_attrs +break heap_hot_indexed_accum_overlaps + +# Merge bitmaps from dead tuples into a target tuple on the page +break heap_hot_indexed_merge_bitmaps_raw + +# Deserialize raw bytes back to Bitmapset +break heap_hot_indexed_deserialize_bitmap + +# ========================================================================= +# 4. INDEX SCAN ā€” HOT chain following with stale-entry detection +# src/backend/access/heap/heapam_indexscan.c +# ========================================================================= + +# Main HOT chain search with indexed update awareness +break heap_hot_search_buffer + +# Redirect-with-data: initialize bitmap accumulator from collapsed redirect +break heapam_indexscan.c:182 + +# Accumulate bitmap from INDEXED_UPDATED tuple in chain +break heapam_indexscan.c:250 + +# Stale entry detection: accumulated bitmap overlaps this index's attrs +break heapam_indexscan.c:297 + +# ========================================================================= +# 5. INDEX SCAN SETUP ā€” indexed_attrs bitmap computation +# src/backend/access/index/indexam.c +# ========================================================================= + +# Compute indexed_attrs for HOT indexed update chain following +break indexam.c:299 + +# ========================================================================= +# 6. INDEX INSERTION ā€” skip unchanged indexes for HOT indexed updates +# src/backend/executor/execIndexing.c +# ========================================================================= + +# Entry: insert/update index tuples +break ExecInsertIndexTuples + +# Index skip decision: skip indexes whose attrs don't overlap modified set +break execIndexing.c:370 + +# ========================================================================= +# 7. PRUNING ā€” chain collapsing and redirect-with-data +# src/backend/access/heap/pruneheap.c +# ========================================================================= + +# Main prune function +break heap_page_prune_and_freeze + +# Per-chain pruning entry +break heap_prune_chain + +# Chain collapsing: collect bitmaps from dead INDEXED_UPDATED intermediates +break pruneheap.c:1802 + +# OR dead tuple bitmaps into combined bitmap +break pruneheap.c:1836 + +# Record redirect-with-data for execute phase +break pruneheap.c:1863 + +# Execute phase: apply redirect-with-data entries on the page +break pruneheap.c:1287 + +# ========================================================================= +# 8. WAL REPLAY ā€” recovery of HOT indexed updates +# src/backend/access/heap/heapam_xlog.c +# ========================================================================= + +# WAL replay for XLOG_HEAP2_INDEXED_UPDATE +break heap_xlog_indexed_update + +# ========================================================================= +# 9. WAL LOGGING ā€” writing HOT indexed update records +# src/backend/access/heap/heapam.c +# ========================================================================= + +# WAL logging for heap updates (handles indexed_update flag) +break log_heap_update + +# Serialize redirect-with-data into WAL record (pruneheap.c) +break pruneheap.c:2936 diff --git a/.local-gitignore b/.local-gitignore new file mode 100644 index 0000000000000..572fbdbd5fb22 --- /dev/null +++ b/.local-gitignore @@ -0,0 +1,21 @@ +# Local development ignores (not tracked in .gitignore) +# To enable: git config core.excludesFile .local-gitignore +.local-gitignore +CA+hUKGKFvu3zyvv3aaj5hHs9VtWcjFAmisOwOc7aOZNc5AF3NA@mail.gmail.com +NOTES +build/ +build-valgrind/ +build-asan/ +install/ +install-valgrind/ +install-asan/ +.direnv/ +.cache/ +.history +test-db/ +log/ +results/ +regression.diffs +regression.out +*.core +core.* diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000000..b8e8a1fdb750f --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1767313136, + "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1777270315, + "narHash": "sha256-yKB4G6cKsQsWN7M6rZGk6gkJPDNPIzT05y4qzRyCDlI=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "6368eda62c9775c38ef7f714b2555a741c20c72d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000000..aae6d54c4c8cf --- /dev/null +++ b/flake.nix @@ -0,0 +1,45 @@ +{ + description = "PostgreSQL development environment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; + nixpkgs-unstable.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + nixpkgs, + nixpkgs-unstable, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + pkgs-unstable = import nixpkgs-unstable { + inherit system; + config.allowUnfree = true; + }; + + shellConfig = import ./shell.nix {inherit pkgs pkgs-unstable system;}; + in { + formatter = pkgs.alejandra; + devShells = { + default = shellConfig.devShell; + gcc = shellConfig.devShell; + clang = shellConfig.clangDevShell; + gcc-musl = shellConfig.muslDevShell; + clang-musl = shellConfig.clangMuslDevShell; + }; + + packages = { + inherit (shellConfig) gdbConfig flameGraphScript pgbenchScript; + }; + + environment.localBinInPath = true; + } + ); +} diff --git a/glibc-no-fortify-warning.patch b/glibc-no-fortify-warning.patch new file mode 100644 index 0000000000000..4657a12adbcc5 --- /dev/null +++ b/glibc-no-fortify-warning.patch @@ -0,0 +1,24 @@ +From 130c231020f97e5eb878cc9fdb2bd9b186a5aa04 Mon Sep 17 00:00:00 2001 +From: Greg Burd +Date: Fri, 24 Oct 2025 11:58:24 -0400 +Subject: [PATCH] no warnings with -O0 and fortify source please + +--- + include/features.h | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/include/features.h b/include/features.h +index 673c4036..a02c8a3f 100644 +--- a/include/features.h ++++ b/include/features.h +@@ -432,7 +432,6 @@ + + #if defined _FORTIFY_SOURCE && _FORTIFY_SOURCE > 0 + # if !defined __OPTIMIZE__ || __OPTIMIZE__ <= 0 +-# warning _FORTIFY_SOURCE requires compiling with optimization (-O) + # elif !__GNUC_PREREQ (4, 1) + # warning _FORTIFY_SOURCE requires GCC 4.1 or later + # elif _FORTIFY_SOURCE > 2 && (__glibc_clang_prereq (9, 0) \ +-- +2.50.1 + diff --git a/pg-aliases.sh b/pg-aliases.sh new file mode 100644 index 0000000000000..0c13adc8f903a --- /dev/null +++ b/pg-aliases.sh @@ -0,0 +1,658 @@ +# PostgreSQL Development Aliases + +# ============================================================ +# Build helpers shared by every variant. +# ============================================================ +pg_clean_for_compiler() { + local current_compiler="$(basename $CC)" + local build_dir="${1:-$PG_BUILD_DIR}" + + if [ -f "$build_dir/compile_commands.json" ]; then + local last_compiler=$(grep -o '/[^/]*/bin/[gc]cc\|/[^/]*/bin/clang' "$build_dir/compile_commands.json" | head -1 | xargs basename 2>/dev/null || echo "unknown") + + if [ "$last_compiler" != "$current_compiler" ] && [ "$last_compiler" != "unknown" ]; then + echo "Detected compiler change from $last_compiler to $current_compiler" + echo "Cleaning build directory..." + trash "$build_dir" 2>/dev/null || rm -rf "$build_dir" + mkdir -p "$build_dir" + fi + fi + + mkdir -p "$build_dir" + echo "$current_compiler" >"$build_dir/.compiler_used" +} + +# ============================================================ +# Core PostgreSQL commands (default/debug build) +# ============================================================ +alias pg-setup=' + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: Could not find perl CORE directory" >&2 + return 1 + fi + + pg_clean_for_compiler "$PG_BUILD_DIR" + + echo "=== PostgreSQL Build Configuration ===" + echo "Compiler: $CC" + echo "LLVM: $(llvm-config --version 2>/dev/null || echo disabled)" + echo "Source: $PG_SOURCE_DIR" + echo "Build: $PG_BUILD_DIR" + echo "Install: $PG_INSTALL_DIR" + echo "======================================" + + env CFLAGS="-I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup $MESON_EXTRA_SETUP \ + --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Db_sanitize=none \ + -Db_lundef=false \ + -Dlz4=enabled \ + -Dzstd=enabled \ + -Dllvm=disabled \ + -Dplperl=enabled \ + -Dplpython=enabled \ + -Dpltcl=enabled \ + -Dlibxml=enabled \ + -Duuid=e2fs \ + -Dlibxslt=enabled \ + -Dssl=openssl \ + -Dldap=disabled \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Ddocs_pdf=enabled \ + -Ddocs_html_style=website \ + --prefix="$PG_INSTALL_DIR" \ + "$PG_BUILD_DIR" \ + "$PG_SOURCE_DIR"' + +alias pg-compdb='compdb -p build/ list > compile_commands.json' +alias pg-build='meson compile -C "$PG_BUILD_DIR"' +alias pg-install='meson install -C "$PG_BUILD_DIR"' +alias pg-test='meson test -q --print-errorlogs -C "$PG_BUILD_DIR"' + +# Clean commands +alias pg-clean='ninja -C "$PG_BUILD_DIR" clean' +alias pg-full-clean='trash "$PG_BUILD_DIR" "$PG_INSTALL_DIR" 2>/dev/null || rm -rf "$PG_BUILD_DIR" "$PG_INSTALL_DIR"; echo "Build and install directories cleaned"' + +# Database management +alias pg-init='trash "$PG_DATA_DIR" 2>/dev/null || rm -rf "$PG_DATA_DIR"; "$PG_INSTALL_DIR/bin/initdb" --debug --no-clean "$PG_DATA_DIR"' + +alias pg-start='ulimit -c unlimited && "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR" -k "$PG_DATA_DIR"' + +alias pg-stop='pkill -f "postgres.*-D.*$PG_DATA_DIR" || true' +alias pg-restart='pg-stop && sleep 2 && pg-start' +alias pg-status='pgrep -f "postgres.*-D.*$PG_DATA_DIR" && echo "PostgreSQL is running" || echo "PostgreSQL is not running"' + +# Client connections +alias pg-psql='"$PG_INSTALL_DIR/bin/psql" -h "$PG_DATA_DIR" postgres' +alias pg-createdb='"$PG_INSTALL_DIR/bin/createdb" -h "$PG_DATA_DIR"' +alias pg-dropdb='"$PG_INSTALL_DIR/bin/dropdb" -h "$PG_DATA_DIR"' + +# ============================================================ +# Debugger attachments +# ============================================================ +alias pg-debug-gdb='gdb -x "$GDBINIT" -x .gdbinit "$PG_INSTALL_DIR/bin/postgres"' +alias pg-debug-lldb='lldb "$PG_INSTALL_DIR/bin/postgres"' +alias pg-debug=' + if command -v gdb >/dev/null 2>&1; then + pg-debug-gdb + elif command -v lldb >/dev/null 2>&1; then + pg-debug-lldb + else + echo "No debugger available (gdb or lldb required)" + fi' + +alias pg-attach-gdb=' + PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -n "$PG_PID" ]; then + echo "Attaching GDB to PostgreSQL process $PG_PID" + gdb -x "$GDBINIT" -x .gdbinit -p "$PG_PID" + else + echo "No PostgreSQL process found" + fi' + +alias pg-attach-lldb=' + PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -n "$PG_PID" ]; then + echo "Attaching LLDB to PostgreSQL process $PG_PID" + lldb -p "$PG_PID" + else + echo "No PostgreSQL process found" + fi' + +alias pg-attach=' + if command -v gdb >/dev/null 2>&1; then + pg-attach-gdb + elif command -v lldb >/dev/null 2>&1; then + pg-attach-lldb + else + echo "No debugger available (gdb or lldb required)" + fi' + +# ============================================================ +# Valgrind-instrumented build and tests +# +# The valgrind build lives in a separate directory so the normal +# build stays warm. Runs use a wrapper dir that shadows `postgres` +# with a valgrind wrapper -- pg_regress finds it via PATH. +# ============================================================ +pg-build-valgrind() { + local bdir="$PG_BUILD_DIR_VALGRIND" + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: PERL_CORE_DIR is not set" >&2 + return 1 + fi + + pg_clean_for_compiler "$bdir" + + echo "=== Configuring Valgrind build in $bdir ===" + env CFLAGS="-Og -ggdb3 -fno-omit-frame-pointer -DUSE_VALGRIND -I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Dllvm=disabled \ + -Dplperl=enabled -Dplpython=enabled -Dpltcl=enabled \ + -Dlz4=enabled -Dzstd=enabled \ + -Dlibxml=enabled -Dlibxslt=enabled -Dssl=openssl -Duuid=e2fs \ + -Dldap=disabled \ + --prefix="$PG_INSTALL_DIR-valgrind" \ + "$bdir" "$PG_SOURCE_DIR" || return 1 + + meson compile -C "$bdir" +} + +# Drop a wrapper directory that shadows the real binaries; `postgres` +# exec's into valgrind, everything else is a symlink. Writes to the +# supplied wrap dir and echoes its path. +_pg_make_valgrind_wrapper() { + local bindir="$1" + local wrapdir="$2" + + mkdir -p "$wrapdir" + cat >"$wrapdir/postgres" <&2 + return 1 + fi + + local tmpbin="$bdir/tmp_install$PG_INSTALL_DIR-valgrind/bin" + if [ ! -x "$tmpbin/postgres" ]; then + echo "Populating tmp_install..." + meson test -C "$bdir" tmp_install install_test_files initdb_cache >/dev/null || return 1 + fi + + local wrap + wrap=$(mktemp -d /tmp/pg-vg-wrap-XXXXXX) + _pg_make_valgrind_wrapper "$tmpbin" "$wrap" + + mkdir -p "$PG_BENCH_DIR" + echo "Valgrind logs: $PG_BENCH_DIR/valgrind-*.log" + echo "Wrapper dir: $wrap (will be removed on exit)" + echo "Expect the regress suite to take 15-45 minutes under valgrind." + + local rc=0 + (cd "$bdir" && PATH="$wrap:$PATH" meson test -t 60 --print-errorlogs regress/regress) || rc=$? + + trash "$wrap" 2>/dev/null || rm -rf "$wrap" + return "$rc" +} + +pg-valgrind-test() { + local bdir="$PG_BUILD_DIR_VALGRIND" + if [ ! -x "$bdir/src/backend/postgres" ]; then + echo "Valgrind build not found; run 'pg-build-valgrind' first." >&2 + return 1 + fi + + echo "This runs the FULL postgres test suite under valgrind." + echo "Expect many hours, and tens of GB of valgrind log output." + echo "Logs: $PG_BENCH_DIR/valgrind-*.log" + local yn + read -r -p "Continue? [y/N] " yn + case "$yn" in + y | Y | yes) ;; + *) echo "Aborted."; return 0 ;; + esac + + local tmpbin="$bdir/tmp_install$PG_INSTALL_DIR-valgrind/bin" + if [ ! -x "$tmpbin/postgres" ]; then + echo "Populating tmp_install..." + meson test -C "$bdir" tmp_install install_test_files initdb_cache >/dev/null || return 1 + fi + + local wrap + wrap=$(mktemp -d /tmp/pg-vg-wrap-XXXXXX) + _pg_make_valgrind_wrapper "$tmpbin" "$wrap" + mkdir -p "$PG_BENCH_DIR" + + local rc=0 + (cd "$bdir" && PATH="$wrap:$PATH" meson test -t 60 --print-errorlogs) || rc=$? + + trash "$wrap" 2>/dev/null || rm -rf "$wrap" + return "$rc" +} + +# ============================================================ +# AddressSanitizer / UndefinedBehaviorSanitizer build and tests +# ============================================================ +pg-build-asan() { + local bdir="$PG_BUILD_DIR_ASAN" + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: PERL_CORE_DIR is not set" >&2 + return 1 + fi + + pg_clean_for_compiler "$bdir" + + echo "=== Configuring ASan+UBSan build in $bdir ===" + env CFLAGS="-Og -ggdb3 -fno-omit-frame-pointer -fsanitize=address,undefined -fno-sanitize-recover=all -I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-fsanitize=address,undefined -L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Dllvm=disabled \ + -Dplperl=enabled -Dplpython=enabled -Dpltcl=enabled \ + -Dlz4=enabled -Dzstd=enabled \ + -Dlibxml=enabled -Dlibxslt=enabled -Dssl=openssl -Duuid=e2fs \ + -Dldap=disabled \ + --prefix="$PG_INSTALL_DIR-asan" \ + "$bdir" "$PG_SOURCE_DIR" || return 1 + + meson compile -C "$bdir" +} + +pg-asan-regress() { + local bdir="$PG_BUILD_DIR_ASAN" + if [ ! -x "$bdir/src/backend/postgres" ]; then + echo "ASan build not found; run 'pg-build-asan' first." >&2 + return 1 + fi + + # halt_on_error=0 lets regress continue past the first diagnostic so + # the whole suite runs; abort_on_error=1 makes each hit fail the test. + ASAN_OPTIONS="halt_on_error=0:abort_on_error=1:detect_leaks=0:print_summary=1:print_stacktrace=1" \ + UBSAN_OPTIONS="halt_on_error=1:abort_on_error=1:print_stacktrace=1:print_summary=1" \ + meson test -t 5 --print-errorlogs -C "$bdir" regress/regress +} + +# ============================================================ +# rr (deterministic record-and-replay) +# Requires kernel.perf_event_paranoid <= 1. rr is the single most +# effective tool for postgres bugs that reproduce intermittently. +# ============================================================ +pg-rr-check() { + if ! command -v rr >/dev/null; then + echo "rr is not installed (expected in the dev shell)." >&2 + return 1 + fi + local paranoid + paranoid=$(cat /proc/sys/kernel/perf_event_paranoid 2>/dev/null || echo 99) + if [ "$paranoid" -gt 1 ]; then + echo "rr requires kernel.perf_event_paranoid <= 1; currently $paranoid" + echo "To enable (root needed):" + echo " echo 1 | sudo tee /proc/sys/kernel/perf_event_paranoid" + return 1 + fi + echo "rr ready (perf_event_paranoid=$paranoid)" +} + +pg-rr-record() { + pg-rr-check >/dev/null || { + pg-rr-check + return 1 + } + ulimit -c unlimited + rr record -- "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR" -k "$PG_DATA_DIR" +} + +pg-rr-replay() { + rr replay "$@" +} + +# ============================================================ +# perf wrappers (parallel to the flame-graph helper) +# ============================================================ +pg-perf-record() { + local pid + pid=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -z "$pid" ]; then + echo "No postgres running under $PG_DATA_DIR" >&2 + return 1 + fi + mkdir -p "$PG_BENCH_DIR" + local out="$PG_BENCH_DIR/perf-$(date +%Y%m%d_%H%M%S).data" + echo "Recording to $out (Ctrl-C to stop)" + perf record -F 997 --call-graph dwarf -p "$pid" -o "$out" "$@" + echo "Saved: $out" +} + +pg-perf-report() { + local data + data=$(ls -t "$PG_BENCH_DIR"/perf-*.data 2>/dev/null | head -1) + if [ -z "$data" ]; then + echo "No perf data in $PG_BENCH_DIR" >&2 + return 1 + fi + echo "Reading $data" + perf report -i "$data" "$@" +} + +pg-perf-annotate() { + local data + data=$(ls -t "$PG_BENCH_DIR"/perf-*.data 2>/dev/null | head -1) + if [ -z "$data" ]; then + echo "No perf data in $PG_BENCH_DIR" >&2 + return 1 + fi + perf annotate -i "$data" "$@" +} + +# ============================================================ +# Single regression test / group runner. +# Runs pg_regress directly against the existing build so you skip the +# full meson-driven suite wrapper. Usage: pg-test-one boolean [name ...] +# ============================================================ +pg-test-one() { + if [ $# -eq 0 ]; then + echo "usage: pg-test-one TESTNAME [TESTNAME ...]" + echo "example: pg-test-one boolean" + return 2 + fi + local bdir="${PG_BUILD_DIR_ONE:-$PG_BUILD_DIR}" + local tmpbin="$bdir/tmp_install$PG_INSTALL_DIR/bin" + if [ ! -x "$tmpbin/postgres" ]; then + echo "Populating tmp_install..." + meson test -C "$bdir" tmp_install install_test_files initdb_cache >/dev/null || return 1 + fi + local outdir + outdir=$(mktemp -d /tmp/pg-test-one-XXXXXX) + echo "Test output: $outdir" + "$bdir/src/test/regress/pg_regress" \ + --bindir="$tmpbin" \ + --inputdir="$PG_SOURCE_DIR/src/test/regress" \ + --expecteddir="$PG_SOURCE_DIR/src/test/regress" \ + --dlpath="$bdir/src/test/regress" \ + --outputdir="$outdir" \ + --temp-instance="$outdir/tmp" \ + --port=40099 \ + "$@" +} + +# Full flame graph / benchmark aliases +alias pg-flame='pg-flame-generate' +alias pg-flame-30='pg-flame-generate 30' +alias pg-flame-60='pg-flame-generate 60' +alias pg-flame-120='pg-flame-generate 120' + +pg-flame-custom() { + local duration=${1:-30} + local output_dir=${2:-$PG_FLAME_DIR} + echo "Generating flame graph for ${duration}s, output to: $output_dir" + pg-flame-generate "$duration" "$output_dir" +} + +alias pg-bench='pg-bench-run' +alias pg-bench-quick='pg-bench-run 5 1 100 1 30 select-only' +alias pg-bench-standard='pg-bench-run 10 2 1000 10 60 tpcb-like' +alias pg-bench-heavy='pg-bench-run 50 4 5000 100 300 tpcb-like' +alias pg-bench-readonly='pg-bench-run 20 4 2000 50 120 select-only' + +pg-bench-custom() { + local clients=${1:-10} + local threads=${2:-2} + local transactions=${3:-1000} + local scale=${4:-10} + local duration=${5:-60} + local test_type=${6:-tpcb-like} + + echo "Running custom benchmark:" + echo " Clients: $clients, Threads: $threads" + echo " Transactions: $transactions, Scale: $scale" + echo " Duration: ${duration}s, Type: $test_type" + + pg-bench-run "$clients" "$threads" "$transactions" "$scale" "$duration" "$test_type" +} + +pg-bench-flame() { + local duration=${1:-60} + local clients=${2:-10} + local scale=${3:-10} + + echo "Running benchmark with flame graph generation" + echo "Duration: ${duration}s, Clients: $clients, Scale: $scale" + + pg-bench-run "$clients" 2 1000 "$scale" "$duration" tpcb-like & + local bench_pid=$! + + sleep 5 + + local flame_duration=$((duration - 10)) + if [ $flame_duration -gt 10 ]; then + pg-flame-generate "$flame_duration" & + local flame_pid=$! + fi + + wait $bench_pid + if [ -n "${flame_pid:-}" ]; then + wait $flame_pid + fi + + echo "Benchmark and flame graph generation completed" +} + +# Live monitoring +alias pg-perf='perf top -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1)' +alias pg-htop='htop -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | tr "\n" "," | sed "s/,$//")' + +pg-stats() { + local duration=${1:-30} + echo "Collecting system stats for ${duration}s..." + + iostat -x 1 "$duration" >"$PG_BENCH_DIR/iostat_$(date +%Y%m%d_%H%M%S).log" & + vmstat 1 "$duration" >"$PG_BENCH_DIR/vmstat_$(date +%Y%m%d_%H%M%S).log" & + + wait + echo "System stats saved to $PG_BENCH_DIR" +} + +# ============================================================ +# Code quality helpers +# ============================================================ +pg-format() { + local since=${1:-HEAD} + + if [ ! -f "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" ]; then + echo "Error: pgindent not found at $PG_SOURCE_DIR/src/tools/pgindent/pgindent" + else + + modified_files=$(git diff --name-only "${since}" | grep -E "\.c$|\.h$") + + if [ -z "$modified_files" ]; then + echo "No modified .c or .h files found" + else + + echo "Formatting modified files with pgindent:" + for file in $modified_files; do + if [ -f "$file" ]; then + echo " Formatting: $file" + "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" "$file" + else + echo " Warning: File not found: $file" + fi + done + + echo "Checking files for whitespace:" + git diff --check "${since}" + fi + fi +} + +pg-tidy() { + local since=${1:-HEAD} + local files + files=$(git diff --name-only "$since" | grep -E "\.(c|h)$") + if [ -z "$files" ]; then + echo "No modified .c or .h files." + return 0 + fi + for f in $files; do + [ -f "$f" ] || continue + echo "clang-tidy: $f" + clang-tidy -p "$PG_BUILD_DIR" "$f" 2>&1 | head -50 + done +} + +pg-spell() { + local since=${1:-HEAD} + local files=$(git diff --name-only "$since" | grep -E '\.(c|h|sgml|md)$') + if [ -z "$files" ]; then + echo "No .c/.h/.sgml/.md files changed since $since" + return 0 + fi + for f in $files; do + [ -f "$f" ] || continue + case "$f" in + *.c | *.h) + grep -nE '^\s*(/\*|\*|//)' "$f" | codespell --stdin-single-line - 2>/dev/null \ + && echo " $f: ok" || true + ;; + *.sgml | *.md) + codespell "$f" || true + ;; + esac + done +} + +# ============================================================ +# Core dump one-shots (one-time, requires root). kernel.core_pattern +# is a system-wide sysctl -- we don't touch it on every shell entry. +# ============================================================ +pg-cores-status() { + echo "ulimit -c: $(ulimit -c)" + echo "kernel.core_pattern: $(cat /proc/sys/kernel/core_pattern 2>/dev/null || echo unreadable)" + echo "cwd: $(pwd)" +} + +pg-enable-cores() { + ulimit -c unlimited + if ! [ -w /proc/sys/kernel/core_pattern ]; then + echo "Setting kernel.core_pattern (requires sudo)..." + echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null || { + echo "Failed to write /proc/sys/kernel/core_pattern" >&2 + return 1 + } + else + echo "core.%p" >/proc/sys/kernel/core_pattern + fi + pg-cores-status +} + +pg-disable-cores() { + ulimit -c 0 + if ! [ -w /proc/sys/kernel/core_pattern ]; then + echo "Restoring kernel.core_pattern to 'core' (requires sudo)..." + echo "core" | sudo tee /proc/sys/kernel/core_pattern >/dev/null || { + echo "Failed to restore /proc/sys/kernel/core_pattern" >&2 + return 1 + } + else + echo "core" >/proc/sys/kernel/core_pattern + fi + pg-cores-status +} + +# ============================================================ +# Logs and results +# ============================================================ +alias pg-log='tail -f "$PG_DATA_DIR/log/postgresql-$(date +%Y-%m-%d).log" 2>/dev/null || echo "No log file found"' +alias pg-log-errors='grep -i error "$PG_DATA_DIR/log/"*.log 2>/dev/null || echo "No error logs found"' + +alias pg-build-log='cat "$PG_BUILD_DIR/meson-logs/meson-log.txt"' +alias pg-build-errors='grep -i error "$PG_BUILD_DIR/meson-logs/meson-log.txt" 2>/dev/null || echo "No build errors found"' + +alias pg-bench-results='ls -la "$PG_BENCH_DIR" && echo "Latest results:" && tail -20 "$PG_BENCH_DIR"/results_*.txt 2>/dev/null | tail -20' +alias pg-flame-results='ls -la "$PG_FLAME_DIR" && echo "Open flame graphs with: firefox $PG_FLAME_DIR/*.svg"' + +pg-clean-results() { + local days=${1:-7} + echo "Cleaning benchmark and flame graph results older than $days days..." + find "$PG_BENCH_DIR" -type f -mtime +$days -delete 2>/dev/null || true + find "$PG_FLAME_DIR" -type f -mtime +$days -delete 2>/dev/null || true + echo "Cleanup completed" +} + +# ============================================================ +# Info +# ============================================================ +alias pg-info=' + echo "=== PostgreSQL Development Environment ===" + echo "Source: $PG_SOURCE_DIR" + echo "Build (default): $PG_BUILD_DIR" + echo "Build (valgrind):$PG_BUILD_DIR_VALGRIND" + echo "Build (asan): $PG_BUILD_DIR_ASAN" + echo "Install: $PG_INSTALL_DIR" + echo "Data: $PG_DATA_DIR" + echo "Benchmarks: $PG_BENCH_DIR" + echo "Flame graphs: $PG_FLAME_DIR" + echo "Compiler: $CC" + echo "" + echo "Available commands:" + echo " Setup/build: pg-setup, pg-build, pg-install" + echo " Database: pg-init, pg-start, pg-stop, pg-psql" + echo " Tests: pg-test, pg-test-one NAME" + echo " Valgrind: pg-build-valgrind, pg-valgrind-regress, pg-valgrind-test" + echo " ASan/UBSan: pg-build-asan, pg-asan-regress" + echo " Debug: pg-debug, pg-attach" + echo " Record/replay: pg-rr-check, pg-rr-record, pg-rr-replay" + echo " Perf: pg-perf-record, pg-perf-report, pg-perf-annotate, pg-perf" + echo " Flame graphs: pg-flame, pg-flame-30, pg-flame-60, pg-flame-custom" + echo " Benchmarks: pg-bench-quick, pg-bench-standard, pg-bench-heavy" + echo " Combined: pg-bench-flame" + echo " Results: pg-bench-results, pg-flame-results" + echo " Logs: pg-log, pg-build-log" + echo " Clean: pg-clean, pg-full-clean, pg-clean-results" + echo " Code quality: pg-format, pg-tidy, pg-spell" + echo " Cores: pg-enable-cores, pg-disable-cores, pg-cores-status" + echo "=========================================="' + +echo "PostgreSQL aliases loaded. Run 'pg-info' for available commands." diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000000..8c738afa789a7 --- /dev/null +++ b/shell.nix @@ -0,0 +1,745 @@ +{ + pkgs, + pkgs-unstable, + system, +}: let + # Create a patched glibc only for the dev shell. + # + # Glibc's features.h emits a `-Wcpp` diagnostic when _FORTIFY_SOURCE is + # defined without an optimization level. Meson's dependency probes + # (notably the libcurl thread-safety check) compile small snippets with + # `-O0 -Werror`, which turns that cpp warning into a hard error and + # breaks reconfigure under our default CFLAGS. The patch simply drops + # the warning. It is scoped to this dev shell only and never leaks + # into system glibc or release builds. + patchedGlibc = pkgs.glibc.overrideAttrs (oldAttrs: { + patches = (oldAttrs.patches or []) ++ [ + ./glibc-no-fortify-warning.patch + ]; + }); + + # Use LLVM for modern PostgreSQL development + llvmPkgs = pkgs-unstable.llvmPackages_21; + + # Configuration constants + config = { + pgSourceDir = "$PWD"; + pgBuildDir = "$PWD/build"; + pgBuildDirValgrind = "$PWD/build-valgrind"; + pgBuildDirAsan = "$PWD/build-asan"; + pgInstallDir = "$PWD/install"; + pgDataDir = "/tmp/test-db-$(basename $PWD)"; + pgBenchDir = "/tmp/pgbench-results-$(basename $PWD)"; + pgFlameDir = "/tmp/flame-graphs-$(basename $PWD)"; + }; + + # Single dependency function that can be used for all environments + getPostgreSQLDeps = muslLibs: + with pkgs; + [ + # Build system (always use host tools) + pkgs-unstable.meson + pkgs-unstable.ninja + pkg-config + autoconf + git + which + binutils + gnumake + mold # fast linker, big wins on large postgres links + + # Parser/lexer tools + bison + flex + + # Perl with required packages + (perl.withPackages (ps: with ps; [IPCRun])) + + # Documentation + docbook_xml_dtd_45 + docbook-xsl-nons + libxslt + libxml2 + fop + + # Development tools (always use host tools) + coreutils + shellcheck + ripgrep + valgrind + curl + uv + pylint + black + lcov + strace + ltrace + perf-tools + linuxPackages.perf + flamegraph + bpftrace # kernel-level tracing (probes, uprobes) + rr # record-and-replay deterministic debugger + htop + iotop + sysstat + ccache + cppcheck + compdb + + # Spell checking + aspell + aspellDicts.en + codespell + + # GCC/GDB + gcc + gdb + + # LLVM toolchain + llvmPkgs.llvm + llvmPkgs.llvm.dev + llvmPkgs.clang-tools + llvmPkgs.lldb + + # Language support + (python3.withPackages (ps: with ps; [requests browser-cookie3])) + tcl + ] + ++ ( + if muslLibs + then [ + # Musl target libraries for cross-compilation + pkgs.pkgsMusl.readline + pkgs.pkgsMusl.zlib + pkgs.pkgsMusl.openssl + pkgs.pkgsMusl.icu + pkgs.pkgsMusl.lz4 + pkgs.pkgsMusl.zstd + pkgs.pkgsMusl.libuuid + pkgs.pkgsMusl.libkrb5 + pkgs.pkgsMusl.linux-pam + pkgs.pkgsMusl.libxcrypt + ] + else [ + # Glibc target libraries + readline + zlib + openssl + icu + lz4 + zstd + libuuid + libkrb5 + linux-pam + libxcrypt + numactl + openldap + liburing + libselinux + patchedGlibc + patchedGlibc.dev + ] + ); + + # GDB configuration for PostgreSQL debugging + gdbConfig = pkgs.writeText "gdbinit-postgres" '' + # PostgreSQL-specific GDB configuration + + # Pretty-print PostgreSQL data structures + define print_node + if $arg0 + printf "Node type: %s\n", nodeTagNames[$arg0->type] + print *$arg0 + else + printf "NULL node\n" + end + end + document print_node + Print a PostgreSQL Node with type information + Usage: print_node + end + + define print_list + set $list = (List*)$arg0 + if $list + printf "List length: %d\n", $list->length + set $cell = $list->head + set $i = 0 + while $cell && $i < $list->length + printf " [%d]: ", $i + print_node $cell->data.ptr_value + set $cell = $cell->next + set $i = $i + 1 + end + else + printf "NULL list\n" + end + end + document print_list + Print a PostgreSQL List structure + Usage: print_list + end + + define print_query + set $query = (Query*)$arg0 + if $query + printf "Query type: %d, command type: %d\n", $query->querySource, $query->commandType + print *$query + else + printf "NULL query\n" + end + end + document print_query + Print a PostgreSQL Query structure + Usage: print_query + end + + define print_relcache + set $rel = (Relation)$arg0 + if $rel + printf "Relation: %s.%s (OID: %u)\n", $rel->rd_rel->relnamespace, $rel->rd_rel->relname.data, $rel->rd_id + printf " natts: %d, relkind: %c\n", $rel->rd_rel->relnatts, $rel->rd_rel->relkind + else + printf "NULL relation\n" + end + end + document print_relcache + Print relation cache entry information + Usage: print_relcache + end + + define print_tupdesc + set $desc = (TupleDesc)$arg0 + if $desc + printf "TupleDesc: %d attributes\n", $desc->natts + set $i = 0 + while $i < $desc->natts + set $attr = $desc->attrs[$i] + printf " [%d]: %s (type: %u, len: %d)\n", $i, $attr->attname.data, $attr->atttypid, $attr->attlen + set $i = $i + 1 + end + else + printf "NULL tuple descriptor\n" + end + end + document print_tupdesc + Print tuple descriptor information + Usage: print_tupdesc + end + + define print_slot + set $slot = (TupleTableSlot*)$arg0 + if $slot + printf "TupleTableSlot: %s\n", $slot->tts_ops->name + printf " empty: %d, shouldFree: %d\n", $slot->tts_empty, $slot->tts_shouldFree + if $slot->tts_tupleDescriptor + print_tupdesc $slot->tts_tupleDescriptor + end + else + printf "NULL slot\n" + end + end + document print_slot + Print tuple table slot information + Usage: print_slot + end + + # Memory context debugging + define print_mcxt + set $context = (MemoryContext)$arg0 + if $context + printf "MemoryContext: %s\n", $context->name + printf " type: %s, parent: %p\n", $context->methods->name, $context->parent + printf " total: %zu, free: %zu\n", $context->mem_allocated, $context->freep - $context->freeptr + else + printf "NULL memory context\n" + end + end + document print_mcxt + Print memory context information + Usage: print_mcxt + end + + # Process debugging + define print_proc + set $proc = (PGPROC*)$arg0 + if $proc + printf "PGPROC: pid=%d, database=%u\n", $proc->pid, $proc->databaseId + printf " waiting: %d, waitStatus: %d\n", $proc->waiting, $proc->waitStatus + else + printf "NULL process\n" + end + end + document print_proc + Print process information + Usage: print_proc + end + + # Set useful defaults + set print pretty on + set print object on + set print static-members off + set print vtbl on + set print demangle on + set demangle-style gnu-v3 + set print sevenbit-strings off + set history save on + set history size 1000 + set history filename ~/.gdb_history_postgres + + # Common breakpoints for PostgreSQL debugging + define pg_break_common + break elog + break errfinish + break ExceptionalCondition + break ProcessInterrupts + end + document pg_break_common + Set common PostgreSQL debugging breakpoints + end + + printf "PostgreSQL GDB configuration loaded.\n" + printf "Available commands: print_node, print_list, print_query, print_relcache,\n" + printf " print_tupdesc, print_slot, print_mcxt, print_proc, pg_break_common\n" + ''; + + # Flame graph generation script + flameGraphScript = pkgs.writeScriptBin "pg-flame-generate" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + DURATION=''${1:-30} + OUTPUT_DIR=''${2:-${config.pgFlameDir}} + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + mkdir -p "$OUTPUT_DIR" + + echo "Generating flame graph for PostgreSQL (duration: ''${DURATION}s)" + + # Find PostgreSQL processes + PG_PIDS=$(pgrep -f "postgres.*-D.*${config.pgDataDir}" || true) + + if [ -z "$PG_PIDS" ]; then + echo "Error: No PostgreSQL processes found" + exit 1 + fi + + echo "Found PostgreSQL processes: $PG_PIDS" + + # Record perf data + PERF_DATA="$OUTPUT_DIR/perf_$TIMESTAMP.data" + echo "Recording perf data to $PERF_DATA" + + ${pkgs.linuxPackages.perf}/bin/perf record \ + -F 997 \ + -g \ + --call-graph dwarf \ + -p "$(echo $PG_PIDS | tr ' ' ',')" \ + -o "$PERF_DATA" \ + sleep "$DURATION" + + # Generate flame graph + FLAME_SVG="$OUTPUT_DIR/postgres_flame_$TIMESTAMP.svg" + echo "Generating flame graph: $FLAME_SVG" + + ${pkgs.linuxPackages.perf}/bin/perf script -i "$PERF_DATA" | \ + ${pkgs.flamegraph}/bin/stackcollapse-perf.pl | \ + ${pkgs.flamegraph}/bin/flamegraph.pl \ + --title "PostgreSQL Flame Graph ($TIMESTAMP)" \ + --width 1200 \ + --height 800 \ + > "$FLAME_SVG" + + echo "Flame graph generated: $FLAME_SVG" + echo "Perf data saved: $PERF_DATA" + + # Generate summary report + REPORT="$OUTPUT_DIR/report_$TIMESTAMP.txt" + echo "Generating performance report: $REPORT" + + { + echo "PostgreSQL Performance Analysis Report" + echo "Generated: $(date)" + echo "Duration: ''${DURATION}s" + echo "Processes: $PG_PIDS" + echo "" + echo "=== Top Functions ===" + ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio --sort comm,dso,symbol | head -50 + echo "" + echo "=== Call Graph ===" + ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio -g --sort comm,dso,symbol | head -100 + } > "$REPORT" + + echo "Report generated: $REPORT" + echo "" + echo "Files created:" + echo " Flame graph: $FLAME_SVG" + echo " Perf data: $PERF_DATA" + echo " Report: $REPORT" + ''; + + # pgbench wrapper script + pgbenchScript = pkgs.writeScriptBin "pg-bench-run" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + # Default parameters + CLIENTS=''${1:-10} + THREADS=''${2:-2} + TRANSACTIONS=''${3:-1000} + SCALE=''${4:-10} + DURATION=''${5:-60} + TEST_TYPE=''${6:-tpcb-like} + + OUTPUT_DIR="${config.pgBenchDir}" + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + mkdir -p "$OUTPUT_DIR" + + echo "=== PostgreSQL Benchmark Configuration ===" + echo "Clients: $CLIENTS" + echo "Threads: $THREADS" + echo "Transactions: $TRANSACTIONS" + echo "Scale factor: $SCALE" + echo "Duration: ''${DURATION}s" + echo "Test type: $TEST_TYPE" + echo "Output directory: $OUTPUT_DIR" + echo "============================================" + + # Check if PostgreSQL is running + if ! pgrep -f "postgres.*-D.*${config.pgDataDir}" >/dev/null; then + echo "Error: PostgreSQL is not running. Start it with 'pg-start'" + exit 1 + fi + + PGBENCH="${config.pgInstallDir}/bin/pgbench" + PSQL="${config.pgInstallDir}/bin/psql" + CREATEDB="${config.pgInstallDir}/bin/createdb" + DROPDB="${config.pgInstallDir}/bin/dropdb" + + DB_NAME="pgbench_test_$TIMESTAMP" + RESULTS_FILE="$OUTPUT_DIR/results_$TIMESTAMP.txt" + LOG_FILE="$OUTPUT_DIR/pgbench_$TIMESTAMP.log" + + echo "Creating test database: $DB_NAME" + "$CREATEDB" -h "${config.pgDataDir}" "$DB_NAME" || { + echo "Failed to create database" + exit 1 + } + + # Initialize pgbench tables + echo "Initializing pgbench tables (scale factor: $SCALE)" + "$PGBENCH" -h "${config.pgDataDir}" -i -s "$SCALE" "$DB_NAME" || { + echo "Failed to initialize pgbench tables" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + exit 1 + } + + # Run benchmark based on test type + echo "Running benchmark..." + + case "$TEST_TYPE" in + "tpcb-like"|"default") + BENCH_ARGS="" + ;; + "select-only") + BENCH_ARGS="-S" + ;; + "simple-update") + BENCH_ARGS="-N" + ;; + "read-write") + BENCH_ARGS="-b select-only@70 -b tpcb-like@30" + ;; + *) + echo "Unknown test type: $TEST_TYPE" + echo "Available types: tpcb-like, select-only, simple-update, read-write" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + exit 1 + ;; + esac + + { + echo "PostgreSQL Benchmark Results" + echo "Generated: $(date)" + echo "Test type: $TEST_TYPE" + echo "Clients: $CLIENTS, Threads: $THREADS" + echo "Transactions: $TRANSACTIONS, Duration: ''${DURATION}s" + echo "Scale factor: $SCALE" + echo "Database: $DB_NAME" + echo "" + echo "=== System Information ===" + echo "CPU: $(nproc) cores" + echo "Memory: $(free -h | grep '^Mem:' | awk '{print $2}')" + echo "Compiler: $CC" + echo "PostgreSQL version: $("$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -t -c "SELECT version();" | head -1)" + echo "" + echo "=== Benchmark Results ===" + } > "$RESULTS_FILE" + + # Run the actual benchmark + "$PGBENCH" \ + -h "${config.pgDataDir}" \ + -c "$CLIENTS" \ + -j "$THREADS" \ + -T "$DURATION" \ + -P 5 \ + --log \ + --log-prefix="$OUTPUT_DIR/pgbench_$TIMESTAMP" \ + $BENCH_ARGS \ + "$DB_NAME" 2>&1 | tee -a "$RESULTS_FILE" + + # Collect additional statistics + { + echo "" + echo "=== Database Statistics ===" + "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c " + SELECT + schemaname, + relname, + n_tup_ins as inserts, + n_tup_upd as updates, + n_tup_del as deletes, + n_live_tup as live_tuples, + n_dead_tup as dead_tuples + FROM pg_stat_user_tables; + " + + echo "" + echo "=== Index Statistics ===" + "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c " + SELECT + schemaname, + relname, + indexrelname, + idx_scan, + idx_tup_read, + idx_tup_fetch + FROM pg_stat_user_indexes; + " + } >> "$RESULTS_FILE" + + # Clean up + echo "Cleaning up test database: $DB_NAME" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + + echo "" + echo "Benchmark completed!" + echo "Results saved to: $RESULTS_FILE" + echo "Transaction logs: $OUTPUT_DIR/pgbench_$TIMESTAMP*" + + # Show summary + echo "" + echo "=== Quick Summary ===" + grep -E "(tps|latency)" "$RESULTS_FILE" | tail -5 + ''; + + # Shared shellHook fragments. Each devShell prepends its own compiler/CFLAGS + # block, then appends the common tail via ${commonHookTail variant}. + commonHookHead = icon: '' + # History configuration + export HISTFILE=.history + export HISTSIZE=1000000 + export HISTFILESIZE=1000000 + + # Clean environment + unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH + + # Essential tools in PATH + export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH" + export PS1="$(echo -e '\u${icon}') {\[$(tput sgr0)\]\[\033[38;5;228m\]\w\[$(tput sgr0)\]\[\033[38;5;15m\]} ($(git rev-parse --abbrev-ref HEAD)) \\$ \[$(tput sgr0)\]" + + # Ccache configuration + export PATH=${pkgs.ccache}/bin:$PATH + export CCACHE_COMPILERCHECK=content + # Loosen a few rules so ccache hits across rebuilds with touched headers. + export CCACHE_SLOPPINESS=pch_defines,time_macros,include_file_mtime,include_file_ctime + export CCACHE_DIR=$HOME/.ccache/pg/$(basename $PWD) + mkdir -p "$CCACHE_DIR" + + # Development tools in PATH + export PATH=${pkgs.clang-tools}/bin:$PATH + export PATH=${pkgs.cppcheck}/bin:$PATH + ''; + + # Tail shared by every devShell: PG env vars, GDB, tool PATH, per-process + # setup and alias load. Kernel core_pattern is NOT touched here -- + # run 'pg-enable-cores' explicitly if you need per-PID cores in CWD. + commonHookTail = label: '' + # PostgreSQL environment + export PG_SOURCE_DIR="${config.pgSourceDir}" + export PG_BUILD_DIR="${config.pgBuildDir}" + export PG_BUILD_DIR_VALGRIND="${config.pgBuildDirValgrind}" + export PG_BUILD_DIR_ASAN="${config.pgBuildDirAsan}" + export PG_INSTALL_DIR="${config.pgInstallDir}" + export PG_DATA_DIR="${config.pgDataDir}" + export PG_BENCH_DIR="${config.pgBenchDir}" + export PG_FLAME_DIR="${config.pgFlameDir}" + export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d) + + # GDB configuration + export GDBINIT="${gdbConfig}" + + # Performance tools in PATH + export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH" + + # Create output directories + mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR" + + # Per-process core dump size limit. Kernel core_pattern is NOT + # touched here -- run 'pg-enable-cores' explicitly when you need + # per-PID cores in CWD. + ulimit -c unlimited + + # Local git excludes + git config core.excludesFile .local-gitignore 2>/dev/null || true + + # Load PostgreSQL development aliases + if [ -f ./pg-aliases.sh ]; then + source ./pg-aliases.sh + else + echo "Warning: pg-aliases.sh not found in current directory" + fi + + echo "" + echo "PostgreSQL Development Environment Ready (${label})" + echo "Run 'pg-info' for available commands" + ''; + + # Development shell (GCC + glibc) + devShell = pkgs.mkShell { + name = "postgresql-dev"; + buildInputs = + (getPostgreSQLDeps false) + ++ [ + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # LLVM configuration + export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config" + export PATH="${llvmPkgs.llvm}/bin:$PATH" + export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH" + export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm" + export LLVM_ROOT="${llvmPkgs.llvm}" + + # PostgreSQL Development CFLAGS + export CFLAGS="" + export CXXFLAGS="" + + # Python UV + UV_PYTHON_DOWNLOADS=never + + # GCC configuration (default compiler) + export CC="${pkgs.gcc}/bin/gcc" + export CXX="${pkgs.gcc}/bin/g++" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: glibc" + echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')" + '' + + (commonHookTail "GCC + glibc"); + }; + + # Clang + glibc variant + clangDevShell = pkgs.mkShell { + name = "postgresql-clang-glibc"; + buildInputs = + (getPostgreSQLDeps false) + ++ [ + llvmPkgs.clang + llvmPkgs.lld + llvmPkgs.compiler-rt + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # LLVM configuration + export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config" + export PATH="${llvmPkgs.llvm}/bin:$PATH" + export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH" + export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm" + export LLVM_ROOT="${llvmPkgs.llvm}" + + # Clang + glibc configuration + export CC="${llvmPkgs.clang}/bin/clang" + export CXX="${llvmPkgs.clang}/bin/clang++" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: glibc" + echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')" + '' + + (commonHookTail "Clang + glibc"); + }; + + # GCC + musl variant (cross-compilation) + muslDevShell = pkgs.mkShell { + name = "postgresql-gcc-musl"; + buildInputs = + (getPostgreSQLDeps true) + ++ [ + pkgs.gcc + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # Cross-compilation to musl with GCC + export CC="${pkgs.gcc}/bin/gcc" + export CXX="${pkgs.gcc}/bin/g++" + + export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig" + export CFLAGS="-ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export CXXFLAGS="-ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export LDFLAGS="-L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -static-libgcc" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: musl (cross-compilation)" + '' + + (commonHookTail "GCC + musl"); + }; + + # Clang + musl variant (cross-compilation) + clangMuslDevShell = pkgs.mkShell { + name = "postgresql-clang-musl"; + buildInputs = + (getPostgreSQLDeps true) + ++ [ + llvmPkgs.clang + llvmPkgs.lld + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # Cross-compilation to musl with clang + export CC="${llvmPkgs.clang}/bin/clang" + export CXX="${llvmPkgs.clang}/bin/clang++" + + export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig" + export CFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export CXXFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export LDFLAGS="--target=x86_64-linux-musl -L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -fuse-ld=lld" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: musl (cross-compilation)" + '' + + (commonHookTail "Clang + musl"); + }; +in { + inherit devShell clangDevShell muslDevShell clangMuslDevShell gdbConfig flameGraphScript pgbenchScript; +} diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 21d00f792d24e..bfd7e84e56bef 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -1243,7 +1243,7 @@ spawn_process(const char *cmdline) char *cmdline2; cmdline2 = psprintf("exec %s", cmdline); - execl(shellprog, shellprog, "-c", cmdline2, (char *) NULL); + execlp(shellprog, shellprog, "-c", cmdline2, (char *) NULL); /* Not using the normal bail() here as we want _exit */ bail_noatexit("could not exec \"%s\": %m", shellprog); } diff --git a/src/tools/pgindent/pgindent b/src/tools/pgindent/pgindent index 004b8fcab0027..747f054351486 100755 --- a/src/tools/pgindent/pgindent +++ b/src/tools/pgindent/pgindent @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Copyright (c) 2021-2026, PostgreSQL Global Development Group From 5813cbb09c08831ff7e1296e69d5ad1e4a585fec Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 9 Jul 2026 12:34:55 -0400 Subject: [PATCH 4/6] Add an opt-in C11 stdatomic.h implementation of the atomics API PostgreSQL requires C11, but not all supported compilers provide a fully functional . This adds an alternative implementation of port/atomics.h built on the standard C11 header, selectable at build time alongside the existing platform-specific implementations. Build-system detection (meson and autoconf): A new option -- meson -Duse_stdatomic=yes|no|auto and autoconf --with-stdatomic=yes|no|auto -- controls the choice: * 'auto' (default): use stdatomic.h if a probe compile succeeds * 'yes': require stdatomic.h (fail if unavailable) * 'no': use the traditional platform-specific implementation Both build systems run the same probe (32-bit fetch_add, a seq_cst thread fence, and a 64-bit compare_exchange) and define USE_STDATOMIC_H when it succeeds and the option is not 'no'. For MSVC, stdatomic.h is only available in Visual Studio 2022 or later with /experimental:c11atomics, which the build adds automatically; earlier MSVC falls back to the traditional implementation. Implementation (src/include/port/atomics/stdatomic_impl.h): All atomic types (pg_atomic_flag, pg_atomic_uint8/16/32/64) are implemented with C11 _Atomic. Memory ordering matches the traditional implementation's observable semantics: * pg_atomic_read/write_* use memory_order_relaxed, matching the plain volatile loads/stores of the traditional path (no fence). * read-modify-write, exchange, and compare-exchange use memory_order_seq_cst, matching the traditional "full barrier" contract. * the _membarrier read/write variants use memory_order_seq_cst. C++ compatibility is provided via for C++11 through C++20, which do not include ; C++23 and later include it directly. Platform-specific spin-delay hints (PAUSE, ISB, YIELD) are factored into a new src/include/port/spin_delay.h, with the shared SpinDelayStatus type and helpers in src/include/port/spin_delay_status.h so both spinlock paths use them. On ARM64, ISB is used instead of YIELD for better backoff under contention; on Windows ARM64 this adds a spin delay where the traditional implementation had none. Wire-up (src/include/port/atomics.h, storage/spin.h, s_lock.c): When USE_STDATOMIC_H is defined, atomics.h includes stdatomic_impl.h; otherwise it includes the arch-*.h and generic-*.h headers exactly as before. Both paths feed the same shared public-API layer (the static inline pg_atomic_* wrappers), so the API is identical either way. For spinlocks, when USE_STDATOMIC_H is defined slock_t is a pg_atomic_uint32 (0 == unlocked, 1 == locked, so a zeroed slock_t is a valid free lock) and the SpinLock* operations are built on the u32 atomic API rather than platform-specific TAS assembly. SpinLockAcquire is a thin wrapper macro over an inline helper: the macro captures the call site's __FILE__/__LINE__/__func__ for stuck-spinlock diagnostics while the helper evaluates its lock argument exactly once. On strong-memory platforms a relaxed-load test precedes the atomic exchange (PG_SPIN_TRY_RELAXED). When USE_STDATOMIC_H is not defined, spinlock behavior is byte-for-byte unchanged from previous PostgreSQL versions. With stdatomic.h, atomic operations no longer depend on backend-only infrastructure, so the frontend include restriction in atomics.h is relaxed for that path. The traditional implementation remains the default , this is an opt-in alternative rather than a replacement. Both code paths provide identical public API, semantics, and roughly equal performance. Co-authored-by: Greg Burd Co-authored-by: Thomas Munro Discussion: https://postgr.es/m/CC76554F-41AC-45AA-AF10-370FEC416498%40greg.burd.me#dd4b139cd557d6e87a9f2d706b6415ae --- configure | 109 ++++ configure.ac | 66 +++ meson.build | 102 ++++ meson_options.txt | 5 + src/backend/storage/lmgr/README | 71 +++ src/backend/storage/lmgr/s_lock.c | 42 +- src/include/pg_config.h.in | 4 + src/include/port/atomics.h | 103 +++- src/include/port/atomics/stdatomic_impl.h | 589 ++++++++++++++++++++++ src/include/port/spin_delay.h | 151 ++++++ src/include/port/spin_delay_status.h | 52 ++ src/include/storage/spin.h | 76 +++ src/test/regress/regress.c | 12 + 13 files changed, 1372 insertions(+), 10 deletions(-) create mode 100644 src/include/port/atomics/stdatomic_impl.h create mode 100644 src/include/port/spin_delay.h create mode 100644 src/include/port/spin_delay_status.h diff --git a/configure b/configure index 35b0b72f0a70f..f9853829bd356 100755 --- a/configure +++ b/configure @@ -759,6 +759,7 @@ LLVM_LIBS CLANG LLVM_CONFIG AWK +with_stdatomic with_llvm have_cxx ac_ct_CXX @@ -857,6 +858,7 @@ with_segsize with_segsize_blocks with_wal_blocksize with_llvm +with_stdatomic enable_depend enable_cassert with_icu @@ -1576,6 +1578,8 @@ Optional Packages: --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] --with-llvm build with LLVM based JIT support + --with-stdatomic[=yes/no/auto] + use C11 stdatomic.h for atomic operations [auto] --without-icu build without ICU support --with-tcl build Tcl modules (PL/Tcl) --with-tclconfig=DIR tclConfig.sh is in DIR @@ -4917,6 +4921,26 @@ fi + +# +# C11 stdatomic.h +# +# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses +# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the +# traditional platform-specific atomics. The actual detection runs later, +# after the compiler and 64-bit integer handling have been configured. + +# Check whether --with-stdatomic was given. +if test "${with_stdatomic+set}" = set; then : + withval=$with_stdatomic; case $withval in + yes | no | auto) ;; + *) as_fn_error $? "invalid argument to --with-stdatomic; use yes, no, or auto" "$LINENO" 5 ;; + esac +else + with_stdatomic=no +fi + + for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17718,6 +17742,91 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi +# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The +# probe mirrors the one used by the meson build: it must compile 32-bit +# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. +if test "$with_stdatomic" != no; then + # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() + # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit + # or weakly-aligned targets. A compile-only check would pass there and then + # fail at final link. Retry with -latomic if the bare link fails. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +$as_echo_n "checking for working C11 stdatomic.h... " >&6; } +if ${pgac_cv_stdatomic+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_cv_stdatomic=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic=yes +else + pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic='yes, with -latomic' +else + pgac_cv_stdatomic=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS=$pgac_save_LIBS +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 +$as_echo "$pgac_cv_stdatomic" >&6; } +else + pgac_cv_stdatomic=no +fi + +if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found" "$LINENO" 5 +fi + +if test "$pgac_cv_stdatomic" != no; then + +$as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h + + if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" + fi +fi + + # Check for __get_cpuid() and __cpuid() { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __get_cpuid" >&5 $as_echo_n "checking for __get_cpuid... " >&6; } diff --git a/configure.ac b/configure.ac index 0e624fe36b96d..a3d4e31a60947 100644 --- a/configure.ac +++ b/configure.ac @@ -443,6 +443,23 @@ choke me PGAC_ARG_BOOL(with, llvm, no, [build with LLVM based JIT support], [AC_DEFINE([USE_LLVM], 1, [Define to 1 to build with LLVM based JIT support. (--with-llvm)])]) AC_SUBST(with_llvm) + +# +# C11 stdatomic.h +# +# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses +# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the +# traditional platform-specific atomics. The actual detection runs later, +# after the compiler and 64-bit integer handling have been configured. +AC_ARG_WITH(stdatomic, + [AS_HELP_STRING([--with-stdatomic@<:@=yes/no/auto@:>@], + [use C11 stdatomic.h for atomic operations @<:@auto@:>@])], + [case $withval in + yes | no | auto) ;; + *) AC_MSG_ERROR([invalid argument to --with-stdatomic; use yes, no, or auto]) ;; + esac], + [with_stdatomic=no]) +AC_SUBST(with_stdatomic) dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2101,6 +2118,55 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS +# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The +# probe mirrors the one used by the meson build: it must compile 32-bit +# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. +if test "$with_stdatomic" != no; then + # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() + # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit + # or weakly-aligned targets. A compile-only check would pass there and then + # fail at final link. Retry with -latomic if the bare link fails. + AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], + [pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) +else + pgac_cv_stdatomic=no +fi + +if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found]) +fi + +if test "$pgac_cv_stdatomic" != no; then + AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations. (--with-stdatomic)]) + if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" + fi +fi + + # Check for __get_cpuid() and __cpuid() AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], diff --git a/meson.build b/meson.build index d88a7a7030878..f4e72386deb7d 100644 --- a/meson.build +++ b/meson.build @@ -674,6 +674,108 @@ if have_cxx and not cxx.compiles(cxx11_test, name: 'C++11') endif +############################################################### +# Check for C11 stdatomic.h support +############################################################### + +stdatomic_test_code = ''' +#include +int main(void) { + _Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + return 0; +} +''' + +has_stdatomic = false +use_stdatomic = get_option('use_stdatomic') +atomic_link_args = [] + +# The stdatomic probe below is a *link* test, not just a compile test: 64-bit +# atomic_compare_exchange_strong() can lower to a libatomic call +# (__atomic_compare_exchange_8) on some 32-bit or weakly-aligned targets. A +# compile-only check would pass there and then fail at final link, so we +# require the probe to link, retrying with -latomic if the bare link fails. + +if cc.get_id() == 'msvc' + msvc_ver = cc.version() + if msvc_ver.version_compare('>=19.30') # Visual Studio 2022+ + message('MSVC version @0@ detected (>= 19.30), checking for stdatomic.h support'.format(msvc_ver)) + # MSVC's requires C11 mode (/std:c11) in addition to + # /experimental:c11atomics; without /std:c11 the header errors out with + # "C atomics require C11 or later". The probe must pass both, since + # test_c_args is not yet populated with the C11 flag at this point. + if cc.links(stdatomic_test_code, + name: 'MSVC stdatomic.h with /experimental:c11atomics', + args: test_c_args + ['/std:c11', '/experimental:c11atomics']) + has_stdatomic = true + message('MSVC stdatomic.h is available with /experimental:c11atomics') + else + message('MSVC stdatomic.h test compilation failed') + endif + else + message('MSVC version @0@ is too old for stdatomic.h (requires >= 19.30)'.format(msvc_ver)) + endif +else + # GCC/Clang and other compilers + if cc.has_header('stdatomic.h', args: test_c_args) + message('stdatomic.h header found, testing that it links...') + if cc.links(stdatomic_test_code, + name: 'stdatomic.h works', + args: test_c_args) + has_stdatomic = true + message('stdatomic.h is available and working') + elif cc.links(stdatomic_test_code, + name: 'stdatomic.h works with -latomic', + args: test_c_args + ['-latomic']) + # Some targets need libatomic for the wider (e.g. 64-bit) operations. + has_stdatomic = true + atomic_link_args = ['-latomic'] + message('stdatomic.h is available and working (requires -latomic)') + else + message('stdatomic.h found but test link failed') + endif + else + message('stdatomic.h header not found') + endif +endif + +# Process the use_stdatomic option +if use_stdatomic == 'yes' + if not has_stdatomic + error('stdatomic.h was requested with -Duse_stdatomic=yes but is not available or not working') + endif + cdata.set('USE_STDATOMIC_H', 1) + message('Using C11 stdatomic.h for atomic operations (forced by -Duse_stdatomic=yes)') + if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] + endif + if atomic_link_args.length() > 0 + os_deps += cc.find_library('atomic') + endif +elif use_stdatomic == 'no' + message('Using traditional platform-specific atomics (forced by -Duse_stdatomic=no)') + # Do not set USE_STDATOMIC_H +elif use_stdatomic == 'auto' + if has_stdatomic + cdata.set('USE_STDATOMIC_H', 1) + message('Using C11 stdatomic.h for atomic operations (auto-detected)') + if cc.get_id() == 'msvc' + cflags += ['/std:c11', '/experimental:c11atomics'] + endif + if atomic_link_args.length() > 0 + os_deps += cc.find_library('atomic') + endif + else + message('Using traditional platform-specific atomics (stdatomic.h not available)') + endif +endif + postgres_inc = [include_directories(postgres_inc_d)] test_lib_d = postgres_lib_d test_c_args = cppflags + cflags diff --git a/meson_options.txt b/meson_options.txt index 6a793f3e47943..f8c5ccc14e114 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,6 +52,11 @@ option('PG_TEST_EXTRA', type: 'string', value: '', option('PG_GIT_REVISION', type: 'string', value: 'HEAD', description: 'git revision to be packaged by pgdist target') +option('use_stdatomic', type: 'combo', + choices: ['auto', 'yes', 'no'], + value: 'no', + description: 'Use C11 stdatomic.h for atomic operations (auto, yes, no)') + # Compilation options diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 45de0fd2bd6f0..460b955944959 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -47,6 +47,77 @@ when the wait time might exceed a few seconds. The rest of this README file discusses the regular lock manager in detail. +Atomic Operations +================= + +PostgreSQL's atomic operations (see src/include/port/atomics.h) can be +implemented using either C11 stdatomic.h or traditional platform-specific +code. The implementation is selected at build time via the USE_STDATOMIC_H +preprocessor definition: + +* C11 stdatomic.h implementation (USE_STDATOMIC_H defined): + Uses the standard C11 header for all atomic operations. + This provides better portability and potentially better compiler + optimizations. Available on: + - MSVC 2022+ (requires /experimental:c11atomics flag) + - GCC 4.9+ and Clang 3.1+ (no special flags needed) + +* Traditional implementation (USE_STDATOMIC_H not defined): + Uses platform-specific implementations that have been battle-tested + in PostgreSQL for many years: + - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h + - Compiler intrinsics: generic-gcc.h, generic-msvc.h + - Fallback implementations: generic.h, fallback.h + +Both implementations provide the same public API and observable semantics. +The choice is made at build configuration time: + + meson setup build -Duse_stdatomic=auto # Auto-detect (default) + meson setup build -Duse_stdatomic=yes # Force stdatomic.h + meson setup build -Duse_stdatomic=no # Force traditional + +The two implementations differ internally in several ways that do not affect +the public API: + +* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering under + stdatomic.h; pg_atomic_write_u32/u64() use relaxed. The seq_cst read was + required for correctness on weak-memory hardware (e.g. RISC-V), where the + weaker relaxed and acquire orderings let a concurrent reader miss a tuple + in a parallel hash join. The write stays relaxed: a plain atomic store + matches the traditional "no barrier" contract for pg_atomic_write and, on + ARM64, avoids the store-release (STLR) serialization that would penalize + contended writes to hot shared state. The pg_atomic_unlocked_write_* + variant also uses relaxed ordering, consistent with its "no guarantees" + contract. On x86 (TSO) a seq_cst load is still a plain mov, so there is no + added cost; + on ARM/RISC-V it adds fence instructions on these paths. + +* Barriers. Each barrier pairs a compiler barrier (atomic_signal_fence) with + the thread fence (atomic_thread_fence), because a bare thread fence orders + only atomic accesses and would let the compiler reorder plain loads/stores + across it, whereas PostgreSQL's barrier contract must order non-atomic + accesses too. This matches the traditional generic-gcc.h barriers. + +* Spinlock flag polarity. The stdatomic.h pg_atomic_flag uses 1=unlocked, + 0=locked (fetch_and-based test-and-set), the inverse of the traditional + 0=unlocked, 1=locked (exchange-based). This is an internal detail with no + external visibility beyond raw values seen in a debugger. + +* Binary compatibility. The atomic types are _Atomic(...) under stdatomic.h + and volatile-struct under the traditional path. The two are not binary + compatible, so objects and shared memory cannot be mixed between builds; + the implementation is fixed at build time, so this is not a concern in + practice. + +* Platform reach. Under stdatomic.h, 64-bit atomics are available on 32-bit + ARM (compiler/runtime may emulate them with locks), atomics are usable from + frontend programs, and the headers are C++-compatible. The traditional + path disables 64-bit atomics on ARM32 and is backend-only. + +For more information on memory barriers and atomic operations, see +src/backend/storage/lmgr/README.barrier. + + Lock Data Structures -------------------- diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index 6df568eccb35a..b3a11fe9fc903 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,7 +51,13 @@ #include #include "common/pg_prng.h" +#ifdef USE_STDATOMIC_H +#include "port/atomics.h" +#include "port/spin_delay.h" +#include "storage/spin.h" +#else #include "storage/s_lock.h" +#endif #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -60,6 +66,7 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L +#ifndef USE_STDATOMIC_H #ifdef S_LOCK_TEST /* * These are needed by pgstat_report_wait_start in the standalone compile of @@ -68,6 +75,7 @@ static uint32 local_my_wait_event_info; uint32 *my_wait_event_info = &local_my_wait_event_info; #endif +#endif /* !USE_STDATOMIC_H */ static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -80,7 +88,7 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if defined(S_LOCK_TEST) +#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) fprintf(stderr, "\nStuck spinlock detected at %s, %s:%d.\n", func, file, line); @@ -101,16 +109,39 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); +#ifdef USE_STDATOMIC_H + for (;;) + { + bool try_to_set; + +#ifdef PG_SPIN_TRY_RELAXED + /* + * Test-and-test-and-set: read the lock first and only attempt the + * exchange when it looks free, to avoid taking cache-line ownership on + * every spin. PG_SPIN_TRY_RELAXED is defined only where this pre-read + * is cheap (x86/x86_64 TSO, where a seq_cst load is a plain load). + */ + try_to_set = (pg_atomic_read_u32(lock) == 0); +#else + try_to_set = true; +#endif + if (try_to_set && pg_atomic_exchange_u32(lock, 1) == 0) + break; + perform_spin_delay(&delayStatus); + } +#else /* !USE_STDATOMIC_H */ while (TAS_SPIN(lock)) { perform_spin_delay(&delayStatus); } +#endif /* USE_STDATOMIC_H */ finish_spin_delay(&delayStatus); return delayStatus.delays; } +#ifndef USE_STDATOMIC_H #ifdef USE_DEFAULT_S_UNLOCK void s_unlock(volatile slock_t *lock) @@ -118,6 +149,7 @@ s_unlock(volatile slock_t *lock) *lock = 0; } #endif +#endif /* !USE_STDATOMIC_H */ /* * Wait while spinning on a contended spinlock. @@ -126,7 +158,11 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ +#ifdef USE_STDATOMIC_H + pg_spin_delay(); +#else SPIN_DELAY(); +#endif /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -149,7 +185,7 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if defined(S_LOCK_TEST) +#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) fprintf(stdout, "*"); fflush(stdout); #endif @@ -232,6 +268,7 @@ update_spins_per_delay(int shared_spins_per_delay) /*****************************************************************************/ +#ifndef USE_STDATOMIC_H #if defined(S_LOCK_TEST) /* @@ -298,3 +335,4 @@ main() } #endif /* S_LOCK_TEST */ +#endif /* !USE_STDATOMIC_H */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 4f8113c144b0c..27dd89765e43a 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -747,6 +747,10 @@ /* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK +/* Define to 1 to use C11 stdatomic.h for atomic operations. + (--with-stdatomic) */ +#undef USE_STDATOMIC_H + /* Define to 1 to use SVE popcount instructions with a runtime check. */ #undef USE_SVE_POPCNT_WITH_RUNTIME_CHECK diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index a605ea81d0760..2058dbb9238c7 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,6 +7,26 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * + * IMPLEMENTATION SELECTION: + * + * PostgreSQL's atomic operations can be implemented using either: + * + * 1. C11 stdatomic.h (when USE_STDATOMIC_H is defined) + * - Uses standard C11 for all atomic operations + * - Better portability and compiler optimizations + * - Requires MSVC 2022+ or GCC 4.9+/Clang 3.1+ + * + * 2. Traditional platform-specific implementations (when USE_STDATOMIC_H is not defined) + * - Uses battle-tested PostgreSQL implementations + * - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h + * - Compiler intrinsics: generic-gcc.h, generic-msvc.h + * - Fallback implementations: generic.h, fallback.h + * + * Both implementations provide identical public API and semantics. + * Selection is made at build time via -Duse_stdatomic=auto/yes/no. + * + * PORTING NOTES: + * * To bring up postgres on a platform/compiler at the very least * implementations for the following operations should be provided: * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() @@ -14,13 +34,8 @@ * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. * - * There exist generic, hardware independent, implementations for several - * compilers which might be sufficient, although possibly not optimal, for a - * new platform. If no such generic implementation is available spinlocks will - * be used to implement the 64-bit parts of the API. - * - * Implement _u64 atomics if and only if your platform can use them - * efficiently (and obviously correctly). + * For new platforms, prefer using stdatomic.h if available, as it reduces + * maintenance burden and leverages compiler-provided implementations. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -38,15 +53,76 @@ #ifndef ATOMICS_H #define ATOMICS_H +/* + * Frontend code restriction + * + * Traditionally, atomics.h could not be included from frontend code because + * the platform-specific implementations often relied on backend-only features. + * + * With the stdatomic.h implementation, this restriction is no longer necessary + * since stdatomic.h is a standard header. However, we keep the restriction for + * the traditional implementation path to maintain compatibility. + */ #ifdef FRONTEND -#error "atomics.h may not be included from frontend code" +#ifndef USE_STDATOMIC_H +#error "atomics.h may not be included from frontend code (use -Duse_stdatomic=yes if atomics are needed)" +#endif #endif #define INSIDE_ATOMICS_H +/* + * The public API and the _impl signatures keep the historical volatile + * qualifier on the atomic pointer, because many callers reach atomic fields + * through volatile-qualified struct pointers. Under C11 the _Atomic types + * carry their own ordering guarantees, so the volatile is redundant for the + * atomic accesses themselves and is accepted (and effectively ignored for the + * atomic operation) by the supported compilers, including MSVC. + */ + #include /* + * PostgreSQL atomics can be implemented using either C11 stdatomic.h + * or platform-specific implementations. The choice is made at build + * configuration time via the USE_STDATOMIC_H preprocessor definition. + * + * When USE_STDATOMIC_H is defined, we use C11 for all + * atomic operations. This provides better portability and potentially + * better compiler optimizations. + * + * When USE_STDATOMIC_H is not defined (default initially), we use + * traditional platform-specific implementations (arch-*.h, generic-*.h) + * that have been battle-tested in PostgreSQL for many years. + * + * Both paths provide identical public API and semantics. + */ + +#ifdef USE_STDATOMIC_H + +/* + * C11 stdatomic.h implementation path + * + * This uses the standard C11 header for atomic operations. + * The type definitions and implementations are in stdatomic_impl.h. + */ + +#include "port/atomics/stdatomic_impl.h" + +/* + * The public API is provided by static inline functions in the common code + * section below. Those functions call _impl functions, which are provided by + * stdatomic_impl.h for this path. + */ + +#else /* !USE_STDATOMIC_H */ + +/* + * Traditional platform-specific implementation path + * + * This is the original PostgreSQL atomics implementation using + * architecture-specific headers and compiler intrinsics. + * * First a set of architecture specific files is included. * * These files can provide the full set of atomics or can do pretty much @@ -116,6 +192,17 @@ */ #include "port/atomics/generic.h" +#endif /* USE_STDATOMIC_H */ + +/* + * Common code for both stdatomic.h and traditional implementations. + * + * The following definitions provide the public API that works identically + * regardless of which implementation is used. The static inline functions + * below call _impl functions which are provided by either stdatomic_impl.h + * (when USE_STDATOMIC_H is defined) or the traditional implementation files + * (generic.h, etc.) otherwise. + */ /* * pg_compiler_barrier - prevent the compiler from moving code across diff --git a/src/include/port/atomics/stdatomic_impl.h b/src/include/port/atomics/stdatomic_impl.h new file mode 100644 index 0000000000000..0bda02d16f419 --- /dev/null +++ b/src/include/port/atomics/stdatomic_impl.h @@ -0,0 +1,589 @@ +/*------------------------------------------------------------------------- + * + * stdatomic_impl.h + * Atomic operations implementation using C11 stdatomic.h + * + * This file provides PostgreSQL atomic operations using the C11 standard + * . It is only included when USE_STDATOMIC_H is defined. + * + * The traditional platform-specific implementation (arch-*.h, generic-*.h, + * fallback.h) remains available and is used when stdatomic.h is not + * available or when explicitly requested via -Duse_stdatomic=no. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/atomics/stdatomic_impl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STDATOMIC_IMPL_H +#define STDATOMIC_IMPL_H + +/* + * Only include this file when USE_STDATOMIC_H is defined at build time. + * This ensures the traditional implementation remains available as fallback. + */ +#ifdef USE_STDATOMIC_H + +/* + * C++ Compatibility + * + * C++11 through C++20 cannot include C's directly. Instead, + * we use C++11's header and map to std::atomic types. C++23 allows + * but we handle older versions. + * + * This follows the approach from C++23 standard section 33.5.12 which + * allows C and C++ atomic types to interoperate. + */ +#if defined(__cplusplus) && __cplusplus < 202302L +extern "C++" +{ +#include + + /* Map to C++ atomic types */ +#define pg_atomic(T) std::atomic + + /* Import memory order constants into global namespace */ + using std::memory_order_relaxed; + using std::memory_order_acquire; + using std::memory_order_release; + using std::memory_order_acq_rel; + using std::memory_order_seq_cst; +} +#else +/* C11 or C++23+: use standard */ +#include +#define pg_atomic(T) _Atomic(T) +#endif + +/* + * Type Definitions + * + * We use C11 _Atomic qualifier (or C++ std::atomic) with PostgreSQL's + * standard integer types. These types are compatible with atomic operations + * and provide sequential consistency guarantees. + * + * Note: pg_atomic(T) is defined above and maps to either _Atomic(T) for C + * or std::atomic for C++. + */ +/* + * pg_atomic_flag is backed by a full 32-bit word, not uint8. On + * architectures without byte-granular atomics (notably RISC-V, whose base + * 'A' extension has only word/doubleword AMOs), a sub-word _Atomic(uint8) + * RMW is emulated by the compiler as a load-reserved/store-conditional of + * the *entire containing 32-bit word*. Because slock_t (a pg_atomic_flag) + * is frequently packed next to other fields, that word-wide RMW can clobber + * the neighbouring bytes, corrupting adjacent shared state under contention. + * Using a 32-bit word makes every flag operation a clean word-aligned AMO + * that never touches neighbouring memory, matching the traditional + * implementation, whose pg_atomic_flag is also 32 bits (generic.h). + */ +typedef pg_atomic(uint32) pg_atomic_flag; +typedef pg_atomic(uint8) pg_atomic_uint8; +typedef pg_atomic(uint16) pg_atomic_uint16; +typedef pg_atomic(uint32) pg_atomic_uint32; +typedef pg_atomic(uint64) pg_atomic_uint64; + +/* + * PostgreSQL's atomics live in shared memory and are accessed from multiple + * *processes*, not just threads. A non-lock-free atomic implemented by the + * compiler/runtime with a lock table (e.g. libatomic) uses process-local + * locks, which would silently fail to provide atomicity across processes and + * corrupt shared state. Require genuine, always-lock-free support at compile + * time for every width we use. (ATOMIC_*_LOCK_FREE is 0 = never lock-free, + * 1 = sometimes / not known at compile time, 2 = always lock-free.) + * + * The == 1 case is a real hazard, not just conservatism: e.g. 64-bit atomics + * on a 32-bit target may be emulated by libatomic with a process-local lock, + * which corrupts cross-process shared memory exactly as == 0 would. So we + * require == 2 everywhere the macro is trustworthy. + * + * The one exception is MSVC compiling as C: its reports + * ATOMIC_*_LOCK_FREE == 1 for every width even though the atomics are in fact + * always lock-free on the hardware we target (in C++ mode the same MSVC + * reports 2, and atomic_is_lock_free() returns true at run time). We skip the + * assertion only in that specific configuration rather than weakening it for + * everyone. + */ +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +StaticAssertDecl(ATOMIC_CHAR_LOCK_FREE == 2, "8-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_SHORT_LOCK_FREE == 2, "16-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_INT_LOCK_FREE == 2, "32-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_LLONG_LOCK_FREE == 2, "64-bit atomics are not lock-free"); +#endif + +/* + * Signal atomic operation support to atomics.h + * + * stdatomic.h provides native support for all atomic types on all platforms + * it compiles on, so we unconditionally define these. PG_HAVE_ATOMIC_U64_SIMULATION + * is intentionally NOT defined -- stdatomic provides real 64-bit atomics. + */ +#define PG_HAVE_ATOMIC_U32_SUPPORT +#define PG_HAVE_ATOMIC_U64_SUPPORT + +/* + * 8-byte single-copy atomicity. + * + * All 64-bit architectures that support C11 stdatomic.h guarantee 8-byte + * single-copy atomicity for aligned loads/stores. This is used by bufmgr.c + * and other performance-critical paths to avoid unnecessary atomic operations. + */ +#if SIZEOF_VOID_P >= 8 +#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY +#endif + +/* + * Memory Barrier Implementation + * + * PostgreSQL's memory barriers must order *non-atomic, non-volatile* accesses + * too, which a bare C11 atomic_thread_fence() does NOT guarantee against + * compiler reordering: atomic_thread_fence only establishes ordering with + * respect to atomic operations, so the compiler is still free to move plain + * loads/stores across it. We therefore pair each thread fence with an + * explicit compiler barrier (atomic_signal_fence), matching the semantics of + * the traditional implementation whose barriers also constrain plain memory + * accesses. Omitting the compiler barrier corrupts lock-free algorithms on + * weak-memory architectures (observed as lost tuples in parallel hash join on + * RISC-V, where the compiler reordered non-atomic hash-table stores across a + * bare acquire/release fence). + * + * These are implementation functions (_impl suffix) mapped to the public API + * by atomics.h. + */ +#define pg_compiler_barrier_impl() atomic_signal_fence(memory_order_seq_cst) + +static inline void +pg_memory_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_seq_cst); +} + +static inline void +pg_read_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_acquire); +} + +static inline void +pg_write_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_release); +} + +/* + * Spin Delay Implementation + * + * On x86, a spinloop without a "pause" instruction can waste a lot of power + * and slow down the adjacent hyperthread. On ARM64, ISB provides better + * backoff than YIELD. This provides platform-specific spin delay hints. + * + * The implementation is in spin_delay.h which directly defines pg_spin_delay_impl(), + * matching the pattern used by the traditional atomics implementation. + */ +#include "port/spin_delay.h" + +/* + * ================================================================= + * pg_atomic_flag Operations + * ================================================================= + * + * Atomic flag type used primarily for spinlocks. + * + * CRITICAL FLAG CONVENTION DIFFERENCE FROM TRADITIONAL IMPLEMENTATION: + * + * stdatomic.h implementation (this file): + * 1 = unlocked, 0 = locked + * + * Traditional implementation (generic.h): + * 0 = unlocked, 1 = locked + * + * This polarity difference is INTENTIONAL and internal to the implementation. + * The public API behavior is identical in both implementations: + * - pg_atomic_init_flag() initializes to unlocked state + * - pg_atomic_test_set_flag() returns true if lock was successfully acquired + * - pg_atomic_clear_flag() releases the lock + * + * The stdatomic.h convention (1=unlocked) is more natural for C11 atomics + * and avoids double-negation logic in the implementation. + */ + +/* + * pg_atomic_init_flag_impl - Initialize atomic flag to unlocked state + * + * In the stdatomic.h implementation, we initialize to 1 (unlocked). + */ +static inline void +pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_init(ptr, 1); /* 1 = unlocked */ +} + +/* + * pg_atomic_test_set_flag_impl - Try to acquire lock + * + * Atomically clears the flag (AND with 0) and returns previous value. + * Returns true if we successfully acquired the lock (old value was 1, + * meaning unlocked), false if already locked (old value was 0). + * + * Memory ordering: acquire (synchronizes-with prior release) + */ +static inline bool +pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) +{ + /* + * AND flag with 0 to clear it (locked). If previous value was 1 + * (unlocked), we succeeded in acquiring. If previous value was 0 + * (already locked), we failed. + * + * Return true if we SUCCEEDED in acquiring the lock. + */ + return atomic_fetch_and_explicit(ptr, 0, memory_order_acquire) != 0; +} + +/* + * pg_atomic_unlocked_test_flag_impl - Test if flag is unlocked without acquiring + * + * Returns true if the flag is currently unlocked (value == 1). + * Uses relaxed ordering since this is typically used for optimization hints. + */ +static inline bool +pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) +{ + return atomic_load_explicit(ptr, memory_order_relaxed) != 0; +} + +/* + * pg_atomic_clear_flag_impl - Release lock + * + * Sets flag to 1 (unlocked). + * Memory ordering: release (makes prior writes visible) + */ +static inline void +pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_store_explicit(ptr, 1, memory_order_release); +} + +/* + * ================================================================= + * pg_atomic_uint32 Operations + * ================================================================= + * + * 32-bit atomic unsigned integer operations. + * All operations use sequential consistency unless otherwise noted. + */ + +/* + * pg_atomic_init_u32_impl - Initialize atomic uint32 + * + * This is not an atomic operation - must only be called during initialization + * when no concurrent access is possible. + */ +static inline void +pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_init(ptr, val); +} + +/* + * pg_atomic_read_u32_impl - Atomically read uint32 value + * + * Memory ordering: sequentially consistent. This is stronger than the + * documented "no barrier semantics" of pg_atomic_read_u32(), and it is + * deliberate: a bare memory_order_relaxed load reintroduces a real, + * reproducible data-loss bug on weak-memory hardware. + * + * The traditional implementation reads through a volatile-qualified pointer. + * A volatile access cannot be reordered *by the compiler* relative to the + * surrounding code, so consumers that publish a pointer with a + * barrier-carrying store and then read+dereference it get the dependent load + * ordered for free. A C11 memory_order_relaxed atomic load does not carry + * that compiler-ordering property, and the parallel hash join relies on it: + * ExecParallelHash reads a freshly published bucket pointer via + * dsa_pointer_atomic_read() and then dereferences the tuple it points to. + * With a relaxed load this was observed to lose a tuple on RISC-V -- + * join_hash "extremely_skewed" returns 19999 instead of 20000, intermittently + * (reproduced here at roughly 1 run in 4 on real rv64 hardware). seq_cst + * restores correctness. + * + * The cost of the seq_cst *load* is not the ARM regression that was measured + * for an earlier revision of this work: microbenchmarking on Graviton showed + * an uncontended ldar is not measurably costlier than a plain ldr, and the + * ~2% aarch64 read-only regression seen previously was traced to the seq_cst + * *store* (STLR) in pg_atomic_write_u32, which is now relaxed (see below). + * So the ordered read is kept for correctness while the write, which was the + * actual measured cost, is not paid. + * + * A cleaner long-term fix is to leave this primitive relaxed (matching the + * documented contract) and give the specific hash-join consumer the ordering + * it needs -- e.g. an acquire read at the call site or + * dsa_pointer_atomic_read_membarrier(). That is a change to core executor / + * dsa code and is intentionally out of scope for the atomics port; it is + * noted as follow-up work rather than folded in here. + */ +static inline uint32 +pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +/* + * pg_atomic_write_u32_impl - Atomically write uint32 value + * + * Memory ordering: relaxed. A plain atomic (non-torn) store matches the + * traditional implementation's "no barrier" contract for pg_atomic_write_u32 + * and, on ARM64, compiles to a plain STR rather than STLR. This matters: + * benchmarking traced the ~2% aarch64 read-only regression of an earlier + * seq_cst-everywhere revision to STLR store-side serialization on hot, + * contended shared lines (LWLock state, buffer headers), not to the reads. + * Making the write relaxed removes that cost. Callers needing an ordered + * write use pg_atomic_write_membarrier_u32() or an explicit barrier. + */ +static inline void +pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_exchange_u32_impl - Atomically exchange uint32 value + * + * Atomically replaces the value with newval and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +/* + * pg_atomic_compare_exchange_u32_impl - Atomic compare-and-swap + * + * Compares *ptr with *expected. If equal, replaces *ptr with newval and + * returns true. If not equal, updates *expected with current *ptr value + * and returns false. + * + * This is the strong version (no spurious failures). + * Memory ordering: seq_cst for both success and failure + */ +static inline bool +pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_add_u32_impl - Atomic fetch-and-add + * + * Atomically adds add_ to *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_sub_u32_impl - Atomic fetch-and-subtract + * + * Atomically subtracts sub_ from *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_and_u32_impl - Atomic fetch-and-AND + * + * Atomically performs *ptr &= and_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_or_u32_impl - Atomic fetch-and-OR + * + * Atomically performs *ptr |= or_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +/* + * pg_atomic_add_fetch_u32_impl - Atomic add-and-fetch + * + * Atomically adds add_ to *ptr and returns the NEW value. + * Implemented using fetch_add + add_ since C11 doesn't have add_fetch. + */ +static inline uint32 +pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +/* + * pg_atomic_sub_fetch_u32_impl - Atomic subtract-and-fetch + * + * Atomically subtracts sub_ from *ptr and returns the NEW value. + */ +static inline uint32 +pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +/* + * pg_atomic_read_membarrier_u32_impl - Read with full memory barrier + * + * Atomic read with sequential consistency. Useful for cases where + * correctness is more important than performance. + */ +static inline uint32 +pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +/* + * pg_atomic_unlocked_write_u32_impl - Non-atomic write + * + * Write without atomicity guarantees. Only safe when exclusive access + * is guaranteed externally. Uses relaxed ordering. + */ +static inline void +pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_write_membarrier_u32_impl - Write with full memory barrier + * + * Atomic write with sequential consistency. + */ +static inline void +pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_seq_cst); +} + +/* + * ================================================================= + * pg_atomic_uint64 Operations + * ================================================================= + * + * 64-bit equivalents of the u32 operations above. See the u32 comments + * for API documentation; semantics and memory ordering are identical. + */ + +static inline void +pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_init(ptr, val); +} + +static inline uint64 +pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +static inline void +pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline uint64 +pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +static inline bool +pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +static inline uint64 +pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +static inline uint64 +pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +static inline void +pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline void +pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_seq_cst); +} + +#endif /* USE_STDATOMIC_H */ + +#endif /* STDATOMIC_IMPL_H */ diff --git a/src/include/port/spin_delay.h b/src/include/port/spin_delay.h new file mode 100644 index 0000000000000..5b3d45587a8cd --- /dev/null +++ b/src/include/port/spin_delay.h @@ -0,0 +1,151 @@ +/*------------------------------------------------------------------------- + * + * spin_delay.h + * Platform-specific spin delay for busy-wait loops + * + * This file provides pg_spin_delay(), a platform-optimized delay instruction + * for use in spinlock contention loops. Different architectures have different + * optimal instructions for indicating to the CPU that we're in a busy-wait. + * + * Key optimizations: + * - x86/x86_64: PAUSE instruction (rep nop) reduces power and helps hyperthreads + * - ARM64: ISB instruction, per the discussion thread below, backs off better + * than YIELD under heavy spinlock contention on high-core-count parts + * - Windows ARM64: __isb() intrinsic (same rationale as ARM64 above) + * + * Discussion: https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_H +#define SPIN_DELAY_H + +#ifdef _MSC_VER +/* + * MSVC intrinsics used below (__isb, _mm_pause, _ReadWriteBarrier) are declared + * in , which also provides 's _mm_pause. Include it at + * file scope rather than relying on another header having pulled it in first. + */ +#include +#endif + +/* + * pg_spin_delay_impl - Execute a platform-specific CPU hint for spin-wait loops + * + * This function should be called inside busy-wait loops to: + * 1. Reduce CPU power consumption during contention + * 2. Improve performance of sibling hyperthreads + * 3. Signal to the CPU that we're in a spin loop + * + * The implementation varies by platform to use the most efficient instruction. + * + * Note: This defines pg_spin_delay_impl() directly, matching the pattern used + * by the traditional atomics implementation (arch-*.h files). + */ +#ifndef PG_HAVE_SPIN_DELAY +#define PG_HAVE_SPIN_DELAY +static inline void +pg_spin_delay_impl(void) +{ +#if defined(__GNUC__) || defined(__INTEL_COMPILER) + + /* + * GCC and Intel compiler: use inline assembly for optimal instructions + */ + +#if defined(__i386__) || defined(__i386) + /* x86 32-bit: PAUSE instruction (encoded as rep nop) */ + __asm__ __volatile__(" rep; nop \n"); +#elif defined(__x86_64__) + /* x86-64: PAUSE instruction */ + __asm__ __volatile__(" rep; nop \n"); +#elif defined(__aarch64__) + /* + * ARM64: ISB (Instruction Synchronization Barrier). Per the discussion + * thread above, ISB backs off better than YIELD in spinlock loops on + * high-core-count ARM64 parts; it forces a pipeline flush that YIELD does + * not. + */ + __asm__ __volatile__(" isb; \n"); +#elif defined(__arm__) || defined(__arm) + /* ARM 32-bit: YIELD hint */ + __asm__ __volatile__(" yield; \n"); +#else + /* + * Other architectures: compiler barrier only + * + * A compiler barrier prevents the compiler from optimizing away the loop, + * even if we don't have an architecture-specific delay instruction. + */ + __asm__ __volatile__("":::"memory"); +#endif + +#elif defined(_MSC_VER) + + /* + * Microsoft Visual C++: use intrinsics (declared in , included + * at file scope above). + */ + +#if defined(_M_ARM64) || defined(_M_ARM64EC) + /* + * Windows ARM64: ISB via intrinsic, matching the GCC/Clang ARM64 path + * above. _ARM64_BARRIER_SY is a full system barrier. + */ + __isb(_ARM64_BARRIER_SY); +#elif defined(_M_AMD64) + /* Windows x86-64: _mm_pause() maps to the PAUSE instruction. */ + _mm_pause(); +#elif defined(_M_IX86) + /* Windows x86 32-bit: MASM syntax for PAUSE (rep nop). */ + __asm rep nop; +#else + /* + * Other Windows architectures: compiler barrier only, matching the + * unknown-compiler fallback below. + */ + _ReadWriteBarrier(); +#endif + +#else + /* + * Unknown compiler: no-op with compiler barrier + * + * At minimum, we need to prevent the compiler from optimizing away the + * spin loop. + */ + (void) 0; +#endif +} +#endif /* PG_HAVE_SPIN_DELAY */ + +/* + * Public spin-delay macro for the stdatomic path. + * + * This is the public pg_spin_delay() entry point used by the stdatomic + * spinlock path; it maps to the pg_spin_delay_impl() defined above. + */ +#ifndef pg_spin_delay +#define pg_spin_delay() pg_spin_delay_impl() +#endif + +/* + * Architectures where a plain load before the atomic exchange reduces + * cache-coherency traffic under spinlock contention (test-and-test-and-set). + * + * Restricted to x86/x86_64: these are TSO and pg_atomic_read_u32() (seq_cst) + * compiles to an ordinary load there, so the pre-read is genuinely cheap. On + * weakly-ordered targets a seq_cst load emits fence instructions, which would + * defeat the point of a cheap pre-check, so they are intentionally omitted. + */ +#if defined(__i386__) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_AMD64) +#define PG_SPIN_TRY_RELAXED +#endif + +#endif /* SPIN_DELAY_H */ diff --git a/src/include/port/spin_delay_status.h b/src/include/port/spin_delay_status.h new file mode 100644 index 0000000000000..738ecb12a4457 --- /dev/null +++ b/src/include/port/spin_delay_status.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * spin_delay_status.h + * SpinDelayStatus type and the spin-delay/backoff helpers + * + * This header holds the delay/backoff bookkeeping used by the spinlock slow + * path (perform_spin_delay / finish_spin_delay in s_lock.c) and by other + * spinlock-like busy-wait loops. It is included by storage/spin.h so the + * definitions live in exactly one place. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay_status.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_STATUS_H +#define SPIN_DELAY_STATUS_H + +#define DEFAULT_SPINS_PER_DELAY 100 + +typedef struct +{ + int spins; + int delays; + int cur_delay; + const char *file; + int line; + const char *func; +} SpinDelayStatus; + +static inline void +init_spin_delay(SpinDelayStatus *status, + const char *file, int line, const char *func) +{ + status->spins = 0; + status->delays = 0; + status->cur_delay = 0; + status->file = file; + status->line = line; + status->func = func; +} + +#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) + +extern void perform_spin_delay(SpinDelayStatus *status); +extern void finish_spin_delay(SpinDelayStatus *status); +extern void set_spins_per_delay(int shared_spins_per_delay); +extern int update_spins_per_delay(int shared_spins_per_delay); + +#endif /* SPIN_DELAY_STATUS_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 9cf6adb671a86..281214822244d 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -44,6 +44,80 @@ #ifndef SPIN_H #define SPIN_H +#ifdef USE_STDATOMIC_H + +/* + * Atomics-based spinlocks. + * + * When stdatomic.h is available, spinlocks are implemented on top of a plain + * 32-bit atomic rather than platform-specific TAS assembly. We deliberately + * do NOT use pg_atomic_flag: that type uses 1==unlocked so it can offer a + * relaxed "unlocked test", whereas a spinlock must be usable when its memory + * has merely been zeroed (much shared-memory state is set up with + * memset(...,0,...) and then relies on embedded spinlocks reading as free). + * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, + * making a zero-initialized slock_t a valid, free lock. + */ +#include "port/atomics.h" + +typedef pg_atomic_uint32 slock_t; + +/* SpinDelayStatus and the spin-delay helpers (perform/finish_spin_delay). */ +#include "port/spin_delay_status.h" + +extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); + +static inline void +SpinLockInit(volatile slock_t *lock) +{ + pg_atomic_init_u32(lock, 0); /* 0 = unlocked */ +} + +/* + * SpinLockAcquire - acquire a spinlock, waiting if necessary. + * + * Fast path: a single atomic exchange (0->1). On failure fall into s_lock(), + * which does the backoff/retry and the "stuck spinlock" diagnostics. + * + * This is split into an inline helper plus a wrapper macro: the macro exists + * only to capture the call site's __FILE__/__LINE__/__func__ for diagnostics, + * while the helper evaluates its lock argument exactly once (a macro doing the + * exchange-then-s_lock inline would evaluate the argument twice) and preserves + * the volatile slock_t * type check. + */ +static inline void +spin_lock_acquire(volatile slock_t *lock, const char *file, int line, + const char *func) +{ + if (pg_atomic_exchange_u32(lock, 1) != 0) + s_lock(lock, file, line, func); +} +#define SpinLockAcquire(lock) \ + spin_lock_acquire((lock), __FILE__, __LINE__, __func__) + +static inline void +SpinLockRelease(volatile slock_t *lock) +{ + /* + * Release the lock with release ordering: every load and store in the + * critical section must complete before the lock is observed free. In the + * stdatomic implementation pg_write_barrier() is a release thread fence + * (atomic_thread_fence with release order), which orders the prior + * critical-section accesses -- loads included -- ahead of the following + * store; the plain store then publishes the unlocked value. This gives + * the same release semantics as the traditional S_UNLOCK() while avoiding + * the store-side serialization (STLR on ARM64) that a seq_cst store adds. + */ + pg_write_barrier(); + pg_atomic_write_u32(lock, 0); +} + +#else /* !USE_STDATOMIC_H */ + +/* + * Traditional spinlock implementation using platform-specific TAS assembly. + * This branch is kept byte-for-byte equivalent to the pre-stdatomic spin.h. + */ #include "storage/s_lock.h" static inline void @@ -64,4 +138,6 @@ SpinLockRelease(volatile slock_t *lock) S_UNLOCK(lock); } +#endif /* USE_STDATOMIC_H */ + #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 6ee6689a46888..df6931a0cf59e 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -651,6 +651,10 @@ test_spinlock(void) * * We embed the spinlock in a struct with other members to test that the * spinlock operations don't perform too wide writes. + * + * Note: slock_t is pg_atomic_uint32 (4 bytes) in the stdatomic path vs + * potentially smaller (e.g., unsigned char) in the traditional path. This + * test verifies no over-wide writes regardless of slock_t size. */ { struct test_lock_struct @@ -668,15 +672,22 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); +#ifndef USE_STDATOMIC_H /* test basic operations via underlying S_* API */ S_INIT_LOCK(&struct_w_lock.lock); S_LOCK(&struct_w_lock.lock); S_UNLOCK(&struct_w_lock.lock); +#endif /* and that "contended" acquisition works */ s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); +#ifdef USE_STDATOMIC_H + SpinLockRelease(&struct_w_lock.lock); +#else S_UNLOCK(&struct_w_lock.lock); +#endif +#ifndef USE_STDATOMIC_H /* * Check, using TAS directly, that a single spin cycle doesn't block * when acquiring an already acquired lock. @@ -694,6 +705,7 @@ test_spinlock(void) S_UNLOCK(&struct_w_lock.lock); #endif /* defined(TAS) */ +#endif /* !USE_STDATOMIC_H */ /* * Verify that after all of this the non-lock contents are still From 60c987a891072fcb9473b4b346cea115ffd7214b Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 9 Jul 2026 12:34:56 -0400 Subject: [PATCH 5/6] [NOT FOR MERGE] Add atomics benchmark harness Python-based A/B benchmark harness for comparing the stdatomic.h and traditional atomic implementations under load. Local development tooling; not part of the submitted series. --- src/test/benchmarks/README.md | 282 ++++++++++++ src/test/benchmarks/atomics_analyzer.py | 295 ++++++++++++ src/test/benchmarks/atomics_benchmark.py | 553 +++++++++++++++++++++++ src/test/benchmarks/atomics_cli.py | 354 +++++++++++++++ src/test/benchmarks/atomics_config.py | 275 +++++++++++ src/test/benchmarks/atomics_workload.py | 304 +++++++++++++ src/test/benchmarks/data_generator.py | 409 +++++++++++++++++ 7 files changed, 2472 insertions(+) create mode 100644 src/test/benchmarks/README.md create mode 100644 src/test/benchmarks/atomics_analyzer.py create mode 100644 src/test/benchmarks/atomics_benchmark.py create mode 100644 src/test/benchmarks/atomics_cli.py create mode 100644 src/test/benchmarks/atomics_config.py create mode 100644 src/test/benchmarks/atomics_workload.py create mode 100644 src/test/benchmarks/data_generator.py diff --git a/src/test/benchmarks/README.md b/src/test/benchmarks/README.md new file mode 100644 index 0000000000000..7d51afa1dc657 --- /dev/null +++ b/src/test/benchmarks/README.md @@ -0,0 +1,282 @@ +# PostgreSQL Atomics Performance Benchmark Suite + +This benchmark suite compares the performance of traditional platform-specific atomics implementation versus the C11 stdatomic.h implementation. + +## Overview + +The benchmark evaluates atomic operations impact across different: +- **Workload patterns**: Full scans, aggregations, index scans, concurrent operations +- **Table schemas**: Narrow (4 cols), medium (11 cols), wide (55 cols) +- **Data distributions**: Random, clustered, low cardinality +- **Data scales**: 10K, 100K, 1M, 10M, 100M rows + +## Prerequisites + +1. **Two PostgreSQL builds**: + ```bash + # Traditional atomics + meson setup build-traditional -Duse_stdatomic=no --buildtype=debugoptimized + meson compile -C build-traditional + + # Stdatomic.h implementation + meson setup build-stdatomic -Duse_stdatomic=yes --buildtype=debugoptimized + meson compile -C build-stdatomic + ``` + +2. **Python dependencies**: + ```bash + pip install asyncpg + ``` + +## Usage + +### Quick Start + +Run with default settings (medium test matrix): + +```bash +cd src/test/benchmarks +python -m atomics_cli +``` + +This will: +- Start two PostgreSQL instances (ports 5433 and 5434) +- Run benchmarks on both implementations +- Compare results +- Save to `atomics_benchmark_results/results.json` + +### Full Matrix + +Run comprehensive tests including large datasets: + +```bash +python -m atomics_cli --full-matrix +``` + +**Warning**: This can take several hours. + +### Custom Configuration + +```bash +# Specify custom build directories +python -m atomics_cli \ + --traditional-build /path/to/build-traditional \ + --stdatomic-build /path/to/build-stdatomic + +# Test specific schema and row counts +python -m atomics_cli --schema medium --rows 100000 1000000 + +# Test specific query patterns +python -m atomics_cli --pattern full_scan --pattern aggregation + +# Adjust measurement precision +python -m atomics_cli --warmup 5 --iterations 20 + +# Verbose logging +python -m atomics_cli -v +``` + +### Analyzing Results + +After running benchmarks, analyze the results: + +```bash +python -m atomics_analyzer atomics_benchmark_results/results.json +``` + +Or save to a file: + +```bash +python -m atomics_analyzer atomics_benchmark_results/results.json \ + --output analysis_report.txt +``` + +## Benchmark Methodology + +### Query Patterns + +1. **full_scan**: Sequential scan - stresses spinlock acquisition and buffer management +2. **column_projection**: Selective scan - tuple visibility checks with atomic reads +3. **filtered_scan**: Index + sequential - mixed contention patterns +4. **aggregation**: Heavy computation - shared buffer access +5. **group_by**: Hash aggregation - memory ordering stress +6. **index_scan**: Index-only - buffer pin/unpin contention + +### Measurement Process + +For each scenario: +1. **Warmup**: Run query N times (default: 3) to prime caches +2. **Measure**: Run query M times (default: 10), record each timing +3. **Report**: Use **median** timing to minimize outlier impact + +### Statistical Significance + +- **Noise threshold**: Ā±2% (measurements within this are considered equivalent) +- **Significant improvement**: >2% faster +- **Significant regression**: >2% slower + +### Good Benchmarking Practices + +1. **Dedicated hardware**: Run on isolated system, no background processes +2. **CPU frequency**: Disable CPU frequency scaling: + ```bash + echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + ``` +3. **NUMA**: Pin PostgreSQL to single NUMA node if applicable +4. **Disk I/O**: Use fast storage (SSD/NVMe) or tmpfs for data directories +5. **Multiple runs**: Run benchmark multiple times, compare results +6. **Baseline**: Establish baseline performance before code changes + +## Output Format + +Results are saved in JSON format: + +```json +{ + "summary": { + "total_scenarios": 54, + "median_speedup": 1.002, + "mean_speedup": 1.005, + "significant_improvements": 5, + "significant_regressions": 2, + "per_pattern_avg_speedup": { + "full_scan": 0.998, + "aggregation": 1.015, + ... + } + }, + "results": [ + { + "schema": "atomics_medium", + "row_count": 100000, + "distribution": "random", + "pattern": "full_scan", + "traditional_ms": 125.3, + "stdatomic_ms": 126.1, + "speedup": 0.994 + }, + ... + ] +} +``` + +## Interpreting Results + +### Speedup Metric + +- **Speedup = traditional_ms / stdatomic_ms** +- `speedup > 1.0`: stdatomic is **faster** +- `speedup < 1.0`: stdatomic is **slower** +- `speedup ā‰ˆ 1.0`: Performance is **equivalent** + +### Example Analysis + +``` +VERDICT: NEUTRAL - Performance is statistically equivalent + +Median speedup: 1.002x (stdatomic ~0.2% faster) +Within measurement noise: 47/54 scenarios (87%) +Significant improvements: 5 (9%) +Significant regressions: 2 (4%) +``` + +**Interpretation**: The stdatomic.h implementation has no material performance impact. + +## Troubleshooting + +### Port Already in Use + +``` +ERROR: Port 5433 already in use +``` + +**Solution**: Stop existing PostgreSQL instances or use different ports: +```bash +python -m atomics_cli --traditional-port 5435 --stdatomic-port 5436 +``` + +### Build Directory Not Found + +``` +ERROR: Traditional build directory not found: ./build-traditional +``` + +**Solution**: Build PostgreSQL first or specify correct path: +```bash +python -m atomics_cli --traditional-build /path/to/build +``` + +### Insufficient Disk Space + +For large datasets, ensure adequate space in `/tmp`: +```bash +df -h /tmp +``` + +Consider using custom data directories: +```bash +python -m atomics_cli \ + --traditional-datadir /mnt/fast-storage/pgdata-trad \ + --stdatomic-datadir /mnt/fast-storage/pgdata-std +``` + +### Connection Timeout + +If PostgreSQL takes >30s to start, check: +1. Disk I/O performance +2. System resources (RAM, CPU) +3. PostgreSQL logs in data directory + +## Architecture + +``` +src/test/benchmarks/ +ā”œā”€ā”€ atomics_config.py - Configuration and schema definitions +ā”œā”€ā”€ atomics_benchmark.py - Main orchestrator +ā”œā”€ā”€ atomics_workload.py - Query generation and execution +ā”œā”€ā”€ atomics_cli.py - Command-line interface +ā”œā”€ā”€ atomics_analyzer.py - Results analysis +ā”œā”€ā”€ data_generator.py - Data generation +ā””ā”€ā”€ README_ATOMICS.md - This file +``` + +## For PostgreSQL Committers + +This benchmark suite is designed for: +- Validating atomics implementation changes +- Performance regression testing +- Platform-specific optimization evaluation +- Memory ordering impact analysis + +### CI Integration + +For automated testing: + +```bash +# Quick smoke test (fast, ~5 minutes) +python -m atomics_cli --schema narrow --rows 10000 --iterations 3 + +# Standard test (moderate, ~30 minutes) +python -m atomics_cli --schema medium --rows 100000 1000000 + +# Full validation (slow, several hours) +python -m atomics_cli --full-matrix +``` + +### Expected Performance + +On x86_64 with strong memory model: +- **Median difference**: Ā±1-2% (within noise) +- **Pattern variation**: Some patterns may show Ā±5% variation +- **Scale impact**: Minimal difference across scales + +On ARM64/RISC-V with weak memory model: +- **Potential regression**: 2-10% (stronger ordering overhead) +- **Workload dependent**: Write-heavy workloads more impacted +- **Trade-off**: Correctness vs. performance + +## References + +- [SEMANTIC_DIFFERENCES.md](../../SEMANTIC_DIFFERENCES.md) - Implementation differences +- [MEMORY_ORDERING_ANALYSIS.md](../../MEMORY_ORDERING_ANALYSIS.md) - Memory ordering decisions +- [PostgreSQL atomics documentation](https://www.postgresql.org/docs/current/atomics.html) diff --git a/src/test/benchmarks/atomics_analyzer.py b/src/test/benchmarks/atomics_analyzer.py new file mode 100644 index 0000000000000..71fb3e98e95a5 --- /dev/null +++ b/src/test/benchmarks/atomics_analyzer.py @@ -0,0 +1,295 @@ +""" +Results analyzer for atomics benchmark comparisons. + +This module provides statistical analysis and visualization of benchmark results, +helping identify performance regressions or improvements from the stdatomic.h +implementation. +""" + +import json +import statistics +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple + + +@dataclass +class ResultComparison: + """Comparison of a single benchmark scenario.""" + schema: str + row_count: int + distribution: str + pattern: str + traditional_ms: float + stdatomic_ms: float + speedup: float + difference_pct: float # (traditional - stdatomic) / traditional * 100 + + +class AtomicsResultAnalyzer: + """Analyzes and compares benchmark results.""" + + def __init__(self, results_file: Path): + self.results_file = results_file + self.comparisons: List[ResultComparison] = [] + self._load_results() + + def _load_results(self): + """Load results from JSON file.""" + with open(self.results_file) as f: + data = json.load(f) + + for r in data.get("results", []): + diff_pct = ( + (r["traditional_ms"] - r["stdatomic_ms"]) / r["traditional_ms"] * 100 + ) + comp = ResultComparison( + schema=r["schema"], + row_count=r["row_count"], + distribution=r["distribution"], + pattern=r["pattern"], + traditional_ms=r["traditional_ms"], + stdatomic_ms=r["stdatomic_ms"], + speedup=r["speedup"], + difference_pct=diff_pct, + ) + self.comparisons.append(comp) + + def analyze_overall(self) -> Dict: + """Analyze overall performance characteristics.""" + if not self.comparisons: + return {} + + speedups = [c.speedup for c in self.comparisons] + diff_pcts = [c.difference_pct for c in self.comparisons] + + # Statistical significance: count how many are outside Ā±2% threshold + significant_improvements = sum(1 for d in diff_pcts if d > 2.0) + significant_regressions = sum(1 for d in diff_pcts if d < -2.0) + within_noise = sum(1 for d in diff_pcts if -2.0 <= d <= 2.0) + + return { + "total_scenarios": len(self.comparisons), + "median_speedup": statistics.median(speedups), + "mean_speedup": statistics.mean(speedups), + "stdev_speedup": statistics.stdev(speedups) if len(speedups) > 1 else 0.0, + "min_speedup": min(speedups), + "max_speedup": max(speedups), + "median_diff_pct": statistics.median(diff_pcts), + "mean_diff_pct": statistics.mean(diff_pcts), + "significant_improvements": significant_improvements, + "significant_regressions": significant_regressions, + "within_noise": within_noise, + } + + def analyze_by_pattern(self) -> Dict[str, Dict]: + """Break down performance by query pattern.""" + pattern_groups = {} + for comp in self.comparisons: + if comp.pattern not in pattern_groups: + pattern_groups[comp.pattern] = [] + pattern_groups[comp.pattern].append(comp) + + results = {} + for pattern, comps in pattern_groups.items(): + speedups = [c.speedup for c in comps] + diff_pcts = [c.difference_pct for c in comps] + + results[pattern] = { + "count": len(comps), + "median_speedup": statistics.median(speedups), + "mean_speedup": statistics.mean(speedups), + "median_diff_pct": statistics.median(diff_pcts), + "mean_diff_pct": statistics.mean(diff_pcts), + "improvements": sum(1 for d in diff_pcts if d > 2.0), + "regressions": sum(1 for d in diff_pcts if d < -2.0), + } + + return results + + def analyze_by_scale(self) -> Dict[int, Dict]: + """Break down performance by data scale.""" + scale_groups = {} + for comp in self.comparisons: + if comp.row_count not in scale_groups: + scale_groups[comp.row_count] = [] + scale_groups[comp.row_count].append(comp) + + results = {} + for row_count, comps in scale_groups.items(): + speedups = [c.speedup for c in comps] + diff_pcts = [c.difference_pct for c in comps] + + results[row_count] = { + "count": len(comps), + "median_speedup": statistics.median(speedups), + "mean_speedup": statistics.mean(speedups), + "median_diff_pct": statistics.median(diff_pcts), + "improvements": sum(1 for d in diff_pcts if d > 2.0), + "regressions": sum(1 for d in diff_pcts if d < -2.0), + } + + return results + + def find_regressions(self, threshold_pct: float = 2.0) -> List[ResultComparison]: + """Find scenarios where stdatomic is slower than threshold.""" + return [c for c in self.comparisons if c.difference_pct < -threshold_pct] + + def find_improvements(self, threshold_pct: float = 2.0) -> List[ResultComparison]: + """Find scenarios where stdatomic is faster than threshold.""" + return [c for c in self.comparisons if c.difference_pct > threshold_pct] + + def generate_report(self, output_file: Optional[Path] = None) -> str: + """Generate a comprehensive text report.""" + lines = [] + lines.append("=" * 80) + lines.append("PostgreSQL Atomics Benchmark Analysis Report") + lines.append("=" * 80) + lines.append("") + + # Overall analysis + overall = self.analyze_overall() + lines.append("OVERALL PERFORMANCE") + lines.append("-" * 80) + lines.append(f"Total scenarios: {overall['total_scenarios']}") + lines.append(f"Median speedup: {overall['median_speedup']:.3f}x") + lines.append(f"Mean speedup: {overall['mean_speedup']:.3f}x Ā± {overall['stdev_speedup']:.3f}") + lines.append(f"Range: {overall['min_speedup']:.3f}x - {overall['max_speedup']:.3f}x") + lines.append("") + lines.append(f"Median difference: {overall['median_diff_pct']:+.2f}%") + lines.append(f"Mean difference: {overall['mean_diff_pct']:+.2f}%") + lines.append("") + lines.append(f"Significant improvements: {overall['significant_improvements']} (>{2.0:.1f}%)") + lines.append(f"Significant regressions: {overall['significant_regressions']} (<-{2.0:.1f}%)") + lines.append(f"Within measurement noise: {overall['within_noise']} (Ā±{2.0:.1f}%)") + lines.append("") + + # Verdict + median_diff = overall["median_diff_pct"] + if abs(median_diff) < 2.0: + verdict = "NEUTRAL - Performance is statistically equivalent" + elif median_diff > 0: + verdict = f"POSITIVE - Stdatomic is faster by {median_diff:.2f}%" + else: + verdict = f"NEGATIVE - Stdatomic is slower by {abs(median_diff):.2f}%" + + lines.append(f"VERDICT: {verdict}") + lines.append("") + + # By pattern + by_pattern = self.analyze_by_pattern() + lines.append("PERFORMANCE BY QUERY PATTERN") + lines.append("-" * 80) + lines.append(f"{'Pattern':<30} {'Median':>10} {'Mean':>10} {'Diff%':>10} {'+/-':>10}") + lines.append("-" * 80) + + for pattern in sorted(by_pattern.keys()): + stats = by_pattern[pattern] + indicator = "+++" if stats["mean_diff_pct"] > 5.0 else " " + indicator = "---" if stats["mean_diff_pct"] < -5.0 else indicator + lines.append( + f"{pattern:<30} {stats['median_speedup']:>9.3f}x " + f"{stats['mean_speedup']:>9.3f}x " + f"{stats['mean_diff_pct']:>9.2f}% " + f"{indicator}" + ) + + lines.append("") + + # By scale + by_scale = self.analyze_by_scale() + lines.append("PERFORMANCE BY DATA SCALE") + lines.append("-" * 80) + lines.append(f"{'Rows':<15} {'Median':>10} {'Mean':>10} {'Diff%':>10} {'Status':<20}") + lines.append("-" * 80) + + for row_count in sorted(by_scale.keys()): + stats = by_scale[row_count] + status = "Faster" if stats["median_diff_pct"] > 2.0 else "Neutral" + status = "Slower" if stats["median_diff_pct"] < -2.0 else status + lines.append( + f"{row_count:<15,} {stats['median_speedup']:>9.3f}x " + f"{stats['mean_speedup']:>9.3f}x " + f"{stats['median_diff_pct']:>9.2f}% " + f"{status:<20}" + ) + + lines.append("") + + # Top regressions + regressions = sorted( + self.find_regressions(2.0), key=lambda c: c.difference_pct + ) + if regressions: + lines.append("TOP REGRESSIONS (stdatomic slower by >2%)") + lines.append("-" * 80) + for comp in regressions[:10]: + lines.append( + f" {comp.difference_pct:>6.2f}% - {comp.pattern:<25} " + f"{comp.schema}/{comp.row_count:,}/{comp.distribution}" + ) + lines.append("") + + # Top improvements + improvements = sorted( + self.find_improvements(2.0), key=lambda c: c.difference_pct, reverse=True + ) + if improvements: + lines.append("TOP IMPROVEMENTS (stdatomic faster by >2%)") + lines.append("-" * 80) + for comp in improvements[:10]: + lines.append( + f" {comp.difference_pct:>6.2f}% - {comp.pattern:<25} " + f"{comp.schema}/{comp.row_count:,}/{comp.distribution}" + ) + lines.append("") + + lines.append("=" * 80) + + report = "\n".join(lines) + + if output_file: + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + f.write(report) + + return report + + +def main(): + """CLI entry point for results analysis.""" + import argparse + + parser = argparse.ArgumentParser( + description="Analyze atomics benchmark results" + ) + parser.add_argument( + "results_file", + type=Path, + help="Path to results.json file", + ) + parser.add_argument( + "--output", + "-o", + type=Path, + help="Output file for report (default: stdout)", + ) + + args = parser.parse_args() + + if not args.results_file.exists(): + print(f"ERROR: Results file not found: {args.results_file}") + return 1 + + analyzer = AtomicsResultAnalyzer(args.results_file) + report = analyzer.generate_report(args.output) + + if not args.output: + print(report) + + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/src/test/benchmarks/atomics_benchmark.py b/src/test/benchmarks/atomics_benchmark.py new file mode 100644 index 0000000000000..5f5207599ad56 --- /dev/null +++ b/src/test/benchmarks/atomics_benchmark.py @@ -0,0 +1,553 @@ +""" +Main benchmark orchestrator for comparing traditional vs stdatomic.h implementations. + +This module coordinates: +- PostgreSQL instance lifecycle management +- Schema creation and data loading +- Workload execution on both implementations +- Results collection and comparison +""" + +import asyncio +import asyncpg +import logging +import os +import shutil +import signal +import subprocess +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from .atomics_config import ( + AtomicsBenchmarkConfig, + DataDistribution, + PostgresInstance, + QueryPattern, + TableSchema, +) +from .data_generator import DataGenerator +from .atomics_workload import AtomicsWorkloadRunner, WorkloadResult + +logger = logging.getLogger(__name__) + + +@dataclass +class BenchmarkMetrics: + """Collected metrics for a benchmark run.""" + implementation: str # "traditional" or "stdatomic" + schema_name: str + row_count: int + distribution: str + load_time_seconds: float + table_size_bytes: int + index_size_bytes: int + shared_buffers_hit_ratio: float = 0.0 + cache_hit_ratio: float = 0.0 + + +@dataclass +class ComparisonResult: + """Result of comparing traditional vs stdatomic for one scenario.""" + schema_name: str + row_count: int + distribution: str + pattern: str + traditional_ms: float + stdatomic_ms: float + speedup: float # stdatomic_ms / traditional_ms (< 1.0 means faster) + traditional_metrics: BenchmarkMetrics + stdatomic_metrics: BenchmarkMetrics + + +class PostgresManager: + """Manages PostgreSQL instance lifecycle.""" + + def __init__(self, instance: PostgresInstance): + self.instance = instance + self.process: Optional[subprocess.Popen] = None + self.pool: Optional[asyncpg.Pool] = None + + async def initialize_cluster(self): + """Initialize a new database cluster if needed.""" + if self.instance.data_dir.exists(): + logger.info( + "Data directory %s exists, skipping initdb", self.instance.data_dir + ) + return + + logger.info("Initializing database cluster at %s", self.instance.data_dir) + self.instance.data_dir.mkdir(parents=True, exist_ok=True) + + # For meson build directories (not installed), create symlink for postgres binary + # in initdb directory so initdb can find it + if not self.instance._is_install_dir(): + initdb_dir = self.instance.initdb_bin.parent + postgres_link = initdb_dir / "postgres" + if not postgres_link.exists(): + postgres_link.symlink_to(self.instance.postgres_bin) + logger.debug("Created postgres symlink: %s -> %s", postgres_link, self.instance.postgres_bin) + + cmd = [ + str(self.instance.initdb_bin), + "-D", + str(self.instance.data_dir), + "--no-locale", + "--encoding=UTF8", + ] + + # Set LD_LIBRARY_PATH for installed directories + env = None + if self.instance._is_install_dir(): + lib_dir = self.instance.build_dir / "lib64" + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = f"{lib_dir}:{env.get('LD_LIBRARY_PATH', '')}" + logger.debug("Setting LD_LIBRARY_PATH=%s", env["LD_LIBRARY_PATH"]) + + result = subprocess.run(cmd, capture_output=True, text=True, env=env) + if result.returncode != 0: + raise RuntimeError(f"initdb failed: {result.stderr}") + + logger.info("Database cluster initialized successfully") + + # Configure for benchmarking + self._configure_for_performance() + + def _configure_for_performance(self): + """Update postgresql.conf for benchmark performance.""" + conf_path = self.instance.data_dir / "postgresql.conf" + + settings = { + "shared_buffers": "1GB", + "effective_cache_size": "4GB", + "maintenance_work_mem": "256MB", + "checkpoint_completion_target": "0.9", + "wal_buffers": "16MB", + "default_statistics_target": "100", + "random_page_cost": "1.1", + "effective_io_concurrency": "200", + "work_mem": "32MB", + "min_wal_size": "1GB", + "max_wal_size": "4GB", + "max_worker_processes": "8", + "max_parallel_workers_per_gather": "4", + "max_parallel_workers": "8", + "max_parallel_maintenance_workers": "4", + # Enable timing + "track_activities": "on", + "track_counts": "on", + "track_io_timing": "on", + "track_functions": "all", + # Reduce checkpointing during benchmark + "checkpoint_timeout": "30min", + } + + with open(conf_path, "a") as f: + f.write("\n\n# Performance settings for benchmarking\n") + for key, value in settings.items(): + f.write(f"{key} = {value}\n") + + logger.info("Configured PostgreSQL for performance benchmarking") + + async def start(self): + """Start the PostgreSQL instance.""" + if self.process is not None: + logger.warning("PostgreSQL is already running") + return + + logger.info( + "Starting PostgreSQL (%s) on port %d", + self.instance.name, + self.instance.port, + ) + + cmd = [ + str(self.instance.postgres_bin), + "-D", + str(self.instance.data_dir), + "-p", + str(self.instance.port), + "-c", + "logging_collector=off", + ] + + # Set LD_LIBRARY_PATH for installed directories + env = None + if self.instance._is_install_dir(): + lib_dir = self.instance.build_dir / "lib64" + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = f"{lib_dir}:{env.get('LD_LIBRARY_PATH', '')}" + + self.process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + # Wait for server to be ready + await self._wait_for_ready() + + # Create connection pool + await self._create_pool() + + logger.info("PostgreSQL (%s) started successfully", self.instance.name) + + async def _wait_for_ready(self, timeout: int = 30): + """Wait for PostgreSQL to be ready to accept connections.""" + start = time.time() + while time.time() - start < timeout: + try: + conn = await asyncpg.connect( + host=self.instance.host, + port=self.instance.port, + user=self.instance.user or os.getenv("USER"), + database="postgres", + timeout=1, + ) + await conn.close() + return + except (asyncpg.CannotConnectNowError, OSError): + await asyncio.sleep(0.5) + + raise RuntimeError( + f"PostgreSQL ({self.instance.name}) did not start within {timeout}s" + ) + + async def _create_pool(self): + """Create connection pool.""" + self.pool = await asyncpg.create_pool( + host=self.instance.host, + port=self.instance.port, + user=self.instance.user or os.getenv("USER"), + database="postgres", + min_size=2, + max_size=10, + ) + + async def stop(self): + """Stop the PostgreSQL instance.""" + if self.process is None: + return + + logger.info("Stopping PostgreSQL (%s)", self.instance.name) + + if self.pool: + await self.pool.close() + self.pool = None + + # Send SIGTERM + self.process.terminate() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.warning("PostgreSQL did not stop gracefully, sending SIGKILL") + self.process.kill() + self.process.wait() + + self.process = None + logger.info("PostgreSQL (%s) stopped", self.instance.name) + + async def execute(self, query: str): + """Execute a query.""" + async with self.pool.acquire() as conn: + return await conn.execute(query) + + async def fetch(self, query: str): + """Fetch query results.""" + async with self.pool.acquire() as conn: + return await conn.fetch(query) + + async def fetchrow(self, query: str): + """Fetch single row.""" + async with self.pool.acquire() as conn: + return await conn.fetchrow(query) + + +class AtomicsBenchmarkSuite: + """Main benchmark orchestrator.""" + + def __init__(self, config: Optional[AtomicsBenchmarkConfig] = None): + self.config = config or AtomicsBenchmarkConfig() + + self.traditional_mgr = PostgresManager(self.config.traditional_instance) + self.stdatomic_mgr = PostgresManager(self.config.stdatomic_instance) + + self.data_generator = DataGenerator(seed=self.config.seed) + self.results: List[ComparisonResult] = [] + + async def setup(self): + """Initialize both PostgreSQL instances.""" + logger.info("Setting up benchmark environment...") + + # Initialize clusters + await self.traditional_mgr.initialize_cluster() + await self.stdatomic_mgr.initialize_cluster() + + # Start instances + await self.traditional_mgr.start() + await self.stdatomic_mgr.start() + + # Create benchmark database + await self.traditional_mgr.execute("CREATE DATABASE benchmark_db") + await self.stdatomic_mgr.execute("CREATE DATABASE benchmark_db") + + logger.info("Benchmark environment ready") + + async def teardown(self): + """Stop both PostgreSQL instances.""" + logger.info("Tearing down benchmark environment...") + await self.traditional_mgr.stop() + await self.stdatomic_mgr.stop() + + async def run_benchmark( + self, + schema: TableSchema, + row_count: int, + distribution: DataDistribution, + ) -> List[ComparisonResult]: + """Run complete benchmark for one scenario, comparing both implementations.""" + logger.info( + "=== Benchmark: %s, %d rows, %s ===", + schema.name, + row_count, + distribution.value, + ) + + results = [] + + # Run on traditional + trad_workload, trad_metrics = await self._run_on_instance( + self.traditional_mgr, "traditional", schema, row_count, distribution + ) + + # Run on stdatomic + std_workload, std_metrics = await self._run_on_instance( + self.stdatomic_mgr, "stdatomic", schema, row_count, distribution + ) + + # Compare results + for trad_result in trad_workload.results: + pattern = trad_result.query_pattern + std_result = next( + (r for r in std_workload.results if r.query_pattern == pattern), None + ) + if std_result is None: + continue + + speedup = trad_result.elapsed_seconds / std_result.elapsed_seconds + + comparison = ComparisonResult( + schema_name=schema.name, + row_count=row_count, + distribution=distribution.value, + pattern=pattern, + traditional_ms=trad_result.elapsed_seconds * 1000, + stdatomic_ms=std_result.elapsed_seconds * 1000, + speedup=speedup, + traditional_metrics=trad_metrics, + stdatomic_metrics=std_metrics, + ) + results.append(comparison) + + return results + + async def _run_on_instance( + self, + mgr: PostgresManager, + impl_name: str, + schema: TableSchema, + row_count: int, + distribution: DataDistribution, + ) -> Tuple[WorkloadResult, BenchmarkMetrics]: + """Run benchmark on a single PostgreSQL instance.""" + # Create table + table_name = f"{schema.name}_{distribution.value}" + await self._create_table(mgr, table_name, schema) + + # Load data + load_start = time.perf_counter() + insert_sql = self.data_generator.generate_server_side_insert( + schema, row_count, distribution, table_suffix=f"_{distribution.value}" + ) + await mgr.execute(insert_sql) + load_time = time.perf_counter() - load_start + + logger.info("%s: loaded %d rows in %.2fs", impl_name, row_count, load_time) + + # Collect pre-workload metrics + metrics = await self._collect_metrics( + mgr, impl_name, table_name, schema.name, row_count, distribution.value + ) + metrics.load_time_seconds = load_time + + # Run workload + runner = AtomicsWorkloadRunner( + mgr.pool, + warmup_iterations=self.config.warmup_iterations, + measure_iterations=self.config.measure_iterations, + ) + + workload_result = await runner.run_workload( + schema=schema, + table_name=table_name, + row_count=row_count, + distribution=distribution.value, + patterns=self.config.query_patterns, + ) + + # Cleanup + await mgr.execute(f"DROP TABLE IF EXISTS {table_name} CASCADE") + + return workload_result, metrics + + async def _create_table( + self, mgr: PostgresManager, table_name: str, schema: TableSchema + ): + """Create a table with the given schema.""" + col_defs = [] + for col_name, col_type in schema.columns: + col_defs.append(f"{col_name} {col_type.value}") + + create_sql = f"CREATE TABLE {table_name} ({', '.join(col_defs)})" + await mgr.execute(create_sql) + + # Create indexes + for idx_col in schema.index_columns: + idx_name = f"{table_name}_{idx_col}_idx" + await mgr.execute( + f"CREATE INDEX {idx_name} ON {table_name} ({idx_col})" + ) + + async def _collect_metrics( + self, + mgr: PostgresManager, + impl_name: str, + table_name: str, + schema_name: str, + row_count: int, + distribution: str, + ) -> BenchmarkMetrics: + """Collect metrics for a table.""" + # Table size + size_row = await mgr.fetchrow( + f"SELECT pg_total_relation_size('{table_name}') as total_size" + ) + table_size = size_row["total_size"] + + # Index size + idx_size_row = await mgr.fetchrow( + f"SELECT pg_indexes_size('{table_name}') as idx_size" + ) + index_size = idx_size_row["idx_size"] + + return BenchmarkMetrics( + implementation=impl_name, + schema_name=schema_name, + row_count=row_count, + distribution=distribution, + load_time_seconds=0.0, # Set by caller + table_size_bytes=table_size, + index_size_bytes=index_size, + ) + + async def run_full_suite(self) -> Dict: + """Run complete benchmark suite.""" + start_time = time.perf_counter() + all_results = [] + + for schema in self.config.schemas: + for row_count in self.config.get_row_counts(): + for distribution in self.config.distributions: + try: + results = await self.run_benchmark( + schema, row_count, distribution + ) + all_results.extend(results) + except Exception as e: + logger.error( + "Benchmark failed for %s/%d/%s: %s", + schema.name, + row_count, + distribution.value, + e, + exc_info=True, + ) + + elapsed_time = time.perf_counter() - start_time + + # Generate summary + summary = self._generate_summary(all_results, elapsed_time) + self.results = all_results + + return summary + + def _generate_summary( + self, results: List[ComparisonResult], elapsed_time: float + ) -> Dict: + """Generate summary statistics.""" + if not results: + return {} + + speedups = [r.speedup for r in results] + + summary = { + "total_scenarios": len(results), + "elapsed_time_seconds": elapsed_time, + "median_speedup": sorted(speedups)[len(speedups) // 2], + "mean_speedup": sum(speedups) / len(speedups), + "min_speedup": min(speedups), + "max_speedup": max(speedups), + "stdatomic_faster_count": sum(1 for s in speedups if s > 1.0), + "traditional_faster_count": sum(1 for s in speedups if s < 1.0), + "neutral_count": sum(1 for s in speedups if s == 1.0), + } + + # Per-pattern breakdown + pattern_speedups = {} + for result in results: + if result.pattern not in pattern_speedups: + pattern_speedups[result.pattern] = [] + pattern_speedups[result.pattern].append(result.speedup) + + summary["per_pattern_avg_speedup"] = { + pattern: sum(speedups) / len(speedups) + for pattern, speedups in pattern_speedups.items() + } + + # Find best/worst scenarios + best = max(results, key=lambda r: r.speedup) + worst = min(results, key=lambda r: r.speedup) + + summary["best_stdatomic_scenario"] = { + "pattern": best.pattern, + "schema": best.schema_name, + "distribution": best.distribution, + "speedup": best.speedup, + } + + summary["worst_stdatomic_scenario"] = { + "pattern": worst.pattern, + "schema": worst.schema_name, + "distribution": worst.distribution, + "speedup": worst.speedup, + } + + return summary + + +async def run_atomics_benchmark( + config: Optional[AtomicsBenchmarkConfig] = None, +) -> Dict: + """Main entry point for running atomics benchmark.""" + suite = AtomicsBenchmarkSuite(config) + + try: + await suite.setup() + summary = await suite.run_full_suite() + return {"summary": summary, "results": suite.results} + finally: + await suite.teardown() diff --git a/src/test/benchmarks/atomics_cli.py b/src/test/benchmarks/atomics_cli.py new file mode 100644 index 0000000000000..20d870dc6b7c2 --- /dev/null +++ b/src/test/benchmarks/atomics_cli.py @@ -0,0 +1,354 @@ +""" +CLI entry point for atomics benchmark suite. + +Usage: + python -m src.test.benchmarks.atomics_cli [OPTIONS] + +Examples: + # Quick run with defaults + python -m src.test.benchmarks.atomics_cli + + # Full matrix + python -m src.test.benchmarks.atomics_cli --full-matrix + + # Specific schema and row count + python -m src.test.benchmarks.atomics_cli --schema medium --rows 100000 + + # Custom build directories + python -m src.test.benchmarks.atomics_cli \\ + --traditional-build ./build-traditional \\ + --stdatomic-build ./build-stdatomic + + # Verbose output + python -m src.test.benchmarks.atomics_cli -v +""" + +import argparse +import asyncio +import json +import logging +import sys +from pathlib import Path + +from .atomics_config import ( + ALL_SCHEMAS, + AtomicsBenchmarkConfig, + DataDistribution, + MEDIUM_SCHEMA, + NARROW_SCHEMA, + PostgresInstance, + QueryPattern, + WIDE_SCHEMA, +) +from .atomics_benchmark import run_atomics_benchmark + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="PostgreSQL Atomics Performance Benchmark Suite", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Build directories + parser.add_argument( + "--traditional-build", + type=Path, + default=Path.cwd() / "build-traditional", + help="Path to traditional atomics build directory", + ) + parser.add_argument( + "--stdatomic-build", + type=Path, + default=Path.cwd() / "build-stdatomic", + help="Path to stdatomic.h build directory", + ) + + # PostgreSQL instance configuration + parser.add_argument( + "--traditional-port", type=int, default=5433, help="Port for traditional instance" + ) + parser.add_argument( + "--stdatomic-port", type=int, default=5434, help="Port for stdatomic instance" + ) + parser.add_argument( + "--traditional-datadir", + type=Path, + default=Path("/tmp/pgdata-traditional"), + help="Data directory for traditional instance", + ) + parser.add_argument( + "--stdatomic-datadir", + type=Path, + default=Path("/tmp/pgdata-stdatomic"), + help="Data directory for stdatomic instance", + ) + + # Test matrix + parser.add_argument( + "--schema", + choices=["narrow", "medium", "wide", "all"], + default="all", + help="Table schema to test (default: all)", + ) + parser.add_argument( + "--rows", + type=int, + nargs="+", + default=None, + help="Row counts to test (default: 10000 100000 1000000)", + ) + parser.add_argument( + "--distribution", + choices=["random", "clustered", "low_cardinality", "high_null", "all"], + default="all", + help="Data distribution (default: all)", + ) + parser.add_argument( + "--pattern", + choices=[p.value for p in QueryPattern] + ["all"], + default="all", + help="Query pattern to test (default: all)", + ) + parser.add_argument( + "--full-matrix", + action="store_true", + help="Run full matrix including large row counts", + ) + + # Execution + parser.add_argument( + "--warmup", type=int, default=3, help="Warmup iterations (default: 3)" + ) + parser.add_argument( + "--iterations", type=int, default=10, help="Measurement iterations (default: 10)" + ) + parser.add_argument( + "--concurrent-clients", + type=int, + default=8, + help="Number of concurrent clients for write tests (default: 8)", + ) + parser.add_argument("--seed", type=int, default=42, help="RNG seed (default: 42)") + + # Output + parser.add_argument( + "--output-dir", + "-o", + default="atomics_benchmark_results", + help="Output directory", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Verbose logging" + ) + + return parser.parse_args() + + +def build_config(args: argparse.Namespace) -> AtomicsBenchmarkConfig: + """Build configuration from command-line arguments.""" + traditional_inst = PostgresInstance( + name="traditional", + build_dir=args.traditional_build, + port=args.traditional_port, + data_dir=args.traditional_datadir, + ) + + stdatomic_inst = PostgresInstance( + name="stdatomic", + build_dir=args.stdatomic_build, + port=args.stdatomic_port, + data_dir=args.stdatomic_datadir, + ) + + schema_map = { + "narrow": [NARROW_SCHEMA], + "medium": [MEDIUM_SCHEMA], + "wide": [WIDE_SCHEMA], + "all": list(ALL_SCHEMAS), + } + schemas = schema_map[args.schema] + + if args.distribution == "all": + distributions = [ + DataDistribution.RANDOM, + DataDistribution.CLUSTERED, + DataDistribution.LOW_CARDINALITY, + ] + else: + distributions = [DataDistribution(args.distribution)] + + if args.pattern == "all": + # Default patterns (exclude concurrent patterns for now) + patterns = [ + QueryPattern.FULL_SCAN, + QueryPattern.COLUMN_PROJECTION, + QueryPattern.FILTERED_SCAN, + QueryPattern.AGGREGATION, + QueryPattern.GROUP_BY, + QueryPattern.INDEX_SCAN, + ] + else: + patterns = [QueryPattern(args.pattern)] + + config = AtomicsBenchmarkConfig( + traditional_instance=traditional_inst, + stdatomic_instance=stdatomic_inst, + schemas=schemas, + distributions=distributions, + query_patterns=patterns, + warmup_iterations=args.warmup, + measure_iterations=args.iterations, + concurrent_clients=args.concurrent_clients, + seed=args.seed, + output_dir=args.output_dir, + full_matrix=args.full_matrix, + verbose=args.verbose, + ) + + if args.rows: + config.row_counts = args.rows + + return config + + +def print_summary(summary: dict): + """Print benchmark summary.""" + print() + print("=" * 70) + print(" ATOMICS BENCHMARK RESULTS SUMMARY") + print("=" * 70) + + if not summary: + print(" No results to display") + return + + print(f" Total scenarios tested: {summary.get('total_scenarios', 0)}") + print(f" Total time: {summary.get('elapsed_time_seconds', 0):.1f}s") + print() + + median = summary.get("median_speedup", 1.0) + mean = summary.get("mean_speedup", 1.0) + print(f" Median speedup (std/trad): {median:.3f}x") + print(f" Mean speedup: {mean:.3f}x") + print(f" Min speedup: {summary.get('min_speedup', 1.0):.3f}x") + print(f" Max speedup: {summary.get('max_speedup', 1.0):.3f}x") + print() + + faster = summary.get("stdatomic_faster_count", 0) + slower = summary.get("traditional_faster_count", 0) + total = summary.get("total_scenarios", 1) + + print(f" Stdatomic faster: {faster} ({100*faster/total:.1f}%)") + print(f" Traditional faster: {slower} ({100*slower/total:.1f}%)") + + if summary.get("per_pattern_avg_speedup"): + print() + print(" Per-pattern average speedup (stdatomic vs traditional):") + for pattern, speedup in sorted(summary["per_pattern_avg_speedup"].items()): + indicator = ">>>" if speedup > 1.05 else " " + indicator = "<<<" if speedup < 0.95 else indicator + print(f" {indicator} {pattern:30s} {speedup:.3f}x") + + if summary.get("best_stdatomic_scenario"): + best = summary["best_stdatomic_scenario"] + print() + print( + f" Best stdatomic: {best['pattern']} on {best['schema']} " + f"({best['distribution']}) = {best['speedup']:.3f}x" + ) + + if summary.get("worst_stdatomic_scenario"): + worst = summary["worst_stdatomic_scenario"] + print( + f" Worst stdatomic: {worst['pattern']} on {worst['schema']} " + f"({worst['distribution']}) = {worst['speedup']:.3f}x" + ) + + print("=" * 70) + + +def main(): + args = parse_args() + + log_level = logging.DEBUG if args.verbose else logging.INFO + logging.basicConfig( + level=log_level, + format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", + datefmt="%H:%M:%S", + ) + + # Verify build directories exist + if not args.traditional_build.exists(): + print(f"ERROR: Traditional build directory not found: {args.traditional_build}") + print("Please build with: meson setup build-traditional -Duse_stdatomic=no") + sys.exit(1) + + if not args.stdatomic_build.exists(): + print(f"ERROR: Stdatomic build directory not found: {args.stdatomic_build}") + print("Please build with: meson setup build-stdatomic -Duse_stdatomic=yes") + sys.exit(1) + + config = build_config(args) + + print("=" * 70) + print(" PostgreSQL Atomics Performance Benchmark") + print("=" * 70) + print(f" Traditional build: {config.traditional_instance.build_dir}") + print(f" Stdatomic build: {config.stdatomic_instance.build_dir}") + print(f" Schemas: {[s.name for s in config.schemas]}") + print(f" Row counts: {config.get_row_counts()}") + print(f" Distributions: {[d.value for d in config.distributions]}") + print(f" Patterns: {[p.value for p in config.query_patterns]}") + print( + f" Iterations: {config.measure_iterations} " + f"(warmup: {config.warmup_iterations})" + ) + print(f" Output: {config.output_dir}") + print("=" * 70) + print() + + try: + report = asyncio.run(run_atomics_benchmark(config)) + except KeyboardInterrupt: + print("\nBenchmark interrupted.") + sys.exit(1) + except Exception as e: + logging.error("Benchmark failed: %s", e, exc_info=True) + sys.exit(1) + + # Print summary + print_summary(report.get("summary", {})) + + # Save results + output_dir = Path(config.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + results_file = output_dir / "results.json" + with open(results_file, "w") as f: + # Convert results to serializable format + results_data = [] + for r in report.get("results", []): + results_data.append( + { + "schema": r.schema_name, + "row_count": r.row_count, + "distribution": r.distribution, + "pattern": r.pattern, + "traditional_ms": r.traditional_ms, + "stdatomic_ms": r.stdatomic_ms, + "speedup": r.speedup, + } + ) + + json.dump( + {"summary": report.get("summary", {}), "results": results_data}, + f, + indent=2, + ) + + print(f"\nResults saved to: {results_file}") + + +if __name__ == "__main__": + main() diff --git a/src/test/benchmarks/atomics_config.py b/src/test/benchmarks/atomics_config.py new file mode 100644 index 0000000000000..0b2c831cbf291 --- /dev/null +++ b/src/test/benchmarks/atomics_config.py @@ -0,0 +1,275 @@ +""" +Benchmark configuration for comparing traditional vs stdatomic.h implementations. + +This module defines the test matrix for evaluating atomics performance impact +across different workload patterns, table schemas, and data distributions. +""" + +import os +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import List, Optional + + +class TableWidth(Enum): + """Table width categories for testing atomic operation overhead.""" + NARROW = "narrow" # 3-5 columns - minimal overhead + MEDIUM = "medium" # 10-30 columns - moderate overhead + WIDE = "wide" # 50-120 columns - heavy overhead + + +class DataDistribution(Enum): + """Data distribution patterns affecting atomic contention.""" + RANDOM = "random" + CLUSTERED = "clustered" + LOW_CARDINALITY = "low_cardinality" + HIGH_NULL = "high_null" + + +class QueryPattern(Enum): + """Query patterns that stress different atomic operations.""" + FULL_SCAN = "full_scan" # Sequential scan - spinlock acquisition + COLUMN_PROJECTION = "column_projection" # Selective scan - tuple visibility checks + FILTERED_SCAN = "filtered_scan" # Index + seq scan - mixed contention + AGGREGATION = "aggregation" # Heavy computation - shared buffer access + GROUP_BY = "group_by" # Hash aggregation - memory ordering stress + INDEX_SCAN = "index_scan" # Index-only - buffer pin/unpin contention + CONCURRENT_INSERT = "concurrent_insert" # High contention writes + CONCURRENT_UPDATE = "concurrent_update" # Lock management stress + MIXED_WORKLOAD = "mixed_workload" # Read/write mix - realistic contention + + +class ColumnType(Enum): + """Column types for schema generation.""" + INT = "integer" + BIGINT = "bigint" + TEXT = "text" + BOOLEAN = "boolean" + UUID = "uuid" + TIMESTAMP = "timestamp" + FLOAT = "double precision" + NUMERIC = "numeric(12,2)" + JSONB = "jsonb" + + +# Row counts for testing atomic overhead at different scales +ROW_COUNTS = [1_000, 10_000, 100_000, 1_000_000, 10_000_000] + +# Default subset for quick runs +DEFAULT_ROW_COUNTS = [10_000, 100_000, 1_000_000] + + +@dataclass +class PostgresInstance: + """Configuration for a PostgreSQL instance (traditional or stdatomic build).""" + name: str # "traditional" or "stdatomic" + build_dir: Path # Path to build or install directory + host: str = "localhost" + port: int = 5432 + data_dir: Optional[Path] = None + user: str = "" + password: str = "" + + def _is_install_dir(self) -> bool: + """Check if build_dir is an install directory (has bin/ subdirectory).""" + return (self.build_dir / "bin").exists() + + @property + def postgres_bin(self) -> Path: + """Path to postgres binary.""" + if self._is_install_dir(): + return self.build_dir / "bin" / "postgres" + return self.build_dir / "src" / "backend" / "postgres" + + @property + def initdb_bin(self) -> Path: + """Path to initdb binary.""" + if self._is_install_dir(): + return self.build_dir / "bin" / "initdb" + return self.build_dir / "src" / "bin" / "initdb" / "initdb" + + @property + def pg_ctl_bin(self) -> Path: + """Path to pg_ctl binary.""" + if self._is_install_dir(): + return self.build_dir / "bin" / "pg_ctl" + return self.build_dir / "src" / "bin" / "pg_ctl" / "pg_ctl" + + def get_dsn(self, database: str = "postgres") -> str: + """Get connection DSN.""" + parts = [ + f"host={self.host}", + f"port={self.port}", + f"dbname={database}", + ] + if self.user: + parts.append(f"user={self.user}") + if self.password: + parts.append(f"password={self.password}") + return " ".join(parts) + + +@dataclass +class TableSchema: + """Defines a table schema for benchmarking.""" + name: str + width: TableWidth + columns: List[tuple] # (col_name, ColumnType) + index_columns: List[str] = field(default_factory=list) + + @property + def column_names(self) -> List[str]: + return [c[0] for c in self.columns] + + @property + def column_types(self) -> List[ColumnType]: + return [c[1] for c in self.columns] + + +# Pre-defined table schemas for the test matrix +NARROW_SCHEMA = TableSchema( + name="atomics_narrow", + width=TableWidth.NARROW, + columns=[ + ("id", ColumnType.BIGINT), + ("val_int", ColumnType.INT), + ("val_text", ColumnType.TEXT), + ("flag", ColumnType.BOOLEAN), + ], + index_columns=["id"], +) + +MEDIUM_SCHEMA = TableSchema( + name="atomics_medium", + width=TableWidth.MEDIUM, + columns=[ + ("id", ColumnType.BIGINT), + ("category", ColumnType.INT), + ("amount", ColumnType.NUMERIC), + ("description", ColumnType.TEXT), + ("is_active", ColumnType.BOOLEAN), + ("created_at", ColumnType.TIMESTAMP), + ("ref_uuid", ColumnType.UUID), + ("score", ColumnType.FLOAT), + ("status_code", ColumnType.INT), + ("notes", ColumnType.TEXT), + ("metadata", ColumnType.JSONB), + ], + index_columns=["id", "category"], +) + + +def _build_wide_columns(): + """Build a wide schema with 55 columns covering all data types.""" + cols = [("id", ColumnType.BIGINT)] + # 8 INT columns + for i in range(1, 9): + cols.append((f"col_int_{i}", ColumnType.INT)) + # 5 BIGINT columns + for i in range(1, 6): + cols.append((f"col_bigint_{i}", ColumnType.BIGINT)) + # 8 TEXT columns + for i in range(1, 9): + cols.append((f"col_text_{i}", ColumnType.TEXT)) + # 6 BOOLEAN columns + for i in range(1, 7): + cols.append((f"col_bool_{i}", ColumnType.BOOLEAN)) + # 5 FLOAT columns + for i in range(1, 6): + cols.append((f"col_float_{i}", ColumnType.FLOAT)) + # 5 NUMERIC columns + for i in range(1, 6): + cols.append((f"col_numeric_{i}", ColumnType.NUMERIC)) + # 5 UUID columns + for i in range(1, 6): + cols.append((f"col_uuid_{i}", ColumnType.UUID)) + # 5 TIMESTAMP columns + for i in range(1, 6): + cols.append((f"col_ts_{i}", ColumnType.TIMESTAMP)) + # 4 JSONB columns + for i in range(1, 5): + cols.append((f"col_jsonb_{i}", ColumnType.JSONB)) + # 3 more INT columns to reach 55 + for i in range(9, 12): + cols.append((f"col_int_{i}", ColumnType.INT)) + return cols + + +WIDE_SCHEMA = TableSchema( + name="atomics_wide", + width=TableWidth.WIDE, + columns=_build_wide_columns(), + index_columns=["id", "col_int_1", "col_text_1"], +) + +ALL_SCHEMAS = [NARROW_SCHEMA, MEDIUM_SCHEMA, WIDE_SCHEMA] + + +@dataclass +class AtomicsBenchmarkConfig: + """Configuration for atomics performance comparison.""" + # PostgreSQL instances to compare + traditional_instance: Optional[PostgresInstance] = None + stdatomic_instance: Optional[PostgresInstance] = None + + # Test matrix + schemas: List[TableSchema] = field(default_factory=lambda: list(ALL_SCHEMAS)) + row_counts: List[int] = field(default_factory=lambda: list(DEFAULT_ROW_COUNTS)) + distributions: List[DataDistribution] = field( + default_factory=lambda: [ + DataDistribution.RANDOM, + DataDistribution.CLUSTERED, + DataDistribution.LOW_CARDINALITY, + ] + ) + query_patterns: List[QueryPattern] = field( + default_factory=lambda: [ + QueryPattern.FULL_SCAN, + QueryPattern.COLUMN_PROJECTION, + QueryPattern.FILTERED_SCAN, + QueryPattern.AGGREGATION, + QueryPattern.GROUP_BY, + QueryPattern.INDEX_SCAN, + ] + ) + + # Execution parameters + warmup_iterations: int = 3 + measure_iterations: int = 10 + concurrent_clients: int = 8 # For concurrent workload tests + seed: int = 42 + + # Output + output_dir: str = "atomics_benchmark_results" + verbose: bool = False + + # Run the full matrix or a reduced subset + full_matrix: bool = False + + def __post_init__(self): + """Initialize default instances if not provided.""" + if self.traditional_instance is None: + build_dir = Path.cwd() / "build-traditional" + if not build_dir.exists(): + build_dir = Path.cwd() / "build" + self.traditional_instance = PostgresInstance( + name="traditional", + build_dir=build_dir, + port=5433, + data_dir=Path("/tmp/pgdata-traditional"), + ) + + if self.stdatomic_instance is None: + build_dir = Path.cwd() / "build-stdatomic" + self.stdatomic_instance = PostgresInstance( + name="stdatomic", + build_dir=build_dir, + port=5434, + data_dir=Path("/tmp/pgdata-stdatomic"), + ) + + def get_row_counts(self) -> List[int]: + if self.full_matrix: + return ROW_COUNTS + return self.row_counts diff --git a/src/test/benchmarks/atomics_workload.py b/src/test/benchmarks/atomics_workload.py new file mode 100644 index 0000000000000..0a64e97453570 --- /dev/null +++ b/src/test/benchmarks/atomics_workload.py @@ -0,0 +1,304 @@ +""" +Workload runner for atomics benchmarking. + +Executes query patterns that stress different atomic operations: +- Buffer management (spinlocks) +- Tuple visibility (atomic reads) +- Lock management (test-and-set) +- Shared memory access (memory ordering) +""" + +import asyncio +import asyncpg +import logging +import time +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from .atomics_config import ColumnType, QueryPattern, TableSchema + +logger = logging.getLogger(__name__) + + +@dataclass +class QueryResult: + """Result of a single query execution.""" + query_pattern: str + table_name: str + implementation: str # "traditional" or "stdatomic" + query_sql: str + elapsed_seconds: float + row_count: int = 0 + explain_plan: Optional[Dict[str, Any]] = None + + +@dataclass +class WorkloadResult: + """Aggregated results for a complete workload run.""" + schema_name: str + row_count: int + distribution: str + implementation: str + results: List[QueryResult] = field(default_factory=list) + + def add(self, result: QueryResult): + self.results.append(result) + + +class AtomicsWorkloadRunner: + """Generates and executes query workloads for atomics testing.""" + + def __init__( + self, + pool: asyncpg.Pool, + warmup_iterations: int = 3, + measure_iterations: int = 10, + ): + self.pool = pool + self.warmup_iterations = warmup_iterations + self.measure_iterations = measure_iterations + + # ------------------------------------------------------------------ + # Query generators per pattern + # ------------------------------------------------------------------ + + def _full_scan_query(self, table_name: str, schema: TableSchema) -> str: + """Full table scan - stresses spinlock acquisition and buffer management.""" + return f"SELECT * FROM {table_name}" + + def _column_projection_query(self, table_name: str, schema: TableSchema) -> str: + """Column projection - tuple visibility checks with atomic reads.""" + cols = [c[0] for c in schema.columns if c[0] != "id"][:2] + if not cols: + cols = [schema.columns[0][0]] + return f"SELECT {', '.join(cols)} FROM {table_name}" + + def _filtered_scan_query(self, table_name: str, schema: TableSchema) -> str: + """Filtered scan - mixed index and sequential access patterns.""" + for col_name, col_type in schema.columns: + if col_type == ColumnType.INT and col_name != "id": + return f"SELECT * FROM {table_name} WHERE {col_name} > 0" + if col_type == ColumnType.BOOLEAN: + return f"SELECT * FROM {table_name} WHERE {col_name} = TRUE" + return f"SELECT * FROM {table_name} WHERE id > 100 AND id <= 10000" + + def _aggregation_query(self, table_name: str, schema: TableSchema) -> str: + """Aggregation - heavy shared buffer access with memory ordering stress.""" + agg_exprs = [] + for col_name, col_type in schema.columns: + if col_type in ( + ColumnType.INT, + ColumnType.BIGINT, + ColumnType.FLOAT, + ColumnType.NUMERIC, + ): + agg_exprs.append(f"SUM({col_name})") + agg_exprs.append(f"AVG({col_name})") + if len(agg_exprs) >= 6: + break + if not agg_exprs: + agg_exprs = ["COUNT(*)"] + return f"SELECT COUNT(*), {', '.join(agg_exprs)} FROM {table_name}" + + def _group_by_query(self, table_name: str, schema: TableSchema) -> str: + """Group-by with hash aggregation - memory ordering and buffer management.""" + group_col = None + agg_col = None + for col_name, col_type in schema.columns: + if col_name == "id": + continue + if col_type in (ColumnType.INT, ColumnType.BOOLEAN) and group_col is None: + group_col = col_name + if ( + col_type + in (ColumnType.FLOAT, ColumnType.NUMERIC, ColumnType.INT, ColumnType.BIGINT) + and agg_col is None + ): + agg_col = col_name + + if group_col is None: + group_col = schema.columns[0][0] + if agg_col is None: + agg_col = "id" + + return ( + f"SELECT {group_col}, COUNT(*), SUM({agg_col}), AVG({agg_col}) " + f"FROM {table_name} GROUP BY {group_col}" + ) + + def _index_scan_query(self, table_name: str, schema: TableSchema) -> str: + """Index scan - buffer pin/unpin contention.""" + return f"SELECT * FROM {table_name} WHERE id BETWEEN 100 AND 200" + + def _concurrent_insert_query(self, table_name: str, schema: TableSchema) -> str: + """Concurrent inserts - high contention lock management.""" + # Generate a simple insert + col_names = schema.column_names + values = [] + for col_name, col_type in schema.columns: + if col_name == "id": + values.append("nextval('seq_concurrent_insert')") + elif col_type == ColumnType.INT: + values.append("42") + elif col_type == ColumnType.BIGINT: + values.append("12345") + elif col_type == ColumnType.TEXT: + values.append("'test'") + elif col_type == ColumnType.BOOLEAN: + values.append("TRUE") + elif col_type == ColumnType.UUID: + values.append("gen_random_uuid()") + elif col_type == ColumnType.TIMESTAMP: + values.append("NOW()") + elif col_type == ColumnType.FLOAT: + values.append("3.14") + elif col_type == ColumnType.NUMERIC: + values.append("99.99") + elif col_type == ColumnType.JSONB: + values.append("'{}'::jsonb") + else: + values.append("NULL") + + return f"INSERT INTO {table_name} ({', '.join(col_names)}) VALUES ({', '.join(values)})" + + def _concurrent_update_query(self, table_name: str, schema: TableSchema) -> str: + """Concurrent updates - lock management and visibility checks.""" + update_col = None + for col_name, col_type in schema.columns: + if col_name != "id" and col_type in (ColumnType.INT, ColumnType.BIGINT): + update_col = col_name + break + + if update_col is None: + update_col = "id" + + return f"UPDATE {table_name} SET {update_col} = {update_col} + 1 WHERE id <= 100" + + def _mixed_workload_query(self, table_name: str, schema: TableSchema) -> str: + """Mixed read/write workload - realistic contention patterns.""" + return f""" + WITH updated AS ( + UPDATE {table_name} SET id = id WHERE id <= 10 RETURNING * + ) + SELECT COUNT(*) FROM {table_name} WHERE id > 10 + """ + + def _get_query( + self, pattern: QueryPattern, table_name: str, schema: TableSchema + ) -> str: + """Get query for a given pattern.""" + generators = { + QueryPattern.FULL_SCAN: self._full_scan_query, + QueryPattern.COLUMN_PROJECTION: self._column_projection_query, + QueryPattern.FILTERED_SCAN: self._filtered_scan_query, + QueryPattern.AGGREGATION: self._aggregation_query, + QueryPattern.GROUP_BY: self._group_by_query, + QueryPattern.INDEX_SCAN: self._index_scan_query, + QueryPattern.CONCURRENT_INSERT: self._concurrent_insert_query, + QueryPattern.CONCURRENT_UPDATE: self._concurrent_update_query, + QueryPattern.MIXED_WORKLOAD: self._mixed_workload_query, + } + gen = generators.get(pattern) + if gen is None: + raise ValueError(f"Unknown query pattern: {pattern}") + return gen(table_name, schema) + + # ------------------------------------------------------------------ + # Execution + # ------------------------------------------------------------------ + + async def _run_single( + self, + query: str, + pattern: QueryPattern, + table_name: str, + implementation: str, + collect_explain: bool = False, + ) -> QueryResult: + """Run a single query, returning timing and optional EXPLAIN data.""" + # Warmup + async with self.pool.acquire() as conn: + for _ in range(self.warmup_iterations): + try: + await conn.fetch(query) + except Exception as e: + logger.warning("Warmup query failed: %s", e) + + # Measure + timings = [] + for _ in range(self.measure_iterations): + start = time.perf_counter() + async with self.pool.acquire() as conn: + try: + rows = await conn.fetch(query) + elapsed = time.perf_counter() - start + timings.append(elapsed) + except Exception as e: + logger.error("Measurement query failed: %s", e) + elapsed = 0.0 + timings.append(elapsed) + + # Use median timing + timings.sort() + median_time = timings[len(timings) // 2] + + # Optional EXPLAIN + explain_plan = None + if collect_explain: + try: + async with self.pool.acquire() as conn: + explain_result = await conn.fetch( + f"EXPLAIN (FORMAT JSON, ANALYZE, BUFFERS) {query}" + ) + if explain_result: + explain_plan = explain_result[0][0] + except Exception as e: + logger.warning("EXPLAIN failed: %s", e) + + return QueryResult( + query_pattern=pattern.value, + table_name=table_name, + implementation=implementation, + query_sql=query, + elapsed_seconds=median_time, + row_count=len(rows) if rows else 0, + explain_plan=explain_plan, + ) + + async def run_workload( + self, + schema: TableSchema, + table_name: str, + row_count: int, + distribution: str, + patterns: List[QueryPattern], + ) -> WorkloadResult: + """Run complete workload and return results.""" + result = WorkloadResult( + schema_name=schema.name, + row_count=row_count, + distribution=distribution, + implementation="", # Set by caller + ) + + for pattern in patterns: + try: + query = self._get_query(pattern, table_name, schema) + query_result = await self._run_single( + query=query, + pattern=pattern, + table_name=table_name, + implementation=result.implementation, + collect_explain=False, # Too expensive for full matrix + ) + result.add(query_result) + + logger.info( + " %s: %.2fms", + pattern.value, + query_result.elapsed_seconds * 1000, + ) + except Exception as e: + logger.error("Pattern %s failed: %s", pattern.value, e, exc_info=True) + + return result diff --git a/src/test/benchmarks/data_generator.py b/src/test/benchmarks/data_generator.py new file mode 100644 index 0000000000000..2366700c6ee9c --- /dev/null +++ b/src/test/benchmarks/data_generator.py @@ -0,0 +1,409 @@ +""" +Reproducible seeded random data generation for benchmark tables. + +Generates SQL INSERT statements or COPY-compatible data for various +column types and data distributions. +""" + +import hashlib +import logging +import random +import uuid +from datetime import datetime, timedelta +from typing import Any, List, Optional + +from .atomics_config import ColumnType, DataDistribution, TableSchema + +logger = logging.getLogger(__name__) + +# Low-cardinality value pools +LOW_CARD_TEXT = [ + "active", "inactive", "pending", "completed", "cancelled", + "processing", "shipped", "returned", "refunded", "on_hold", +] +LOW_CARD_INT_RANGE = 20 +LOW_CARD_STATUS_CODES = [100, 200, 201, 301, 400, 403, 404, 500, 502, 503] + +# Clustered parameters +CLUSTER_CENTERS = 5 +CLUSTER_SPREAD = 100 + +# Base timestamp for reproducible timestamp generation +BASE_TS = datetime(2020, 1, 1) + + +class DataGenerator: + """Generates reproducible test data for benchmark tables.""" + + def __init__(self, seed: int = 42): + self.seed = seed + self._rng = random.Random(seed) + + def reset(self): + """Reset the RNG to produce identical sequences.""" + self._rng = random.Random(self.seed) + + # ------------------------------------------------------------------ + # Value generators per column type and distribution + # ------------------------------------------------------------------ + + def _gen_int(self, dist: DataDistribution, row_idx: int) -> int: + if dist == DataDistribution.RANDOM: + return self._rng.randint(-2_147_483_648, 2_147_483_647) + elif dist == DataDistribution.CLUSTERED: + center = (row_idx % CLUSTER_CENTERS) * 1_000_000 + return center + self._rng.randint(-CLUSTER_SPREAD, CLUSTER_SPREAD) + else: # LOW_CARDINALITY + return self._rng.choice(LOW_CARD_STATUS_CODES) + + def _gen_bigint(self, dist: DataDistribution, row_idx: int) -> int: + if dist == DataDistribution.RANDOM: + return self._rng.randint(0, 2**62) + elif dist == DataDistribution.CLUSTERED: + center = (row_idx % CLUSTER_CENTERS) * 10_000_000_000 + return center + self._rng.randint(-1000, 1000) + else: + return self._rng.randint(1, LOW_CARD_INT_RANGE) + + def _gen_text(self, dist: DataDistribution, row_idx: int) -> str: + if dist == DataDistribution.RANDOM: + # MD5-like random string + h = hashlib.md5(f"{self.seed}-{row_idx}-{self._rng.random()}".encode()) + return h.hexdigest() + elif dist == DataDistribution.CLUSTERED: + group = row_idx % CLUSTER_CENTERS + suffix = self._rng.randint(0, CLUSTER_SPREAD) + return f"group_{group}_item_{suffix}" + else: + return self._rng.choice(LOW_CARD_TEXT) + + def _gen_boolean(self, dist: DataDistribution, row_idx: int) -> bool: + if dist == DataDistribution.RANDOM: + return self._rng.random() < 0.5 + elif dist == DataDistribution.CLUSTERED: + # Runs of True/False + return (row_idx // 100) % 2 == 0 + else: + # Heavily skewed: 95% True + return self._rng.random() < 0.95 + + def _gen_uuid(self, dist: DataDistribution, row_idx: int) -> str: + if dist == DataDistribution.LOW_CARDINALITY: + # Only 10 distinct UUIDs + idx = row_idx % 10 + return str(uuid.UUID(int=idx + 1)) + # For RANDOM and CLUSTERED, use seeded generation + bits = self._rng.getrandbits(128) + return str(uuid.UUID(int=bits, version=4)) + + def _gen_timestamp(self, dist: DataDistribution, row_idx: int) -> str: + if dist == DataDistribution.RANDOM: + days = self._rng.randint(0, 1825) # ~5 years + secs = self._rng.randint(0, 86400) + ts = BASE_TS + timedelta(days=days, seconds=secs) + elif dist == DataDistribution.CLUSTERED: + # Clustered around specific dates + center_day = (row_idx % CLUSTER_CENTERS) * 365 + offset = self._rng.randint(-30, 30) + ts = BASE_TS + timedelta(days=center_day + offset) + else: + # Low cardinality: 10 distinct dates + day_idx = row_idx % 10 + ts = BASE_TS + timedelta(days=day_idx * 100) + return ts.strftime("%Y-%m-%d %H:%M:%S") + + def _gen_float(self, dist: DataDistribution, row_idx: int) -> float: + if dist == DataDistribution.RANDOM: + return self._rng.uniform(-1e6, 1e6) + elif dist == DataDistribution.CLUSTERED: + center = (row_idx % CLUSTER_CENTERS) * 1000.0 + return center + self._rng.gauss(0, 10) + else: + return self._rng.choice([0.0, 1.0, 10.0, 100.0, 1000.0]) + + def _gen_numeric(self, dist: DataDistribution, row_idx: int) -> str: + val = self._gen_float(dist, row_idx) + return f"{val:.2f}" + + def _gen_jsonb(self, dist: DataDistribution, row_idx: int) -> str: + import json + if dist == DataDistribution.RANDOM: + obj = { + "key": self._rng.randint(1, 100000), + "label": hashlib.md5(f"{self.seed}-json-{row_idx}".encode()).hexdigest()[:8], + "value": round(self._rng.uniform(0, 1000), 2), + "active": self._rng.random() < 0.5, + } + elif dist == DataDistribution.CLUSTERED: + group = row_idx % CLUSTER_CENTERS + obj = { + "group": group, + "label": f"cluster_{group}", + "value": group * 100 + self._rng.randint(0, CLUSTER_SPREAD), + } + elif dist == DataDistribution.HIGH_NULL: + # HIGH_NULL: return None most of the time (handled in _gen_value) + obj = {"id": row_idx % 10, "status": self._rng.choice(LOW_CARD_TEXT)} + else: # LOW_CARDINALITY + obj = {"id": row_idx % 10, "status": self._rng.choice(LOW_CARD_TEXT)} + return json.dumps(obj) + + def _gen_value( + self, col_type: ColumnType, dist: DataDistribution, row_idx: int + ) -> Any: + # HIGH_NULL distribution: ~80% of non-id values are NULL + if dist == DataDistribution.HIGH_NULL and col_type != ColumnType.BIGINT: + if self._rng.random() < 0.80: + return None + + generators = { + ColumnType.INT: self._gen_int, + ColumnType.BIGINT: self._gen_bigint, + ColumnType.TEXT: self._gen_text, + ColumnType.BOOLEAN: self._gen_boolean, + ColumnType.UUID: self._gen_uuid, + ColumnType.TIMESTAMP: self._gen_timestamp, + ColumnType.FLOAT: self._gen_float, + ColumnType.NUMERIC: self._gen_numeric, + ColumnType.JSONB: self._gen_jsonb, + } + gen = generators.get(col_type) + if gen is None: + raise ValueError(f"Unsupported column type: {col_type}") + return gen(dist, row_idx) + + # ------------------------------------------------------------------ + # SQL generation helpers + # ------------------------------------------------------------------ + + def generate_insert_sql( + self, + schema: TableSchema, + row_count: int, + dist: DataDistribution, + table_suffix: str = "", + batch_size: int = 1000, + ) -> List[str]: + """Generate INSERT statements in batches for the given schema. + + Returns a list of SQL strings, each inserting up to batch_size rows. + The ``id`` column is always set to the sequential row index. + """ + self.reset() + col_defs = ", ".join(schema.column_names) + statements = [] + + for batch_start in range(0, row_count, batch_size): + batch_end = min(batch_start + batch_size, row_count) + rows_sql = [] + for i in range(batch_start, batch_end): + vals = [] + for col_name, col_type in schema.columns: + if col_name == "id": + vals.append(str(i + 1)) + else: + v = self._gen_value(col_type, dist, i) + vals.append(self._sql_literal(v, col_type)) + rows_sql.append(f"({', '.join(vals)})") + + table_name = f"{schema.name}{table_suffix}" + stmt = f"INSERT INTO {table_name} ({col_defs}) VALUES\n" + stmt += ",\n".join(rows_sql) + statements.append(stmt) + + return statements + + def generate_copy_data( + self, + schema: TableSchema, + row_count: int, + dist: DataDistribution, + ) -> str: + """Generate tab-separated COPY data for the given schema. + + Returns a single string suitable for COPY ... FROM STDIN. + """ + self.reset() + lines = [] + for i in range(row_count): + vals = [] + for col_name, col_type in schema.columns: + if col_name == "id": + vals.append(str(i + 1)) + else: + v = self._gen_value(col_type, dist, i) + vals.append(self._copy_literal(v, col_type)) + lines.append("\t".join(vals)) + return "\n".join(lines) + + def generate_server_side_insert( + self, + schema: TableSchema, + row_count: int, + dist: DataDistribution, + table_suffix: str = "", + ) -> str: + """Generate a single INSERT ... SELECT generate_series SQL statement. + + This is much faster for large datasets because it runs entirely + server-side without sending row data over the wire. + """ + table_name = f"{schema.name}{table_suffix}" + col_exprs = [] + for col_name, col_type in schema.columns: + if col_name == "id": + col_exprs.append("g AS id") + else: + col_exprs.append( + f"{self._server_side_expr(col_name, col_type, dist, row_count)} AS {col_name}" + ) + + select_list = ",\n ".join(col_exprs) + return ( + f"INSERT INTO {table_name} ({', '.join(schema.column_names)})\n" + f"SELECT {select_list}\n" + f"FROM generate_series(1, {row_count}) AS g" + ) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _sql_literal(value: Any, col_type: ColumnType) -> str: + if value is None: + return "NULL" + if col_type in (ColumnType.TEXT, ColumnType.UUID, ColumnType.TIMESTAMP): + escaped = str(value).replace("'", "''") + return f"'{escaped}'" + if col_type == ColumnType.JSONB: + escaped = str(value).replace("'", "''") + return f"'{escaped}'::jsonb" + if col_type == ColumnType.BOOLEAN: + return "TRUE" if value else "FALSE" + if col_type == ColumnType.NUMERIC: + return str(value) + return str(value) + + @staticmethod + def _copy_literal(value: Any, col_type: ColumnType) -> str: + if value is None: + return "\\N" + if col_type == ColumnType.BOOLEAN: + return "t" if value else "f" + return str(value) + + def _server_side_expr( + self, + col_name: str, + col_type: ColumnType, + dist: DataDistribution, + row_count: int, + ) -> str: + """Return a SQL expression that produces the desired distribution + server-side using generate_series variable ``g``.""" + + seed_val = self.seed + + # HIGH_NULL: wrap the underlying RANDOM expression so ~80% are NULL + if dist == DataDistribution.HIGH_NULL and col_type != ColumnType.BIGINT: + inner = self._server_side_expr( + col_name, col_type, DataDistribution.RANDOM, row_count + ) + return f"CASE WHEN abs(hashint4(g + {seed_val} + 99)) % 5 = 0 THEN {inner} ELSE NULL END" + + if col_type == ColumnType.INT: + if dist == DataDistribution.RANDOM: + return f"(hashint4(g + {seed_val}) % 2147483647)::integer" + elif dist == DataDistribution.CLUSTERED: + return f"((g % {CLUSTER_CENTERS}) * 1000000 + (hashint4(g + {seed_val}) % {CLUSTER_SPREAD}))::integer" + else: + codes = ",".join(str(c) for c in LOW_CARD_STATUS_CODES) + return f"(ARRAY[{codes}])[1 + abs(hashint4(g + {seed_val})) % {len(LOW_CARD_STATUS_CODES)}]" + + if col_type == ColumnType.BIGINT: + if dist == DataDistribution.RANDOM: + return f"(hashint8(g::bigint + {seed_val}) & x'3FFFFFFFFFFFFFFF'::bigint)::bigint" + elif dist == DataDistribution.CLUSTERED: + return f"((g % {CLUSTER_CENTERS})::bigint * 10000000000 + (hashint4(g + {seed_val}) % 1000)::bigint)" + else: + return f"(1 + abs(hashint4(g + {seed_val})) % {LOW_CARD_INT_RANGE})::bigint" + + if col_type == ColumnType.TEXT: + if dist == DataDistribution.RANDOM: + return f"md5(g::text || '{seed_val}')" + elif dist == DataDistribution.CLUSTERED: + return f"'group_' || (g % {CLUSTER_CENTERS})::text || '_item_' || (abs(hashint4(g + {seed_val})) % {CLUSTER_SPREAD})::text" + else: + texts = ",".join(f"'{t}'" for t in LOW_CARD_TEXT) + return f"(ARRAY[{texts}])[1 + abs(hashint4(g + {seed_val})) % {len(LOW_CARD_TEXT)}]" + + if col_type == ColumnType.BOOLEAN: + if dist == DataDistribution.RANDOM: + return f"(hashint4(g + {seed_val}) % 2 = 0)" + elif dist == DataDistribution.CLUSTERED: + return f"((g / 100) % 2 = 0)" + else: + return f"(abs(hashint4(g + {seed_val})) % 20 != 0)" + + if col_type == ColumnType.UUID: + if dist == DataDistribution.LOW_CARDINALITY: + return f"(lpad(((g % 10) + 1)::text, 32, '0'))::uuid" + return f"md5(g::text || '{seed_val}' || random()::text)::uuid" + + if col_type == ColumnType.TIMESTAMP: + if dist == DataDistribution.RANDOM: + return f"'2020-01-01'::timestamp + (abs(hashint4(g + {seed_val})) % 157680000) * interval '1 second'" + elif dist == DataDistribution.CLUSTERED: + return f"'2020-01-01'::timestamp + ((g % {CLUSTER_CENTERS}) * 365 + (abs(hashint4(g + {seed_val})) % 60) - 30) * interval '1 day'" + else: + return f"'2020-01-01'::timestamp + ((g % 10) * 100) * interval '1 day'" + + if col_type == ColumnType.FLOAT: + if dist == DataDistribution.RANDOM: + return f"(hashint4(g + {seed_val})::double precision / 2147483647.0 * 2000000 - 1000000)" + elif dist == DataDistribution.CLUSTERED: + return f"((g % {CLUSTER_CENTERS}) * 1000.0 + (hashint4(g + {seed_val}) % 100)::double precision / 10.0)" + else: + return f"(ARRAY[0.0, 1.0, 10.0, 100.0, 1000.0])[1 + abs(hashint4(g + {seed_val})) % 5]" + + if col_type == ColumnType.NUMERIC: + if dist == DataDistribution.RANDOM: + return f"round((hashint4(g + {seed_val})::numeric / 2147483647.0 * 2000000 - 1000000), 2)" + elif dist == DataDistribution.CLUSTERED: + return f"round(((g % {CLUSTER_CENTERS}) * 1000.0 + (hashint4(g + {seed_val}) % 100)::numeric / 10.0), 2)" + else: + return f"(ARRAY[0.00, 1.00, 10.00, 100.00, 1000.00])[1 + abs(hashint4(g + {seed_val})) % 5]::numeric(12,2)" + + if col_type == ColumnType.JSONB: + if dist == DataDistribution.RANDOM: + return ( + f"jsonb_build_object(" + f"'key', abs(hashint4(g + {seed_val})) % 100000, " + f"'label', left(md5(g::text || '{seed_val}'), 8), " + f"'value', round((hashint4(g + {seed_val})::numeric / 2147483647.0 * 1000), 2), " + f"'active', (hashint4(g + {seed_val}) % 2 = 0))" + ) + elif dist == DataDistribution.CLUSTERED: + return ( + f"jsonb_build_object(" + f"'group', g % {CLUSTER_CENTERS}, " + f"'label', 'cluster_' || (g % {CLUSTER_CENTERS})::text, " + f"'value', (g % {CLUSTER_CENTERS}) * 100 + abs(hashint4(g + {seed_val})) % {CLUSTER_SPREAD})" + ) + elif dist == DataDistribution.HIGH_NULL: + return ( + f"CASE WHEN abs(hashint4(g + {seed_val})) % 5 = 0 THEN " + f"jsonb_build_object('id', g % 10, 'status', " + f"(ARRAY[{','.join(repr(t) for t in LOW_CARD_TEXT)}])" + f"[1 + abs(hashint4(g + {seed_val} + 1)) % {len(LOW_CARD_TEXT)}]) " + f"ELSE NULL END" + ) + else: # LOW_CARDINALITY + texts = ",".join(f"'{t}'" for t in LOW_CARD_TEXT) + return ( + f"jsonb_build_object('id', g % 10, 'status', " + f"(ARRAY[{texts}])[1 + abs(hashint4(g + {seed_val})) % {len(LOW_CARD_TEXT)}])" + ) + + raise ValueError(f"Unsupported column type for server-side generation: {col_type}") From fc39412f51d2726c696eb866a3b43a483fc4f7ee Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 11 Jul 2026 18:42:50 -0400 Subject: [PATCH 6/6] [NOT FOR MERGE... yet.] Remove traditional atomics; use C11 stdatomic.h exclusively This commit is included to show the intended long-term destination of the series, and to let reviewers and the buildfarm exercise the stdatomic-only end state -- but it is deliberately NOT proposed for commit yet. The plan is to ship the earlier patches (which add C11 as a *selectable* implementation alongside the traditional platform-specific code), let the stdatomic path prove itself in the field across the buildfarm and at least one release, and only THEN remove the traditional implementation via a commit like this one. Removing the battle-tested arch-*/generic-* code on day one is not being asked for here; keeping both paths until stdatomic has earned trust is. Everything below describes what this (eventual) removal commit does. This collapses the dual-path atomics into a single stdatomic-only implementation, mirroring the destination state of the series. Rather than selecting between C11 and the platform-specific implementations at build time, stdatomic.h is now the sole backend for PostgreSQL's atomic operations and USE_STDATOMIC_H is effectively always-on. The platform-specific implementation files are deleted outright: - src/include/port/atomics/arch-arm.h - src/include/port/atomics/arch-ppc.h - src/include/port/atomics/arch-x86.h - src/include/port/atomics/generic.h - src/include/port/atomics/generic-gcc.h - src/include/port/atomics/generic-msvc.h - src/include/port/atomics/fallback.h atomics.h drops its #ifdef USE_STDATOMIC_H / #else conditional and the now-dead includes of the arch-*/generic-*/fallback headers, keeping only the stdatomic_impl.h include and the common public-API layer of static inline pg_atomic_* wrappers. storage/spin.h and s_lock.c lose their traditional-spinlock (#else) branches, so only the pg_atomic_flag-based spinlock path remains; the s_lock.h includes that served only that path are removed. Detection is retained but now hard-errors when a working C11 stdatomic.h is unavailable, in both meson.build and configure.ac, instead of silently falling back. The only remaining hand-written platform hint is the CPU spin-delay instruction in src/include/port/spin_delay.h, which stdatomic.h does not cover; it is intentionally left in place along with spin_delay_status.h and stdatomic_impl.h. This depends on the prior commits in the series (build-time detection, stdatomic_impl.h, conditional wire-up, documentation, and the test_atomics module). Co-authored-by: Greg Burd Co-authored-by: Thomas Munro --- configure | 80 +-- configure.ac | 79 +-- meson.build | 45 +- meson_options.txt | 5 - src/backend/storage/lmgr/.gitignore | 1 - src/backend/storage/lmgr/Makefile | 9 - src/backend/storage/lmgr/README | 59 +- src/backend/storage/lmgr/s_lock.c | 119 ---- src/include/port/atomics.h | 173 +---- src/include/port/atomics/arch-arm.h | 32 - src/include/port/atomics/arch-ppc.h | 256 -------- src/include/port/atomics/arch-x86.h | 189 ------ src/include/port/atomics/fallback.h | 42 -- src/include/port/atomics/generic-gcc.h | 326 ---------- src/include/port/atomics/generic-msvc.h | 115 ---- src/include/port/atomics/generic.h | 430 ------------ src/include/port/atomics/stdatomic_impl.h | 16 +- src/include/storage/s_lock.h | 753 ---------------------- src/include/storage/spin.h | 49 +- src/test/regress/regress.c | 38 +- src/tools/pginclude/headerscheck | 10 - 21 files changed, 123 insertions(+), 2703 deletions(-) delete mode 100644 src/include/port/atomics/arch-arm.h delete mode 100644 src/include/port/atomics/arch-ppc.h delete mode 100644 src/include/port/atomics/arch-x86.h delete mode 100644 src/include/port/atomics/fallback.h delete mode 100644 src/include/port/atomics/generic-gcc.h delete mode 100644 src/include/port/atomics/generic-msvc.h delete mode 100644 src/include/port/atomics/generic.h delete mode 100644 src/include/storage/s_lock.h diff --git a/configure b/configure index f9853829bd356..8e0f521e24750 100755 --- a/configure +++ b/configure @@ -759,7 +759,6 @@ LLVM_LIBS CLANG LLVM_CONFIG AWK -with_stdatomic with_llvm have_cxx ac_ct_CXX @@ -858,7 +857,6 @@ with_segsize with_segsize_blocks with_wal_blocksize with_llvm -with_stdatomic enable_depend enable_cassert with_icu @@ -1578,8 +1576,6 @@ Optional Packages: --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] --with-llvm build with LLVM based JIT support - --with-stdatomic[=yes/no/auto] - use C11 stdatomic.h for atomic operations [auto] --without-icu build without ICU support --with-tcl build Tcl modules (PL/Tcl) --with-tclconfig=DIR tclConfig.sh is in DIR @@ -4925,22 +4921,9 @@ fi # # C11 stdatomic.h # -# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses -# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the -# traditional platform-specific atomics. The actual detection runs later, -# after the compiler and 64-bit integer handling have been configured. - -# Check whether --with-stdatomic was given. -if test "${with_stdatomic+set}" = set; then : - withval=$with_stdatomic; case $withval in - yes | no | auto) ;; - *) as_fn_error $? "invalid argument to --with-stdatomic; use yes, no, or auto" "$LINENO" 5 ;; - esac -else - with_stdatomic=no -fi - - +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17742,32 +17725,30 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi -# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The -# probe mirrors the one used by the meson build: it must compile 32-bit -# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. -if test "$with_stdatomic" != no; then - # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() - # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit - # or weakly-aligned targets. A compile-only check would pass there and then - # fail at final link. Retry with -latomic if the bare link fails. - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 $as_echo_n "checking for working C11 stdatomic.h... " >&6; } if ${pgac_cv_stdatomic+:} false; then : $as_echo_n "(cached) " >&6 else pgac_cv_stdatomic=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { _Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); ; return 0; @@ -17777,19 +17758,19 @@ if ac_fn_c_try_link "$LINENO"; then : pgac_cv_stdatomic=yes else pgac_save_LIBS=$LIBS - LIBS="$LIBS -latomic" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { _Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); ; return 0; @@ -17802,28 +17783,23 @@ else fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext - LIBS=$pgac_save_LIBS + LIBS=$pgac_save_LIBS fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 $as_echo "$pgac_cv_stdatomic" >&6; } -else - pgac_cv_stdatomic=no -fi -if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then - as_fn_error $? "--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found" "$LINENO" 5 +if test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "a working C11 stdatomic.h is required (it is the only atomics implementation)" "$LINENO" 5 fi -if test "$pgac_cv_stdatomic" != no; then $as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h - if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then - LIBS="$LIBS -latomic" - fi +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" fi diff --git a/configure.ac b/configure.ac index a3d4e31a60947..da7a6e1014d9d 100644 --- a/configure.ac +++ b/configure.ac @@ -447,19 +447,9 @@ AC_SUBST(with_llvm) # # C11 stdatomic.h # -# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses -# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the -# traditional platform-specific atomics. The actual detection runs later, -# after the compiler and 64-bit integer handling have been configured. -AC_ARG_WITH(stdatomic, - [AS_HELP_STRING([--with-stdatomic@<:@=yes/no/auto@:>@], - [use C11 stdatomic.h for atomic operations @<:@auto@:>@])], - [case $withval in - yes | no | auto) ;; - *) AC_MSG_ERROR([invalid argument to --with-stdatomic; use yes, no, or auto]) ;; - esac], - [with_stdatomic=no]) -AC_SUBST(with_stdatomic) +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2118,16 +2108,25 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS -# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The -# probe mirrors the one used by the meson build: it must compile 32-bit -# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. -if test "$with_stdatomic" != no; then - # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() - # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit - # or weakly-aligned targets. A compile-only check would pass there and then - # fail at final link. Retry with -latomic if the bare link fails. - AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], - [pgac_cv_stdatomic=no +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], +[pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], [[_Atomic int x = 0; atomic_fetch_add(&x, 1); @@ -2136,34 +2135,18 @@ if test "$with_stdatomic" != no; then unsigned long long expected = 0; atomic_compare_exchange_strong(&y, &expected, 42); ]])], - [pgac_cv_stdatomic=yes], - [pgac_save_LIBS=$LIBS - LIBS="$LIBS -latomic" - AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], - [[_Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); - ]])], - [pgac_cv_stdatomic='yes, with -latomic'], - [pgac_cv_stdatomic=no]) - LIBS=$pgac_save_LIBS])]) -else - pgac_cv_stdatomic=no -fi + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) -if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then - AC_MSG_ERROR([--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found]) +if test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([a working C11 stdatomic.h is required (it is the only atomics implementation)]) fi -if test "$pgac_cv_stdatomic" != no; then - AC_DEFINE([USE_STDATOMIC_H], 1, - [Define to 1 to use C11 stdatomic.h for atomic operations. (--with-stdatomic)]) - if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then - LIBS="$LIBS -latomic" - fi +AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations.]) +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" fi diff --git a/meson.build b/meson.build index f4e72386deb7d..9132df40b167d 100644 --- a/meson.build +++ b/meson.build @@ -692,7 +692,6 @@ int main(void) { ''' has_stdatomic = false -use_stdatomic = get_option('use_stdatomic') atomic_link_args = [] # The stdatomic probe below is a *link* test, not just a compile test: 64-bit @@ -744,36 +743,20 @@ else endif endif -# Process the use_stdatomic option -if use_stdatomic == 'yes' - if not has_stdatomic - error('stdatomic.h was requested with -Duse_stdatomic=yes but is not available or not working') - endif - cdata.set('USE_STDATOMIC_H', 1) - message('Using C11 stdatomic.h for atomic operations (forced by -Duse_stdatomic=yes)') - if cc.get_id() == 'msvc' - # MSVC needs C11 mode plus the experimental atomics switch for . - cflags += ['/std:c11', '/experimental:c11atomics'] - endif - if atomic_link_args.length() > 0 - os_deps += cc.find_library('atomic') - endif -elif use_stdatomic == 'no' - message('Using traditional platform-specific atomics (forced by -Duse_stdatomic=no)') - # Do not set USE_STDATOMIC_H -elif use_stdatomic == 'auto' - if has_stdatomic - cdata.set('USE_STDATOMIC_H', 1) - message('Using C11 stdatomic.h for atomic operations (auto-detected)') - if cc.get_id() == 'msvc' - cflags += ['/std:c11', '/experimental:c11atomics'] - endif - if atomic_link_args.length() > 0 - os_deps += cc.find_library('atomic') - endif - else - message('Using traditional platform-specific atomics (stdatomic.h not available)') - endif +# stdatomic.h is the sole atomics implementation, so a missing or broken +# header is a hard error. +if not has_stdatomic + error('a working C11 stdatomic.h is required (it is the only atomics implementation)') +endif +cdata.set('USE_STDATOMIC_H', 1) +message('Using C11 stdatomic.h for atomic operations') +if atomic_link_args.length() > 0 + # libatomic is needed for atomics the compiler cannot inline on this target. + os_deps += cc.find_library('atomic') +endif +if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] endif postgres_inc = [include_directories(postgres_inc_d)] diff --git a/meson_options.txt b/meson_options.txt index f8c5ccc14e114..6a793f3e47943 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,11 +52,6 @@ option('PG_TEST_EXTRA', type: 'string', value: '', option('PG_GIT_REVISION', type: 'string', value: 'HEAD', description: 'git revision to be packaged by pgdist target') -option('use_stdatomic', type: 'combo', - choices: ['auto', 'yes', 'no'], - value: 'no', - description: 'Use C11 stdatomic.h for atomic operations (auto, yes, no)') - # Compilation options diff --git a/src/backend/storage/lmgr/.gitignore b/src/backend/storage/lmgr/.gitignore index 8e5b734f15296..209c8be722369 100644 --- a/src/backend/storage/lmgr/.gitignore +++ b/src/backend/storage/lmgr/.gitignore @@ -1,2 +1 @@ /lwlocknames.h -/s_lock_test diff --git a/src/backend/storage/lmgr/Makefile b/src/backend/storage/lmgr/Makefile index a5fbc24ddad6e..a60e70ef0e1e7 100644 --- a/src/backend/storage/lmgr/Makefile +++ b/src/backend/storage/lmgr/Makefile @@ -24,17 +24,8 @@ OBJS = \ include $(top_srcdir)/src/backend/common.mk -s_lock_test: s_lock.c $(top_builddir)/src/common/libpgcommon.a $(top_builddir)/src/port/libpgport.a - $(CC) $(CPPFLAGS) $(CFLAGS) -DS_LOCK_TEST=1 $(srcdir)/s_lock.c \ - -L $(top_builddir)/src/common -lpgcommon \ - -L $(top_builddir)/src/port -lpgport -lm -o s_lock_test - lwlocknames.h: ../../../include/storage/lwlocklist.h ../../utils/activity/wait_event_names.txt generate-lwlocknames.pl $(PERL) $(srcdir)/generate-lwlocknames.pl $^ -check: s_lock_test - ./s_lock_test - clean: - rm -f s_lock_test rm -f lwlocknames.h diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 460b955944959..b55911396db1f 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -50,41 +50,28 @@ The rest of this README file discusses the regular lock manager in detail. Atomic Operations ================= -PostgreSQL's atomic operations (see src/include/port/atomics.h) can be -implemented using either C11 stdatomic.h or traditional platform-specific -code. The implementation is selected at build time via the USE_STDATOMIC_H -preprocessor definition: - -* C11 stdatomic.h implementation (USE_STDATOMIC_H defined): - Uses the standard C11 header for all atomic operations. - This provides better portability and potentially better compiler - optimizations. Available on: +PostgreSQL's atomic operations (see src/include/port/atomics.h) are +implemented on top of the standard C11 header, which is the +sole atomics implementation. A working stdatomic.h is required at build +time; the USE_STDATOMIC_H preprocessor symbol is always defined. Available +on: - MSVC 2022+ (requires /experimental:c11atomics flag) - GCC 4.9+ and Clang 3.1+ (no special flags needed) -* Traditional implementation (USE_STDATOMIC_H not defined): - Uses platform-specific implementations that have been battle-tested - in PostgreSQL for many years: - - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h - - Compiler intrinsics: generic-gcc.h, generic-msvc.h - - Fallback implementations: generic.h, fallback.h +There is no build option to select the atomics implementation: stdatomic.h +is probed unconditionally and a missing or non-working header is a hard +configure/build error, since there is no other implementation to fall back +to. -Both implementations provide the same public API and observable semantics. -The choice is made at build configuration time: +Several internal properties of the implementation are worth noting; none of +them affect the public API: - meson setup build -Duse_stdatomic=auto # Auto-detect (default) - meson setup build -Duse_stdatomic=yes # Force stdatomic.h - meson setup build -Duse_stdatomic=no # Force traditional - -The two implementations differ internally in several ways that do not affect -the public API: - -* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering under - stdatomic.h; pg_atomic_write_u32/u64() use relaxed. The seq_cst read was +* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering; + pg_atomic_write_u32/u64() use relaxed. The seq_cst read is required for correctness on weak-memory hardware (e.g. RISC-V), where the weaker relaxed and acquire orderings let a concurrent reader miss a tuple in a parallel hash join. The write stays relaxed: a plain atomic store - matches the traditional "no barrier" contract for pg_atomic_write and, on + matches the "no barrier" contract for pg_atomic_write and, on ARM64, avoids the store-release (STLR) serialization that would penalize contended writes to hot shared state. The pg_atomic_unlocked_write_* variant also uses relaxed ordering, consistent with its "no guarantees" @@ -96,23 +83,15 @@ the public API: the thread fence (atomic_thread_fence), because a bare thread fence orders only atomic accesses and would let the compiler reorder plain loads/stores across it, whereas PostgreSQL's barrier contract must order non-atomic - accesses too. This matches the traditional generic-gcc.h barriers. + accesses too. -* Spinlock flag polarity. The stdatomic.h pg_atomic_flag uses 1=unlocked, - 0=locked (fetch_and-based test-and-set), the inverse of the traditional - 0=unlocked, 1=locked (exchange-based). This is an internal detail with no +* Spinlock flag polarity. The pg_atomic_flag uses 1=unlocked, + 0=locked (fetch_and-based test-and-set). This is an internal detail with no external visibility beyond raw values seen in a debugger. -* Binary compatibility. The atomic types are _Atomic(...) under stdatomic.h - and volatile-struct under the traditional path. The two are not binary - compatible, so objects and shared memory cannot be mixed between builds; - the implementation is fixed at build time, so this is not a concern in - practice. - -* Platform reach. Under stdatomic.h, 64-bit atomics are available on 32-bit +* Platform reach. 64-bit atomics are available on 32-bit ARM (compiler/runtime may emulate them with locks), atomics are usable from - frontend programs, and the headers are C++-compatible. The traditional - path disables 64-bit atomics on ARM32 and is backend-only. + frontend programs, and the headers are C++-compatible. For more information on memory barriers and atomic operations, see src/backend/storage/lmgr/README.barrier. diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index b3a11fe9fc903..8caaa562cedb2 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,13 +51,9 @@ #include #include "common/pg_prng.h" -#ifdef USE_STDATOMIC_H #include "port/atomics.h" #include "port/spin_delay.h" #include "storage/spin.h" -#else -#include "storage/s_lock.h" -#endif #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -66,17 +62,6 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L -#ifndef USE_STDATOMIC_H -#ifdef S_LOCK_TEST -/* - * These are needed by pgstat_report_wait_start in the standalone compile of - * s_lock_test. - */ -static uint32 local_my_wait_event_info; -uint32 *my_wait_event_info = &local_my_wait_event_info; -#endif -#endif /* !USE_STDATOMIC_H */ - static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -88,15 +73,8 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) - fprintf(stderr, - "\nStuck spinlock detected at %s, %s:%d.\n", - func, file, line); - exit(1); -#else elog(PANIC, "stuck spinlock detected at %s, %s:%d", func, file, line); -#endif } /* @@ -109,7 +87,6 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); -#ifdef USE_STDATOMIC_H for (;;) { bool try_to_set; @@ -129,28 +106,12 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) break; perform_spin_delay(&delayStatus); } -#else /* !USE_STDATOMIC_H */ - while (TAS_SPIN(lock)) - { - perform_spin_delay(&delayStatus); - } -#endif /* USE_STDATOMIC_H */ finish_spin_delay(&delayStatus); return delayStatus.delays; } -#ifndef USE_STDATOMIC_H -#ifdef USE_DEFAULT_S_UNLOCK -void -s_unlock(volatile slock_t *lock) -{ - *lock = 0; -} -#endif -#endif /* !USE_STDATOMIC_H */ - /* * Wait while spinning on a contended spinlock. */ @@ -158,11 +119,7 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ -#ifdef USE_STDATOMIC_H pg_spin_delay(); -#else - SPIN_DELAY(); -#endif /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -185,11 +142,6 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) - fprintf(stdout, "*"); - fflush(stdout); -#endif - /* increase delay by a random fraction between 1X and 2X */ status->cur_delay += (int) (status->cur_delay * pg_prng_double(&pg_global_prng_state) + 0.5); @@ -265,74 +217,3 @@ update_spins_per_delay(int shared_spins_per_delay) */ return (shared_spins_per_delay * 15 + spins_per_delay) / 16; } - - -/*****************************************************************************/ -#ifndef USE_STDATOMIC_H -#if defined(S_LOCK_TEST) - -/* - * test program for verifying a port's spinlock support. - */ - -struct test_lock_struct -{ - char pad1; - slock_t lock; - char pad2; -}; - -volatile struct test_lock_struct test_lock; - -int -main() -{ - pg_prng_seed(&pg_global_prng_state, (uint64) time(NULL)); - - test_lock.pad1 = test_lock.pad2 = 0x44; - - S_INIT_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_UNLOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - printf("S_LOCK_TEST: this will print %d stars and then\n", NUM_DELAYS); - printf(" exit with a 'stuck spinlock' message\n"); - printf(" if S_LOCK() and TAS() are working.\n"); - fflush(stdout); - - s_lock(&test_lock.lock, __FILE__, __LINE__, __func__); - - printf("S_LOCK_TEST: failed, lock not locked\n"); - return 1; -} - -#endif /* S_LOCK_TEST */ -#endif /* !USE_STDATOMIC_H */ diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index 2058dbb9238c7..9533b1e42bf6f 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,35 +7,18 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * - * IMPLEMENTATION SELECTION: + * IMPLEMENTATION: * - * PostgreSQL's atomic operations can be implemented using either: + * PostgreSQL's atomic operations are implemented on top of C11 + * (see stdatomic_impl.h). This standard header supplies all + * atomic types and operations, so no architecture-specific assembly or + * compiler intrinsics are required here. A working C11 stdatomic.h is a + * hard build requirement (USE_STDATOMIC_H is always defined); the former + * platform-specific implementations (arch-*.h, generic-*.h, fallback.h) + * have been removed. * - * 1. C11 stdatomic.h (when USE_STDATOMIC_H is defined) - * - Uses standard C11 for all atomic operations - * - Better portability and compiler optimizations - * - Requires MSVC 2022+ or GCC 4.9+/Clang 3.1+ - * - * 2. Traditional platform-specific implementations (when USE_STDATOMIC_H is not defined) - * - Uses battle-tested PostgreSQL implementations - * - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h - * - Compiler intrinsics: generic-gcc.h, generic-msvc.h - * - Fallback implementations: generic.h, fallback.h - * - * Both implementations provide identical public API and semantics. - * Selection is made at build time via -Duse_stdatomic=auto/yes/no. - * - * PORTING NOTES: - * - * To bring up postgres on a platform/compiler at the very least - * implementations for the following operations should be provided: - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() - * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. - * - * For new platforms, prefer using stdatomic.h if available, as it reduces - * maintenance burden and leverages compiler-provided implementations. + * The only remaining hand-written platform hint is the CPU spin-delay + * instruction in port/spin_delay.h, which stdatomic.h does not cover. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -54,21 +37,10 @@ #define ATOMICS_H /* - * Frontend code restriction - * - * Traditionally, atomics.h could not be included from frontend code because - * the platform-specific implementations often relied on backend-only features. - * - * With the stdatomic.h implementation, this restriction is no longer necessary - * since stdatomic.h is a standard header. However, we keep the restriction for - * the traditional implementation path to maintain compatibility. + * With the stdatomic.h implementation there is no longer any restriction on + * including atomics.h from frontend code, since stdatomic.h is a standard + * header with no backend-only dependencies. */ -#ifdef FRONTEND -#ifndef USE_STDATOMIC_H -#error "atomics.h may not be included from frontend code (use -Duse_stdatomic=yes if atomics are needed)" -#endif -#endif - #define INSIDE_ATOMICS_H /* @@ -83,125 +55,14 @@ #include /* - * PostgreSQL atomics can be implemented using either C11 stdatomic.h - * or platform-specific implementations. The choice is made at build - * configuration time via the USE_STDATOMIC_H preprocessor definition. - * - * When USE_STDATOMIC_H is defined, we use C11 for all - * atomic operations. This provides better portability and potentially - * better compiler optimizations. - * - * When USE_STDATOMIC_H is not defined (default initially), we use - * traditional platform-specific implementations (arch-*.h, generic-*.h) - * that have been battle-tested in PostgreSQL for many years. - * - * Both paths provide identical public API and semantics. - */ - -#ifdef USE_STDATOMIC_H - -/* - * C11 stdatomic.h implementation path - * - * This uses the standard C11 header for atomic operations. - * The type definitions and implementations are in stdatomic_impl.h. + * All atomic types and _impl operations come from C11 , wrapped + * in stdatomic_impl.h. */ - #include "port/atomics/stdatomic_impl.h" /* - * The public API is provided by static inline functions in the common code - * section below. Those functions call _impl functions, which are provided by - * stdatomic_impl.h for this path. - */ - -#else /* !USE_STDATOMIC_H */ - -/* - * Traditional platform-specific implementation path - * - * This is the original PostgreSQL atomics implementation using - * architecture-specific headers and compiler intrinsics. - * - * First a set of architecture specific files is included. - * - * These files can provide the full set of atomics or can do pretty much - * nothing if all the compilers commonly used on these platforms provide - * usable generics. - * - * Don't add an inline assembly of the actual atomic operations if all the - * common implementations of your platform provide intrinsics. Intrinsics are - * much easier to understand and potentially support more architectures. - * - * It will often make sense to define memory barrier semantics here, since - * e.g. generic compiler intrinsics for x86 memory barriers can't know that - * postgres doesn't need x86 read/write barriers do anything more than a - * compiler barrier. - * - */ -#if defined(__arm__) || defined(__aarch64__) -#include "port/atomics/arch-arm.h" -#elif defined(__i386__) || defined(__x86_64__) -#include "port/atomics/arch-x86.h" -#elif defined(__powerpc__) || defined(__powerpc64__) -#include "port/atomics/arch-ppc.h" -#endif - -/* - * Compiler specific, but architecture independent implementations. - * - * Provide architecture independent implementations of the atomic - * facilities. At the very least compiler barriers should be provided, but a - * full implementation of - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * using compiler intrinsics are a good idea. - */ -/* - * gcc or compatible, including clang and icc. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#include "port/atomics/generic-gcc.h" -#elif defined(_MSC_VER) -#include "port/atomics/generic-msvc.h" -#else -/* Unknown compiler. */ -#endif - -/* Fail if we couldn't find implementations of required facilities. */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#error "could not find an implementation of pg_atomic_uint32" -#endif -#if !defined(pg_compiler_barrier_impl) -#error "could not find an implementation of pg_compiler_barrier" -#endif -#if !defined(pg_memory_barrier_impl) -#error "could not find an implementation of pg_memory_barrier_impl" -#endif - - -/* - * Provide a spinlock-based implementation of the 64 bit variants, if - * necessary. - */ -#include "port/atomics/fallback.h" - -/* - * Provide additional operations using supported infrastructure. These are - * expected to be efficient if the underlying atomic operations are efficient. - */ -#include "port/atomics/generic.h" - -#endif /* USE_STDATOMIC_H */ - -/* - * Common code for both stdatomic.h and traditional implementations. - * - * The following definitions provide the public API that works identically - * regardless of which implementation is used. The static inline functions - * below call _impl functions which are provided by either stdatomic_impl.h - * (when USE_STDATOMIC_H is defined) or the traditional implementation files - * (generic.h, etc.) otherwise. + * The public API below is provided by static inline functions that call the + * _impl functions supplied by stdatomic_impl.h. */ /* diff --git a/src/include/port/atomics/arch-arm.h b/src/include/port/atomics/arch-arm.h deleted file mode 100644 index 90280c7b751a9..0000000000000 --- a/src/include/port/atomics/arch-arm.h +++ /dev/null @@ -1,32 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-arm.h - * Atomic operations considerations specific to ARM - * - * Portions Copyright (c) 2013-2026, PostgreSQL Global Development Group - * - * NOTES: - * - * src/include/port/atomics/arch-arm.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * 64 bit atomics on ARM32 are implemented using kernel fallbacks and thus - * might be slow, so disable entirely. On ARM64 that problem doesn't exist. - */ -#if !defined(__aarch64__) -#define PG_DISABLE_64_BIT_ATOMICS -#else -/* - * Architecture Reference Manual for ARMv8 states aligned read/write to/from - * general purpose register is atomic. - */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* __aarch64__ */ diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h deleted file mode 100644 index 65ced2c6d3a23..0000000000000 --- a/src/include/port/atomics/arch-ppc.h +++ /dev/null @@ -1,256 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-ppc.h - * Atomic operations considerations specific to PowerPC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-ppc.h - * - *------------------------------------------------------------------------- - */ - -#if defined(__GNUC__) - -/* - * lwsync orders loads with respect to each other, and similarly with stores. - * But a load can be performed before a subsequent store, so sync must be used - * for a full memory barrier. - */ -#define pg_memory_barrier_impl() __asm__ __volatile__ ("sync" : : : "memory") -#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* 64bit atomics are only supported in 64bit mode */ -#if SIZEOF_VOID_P >= 8 -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif - -/* - * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but - * code generation differs at the end. __atomic_compare_exchange_n(): - * 100: isync - * 104: mfcr r3 - * 108: rlwinm r3,r3,3,31,31 - * 10c: bne 120 <.eb+0x10> - * 110: clrldi r3,r3,63 - * 114: addi r1,r1,112 - * 118: blr - * 11c: nop - * 120: clrldi r3,r3,63 - * 124: stw r9,0(r4) - * 128: addi r1,r1,112 - * 12c: blr - * - * This: - * f0: isync - * f4: mfcr r9 - * f8: rldicl. r3,r9,35,63 - * fc: bne 104 <.eb> - * 100: stw r10,0(r4) - * 104: addi r1,r1,112 - * 108: blr - * - * This implementation may or may not have materially different performance. - * It's not exploiting the fact that cr0 still holds the relevant comparison - * bits, set during the __asm__. One could fix that by moving more code into - * the __asm__. (That would remove the freedom to eliminate dead stores when - * the caller ignores "expected", but few callers do.) - * - * Recognizing constant "newval" would be superfluous, because there's no - * immediate-operand version of stwcx. - */ -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - uint32 found; - uint32 condition_register; - bool ret; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int32) *expected <= PG_INT16_MAX && - (int32) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpwi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpw %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -/* - * This mirrors gcc __sync_fetch_and_add(). - * - * Like tas(), use constraint "=&b" to avoid allocating r0. - */ -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 _t; - uint32 res; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#ifdef PG_HAVE_ATOMIC_U64_SUPPORT - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - uint64 found; - uint32 condition_register; - bool ret; - - AssertPointerAlignment(expected, 8); - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int64) *expected <= PG_INT16_MAX && - (int64) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpdi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpd %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 _t; - uint64 res; - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ - -/* per architecture manual doubleword accesses have single copy atomicity */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h deleted file mode 100644 index 231831dcf5979..0000000000000 --- a/src/include/port/atomics/arch-x86.h +++ /dev/null @@ -1,189 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-x86.h - * Atomic operations considerations specific to intel x86 - * - * Note that we actually require a 486 upwards because the 386 doesn't have - * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere - * anymore that's not much of a restriction luckily. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-x86.h - * - *------------------------------------------------------------------------- - */ - -/* - * Both 32 and 64 bit x86 do not allow loads to be reordered with other loads, - * or stores to be reordered with other stores, but a load can be performed - * before a subsequent store. - * - * Technically, some x86-ish chips support uncached memory access and/or - * special instructions that are weakly ordered. In those cases we'd need - * the read and write barriers to be lfence and sfence. But since we don't - * do those things, a compiler barrier should be enough. - * - * "lock; addl" has worked for longer than "mfence". It's also rumored to be - * faster in many scenarios. - */ - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#if defined(__i386__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory", "cc") -#elif defined(__x86_64__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%rsp)" : : : "memory", "cc") -#endif -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -#define pg_read_barrier_impl() pg_compiler_barrier_impl() -#define pg_write_barrier_impl() pg_compiler_barrier_impl() - -/* - * Provide implementation for atomics using inline assembly on x86 gcc. It's - * nice to support older gcc's and the compare/exchange implementation here is - * actually more efficient than the * __sync variant. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - volatile char value; -} pg_atomic_flag; - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* - * It's too complicated to write inline asm for 64bit types on 32bit and the - * 486 can't do it anyway. - */ -#ifdef __x86_64__ -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - /* alignment guaranteed due to being on a 64bit platform */ - volatile uint64 value; -} pg_atomic_uint64; -#endif /* __x86_64__ */ - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - char _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(ptr->value) -: -: "memory"); - return _res == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* - * On a TSO architecture like x86 it's sufficient to use a compiler - * barrier to achieve release semantics. - */ - __asm__ __volatile__("" ::: "memory"); - ptr->value = 0; -} - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - char ret; - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgl %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 res; - __asm__ __volatile__( - " lock \n" - " xaddl %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#ifdef __x86_64__ - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - char ret; - - AssertPointerAlignment(expected, 8); - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgq %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 res; - __asm__ __volatile__( - " lock \n" - " xaddq %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -/* - * 8 byte reads / writes have single-copy atomicity on all x86-64 cpus. - */ -#if defined(__x86_64__) -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* 8 byte single-copy atomicity */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h deleted file mode 100644 index f37225189fc3b..0000000000000 --- a/src/include/port/atomics/fallback.h +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fallback.h - * Fallback for platforms without 64 bit atomics support. Slower - * than native atomics support, but not unusably slow. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/fallback.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - - -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) - -#define PG_HAVE_ATOMIC_U64_SIMULATION - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - int sema; - volatile uint64 value; -} pg_atomic_uint64; - -#define PG_HAVE_ATOMIC_INIT_U64 -extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_); - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval); - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h deleted file mode 100644 index 5bfce82f687e9..0000000000000 --- a/src/include/port/atomics/generic-gcc.h +++ /dev/null @@ -1,326 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-gcc.h - * Atomic operations, implemented using gcc (or compatible) intrinsics. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Legacy __sync Built-in Functions for Atomic Memory Access - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html - * * Built-in functions for memory model aware atomic operations - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html - * - * src/include/port/atomics/generic-gcc.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * An empty asm block should be a sufficient compiler barrier. - */ -#define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory") - -/* - * If we're on GCC, we should be able to get a memory barrier - * out of this compiler built-in. But we prefer to rely on platform specific - * definitions where possible, and use this only as a fallback. - */ -#if !defined(pg_memory_barrier_impl) -# if defined(HAVE_GCC__ATOMIC_INT32_CAS) -# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST) -# elif defined(__GNUC__) -# define pg_memory_barrier_impl() __sync_synchronize() -# endif -#endif /* !defined(pg_memory_barrier_impl) */ - -#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* acquire semantics include read barrier semantics */ -# define pg_read_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_ACQUIRE); \ -} while (0) -#endif - -#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* release semantics include write barrier semantics */ -# define pg_write_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_RELEASE); \ -} while (0) -#endif - - -/* generic gcc based atomic flag implementation */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \ - && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS)) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - /* - * If we have a choice, use int-width TAS, because that is more efficient - * and/or more reliably implemented on most non-Intel platforms. (Note - * that this code isn't used on x86[_64]; see arch-x86.h for that.) - */ -#ifdef HAVE_GCC__SYNC_INT32_TAS - volatile int value; -#else - volatile char value; -#endif -} pg_atomic_flag; - -#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */ - -/* generic gcc based atomic uint32 implementation */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \ - && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS)) - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */ - -/* generic gcc based atomic uint64 implementation */ -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \ - && !defined(PG_DISABLE_64_BIT_ATOMICS) \ - && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS)) - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ - -#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT - -#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) - -#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* NB: only an acquire barrier, not a full one */ - /* some platform only support a 1 here */ - return __sync_lock_test_and_set(&ptr->value, 1) == 0; -} -#endif - -#endif /* defined(HAVE_GCC__SYNC_*_TAS) */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return ptr->value == 0; -} -#endif - -#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - __sync_lock_release(&ptr->value); -} -#endif - -#ifndef PG_HAVE_ATOMIC_INIT_FLAG -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} -#endif - -#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */ - -/* prefer __atomic, it has a better API */ -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - /* FIXME: we can probably use a lower consistency model */ - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 32-bit __atomic_compare_exchange_n() implies - * the availability of 32-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - - -#if !defined(PG_DISABLE_64_BIT_ATOMICS) - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - AssertPointerAlignment(expected, 8); - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - - AssertPointerAlignment(expected, 8); - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 64-bit __atomic_compare_exchange_n() implies - * the availability of 64-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - -#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h deleted file mode 100644 index 6d09cd0e0438e..0000000000000 --- a/src/include/port/atomics/generic-msvc.h +++ /dev/null @@ -1,115 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-msvc.h - * Atomic operations support when using MSVC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Interlocked Variable Access - * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx - * - * src/include/port/atomics/generic-msvc.h - * - *------------------------------------------------------------------------- - */ -#include - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -#pragma intrinsic(_ReadWriteBarrier) -#define pg_compiler_barrier_impl() _ReadWriteBarrier() - -#ifndef pg_memory_barrier_impl -#define pg_memory_barrier_impl() MemoryBarrier() -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = InterlockedCompareExchange(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return InterlockedExchange(&ptr->value, newval); -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return InterlockedExchangeAdd(&ptr->value, add_); -} - -/* - * The non-intrinsics versions are only available in vista upwards, so use the - * intrinsic version. Only supported on >486, but we require XP as a minimum - * baseline, which doesn't support the 486, so we don't need to add checks for - * that case. - */ -#pragma intrinsic(_InterlockedCompareExchange64) - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - current = _InterlockedCompareExchange64(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -/* Only implemented on 64bit builds */ -#ifdef _WIN64 - -#pragma intrinsic(_InterlockedExchange64) - -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return _InterlockedExchange64(&ptr->value, newval); -} - -#pragma intrinsic(_InterlockedExchangeAdd64) - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return _InterlockedExchangeAdd64(&ptr->value, add_); -} - -#endif /* _WIN64 */ diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h deleted file mode 100644 index daa772e9a6d00..0000000000000 --- a/src/include/port/atomics/generic.h +++ /dev/null @@ -1,430 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic.h - * Implement higher level operations based on some lower level atomic - * operations. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/generic.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - -/* - * If read or write barriers are undefined, we upgrade them to full memory - * barriers. - */ -#if !defined(pg_read_barrier_impl) -# define pg_read_barrier_impl pg_memory_barrier_impl -#endif -#if !defined(pg_write_barrier_impl) -# define pg_write_barrier_impl pg_memory_barrier_impl -#endif - -/* provide fallback */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef pg_atomic_uint32 pg_atomic_flag; -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U32 -#define PG_HAVE_ATOMIC_READ_U32 -static inline uint32 -pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return ptr->value; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U32 -#define PG_HAVE_ATOMIC_WRITE_U32 -static inline void -pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -static inline void -pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -/* - * provide fallback for test_and_set using atomic_exchange if available - */ -#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_exchange_u32_impl(ptr, 1) == 0; -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -/* - * provide fallback for test_and_set using atomic_compare_exchange if - * available. - */ -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - uint32 value = 0; - return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1); -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) -# error "No pg_atomic_test_and_set provided" -#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ - - -#ifndef PG_HAVE_ATOMIC_INIT_U32 -#define PG_HAVE_ATOMIC_INIT_U32 -static inline void -pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_ADD_FETCH_U32 -static inline uint32 -pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) -#define PG_HAVE_ATOMIC_SUB_FETCH_U32 -static inline uint32 -pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32 -static inline uint32 -pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return pg_atomic_fetch_add_u32_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32 -static inline void -pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - (void) pg_atomic_exchange_u32_impl(ptr, val); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U64 -#define PG_HAVE_ATOMIC_WRITE_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * On this platform aligned 64bit writes are guaranteed to be atomic, - * except if using the fallback implementation, where can't guarantee the - * required alignment. - */ - AssertPointerAlignment(ptr, 8); - ptr->value = val; -} - -#else - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * 64 bit writes aren't safe on all platforms. In the generic - * implementation implement them as an atomic exchange. - */ - pg_atomic_exchange_u64_impl(ptr, val); -} - -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_WRITE_U64 */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -static inline void -pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U64 -#define PG_HAVE_ATOMIC_READ_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - /* - * On this platform aligned 64-bit reads are guaranteed to be atomic. - */ - AssertPointerAlignment(ptr, 8); - return ptr->value; -} - -#else - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - uint64 old = 0; - - /* - * 64-bit reads aren't atomic on all platforms. In the generic - * implementation implement them as a compare/exchange with 0. That'll - * fail or succeed, but always return the old value. Possibly might store - * a 0, but only if the previous value also was a 0 - i.e. harmless. - */ - pg_atomic_compare_exchange_u64_impl(ptr, &old, 0); - - return old; -} -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_READ_U64 */ - -#ifndef PG_HAVE_ATOMIC_INIT_U64 -#define PG_HAVE_ATOMIC_INIT_U64 -static inline void -pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_ADD_FETCH_U64 -static inline uint64 -pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) -#define PG_HAVE_ATOMIC_SUB_FETCH_U64 -static inline uint64 -pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64 -static inline uint64 -pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - return pg_atomic_fetch_add_u64_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64 -static inline void -pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - (void) pg_atomic_exchange_u64_impl(ptr, val); -} -#endif diff --git a/src/include/port/atomics/stdatomic_impl.h b/src/include/port/atomics/stdatomic_impl.h index 0bda02d16f419..7519a096e07c0 100644 --- a/src/include/port/atomics/stdatomic_impl.h +++ b/src/include/port/atomics/stdatomic_impl.h @@ -4,11 +4,9 @@ * Atomic operations implementation using C11 stdatomic.h * * This file provides PostgreSQL atomic operations using the C11 standard - * . It is only included when USE_STDATOMIC_H is defined. - * - * The traditional platform-specific implementation (arch-*.h, generic-*.h, - * fallback.h) remains available and is used when stdatomic.h is not - * available or when explicitly requested via -Duse_stdatomic=no. + * . As of this commit it is the sole atomics implementation; + * the former platform-specific paths (arch-*.h, generic-*.h, fallback.h) + * have been removed and USE_STDATOMIC_H is always defined. * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -21,12 +19,6 @@ #ifndef STDATOMIC_IMPL_H #define STDATOMIC_IMPL_H -/* - * Only include this file when USE_STDATOMIC_H is defined at build time. - * This ensures the traditional implementation remains available as fallback. - */ -#ifdef USE_STDATOMIC_H - /* * C++ Compatibility * @@ -584,6 +576,4 @@ pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) atomic_store_explicit(ptr, val, memory_order_seq_cst); } -#endif /* USE_STDATOMIC_H */ - #endif /* STDATOMIC_IMPL_H */ diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h deleted file mode 100644 index 96fcaaac01e76..0000000000000 --- a/src/include/storage/s_lock.h +++ /dev/null @@ -1,753 +0,0 @@ -/*------------------------------------------------------------------------- - * - * s_lock.h - * Implementation of spinlocks. - * - * NOTE: none of the macros in this file are intended to be called directly. - * Call them through the macros in spin.h. - * - * The following hardware-dependent macros must be provided for each - * supported platform: - * - * void S_INIT_LOCK(slock_t *lock) - * Initialize a spinlock (to the unlocked state). - * - * int S_LOCK(slock_t *lock) - * Acquire a spinlock, waiting if necessary. - * Time out and abort() if unable to acquire the lock in a - * "reasonable" amount of time --- typically ~ 1 minute. - * Should return number of "delays"; see s_lock.c - * - * void S_UNLOCK(slock_t *lock) - * Unlock a previously acquired lock. - * - * void SPIN_DELAY(void) - * Delay operation to occur inside spinlock wait loop. - * - * Note to implementors: there are default implementations for all these - * macros at the bottom of the file. Check if your platform can use - * these or needs to override them. - * - * Usually, S_LOCK() is implemented in terms of even lower-level macros - * TAS() and TAS_SPIN(): - * - * int TAS(slock_t *lock) - * Atomic test-and-set instruction. Attempt to acquire the lock, - * but do *not* wait. Returns 0 if successful, nonzero if unable - * to acquire the lock. - * - * int TAS_SPIN(slock_t *lock) - * Like TAS(), but this version is used when waiting for a lock - * previously found to be contended. By default, this is the - * same as TAS(), but on some architectures it's better to poll a - * contended lock using an unlocked instruction and retry the - * atomic test-and-set only when it appears free. - * - * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. - * - * CAUTION: on some platforms TAS() and/or TAS_SPIN() may sometimes report - * failure to acquire a lock even when the lock is not locked. For example, - * on Alpha TAS() will "fail" if interrupted. Therefore a retry loop must - * always be used, even if you are certain the lock is free. - * - * It is the responsibility of these macros to make sure that the compiler - * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. Prior to PostgreSQL 9.5, this - * was the caller's responsibility, which meant that callers had to use - * volatile-qualified pointers to refer to both the spinlock itself and the - * shared data being accessed within the spinlocked critical section. This - * was notationally awkward, easy to forget (and thus error-prone), and - * prevented some useful compiler optimizations. For these reasons, we - * now require that the macros themselves prevent compiler re-ordering, - * so that the caller doesn't need to take special precautions. - * - * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and - * S_UNLOCK() macros must further include hardware-level memory fence - * instructions to prevent similar re-ordering at the hardware level. - * TAS() and TAS_SPIN() must guarantee that loads and stores issued after - * the macro are not executed until the lock has been obtained. Conversely, - * S_UNLOCK() must guarantee that loads and stores issued before the macro - * have been executed before the lock is released. - * - * On most supported platforms, TAS() uses a tas() function written - * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent OS-supplied mutex routines could be used too. - * - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/storage/s_lock.h - * - *------------------------------------------------------------------------- - */ -#ifndef S_LOCK_H -#define S_LOCK_H - -#ifdef FRONTEND -#error "s_lock.h may not be included from frontend code" -#endif - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -/************************************************************************* - * All the gcc inlines - * Gcc consistently defines the CPU as __cpu__. - * Other compilers use __cpu or __cpu__ so we test for both in those cases. - */ - -/*---------- - * Standard gcc asm format (assuming "volatile slock_t *lock"): - - __asm__ __volatile__( - " instruction \n" - " instruction \n" - " instruction \n" -: "=r"(_res), "+m"(*lock) // return register, in/out lock value -: "r"(lock) // lock pointer, in input register -: "memory", "cc"); // show clobbered registers here - - * The output-operands list (after first colon) should always include - * "+m"(*lock), whether or not the asm code actually refers to this - * operand directly. This ensures that gcc believes the value in the - * lock variable is used and set by the asm code. Also, the clobbers - * list (after third colon) should always include "memory"; this prevents - * gcc from thinking it can cache the values of shared-memory fields - * across the asm code. Add "cc" if your asm code changes the condition - * code register, and also list any temp registers the code uses. - * - * If you need branch target labels within the asm block, include "%=" - * in the label names to make them distinct across multiple asm blocks - * within a source file. - *---------- - */ - - -#ifdef __i386__ /* 32-bit i386 */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - /* - * Use a non-locking test before asserting the bus lock. Note that the - * extra test appears to be a small loss on some x86 platforms and a small - * win on others; it's by no means clear that we should keep it. - * - * When this was last tested, we didn't have separate TAS() and TAS_SPIN() - * macros. Nowadays it probably would be better to do a non-locking test - * in TAS_SPIN() but not in TAS(), like on x86_64, but no-one's done the - * testing to verify that. Without some empirical evidence, better to - * leave it alone. - */ - __asm__ __volatile__( - " cmpb $0,%1 \n" - " jne TAS%=_out \n" - " lock \n" - " xchgb %0,%1 \n" - "TAS%=_out: \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * This sequence is equivalent to the PAUSE instruction ("rep" is - * ignored by old IA32 processors if the following instruction is - * not a string operation); the IA-32 Architecture Software - * Developer's Manual, Vol. 3, Section 7.7.2 describes why using - * PAUSE in the inner loop of a spin lock is necessary for good - * performance: - * - * The PAUSE instruction improves the performance of IA-32 - * processors supporting Hyper-Threading Technology when - * executing spin-wait loops and other routines where one - * thread is accessing a shared lock or semaphore in a tight - * polling loop. When executing a spin-wait loop, the - * processor can suffer a severe performance penalty when - * exiting the loop because it detects a possible memory order - * violation and flushes the core processor's pipeline. The - * PAUSE instruction provides a hint to the processor that the - * code sequence is a spin-wait loop. The processor uses this - * hint to avoid the memory order violation and prevent the - * pipeline flush. In addition, the PAUSE instruction - * de-pipelines the spin-wait loop to prevent it from - * consuming execution resources excessively. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __i386__ */ - - -#ifdef __x86_64__ /* AMD Opteron, Intel EM64T */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -/* - * On Intel EM64T, it's a win to use a non-locking test before the xchg proper, - * but only when spinning. - * - * See also Implementing Scalable Atomic Locks for Multi-Core Intel(tm) EM64T - * and IA32, by Michael Chynoweth and Mary R. Lee. As of this writing, it is - * available at: - * http://software.intel.com/en-us/articles/implementing-scalable-atomic-locks-for-multi-core-intel-em64t-and-ia32-architectures - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Adding a PAUSE in the spin delay loop is demonstrably a no-op on - * Opteron, but it may be of some use on EM64T, so we keep it. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __x86_64__ */ - - -/* - * On ARM and ARM64, we use __sync_lock_test_and_set(int *, int) if available. - * - * We use the int-width variant of the builtin because it works on more chips - * than other widths. - */ -#if defined(__arm__) || defined(__aarch64__) -#ifdef HAVE_GCC__SYNC_INT32_TAS -#define HAS_TEST_AND_SET - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#if defined(__aarch64__) - -/* - * On ARM64, it's a win to use a non-locking test before the TAS proper. It - * may be a win on 32-bit ARM, too, but nobody's tested it yet. - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Using an ISB instruction to delay in spinlock loops appears beneficial - * on high-core-count ARM64 processors. It seems mostly a wash for smaller - * gear, and ISB doesn't exist at all on pre-v7 ARM chips. - */ - __asm__ __volatile__( - " isb; \n"); -} - -#endif /* __aarch64__ */ -#endif /* HAVE_GCC__SYNC_INT32_TAS */ -#endif /* __arm__ || __aarch64__ */ - - -/* S/390 and S/390x Linux (32- and 64-bit zSeries) */ -#if defined(__s390__) || defined(__s390x__) -#define HAS_TEST_AND_SET - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - int _res = 0; - - __asm__ __volatile__( - " cs %0,%3,0(%2) \n" -: "+d"(_res), "+m"(*lock) -: "a"(lock), "d"(1) -: "memory", "cc"); - return _res; -} - -#endif /* __s390__ || __s390x__ */ - - -#if defined(__sparc__) /* Sparc */ -/* - * Solaris has always run sparc processors in TSO (total store) mode, but - * linux didn't use to and the *BSDs still don't. So, be careful about - * acquire/release semantics. The CPU will treat superfluous members as - * NOPs, so it's just code space. - */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res; - - /* - * "cas" would be better than "ldstub", but it is only present on - * sparcv8plus and later, while some platforms still support sparcv7 or - * sparcv8. Also, "cas" requires that the system be running in TSO mode. - */ - __asm__ __volatile__( - " ldstub [%2], %0 \n" -: "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory"); -#if defined(__sparcv7) || defined(__sparc_v7__) - /* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) - /* stbar is available (and required for both PSO, RMO), membar isn't */ - __asm__ __volatile__ ("stbar \n":::"memory"); -#else - /* - * #LoadStore (RMO) | #LoadLoad (RMO) together are the appropriate acquire - * barrier for sparcv8+ upwards. - */ - __asm__ __volatile__ ("membar #LoadStore | #LoadLoad \n":::"memory"); -#endif - return (int) _res; -} - -#if defined(__sparcv7) || defined(__sparc_v7__) -/* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. We fall through to the default gcc definition of - * S_UNLOCK in this case. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) -/* stbar is available (and required for both PSO, RMO), membar isn't */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("stbar \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#else -/* - * #LoadStore (RMO) | #StoreStore (RMO, PSO) together are the appropriate - * release barrier for sparcv8+ upwards. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("membar #LoadStore | #StoreStore \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#endif - -#endif /* __sparc__ */ - - -/* PowerPC */ -#if defined(__powerpc__) || defined(__powerpc64__) -#define HAS_TEST_AND_SET - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* On PPC, it's a win to use a non-locking test before the lwarx */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -/* - * The second operand of addi can hold a constant zero or a register number, - * hence constraint "=&b" to avoid allocating r0. "b" stands for "address - * base register"; most operands having this register-or-zero property are - * address bases, e.g. the second operand of lwax. - * - * NOTE: per the Enhanced PowerPC Architecture manual, v1.0 dated 7-May-2002, - * an isync is a sufficient synchronization barrier after a lwarx/stwcx loop. - * But if the spinlock is in ordinary memory, we can use lwsync instead for - * better performance. - */ -static inline int -tas(volatile slock_t *lock) -{ - slock_t _t; - int _res; - - __asm__ __volatile__( -" lwarx %0,0,%3,1 \n" -" cmpwi %0,0 \n" -" bne TAS%=_fail \n" -" addi %0,%0,1 \n" -" stwcx. %0,0,%3 \n" -" beq TAS%=_ok \n" -"TAS%=_fail: \n" -" li %1,1 \n" -" b TAS%=_out \n" -"TAS%=_ok: \n" -" lwsync \n" -" li %1,0 \n" -"TAS%=_out: \n" -: "=&b"(_t), "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory", "cc"); - return _res; -} - -/* - * PowerPC S_UNLOCK is almost standard but requires a "sync" instruction. - * But we can use lwsync instead for better performance. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ (" lwsync \n" ::: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* powerpc */ - - -#if defined(__mips__) && !defined(__sgi) /* non-SGI MIPS */ -#define HAS_TEST_AND_SET - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* - * Original MIPS-I processors lacked the LL/SC instructions, but if we are - * so unfortunate as to be running on one of those, we expect that the kernel - * will handle the illegal-instruction traps and emulate them for us. On - * anything newer (and really, MIPS-I is extinct) LL/SC is the only sane - * choice because any other synchronization method must involve a kernel - * call. Unfortunately, many toolchains still default to MIPS-I as the - * codegen target; if the symbol __mips shows that that's the case, we - * have to force the assembler to accept LL/SC. - * - * R10000 and up processors require a separate SYNC, which has the same - * issues as LL/SC. - */ -#if __mips < 2 -#define MIPS_SET_MIPS2 " .set mips2 \n" -#else -#define MIPS_SET_MIPS2 -#endif - -static inline int -tas(volatile slock_t *lock) -{ - volatile slock_t *_l = lock; - int _res; - int _tmp; - - __asm__ __volatile__( - " .set push \n" - MIPS_SET_MIPS2 - " .set noreorder \n" - " .set nomacro \n" - " ll %0, %2 \n" - " or %1, %0, 1 \n" - " sc %1, %2 \n" - " xori %1, 1 \n" - " or %0, %0, %1 \n" - " sync \n" - " .set pop " -: "=&r" (_res), "=&r" (_tmp), "+R" (*_l) -: /* no inputs */ -: "memory"); - return _res; -} - -/* MIPS S_UNLOCK is almost standard but requires a "sync" instruction */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__( \ - " .set push \n" \ - MIPS_SET_MIPS2 \ - " .set noreorder \n" \ - " .set nomacro \n" \ - " sync \n" \ - " .set pop " \ -: /* no outputs */ \ -: /* no inputs */ \ -: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* __mips__ && !__sgi */ - - - -/* - * If we have no platform-specific knowledge, but we found that the compiler - * provides __sync_lock_test_and_set(), use that. Prefer the int-width - * version over the char-width version if we have both, on the rather dubious - * grounds that that's known to be more likely to work in the ARM ecosystem. - * (But we dealt with ARM above.) - */ -#if !defined(HAS_TEST_AND_SET) - -#if defined(HAVE_GCC__SYNC_INT32_TAS) -#define HAS_TEST_AND_SET - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#elif defined(HAVE_GCC__SYNC_CHAR_TAS) -#define HAS_TEST_AND_SET - -#define TAS(lock) tas(lock) - -typedef char slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#endif /* HAVE_GCC__SYNC_INT32_TAS */ - -#endif /* !defined(HAS_TEST_AND_SET) */ - - -/* - * Default implementation of S_UNLOCK() for gcc/icc. - * - * Note that this implementation is unsafe for any platform that can reorder - * a memory access (either load or store) after a following store. That - * happens not to be possible on x86 and most legacy architectures (some are - * single-processor!), but many modern systems have weaker memory ordering. - * Those that do must define their own version of S_UNLOCK() rather than - * relying on this one. - */ -#if !defined(S_UNLOCK) -#define S_UNLOCK(lock) \ - do { __asm__ __volatile__("" : : : "memory"); *(lock) = 0; } while (0) -#endif - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - - -/* - * --------------------------------------------------------------------- - * Platforms that use non-gcc inline assembly: - * --------------------------------------------------------------------- - */ - -#if !defined(HAS_TEST_AND_SET) /* We didn't trigger above, let's try here */ - -#ifdef _MSC_VER -typedef LONG slock_t; - -#define HAS_TEST_AND_SET -#define TAS(lock) (InterlockedCompareExchange(lock, 1, 0)) - -#define SPIN_DELAY() spin_delay() - -#ifdef __aarch64__ -static __forceinline void -spin_delay(void) -{ - /* - * Research indicates ISB is better than __yield() on AArch64. See - * https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com. - */ - __isb(_ARM64_BARRIER_SY); -} -#elif defined(_WIN64) -static __forceinline void -spin_delay(void) -{ - /* - * If using Visual C++ on Win64, inline assembly is unavailable. - * Use a _mm_pause intrinsic instead of rep nop. - */ - _mm_pause(); -} -#else -static __forceinline void -spin_delay(void) -{ - /* See comment for gcc code. Same code, MASM syntax */ - __asm rep nop; -} -#endif - -#include - -#ifdef __aarch64__ - -/* _ReadWriteBarrier() is insufficient on non-TSO architectures. */ -#pragma intrinsic(_InterlockedExchange) -#define S_UNLOCK(lock) _InterlockedExchange(lock, 0) - -#else - -#pragma intrinsic(_ReadWriteBarrier) -#define S_UNLOCK(lock) \ - do { _ReadWriteBarrier(); (*(lock)) = 0; } while (0) - -#endif -#endif - - -#endif /* !defined(HAS_TEST_AND_SET) */ - - -/* Blow up if we didn't have any way to do spinlocks */ -#ifndef HAS_TEST_AND_SET -#error PostgreSQL does not have spinlock support on this platform. Please report this to pgsql-bugs@lists.postgresql.org. -#endif - - -/* - * Default Definitions - override these above as needed. - */ - -#if !defined(S_LOCK) -#define S_LOCK(lock) \ - (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#endif /* S_LOCK */ - -#if !defined(S_UNLOCK) -/* - * Our default implementation of S_UNLOCK is essentially *(lock) = 0. This - * is unsafe if the platform can reorder a memory access (either load or - * store) after a following store; platforms where this is possible must - * define their own S_UNLOCK. But CPU reordering is not the only concern: - * if we simply defined S_UNLOCK() as an inline macro, the compiler might - * reorder instructions from inside the critical section to occur after the - * lock release. Since the compiler probably can't know what the external - * function s_unlock is doing, putting the same logic there should be adequate. - * A sufficiently-smart globally optimizing compiler could break that - * assumption, though, and the cost of a function call for every spinlock - * release may hurt performance significantly, so we use this implementation - * only for platforms where we don't know of a suitable intrinsic. For the - * most part, those are relatively obscure platform/compiler combinations to - * which the PostgreSQL project does not have access. - */ -#define USE_DEFAULT_S_UNLOCK -extern void s_unlock(volatile slock_t *lock); -#define S_UNLOCK(lock) s_unlock(lock) -#endif /* S_UNLOCK */ - -#if !defined(S_INIT_LOCK) -#define S_INIT_LOCK(lock) S_UNLOCK(lock) -#endif /* S_INIT_LOCK */ - -#if !defined(SPIN_DELAY) -#define SPIN_DELAY() ((void) 0) -#endif /* SPIN_DELAY */ - -#if !defined(TAS) -extern int tas(volatile slock_t *lock); /* in port/.../tas.s, or - * s_lock.c */ - -#define TAS(lock) tas(lock) -#endif /* TAS */ - -#if !defined(TAS_SPIN) -#define TAS_SPIN(lock) TAS(lock) -#endif /* TAS_SPIN */ - - -/* - * Platform-independent out-of-line support routines - */ -extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); - -/* Support for dynamic adjustment of spins_per_delay */ -#define DEFAULT_SPINS_PER_DELAY 100 - -extern void set_spins_per_delay(int shared_spins_per_delay); -extern int update_spins_per_delay(int shared_spins_per_delay); - -/* - * Support for spin delay which is useful in various places where - * spinlock-like procedures take place. - */ -typedef struct -{ - int spins; - int delays; - int cur_delay; - const char *file; - int line; - const char *func; -} SpinDelayStatus; - -static inline void -init_spin_delay(SpinDelayStatus *status, - const char *file, int line, const char *func) -{ - status->spins = 0; - status->delays = 0; - status->cur_delay = 0; - status->file = file; - status->line = line; - status->func = func; -} - -#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) -extern void perform_spin_delay(SpinDelayStatus *status); -extern void finish_spin_delay(SpinDelayStatus *status); - -#endif /* S_LOCK_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 281214822244d..71cd69dc6947c 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -28,10 +28,9 @@ * for a CHECK_FOR_INTERRUPTS() to occur while holding a spinlock, and so * it is not necessary to do HOLD/RESUME_INTERRUPTS() in these functions. * - * These functions are implemented in terms of hardware-dependent macros - * supplied by s_lock.h. There is not currently any extra functionality - * added by this header, but there has been in the past and may someday - * be again. + * These functions are implemented on top of the C11 atomic + * operations in port/atomics.h; the contended-acquire slow path lives in + * s_lock() in s_lock.c. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -44,19 +43,17 @@ #ifndef SPIN_H #define SPIN_H -#ifdef USE_STDATOMIC_H - /* * Atomics-based spinlocks. * - * When stdatomic.h is available, spinlocks are implemented on top of a plain - * 32-bit atomic rather than platform-specific TAS assembly. We deliberately - * do NOT use pg_atomic_flag: that type uses 1==unlocked so it can offer a - * relaxed "unlocked test", whereas a spinlock must be usable when its memory - * has merely been zeroed (much shared-memory state is set up with + * Spinlocks are implemented directly on top of a plain 32-bit atomic rather + * than platform-specific TAS assembly. We deliberately do NOT use + * pg_atomic_flag: that type uses 1==unlocked so it can offer a relaxed + * "unlocked test", whereas a spinlock must be usable when its memory has + * merely been zeroed (a great deal of shared-memory state is set up with * memset(...,0,...) and then relies on embedded spinlocks reading as free). * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, - * making a zero-initialized slock_t a valid, free lock. + * which makes a zero-initialized slock_t a valid, free lock. */ #include "port/atomics.h" @@ -112,32 +109,4 @@ SpinLockRelease(volatile slock_t *lock) pg_atomic_write_u32(lock, 0); } -#else /* !USE_STDATOMIC_H */ - -/* - * Traditional spinlock implementation using platform-specific TAS assembly. - * This branch is kept byte-for-byte equivalent to the pre-stdatomic spin.h. - */ -#include "storage/s_lock.h" - -static inline void -SpinLockInit(volatile slock_t *lock) -{ - S_INIT_LOCK(lock); -} - -static inline void -SpinLockAcquire(volatile slock_t *lock) -{ - S_LOCK(lock); -} - -static inline void -SpinLockRelease(volatile slock_t *lock) -{ - S_UNLOCK(lock); -} - -#endif /* USE_STDATOMIC_H */ - #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index df6931a0cf59e..c0e993fd453e1 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -650,11 +650,8 @@ test_spinlock(void) * Basic tests for spinlocks, as well as the underlying operations. * * We embed the spinlock in a struct with other members to test that the - * spinlock operations don't perform too wide writes. - * - * Note: slock_t is pg_atomic_uint32 (4 bytes) in the stdatomic path vs - * potentially smaller (e.g., unsigned char) in the traditional path. This - * test verifies no over-wide writes regardless of slock_t size. + * spinlock operations don't perform too wide writes. slock_t is a + * pg_atomic_uint32 (4 bytes); this test verifies no over-wide writes. */ { struct test_lock_struct @@ -672,40 +669,9 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); -#ifndef USE_STDATOMIC_H - /* test basic operations via underlying S_* API */ - S_INIT_LOCK(&struct_w_lock.lock); - S_LOCK(&struct_w_lock.lock); - S_UNLOCK(&struct_w_lock.lock); -#endif - /* and that "contended" acquisition works */ s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); -#ifdef USE_STDATOMIC_H SpinLockRelease(&struct_w_lock.lock); -#else - S_UNLOCK(&struct_w_lock.lock); -#endif - -#ifndef USE_STDATOMIC_H - /* - * Check, using TAS directly, that a single spin cycle doesn't block - * when acquiring an already acquired lock. - */ -#ifdef TAS - S_LOCK(&struct_w_lock.lock); - - if (!TAS(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); - -#ifdef TAS_SPIN - if (!TAS_SPIN(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); -#endif /* defined(TAS_SPIN) */ - - S_UNLOCK(&struct_w_lock.lock); -#endif /* defined(TAS) */ -#endif /* !USE_STDATOMIC_H */ /* * Verify that after all of this the non-lock contents are still diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck index 785d6f867ad09..378100acf2c64 100755 --- a/src/tools/pginclude/headerscheck +++ b/src/tools/pginclude/headerscheck @@ -107,16 +107,6 @@ do test "$f" = src/include/port/win32ntdll.h && continue test "$f" = src/port/pthread-win32.h && continue - # Likewise, these files are platform-specific, and the one - # relevant to our platform will be included by atomics.h. - test "$f" = src/include/port/atomics/arch-arm.h && continue - test "$f" = src/include/port/atomics/arch-ppc.h && continue - test "$f" = src/include/port/atomics/arch-x86.h && continue - test "$f" = src/include/port/atomics/fallback.h && continue - test "$f" = src/include/port/atomics/generic.h && continue - test "$f" = src/include/port/atomics/generic-gcc.h && continue - test "$f" = src/include/port/atomics/generic-msvc.h && continue - # sepgsql.h depends on headers that aren't there on most platforms. test "$f" = contrib/sepgsql/sepgsql.h && continue