Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 167 additions & 2 deletions .github/scripts/update_metrics.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"""Refresh METRICS.md from PyPI + GitHub APIs.
"""Refresh METRICS.md and download charts from PyPI + GitHub APIs.

Pulled signals:
- PyPI download counts (last day / week / month) via pypistats.org JSON API.
- GitHub repo metadata (stars, forks, watchers, open issues/PRs).
- GitHub traffic (clones, views, referrers, paths) — last 14 days.
- PyPI daily download chart via pepy.tech.

Plugin install proxy: `/plugin marketplace add <owner>/<repo>` is a git
clone under the hood, so daily clone counts approximate plugin installs.
Expand All @@ -19,12 +20,14 @@
import sys
import urllib.error
import urllib.request
from datetime import datetime, timezone
from datetime import date, datetime, timedelta, timezone
from html import escape
from pathlib import Path

REPO = os.environ.get("GITHUB_REPOSITORY", "raintree-technology/docpull")
PKG = os.environ.get("PYPI_PACKAGE", "docpull")
OUTPUT = Path(os.environ.get("METRICS_OUTPUT", "METRICS.md"))
DOWNLOAD_CHART_OUTPUT = Path(os.environ.get("DOWNLOAD_CHART_OUTPUT", "docs/downloads-history.svg"))


def gh(path: str) -> dict | list:
Expand Down Expand Up @@ -66,11 +69,37 @@ def pypistats(path: str) -> dict:
return json.loads(r.read().decode("utf-8"))


def pepy_project() -> dict:
"""GET project download history from Pepy.

Pepy exposes all-time totals plus a per-version daily breakdown for the
last roughly 90 days. We aggregate that version map into daily totals for
the README chart.
"""
url = f"https://pepy.tech/api/v2/projects/{PKG}"
req = urllib.request.Request(
url,
headers={"User-Agent": f"{REPO}-metrics-workflow (+https://github.com/{REPO})"},
)
with urllib.request.urlopen(req, timeout=30) as r:
return json.loads(r.read().decode("utf-8"))


def fmt(n: int | float) -> str:
"""Format a number with thousands separators."""
return f"{int(n):,}"


def short_fmt(n: int | float) -> str:
"""Compact label for chart axes."""
n = int(n)
if abs(n) >= 1_000_000:
return f"{n / 1_000_000:.1f}M"
if abs(n) >= 1_000:
return f"{n / 1_000:.1f}k"
return str(n)


def safe_get(fn, default, *, on_error: list[str] | None = None):
"""Best-effort wrapper — never let a transient API hiccup blank METRICS.md.

Expand All @@ -93,6 +122,139 @@ def safe_get(fn, default, *, on_error: list[str] | None = None):
return default


def download_series(pepy: dict, *, days: int = 90) -> list[tuple[date, int]]:
"""Aggregate Pepy's per-version daily downloads into one daily series."""
raw = pepy.get("downloads", {})
totals: dict[date, int] = {}
for day, versions in raw.items():
try:
day_key = date.fromisoformat(day)
except ValueError:
continue
if isinstance(versions, dict):
totals[day_key] = sum(int(v or 0) for v in versions.values())

end = max(totals) if totals else datetime.now(timezone.utc).date()
start = end - timedelta(days=days - 1)
return [(start + timedelta(days=i), totals.get(start + timedelta(days=i), 0)) for i in range(days)]


def render_download_chart(series: list[tuple[date, int]], *, total_downloads: int) -> str:
"""Render a dependency-free SVG chart for README embedding."""
width = 920
height = 360
left = 70
right = 28
top = 72
bottom = 64
chart_w = width - left - right
chart_h = height - top - bottom
max_y = max([value for _, value in series] + [1])
# Give the line a little headroom and round the top tick to a clean value.
top_tick = max(10, int(((max_y * 1.18) + 9) // 10 * 10))

def x_for(i: int) -> float:
if len(series) <= 1:
return left
return left + (i / (len(series) - 1)) * chart_w

def y_for(value: int | float) -> float:
return top + chart_h - (float(value) / top_tick) * chart_h

points = [(x_for(i), y_for(value)) for i, (_, value) in enumerate(series)]
line_points = " ".join(f"{x:.1f},{y:.1f}" for x, y in points)
area_points = (
f"{left:.1f},{top + chart_h:.1f} " + line_points + f" {left + chart_w:.1f},{top + chart_h:.1f}"
)

grid_lines: list[str] = []
axis_labels: list[str] = []
for tick in range(5):
value = top_tick * tick / 4
y = y_for(value)
grid_lines.append(
f'<line x1="{left}" y1="{y:.1f}" x2="{left + chart_w}" y2="{y:.1f}" '
'stroke="#e5e7eb" stroke-width="1" />'
)
axis_labels.append(
f'<text x="{left - 14}" y="{y + 4:.1f}" text-anchor="end" '
f'font-size="12" fill="#6b7280">{escape(short_fmt(value))}</text>'
)

month_labels: list[str] = []
seen_months: set[tuple[int, int]] = set()
for i, (day, _) in enumerate(series):
key = (day.year, day.month)
if key in seen_months:
continue
seen_months.add(key)
x = x_for(i)
month_labels.append(
f'<text x="{x:.1f}" y="{height - 26}" text-anchor="middle" '
f'font-size="12" fill="#6b7280">{escape(day.strftime("%b %d"))}</text>'
)

start_label = series[0][0].strftime("%b %-d, %Y") if series else ""
end_label = series[-1][0].strftime("%b %-d, %Y") if series else ""
latest_value = series[-1][1] if series else 0
total_window = sum(value for _, value in series)
subtitle = (
f"Daily downloads, last {len(series)} days · {fmt(total_window)} in window · "
f"{fmt(total_downloads)} all time"
)
generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
svg_lines = [
(
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
f'viewBox="0 0 {width} {height}" role="img" aria-labelledby="title desc">'
),
f' <title id="title">{escape(PKG)} PyPI download history</title>',
(
f' <desc id="desc">Daily PyPI downloads from Pepy for {escape(PKG)} '
f"between {escape(start_label)} and {escape(end_label)}.</desc>"
),
f' <rect width="{width}" height="{height}" rx="16" fill="#ffffff" />',
(
f' <text x="{left}" y="34" font-size="22" font-weight="700" '
f'fill="#111827">{escape(PKG)} PyPI downloads</text>'
),
f' <text x="{left}" y="56" font-size="13" fill="#6b7280">{escape(subtitle)}</text>',
(
f' <text x="{width - right}" y="34" text-anchor="end" font-size="20" '
f'font-weight="700" fill="#2563eb">{escape(short_fmt(latest_value))}</text>'
),
f' <text x="{width - right}" y="55" text-anchor="end" font-size="12" '
'fill="#6b7280">latest day</text>',
*grid_lines,
*axis_labels,
(
f' <line x1="{left}" y1="{top + chart_h}" x2="{left + chart_w}" '
f'y2="{top + chart_h}" stroke="#d1d5db" stroke-width="1.5" />'
),
f' <polygon points="{area_points}" fill="#dbeafe" opacity="0.85" />',
(
f' <polyline points="{line_points}" fill="none" stroke="#2563eb" '
'stroke-width="3" stroke-linecap="round" stroke-linejoin="round" />'
),
*month_labels,
(f' <text x="{left}" y="{height - 10}" font-size="11" fill="#9ca3af">Source: pepy.tech API</text>'),
(
f' <text x="{width - right}" y="{height - 10}" text-anchor="end" '
f'font-size="11" fill="#9ca3af">Generated {escape(generated_at)}</text>'
),
"</svg>",
"",
]
return "\n".join(svg_lines)


def write_download_chart(pepy: dict) -> None:
series = download_series(pepy)
total_downloads = int(pepy.get("total_downloads", 0) or 0)
DOWNLOAD_CHART_OUTPUT.parent.mkdir(parents=True, exist_ok=True)
DOWNLOAD_CHART_OUTPUT.write_text(render_download_chart(series, total_downloads=total_downloads))


def main() -> int:
now = datetime.now(timezone.utc)
timestamp = now.strftime("%Y-%m-%d %H:%M UTC")
Expand Down Expand Up @@ -121,7 +283,10 @@ def main() -> int:

pypi_errors: list[str] = []
recent = safe_get(lambda: pypistats("/recent")["data"], {}, on_error=pypi_errors)
pepy = safe_get(lambda: pepy_project(), {}, on_error=pypi_errors)
pypi_blocked = bool(pypi_errors)
if pepy:
write_download_chart(pepy)

stars = repo.get("stargazers_count", 0)
forks = repo.get("forks_count", 0)
Expand Down
9 changes: 6 additions & 3 deletions .github/workflows/metrics.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
name: Update metrics

# Refreshes METRICS.md from PyPI + GitHub APIs.
# Refreshes METRICS.md and the README downloads chart from PyPI + GitHub APIs.
#
# Signals:
# - PyPI downloads (24h / 7d / 30d) via pypistats.org JSON API.
# - PyPI downloads chart via pepy.tech JSON API.
# - GitHub repo metadata (stars, forks, watchers, open issues/PRs).
# - GitHub traffic (clones, views, referrers, paths) — last 14 days.
#
Expand Down Expand Up @@ -48,7 +49,7 @@ jobs:
with:
python-version: "3.11"

- name: Refresh METRICS.md
- name: Refresh metrics artifacts
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
METRICS_TOKEN: ${{ secrets.METRICS_TOKEN }}
Expand All @@ -68,7 +69,9 @@ jobs:
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
continue-on-error: true
with:
add-paths: METRICS.md
add-paths: |
METRICS.md
docs/downloads-history.svg
branch: automation/metrics-refresh
delete-branch: true
commit-message: "chore(metrics): refresh metrics"
Expand Down
65 changes: 35 additions & 30 deletions METRICS.md
Original file line number Diff line number Diff line change
@@ -1,61 +1,66 @@
# docpull metrics

_Last updated: 2026-05-22 11:59 UTC. Auto-generated by `.github/workflows/metrics.yml`; do not edit by hand._
_Last updated: 2026-06-12 18:16 UTC. Auto-generated by `.github/workflows/metrics.yml`; do not edit by hand._

## Snapshot

| Metric | Value |
|---|---|
| PyPI downloads (last 24h) | 0 |
| PyPI downloads (last 7d) | 12 |
| PyPI downloads (last 30d) | 763 |
| PyPI downloads (last 24h) | 12 |
| PyPI downloads (last 7d) | 425 |
| PyPI downloads (last 30d) | 933 |
| GitHub stars | 21 |
| GitHub forks | 2 |
| GitHub watchers | 1 |
| Open issues | 0 |
| Open PRs | 0 |
| Repo clones (last 14d) | 219 |
| Unique cloners (last 14d) | 105 |
| Repo views (last 14d) | 15 |
| Unique visitors (last 14d) | 10 |
| Open PRs | 6 |
| Repo clones (last 14d) | 3,029 |
| Unique cloners (last 14d) | 425 |
| Repo views (last 14d) | 95 |
| Unique visitors (last 14d) | 9 |

## Plugin install proxy: daily clones (last 14d)

`/plugin marketplace add raintree-technology/docpull` is a git clone under the hood, so daily clone counts approximate plugin installs.

| Date | Clones | Unique cloners |
|---|---|---|
| 2026-05-21 | 25 | 14 |
| 2026-05-20 | 14 | 6 |
| 2026-05-19 | 9 | 7 |
| 2026-05-18 | 6 | 5 |
| 2026-05-17 | 12 | 7 |
| 2026-05-16 | 23 | 13 |
| 2026-05-15 | 10 | 6 |
| 2026-05-14 | 26 | 13 |
| 2026-05-13 | 9 | 7 |
| 2026-05-12 | 13 | 8 |
| 2026-05-11 | 16 | 10 |
| 2026-05-10 | 15 | 10 |
| 2026-05-09 | 22 | 13 |
| 2026-05-08 | 19 | 8 |
| 2026-06-10 | 121 | 26 |
| 2026-06-09 | 173 | 40 |
| 2026-06-08 | 315 | 68 |
| 2026-06-07 | 33 | 13 |
| 2026-06-06 | 422 | 78 |
| 2026-06-05 | 6 | 5 |
| 2026-06-04 | 489 | 92 |
| 2026-06-03 | 125 | 22 |
| 2026-06-02 | 16 | 7 |
| 2026-06-01 | 10 | 7 |
| 2026-05-31 | 23 | 13 |
| 2026-05-30 | 159 | 19 |
| 2026-05-29 | 1,135 | 122 |
| 2026-05-28 | 2 | 1 |

## Top referrers (last 14d)

| Source | Views | Unique |
|---|---|---|
| github.com | 6 | 5 |
| Google | 1 | 1 |
| github.com | 17 | 6 |
| Google | 2 | 2 |

## Top paths (last 14d)

| Path | Views | Unique |
|---|---|---|
| `/` | 11 | 9 |
| `/blob/main/.editorconfig` | 1 | 1 |
| `/blob/main/.gitignore` | 1 | 1 |
| `/graphs/contributors` | 1 | 1 |
| `/tree/main/mcp` | 1 | 1 |
| `/` | 43 | 8 |
| `/pulls` | 11 | 1 |
| `/pull/49` | 4 | 1 |
| `/branches` | 3 | 1 |
| `/pull/61` | 3 | 1 |
| `/actions/runs/26967522318/job/79574197483` | 2 | 1 |
| `/actions/runs/27053968208/job/79854561710` | 2 | 1 |
| `/actions/runs/27073500057` | 2 | 1 |
| `/pull/43` | 2 | 1 |
| `/pull/63` | 2 | 1 |

## Drill deeper

Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@
</a>
</p>

## Download History

<a href="https://pepy.tech/project/docpull">
<img alt="PyPI download history chart for docpull" src="docs/downloads-history.svg" />
</a>

## Star History

<a href="https://star-history.com/#raintree-technology/docpull&Date">
Expand Down
28 changes: 28 additions & 0 deletions docs/downloads-history.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading