Skip to content

Commit 7cb13a4

Browse files
henrymcconvillehenrymcconville
authored
Reinstate GITHUB_RATE_LIMIT for quota-based token refreshes (#136)
* re-add rate limit threshold
1 parent f6b3669 commit 7cb13a4

File tree

5 files changed

+258
-1
lines changed

5 files changed

+258
-1
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ This exporter is configured via environment variables. All variables are optiona
1818
| `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | |
1919
| `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | |
2020
| `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` |
21+
| `GITHUB_RATE_LIMIT` | Core API quota threshold for proactive GitHub App token refresh. When the remaining `core` requests drop below this value, a new installation token is requested. `0` disables this behaviour. | `0` |
2122
| `GITHUB_RESULTS_PER_PAGE` | Number of results to request per page from the GitHub API (max 100). | `100` |
2223
| `FETCH_REPO_RELEASES_ENABLED` | Whether to fetch repository release metrics (`true` or `false`). | `true` |
2324
| `FETCH_ORGS_CONCURRENCY` | Number of concurrent requests to make when fetching organization data. | `1` |

‎config/config.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ type Config struct {
2828
GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"`
2929
GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"`
3030
GitHubRateLimitEnabled bool `envconfig:"GITHUB_RATE_LIMIT_ENABLED" required:"false" default:"true"`
31+
GitHubRateLimit float64 `envconfig:"GITHUB_RATE_LIMIT" required:"false" default:"0"`
3132
FetchRepoReleasesEnabled bool `envconfig:"FETCH_REPO_RELEASES_ENABLED" required:"false" default:"true"`
3233
FetchOrgsConcurrency int `envconfig:"FETCH_ORGS_CONCURRENCY" required:"false" default:"1"`
3334
FetchOrgReposConcurrency int `envconfig:"FETCH_ORG_REPOS_CONCURRENCY" required:"false" default:"1"`
@@ -88,7 +89,7 @@ func (c *Config) GetClient() (*github.Client, error) {
8889

8990
// Add custom transport for GitHub App authentication if enabled
9091
if c.GitHubApp {
91-
itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath)
92+
itr, err := ghinstallation.NewKeyFromFile(transport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath)
9293
if err != nil {
9394
return nil, fmt.Errorf("creating GitHub App installation transport: %v", err)
9495
}

‎config/config_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ func TestConfig(t *testing.T) {
3434
GitHubApp: false,
3535
GitHubAppConfig: nil,
3636
GitHubRateLimitEnabled: true,
37+
GitHubRateLimit: 0,
3738
FetchRepoReleasesEnabled: true,
3839
FetchOrgsConcurrency: 1,
3940
FetchOrgReposConcurrency: 1,
@@ -87,6 +88,7 @@ func TestConfig(t *testing.T) {
8788
GitHubApp: false,
8889
GitHubAppConfig: nil,
8990
GitHubRateLimitEnabled: false,
91+
GitHubRateLimit: 0,
9092
FetchRepoReleasesEnabled: false,
9193
FetchOrgsConcurrency: 2,
9294
FetchOrgReposConcurrency: 3,
@@ -95,6 +97,37 @@ func TestConfig(t *testing.T) {
9597
},
9698
expectedErr: nil,
9799
},
100+
{
101+
name: "github rate limit threshold config",
102+
envVars: map[string]string{
103+
"GITHUB_RATE_LIMIT": "15000",
104+
},
105+
expectedCfg: &Config{
106+
MetricsPath: "/metrics",
107+
ListenPort: "9171",
108+
LogLevel: "INFO",
109+
ApiUrl: &url.URL{
110+
Scheme: "https",
111+
Host: "api.github.com",
112+
},
113+
Repositories: []string{},
114+
Organisations: []string{},
115+
Users: []string{},
116+
GitHubResultsPerPage: 100,
117+
GithubToken: "",
118+
GithubTokenFile: "",
119+
GitHubApp: false,
120+
GitHubAppConfig: nil,
121+
GitHubRateLimitEnabled: true,
122+
GitHubRateLimit: 15000,
123+
FetchRepoReleasesEnabled: true,
124+
FetchOrgsConcurrency: 1,
125+
FetchOrgReposConcurrency: 1,
126+
FetchReposConcurrency: 1,
127+
FetchUsersConcurrency: 1,
128+
},
129+
expectedErr: nil,
130+
},
98131
{
99132
name: "invalid url",
100133
expectedCfg: nil,

‎exporter/prometheus.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
5757
return
5858
}
5959

60+
// Refresh client if rate limit threshold enabled
61+
e.rateLimitRefresh(r)
62+
6063
// Set prometheus gauge metrics using the data gathered
6164
err = e.processMetrics(data, r, ch)
6265
if err != nil {
@@ -107,6 +110,32 @@ func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) {
107110
return &rls, nil
108111
}
109112

113+
func (e *Exporter) rateLimitRefresh(rates *[]RateLimit) {
114+
if e.GitHubRateLimit <= 0 || rates == nil {
115+
return
116+
}
117+
118+
for _, rl := range *rates {
119+
if rl.Resource == "core" && rl.Remaining <= e.GitHubRateLimit {
120+
log.Infof("GitHub API core rate limit is low (%.0f remaining, threshold: %.0f)", rl.Remaining, e.GitHubRateLimit)
121+
if !e.GitHubApp {
122+
log.Warn("GITHUB_RATE_LIMIT threshold breached but GitHub App auth is not configured; cannot refresh token automatically")
123+
return
124+
}
125+
126+
log.Info("Refreshing GitHub App installation token due to low rate limit")
127+
newClient, err := e.GetClient()
128+
if err != nil {
129+
log.Errorf("refreshing GitHub App client after low rate limit: %v", err)
130+
return
131+
}
132+
133+
e.Client = newClient
134+
return
135+
}
136+
}
137+
}
138+
110139
// getRepoMetrics fetches metrics for the configured repositories
111140
func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) {
112141
var data []*Datum

‎exporter/prometheus_test.go‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
package exporter
2+
3+
import (
4+
"crypto/rand"
5+
"crypto/rsa"
6+
"crypto/x509"
7+
"encoding/pem"
8+
"os"
9+
"testing"
10+
11+
"github.com/githubexporter/github-exporter/config"
12+
"github.com/google/go-github/v76/github"
13+
"github.com/stretchr/testify/assert"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
// writeTempRSAKey generates a 2048-bit RSA private key and writes it as a
18+
// PKCS#1 PEM file to a temp path. The file is removed automatically when the
19+
// test finishes.
20+
func writeTempRSAKey(t *testing.T) string {
21+
t.Helper()
22+
key, err := rsa.GenerateKey(rand.Reader, 2048)
23+
require.NoError(t, err)
24+
25+
f, err := os.CreateTemp("", "test-github-app-key-*.pem")
26+
require.NoError(t, err)
27+
defer f.Close()
28+
29+
err = pem.Encode(f, &pem.Block{
30+
Type: "RSA PRIVATE KEY",
31+
Bytes: x509.MarshalPKCS1PrivateKey(key),
32+
})
33+
require.NoError(t, err)
34+
35+
t.Cleanup(func() { os.Remove(f.Name()) })
36+
37+
return f.Name()
38+
}
39+
40+
func newTestExporter(cfg config.Config) *Exporter {
41+
return &Exporter{
42+
Client: github.NewClient(nil),
43+
Config: cfg,
44+
}
45+
}
46+
47+
func TestRateLimitRefresh(t *testing.T) {
48+
// Shared rate-limit slices used across sub-tests.
49+
aboveThreshold := []RateLimit{{Resource: "core", Remaining: 200, Limit: 5000}}
50+
atThreshold := []RateLimit{{Resource: "core", Remaining: 100, Limit: 5000}}
51+
belowThreshold := []RateLimit{{Resource: "core", Remaining: 50, Limit: 5000}}
52+
noCoreEntry := []RateLimit{{Resource: "search", Remaining: 5, Limit: 30}}
53+
54+
baseCfg := config.Config{
55+
GitHubResultsPerPage: 100,
56+
FetchReposConcurrency: 1,
57+
FetchOrgsConcurrency: 1,
58+
FetchOrgReposConcurrency: 1,
59+
FetchUsersConcurrency: 1,
60+
}
61+
62+
t.Run("no-op when GitHubRateLimit is zero", func(t *testing.T) {
63+
cfg := baseCfg
64+
cfg.GitHubRateLimit = 0
65+
e := newTestExporter(cfg)
66+
original := e.Client
67+
68+
e.rateLimitRefresh(&atThreshold)
69+
70+
assert.Same(t, original, e.Client)
71+
})
72+
73+
t.Run("no-op when GitHubRateLimit is negative", func(t *testing.T) {
74+
cfg := baseCfg
75+
cfg.GitHubRateLimit = -1
76+
e := newTestExporter(cfg)
77+
original := e.Client
78+
79+
e.rateLimitRefresh(&atThreshold)
80+
81+
assert.Same(t, original, e.Client)
82+
})
83+
84+
t.Run("no-op when rates is nil", func(t *testing.T) {
85+
cfg := baseCfg
86+
cfg.GitHubRateLimit = 100
87+
e := newTestExporter(cfg)
88+
original := e.Client
89+
90+
e.rateLimitRefresh(nil)
91+
92+
assert.Same(t, original, e.Client)
93+
})
94+
95+
t.Run("no-op when core remaining is above threshold", func(t *testing.T) {
96+
cfg := baseCfg
97+
cfg.GitHubRateLimit = 100 // remaining=200 > threshold=100
98+
e := newTestExporter(cfg)
99+
original := e.Client
100+
101+
e.rateLimitRefresh(&aboveThreshold)
102+
103+
assert.Same(t, original, e.Client)
104+
})
105+
106+
t.Run("no-op when rates has no core entry", func(t *testing.T) {
107+
cfg := baseCfg
108+
cfg.GitHubRateLimit = 100
109+
e := newTestExporter(cfg)
110+
original := e.Client
111+
112+
e.rateLimitRefresh(&noCoreEntry)
113+
114+
assert.Same(t, original, e.Client)
115+
})
116+
117+
t.Run("no-op when GitHubApp is false and core is at threshold", func(t *testing.T) {
118+
cfg := baseCfg
119+
cfg.GitHubRateLimit = 100 // remaining=100 <= threshold=100
120+
cfg.GitHubApp = false
121+
e := newTestExporter(cfg)
122+
original := e.Client
123+
124+
e.rateLimitRefresh(&atThreshold)
125+
126+
assert.Same(t, original, e.Client)
127+
})
128+
129+
t.Run("no-op when GitHubApp is false and core is below threshold", func(t *testing.T) {
130+
cfg := baseCfg
131+
cfg.GitHubRateLimit = 100 // remaining=50 < threshold=100
132+
cfg.GitHubApp = false
133+
e := newTestExporter(cfg)
134+
original := e.Client
135+
136+
e.rateLimitRefresh(&belowThreshold)
137+
138+
assert.Same(t, original, e.Client)
139+
})
140+
141+
t.Run("no-op when GitHubApp is true but GetClient fails", func(t *testing.T) {
142+
cfg := baseCfg
143+
cfg.GitHubRateLimit = 100
144+
cfg.GitHubApp = true
145+
cfg.GitHubAppConfig = &config.GitHubAppConfig{
146+
GitHubAppKeyPath: "/nonexistent/key.pem",
147+
GitHubAppId: 1,
148+
GitHubAppInstallationId: 1,
149+
}
150+
e := newTestExporter(cfg)
151+
original := e.Client
152+
153+
e.rateLimitRefresh(&atThreshold)
154+
155+
assert.Same(t, original, e.Client)
156+
})
157+
158+
t.Run("replaces client when GitHubApp is true and core is at threshold", func(t *testing.T) {
159+
keyPath := writeTempRSAKey(t)
160+
cfg := baseCfg
161+
cfg.GitHubRateLimit = 100 // remaining=100 <= threshold=100
162+
cfg.GitHubApp = true
163+
cfg.GitHubAppConfig = &config.GitHubAppConfig{
164+
GitHubAppKeyPath: keyPath,
165+
GitHubAppId: 1,
166+
GitHubAppInstallationId: 1,
167+
}
168+
e := newTestExporter(cfg)
169+
original := e.Client
170+
171+
e.rateLimitRefresh(&atThreshold)
172+
173+
assert.NotSame(t, original, e.Client, "expected client to be replaced after rate limit refresh")
174+
})
175+
176+
t.Run("replaces client when GitHubApp is true and core is below threshold", func(t *testing.T) {
177+
keyPath := writeTempRSAKey(t)
178+
cfg := baseCfg
179+
cfg.GitHubRateLimit = 100 // remaining=50 < threshold=100
180+
cfg.GitHubApp = true
181+
cfg.GitHubAppConfig = &config.GitHubAppConfig{
182+
GitHubAppKeyPath: keyPath,
183+
GitHubAppId: 1,
184+
GitHubAppInstallationId: 1,
185+
}
186+
e := newTestExporter(cfg)
187+
original := e.Client
188+
189+
e.rateLimitRefresh(&belowThreshold)
190+
191+
assert.NotSame(t, original, e.Client, "expected client to be replaced after rate limit refresh")
192+
})
193+
}

0 commit comments

Comments
 (0)