diff --git a/backend/index.js b/backend/index.js index 0028566727..b6d908ac94 100644 --- a/backend/index.js +++ b/backend/index.js @@ -3,47 +3,92 @@ import app from "./app.js"; import internalCertificate from "./internal/certificate.js"; import internalIpRanges from "./internal/ip_ranges.js"; +import internalIpRangesEO from "./internal/ip_ranges_eo.js"; import { global as logger } from "./logger.js"; import { migrateUp } from "./migrate.js"; import { getCompiledSchema } from "./schema/index.js"; import setup from "./setup.js"; const IP_RANGES_FETCH_ENABLED = process.env.IP_RANGES_FETCH_ENABLED !== "false"; +const EO_IP_RANGES_FETCH_ENABLED = process.env.EO_IP_RANGES_FETCH_ENABLED === "true"; + +// Timer env: +// 'true' = always, 'false' = never, +// anything else (including unset/'auto') = enable on successful fetch +const IP_RANGES_TIMER_ENABLED = process.env.IP_RANGES_TIMER_ENABLED; +const EO_IP_RANGES_TIMER_ENABLED = process.env.EO_IP_RANGES_TIMER_ENABLED; async function appStart() { - return migrateUp() - .then(setup) - .then(getCompiledSchema) - .then(() => { - if (!IP_RANGES_FETCH_ENABLED) { - logger.info("IP Ranges fetch is disabled by environment variable"); - return; + try { + await migrateUp(); + await setup(); + await getCompiledSchema(); + + // IP Ranges - Cloudflare and Cloudfront + if (IP_RANGES_FETCH_ENABLED) { + let ipRangesFetchSucceeded = false; + try { + await internalIpRanges.fetch(); + ipRangesFetchSucceeded = true; + logger.info("IP Ranges initial fetch succeeded."); + } catch (err) { + logger.error(`IP·Ranges·initial·fetch·failed:·${err.message}`); } - logger.info("IP Ranges fetch is enabled"); - return internalIpRanges.fetch().catch((err) => { - logger.error("IP Ranges fetch failed, continuing anyway:", err.message); - }); - }) - .then(() => { - internalCertificate.initTimer(); - internalIpRanges.initTimer(); - - const server = app.listen(3000, () => { - logger.info(`Backend PID ${process.pid} listening on port 3000 ...`); - - process.on("SIGTERM", () => { - logger.info(`PID ${process.pid} received SIGTERM`); - server.close(() => { - logger.info("Stopping."); - process.exit(0); - }); + + if ( + IP_RANGES_TIMER_ENABLED === 'true' || + (IP_RANGES_TIMER_ENABLED !== 'false' && ipRangesFetchSucceeded) + ) { + internalIpRanges.initTimer(); + logger.info("IP Ranges timer enabled."); + } else { + logger.info("IP Ranges timer not enabled."); + } + } else { + logger.info("IP Ranges fetch is disabled by environment variable"); + } + + // EO IP Ranges - EdgeOne + if (EO_IP_RANGES_FETCH_ENABLED) { + let eoIpRangesFetchSucceeded = false; + try { + await internalIpRangesEO.fetch(); + eoIpRangesFetchSucceeded = true; + logger.info("EO IP Ranges initial fetch succeeded."); + } catch (err) { + logger.error(`EO·IP·Ranges·initial·fetch·failed:·${err.message}`); + } + + if ( + EO_IP_RANGES_TIMER_ENABLED === 'true' || + (EO_IP_RANGES_TIMER_ENABLED !== 'false' && eoIpRangesFetchSucceeded) + ) { + internalIpRangesEO.initTimer(); + logger.info("EO IP Ranges timer enabled."); + } else { + logger.info("EO IP Ranges timer not enabled."); + } + } else { + logger.info("EO IP Ranges fetch is disabled by environment variable"); + } + + internalCertificate.initTimer(); + + const server = app.listen(3000, () => { + logger.info(`Backend PID ${process.pid} listening on port 3000 ...`); + + process.on("SIGTERM", () => { + logger.info(`PID ${process.pid} received SIGTERM`); + server.close(() => { + logger.info("Stopping."); + process.exit(0); }); }); - }) - .catch((err) => { - logger.error(`Startup Error: ${err.message}`, err); - setTimeout(appStart, 1000); }); + } catch (err) { + logger.error(`Startup Error: ${err.message}`, err); + setTimeout(appStart, 1000); + } } try { diff --git a/backend/internal/ip_ranges_eo.js b/backend/internal/ip_ranges_eo.js new file mode 100644 index 0000000000..8bb1f222ed --- /dev/null +++ b/backend/internal/ip_ranges_eo.js @@ -0,0 +1,289 @@ +import fs from "node:fs"; +import https from "node:https"; +import crypto from "node:crypto"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { ProxyAgent } from "proxy-agent"; +import errs from "../lib/error.js"; +import utils from "../lib/utils.js"; +import { ipRangesEO as logger } from "../logger.js"; +import internalNginx from "./nginx.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// ==== EdgeOne Environment Variables ==== +const EO_AUTO_CONFIRM_ENABLED = process.env.EO_AUTO_CONFIRM_ENABLED === "true"; +// Mainland China: teo.tencentcloudapi.com +// International: teo.intl.tencentcloudapi.com +const EO_API_BASE = process.env.EO_API_BASE || "teo.tencentcloudapi.com"; +const EO_API_SECRET_ID = process.env.EO_API_SECRET_ID || ""; +const EO_API_SECRET_KEY = process.env.EO_API_SECRET_KEY || ""; +const EO_ZONE_IDS = process.env.EO_ZONE_IDS ? process.env.EO_ZONE_IDS.split(",") : []; +// Default: 3 days (1000 * 60 * 60 * 72) +const EO_IP_RANGES_FETCH_INTERVAL = Number.parseInt(process.env.EO_IP_RANGES_FETCH_INTERVAL || "", 10) || 259200000; +const EO_IP_RANGES_DEBUG = process.env.EO_IP_RANGES_DEBUG === "true"; + +const internalIpRangesEo = { + interval: null, + interval_timeout: EO_IP_RANGES_FETCH_INTERVAL, + interval_processing: false, + iteration_count: 0, + + initTimer: () => { + logger.info("EdgeOne IP Ranges Renewal Timer initialized"); + if (internalIpRangesEo.interval) { + clearInterval(internalIpRangesEo.interval); + } + internalIpRangesEo.interval = setInterval( + internalIpRangesEo.fetch, + internalIpRangesEo.interval_timeout + ); + }, + + /** + * Makes a signed request to Tencent Cloud API v3 + * Returns a Promise that resolves with the response body string + */ + eoApiCall: (my_action, my_payload) => { + return new Promise((resolve, reject) => { + // Tencent Cloud API Signature v3 Logic + // Ref: https://cloud.tencent.com/document/product/213/30654 + + function sha256(message, secret, encoding) { + const hmac = crypto.createHmac("sha256", secret || ""); + return hmac.update(message).digest(encoding); + } + function getHash(message, encoding = "hex") { + const hash = crypto.createHash("sha256"); + return hash.update(message).digest(encoding); + } + function getDate(timestamp) { + const date = new Date(timestamp * 1000); + const year = date.getUTCFullYear(); + const month = (`0${date.getUTCMonth() + 1}`).slice(-2); + const day = (`0${date.getUTCDate()}`).slice(-2); + return `${year}-${month}-${day}`; + } + + const SECRET_ID = EO_API_SECRET_ID; + const SECRET_KEY = EO_API_SECRET_KEY; + + if (!SECRET_ID || !SECRET_KEY) { + reject(new Error("Missing EO_API_SECRET_ID or EO_API_SECRET_KEY")); + return; + } + + const host = EO_API_BASE; + const service = "teo"; + const action = my_action; + const version = "2022-09-01"; + const timestamp = Number.parseInt(String(Date.now() / 1000), 10); + const date = getDate(timestamp); + const payload = my_payload; + + // 1. Canonical Request + const signedHeaders = "content-type;host"; + const hashedRequestPayload = getHash(payload); + const httpRequestMethod = "POST"; + const canonicalUri = "/"; + const canonicalQueryString = ""; + const canonicalHeaders = `content-type:application/json; charset=utf-8\nhost:${host}\n`; + + const canonicalRequest = + httpRequestMethod + "\n" + + canonicalUri + "\n" + + canonicalQueryString + "\n" + + canonicalHeaders + "\n" + + signedHeaders + "\n" + + hashedRequestPayload; + + // 2. String to Sign + const algorithm = "TC3-HMAC-SHA256"; + const hashedCanonicalRequest = getHash(canonicalRequest); + const credentialScope = `${date}/${service}/tc3_request`; + const stringToSign = + algorithm + "\n" + + timestamp + "\n" + + credentialScope + "\n" + + hashedCanonicalRequest; + + // 3. Calculate Signature + const kDate = sha256(date, `TC3${SECRET_KEY}`); + const kService = sha256(service, kDate); + const kSigning = sha256("tc3_request", kService); + const signature = sha256(stringToSign, kSigning, "hex"); + + // 4. Authorization Header + const authorization = + algorithm + " " + + "Credential=" + SECRET_ID + "/" + credentialScope + ", " + + "SignedHeaders=" + signedHeaders + ", " + + "Signature=" + signature; + + const headers = { + Authorization: authorization, + "Content-Type": "application/json; charset=utf-8", + Host: host, + "X-TC-Action": action, + "X-TC-Timestamp": timestamp, + "X-TC-Version": version, + }; + + // Use ProxyAgent to support HTTP proxies if configured in env + const agent = new ProxyAgent(); + const options = { + hostname: host, + method: httpRequestMethod, + headers, + agent, + }; + + const req = https.request(options, (res) => { + let data = ""; + res.on("data", (chunk) => { + data += chunk; + }); + + res.on("end", () => { + if (EO_IP_RANGES_DEBUG) { + logger.debug(`eoApiCall(${action}, ${payload}) response: ${data}`); + } + resolve(data); + }); + }); + + req.on("error", (error) => { + logger.error(`eoApiCall(${action}, ${payload}) error: ${error}`); + reject(error); + }); + + req.write(payload); + req.end(); + }); + }, + + eoDescribeOriginACL: (zoneId) => { + return internalIpRangesEo.eoApiCall("DescribeOriginACL", JSON.stringify({ ZoneId: zoneId })); + }, + + eoConfirmOriginACLUpdate: (zoneId) => { + return internalIpRangesEo.eoApiCall("ConfirmOriginACLUpdate", JSON.stringify({ ZoneId: zoneId })); + }, + + /** + * Main fetch method for EdgeOne. + * Fetches IP ranges from EdgeOne API for each Zone ID, merges them, and writes to config. + */ + fetch: async () => { + if (internalIpRangesEo.interval_processing) { + return; + } + + // If no zone IDs configured, nothing to do + if (EO_ZONE_IDS.length === 0) { + return; + } + + internalIpRangesEo.interval_processing = true; + logger.info(`Fetching EdgeOne IP Ranges from API: ${EO_API_BASE}`); + + try { + const ip_ranges_4 = []; + const ip_ranges_6 = []; + + for (const zoneId of EO_ZONE_IDS) { + if (!zoneId.trim()) continue; + + try { + logger.info(`zone ID: ${zoneId}, Fetching new config`); + const response = await internalIpRangesEo.eoDescribeOriginACL(zoneId.trim()); + const jsonResponse = JSON.parse(response); + + // Check for API errors + if (jsonResponse.Response?.Error) { + throw new Error(`API Error: ${jsonResponse.Response.Error.Message}`); + } + + const aclInfo = jsonResponse?.Response?.OriginACLInfo; + + if (aclInfo?.CurrentOriginACL?.EntireAddresses) { + const addresses = aclInfo.CurrentOriginACL.EntireAddresses; + if (Array.isArray(addresses.IPv4)) { + ip_ranges_4.push(...addresses.IPv4); + } + if (Array.isArray(addresses.IPv6)) { + ip_ranges_6.push(...addresses.IPv6); + } + } + + // If NextOriginACL returns not null, it indicates new origin IP ranges are available for update. + if (Object.is(aclInfo.NextOriginACL, null)) continue; + logger.info(`zone ID: ${zoneId}, ACL update pending`); + + // Auto confirm if there is a pending update + if (!EO_AUTO_CONFIRM_ENABLED) continue; + logger.info(`zone ID: ${zoneId}, Auto-confirming ACL update`); + await internalIpRangesEo.eoConfirmOriginACLUpdate(zoneId.trim()); + + } catch (zoneErr) { + logger.error(`zone ID: ${zoneId}, Failed to fetch/process: ${zoneErr.message}`); + } + } + + const ip_ranges = [...ip_ranges_4, ...ip_ranges_6]; + + // De-duplicate ranges + const unique_ip_ranges = [...new Set(ip_ranges)]; + + if (unique_ip_ranges.length > 0) { + // Generate config + await internalIpRangesEo.generateConfig(unique_ip_ranges); + + // Reload nginx + if (internalIpRangesEo.iteration_count > 0) { + await internalNginx.reload(); + } + internalIpRangesEo.iteration_count++; + } else { + logger.warn("EdgeOne IP fetch resulted in 0 IPs, skipping config update."); + } + + } catch (err) { + logger.fatal(`EdgeOne IP range fetch failed: ${err.message}`); + } + + internalIpRangesEo.interval_processing = false; + }, + + /** + * @param {Array} ip_ranges + * @returns {Promise} + */ + generateConfig: (ip_ranges) => { + const renderEngine = utils.getRenderEngine(); + return new Promise((resolve, reject) => { + let template = null; + const filename = "/etc/nginx/conf.d/include/ip_ranges_eo.conf"; + try { + template = fs.readFileSync(`${__dirname}/../templates/ip_ranges.conf`, { encoding: "utf8" }); + } catch (err) { + reject(new errs.ConfigurationError(err.message)); + return; + } + + renderEngine + .parseAndRender(template, { ip_ranges: ip_ranges }) + .then((config_text) => { + fs.writeFileSync(filename, config_text, { encoding: "utf8" }); + resolve(true); + }) + .catch((err) => { + logger.warn(`Could not write ${filename}: ${err.message}`); + reject(new errs.ConfigurationError(err.message)); + }); + }); + }, +}; + +export default internalIpRangesEo; diff --git a/backend/logger.js b/backend/logger.js index 2b60dbff7b..a545e66798 100644 --- a/backend/logger.js +++ b/backend/logger.js @@ -15,6 +15,7 @@ const certbot = new signale.Signale({ scope: "Certbot ", ...opts }); const importer = new signale.Signale({ scope: "Importer ", ...opts }); const setup = new signale.Signale({ scope: "Setup ", ...opts }); const ipRanges = new signale.Signale({ scope: "IP Ranges", ...opts }); +const ipRangesEO = new signale.Signale({ scope: "EO Ranges", ...opts }); const remoteVersion = new signale.Signale({ scope: "Remote Version", ...opts }); const debug = (logger, ...args) => { @@ -23,4 +24,4 @@ const debug = (logger, ...args) => { } }; -export { debug, global, migrate, express, access, nginx, ssl, certbot, importer, setup, ipRanges, remoteVersion }; +export { debug, global, migrate, express, access, nginx, ssl, certbot, importer, setup, ipRanges, ipRangesEO, remoteVersion }; diff --git a/docker/rootfs/etc/nginx/conf.d/include/ip_ranges_eo.conf b/docker/rootfs/etc/nginx/conf.d/include/ip_ranges_eo.conf new file mode 100644 index 0000000000..e1e7a54886 --- /dev/null +++ b/docker/rootfs/etc/nginx/conf.d/include/ip_ranges_eo.conf @@ -0,0 +1,2 @@ +# This should be left blank, it is populated programatically by the application backend. +# EdgeOne IP ranges fetch is disabled by default, it requires setup in environment variables. diff --git a/docker/rootfs/etc/nginx/nginx.conf b/docker/rootfs/etc/nginx/nginx.conf index bdba3b3055..4e07287e34 100644 --- a/docker/rootfs/etc/nginx/nginx.conf +++ b/docker/rootfs/etc/nginx/nginx.conf @@ -77,8 +77,9 @@ http { set_real_ip_from 192.168.0.0/16; # NPM generated CDN ip ranges: include conf.d/include/ip_ranges[.]conf; + include conf.d/include/ip_ranges_eo[.]conf; # always put the following 2 lines after ip subnets: - real_ip_header X-Real-IP; + real_ip_header X-Forwarded-For; real_ip_recursive on; # Custom diff --git a/docs/src/advanced-config/index.md b/docs/src/advanced-config/index.md index 3ab04ce25b..4910874474 100644 --- a/docs/src/advanced-config/index.md +++ b/docs/src/advanced-config/index.md @@ -159,13 +159,98 @@ The easy fix is to add a Docker environment variable to the Nginx Proxy Manager DISABLE_IPV6: 'true' ``` -## Disabling IP Ranges Fetch +## Managing CDN IP Ranges -By default, NPM fetches IP ranges from CloudFront and Cloudflare during application startup. In environments with limited internet access or to speed up container startup, this fetch can be disabled: +By default, NPM fetches IP ranges from common CDN providers on application startup and updates them periodically in the background. You can control fetching and updating for each CDN source using environment variables. + +### Enable/Disable CDN IP Range Fetching + +These environment variables control whether NPM will ever fetch from each CDN provider. +If set to `'false'`, no fetch will be performed and no update timer will be set. + +- **`IP_RANGES_FETCH_ENABLED`** + - Controls fetching of CloudFront & Cloudflare IP ranges + - `'true'`, Unset (default): Allow fetching and optional periodic updates + - `'false'`: Completely disables fetching and timer + +- **`EO_IP_RANGES_FETCH_ENABLED`** + - Controls fetching of EdgeOne IP ranges + - `'true'`: Allow fetching and optional periodic updates + - `'false'`, Unset (default): Completely disables fetching and timer + +### Auto-Update Timers + +These control whether the application will keep updating the CDN IP ranges in the background, if fetching for that CDN is enabled: + +- **`IP_RANGES_TIMER_ENABLED`**: Controls the CloudFront/Cloudflare auto-update timer +- **`EO_IP_RANGES_TIMER_ENABLED`**: Controls the EdgeOne auto-update timer + +Possible values: + +- `'true'`: Always enable the timer, even if the initial fetch fails +- `'false'`: Never enable the timer after startup +- `'auto'`, Unset (default): Enable the timer only if the initial fetch succeeds + +### Example docker-compose configuration ```yml - environment: - IP_RANGES_FETCH_ENABLED: 'false' +environment: + IP_RANGES_FETCH_ENABLED: 'true' # Enable CloudFront & Cloudflare fetching (and timer, if configured) + IP_RANGES_TIMER_ENABLED: 'auto' # Timer runs only if initial fetch succeeded + EO_IP_RANGES_FETCH_ENABLED: 'false' # Disable EdgeOne IP ranges completely (no fetch, no timer) +``` + +Note: For EdgeOne, further config and credentials are required (see next section). + +## Enabling EdgeOne IP Ranges Fetch \( [Origin Protection](https://www.tencentcloud.com/document/product/1145/48535) \) + +> IP ranges here is only used for extracting client IP from header, not actually protecting the server at network level. + +EdgeOne API requires tencent cloud credentials to work, you should get one first, follow the steps +1. navigate to tencent cloud console - Cloud Access Management - Policies, create a custom policy - Create according to the policy syntax - select Blank Template, set its name, paste below snippet + +```json +{ + "statement": [ + { + "action": [ + "teo:DescribeOriginACL", + "teo:ConfirmOriginACLUpdate" + ], + "effect": "allow", + "resource": [ + "*" + ] + } + ], + "version": "2.0" +} +``` +- [DescribeOriginACL](https://www.tencentcloud.com/document/product/1145/71118) query EdgeOne server IP range for a given zone. +- [ConfirmOriginACLUpdate](https://www.tencentcloud.com/document/product/1145/71119) confirms the latest IP ranges have been applied. + +2. create a new user: User List - Create User - Custom + - Select a type: Accessible resources and message reception + - Fill in the user information: Enable Programming access + - Set user permissions: choose the policy we just created + - once completed, save its `SecretId` and `SecretKey` + +4. fill in these environment variables + +> `EO_API_BASE` +> - Mainland China: `teo.tencentcloudapi.com` +> - International: `teo.intl.tencentcloudapi.com` + +```yml + environment: + EO_IP_RANGES_FETCH_ENABLED: 'false' # Controls whether EdgeOne IP fetches run + EO_AUTO_CONFIRM_ENABLED: 'false' # Calls ConfirmOriginACLUpdate if an update is found + EO_API_BASE: '' # EdgeOne API endpoint + EO_API_SECRET_ID: '' # API credential: Secret ID + EO_API_SECRET_KEY: '' # API credential: Secret Key + EO_ZONE_IDS: '' # Comma separated Zone IDs to fetch and merge + EO_IP_RANGES_FETCH_INTERVAL: '259200000' # (ms) How often to fetch, default 3 days + EO_IP_RANGES_DEBUG: 'false' # Print debug logs for troubleshooting ``` ## Custom Nginx Configurations