Custom request headers

lib/request.js

@@ -152,17 +152,13 @@ export default function(options, iframely_options, callback) {
             });
         }
 
-        if (!options.headers || !options.headers['User-Agent']) {
-            options.headers = options.headers || {};
-            options.headers['User-Agent'] = CONFIG.USER_AGENT;
-        }
-
         iframely_options.registerFetch({
             source: 'api',
             url: options.uri
         });
 
-        fetchData(utils.prepareRequestOptions(options, iframely_options))
+        utils.prepareRequestOptions(options, iframely_options)
+            .then(request_options => fetchData(request_options))
             .then(result => {
                 finish(null, result);
             })

lib/utils.js

@@ -82,99 +82,140 @@ export function getMaxCacheTTL(url, options, default_min_ttl) {
 };
 
 export function prepareRequestOptions(request_options, options) {
+    return new Promise((resolve, reject) => {
+        try {
+            if (request_options.url && !request_options.uri) {
+                request_options.uri = request_options.url;
+            }
+            var uri = request_options.uri;
+            var original_uri = uri;
+            delete request_options.url;
 
-    if (request_options.url && !request_options.uri) {
-        request_options.uri = request_options.url;
-    }
-    var uri = request_options.uri;
-    delete request_options.url;
-
-    var disable_language = false;
+            var disable_language = false;
+            var request_sig = false;
 
-    function setPrerender(prerender_option) {
-        if (options?.allowPrerender && CONFIG.PRERENDER_URL) {
-            request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
-
-            // Use string `proxy.prerender` as additional prerender param.
-            if (typeof prerender_option === 'string') {
-                request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`;
+            function setPrerender(prerender_option) {
+                if (options?.allowPrerender && CONFIG.PRERENDER_URL) {
+                    request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
+
+                    // Use string `proxy.prerender` as additional prerender param.
+                    if (typeof prerender_option === 'string') {
+                        request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`;
+                    }
+                }
             }
-        }
-    }
 
-    var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender');
-    if (enable_domain_prerender) {
-        setPrerender(enable_domain_prerender);
-    }
+            var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender');
+            if (enable_domain_prerender) {
+                setPrerender(enable_domain_prerender);
+            }
 
-    if (CONFIG.PROXY || (options && options.proxy)) {
+            request_options.headers = request_options.headers || {};
+
+            if (CONFIG.PROXY || options?.proxy) {
+
+                var proxy = options?.proxy || getCustomProxyForUri(uri, options);
+                if (proxy) {
+                    if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) {
+                        setPrerender(proxy.prerender);
+
+                    } else if (proxy.proxy && CONFIG.PROXY_URL) {
+                        request_options.uri = /{url}/.test(CONFIG.PROXY_URL) 
+                            ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri))
+                            : CONFIG.PROXY_URL + encodeURIComponent(uri);
+                    } else if (proxy.proxy_url && /{url}/.test(proxy.proxy_url)) {
+                        request_options.uri = proxy.proxy_url.replace(/{url}/, encodeURIComponent(uri));
+                    }
+                    if (proxy.user_agent) {
+                        request_options.headers['User-Agent'] = proxy.user_agent;
+                    }
+                    if (proxy.headers) {
+                        request_options.headers = request_options.headers || {};
+                        Object.assign(request_options.headers, proxy.headers)
+                    }
+                    if (proxy.request_options) {
+                        Object.assign(request_options, proxy.request_options);
+                    }
+                    if (proxy.disable_http2) {
+                        request_options.disable_http2 = true;
+                    }
+                    if (proxy.disable_language) {
+                        disable_language = true;
+                    }
+                    if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) {
+                        options.maxRedirects = proxy.maxredirects;
+                    }
+                    if (proxy && proxy.timeout > 0) {
+                        request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000;
+                    }
+                    if (proxy.sig) {
+                        request_sig = true;
+                    }
+                }
+            }
 
-        var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options);
-        if (proxy) {
-            if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) {
-                setPrerender(proxy.prerender);
-
-            } else if (proxy.proxy && CONFIG.PROXY_URL) {
-                request_options.uri = /{url}/.test(CONFIG.PROXY_URL) 
-                    ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri))
-                    : CONFIG.PROXY_URL + encodeURIComponent(uri);
-            } else if (proxy.proxy_url && /{url}/.test(proxy.proxy_url)) {
-                request_options.uri = proxy.proxy_url.replace(/{url}/, encodeURIComponent(uri));
+            if (options?.user_agent) {
+                // Let `options.user_agent` be prioritized over `proxy.user_agent`.
+                request_options.headers['User-Agent'] = options?.user_agent;
+            } else if (!request_options.headers['User-Agent']) {
+                // ...otherwise use default user agent.
+                request_options.headers['User-Agent'] = CONFIG.USER_AGENT;
             }
-            if (proxy.user_agent) {
-                request_options.headers = request_options.headers || {};
-                request_options.headers['User-Agent'] = proxy.user_agent;
+
+            // Add user agent extension to default agent if avaliable.
+            if (options?.getProviderOptions 
+                && (options.getProviderOptions('app.name') || options.getProviderOptions('app.ua_extension'))
+                && request_options.headers
+                && request_options.headers['User-Agent'] === CONFIG.USER_AGENT) {
+
+                var ext = options.getProviderOptions('app.ua_extension', options.getProviderOptions('app.name'));
+                if (/^[a-zA-Z0-9\.\s_-]+$/.test(ext) && ext.length > 1) {
+                    ext =  ext[0].toUpperCase() + ext.slice(1)
+                    request_options.headers['User-Agent'] += ' ' + ext;
+                }
             }
-            if (proxy.headers) {
+
+            var lang = options?.getProviderOptions && options.getProviderOptions('locale') || '';
+            if (!disable_language) {
                 request_options.headers = request_options.headers || {};
-                Object.assign(request_options.headers, proxy.headers)
-            }
-            if (proxy.request_options) {
-                Object.assign(request_options, proxy.request_options);
+                request_options.headers['Accept-Language'] = 
+                    lang /* !== '' */
+                        ? lang.replace('_', '-') 
+                            + (/\-/.test(lang) ? `,${lang.replace(/\-.*$/, '')};q=0.9` : '')
+                            + ',en;q=0.7,*;q=0.5'
+                        : 'en,*';
             }
-            if (proxy.disable_http2) {
-                request_options.disable_http2 = true;
-            }
-            if (proxy.disable_language) {
-                disable_language = true;
-            }
-            if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) {
-                options.maxRedirects = proxy.maxredirects;
-            }
-            if (proxy && proxy.timeout > 0) {
-                request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000;
-            }
-        }
-    }
-
-    if (options && options.getProviderOptions 
-        && (options.getProviderOptions('app.name') || options.getProviderOptions('app.ua_extension'))
-        && request_options.headers
-        && request_options.headers['User-Agent'] === CONFIG.USER_AGENT) {
-
-        var ext = options.getProviderOptions('app.ua_extension', options.getProviderOptions('app.name'));
-        if ( /^[a-zA-Z0-9\.\s_-]+$/.test(ext) && ext.length > 1) {
-            ext =  ext[0].toUpperCase() + ext.slice(1)
-            request_options.headers['User-Agent'] += ' ' + ext;
-        }        
-    } 
-
-    var lang = options?.getProviderOptions && options.getProviderOptions('locale') || '';
-    if (!disable_language) {
-        request_options.headers = request_options.headers || {};
-        request_options.headers['Accept-Language'] = 
-            lang /* !== '' */
-                ? lang.replace('_', '-') 
-                    + (/\-/.test(lang) ? `,${lang.replace(/\-.*$/, '')};q=0.9` : '')
-                    + ',en;q=0.7,*;q=0.5'
-                : 'en,*';
-    }
 
-    prepareEncodedUri(request_options, 'uri');
+            prepareEncodedUri(request_options, 'uri');
 
-    setCookieFromJar(uri, request_options.headers, options?.jar)
+            setCookieFromJar(uri, request_options.headers, options?.jar)
 
-    return request_options;
+            if ((options?.sig || request_sig) && options?.getSigHeaders) {
+                try {
+                    options.getSigHeaders(original_uri, function(error, headers) {
+                        if (error) {
+                            // Skip error.
+                            console.error('getSigHeaders error', error);
+                            resolve(request_options);
+                        } else {
+                            request_options.headers = request_options.headers || {};
+                            Object.assign(request_options.headers, headers);
+                            console.log('--- debug used headers', request_options.headers);
+                            resolve(request_options);
+                        }
+                    });
+                } catch(ex) {
+                    console.error('options.getSigHeaders exception', ex);
+                    resolve(request_options);    
+                }
+            } else {
+                resolve(request_options);
+            }
+
+        } catch(ex) {
+            reject(ex);
+        }
+    });
 };
 
 /**
@@ -207,21 +248,23 @@ export function prepareRequestOptions(request_options, options) {
         follow = 0;
     }
 
-    var request_options = prepareRequestOptions({
-        // Reviewed.
-        uri: url,
-        method: 'GET',
-        headers: {
-            'User-Agent': options.user_agent || CONFIG.USER_AGENT,
-            'Accept': '*/*'
-        },
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        redirect: redirect,
-        follow: follow
-    }, options);
-
     try {
-        fetchStreamKeepAlive(request_options)
+        var request_options;
+        prepareRequestOptions({
+            // Reviewed.
+            uri: url,
+            method: 'GET',
+            headers: {
+                'Accept': '*/*'
+            },
+            timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+            redirect: redirect,
+            follow: follow
+        }, options)
+            .then(_request_options => {
+                request_options = _request_options;
+                return fetchStreamKeepAlive(request_options)
+            })
             .then(stream => {
 
                 // Custom inner redirect logic with cookies.
@@ -281,23 +324,23 @@ var getHead = function(url, options, callbacks) {
         follow = 0;
     }
 
-    var request_options = prepareRequestOptions({
-        // jar: jar,
-
-        // Reviewed.
-        uri: url,
-        method: 'HEAD',
-        headers: {
-            'User-Agent': CONFIG.USER_AGENT
-        },
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        redirect: redirect,
-        follow: follow
-        // No abort controller for head.
-    }, options);
-
     try {
-        fetchStreamAuthorized(request_options)
+        var request_options;
+        prepareRequestOptions({
+            // jar: jar,
+
+            // Reviewed.
+            uri: url,
+            method: 'HEAD',
+            timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+            redirect: redirect,
+            follow: follow
+            // No abort controller for head.
+        }, options)
+            .then(_request_options => {
+                request_options = _request_options;
+                return fetchStreamAuthorized(request_options);
+            })
             .then(response => {
                 callbacks.onResponse && callbacks.onResponse(response, request_options);
             })
@@ -1237,19 +1280,15 @@ export function generateLinksHtml(data, options) {
 
 var getUriStatusPrivate = function(uri, options, cb) {
 
-    var request_options = prepareRequestOptions({
-        // Reviewed.
-        uri: uri,
-        method: 'GET',
-        headers: {
-            'User-Agent': CONFIG.USER_AGENT
-        },
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        follow: CONFIG.MAX_REDIRECTS
-    });
-
     try {
-        fetchStream(request_options)
+        prepareRequestOptions({
+            // Reviewed.
+            uri: uri,
+            method: 'GET',
+            timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+            follow: CONFIG.MAX_REDIRECTS
+        })
+            .then(request_options => fetchStream(request_options))
             .then(res => {
                 // TODO: may cause AbortError before cb.
                 res.abortController.abort();

modules/api/utils.js

@@ -1,3 +1,5 @@
+import { fetchData } from "../../lib/fetch.js";
+
 var _RE = /^_.+/;
 
 export function getProviderOptionsQuery(query) {
@@ -54,4 +56,24 @@ export function getProviderOptionsFromQuery(query) {
     }
 
     return providerOptions;
-}
+}
+
+function getSigHeaders(url, cb) {
+    const sigUrl = new URL(CONFIG.SIG_API);
+    sigUrl.searchParams.append('url', url);
+    fetchData({
+        uri: sigUrl,
+        json: true
+    })
+    .then(result => {
+        console.log('-- got sig headers ', result.data)
+        cb(null, result.data);
+    })
+    .catch(cb);;
+}
+
+export function getSigHeadersFunction() {
+    if (CONFIG.SIG_API) {
+        return getSigHeaders;
+    }
+}

modules/api/views.js

@@ -144,7 +144,9 @@ export default function(app) {
                     maxWidth: getIntParam(req, 'maxwidth') || getIntParam(req, 'max-width'),
                     promoUri: req.query.promoUri,
                     refresh: getBooleanParam(req, 'refresh'),
-                    providerOptions: getProviderOptionsFromQuery(req.query)
+                    providerOptions: getProviderOptionsFromQuery(req.query),
+                    getSigHeaders: apiUtils.getSigHeadersFunction(),
+                    sig: !!CONFIG.SIG_API
                 }, cb);
             }