Overwrite default requests user-agent (#7)

* Overwrite default requests user-agent

Author: daolf

scrapingbee/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.1.5'

scrapingbee/client.py

@@ -1,5 +1,6 @@
 from requests import request, Response
 
+from .default_headers import default_headers
 from .utils import get_scrapingbee_url, process_headers
 
 
@@ -19,6 +20,9 @@ def request(self, method: str, url: str, params: dict = None, data: dict = None,
         if headers:
             headers = process_headers(headers)
             params['forward_headers'] = True
+        else:
+            headers = {}
+        headers.update(default_headers)
 
         # Add cookies to params
         if cookies:

scrapingbee/default_headers.py

@@ -0,0 +1,4 @@
+from .__version__ import __version__
+
+# Add User-Agent identifying sdk
+default_headers = {"User-Agent": f"ScrapingBee-Python/{__version__}"}

setup.py

@@ -1,8 +1,15 @@
+import os
+
 from setuptools import setup
 
+about = {}
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'scrapingbee', '__version__.py'), 'r', encoding='utf-8') as f:
+    exec(f.read(), about)
+
 setup(
     name='scrapingbee',
-    version='1.1.4',
+    version=about['__version__'],
     url='https://github.com/scrapingbee/scrapingbee-python',
     description='ScrapingBee Python SDK',
     long_description=open('README.md').read(),

tests/test_client.py

@@ -3,6 +3,7 @@
 import pytest
 
 from scrapingbee import ScrapingBeeClient
+from scrapingbee.default_headers import default_headers
 
 
 @pytest.fixture(scope='module')
@@ -20,7 +21,7 @@ def test_get(mock_request, client):
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org',
         data=None,
-        headers=None
+        headers=default_headers
     )
 
 
@@ -34,7 +35,7 @@ def test_get_with_params(mock_request, client):
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&render_js=True',
         data=None,
-        headers=None,
+        headers=default_headers,
     )
 
 
@@ -48,7 +49,8 @@ def test_get_with_headers(mock_request, client):
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&forward_headers=True',
         data=None,
-        headers={'Spb-Content-Type': 'text/html; charset=utf-8'},
+        headers={'Spb-Content-Type': 'text/html; charset=utf-8',
+                 **default_headers},
     )
 
 
@@ -65,7 +67,7 @@ def test_get_with_cookies(mock_request, client):
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&cookies=name_1=value_1;name_2=value_2',
         data=None,
-        headers=None,
+        headers=default_headers,
     )
 
 
@@ -86,7 +88,7 @@ def test_get_with_extract_rules(mock_request, client):
         'extract_rules=%7B%22title%22%3A%20%22h1%22%2C%20%22'
         'subtitle%22%3A%20%22%23subtitle%22%7D',
         data=None,
-        headers=None,
+        headers=default_headers,
     )
 
 
@@ -99,5 +101,5 @@ def test_post(mock_request, client):
         'POST',
         'https://app.scrapingbee.com/api/v1/?api_key=API_KEY&url=https%3A//httpbin.org',
         data={'KEY_1': 'VALUE_1'},
-        headers=None
+        headers=default_headers
     )