diff --git a/README.md b/README.md index ecb2e45..a3773a5 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,49 @@ query string (query), optional prefixes (prefix) if you do not want to use the s the operators of the endpoint know who you are, especially if you execute many queries on the endpoint. This allows the operators of the endpoint to contact you (e.g. specify a email address or the URL to your bot code repository.) +For authenticated SPARQL endpoints, an optional `auth` parameter can be passed through to `requests.post()`. This means any +authentication object or value accepted by `requests` can be used here. + +Example using HTTP basic auth: + +```Python + from wikidataintegrator import wdi_core + + query = """ + SELECT * WHERE { + ?item ?p ?o . + } LIMIT 10 + """ + + results = wdi_core.WDItemEngine.execute_sparql_query( + query, + endpoint='https://example.org/sparql', + auth=('username', 'password') + ) +``` + +Example using a custom requests auth object: + +```Python + from requests_oauthlib import OAuth1 + from wikidataintegrator import wdi_core + + auth = OAuth1( + client_key='consumer-key', + client_secret='consumer-secret', + resource_owner_key='access-token', + resource_owner_secret='access-secret' + ) + + results = wdi_core.WDItemEngine.execute_sparql_query( + 'SELECT * WHERE { ?item ?p ?o . } LIMIT 10', + endpoint='https://example.org/sparql', + auth=auth + ) +``` + +If `auth` is not provided, the existing unauthenticated behavior is unchanged. + ## Logging ## The method wdi_core.WDItemEngine.log() allows for using the Python built in logging functionality to collect errors and other logs. It takes two parameters, the log level (level) and the log message (message). It is advisable to separate log file columns by colons diff --git a/wikidataintegrator/sdc_core.py b/wikidataintegrator/sdc_core.py index 51419fc..fa02031 100755 --- a/wikidataintegrator/sdc_core.py +++ b/wikidataintegrator/sdc_core.py @@ -63,7 +63,7 @@ def get_rdf(wd_item_id='', format="turtle", mediawiki_api_url=None): @staticmethod @wdi_backoff() - def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60): + def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None): """ Static method which can be used to execute any SPARQL query @@ -76,6 +76,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_ :type user_agent: str :param max_retries: The number time this function should retry in case of header reports. :param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable. + :param auth: Optional requests-compatible authentication for protected SPARQL endpoints. :return: The results of the query are returned in JSON format """ @@ -98,7 +99,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_ for n in range(max_retries): try: - response = requests.post(sparql_endpoint_url, params=params, headers=headers) + response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth) except requests.exceptions.ConnectionError as e: print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after)) time.sleep(retry_after) @@ -1528,7 +1529,7 @@ def generate_item_instances(cls, items, mediawiki_api_url=None, login=None, @staticmethod @wdi_backoff() def execute_sparql_query(query, prefix=None, endpoint=None, - user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60): + user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None): """ Static method which can be used to execute any SPARQL query @@ -1540,6 +1541,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, :type user_agent: str :param max_retries: The number time this function should retry in case of header reports. :param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable. + :param auth: Optional requests-compatible authentication for protected SPARQL endpoints. :return: The results of the query are returned in JSON format """ @@ -1562,7 +1564,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, for n in range(max_retries): try: - response = requests.post(sparql_endpoint_url, params=params, headers=headers) + response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth) except requests.exceptions.ConnectionError as e: print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after)) time.sleep(retry_after) diff --git a/wikidataintegrator/tests/tests.py b/wikidataintegrator/tests/tests.py index 39dba2e..93ab855 100644 --- a/wikidataintegrator/tests/tests.py +++ b/wikidataintegrator/tests/tests.py @@ -1,10 +1,11 @@ import copy import pprint import unittest +from unittest.mock import patch import requests -from wikidataintegrator import wdi_core, wdi_fastrun +from wikidataintegrator import sdc_core, wdi_core, wdi_fastrun from wikidataintegrator.wdi_core import WDApiError __author__ = 'Sebastian Burgstaller-Muehlbacher' @@ -24,6 +25,30 @@ def test_all(self): params={'format': 'json', 'action': 'wbgetentities', 'ids': 'Q42'}) +class TestSparqlAuth(unittest.TestCase): + @patch('wikidataintegrator.wdi_core.requests.post') + def test_wdi_core_execute_sparql_query_passes_auth(self, mock_post): + mock_post.return_value.status_code = 200 + mock_post.return_value.headers = {} + mock_post.return_value.json.return_value = {'results': {'bindings': []}} + + auth = ('user', 'pass') + wdi_core.WDItemEngine.execute_sparql_query('SELECT * WHERE {}', auth=auth, max_retries=1) + + assert mock_post.call_args.kwargs['auth'] == auth + + @patch('wikidataintegrator.sdc_core.requests.post') + def test_sdc_core_execute_sparql_query_passes_auth(self, mock_post): + mock_post.return_value.status_code = 200 + mock_post.return_value.headers = {} + mock_post.return_value.json.return_value = {'results': {'bindings': []}} + + auth = ('user', 'pass') + sdc_core.WDItemEngine.execute_sparql_query('SELECT * WHERE {}', auth=auth, max_retries=1) + + assert mock_post.call_args.kwargs['auth'] == auth + + class TestDataType(unittest.TestCase): def test_wd_quantity(self): dt = wdi_core.WDQuantity(value='34', prop_nr='P43') diff --git a/wikidataintegrator/wdi_core.py b/wikidataintegrator/wdi_core.py index 0ee2ab8..f667d3e 100755 --- a/wikidataintegrator/wdi_core.py +++ b/wikidataintegrator/wdi_core.py @@ -82,7 +82,7 @@ def get_linked_by(qid, mediawiki_api_url=None): @staticmethod @wdi_backoff() - def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60): + def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None): """ Static method which can be used to execute any SPARQL query @@ -95,6 +95,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_ :type user_agent: str :param max_retries: The number time this function should retry in case of header reports. :param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable. + :param auth: Optional requests-compatible authentication for protected SPARQL endpoints. :return: The results of the query are returned in JSON format """ @@ -117,7 +118,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_ for n in range(max_retries): try: - response = requests.post(sparql_endpoint_url, params=params, headers=headers) + response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth) except requests.exceptions.ConnectionError as e: print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after)) time.sleep(retry_after) @@ -1550,7 +1551,7 @@ def generate_item_instances(cls, items, mediawiki_api_url=None, login=None, @staticmethod @wdi_backoff() def execute_sparql_query(query, prefix=None, endpoint=None, - user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60): + user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None): """ Static method which can be used to execute any SPARQL query @@ -1562,6 +1563,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, :type user_agent: str :param max_retries: The number time this function should retry in case of header reports. :param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable. + :param auth: Optional requests-compatible authentication for protected SPARQL endpoints. :return: The results of the query are returned in JSON format """ @@ -1584,7 +1586,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, for n in range(max_retries): try: - response = requests.post(sparql_endpoint_url, params=params, headers=headers) + response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth) except requests.exceptions.ConnectionError as e: print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after)) time.sleep(retry_after)