Skip to content

feat(auth): Allow to use protected SPARQL endpoints using requests au… #209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kwisatz wants to merge 1 commit into
base: main
Choose a base branch
from
+81 −9
Open

feat(auth): Allow to use protected SPARQL endpoints using requests au… #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,49 @@ query string (query), optional prefixes (prefix) if you do not want to use the s
the operators of the endpoint know who you are, especially if you execute many queries on the endpoint. This allows the operators of
the endpoint to contact you (e.g. specify a email address or the URL to your bot code repository.)

For authenticated SPARQL endpoints, an optional `auth` parameter can be passed through to `requests.post()`. This means any
authentication object or value accepted by `requests` can be used here.

Example using HTTP basic auth:

```Python
from wikidataintegrator import wdi_core

query = """
SELECT * WHERE {
?item ?p ?o .
} LIMIT 10
"""

results = wdi_core.WDItemEngine.execute_sparql_query(
query,
endpoint='https://example.org/sparql',
auth=('username', 'password')
)
```

Example using a custom requests auth object:

```Python
from requests_oauthlib import OAuth1
from wikidataintegrator import wdi_core

auth = OAuth1(
client_key='consumer-key',
client_secret='consumer-secret',
resource_owner_key='access-token',
resource_owner_secret='access-secret'
)

results = wdi_core.WDItemEngine.execute_sparql_query(
'SELECT * WHERE { ?item ?p ?o . } LIMIT 10',
endpoint='https://example.org/sparql',
auth=auth
)
```

If `auth` is not provided, the existing unauthenticated behavior is unchanged.

## Logging ##
The method wdi_core.WDItemEngine.log() allows for using the Python built in logging functionality to collect errors and other logs.
It takes two parameters, the log level (level) and the log message (message). It is advisable to separate log file columns by colons
Expand Down
10 changes: 6 additions & 4 deletions wikidataintegrator/sdc_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def get_rdf(wd_item_id='', format="turtle", mediawiki_api_url=None):

@staticmethod
@wdi_backoff()
def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60):
def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None):

"""
Static method which can be used to execute any SPARQL query
Expand All @@ -76,6 +76,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_
:type user_agent: str
:param max_retries: The number time this function should retry in case of header reports.
:param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable.
:param auth: Optional requests-compatible authentication for protected SPARQL endpoints.
:return: The results of the query are returned in JSON format
"""

Expand All @@ -98,7 +99,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_

for n in range(max_retries):
try:
response = requests.post(sparql_endpoint_url, params=params, headers=headers)
response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth)
except requests.exceptions.ConnectionError as e:
print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after))
time.sleep(retry_after)
Expand Down Expand Up @@ -1528,7 +1529,7 @@ def generate_item_instances(cls, items, mediawiki_api_url=None, login=None,
@staticmethod
@wdi_backoff()
def execute_sparql_query(query, prefix=None, endpoint=None,
user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60):
user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None):
"""
Static method which can be used to execute any SPARQL query

Expand All @@ -1540,6 +1541,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None,
:type user_agent: str
:param max_retries: The number time this function should retry in case of header reports.
:param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable.
:param auth: Optional requests-compatible authentication for protected SPARQL endpoints.
:return: The results of the query are returned in JSON format
"""

Expand All @@ -1562,7 +1564,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None,

for n in range(max_retries):
try:
response = requests.post(sparql_endpoint_url, params=params, headers=headers)
response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth)
except requests.exceptions.ConnectionError as e:
print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after))
time.sleep(retry_after)
Expand Down
27 changes: 26 additions & 1 deletion wikidataintegrator/tests/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import copy
import pprint
import unittest
from unittest.mock import patch

import requests

from wikidataintegrator import wdi_core, wdi_fastrun
from wikidataintegrator import sdc_core, wdi_core, wdi_fastrun
from wikidataintegrator.wdi_core import WDApiError

__author__ = 'Sebastian Burgstaller-Muehlbacher'
Expand All @@ -24,6 +25,30 @@ def test_all(self):
params={'format': 'json', 'action': 'wbgetentities', 'ids': 'Q42'})


class TestSparqlAuth(unittest.TestCase):
@patch('wikidataintegrator.wdi_core.requests.post')
def test_wdi_core_execute_sparql_query_passes_auth(self, mock_post):
mock_post.return_value.status_code = 200
mock_post.return_value.headers = {}
mock_post.return_value.json.return_value = {'results': {'bindings': []}}

auth = ('user', 'pass')
wdi_core.WDItemEngine.execute_sparql_query('SELECT * WHERE {}', auth=auth, max_retries=1)

assert mock_post.call_args.kwargs['auth'] == auth

@patch('wikidataintegrator.sdc_core.requests.post')
def test_sdc_core_execute_sparql_query_passes_auth(self, mock_post):
mock_post.return_value.status_code = 200
mock_post.return_value.headers = {}
mock_post.return_value.json.return_value = {'results': {'bindings': []}}

auth = ('user', 'pass')
sdc_core.WDItemEngine.execute_sparql_query('SELECT * WHERE {}', auth=auth, max_retries=1)

assert mock_post.call_args.kwargs['auth'] == auth


class TestDataType(unittest.TestCase):
def test_wd_quantity(self):
dt = wdi_core.WDQuantity(value='34', prop_nr='P43')
Expand Down
10 changes: 6 additions & 4 deletions wikidataintegrator/wdi_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def get_linked_by(qid, mediawiki_api_url=None):

@staticmethod
@wdi_backoff()
def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60):
def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None):

"""
Static method which can be used to execute any SPARQL query
Expand All @@ -95,6 +95,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_
:type user_agent: str
:param max_retries: The number time this function should retry in case of header reports.
:param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable.
:param auth: Optional requests-compatible authentication for protected SPARQL endpoints.
:return: The results of the query are returned in JSON format
"""

Expand All @@ -117,7 +118,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, as_

for n in range(max_retries):
try:
response = requests.post(sparql_endpoint_url, params=params, headers=headers)
response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth)
except requests.exceptions.ConnectionError as e:
print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after))
time.sleep(retry_after)
Expand Down Expand Up @@ -1550,7 +1551,7 @@ def generate_item_instances(cls, items, mediawiki_api_url=None, login=None,
@staticmethod
@wdi_backoff()
def execute_sparql_query(query, prefix=None, endpoint=None,
user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60):
user_agent=None, as_dataframe=False, max_retries=1000, retry_after=60, auth=None):
"""
Static method which can be used to execute any SPARQL query

Expand All @@ -1562,6 +1563,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None,
:type user_agent: str
:param max_retries: The number time this function should retry in case of header reports.
:param retry_after: the number of seconds should wait upon receiving either an error code or the WDQS is not reachable.
:param auth: Optional requests-compatible authentication for protected SPARQL endpoints.
:return: The results of the query are returned in JSON format
"""

Expand All @@ -1584,7 +1586,7 @@ def execute_sparql_query(query, prefix=None, endpoint=None,

for n in range(max_retries):
try:
response = requests.post(sparql_endpoint_url, params=params, headers=headers)
response = requests.post(sparql_endpoint_url, params=params, headers=headers, auth=auth)
except requests.exceptions.ConnectionError as e:
print("Connection error: {}. Sleeping for {} seconds.".format(e, retry_after))
time.sleep(retry_after)
Expand Down