diff --git a/requirements.txt b/requirements.txt index bd6f234..e69de29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +0,0 @@ -requests==2.32.4 diff --git a/tom/changelog.py b/tom/changelog.py index 1974e6b..120d367 100644 --- a/tom/changelog.py +++ b/tom/changelog.py @@ -1,6 +1,5 @@ import re import os -import requests import subprocess import datetime import hashlib @@ -125,7 +124,7 @@ def generate_changelog_in_repo(self, branch, repo): return False # find old and new versions - (header, old_version, old_changelog) = self.split_changelog_into_parts( + header, old_version, old_changelog = self.split_changelog_into_parts( changelog_filename, repo ) if header != "": diff --git a/tom/dependencies.py b/tom/dependencies.py index 370b74a..2e8c0ee 100644 --- a/tom/dependencies.py +++ b/tom/dependencies.py @@ -1,6 +1,5 @@ import re import json -import requests import collections import datetime import hashlib @@ -134,19 +133,23 @@ def checkfile(self, url, sha256=False): True, False, or sha256 of a linked file """ log.debug("checking URL: " + url) + + is_head = not sha256 and url.startswith("http") + method = "HEAD" if is_head else "GET" try: - if not sha256 and url.startswith("http"): - log.debug("testing with HEAD") - r = requests.head(url) - return r.status_code >= 200 and r.status_code < 300 - else: + req = urllib.request.Request(url, method=method) + with urllib.request.urlopen(req) as f: + if is_head: + log.debug("testing with HEAD") + return True + log.debug("getting whole file") m = hashlib.sha256() - with urllib.request.urlopen(url) as f: + + data = f.read(4096) + while data: + m.update(data) data = f.read(4096) - while data: - m.update(data) - data = f.read(4096) return m.hexdigest() except: return False @@ -222,7 +225,8 @@ def get_version_from_monitoring(self, dep): id = self.monitoring_ids[dep] url = "https://release-monitoring.org/api/v2/versions/?project_id={}".format(id) try: - data = requests.get(url).json() + with urllib.request.urlopen(url) as r: + data = json.loads(r) except: raise ReleaseMonitoringException( "Failed to do a request to release-monitoring.org website" @@ -255,7 +259,7 @@ def get_current_version(self, dep): dist_file = self.buildscripts.get_file(dist_file_path) dist_file = dist_file.strip() old_filename = re.sub(".* ", "", dist_file) - (old_version, separator) = self.extract_version_from_filename(dep, old_filename) + old_version, separator = self.extract_version_from_filename(dep, old_filename) return old_version def patch_spec_file(self, spec_file_path, old_version, new_version): @@ -283,7 +287,7 @@ def update_single_dep(self, dep): source_file = source_file.strip() old_filename = re.sub(".* ", "", dist_file) old_url = "{}{}".format(source_file, old_filename) - (old_version, separator) = self.extract_version_from_filename(dep, old_filename) + old_version, separator = self.extract_version_from_filename(dep, old_filename) new_version = self.get_version_from_monitoring(dep) if not new_version: log.warning( diff --git a/tom/github.py b/tom/github.py index b73f3c1..0463aaf 100644 --- a/tom/github.py +++ b/tom/github.py @@ -1,12 +1,12 @@ import random -import requests import re import os import sys import json import datetime +import urllib.error +import urllib.request import logging as log -import requests from copy import copy from tom.utils import pretty, write_json @@ -36,15 +36,16 @@ def get(self, path): if path in self.get_cache: log.debug("Found in cache") return self.get_cache[path] - r = requests.get(path, headers=self.headers) - log.debug("RESPONSE {}".format(r.status_code)) - if not (200 <= r.status_code < 300): - sys.exit("Non-success API response {} for '{}'".format(r.status_code, path)) + req = urllib.request.Request(path, headers=self.headers) + try: + with urllib.request.urlopen(req) as r: + data = json.load(r) + log.debug("RESPONSE {}".format(r.code)) + except urllib.error.HTTPError as e: + sys.exit("Non-success API response {} for '{}'".format(e.code, path)) - assert r.status_code >= 200 and r.status_code < 300 - data = r.json() - log.debug(pretty(data)) + log.debug(json.dumps(data)) self.get_cache[path] = data if ( not isinstance(data, list) @@ -74,12 +75,19 @@ def post(self, path, data, check_status_code=True): return None self.api_log("POST {} {}".format(path, data)) path = self.path(path) - r = requests.post(path, headers=self.headers, json=data) - log.debug("RESPONSE {}".format(r.status_code)) - if check_status_code: - assert r.status_code >= 200 and r.status_code < 300, r.text - data = r.json() - log.debug(pretty(data)) + + json_data = json.dumps(data).encode("utf-8") + req = urllib.request.Request(path, headers=self.headers, data=json_data) + try: + with urllib.request.urlopen(req) as r: + log.debug("RESPONSE {}".format(r.status_code)) + data = json.load(r) + log.debug(r.read().decode()) + except urllib.error.HTTPError as e: + log.error(e.headers) + log.error(e.read().decode()) + raise AssertionError("HTTP response {} from GitHub".format(e.code)) + return data def create_pr( @@ -388,7 +396,7 @@ def create_prs_from_slack(self): last_branch = self.find_last_branch_in_repo(username, repo) log.info("last branch: " + last_branch) # now, try to find a parent for it. - (parent_repo, parent_branch) = self.find_parent(username, repo, last_branch) + parent_repo, parent_branch = self.find_parent(username, repo, last_branch) log.info("parent branch: " + parent_branch) pr_text = self.github.create_pr( parent_repo, parent_branch, username, last_branch, last_branch + " PR", "" diff --git a/tom/jenkins.py b/tom/jenkins.py index 6a93325..640bd81 100644 --- a/tom/jenkins.py +++ b/tom/jenkins.py @@ -1,11 +1,14 @@ import logging as log from time import sleep -import requests -from requests.auth import HTTPBasicAuth from tom.utils import pretty import os +import base64 +import json from typing import Dict +import urllib.request +import urllib.error + class Jenkins: def __init__(self, url, job, secrets, username): @@ -20,7 +23,7 @@ def __init__(self, url, job, secrets, username): self.crumb = crumb self.username = username - self.auth = HTTPBasicAuth(user, token) + self.auth = base64.b64encode(("%s:%s" % (user, token)).encode()).decode() self.headers = {} if crumb: @@ -34,17 +37,20 @@ def post(self, path, data): if os.getenv("TOM") == "PASSIVE": print("Would post: " + path) return None - r = requests.post(path, data=data, headers=self.headers, auth=self.auth) - if not (200 <= r.status_code < 300): - log.error("Unexpected HTTP response from Jenkins: {}".format(r.status_code)) - log.error(str(r.headers)) - log.error(str(r.text)) - raise AssertionError("HTTP response {} from Jenkins".format(r.status_code)) - + json_data = json.dumps(data).encode("utf-8") + req = urllib.request.Request(path, data=json_data, headers=self.headers) + req.add_header("Authorization", "Basic %s" % self.auth) try: - return r.headers, r.json() - except: - return r.headers, r.text + with urllib.request.urlopen(req) as r: + try: + parsed = json.load(r) + return r.headers, parsed + except: + return r.headers, r.read().decode() + except urllib.error.HTTPError as e: + log.error(e.headers) + log.error(e.read().decode()) + raise AssertionError("HTTP response {} from Jenkins".format(e.code)) def trigger( self, @@ -144,9 +150,16 @@ def wait_for_queue(self, url): while "executable" not in queue_item: log.info("Waiting for jenkins build in queue") sleep(1) - r = requests.get(url + "api/json", headers=self.headers, auth=self.auth) - assert r.status_code >= 200 and r.status_code < 300 - queue_item = r.json() + req = urllib.request.Request("{}api/json".format(url), headers=self.headers) + req.add_header("Authorization", "Basic %s" % self.auth) + try: + with urllib.request.urlopen(req) as r: + queue_item = json.load(r) + except urllib.error.HTTPError as e: + log.error("Unexpected HTTP response from Jenkins: {}".format(e.code)) + log.error(e.headers) + log.error(e.read().decode()) + raise AssertionError("HTTP response {} from Jenkins".format(e.code)) log.debug(pretty(queue_item)) num = queue_item["executable"]["number"] diff --git a/tom/packages.py b/tom/packages.py index 52d8b06..2f26032 100644 --- a/tom/packages.py +++ b/tom/packages.py @@ -1,8 +1,9 @@ import re import json -import requests import collections import datetime +import urllib.request +import urllib.error from tom.git import GitRepo @@ -51,6 +52,18 @@ def is_branch_or_version(string): return None +def fetch_json(url): + try: + with urllib.request.urlopen(url) as r: + return json.load(r) + except urllib.error.HTTPError as e: + raise URLDownloadFailureException( + "failed to download %s, return code %d" % (url, e.code) + ) + except json.decoder.JSONDecodeError as e: + raise JSONParsingError("file %s is not a valid JSON" % url) from e + + class PackageMapper: """Class responsible for updating packages_mapping.json file Currently it's tailored for cfengine needs. @@ -125,18 +138,8 @@ def run(self, inputs): releases_url = ( "https://cfengine.com/release-data/%s/releases.json" % product ) - releases_request = requests.get(releases_url) - if not releases_request.ok: - raise URLDownloadFailureException( - "failed to download %s, return code %d" - % (releases_url, releases_request.status_code) - ) - try: - releases_data = releases_request.json() - except json.decoder.JSONDecodeError as e: - raise JSONParsingError( - "file %s is not a valid JSON" % releases_url - ) from e + releases_data = fetch_json(releases_url) + if "releases" not in releases_data: raise JSONStructureError('no "releases" in %s JSON' % releases_url) if value_type == "version": @@ -233,16 +236,7 @@ def run(self, inputs): def collect_packages(self, url): """Given a release URL, returns a dict where keys are platform names, and values are {'url': 'http...'}""" - release_request = requests.get(url) - if not release_request.ok: - raise URLDownloadFailureException( - "failed to download %s, return code %d" - % (url, release_request.status_code) - ) - try: - release_data = release_request.json() - except json.decoder.JSONDecodeError as e: - raise JSONParsingError("file %s is not a valid JSON" % url) from e + release_data = fetch_json(url) if "artifacts" not in release_data: raise JSONStructureError('no "artifacts" in %s JSON' % url) # release_data['artifacts'] is a dictionary (called 'table'), each element is a list diff --git a/tom/slack.py b/tom/slack.py index a0e0d74..c6181ec 100644 --- a/tom/slack.py +++ b/tom/slack.py @@ -3,8 +3,9 @@ import re import sys import json -import requests import logging as log +import urllib.request +import urllib.error from tom.utils import pretty @@ -36,13 +37,25 @@ def post(self, url, data={}): url = self.api(url) if not "token" in data: data["token"] = self.bot_token - r = requests.post(url, data) - assert r.status_code >= 200 and r.status_code < 300 + + json_data = json.dumps(data).encode("utf-8") + req = urllib.request.Request( + url, + data=json_data, + method="POST", + headers={"Content-Type": "application/json"}, + ) try: - log.debug(pretty(r.json())) - return r.json() - except: - log.debug(pretty(r.text)) + with urllib.request.urlopen(req) as r: + try: + parsed = json.load(r) + log.debug(r.read().decode()) + return parsed + except json.JSONDecodeError: + log.debug(r.read().decode()) + return False + + except urllib.error.HTTPError: return False def send_message(self, channel, text): diff --git a/tom/tag.py b/tom/tag.py index cb83a36..62e1c9b 100644 --- a/tom/tag.py +++ b/tom/tag.py @@ -1,6 +1,5 @@ import re import os -import requests import subprocess import datetime import hashlib diff --git a/tom/utils.py b/tom/utils.py index 625b055..0854095 100644 --- a/tom/utils.py +++ b/tom/utils.py @@ -1,6 +1,8 @@ import sys import json import hashlib +import urllib.request +import urllib.error def read_json(path):