Migration tool

.gitignore

@@ -1,6 +1,7 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
+.pytest_cache/
 
 # C extensions
 *.so
@@ -55,3 +56,6 @@ docs/_build/
 
 # PyBuilder
 target/
+
+# PyCharm
+.idea/

teamscale_client/client.py

@@ -22,8 +22,10 @@ class TeamscaleClient:
         username (str): The username to use for authentication
         access_token (str): The IDE access token to use for authentication
         project (str): The id of the project on which to work
-        sslverify: See requests' verify parameter in http://docs.python-requests.org/en/latest/user/advanced/#ssl-cert-verification
-        timeout (float): TTFB timeout in seconds, see http://docs.python-requests.org/en/master/user/quickstart/#timeouts
+        sslverify: See requests' verify parameter in
+            http://docs.python-requests.org/en/latest/user/advanced/#ssl-cert-verification
+        timeout (float): TTFB timeout in seconds,
+            see http://docs.python-requests.org/en/master/user/quickstart/#timeouts
         branch: The branch name for which to upload/retrieve data
     """
 
@@ -45,16 +47,16 @@ def check_api_version(self):
         """
         url = self.get_global_service_url('service-api-info')
         response = self.get(url)
-        apiVersion = response.json()['apiVersion']
-        if apiVersion < 3:
+        api_version = response.json()['apiVersion']
+        if api_version < 3:
             raise ServiceError("Server api version " + str(
-                apiVersion) + " too low and not compatible. This client requires Teamscale 3.2 or newer.");
+                api_version) + " too low and not compatible. This client requires Teamscale 3.2 or newer.")
 
     def get(self, url, parameters=None):
         """Sends a GET request to the given service url.
 
         Args:
-            url (str):  The URL for which to execute a PUT request
+            url (str):  The URL for which to execute a GET request
             parameters (dict): parameters to attach to the url
 
         Returns:
@@ -67,7 +69,7 @@ def get(self, url, parameters=None):
         response = requests.get(url, params=parameters, auth=self.auth_header, verify=self.sslverify, headers=headers,
                                 timeout=self.timeout)
         if response.status_code != 200:
-            raise ServiceError("ERROR: GET {url}: {r.status_code}:{r.text}".format(url=url, r=response))
+            raise ServiceError("GET", url, response)
         return response
 
     def put(self, url, json=None, parameters=None, data=None):
@@ -85,12 +87,12 @@ def put(self, url, json=None, parameters=None, data=None):
         Raises:
             ServiceError: If anything goes wrong
         """
-        headers = {'Accept': 'application/json','Content-Type': 'application/json'}
+        headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
         response = requests.put(url, params=parameters, json=json, data=data,
                                 headers=headers, auth=self.auth_header,
                                 verify=self.sslverify, timeout=self.timeout)
         if response.status_code != 200:
-            raise ServiceError("ERROR: PUT {url}: {r.status_code}:{r.text}".format(url=url, r=response))
+            raise ServiceError("PUT", url, response)
         return response
 
     def delete(self, url, parameters=None):
@@ -109,7 +111,7 @@ def delete(self, url, parameters=None):
         response = requests.delete(url, params=parameters, auth=self.auth_header, verify=self.sslverify,
                                    timeout=self.timeout)
         if response.status_code != 200:
-            raise ServiceError("ERROR: PUT {url}: {r.status_code}:{r.text}".format(url=url, r=response))
+            raise ServiceError("PUT", url, response)
         return response
 
     def add_findings_group(self, name, mapping_pattern):
@@ -218,7 +220,6 @@ def add_metric_descriptions(self, metric_descriptions):
         service_url = self.get_global_service_url("external-metric")
         return self.put(service_url, data=to_json(metric_descriptions))
 
-
     def upload_coverage_data(self, coverage_files, coverage_format, timestamp, message, partition):
         """Upload coverage reports to Teamscale. It is expected that the given coverage report files can be read from the filesystem.
 
@@ -266,14 +267,14 @@ def upload_report(self, report_files, report_format, timestamp, message, partiti
         response = requests.post(service_url, params=parameters, auth=self.auth_header, verify=self.sslverify,
                                  files=multiple_files, timeout=self.timeout)
         if response.status_code != 200:
-            raise ServiceError("ERROR: POST {url}: {r.status_code}:{r.text}".format(url=service_url, r=response))
+            raise ServiceError("POST", service_url, response)
         return response
 
     def upload_architectures(self, architectures, timestamp, message):
         """Upload architectures to Teamscale. It is expected that the given architectures can be be read from the filesystem.
 
         Args:
-            architectures (dict): mappping of teamscale paths to architecture files that should be uploaded. Files must be readable.
+            architectures (dict): mapping of teamscale paths to architecture files that should be uploaded. Files must be readable.
             timestamp (datetime.datetime): timestamp for which to upload the data
             message (str): The message to use for the generated upload commit
 
@@ -292,7 +293,7 @@ def upload_architectures(self, architectures, timestamp, message):
         response = requests.post(service_url, params=parameters, auth=self.auth_header, verify=self.sslverify,
                                  files=architecture_files, timeout=self.timeout)
         if response.status_code != 200:
-            raise ServiceError("ERROR: GET {url}: {r.status_code}:{r.text}".format(url=service_url, r=response))
+            raise ServiceError("GET", service_url, response)
         return response
 
     def upload_non_code_metrics(self, metrics, timestamp, message, partition):
@@ -329,7 +330,7 @@ def get_baselines(self):
         response = requests.get(service_url, params=parameters, auth=self.auth_header, verify=self.sslverify,
                                 headers=headers, timeout=self.timeout)
         if response.status_code != 200:
-            raise ServiceError("ERROR: GET {url}: {r.status_code}:{r.text}".format(url=service_url, r=response))
+            raise ServiceError("GET", service_url, response)
         return [Baseline(x['name'], x['description'], timestamp=x['timestamp']) for x in response.json()]
 
     def delete_baseline(self, baseline_name):
@@ -383,6 +384,11 @@ def get_projects(self):
                         creation_timestamp=x['creationTimestamp'], alias=x.get('alias'),
                         deleting=x['deleting'], reanalyzing=x['reanalyzing']) for x in response.json()]
 
+    def get_version(self):
+        """ Retrieves the teamscale version """
+        response_text = self.get(self.get_global_service_url("health-metrics"), {"metric": "version"}).text
+        return response_text.split()[1]
+
     def create_project(self, project_configuration):
         """Creates a project with the specified configuration in Teamscale.
 
@@ -394,7 +400,7 @@ def create_project(self, project_configuration):
         Raises:
             ServiceError: If anything goes wrong.
         """
-        return self._add_project(project_configuration, perfrom_update_call=False)
+        return self._add_project(project_configuration, perform_update_call=False)
 
     def update_project(self, project_configuration):
         """Updates an existing project in Teamscale with the given configuration. The id of the existing project is
@@ -408,20 +414,20 @@ def update_project(self, project_configuration):
         Raises:
             ServiceError: If anything goes wrong.
         """
-        return self._add_project(project_configuration, perfrom_update_call=True)
+        return self._add_project(project_configuration, perform_update_call=True)
 
-    def _add_project(self, project_configuration, perfrom_update_call):
-        """Adds a project to Teamscale. The parameter `perfrom_update_call` specifies, whether an update call should be
+    def _add_project(self, project_configuration, perform_update_call):
+        """Adds a project to Teamscale. The parameter `perform_update_call` specifies, whether an update call should be
         made:
-        - If `perfrom_update_call` is set to `True`, re-adding a project with an existing id will update the original
+        - If `perform_update_call` is set to `True`, re-adding a project with an existing id will update the original
         project.
-        - If `perfrom_update_call` is set to `False`, re-adding a project with an existing id will result in an error.
-        - Further, if `perfrom_update_call` is set to `True`, but no project with the specified id exists, an error is
+        - If `perform_update_call` is set to `False`, re-adding a project with an existing id will result in an error.
+        - Further, if `perform_update_call` is set to `True`, but no project with the specified id exists, an error is
         thrown as well.
 
         Args:
             project_configuration (data.ProjectConfiguration): The project that is to be created (or updated).
-            perfrom_update_call (bool): Whether to perform an update call.
+            perform_update_call (bool): Whether to perform an update call.
         Returns:
             requests.Response: object generated by the upload request.
 
@@ -430,15 +436,13 @@ def _add_project(self, project_configuration, perfrom_update_call):
         """
         service_url = self.get_global_service_url("create-project")
         parameters = {
-            "only-config-update": perfrom_update_call
+            "only-config-update": perform_update_call
         }
         response = self.put(service_url, parameters=parameters, data=to_json(project_configuration))
 
         response_message = TeamscaleClient._get_response_message(response)
         if response_message != 'success':
-            raise ServiceError(
-                "ERROR: GET {url}: {status_code}:{message}".format(url=service_url, status_code=response.status_code,
-                                                                   message=response_message))
+            raise ServiceError("GET", service_url, response)
         return response
 
     @staticmethod

teamscale_client/constants.py

@@ -113,6 +113,7 @@ class ReportFormats:
 
     FINDBUGS = "FINDBUGS"
 
+
 class UnitTestReportFormats:
     """Reports for unit test results that Teamscale understands."""
 

teamscale_client/data.py

@@ -5,8 +5,7 @@
 import datetime
 import time
 
-from teamscale_client.constants import Assessment, MetricAggregation, MetricValueType, MetricProperties, \
-    AssessmentMetricColors, ConnectorType
+from teamscale_client.constants import Assessment, MetricAggregation, MetricValueType, MetricProperties, ConnectorType
 from teamscale_client.utils import auto_str
 
 
@@ -17,7 +16,7 @@ class Finding(object):
     Args:
         finding_type_id (str): The type id that this finding belongs to.
         message (str): The main finding message
-        assesssment (constants.Assessment): The assessment this finding should have. Default is `YELLOW`.
+        assessment (constants.Assessment): The assessment this finding should have. Default is `YELLOW`.
                                             This value is only important if in Teamscale the finding enablement
                                             is set to auto, otherwise the setting from Teamscale will be used.
         start_offset (int): Offset from the beginning of the file, where the finding area starts
@@ -66,8 +65,10 @@ def __init__(self, findings, path, content=None):
 
     Args:
         typeid (str): The id used to reference the finding type.
-        description (str): The text to display that explains what this finding type is about (and ideally how to fix it). This text will be the same for each concrete instance of the finding.
-        enablement (constants.Enablement): Describes the default enablement setting for this finding type, used when it is added to the analysis profile.
+            description (str): The text to display that explains what this finding type is about 
+            (and ideally how to fix it) This text will be the same for each concrete instance of the finding.
+        enablement (constants.Enablement): Describes the default enablement setting for this finding type, used when it 
+            is added to the analysis profile.
 """
 
 
@@ -181,7 +182,14 @@ def _set_date(self, date_object):
 
 class ServiceError(Exception):
     """Teamscale service returned an error."""
-    pass
+
+    def __init__(self, method, url, response):
+        self.message = "ERROR: {0} {1}: {r.status_code}:{r.text}".format(method, url, r=response)
+        self.response = response
+
+    # This prevents the compressing of a response into a single line, making it unreadable (no repr()!)
+    def __str__(self):
+        return self.message
 
 
 @auto_str

tests/task_migrator_test.py

@@ -0,0 +1,62 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import responses
+from tools.migration.task_migrator import TaskMigrator
+from test_utils import get_global_service_mock
+from copy import deepcopy
+
+URL = "http://localhost:8080"
+CONFIG = {
+    "old_instance": {
+        "url": URL,
+        "project": "old",
+        "user": "admin",
+        "token": "token"
+    },
+    "new_instance": {
+        "url": URL,
+        "project": "new",
+        "user": "admin",
+        "token": "token"
+    }
+}
+
+
+class TestTaskMigrator:
+    """ Simple class for bundling the test for the task migration. """
+    @staticmethod
+    def get_migrator(config):
+        """
+        Returns a task migrator with the given config.
+        For an example config look at CONFIG
+        """
+        TestTaskMigrator.create_necessary_client_responses(URL)
+        return TaskMigrator(config, False)
+
+    @staticmethod
+    def create_necessary_client_responses(url, version=40000):
+        """ Creates responses which are necessary to create a client """
+        responses.add(responses.GET, get_global_service_mock(url, "service-api-info"),
+                      status=200, content_type="application/json", body='{ "apiVersion": 3 }')
+        responses.add(responses.GET, get_global_service_mock(url, "health-metrics"),
+                      status=200, content_type="text/plain", body="version %s 0" % version)
+
+    def get_default_migrator(self):
+        """ Returns the migrator with the default settings """
+        return self.get_migrator(CONFIG)
+
+    @responses.activate
+    def test_different_versions(self, caplog):
+        """ Tests the case where we want to migrate between two TS-instances with a different
+        version. A warning should be logged and the version_match flag should be False.
+        """
+        config = deepcopy(CONFIG)
+        new_url = "http://localhost:8081"
+        config["new_instance"]["url"] = new_url
+        self.create_necessary_client_responses(new_url, version=30000)
+        migrator = self.get_migrator(config)
+
+        warning = list(filter(lambda x: x.levelname == "WARNING" and "version" in x.message, caplog.records))
+        assert len(warning) == 1, "Missing warning about version mismatch"
+        assert not migrator.versions_match, "Flag 'versions_match' should be False"

tests/test_utils.py

@@ -0,0 +1,5 @@
+import re
+
+def get_global_service_mock(url, service_id):
+    """ Creates a url for a global service with the given url and service """
+    return re.compile(r'%s/%s/.*' % (url, service_id))

tools/migration/batch_config.template

@@ -0,0 +1,16 @@
+{
+    "old_instance" : {
+        "url": "http://localhost:8080",
+        "user": "user",
+        "token": "tokentoken"
+    },
+    "new_instance" : {
+        "url": "http://localhost:8080",
+        "user": "user",
+        "token": "tokentoken"
+    },
+    "project_mappings": [
+        {"from": "a", "to": "b"},
+        {"from": "c", "to": "d"}
+    ]
+}

tools/migration/batch_migrator.py

@@ -0,0 +1,47 @@
+import argparse
+import json
+
+from pathlib import Path
+from task_migrator import TaskMigrator
+from blacklist_migrator import BlacklistMigrator
+from migrator_base import create_logger
+
+
+def main():
+    """ Migrates the blacklists and tasks of multiple projects from one server to the projects to the other.
+    It automatically reads the arguments from the command line. """
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("config", help="The path to the config file. Needs to be in a specific format, "
+                                       "see batch_config.template.")
+    args = parser.parse_args()
+    logger = create_logger(name="batch")
+    config_file = Path(args.config)
+    if not config_file.exists():
+        logger.error("Config file does not exist")
+
+    with config_file.open() as file:
+        data = json.load(file)
+        base_config = {x: data[x] for x in ["old_instance", "new_instance"]}
+
+        for mapping in data["project_mappings"]:
+            migrate(base_config, mapping, logger)
+
+
+def migrate(base_config, mapping, logger):
+    """ Migrates the blacklist and the tasks of between the projects defined in the mapping.
+    The servers containing the project are defined in the base config.
+    """
+    if not all(key in mapping for key in ("from", "to")):
+        logger.error("Project mapping is malformed: %s" % mapping)
+        return None
+
+    base_config["old_instance"]["project"] = mapping["from"]
+    base_config["new_instance"]["project"] = mapping["to"]
+
+    logger.info("Migrating from '{from}' to '{to}'".format(**mapping))
+    BlacklistMigrator(base_config).migrate()
+    TaskMigrator(base_config).migrate()
+
+
+if __name__ == "__main__":
+    main()