From e4d5bd8d2989dcba32388c20378068900ee0146a Mon Sep 17 00:00:00 2001 From: Shem Manyu Date: Tue, 9 Apr 2019 17:08:57 +0300 Subject: [PATCH] removed redundant get functions in repo processor --- .gitignore | 2 ++ compiler/repo_processor.py | 64 +++++++---------------------------- simple_grid_yaml_compiler.py | 16 ++++----- tests/tests_repo_processor.py | 18 +++++----- 4 files changed, 32 insertions(+), 68 deletions(-) diff --git a/.gitignore b/.gitignore index 894a44c..43681bc 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,5 @@ venv.bak/ # mypy .mypy_cache/ + +.vscode \ No newline at end of file diff --git a/compiler/repo_processor.py b/compiler/repo_processor.py index 5576c73..127ec0e 100644 --- a/compiler/repo_processor.py +++ b/compiler/repo_processor.py @@ -3,17 +3,13 @@ from urlparse import urlparse, urljoin -def generate_default_file_name(repo_info): - return './.temp/' + repo_info['repo_name'] + '_defaults.yaml' - - -def generate_config_schema_file_name(repo_info): - return './.temp/' + repo_info['repo_name'] + '_schema.yaml' - - -def generate_meta_info_file_name(repo_info): - return './.temp/' + repo_info['repo_name'] + '_info.yaml' - +def generate_file_name(repo_info, file): + return { + "meta-info.yaml": './.temp/' + repo_info['repo_name'] + '_info.yaml', + "config-schema.yaml": './.temp/' + repo_info['repo_name'] + '_schema.yaml', + "site_level_configuration_defaults.yaml": './.temp/' + repo_info['repo_name'] + '_defaults.yaml', + "default-data.yaml": './.temp/' + repo_info['repo_name'] + '_defaults.yaml' + }[file] def analyse_repo_url(repo_url): repo_analysis = re.search('//.*/(.*)/(.*)', repo_url) @@ -46,55 +42,21 @@ def augment_meta_info(meta_info_file): meta_info.write(augmented_meta_info) return meta_info -def get_meta_info(repo_url): +def get_repository_file(repo_url, file, is_meta_file=False): try: base_url = urlparse("https://raw.githubusercontent.com/") repo_info = analyse_repo_url(repo_url) - repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], 'meta-info.yaml'] + repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], file] relative_url = urlparse("/".join(x.strip() for x in repo_info_list)) meta_info_url = urljoin(base_url.geturl(), relative_url.geturl()) response = urllib2.urlopen(meta_info_url) meta_info = response.read() - fname = generate_meta_info_file_name(repo_info) + fname = generate_file_name(repo_info, file) with open(fname, 'w') as f: f.write(meta_info) f.close() - return augment_meta_info(fname) - except Exception as ex: - print ex.message - - -def get_default_values(repo_url, default_file_name): - try: - default_data_base_url = urlparse("https://raw.githubusercontent.com/") - repo_info = analyse_repo_url(repo_url) - repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], default_file_name] - default_data_relative_url = urlparse("/".join(x.strip() for x in repo_info_list)) - default_data_url = urljoin(default_data_base_url.geturl(), default_data_relative_url.geturl()) - response = urllib2.urlopen(default_data_url) - default_data = response.read() - fname = generate_default_file_name(repo_info) - with open(fname, 'w') as f: - f.write(default_data) - f.close() + if is_meta_file: + return augment_meta_info(fname) return fname - except Exception as ex: - print(ex.message) - - -def get_config_schema(repo_url): - try: - base_url= urlparse("https://raw.githubusercontent.com/") - repo_info = analyse_repo_url(repo_url) - repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], 'config-schema.yaml'] - relative_url = urlparse("/".join(x.strip() for x in repo_info_list)) - config_schema_url = urljoin(base_url.geturl(), relative_url.geturl()) - response = urllib2.urlopen(config_schema_url) - config_schema = response.read() - fname = generate_config_schema_file_name(repo_info) - with open(fname, 'w') as f: - f.write(config_schema) - f.close() - return f except Exception as ex: - print(ex.message) + print ex.message \ No newline at end of file diff --git a/simple_grid_yaml_compiler.py b/simple_grid_yaml_compiler.py index d0a525f..aa16b66 100644 --- a/simple_grid_yaml_compiler.py +++ b/simple_grid_yaml_compiler.py @@ -10,14 +10,14 @@ # OUTPUT: include statements for default files and meta-info files of repositories + raw site level-config file def phase_1(site_level_configuration_file): # fetch repo and get default_values.yaml - main_default_values_file = repo_processor.get_default_values("https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults", "site_level_configuration_defaults.yaml" ) + main_default_values_file = repo_processor.get_repository_file("https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults", "site_level_configuration_defaults.yaml" ) repo_urls = lexemes.get_repo_list(site_level_configuration_file) print(repo_urls) file_names_repository_default = [main_default_values_file] file_names_repository_meta = [] for url in repo_urls: - file_names_repository_default.append(repo_processor.get_default_values(url, 'default-data.yaml')) - file_names_repository_meta.append(repo_processor.get_meta_info(url).name) + file_names_repository_default.append(repo_processor.get_repository_file(url, 'default-data.yaml')) + file_names_repository_meta.append(repo_processor.get_repository_file(url, 'meta-info.yaml', True).name) all_includes = file_names_repository_meta + file_names_repository_default includes_yaml_file = yaml_augmentation.add_include_statements(all_includes, site_level_configuration_file) return includes_yaml_file, repo_urls @@ -54,15 +54,15 @@ def phase_5(phase_4_output, runtime_vars, yaml): for component_section in lightweight_component: if component_section == 'config': repo_url = lightweight_component['repository_url'] - repo_processor.get_config_schema(repo_url) - repo_processor.get_meta_info(repo_url) + repo_processor.get_repository_file(repo_url, 'config-schema.yaml') + repo_processor.get_repository_file(repo_url, 'meta-info.yaml', True) repo_info = repo_processor.analyse_repo_url(repo_url) - config_schema_file_name = repo_processor.generate_config_schema_file_name(repo_info) + config_schema_file_name = repo_processor.generate_file_name(repo_info, 'config-schema.yaml') config_schema_file = open(config_schema_file_name, 'r') - meta_info_file = repo_processor.generate_meta_info_file_name(repo_info) + meta_info_file = repo_processor.generate_file_name(repo_info, 'meta-info.yaml') meta_info_parent_name = repo_processor.generate_meta_info_parent_name(meta_info_file) meta_info = data[meta_info_parent_name] - default_data_file_name = repo_processor.generate_default_file_name(repo_info) + default_data_file_name = repo_processor.generate_file_name(repo_info, 'default-data.yaml') default_data_runtime_file = open(default_data_file_name + ".runtime", 'w') default_data_file = open(default_data_file_name, 'r') default_data_runtime_file.write(runtime_vars) diff --git a/tests/tests_repo_processor.py b/tests/tests_repo_processor.py index 5e8d3ff..a9d5fb9 100644 --- a/tests/tests_repo_processor.py +++ b/tests/tests_repo_processor.py @@ -1,4 +1,4 @@ -from compiler.repo_processor import analyse_repo_url, generate_default_file_name, generate_config_schema_file_name, generate_meta_info_file_name, get_default_values, get_config_schema, get_meta_info, augment_meta_info, generate_meta_info_parent_name +from compiler.repo_processor import analyse_repo_url, generate_file_name, augment_meta_info, generate_meta_info_parent_name, get_repository_file from os import mkdir, remove from shutil import rmtree @@ -27,7 +27,7 @@ def test_generate_default_file_name(self): expected_output = "./.temp/simple_grid_yaml_compiler_defaults.yaml" - self.assertEqual(generate_default_file_name(repo_info), expected_output) + self.assertEqual(generate_file_name(repo_info, "site_level_configuration_defaults.yaml"), expected_output) def test_generate_config_schema_file_name(self): repo_info = { @@ -38,7 +38,7 @@ def test_generate_config_schema_file_name(self): expected_output = "./.temp/simple_grid_yaml_compiler_schema.yaml" - self.assertEqual(generate_config_schema_file_name(repo_info), expected_output) + self.assertEqual(generate_file_name(repo_info, "config-schema.yaml"), expected_output) def test_generate_meta_info_file_name(self): repo_info = { @@ -49,13 +49,13 @@ def test_generate_meta_info_file_name(self): expected_output = "./.temp/simple_grid_yaml_compiler_info.yaml" - self.assertEqual(generate_meta_info_file_name(repo_info), expected_output) + self.assertEqual(generate_file_name(repo_info, "meta-info.yaml"), expected_output) def test_get_default_values(self): repo_url = "https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults" defaults_file = "site_level_configuration_defaults.yaml" - fname = get_default_values(repo_url, defaults_file) + fname = get_repository_file(repo_url, defaults_file) with open(fname, "r") as file: output = file.read() @@ -68,10 +68,10 @@ def test_get_default_values(self): def test_get_config_schema(self): repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream" - get_config_schema(repo_url) + get_repository_file(repo_url, "config-schema.yaml") repo_info = analyse_repo_url(repo_url) - fname = generate_config_schema_file_name(repo_info) + fname = generate_file_name(repo_info, "config-schema.yaml") with open(fname, "r") as file: output = file.read() @@ -84,10 +84,10 @@ def test_get_config_schema(self): def test_get_meta_info(self): repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream" - get_meta_info(repo_url) + get_repository_file(repo_url, "meta-info.yaml",True) repo_info = analyse_repo_url(repo_url) - fname = generate_meta_info_file_name(repo_info) + fname = generate_file_name(repo_info, "meta-info.yaml") with open(fname, "r") as file: output = file.read()