Skip to content

removed redundant get functions in repo processor #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ShemManyu wants to merge 1 commit into
base: master
Choose a base branch
from
Open

removed redundant get functions in repo processor #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,5 @@ venv.bak/

# mypy
.mypy_cache/

.vscode
64 changes: 13 additions & 51 deletions compiler/repo_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,13 @@
from urlparse import urlparse, urljoin


def generate_default_file_name(repo_info):
return './.temp/' + repo_info['repo_name'] + '_defaults.yaml'


def generate_config_schema_file_name(repo_info):
return './.temp/' + repo_info['repo_name'] + '_schema.yaml'


def generate_meta_info_file_name(repo_info):
return './.temp/' + repo_info['repo_name'] + '_info.yaml'

def generate_file_name(repo_info, file):
return {
"meta-info.yaml": './.temp/' + repo_info['repo_name'] + '_info.yaml',
"config-schema.yaml": './.temp/' + repo_info['repo_name'] + '_schema.yaml',
"site_level_configuration_defaults.yaml": './.temp/' + repo_info['repo_name'] + '_defaults.yaml',
"default-data.yaml": './.temp/' + repo_info['repo_name'] + '_defaults.yaml'
}[file]

def analyse_repo_url(repo_url):
repo_analysis = re.search('//.*/(.*)/(.*)', repo_url)
Expand Down Expand Up @@ -46,55 +42,21 @@ def augment_meta_info(meta_info_file):
meta_info.write(augmented_meta_info)
return meta_info

def get_meta_info(repo_url):
def get_repository_file(repo_url, file, is_meta_file=False):
try:
base_url = urlparse("https://raw.githubusercontent.com/")
repo_info = analyse_repo_url(repo_url)
repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], 'meta-info.yaml']
repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], file]
relative_url = urlparse("/".join(x.strip() for x in repo_info_list))
meta_info_url = urljoin(base_url.geturl(), relative_url.geturl())
response = urllib2.urlopen(meta_info_url)
meta_info = response.read()
fname = generate_meta_info_file_name(repo_info)
fname = generate_file_name(repo_info, file)
with open(fname, 'w') as f:
f.write(meta_info)
f.close()
return augment_meta_info(fname)
except Exception as ex:
print ex.message


def get_default_values(repo_url, default_file_name):
try:
default_data_base_url = urlparse("https://raw.githubusercontent.com/")
repo_info = analyse_repo_url(repo_url)
repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], default_file_name]
default_data_relative_url = urlparse("/".join(x.strip() for x in repo_info_list))
default_data_url = urljoin(default_data_base_url.geturl(), default_data_relative_url.geturl())
response = urllib2.urlopen(default_data_url)
default_data = response.read()
fname = generate_default_file_name(repo_info)
with open(fname, 'w') as f:
f.write(default_data)
f.close()
if is_meta_file:
return augment_meta_info(fname)
return fname
except Exception as ex:
print(ex.message)


def get_config_schema(repo_url):
try:
base_url= urlparse("https://raw.githubusercontent.com/")
repo_info = analyse_repo_url(repo_url)
repo_info_list = [repo_info['org_name'], repo_info['repo_name'], repo_info['branch_name'], 'config-schema.yaml']
relative_url = urlparse("/".join(x.strip() for x in repo_info_list))
config_schema_url = urljoin(base_url.geturl(), relative_url.geturl())
response = urllib2.urlopen(config_schema_url)
config_schema = response.read()
fname = generate_config_schema_file_name(repo_info)
with open(fname, 'w') as f:
f.write(config_schema)
f.close()
return f
except Exception as ex:
print(ex.message)
print ex.message
16 changes: 8 additions & 8 deletions simple_grid_yaml_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@
# OUTPUT: include statements for default files and meta-info files of repositories + raw site level-config file
def phase_1(site_level_configuration_file):
# fetch repo and get default_values.yaml
main_default_values_file = repo_processor.get_default_values("https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults", "site_level_configuration_defaults.yaml" )
main_default_values_file = repo_processor.get_repository_file("https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults", "site_level_configuration_defaults.yaml" )
repo_urls = lexemes.get_repo_list(site_level_configuration_file)
print(repo_urls)
file_names_repository_default = [main_default_values_file]
file_names_repository_meta = []
for url in repo_urls:
file_names_repository_default.append(repo_processor.get_default_values(url, 'default-data.yaml'))
file_names_repository_meta.append(repo_processor.get_meta_info(url).name)
file_names_repository_default.append(repo_processor.get_repository_file(url, 'default-data.yaml'))
file_names_repository_meta.append(repo_processor.get_repository_file(url, 'meta-info.yaml', True).name)
all_includes = file_names_repository_meta + file_names_repository_default
includes_yaml_file = yaml_augmentation.add_include_statements(all_includes, site_level_configuration_file)
return includes_yaml_file, repo_urls
Expand Down Expand Up @@ -54,15 +54,15 @@ def phase_5(phase_4_output, runtime_vars, yaml):
for component_section in lightweight_component:
if component_section == 'config':
repo_url = lightweight_component['repository_url']
repo_processor.get_config_schema(repo_url)
repo_processor.get_meta_info(repo_url)
repo_processor.get_repository_file(repo_url, 'config-schema.yaml')
repo_processor.get_repository_file(repo_url, 'meta-info.yaml', True)
repo_info = repo_processor.analyse_repo_url(repo_url)
config_schema_file_name = repo_processor.generate_config_schema_file_name(repo_info)
config_schema_file_name = repo_processor.generate_file_name(repo_info, 'config-schema.yaml')
config_schema_file = open(config_schema_file_name, 'r')
meta_info_file = repo_processor.generate_meta_info_file_name(repo_info)
meta_info_file = repo_processor.generate_file_name(repo_info, 'meta-info.yaml')
meta_info_parent_name = repo_processor.generate_meta_info_parent_name(meta_info_file)
meta_info = data[meta_info_parent_name]
default_data_file_name = repo_processor.generate_default_file_name(repo_info)
default_data_file_name = repo_processor.generate_file_name(repo_info, 'default-data.yaml')
default_data_runtime_file = open(default_data_file_name + ".runtime", 'w')
default_data_file = open(default_data_file_name, 'r')
default_data_runtime_file.write(runtime_vars)
Expand Down
18 changes: 9 additions & 9 deletions tests/tests_repo_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from compiler.repo_processor import analyse_repo_url, generate_default_file_name, generate_config_schema_file_name, generate_meta_info_file_name, get_default_values, get_config_schema, get_meta_info, augment_meta_info, generate_meta_info_parent_name
from compiler.repo_processor import analyse_repo_url, generate_file_name, augment_meta_info, generate_meta_info_parent_name, get_repository_file

from os import mkdir, remove
from shutil import rmtree
Expand Down Expand Up @@ -27,7 +27,7 @@ def test_generate_default_file_name(self):

expected_output = "./.temp/simple_grid_yaml_compiler_defaults.yaml"

self.assertEqual(generate_default_file_name(repo_info), expected_output)
self.assertEqual(generate_file_name(repo_info, "site_level_configuration_defaults.yaml"), expected_output)

def test_generate_config_schema_file_name(self):
repo_info = {
Expand All @@ -38,7 +38,7 @@ def test_generate_config_schema_file_name(self):

expected_output = "./.temp/simple_grid_yaml_compiler_schema.yaml"

self.assertEqual(generate_config_schema_file_name(repo_info), expected_output)
self.assertEqual(generate_file_name(repo_info, "config-schema.yaml"), expected_output)

def test_generate_meta_info_file_name(self):
repo_info = {
Expand All @@ -49,13 +49,13 @@ def test_generate_meta_info_file_name(self):

expected_output = "./.temp/simple_grid_yaml_compiler_info.yaml"

self.assertEqual(generate_meta_info_file_name(repo_info), expected_output)
self.assertEqual(generate_file_name(repo_info, "meta-info.yaml"), expected_output)

def test_get_default_values(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults"
defaults_file = "site_level_configuration_defaults.yaml"

fname = get_default_values(repo_url, defaults_file)
fname = get_repository_file(repo_url, defaults_file)

with open(fname, "r") as file:
output = file.read()
Expand All @@ -68,10 +68,10 @@ def test_get_default_values(self):
def test_get_config_schema(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream"

get_config_schema(repo_url)
get_repository_file(repo_url, "config-schema.yaml")

repo_info = analyse_repo_url(repo_url)
fname = generate_config_schema_file_name(repo_info)
fname = generate_file_name(repo_info, "config-schema.yaml")

with open(fname, "r") as file:
output = file.read()
Expand All @@ -84,10 +84,10 @@ def test_get_config_schema(self):
def test_get_meta_info(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream"

get_meta_info(repo_url)
get_repository_file(repo_url, "meta-info.yaml",True)

repo_info = analyse_repo_url(repo_url)
fname = generate_meta_info_file_name(repo_info)
fname = generate_file_name(repo_info, "meta-info.yaml")

with open(fname, "r") as file:
output = file.read()
Expand Down