added unittest for compiler/lexemes.py

compiler/lexemes.py

@@ -35,4 +35,4 @@ def parse_for_variable_hierarchies(data, keyword):
         for sub_data_key in data:
             sub_data_updated = parse_for_variable_hierarchies(data[sub_data_key], keyword)
             data_updated[sub_data_key] = sub_data_updated
-        return data_updated
+        return data_updated

compiler/repo_processor.py

@@ -19,7 +19,7 @@ def analyse_repo_url(repo_url):
     repo_analysis = re.search('//.*/(.*)/(.*)', repo_url)
     org_name = repo_analysis.group(1)
     repo_name = repo_analysis.group(2)
-    ##TODO fetch branch info
+    #TODO fetch branch info
     branch = 'master'
     return {
         'org_name':org_name,

test_input_data.json

@@ -0,0 +1 @@
+{"meta_info_wn-pbs": {"component": "WN-Pbs", "type": "worker_node", "version": "2.5.13", "docker_hub_tag": "", "site_level_config_version": 1.0, "default_var_prefix": "pbs_default_var", "runtime_vars": ["pbs_runtime_var_ce_host"], "docker_run_parameters": {"privileged": true, "tty": true, "detached": true, "command": "/bin/bash"}}, "default_group_opssgm": {"gid": 46001, "name": "opssgm"}, "vars": ["192.168.0.4", "192.168.0.5"], "default_pool_accounts_alicesgm": {"base_name": "sgmali", "initial_uid": 60101, "users_num": 10, "primary_group": {"gid": 1397, "name": "alicesgm"}, "secondary_groups": [{"gid": 1395, "name": "alice"}]}, "default_group_ops_name": "ops", "site": {"name": "simple-dev-site", "email": "mayank.sharma@cern.ch", "latitude": 46.3, "longitude": 6.2, "location": "CERN, Geneva, Switzerland", "description": "A demo site demonstrating the SIMPLE framework for site setup", "website": "https://home.cern", "support_website": "https://groups.google.com/forum/#!forum/wlcg-lightweight-sites", "support_email": "mayank.sharma@cern.ch", "security_email": "mayank.sharma@cern.ch", "grid": "wlcg", "tier": 3, "bdii_host": "bdii.cern.ch", "use_argus": false}, "cream_runtime_variables": ["lightweight_component01.cern.ch"], "lightweight_components": [{"lifecycle_hooks": {"pre_config": ["/etc/simple_grid/lifecycle/ce_pre_config.sh"], "pre_init": ["/etc/simple_grid/lifecycle/ce_pre_inst1.sh"], "post_init": ["/etc/simple_grid/lifecycle/ce_post_inst1.sh"]}, "name": "Cream-CE", "deploy": [{"node": "lightweight_component01.cern.ch", "container_count": 1}], "config": {"ce_si00": 381, "cream_db_password": "str()","cemon_host": {"__from__": "lightweight_component01.cern.ch"}, "ce_runtimeenv": "EMI-3", "ce_close_se": "SE3", "ce_cpu_vendor": "Intel", "ce_minvirtmem": 8192, "ce_close_se3_access_point": "/not/used", "batch_log_dir": "/var/spool/pbs", "apel_db_password": "str()", "ce_inboundip": false, "blparser_with_updater_notifier": true, "ce_logcpu": 0, "ce_os_arch": "str()", "apel_mysql_host": {"__from__": "lightweight_component01.cern.ch"}, "ce_otherdescr": "Cores=4", "cream_db_user": "str()", "ce_sf00": 0, "ce_minphysmem": 2000, "ce_smpsize": 8, "ce_physcpu": 0, "ce_outboundip": true, "se_list": {"__from__": "not-used.some-domain"}, "blparser_host": {"__from__": "lightweight_component01.cern.ch"}, "globus_tcp_port_range": {"begin": 20000, "end": 21000}, "ce_capability": "CPUScalingReferenceSI00=1000", "mysql_password": "fteu5746hg", "ce_cpu_model": "Xeon", "ce_cpu_speed": 3000, "queues": {"alice": [{"voms_fqan": "/alice", "pool_accounts": [{"base_name": "alice", "initial_uid": 10417, "users_num": 5, "primary_group": {"gid": 1395, "name": "alice"}, "secondary_groups": []}], "vo": {"name": "alice", "servers": [{"server": "voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/not/used"}}, {"voms_fqan": "/alice/ROLE=lcgadmin", "pool_accounts": [{"base_name": "alice", "initial_uid": 10417, "users_num": 5, "primary_group": {"gid": 1395, "name": "alice"}, "secondary_groups": []}, {"base_name": "sgmali", "initial_uid": 60101, "users_num": 10, "primary_group": {"gid": 1397, "name": "alicesgm"}, "secondary_groups": [{"gid": 1395, "name": "alice"}]}], "vo": {"name": "alice", "servers": [{"server": "voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/not/used"}}]}, "ce_close_se3_host": "not-used.some-domain", "se_mount_info_list": "none"}, "repository_revision": "master", "supplemental_config": {"some_additional_parameter": "some_value"}, "repository_url": "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream", "type": "compute_element", "execution_id": 0, "preferred_tech_stack": {"level_2_configuration": "yaim"}}, {"lifecycle_hooks": {"pre_config": ["/etc/simple_grid/lifecycle/wn_pre_config.sh"], "pre_init": ["/etc/simple_grid/lifecycle/wn_pre_inst1.sh"], "post_init": ["/etc/simple_grid/lifecycle/wn_post_inst1.sh"]}, "name": "WN-Pbs", "deploy": [{"node": "lightweight_component02.cern.ch", "container_count": 2}], "config": {"px_host": "myproxy.cern.ch", "batch_server": "pbs", "ce_smpsize": 8, "ce_close_se3_access_point": "/not/used", "ce_host": "lightweight_component01.cern.ch", "ce_close_se": "SE3", "se_list": {"__from__": "not-used.some-domain"}, "ce_close_se3_host": "not-used.some-domain", "se_mount_info_list": "none"}, "repository_revision": "master", "supplemental_config": {"some_additional_parameter": "some_value"}, "repository_url": "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_wn_pbs", "type": "worker_node", "execution_id": 1, "preferred_tech_stack": {"level_2_configuration": "yaim"}}], "default_pool_accounts_alice": {"base_name": "alice", "initial_uid": 10417, "users_num": 5, "primary_group": {"gid": 1395, "name": "alice"}, "secondary_groups": []}, "default_group_dteamsgm": {"gid": 2680, "name": "dteamsgm"}, "preferred_tech_stack": {"level_1_configuration": "puppet", "level_2_configuration": "puppet", "container_orchestration": "docker-swarm", "container": "docker"}, "default_pool_accounts_dteam": {"base_name": "dteam", "initial_uid": 18118, "users_num": 10, "primary_group": {"gid": 2688, "name": "dteam"}, "secondary_groups": []}, "default_vo_dteam": {"name": "dteam", "servers": [{"server": "voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/dteam"}, "default_group_ops": {"gid": 45000, "name": "ops"}, "default_group_alicesgm_name": "alicesgm", "pbs_runtime_variables": ["lightweight_component01.cern.ch"], "site_infrastructure": [{"fqdn": "lightweight_component01.cern.ch", "ip_address": "192.168.0.4"}, {"fqdn": "lightweight_component02.cern.ch", "ip_address": "192.168.0.5"}], "default_pool_accounts_dteamsgm": {"base_name": "sgmdtm", "initial_uid": 60501, "users_num": 5, "primary_group": {"gid": 2680, "name": "dteamsgm"}, "secondary_groups": [{"gid": 2688, "name": "dteam"}]}, "default_group_alice_name": "alice", "default_vo_generic_dn_ca_cern": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority", "default_group_alicesgm": {"gid": 1397,"name": "alicesgm"}, "supported_virtual_organizations": [{"name": "alice", "servers": [{"server": "voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/not/used"}, {"name": "dteam", "servers": [{"server": "voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/dteam"}, {"name": "ops", "servers": [{"server": "voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/ops"}], "voms_config": [{"voms_fqan": "/alice", "pool_accounts": [{"base_name": "alice", "initial_uid": 10417, "users_num": 5, "primary_group": {"gid": 1395, "name": "alice"}, "secondary_groups": []}], "vo": {"name": "alice", "servers": [{"server": "voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/not/used"}}, {"voms_fqan": "/alice/ROLE=lcgadmin", "pool_accounts": [{"base_name": "alice", "initial_uid": 10417, "users_num": 5, "primary_group": {"gid": 1395, "name": "alice"}, "secondary_groups": []}, {"base_name": "sgmali", "initial_uid": 60101, "users_num": 10, "primary_group": {"gid": 1397, "name": "alicesgm"}, "secondary_groups": [{"gid": 1395, "name": "alice"}]}], "vo": {"name": "alice", "servers": [{"server": "voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/not/used"}, "comment": "sgm"}, {"voms_fqan": "/dteam", "pool_accounts": [{"base_name": "dteam", "initial_uid": 18118, "users_num": 10, "primary_group": {"gid": 2688, "name": "dteam"}, "secondary_groups": []}], "vo": {"name": "dteam", "servers": [{"server": "voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/dteam"}}, {"voms_fqan": "/dteam/ROLE=lcgadmin", "pool_accounts": [{"base_name": "sgmdtm", "initial_uid": 60501, "users_num": 5, "primary_group": {"gid": 2680, "name": "dteamsgm"}, "secondary_groups": [{"gid": 2688, "name": "dteam"}]}, {"base_name": "dteam", "initial_uid": 18118, "users_num": 10, "primary_group": {"gid": 2688, "name": "dteam"}, "secondary_groups": []}], "vo": {"name": "dteam", "servers": [{"server": "voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15001", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/dteam"}, "comment": "sgm"}, {"voms_fqan": "/ops", "vo": {"name": "ops", "servers": [{"server": "voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/ops"}, "pool_accounts": [{"base_name": "ops", "initial_uid": 45001, "users_num": 10, "primary_group": {"gid": 45000, "name": "ops"}, "secondary_groups": []}]}, {"voms_fqan": "/ops/ROLE=lcgadmin", "pool_accounts": [{"base_name": "sgmops", "initial_uid": 60701, "users_num": 10, "primary_group": {"gid": 45000, "name": "ops"}, "secondary_groups": [{"gid": 46001, "name": "opssgm"}]}, {"base_name": "ops", "initial_uid": 45001, "users_num": 10, "primary_group": {"gid": 45000, "name": "ops"}, "secondary_groups": []}], "vo": {"name": "ops", "servers": [{"server": "voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/ops"}, "comment": "sgm"}], "default_vo_alice": {"name": "alice", "servers": [{"server": "voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15000", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/not/used"}, "default_vo_generic_server_primary": "voms2.cern.ch", "default_vo_ops": {"name": "ops", "servers": [{"server": "voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}, {"server": "lcg-voms2.cern.ch", "port": "15009", "dn": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "ca_dn": "/DC=ch/DC=cern/CN=CERN Grid Certification Authority"}], "default_se": "not_used.some-domain", "sw_dir": ".", "storage_dir": "/storage/ops"}, "default_group_dteam": {"gid": 2688, "name": "dteam"}, "meta_info_cream-ce": {"component": "Cream-CE", "type": "compute_element", "version": 1.2, "docker_hub_tag": "", "site_level_config_version": 1.0, "default_var_prefix": "cream_default_var", "runtime_vars": ["cream_runtime_var_ce_host"], "host_requirements": {"firewall": [{"ports": 22, "protocol": "tcp", "action": "accept", "ipv6": true}, {"ports": 8443, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": "20000-21000", "protocol":"tcp", "action": "accept", "ipv6": false}, {"ports": 49152, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 49155, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 49152, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 9002, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 9002, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 9002, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 9001, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 3306, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 2170, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 33333, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 56565, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 56554, "protocol": "tcp", "action": "accept", "ipv6": false}, {"ports": 40559, "protocol": "tcp", "action": "accept", "ipv6": false}], "cvmfs": true, "host_certificates": true}, "docker_run_parameters": {"privileged": true, "hostname": "lightweight_component01.cern.ch", "ports": ["8443:8443", "2811:2811", "20000-21000:20000-21000", "49152:49152", "49155:49155", "9002:9002", "9001:9001", "3306:3306", "2170:2170", "33333:33333", "56565:56565", "56554:56554", "40559:40559"], "tty": true, "interactive": true, "detached": true, "command": "/bin/bash"}}, "default_vo_generic_dn_server_secondary": "/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch", "default_pool_account_ops": {"base_name": "ops", "initial_uid": 45001, "users_num": 10, "primary_group": {"gid": 45000, "name": "ops"}, "secondary_groups": []}, "wn-pbs-expected-from-site-level-config": {"pbs_default_var_px_host": "myproxy.cern.ch", "pbs_default_var_ce_host": {"__from__": "lightweight_component01.cern.ch"}, "pbs_default_var_ce_smpsize": 8, "pbs_default_var_batch_server": {"__from__": "lightweight_component01.cern.ch"}, "pbs_default_var_ce_close_se": "SE3", "pbs_default_var_ce_close_se3_access_point": "/not/used", "pbs_default_var_ce_close_se3_host": "not-used.some-domain", "pbs_default_var_se_list": {"__from__": "not-used.some-domain"}, "pbs_default_var_se_mount_info_list": "none"}, "default_group_alice": {"gid": 1395, "name": "alice"}, "default_vo_generic_server_secondary": "lcg-voms2.cern.ch", "default_group_opssgm_name": "opssgm", "default_vo_generic_dn_server_primary": "/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch", "cream-ce-expected-from-site-level-config": {"cream_default_var_globus_tcp_port_range": {"begin": 20000, "end": 21000}, "cream_default_var_ce_capability": "CPUScalingReferenceSI00=1000", "cream_default_var_ce_cpu_model": "Xeon", "cream_default_var_ce_cpu_speed": 2800, "cream_default_var_ce_cpu_vendor": "intel", "cream_default_var_ce_inboundip": false, "cream_default_var_ce_logcpu": 0, "cream_default_var_ce_minphysmem": 2000, "cream_default_var_ce_minvirtmem": 8192, "cream_default_var_ce_os_arch": "x86_64", "cream_default_var_ce_otherdescr": "Cores=4", "cream_default_var_ce_outboundip": true, "cream_default_var_ce_physcpu": 0, "cream_default_var_ce_runtimeenv": "EMI-3", "cream_default_var_ce_sf00": 0, "cream_default_var_ce_si00": 381, "cream_default_var_ce_smpsize": 8, "cream_default_var_batch_log_dir": "/var/spool/pbs", "cream_default_var_blparser_with_updater_notifier": true, "cream_default_var_blparser_host": {"__from__": "lightweight_component01.cern.ch"}, "cream_default_var_cemon_host": {"__from__": "lightweight_component01.cern.ch"}, "cream_default_var_cream_db_user": "cream", "cream_default_var_apel_mysql_host": {"__from__": "lightweight_component01.cern.ch"}, "cream_default_var_ce_close_se": "SE3", "cream_default_var_ce_close_se3_access_point": "/not/used", "cream_default_var_ce_close_se3_host": "not-used.some-domain", "cream_default_var_se_list": {"__from__": "not-used.some-domain"}, "cream_default_var_se_mount_info_list": "none"}, "default_group_dteamsgm_name": "dteamsgm", "default_pool_account_opssgm": {"base_name": "sgmops", "initial_uid": 60701, "users_num": 10, "primary_group": {"gid": 45000, "name": "ops"}, "secondary_groups": [{"gid": 46001, "name": "opssgm"}]}, "default_group_dteam_name": "dteam"}