From 8f9d1969d3fe1a28e4e9129e878e95d3446d4646 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Wed, 4 Mar 2026 13:02:31 +0100 Subject: [PATCH] feat: migrate to new template --- .tool-versions | 4 +- ansible/ansible.cfg | 21 +- ansible/group_vars/all/defaults.yaml | 2 + ansible/requirements.yaml | 6 +- setup.sh | 6 +- terraform/devnet-4/ansible_inventory.tmpl | 91 ++++++ terraform/devnet-4/cloudflare.tf | 60 ++++ terraform/devnet-4/digitalocean.tf | 167 +++++++++++ terraform/devnet-4/firewall.tf | 330 ++++++++++++++++++++++ terraform/devnet-4/hetzner.tf | 177 ++++++++++++ terraform/devnet-4/main.tf | 98 +++++++ terraform/devnet-4/nodes.tf | 36 +++ terraform/devnet-4/outputs.tf | 118 ++++++++ terraform/devnet-4/ssh_config.tmpl | 16 ++ 14 files changed, 1119 insertions(+), 13 deletions(-) create mode 100644 terraform/devnet-4/ansible_inventory.tmpl create mode 100644 terraform/devnet-4/cloudflare.tf create mode 100644 terraform/devnet-4/digitalocean.tf create mode 100644 terraform/devnet-4/firewall.tf create mode 100644 terraform/devnet-4/hetzner.tf create mode 100644 terraform/devnet-4/main.tf create mode 100644 terraform/devnet-4/nodes.tf create mode 100644 terraform/devnet-4/outputs.tf create mode 100644 terraform/devnet-4/ssh_config.tmpl diff --git a/.tool-versions b/.tool-versions index 8f86a40..4976a0d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,8 +1,8 @@ age 1.2.1 helm 3.15.3 sops 3.8.1 -shellcheck 0.10.0 -python 3.12.1 +shellcheck 0.11.0 +python 3.12.12 awscli 2.22.26 yq 4.44.6 terraform 1.9.0 diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 4f1a5b3..ed68332 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,22 +1,25 @@ [defaults] ansible_managed = Ansible managed. Don't change this file manually. Template info: {{{{ (template_fullpath | replace(playbook_dir,'')) }}}} -stdout_callback = yaml -inventory = inventories/devnet-3/inventory.ini +stdout_callback = default +result_format = yaml +inventory = inventories/devnet-4/inventory.ini roles_path = vendor/roles/:roles collections_path = vendor/collections -forks = 50 +forks = 100 timeout = 60 retry_files_enabled = False host_key_checking = False vars_plugins_enabled = host_group_vars,community.sops.sops - +vars_plugin_stage = inventory +strategy = mitogen_free +strategy_plugins = vendor/mitogen-0.3.43/ansible_mitogen/plugins/strategy # Persist facts locally so that they can be used within multiple runs fact_caching = jsonfile # Keep facts forever fact_caching_timeout = 0 # Where to store the fact cache -fact_caching_connection = tmp/devnet-3/ +fact_caching_connection = tmp/devnet-4/ [inventory] enable_plugins = script, yaml, ini @@ -24,3 +27,11 @@ enable_plugins = script, yaml, ini [ssh_connection] ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s pipelining = true + +[mitogen] +# Optional: Adjust the size of the connection pool (default is 16) +# Increase if you have many hosts +# mitogen_pool_size = 32 + +# Optional: Use fork isolation for better module compatibility (default is fork) +# mitogen_task_isolation = fork diff --git a/ansible/group_vars/all/defaults.yaml b/ansible/group_vars/all/defaults.yaml index 163b37e..59cf6ca 100644 --- a/ansible/group_vars/all/defaults.yaml +++ b/ansible/group_vars/all/defaults.yaml @@ -59,6 +59,8 @@ docker_daemon_options: "log-opts": "max-size": "500m" "max-file": "8" + "registry-mirrors": ["https://docker-cache.ethquokkaops.io","https://docker.ethquokkaops.io/gh"] + "features": { "containerd-snapshotter": false } # This is a temp fix for a docker 29 in combinations with our pull through cache, can be safely removed once the cache is working again with docker 29+ # role: ethpandaops.general.prometheus prometheus_container_networks: "{{ docker_networks_shared }}" diff --git a/ansible/requirements.yaml b/ansible/requirements.yaml index 94c98db..517a55b 100644 --- a/ansible/requirements.yaml +++ b/ansible/requirements.yaml @@ -2,15 +2,15 @@ roles: - name: gantsign.oh-my-zsh version: "2.7.0" - name: geerlingguy.docker - version: "6.0.3" + version: "8.0.0" - name: geerlingguy.firewall version: "2.5.0" - src: geerlingguy.pip - version: "3.0.3" + version: "3.1.2" - name: robertdebock.fail2ban version: "4.2.3" - name: gantsign.golang - version: "3.4.0" + version: "3.5.0" collections: - name: ansible.posix diff --git a/setup.sh b/setup.sh index 07d56ca..2d136d7 100755 --- a/setup.sh +++ b/setup.sh @@ -9,9 +9,9 @@ asdf plugin add age https://github.com/threkk/asdf-age.git || true asdf plugin add shellcheck https://github.com/luizm/asdf-shellcheck.git || true asdf plugin add sops https://github.com/feniix/asdf-sops.git || true asdf plugin add terraform https://github.com/asdf-community/asdf-hashicorp.git || true -asdf plugin-add helm https://github.com/Antiarchitect/asdf-helm.git || true -asdf plugin-add python || true -asdf plugin-add yq https://github.com/sudermanjr/asdf-yq.git || true +asdf plugin add helm https://github.com/Antiarchitect/asdf-helm.git || true +asdf plugin add python || true +asdf plugin add yq https://github.com/sudermanjr/asdf-yq.git || true asdf plugin add awscli || true asdf install diff --git a/terraform/devnet-4/ansible_inventory.tmpl b/terraform/devnet-4/ansible_inventory.tmpl new file mode 100644 index 0000000..e038b68 --- /dev/null +++ b/terraform/devnet-4/ansible_inventory.tmpl @@ -0,0 +1,91 @@ +localhost + +[all:vars] +ethereum_network_name=${ethereum_network_name} + +%{ for gid, group in groups ~} +[${replace(gid, "-", "_")}] +%{ for key, host in hosts ~} +%{ if host.group == gid ~} +${host.hostname} ansible_host=${host.ip} ipv6=${host.ipv6} cloud=${host.cloud} cloud_region=${host.region} arch=${host.arch} ethereum_node_cl_supernode_enabled=${title(host.supernode)} %{ if tonumber(host.validator_end) > 0 }validator_start=${host.validator_start} validator_end=${host.validator_end}%{ endif } +%{ endif ~} +%{ endfor ~} +%{ if gid == "lighthouse-reth" ~} +%{ for key, host in hosts ~} +%{ if host.group == "mev-relay" ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} +%{ endif ~} + +%{ endfor ~} + +%{ if !contains(keys(groups), "lighthouse-reth") ~} +[lighthouse_reth] +%{ for key, host in hosts ~} +%{ if host.group == "mev-relay" ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} +%{ endif ~} + +# Consensus client groups + +%{ for cl in ["lighthouse", "lodestar", "nimbus", "teku", "prysm", "grandine"] ~} +[${cl}:children] +%{ for gid, group in groups ~} +%{ if split("-", gid)[0] == "${cl}" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ endfor ~} +%{ if cl == "lighthouse" && contains(keys(groups), "mev-relay") ~} +mev_relay +%{ endif ~} +%{ endfor ~} + +# Execution client groups + +%{ for el in ["besu", "ethereumjs", "geth", "nethermind", "erigon", "reth", "nimbusel", "ethrex"] ~} +[${el}:children] +%{ for gid, group in groups ~} +%{ if split("-", gid)[0] != "bootnode" && split("-", gid)[0] != "mev" ~} +%{ if length(split("-", gid)) >= 2 && split("-", gid)[1] == "${el}" ~} +${replace(gid, "-", "_")} +%{ endif ~} +%{ endif ~} +%{ endfor ~} +%{ if el == "reth" && contains(keys(groups), "mev-relay") ~} +mev_relay +%{ endif ~} +%{ endfor ~} + +# Global groups + +[consensus_node:children] +%{ for x,y in merge( { for gid, group in groups : split("-", gid)[0] => true... if split("-", gid)[0] != "bootnode" && split("-", gid)[0] != "mev" } ) ~} +${x} +%{ endfor ~} + +[execution_node:children] +%{ for x,y in merge( { for gid, group in groups : split("-", gid)[1] => true... if split("-", gid)[0] != "bootnode" && split("-", gid)[0] != "mev" && length(split("-", gid)) >= 2 } ) ~} +${x} +%{ endfor ~} + +[ethereum_node:children] +consensus_node +execution_node + +%{ if contains(keys(groups), "bootnode") ~} +[dns_server:children] +bootnode +%{ endif ~} + +[mev_boost:children] +consensus_node + +[arm] +%{ for key, host in hosts ~} +%{ if can(regex("arm", key)) ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} diff --git a/terraform/devnet-4/cloudflare.tf b/terraform/devnet-4/cloudflare.tf new file mode 100644 index 0000000..67bd810 --- /dev/null +++ b/terraform/devnet-4/cloudflare.tf @@ -0,0 +1,60 @@ + +//////////////////////////////////////////////////////////////////////////////////////// +// DNS NAMES +//////////////////////////////////////////////////////////////////////////////////////// + +data "cloudflare_zone" "default" { + name = "ethpandaops.io" +} + +locals { + # Combine bootnodes from both providers + bootnodes = merge( + { + for vm in local.digitalocean_vms : vm.id => { + name = vm.name + has_ipv6 = vm.ipv6 + ipv4 = digitalocean_droplet.main[vm.id].ipv4_address + ipv6 = try(digitalocean_droplet.main[vm.id].ipv6_address, null) + } if can(regex("bootnode", vm.name)) + }, + { + for vm in local.hcloud_vms : vm.id => { + name = vm.name + has_ipv6 = vm.ipv6_enabled + ipv4 = hcloud_server.main[vm.id].ipv4_address + ipv6 = try(hcloud_server.main[vm.id].ipv6_address, null) + } if can(regex("bootnode", vm.name)) + } + ) +} + +resource "cloudflare_record" "server_record_v4" { + for_each = local.bootnodes + zone_id = data.cloudflare_zone.default.id + name = "${each.value.name}.${var.ethereum_network}" + type = "A" + value = each.value.ipv4 + proxied = false + ttl = 120 +} + +resource "cloudflare_record" "server_record_v6" { + for_each = { for k, v in local.bootnodes : k => v if v.has_ipv6 } + zone_id = data.cloudflare_zone.default.id + name = "${each.value.name}.${var.ethereum_network}" + type = "AAAA" + value = each.value.ipv6 + proxied = false + ttl = 120 +} + +resource "cloudflare_record" "server_record_ns" { + for_each = local.bootnodes + zone_id = data.cloudflare_zone.default.id + name = "srv.${var.ethereum_network}" + type = "NS" + value = "${each.value.name}.${var.ethereum_network}.${data.cloudflare_zone.default.name}" + proxied = false + ttl = 120 +} diff --git a/terraform/devnet-4/digitalocean.tf b/terraform/devnet-4/digitalocean.tf new file mode 100644 index 0000000..66138ac --- /dev/null +++ b/terraform/devnet-4/digitalocean.tf @@ -0,0 +1,167 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// VARIABLES +//////////////////////////////////////////////////////////////////////////////////////// +variable "digitalocean_project_name" { + type = string + default = "Perf" +} + +variable "digitalocean_ssh_key_name" { + type = string + default = "shared-devops-eth2" +} + +variable "digitalocean_supernode_size" { + type = string + default = "s-8vcpu-32gb-640gb-intel" +} + +variable "digitalocean_fullnode_size" { + type = string + default = "s-8vcpu-16gb" +} + +variable "digitalocean_regions" { + default = [ + "nyc1", + "sgp1", + "lon1", + "nyc3", + "ams3", + "fra1", + "tor1", + "blr1", + "sfo3", + "syd1" + ] +} + +//////////////////////////////////////////////////////////////////////////////////////// +// LOCALS +//////////////////////////////////////////////////////////////////////////////////////// +locals { + digitalocean_vpcs = { + for region in var.digitalocean_regions : region => { + name = "${var.ethereum_network}-${region}" + region = region + ip_range = cidrsubnet(var.base_cidr_block, 8, index(var.digitalocean_regions, region)) + } + } +} + +locals { + digitalocean_vm_groups = flatten([ + for node in local.digitalocean_nodes : [ + for i in range(0, node.count) : { + group_name = node.name + id = "${node.name}-${node.start_index + i + 1}" + vms = { + "${i + 1}" = { + # Validator range for this instance + val_start = node.validator_start + (i * (node.validator_end - node.validator_start) / node.count) + val_end = min( + node.validator_start + ((i + 1) * (node.validator_end - node.validator_start) / node.count), + node.validator_end + ) + validator_count = node.count > 0 ? (node.validator_end - node.validator_start) / node.count : 0 + + # Supernode: explicit > bootnode/mev > validator_count >= 128 + supernode = ( + node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) + + region = node.region != null ? node.region : var.digitalocean_regions[i % length(var.digitalocean_regions)] + ipv6 = node.ipv6 + arch = "amd64" + } + } + } + ] + ]) +} + +locals { + digitalocean_default_region = "ams3" + digitalocean_default_size = var.digitalocean_fullnode_size + digitalocean_default_image = "debian-13-x64" + digitalocean_global_tags = [ + "Owner:Devops", + "EthNetwork:${var.ethereum_network}" + ] + + digitalocean_vms = flatten([ + for group in local.digitalocean_vm_groups : [ + for vm_key, vm in group.vms : { + id = group.id + group_key = group.group_name + vm_key = vm_key + + name = group.id + ssh_keys = [data.digitalocean_ssh_key.main.fingerprint] + region = vm.region + image = local.digitalocean_default_image + size = vm.supernode ? var.digitalocean_supernode_size : var.digitalocean_fullnode_size + resize_disk = true + monitoring = true + backups = false + ipv6 = vm.ipv6 + vpc_uuid = digitalocean_vpc.main[vm.region].id + + tags = concat(local.digitalocean_global_tags, [ + "group_name:${group.group_name}", + "val_start:${vm.val_start}", + "val_end:${vm.val_end}", + "supernode:${vm.supernode ? "True" : "False"}", + "arch:${vm.arch}", + ], compact([ + can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null, + can(regex("mev-relay", group.group_name)) ? "mev-relay:${var.ethereum_network}" : null + ])) + } + ] + ]) +} + +//////////////////////////////////////////////////////////////////////////////////////// +// DIGITALOCEAN RESOURCES +//////////////////////////////////////////////////////////////////////////////////////// +data "digitalocean_project" "main" { + name = var.digitalocean_project_name +} + +data "digitalocean_ssh_key" "main" { + name = var.digitalocean_ssh_key_name +} + +resource "digitalocean_vpc" "main" { + for_each = local.digitalocean_vpcs + + name = each.value["name"] + region = each.value["region"] + ip_range = each.value["ip_range"] +} + +resource "digitalocean_droplet" "main" { + for_each = { + for vm in local.digitalocean_vms : vm.id => vm + } + name = "${var.ethereum_network}-${each.value.name}" + region = each.value.region + ssh_keys = each.value.ssh_keys + image = each.value.image + size = each.value.size + resize_disk = each.value.resize_disk + monitoring = each.value.monitoring + backups = each.value.backups + ipv6 = each.value.ipv6 + vpc_uuid = each.value.vpc_uuid + tags = each.value.tags +} + +resource "digitalocean_project_resources" "droplets" { + for_each = digitalocean_droplet.main + project = data.digitalocean_project.main.id + resources = [each.value.urn] +} diff --git a/terraform/devnet-4/firewall.tf b/terraform/devnet-4/firewall.tf new file mode 100644 index 0000000..285b559 --- /dev/null +++ b/terraform/devnet-4/firewall.tf @@ -0,0 +1,330 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// DIGITALOCEAN FIREWALLS +//////////////////////////////////////////////////////////////////////////////////////// +resource "digitalocean_firewall" "main" { + count = length(local.digitalocean_vms) > 0 ? 1 : 0 + name = "${var.ethereum_network}-nodes" + tags = [ + "EthNetwork:${var.ethereum_network}" + ] + + // SSH + inbound_rule { + protocol = "tcp" + port_range = "22" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Allow all inbound ICMP + inbound_rule { + protocol = "icmp" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Nginx / Web + inbound_rule { + protocol = "tcp" + port_range = "80" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "tcp" + port_range = "443" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Consensus layer p2p port + inbound_rule { + protocol = "tcp" + port_range = "9000-9001" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "9000-9002" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Execution layer p2p Port + inbound_rule { + protocol = "tcp" + port_range = "30303" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "30303" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "tcp" + port_range = "42069" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "42069" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Engine rpc-snooper api + inbound_rule { + protocol = "tcp" + port_range = "8961" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Allow all outbound traffic + outbound_rule { + protocol = "tcp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + outbound_rule { + protocol = "udp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + outbound_rule { + protocol = "icmp" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + depends_on = [digitalocean_project_resources.droplets] +} + +resource "digitalocean_firewall" "bootnode" { + count = contains(keys(digitalocean_droplet.main), "bootnode-1") ? 1 : 0 + name = "${var.ethereum_network}-nodes-bootnode" + tags = [ + "bootnode:${var.ethereum_network}" + ] + + // DNS + inbound_rule { + protocol = "tcp" + port_range = "53" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "53" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + // Bootnodoor P2P + inbound_rule { + protocol = "tcp" + port_range = "9010" + source_addresses = ["0.0.0.0/0", "::/0"] + } + inbound_rule { + protocol = "udp" + port_range = "9010" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + depends_on = [digitalocean_project_resources.droplets] +} + +resource "digitalocean_firewall" "mev_relay" { + count = contains(keys(digitalocean_droplet.main), "mev-relay-1") ? 1 : 0 + name = "${var.ethereum_network}-nodes-mev-relay" + tags = ["mev-relay:${var.ethereum_network}"] + + // mev-relay ports + inbound_rule { + protocol = "tcp" + port_range = "9060-9062" + source_addresses = ["0.0.0.0/0", "::/0"] + } + depends_on = [digitalocean_project_resources.droplets] +} + +//////////////////////////////////////////////////////////////////////////////////////// +// HETZNER FIREWALLS +//////////////////////////////////////////////////////////////////////////////////////// +resource "hcloud_firewall" "machine_firewall" { + count = local.hetzner_has_servers ? 1 : 0 + name = "${var.ethereum_network}-firewall" + + apply_to { + label_selector = "EthNetwork=${var.ethereum_network}" + } + + // SSH + rule { + description = "Allow SSH" + direction = "in" + protocol = "tcp" + port = "22" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Allow all inbound ICMP + rule { + description = "Allow all inbound ICMP" + direction = "in" + protocol = "icmp" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Nginx / Web + rule { + description = "Allow HTTP" + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow HTTPS" + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Consensus layer p2p port + rule { + description = "Allow consensus p2p port TCP" + direction = "in" + protocol = "tcp" + port = "9000-9002" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow consensus p2p port UDP" + direction = "in" + protocol = "udp" + port = "9000-9002" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Execution layer p2p Port + rule { + description = "Allow execution p2p port TCP" + direction = "in" + protocol = "tcp" + port = "30303" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow execution p2p port UDP" + direction = "in" + protocol = "udp" + port = "30303" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow execution torrent port TCP" + direction = "in" + protocol = "tcp" + port = "42069" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow execution torrent port UDP" + direction = "in" + protocol = "udp" + port = "42069" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Engine rpc-snooper api + rule { + description = "Allow engine snooper api port TCP" + direction = "in" + protocol = "tcp" + port = "8961" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Allow all outbound traffic + rule { + description = "Allow all outbound traffic TCP" + direction = "out" + protocol = "tcp" + port = "1-65535" + destination_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow all outbound traffic UDP" + direction = "out" + protocol = "udp" + port = "1-65535" + destination_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + description = "Allow all outbound traffic ICMP" + direction = "out" + protocol = "icmp" + destination_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_firewall" "bootnode_firewall" { + count = contains(keys(hcloud_server.main), "bootnode-1") ? 1 : 0 + name = "${var.ethereum_network}-bootnode-firewall" + + apply_to { + label_selector = "bootnode=${var.ethereum_network}" + } + + // DNS + rule { + description = "Allow DNS UDP" + direction = "in" + protocol = "udp" + port = "53" + source_ips = ["0.0.0.0/0", "::/0"] + } + rule { + description = "Allow DNS TCP" + direction = "in" + protocol = "tcp" + port = "53" + source_ips = ["0.0.0.0/0", "::/0"] + } + + // Bootnodoor P2P + rule { + description = "Allow Bootnodoor P2P port TCP" + direction = "in" + protocol = "tcp" + port = "9010" + source_ips = ["0.0.0.0/0", "::/0"] + } + rule { + description = "Allow Bootnodoor P2P port UDP" + direction = "in" + protocol = "udp" + port = "9010" + source_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_firewall" "mev_relay_firewall" { + count = contains(keys(hcloud_server.main), "mev-relay-1") ? 1 : 0 + name = "${var.ethereum_network}-mev-relay-firewall" + + apply_to { + label_selector = "mev=${var.ethereum_network}" + } + + // mev-relay ports + rule { + description = "Allow MEV Relay ports" + direction = "in" + protocol = "tcp" + port = "9060-9062" + source_ips = ["0.0.0.0/0", "::/0"] + } +} diff --git a/terraform/devnet-4/hetzner.tf b/terraform/devnet-4/hetzner.tf new file mode 100644 index 0000000..0fec9d7 --- /dev/null +++ b/terraform/devnet-4/hetzner.tf @@ -0,0 +1,177 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// VARIABLES +//////////////////////////////////////////////////////////////////////////////////////// +variable "hcloud_ssh_key_fingerprint" { + type = string + default = "d6:76:2d:9c:5b:33:80:ff:0f:09:a2:10:9b:58:7e:dc" +} + +variable "hetzner_supernode_size" { + type = string + default = "cax41" +} + +variable "hetzner_fullnode_size" { + type = string + default = "cax31" +} + +variable "hetzner_regions" { + default = [ + "nbg1", + "fsn1", + "hel1" + ] +} + +//////////////////////////////////////////////////////////////////////////////////////// +// LOCALS +//////////////////////////////////////////////////////////////////////////////////////// +locals { + hetzner_has_servers = length(local.hetzner_nodes) > 0 + + hetzner_network = { + for region in var.hetzner_regions : region => { + name = "${var.ethereum_network}-${region}" + ip_range = cidrsubnet(var.base_cidr_block, 8, index(var.hetzner_regions, region)) + } + } + hetzner_network_subnets = { + for region in var.hetzner_regions : region => { + zone = "eu-central" + ip_range = cidrsubnet(var.base_cidr_block, 8, index(var.hetzner_regions, region)) + } + } +} + +locals { + hetzner_vm_groups = flatten([ + for node in local.hetzner_nodes : [ + for i in range(0, node.count) : { + group_name = node.name + id = "${node.name}-${node.start_index + i + 1}" + vms = { + "${i + 1}" = { + # Validator range for this instance + val_start = node.validator_start + (i * (node.validator_end - node.validator_start) / node.count) + val_end = min( + node.validator_start + ((i + 1) * (node.validator_end - node.validator_start) / node.count), + node.validator_end + ) + validator_count = node.count > 0 ? (node.validator_end - node.validator_start) / node.count : 0 + + # Supernode: explicit > bootnode/mev > validator_count >= 128 + supernode = ( + node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) + + # Size: explicit > supernode-based default + size = ( + node.size != null ? node.size : + (node.supernode != null ? node.supernode : + can(regex("(bootnode|mev)", node.name)) ? true : + (node.count > 0 ? (node.validator_end - node.validator_start) / node.count >= 128 : false) + ) ? var.hetzner_supernode_size : var.hetzner_fullnode_size + ) + + location = node.location != null ? node.location : var.hetzner_regions[i % length(var.hetzner_regions)] + ipv4_enabled = node.ipv4_enabled + ipv6_enabled = node.ipv6_enabled + } + } + } + ] + ]) +} + +locals { + hcloud_default_location = "nbg1" + hcloud_default_image = "debian-13" + hcloud_default_server_type = var.hetzner_fullnode_size + hcloud_global_labels = [ + "Owner:Devops", + "EthNetwork:${var.ethereum_network}" + ] + + hcloud_vms = flatten([ + for group in local.hetzner_vm_groups : [ + for vm_key, vm in group.vms : { + id = group.id + group_key = group.group_name + vm_key = vm_key + + name = group.id + ipv4_enabled = vm.ipv4_enabled + ipv6_enabled = vm.ipv6_enabled + ssh_keys = local.hetzner_has_servers ? [data.hcloud_ssh_key.main[0].id] : [] + location = vm.location + image = local.hcloud_default_image + server_type = vm.size + backups = false + + # Architecture: cax* = ARM64, everything else = AMD64 + arch = can(regex("^cax", vm.size)) ? "arm64" : "amd64" + + labels = concat(local.hcloud_global_labels, [ + "group_name:${group.group_name}", + "val_start:${vm.val_start}", + "val_end:${vm.val_end}", + "supernode:${vm.supernode ? "True" : "False"}", + "arch:${can(regex("^cax", vm.size)) ? "arm64" : "amd64"}", + ], compact([ + can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null, + can(regex("mev-relay", group.group_name)) ? "mev:${var.ethereum_network}" : null + ])) + } + ] + ]) +} + +//////////////////////////////////////////////////////////////////////////////////////// +// HETZNER RESOURCES +//////////////////////////////////////////////////////////////////////////////////////// +resource "hcloud_network" "main" { + for_each = local.hetzner_has_servers ? local.hetzner_network : {} + name = each.value.name + ip_range = each.value.ip_range +} + +resource "hcloud_network_subnet" "main" { + for_each = local.hetzner_has_servers ? local.hetzner_network_subnets : {} + network_id = hcloud_network.main[each.key].id + type = "cloud" + network_zone = each.value.zone + ip_range = each.value.ip_range +} + +data "hcloud_ssh_key" "main" { + count = local.hetzner_has_servers ? 1 : 0 + fingerprint = var.hcloud_ssh_key_fingerprint +} + +resource "hcloud_server" "main" { + for_each = { + for vm in local.hcloud_vms : vm.id => vm + } + name = "${var.ethereum_network}-${each.value.name}" + image = each.value.image + server_type = each.value.server_type + location = each.value.location + ssh_keys = each.value.ssh_keys + backups = each.value.backups + labels = { for label in each.value.labels : split(":", label)[0] => split(":", label)[1] } + public_net { + ipv4_enabled = each.value.ipv4_enabled + ipv6_enabled = each.value.ipv6_enabled + } +} + +resource "hcloud_server_network" "main" { + for_each = { + for vm in local.hcloud_vms : vm.id => vm + } + server_id = hcloud_server.main[each.key].id + network_id = hcloud_network.main[each.value.location].id +} diff --git a/terraform/devnet-4/main.tf b/terraform/devnet-4/main.tf new file mode 100644 index 0000000..298e90c --- /dev/null +++ b/terraform/devnet-4/main.tf @@ -0,0 +1,98 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// TERRAFORM PROVIDERS & BACKEND +//////////////////////////////////////////////////////////////////////////////////////// +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.28" + } + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 3.0" + } + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.42.1" + } + } +} + +terraform { + backend "s3" { + skip_credentials_validation = true + skip_metadata_api_check = true + endpoints = { s3 = "https://fra1.digitaloceanspaces.com" } + skip_requesting_account_id = true + skip_s3_checksum = true + region = "us-east-1" + bucket = "merge-testnets" + key = "infrastructure/perf-devnet-4/terraform.tfstate" + } +} + +provider "digitalocean" { + http_retry_max = 20 +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +provider "hcloud" { + token = var.perf_hcloud_token +} + +//////////////////////////////////////////////////////////////////////////////////////// +// VARIABLES +//////////////////////////////////////////////////////////////////////////////////////// +variable "cloudflare_api_token" { + type = string + sensitive = true + description = "Cloudflare API Token" +} + +variable "perf_hcloud_token" { + type = string + sensitive = true + default = "" + description = "Hetzner Cloud API Token (optional if not using Hetzner)" +} + +variable "ethereum_network" { + type = string + default = "perf-devnet-4" +} + +variable "base_cidr_block" { + default = "10.2.0.0/16" +} + +//////////////////////////////////////////////////////////////////////////////////////// +// LOCALS +//////////////////////////////////////////////////////////////////////////////////////// +locals { + # Normalize node entries with defaults and calculate starting index for continuous numbering + nodes_normalized = [ + for idx, node in var.nodes : { + name = node.name + count = node.count + cloud = node.cloud + validator_start = try(node.validator_start, 0) + validator_end = try(node.validator_end, 0) + size = try(node.size, null) + region = try(node.region, null) + location = try(node.location, try(node.region, null)) + supernode = try(node.supernode, null) + ipv6 = try(node.ipv6, true) + ipv4_enabled = try(node.ipv4_enabled, true) + ipv6_enabled = try(node.ipv6_enabled, true) + # Calculate starting index: sum of counts from all previous entries with same name + start_index = sum(concat([for i, n in var.nodes : n.count if i < idx && n.name == node.name], [0])) + } + ] + + # Filter by cloud provider (only nodes with count > 0) + digitalocean_nodes = [for n in local.nodes_normalized : n if n.cloud == "digitalocean" && n.count > 0] + hetzner_nodes = [for n in local.nodes_normalized : n if n.cloud == "hetzner" && n.count > 0] +} diff --git a/terraform/devnet-4/nodes.tf b/terraform/devnet-4/nodes.tf new file mode 100644 index 0000000..7e51823 --- /dev/null +++ b/terraform/devnet-4/nodes.tf @@ -0,0 +1,36 @@ +######################################################################################## +# NODE DEFINITIONS +# +# Define your fleet as a list of node entries. Each entry supports: +# +# Required: +# - name : Node type (e.g., "lighthouse-geth-super", "bootnode") +# - count : Number of instances +# - cloud : "digitalocean" or "hetzner" +# +# Optional: +# - validator_start : First validator index (default: 0) +# - validator_end : Last validator index (default: 0) +# - size : Instance size override (provider-specific) +# - region : Region override (digitalocean) or location (hetzner) +# - supernode : Force supernode=true/false (auto-detected from name) +# +# Examples: +# { name = "bootnode", count = 1, cloud = "digitalocean" } +# { name = "lighthouse-geth-super", count = 2, cloud = "hetzner", validator_start = 0, validator_end = 200 } +# { name = "mev-relay", count = 1, cloud = "hetzner", size = "ccx53" } +# +######################################################################################## + +variable "nodes" { + description = "List of node definitions for the devnet" + default = [ + { name = "bootnode", count = 1, cloud = "hetzner" }, + { name = "lighthouse-geth", count = 1, cloud = "hetzner", validator_start = 0, validator_end = 100 }, + { name = "lighthouse-nethermind", count = 1, cloud = "hetzner", validator_start = 100, validator_end = 200 }, + { name = "lighthouse-reth", count = 1, cloud = "hetzner", validator_start = 200, validator_end = 300 }, + { name = "prysm-geth", count = 1, cloud = "hetzner", validator_start = 300, validator_end = 400 }, + { name = "nimbus-besu", count = 1, cloud = "hetzner", validator_start = 400, validator_end = 500 }, + { name = "teku-erigon", count = 1, cloud = "hetzner", validator_start = 500, validator_end = 600 }, + ] +} diff --git a/terraform/devnet-4/outputs.tf b/terraform/devnet-4/outputs.tf new file mode 100644 index 0000000..eca43ab --- /dev/null +++ b/terraform/devnet-4/outputs.tf @@ -0,0 +1,118 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// GENERATED FILES AND OUTPUTS +//////////////////////////////////////////////////////////////////////////////////////// + +resource "local_file" "ansible_inventory" { + content = templatefile("ansible_inventory.tmpl", + { + ethereum_network_name = "${var.ethereum_network}" + groups = merge( + { for group in local.digitalocean_vm_groups : "${group.group_name}" => true... }, + { for group in local.hetzner_vm_groups : "${group.group_name}" => true... }, + ) + hosts = merge( + { + for key, server in digitalocean_droplet.main : "do.${key}" => { + ip = "${server.ipv4_address}" + ipv6 = try(server.ipv6_address, "none") + group = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^group_name:", tag))][0], "unknown") + validator_start = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^val_start:", tag))][0], 0) + validator_end = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^val_end:", tag))][0], 0) + supernode = try(title([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^supernode:", tag))][0]), "undefined") + arch = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^arch:", tag))][0], "amd64") + tags = "${server.tags}" + hostname = "${split(".", key)[0]}" + cloud = "digitalocean" + region = "${server.region}" + } + }, + { + for key, server in hcloud_server.main : "${key}" => { + ip = coalesce(server.ipv4_address, (try(server.ipv6_address, ""))) + ipv6 = coalesce(server.ipv6_address, "") + group = server.labels.group_name + validator_start = server.labels.val_start + validator_end = server.labels.val_end + supernode = server.labels.supernode + arch = server.labels.arch + tags = server.labels + hostname = split(".", key)[0] + cloud = "hetzner" + region = server.datacenter + } + } + ) + } + ) + filename = "../../ansible/inventories/devnet-4/inventory.ini" +} + +locals { + ssh_config_path = pathexpand("~/.ssh/config.d/ssh_config.${var.ethereum_network}") +} + +resource "local_file" "ssh_config" { + content = templatefile("${path.module}/ssh_config.tmpl", + { + ethereum_network = var.ethereum_network + hosts = merge( + { + for key, server in digitalocean_droplet.main : "${var.ethereum_network}-${key}" => { + hostname = server.ipv4_address + private_ip = server.ipv4_address_private + name = key + user = "devops" + } + }, + { + for key, server in hcloud_server.main : "${var.ethereum_network}-${key}" => { + hostname = coalesce(server.ipv4_address, (try(server.ipv6_address, ""))) + private_ip = try(hcloud_server_network.main[key].ip, "") + name = key + user = "devops" + } + } + ) + } + ) + filename = local.ssh_config_path + + depends_on = [digitalocean_droplet.main, hcloud_server.main] + + lifecycle { + create_before_destroy = true + } +} + +resource "null_resource" "ssh_config_cleanup" { + triggers = { + ssh_config_path = local.ssh_config_path + } + + provisioner "local-exec" { + when = destroy + command = "rm -f ${self.triggers.ssh_config_path} || true" + } + + depends_on = [local_file.ssh_config] +} + +output "ssh_config_file" { + value = "SSH config generated at: ${local.ssh_config_path}" + description = "Path to the generated SSH config file" +} + +output "digitalocean_server_count" { + value = length(digitalocean_droplet.main) + description = "Number of DigitalOcean servers created" +} + +output "hetzner_server_count" { + value = length(hcloud_server.main) + description = "Number of Hetzner servers created" +} + +output "total_server_count" { + value = length(digitalocean_droplet.main) + length(hcloud_server.main) + description = "Total number of servers created across all providers" +} diff --git a/terraform/devnet-4/ssh_config.tmpl b/terraform/devnet-4/ssh_config.tmpl new file mode 100644 index 0000000..d2394b4 --- /dev/null +++ b/terraform/devnet-4/ssh_config.tmpl @@ -0,0 +1,16 @@ +# SSH Config for ${ethereum_network} devnet +# Generated by Terraform +# Usage: Include this file in your ~/.ssh/config with: +# Include /path/to/this/ssh_config +# Or copy entries to your main SSH config file + +%{ for host_key, host in hosts ~} +Host ${host_key} + HostName ${host.hostname} + User ${host.user} + Port 22 + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + LogLevel ERROR + +%{ endfor ~}