Feature/registry drone roles

bin/create_gandi_vm.sh

@@ -40,3 +40,6 @@ gandi vm create --memory 1024 --hostname worker1 --image "Debian 8" --size 10G -
 # SMTP
 # vlan ip range for smtp services : 192.168.1.32/29
 gandi vm create --memory 2048 --hostname mail1 --image "Debian 8" --size 20G --datacenter FR-SD3 --vlan alpha_vlan --ip 192.168.1.33 --ip-version 4
+
+# Registry
+gandi vm create --memory 2048 --hostname registry1 --image "Debian 8" --size 50G --datacenter FR-SD3 --ip-version 4

deploy-registry.yaml

@@ -0,0 +1,7 @@
+- hosts: registry
+  roles:
+    - common-no-vlan
+    - nginx
+    - docker
+    - registry
+    - drone

hosts.template

@@ -4,12 +4,14 @@ caliopen
 gateway
 storage
 smtp
+citools
 
 [services:vars]
 dist_directory=./dist
 object_store_access_key=SZ1BBGKTD2N13E0W5L8N
 object_store_secret_key=qTsjiThBQA2NH6ZO32tCwCC6wcC8ValVLR16XUsB
-caliopen_domain_name=alpha.caliopen.org
+caliopen_domain_base=caliopen.org
+caliopen_domain_name=alpha.{{ caliopen_domain_base }}
 caliopen_nameservers=["155.133.128.67", "155.133.128.65"]
 
 # Vault
@@ -20,6 +22,20 @@ vault_worker_password=TO_BE_DEFINED
 vault_cert_path=/etc/vault/alpha.caliopen.org.crt
 vault_key_path=/etc/vault/alpha.caliopen.org.key
 
+# Docker registry
+registry_path=/etc/docker-registry
+
+# Drone
+drone_path=/etc/drone
+# Github OAuth
+DRONE_GITHUB_CLIENT=
+DRONE_GITHUB_SECRET=
+# Agent/Server communication
+DRONE_SECRET=this_should_be_a_secret
+# List of admins, Github usernames
+DRONE_ADMIN=
+DRONE_HOST=drone.{{ caliopen_domain_base }}
+
 # Version of installed software out of host packaging
 
 # monitoring platform
@@ -61,6 +77,9 @@ cache
 mq
 object_store
 
+[citools:children]
+registry
+
 [store]
 store1 ansible_host=ip_store1 ansible_user=root backend_ip=backend_store1
 store2 ansible_host=ip_store2 ansible_user=root backend_ip=backend_store2
@@ -107,3 +126,6 @@ mail1 ansible_host=ip_mail1 ansible_user=root backend_ip=backend_mail1
 
 [logstash]
 logstash1 ansible_host=ip_logstash1 ansible_user=root backend_ip=backend_logstash1
+
+[registry]
+registry1 ansible_host=ip_registry1 ansible_user=root backend_ip=backend_registry1

roles/common-no-vlan/files/ssh_authorized_keys

@@ -0,0 +1,8 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5+2ViaP1ktWlzMCY4IOJOV1K0TH1GstHzoMdeIf9ihiSz7nR7wKcYJMC6KlmOYVQzftENXHQZAtbL4tVPLpLWXN+4fCn+pbQVu47P3QCH9Ez0d23p4byZl5h+qyx0dJv/ltc71X6NIvHH2WXmvvy+Bda4b1NVpJN/voiMoihipsjPPeL+s6B+3dw6PD3h5vvzvJCrfkKGijoT74+BbjYimwmNsaDRQH9tIMaTVeV7ZIe9qfxg5fkg4WsFl9mzikbqYzdBgiC2XeK/L4w3FJONALAEy7FTsUdNaenKxTn4zw/9qdV20TqYEyCbYlANS+2NMLYxeSqdpYB3yvePoucOw== mric@gandi.net
+ssh-dss AAAAB3NzaC1kc3MAAACBAM686CNkUeMiHvr/1tj4zRaJMqAgZAFCuX6WmocNHleTLG2yWcQPAIXKONp++AJ78woEERCTB2otJSsP4Ur8q/K95UiPYmtRJ/wwTI4ojrCk4BmK9KK2hb0OONOL0SvX/sUZlddFtAZ2xnSFD6YC4gtANE1nnojo2/BOrgs9h13tAAAAFQCkqnmRZOK29LK8OPI+095IzI0YMQAAAIAf3BB/TX2mZWGtB9PivKybt+QPMx5YWA43jK6NippTIVq60ihvcnVKpAQDt0llZn4J5qoEgVHwELr+4F6vMz2HP3ZviQ3c/4hlIpfknVsFLgMkJynKZaJLTe+Afwv1r+8DAA2+/SvtwLjFIDcbkTgGdxiyInD8rDyprKQ7nI3sNwAAAIABuUMiFMmpkARmatAJoXjFm2V1JIyycuJdMqJMUoq9m7kjJB4r55+eTLEtIvtBs/LnlAUTl2kCQszEax4VlLGiEEH/hWryaePRuosEv1issiISiluJmIQcJU+vgAHApyGH6uVCWzoc58or5rnQto22MEcH/qHIggTuKIfQvz8Hhg== esion99@gmail.com
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCzWy1ckqww3CP8BPB56tfHwN7Wh+/4dpByhwTpb5lXpfjNvI3KNaz3835oa/cHvxt34alBHdhG65eUQOYOX3frXGEPxFPCjADPBh0+WAa00IpFLCyvsVxOutBU9Fu5eGVVcmIC5gq4M21zUppbiFzppc/7a8W/xz4W17LzuqzRRaerz5WcYNE5uf+arqW2zrmWA5Nhwjo2C1Q0qB8a+6Nvg14OAKyYL8P2eTUmzmK+Dv3oCwgqLZalx1djjmFv3N6SWdOLK92Jq3/b6Xb9RHCALP1/wnVtmzKFaCtspBpDs9eU6f6G4hP67sRobimMWdB/EibLlLDw3Sul6j6CK6Qd8QMJwR5cU2+lmSQXlwlvRmgvhayPUNJEwR0nY3uiHoybJzClF3LOljn7RkqBUY6ud3y1L1OUbJvUYw9ou6gd61HGqXFjuD8hLDRBdCaAzlPm1Wm6eAY01bLOcKcXdqNJOoSh/ZKAdT5VvmPZgbit9c5OitujNGs3wkI2O+DydMa/UBJbdzQdh+QZLVFzQ1qO2BPkQCQ2TZBad0YqtlTxGiB+cEwIx4dkYfJcbWcrVZs04g24otMQOnb+1KcISKcqVR/qOWvFzaBMa8uXu6JoDE2qL6R1q+uls+gMFFIKoy16C3skg7jxydlgXEfiURWNy3RkIEgMSEkGz9dxDv8UAQ== david.epely@gandi.net
+ssh-dss AAAAB3NzaC1kc3MAAACBAMZ50GLnHEvUFvMQsHBRdlUka0ikKOmpTt7etALYXfDSsR6ItF1r3evY74bDGrQjD6SwOHiwfnDsUH6pw8OydHH17803JBhUjDe2/NEhmaAqOmI+sLTknHVJZwDEVLhvXBaFTQcT4qEGgG0XYxM+EZ7vBirodzvGwBpwXnmMlNA3AAAAFQDVlmYkBh8rhBbJwZpmWGaGDPjFBQAAAIEAl4shqD8f64XTqcaJ456j5vghvElyPN1T8JoqlUg7iyzFeRnHlcGZkN3nw+aB1DVHIRIBA5hGk0TOCXgGRNkcgtDn6Wj9R0A2Wow/reFFPjio+NIM5YaNVqUPirFMWOO+cVhCaN3RvIHjWzGM+oQjk2LGxpN/Tzh2UyF7O3FLU6MAAACAcH39fUlEERGhrfx6cyKv7jjoChOsHKc/Wix3QyU7jq7Ta5iVjOGBOR1nYL5JKJWnEhIx/g5N+NAuvvzLIe/x4vlFo8PLk0obCXDeOsucuwWpcCldYE3GqTJZzxnCOIkdjooszQhztmxe1bBP/Aib7KyogaJmKrQv6BOjWQhx/TE= laurent@brasil
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBPQJeSOFDn9N73xOkJvWS97CvGQLarKI6n2kaA4cLzx root@argentina
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCm70qHMJSqaQKJdoNOzrfCo+1pzlhqHIFBozI5VCiMwA6Nr2qEHD+VgSbhDByX0xdgv9cwIcxvVzWMZPn8QM2ZewvNgPHoQxH4ae3pWjTM+W8qqaNjBfWuarinwt7gO8jT8i55AcMa5ctihvXWE3jTM6EHcaKTngFD1NYFj5tS4Zrw9a5nK1ZRsMrPF6Wte9S3e2PWiPYiT8uCauNUB5Xi6r1BxzMtviJddZmv0r4WQL3QD672Gmia6xhIybiIFTOID+N4cAARKZKh7WSlcx4qA1umWLd1nst5HgyK4SfFhSPd+2XJLsPc1cZpVVfjJRGomLi7yxu4P8VMaKwwCiuj stan@BobyLap
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdqFMRQx+OI+3b8st+ho1Ioqp3wiQqqMlMKiNjDC1rUiQ5sGhGT1uxBzxWSA8UjyfJmYaJhllfe5Tjp2D1lUyXX2tX0QCMM1doHON/29wjBXxBgtP7i5focNAv6KP2suSuyFuIRpP3MuEyieQgyH0atL1FxNpQIrvnOrdiw609T4xfLTWfad+jjtIo3qq3Rvb7TpI9h0lBcgJEHPSjsapYenFPNCaRE+3oye37OtYdWaF9ozdHkRBDj8mp23bitJSwltYOhYZlVs8fVyBr30+z4tSwNMizl7DCrr+rJFBCRwoHUOLo82LuJf1ivQwu3mC77JJgWsiycMYnKPOamDwv pablo@pablo
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDTHau8idcljL+CRudl/i0HPzKbTxYk+qSJg26ql/K+BJF5uS360jYt9MW+C4qLSfYBtY0Ywcb7ePeO+1I/BjAdf6BtzxnNCYDnKptPMlkq9E0caMcDuuYFNG8BZQ6HcVmIbCns6aa0cXRGZcIBeHFAWfveocPdpcnTOXbFItV4ndFhm+KsCaJ9uQUxmJZMuUYoA/mmIgizddevh+bWMyN2/ntLhCvXucKti79sUoGVo5Ihk4KKXYxmZkWMJeY6y72TMeMx/KOJuoI5bkaLl/Y7oF8g7gzghmF56yI0uev83CyY1Mi+nF+qYqcjA9W9UVzx/xlkt3vhVyQ0C7hywoW1SPultSOr2ovtaEEFG16phOhWCNTwc1fUBwDv+UHHSgDjUQw1XBeMpTyW3d0Kys6bqvdisT9Z8n364BNDs2wC5VpctOLMyWXOt7stql56625fjMTGZYxD/+b4IvyW+qP9JRSjeeqb03PXoT+45V57s6UxJpIihjZSrEqJ1ZXAD0tLv3Fsj1KHNQi8da+P0ph33XeplCGPhRBohiL3obxWSDZ/WX0MaX4tgICqDHAkSGrDJkeF8dgNAtO2+m44oty6U/ztnuSKQGBXACxbpS+M8YfPFTyBkjrOtfqk0R/MgdG2gxvPyYPG3ody20QcYzTbG9LrtOReN58kPt4D/3Mp2w== dev@lanza.fr

roles/common-no-vlan/tasks/main.yml

@@ -0,0 +1,7 @@
+- name: install ssh authorized keys
+  copy: src=ssh_authorized_keys dest=/root/.ssh/authorized_keys mode=0600
+  tags:
+    - ssh
+
+- name: upgrade packages
+  shell: apt-get upgrade -y

roles/docker/tasks/main.yml

@@ -0,0 +1,35 @@
+- name: install dependencies
+  apt: package={{ item }} state=installed update_cache=yes
+  with_items:
+    - apt-transport-https
+    - curl
+    - ca-certificates
+    - gnupg2
+    - software-properties-common
+
+- name: add apt key
+  apt_key:
+    url: https://download.docker.com/linux/debian/gpg
+    state: present
+
+- name: Add docker apt repo
+  apt_repository:
+    repo: "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
+    filename: docker
+    state: present
+
+- name: install docker ce
+  apt: package={{ item }} update_cache=yes
+  with_items:
+    - docker-ce
+
+- name: start docker
+  service:
+    name: docker
+    state: restarted
+
+- name: install docker compose
+  get_url:
+    url: https://github.com/docker/compose/releases/download/1.22.0/docker-compose-Linux-x86_64
+    dest: /usr/local/bin/docker-compose
+    mode: 0550

roles/drone/tasks/main.yml

@@ -0,0 +1,20 @@
+- name: create drone directory
+  file:
+    path: "{{ drone_path }}"
+    state: directory
+
+- name: copy compose file for registry server
+  template: src=docker-compose.yml.j2 dest={{ drone_path }}/docker-compose.yml
+
+- name: start drone
+  shell: docker-compose -f {{ drone_path }}/docker-compose.yml up -d
+
+- name: configure nginx vhost
+  template:
+    src: drone.nginx.j2
+    dest: /etc/nginx/sites-enabled/drone
+
+- name: restart service nginx
+  service:
+    name: nginx
+    state: restarted

roles/drone/templates/docker-compose.yml.j2

@@ -0,0 +1,33 @@
+version: '2'
+
+services:
+
+  drone-server:
+    image: drone/drone
+    ports:
+      - 80:8000
+      - 9000
+    volumes:
+      - ./drone/:/var/lib/drone/
+    restart: always
+    environment:
+      - DRONE_OPEN=true
+      - DRONE_HOST={{ DRONE_HOST }}
+      - DRONE_GITHUB=true
+      - DRONE_ORGS=CaliOpen
+      - DRONE_GITHUB_CLIENT={{ DRONE_GITHUB_CLIENT }}
+      - DRONE_GITHUB_SECRET={{ DRONE_GITHUB_SECRET }}
+      - DRONE_SECRET={{ DRONE_SECRET }}
+      - DRONE_ADMIN={{ DRONE_ADMIN }}
+
+  drone-agent:
+    image: drone/agent
+    command: agent
+    restart: always
+    depends_on:
+      - drone-server
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      - DRONE_SERVER=drone-server:9000
+      - DRONE_SECRET={{ DRONE_SECRET }}

roles/drone/templates/drone.nginx.j2

@@ -0,0 +1,29 @@
+upstream drone {
+    server 127.0.0.1:8000;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name drone.{{ caliopen_domain_base }};
+
+    ssl_certificate     /etc/nginx/certs/{{ caliopen_domain_base }}.crt;
+    ssl_certificate_key /etc/nginx/certs/{{ caliopen_domain_base }}.key;
+    ssl_prefer_server_ciphers On;
+    ssl_protocols TLSv1.1 TLSv1.2;
+    ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS;
+    ssl_session_cache shared:SSL:10m;
+
+    location / {
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $http_host;
+
+        proxy_pass http://drone;
+        proxy_redirect off;
+        proxy_http_version 1.1;
+        proxy_buffering off;
+
+        chunked_transfer_encoding off;
+    }
+}

roles/nginx/tasks/main.yml

@@ -15,15 +15,19 @@
   with_items:
     - "{{ caliopen_domain_name }}.crt"
     - "{{ caliopen_domain_name }}.key"
+    - "{{ caliopen_domain_base }}.crt"
+    - "{{ caliopen_domain_base }}.key"
 
 - name: install prometheus nginx metric exporter
   git:
     repo: "https://github.com/knyar/nginx-lua-prometheus.git"
     dest: /srv/nginx-lua-prometheus
+  tags: monitoring
 
 - name: configure lua
   template: src=lua.conf.j2 dest=/etc/nginx/conf.d/lua.conf
   notify: restart nginx
+  tags: monitoring
 
 - name: remove default nginx site
   file:

roles/registry/files/docker-compose.yml

@@ -0,0 +1,10 @@
+version: '2'
+
+services:
+
+  registry:
+    image: registry:2
+    ports:
+      - 127.0.0.1:5000:5000
+    volumes:
+      - ./data:/var/lib/registry

roles/registry/files/registry.htpasswd

@@ -0,0 +1,9 @@
+$ANSIBLE_VAULT;1.1;AES256
+38343563333862323230616439303037656531306339656132306539616132336336306639633435
+6135323263643732326538376531323234626235303935660a366437333130323531333765343965
+34363962323665386161633939613337663334616266646235663064303965623062333663636162
+6162386564383466320a613162383438303131336566336163376637363465653264643038646364
+38396436393663343432333830333236383433633361393638393433383563633437666137383132
+62623633616639653832653235643665323734393137636331613065616461313131316339396531
+31303030656564383632643237363130353664643233313137303632396465323962363638383436
+66306230373632383730

roles/registry/tasks/main.yml

@@ -0,0 +1,30 @@
+- name: create docker-registry directory
+  file:
+    path: "{{ registry_path }}"
+    state: directory
+
+- name: copy compose file for registry server
+  copy: src=docker-compose.yml dest={{ registry_path }}/docker-compose.yml
+
+- name: start docker-registry
+  shell: docker-compose -f {{ registry_path }}/docker-compose.yml up -d
+
+- name: create auth files directory
+  file:
+    path: /etc/nginx/auth
+    state: directory
+
+- name: copy registry pass file
+  copy:
+    src: registry.htpasswd
+    dest: /etc/nginx/auth/registry.htpasswd
+
+- name: configure nginx vhost
+  template:
+    src: docker-registry.nginx.j2
+    dest: /etc/nginx/sites-enabled/docker-registry 
+
+- name: restart service nginx
+  service:
+    name: nginx
+    state: restarted

roles/registry/templates/docker-registry.nginx.j2

@@ -0,0 +1,76 @@
+upstream docker-registry {
+  server 127.0.0.1:5000;
+}
+
+map $upstream_http_docker_distribution_api_version $docker_distribution_api_version {
+  '' 'registry/2.0';
+}
+
+server {
+  listen 443 ssl;
+  listen [::]:443 ssl;
+  server_name registry.{{ caliopen_domain_base }};
+
+  ssl_certificate     /etc/nginx/certs/{{ caliopen_domain_base }}.crt;
+  ssl_certificate_key /etc/nginx/certs/{{ caliopen_domain_base }}.key;
+  ssl_prefer_server_ciphers On;
+  ssl_protocols TLSv1.1 TLSv1.2;
+  ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS;
+  ssl_session_cache shared:SSL:10m;
+
+  client_max_body_size 0;
+  chunked_transfer_encoding on;
+
+  location /v2/ {
+
+    if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) {
+      return 404;
+    }
+
+    auth_basic "Registry realm";
+    auth_basic_user_file /etc/nginx/auth/registry.htpasswd;
+
+    add_header 'Docker-Distribution-Api-Version' $docker_distribution_api_version always;
+
+    proxy_pass http://docker-registry;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_read_timeout 900;
+  }
+}
+
+server {
+  listen 443 ssl;
+  listen [::]:443 ssl;
+  server_name public-registry.{{ caliopen_domain_base }};
+
+  ssl_certificate     /etc/nginx/certs/{{ caliopen_domain_base }}.crt;
+  ssl_certificate_key /etc/nginx/certs/{{ caliopen_domain_base }}.key;
+  ssl_prefer_server_ciphers On;
+  ssl_protocols TLSv1.1 TLSv1.2;
+  ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS;
+  ssl_session_cache shared:SSL:10m;
+
+  client_max_body_size 0;
+  chunked_transfer_encoding on;
+
+  if ($request_method !~ ^(GET|HEAD)$ ) {
+    return 444;
+  }
+
+  location /v2/ {
+
+    if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) {
+      return 404;
+    }
+
+    auth_basic off;
+
+    add_header 'Docker-Distribution-Api-Version' $docker_distribution_api_version always;
+
+    proxy_pass http://docker-registry;
+    proxy_read_timeout 900;
+  }
+}