Files, scripts and documentation for the LDCS system

maintenance/README.md

@@ -0,0 +1,14 @@
+#
+#  LDCS System maintenance files
+#
+
+The maintenance folder contains files that are used to 
+maintain the LDCS system. 
+
+  * `README.md` - this file
+  * `configFiles/` - configuration files for the various subsystems
+  * `docker/` - docker files to bring up the various systems
+  * `tools/` - scripts and tools for various purposes
+  * `docs/` - quick reference documentation
+
+

maintenance/configFiles/rucio-server/README.md

@@ -0,0 +1,5 @@
+Each configuration file in this folder should contain a comment stating where
+the file should be located in the target service.
+
+Note that default passwords are in cleartext. A production setup should at
+least create a new password.

maintenance/configFiles/rucio-server/alembic.ini

@@ -0,0 +1,68 @@
+# Copyright European Organization for Nuclear Research (CERN)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Authors:
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2014
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2019
+
+# A generic, single database configuration.
+
+# LDCS: this file should be placed in
+# /opt/rucio/etc
+
+[alembic]
+# path to migration scripts
+script_location = /usr/lib/python2.7/site-packages/rucio/db/sqla/migrate_repo/
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+sqlalchemy.url = postgresql://rucio:secret@ruciodb/rucio
+version_table_schema = rucio
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
+

maintenance/configFiles/rucio-server/aliases.conf

@@ -0,0 +1,27 @@
+# LDCS: this file should be placed in
+# /opt/rucio/etc
+
+WSGIScriptAlias /ping                   /usr/lib/python2.7/site-packages/rucio/web/rest/ping.py
+WSGIScriptAlias /accounts               /usr/lib/python2.7/site-packages/rucio/web/rest/account.py
+WSGIScriptAlias /accountlimits          /usr/lib/python2.7/site-packages/rucio/web/rest/account_limit.py
+#WSGIScriptAlias /auth                   /usr/lib/python2.7/site-packages/rucio/web/rest/authentication.py process-group=rucio application-group=rucio
+WSGIScriptAlias /auth                   /usr/lib/python2.7/site-packages/rucio/web/rest/authentication.py
+WSGIScriptAlias /config                 /usr/lib/python2.7/site-packages/rucio/web/rest/config.py
+WSGIScriptAlias /dids                   /usr/lib/python2.7/site-packages/rucio/web/rest/did.py
+WSGIScriptAlias /export                 /usr/lib/python2.7/site-packages/rucio/web/rest/exporter.py
+WSGIScriptAlias /identities             /usr/lib/python2.7/site-packages/rucio/web/rest/identity.py
+WSGIScriptAlias /import                 /usr/lib/python2.7/site-packages/rucio/web/rest/importer.py
+WSGIScriptAlias /heartbeats             /usr/lib/python2.7/site-packages/rucio/web/rest/heartbeat.py
+WSGIScriptAlias /locks                  /usr/lib/python2.7/site-packages/rucio/web/rest/lock.py
+WSGIScriptAlias /meta                   /usr/lib/python2.7/site-packages/rucio/web/rest/meta.py
+WSGIScriptAlias /ping                   /usr/lib/python2.7/site-packages/rucio/web/rest/ping.py
+WSGIScriptAlias /redirect               /usr/lib/python2.7/site-packages/rucio/web/rest/redirect.py
+WSGIScriptAlias /replicas               /usr/lib/python2.7/site-packages/rucio/web/rest/replica.py
+WSGIScriptAlias /requests               /usr/lib/python2.7/site-packages/rucio/web/rest/request.py
+WSGIScriptAlias /rses                   /usr/lib/python2.7/site-packages/rucio/web/rest/rse.py
+WSGIScriptAlias /rules                  /usr/lib/python2.7/site-packages/rucio/web/rest/rule.py
+WSGIScriptAlias /scopes                 /usr/lib/python2.7/site-packages/rucio/web/rest/scope.py
+WSGIScriptAlias /subscriptions          /usr/lib/python2.7/site-packages/rucio/web/rest/subscription.py
+WSGIScriptAlias /objectstores           /usr/lib/python2.7/site-packages/rucio/web/rest/objectstore.py
+WSGIScriptAlias /lifetime_exceptions    /usr/lib/python2.7/site-packages/rucio/web/rest/lifetime_exception.py
+

maintenance/configFiles/rucio-server/rucio.cfg

@@ -0,0 +1,51 @@
+# Copyright European Organization for Nuclear Research (CERN)
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Authors:
+# - Vincent Garonne, <vincent.garonne@cern.ch>, 2013
+
+# LDCS: this file should be placed in
+# /opt/rucio/etc/
+
+[common]
+logdir = /var/log/rucio
+loglevel = WARNING
+mailtemplatedir = /opt/rucio/etc/mail_templates
+[api]
+endpoints=accountlimits,accounts,auth,config,credentials,dids,export,heartbeats,identities,import,lifetime_exceptions,locks,meta,ping,redirect,replicas,requests,rses,rules,scopes,subscriptions
+[database]
+default = postgresql://rucio:secret@ruciodb/rucio
+pool_reset_on_return = rollback
+echo = 0
+pool_recycle = 600
+[bootstrap]
+userpass_identity = ddmlab
+userpass_pwd = secret
+userpass_email = rucio-dev@cern.ch
+[alembic]
+cfg = /opt/rucio/etc/alembic.ini
+[monitor]
+carbon_server = localhost
+carbon_port = 8125
+user_scope = default_docker
+[trace]
+tracedir = /var/log/rucio/trace
+brokers = localhost
+port = 61013
+topic = /topic/rucio.tracer
+[nongrid_trace]
+tracedir = /var/log/rucio/trace
+brokers = localhost
+port = 61013
+topic = /topic/rucio.tracer
+[policy]
+permission = generic
+schema = generic
+lfn2pfn_algorithm_default = hash
+support = https://github.com/rucio/rucio/issues/
+support_rucio = https://github.com/rucio/rucio/issues/
+[webui]
+usercert = /opt/rucio/etc/usercert_with_key.pem
+

maintenance/docker/LDCS-rucio/README.md

@@ -0,0 +1,38 @@
+# Docker compose file for rucio server
+
+1) Copy all these files in a folder such as
+
+```
+   cp -r ../LDCS_rucio ~
+   cd ~/LDCS_rucio
+```
+
+2) Add values to the variables in `env` and rename the file to .env
+
+```  
+   mv env .env
+```
+
+3) Switch to root (mainly for access to port 443)
+
+```
+   sudo -s
+```
+
+4) Deploy and edit the relevant configuration files in ../configFiles. Create folders if needed.
+Make sure passwords and paths match between the configFiles and the docker-compose file
+
+5) Start all the services with docker-compose
+
+```
+docker compose -f docker-compose-rucioserver-postgres-daemons.yml up -d
+
+```
+6) To stop all the services:
+```
+docker compose -f docker-compose-rucioserver-postgres-daemons.yml down
+
+```
+
+Refer to docker compose documentation for further information on how to interact with each single service.
+

maintenance/docker/LDCS-rucio/docker-compose-rucioserver-postgres-daemons.yml

@@ -0,0 +1,65 @@
+services:
+  rucio-server:
+    image: rucio/rucio-server:release-1.21.6
+    hostname: ${LDCS_RUCIO_FQDN}
+    ports:
+      - "443:443"
+    volumes:
+      - /etc/grid-security/:/etc/grid-security/
+#      - /etc/grid-security/hostcert.pem:/etc/grid-security/hostcert.pem
+#      - /etc/grid-security/hostkey.pem:/etc/grid-security/hostkey.pem
+      - /opt/rucio/etc:/opt/rucio/etc
+      # comment out the following to use docker compose logs only. This logs to the host system, when we have space
+      - /var/log/rucio:/var/log/rucio
+    environment:
+      - RUCIO_ENABLE_SSL=True
+      - RUCIO_HOSTNAME=${LDCS_RUCIO_FQDN}
+      - RUCIO_CFG_DATABASE_DEFAULT=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@ruciodb/${POSTGRES_DB}
+      - RUCIO_CA_PATH=/etc/grid-security/certificates
+      - RUCIO_ENABLE_LOGFILE=True
+      - RUCIO_DEFINE_ALIASES=True
+  ruciodb:
+    image: postgres:11
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    ports:
+      - "5432:5432"
+    command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"]
+    volumes:
+      - vol-ruciodb-data:/var/lib/postgresql/data
+   rucio-daemons-abacus:
+     image: rucio/rucio-daemons:release-1.21.6
+     volumes:
+       - /opt/rucio/etc/rucio.cfg:/opt/rucio/etc/rucio.cfg
+       # comment out the following to use docker compose logs only. This logs to the host system, when we have space
+       - /var/log/rucio:/var/log/rucio
+     environment:
+       - RUCIO_DAEMON=abacus-rse
+       - RUCIO_ENABLE_LOGS=True
+   rucio-daemons-reaper:
+     image: rucio/rucio-daemons:release-1.21.6
+     volumes:
+       - /opt/rucio/etc/rucio.cfg:/opt/rucio/etc/rucio.cfg
+       - /home/almalinux/act/ldmx.long.proxy.root:/opt/rucio/etc/x509
+       - /etc/grid-security/certificates:/etc/grid-security/certificates
+       # comment out the following to use docker compose logs only. This logs to the host system, when we have space
+       - /var/log/rucio:/var/log/rucio
+     environment:
+       - RUCIO_DAEMON=reaper
+       - RUCIO_ENABLE_LOGS=True 
+       - X509_USER_PROXY=/opt/rucio/etc/x509
+   rucio-daemons-undertaker:
+     image: rucio/rucio-daemons:release-1.21.6
+     volumes:
+       - /opt/rucio/etc/rucio.cfg:/opt/rucio/etc/rucio.cfg
+       # comment out the following to use docker compose logs only. This logs to the host system, when we have space
+       - /var/log/rucio:/var/log/rucio
+     environment:
+      - RUCIO_DAEMON=undertaker
+      - RUCIO_ENABLE_LOGS=True
+
+volumes:
+  vol-ruciodb-data:
+

maintenance/docker/LDCS-rucio/env

@@ -0,0 +1,7 @@
+# fill the blanks below and rename this file to .env for docker to parse it
+# it must be in the same folder as the docker-compose file
+LDCS_RUCIO_FQDN=
+POSTGRES_USER=
+POSTGRES_DB=
+POSTGRES_PASSWORD=
+

maintenance/tools/CAtools/README.md

@@ -0,0 +1 @@
+Suite of tools to manage CA and certificates

maintenance/tools/CAtools/generateCA.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -x
+
+SUBJECT='/DC=org/DC=nordugrid/DC=ARC/O=LDMX/CN=LDCS CA'
+CANAME='LDCS-CA'
+MESSAGEDIGEST='sha512'
+VALIDITYPERIOD='1460'
+
+
+# Generate key
+openssl genrsa -out $CANAME.key 4096 
+
+# Generate self-signed CSR and cert
+openssl req -x509 -new -${MESSAGEDIGEST} -subj "$SUBJECT" -key $CANAME.key -days $VALIDITYPERIOD -out $CANAME.pem
+
+# Generate signing policy
+cat << EOF > $CANAME.signing_policy
+access_id_CA  X509   '/DC=org/DC=nordugrid/DC=ARC/O=LDMX/CN=LDCS CA'
+pos_rights    globus CA:sign
+cond_subjects globus '"/DC=org/DC=nordugrid/DC=ARC/O=LDMX/*"'
+EOF
+
+# Generate hash links
+CERTHASH=$(openssl x509 -subject_hash -subject_hash_old -noout -in $CANAME.pem)
+
+for h in $CERTHASH; do 
+   ln -s $CANAME.pem $h.0
+   ln -s $CANAME.signing_policy $h.signing_policy
+done
+
+
+

maintenance/tools/CAtools/generatehostcert.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -x
+
+#TODO: add defaults
+CADIR=${2:-CA/}
+CANAME='LDCS-CA'
+CACERT=$CADIR/$CANAME.pem
+CAKEY=$CADIR/$CANAME.key
+MESSAGEDIGEST='sha512'
+
+HOSTNAME=$1
+SUBJECTHEAD='/DC=org/DC=nordugrid/DC=ARC/O=LDMX/CN=host\/'
+SUBJECT="$SUBJECTHEAD$HOSTNAME"
+
+# Generate hostkey
+
+openssl genrsa -out $HOSTNAME.key 4096
+
+# Generate csr
+openssl req -new -$MESSAGEDIGEST -subj "$SUBJECT" -key $HOSTNAME.key -out $HOSTNAME.csr
+
+#generate config
+
+cat << EOF > x509v3_config-$HOSTNAME
+basicConstraints=CA:FALSE
+keyUsage=digitalSignature, nonRepudiation, keyEncipherment
+subjectAltName=DNS:$HOSTNAME
+EOF
+
+# Sign certificate with CA
+
+openssl x509 -req -$MESSAGEDIGEST -in $HOSTNAME.csr -CA $CACERT -CAkey $CAKEY -CAcreateserial -extfile x509v3_config-$HOSTNAME -out $HOSTNAME.pem -days 365
+
+
+
+

maintenance/tools/CAtools/generateusercerts.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -x
+
+#TODO: add better defaults
+CADIR=${2:-CA/}
+CANAME='LDCS-CA'
+CACERT=$CADIR/$CANAME.pem
+CAKEY=$CADIR/$CANAME.key
+MESSAGEDIGEST='sha512'
+
+USERNAME=${1:-'Simulation Agent'}
+# Avoid blank spaces in filenames
+USERNAMEDASHES=$(echo $USERNAME | tr ' ' '-')
+SUBJECTHEAD='/DC=org/DC=nordugrid/DC=ARC/O=LDMX/CN='
+SUBJECT="$SUBJECTHEAD$USERNAME"
+
+# Generate hostkey
+
+openssl genrsa -out userkey-$USERNAMEDASHES.key 4096
+
+# Generate csr
+openssl req -new -$MESSAGEDIGEST -subj "$SUBJECT" -key userkey-$USERNAMEDASHES.key -out usercert-$USERNAMEDASHES.csr
+
+#generate config
+
+cat << EOF > x509v3_config-$USERNAMEDASHES
+basicConstraints=CA:FALSE
+keyUsage=digitalSignature, nonRepudiation, keyEncipherment
+EOF
+
+# Sign certificate with CA
+
+openssl x509 -req -$MESSAGEDIGEST -in usercert-$USERNAMEDASHES.csr -CA $CACERT -CAkey $CAKEY -CAcreateserial -extfile x509v3_config-$USERNAMEDASHES -out usercert-$USERNAMEDASHES.pem -days 365
+
+
+
+