From 01c3e5374d1cbcdb9e2a851d39b408a219a239b2 Mon Sep 17 00:00:00 2001 From: Adrian Amaglio Date: Fri, 15 Sep 2023 10:57:47 +0200 Subject: [PATCH] certs and stuff --- .../deploy_all/files/bin/deploy_service.sh | 2 + .../roles/deploy_all/files/bin/gen_env.sh | 1 + .../roles/deploy_all/files/bin/resolvable.sh | 14 ++ .../roles/deploy_all/files/bin/template.sh | 1 + .../jean-cloud-common/files/certs.priv.pub | 1 + .../roles/jean-cloud-common/tasks/main.yml | 16 +++ provisioning/services_nougaro.yml | 64 --------- provisioning/services_vandamme.yml | 132 ------------------ services/_ssh/docker-compose.yml | 15 -- .../letsencrypt.jean-cloud.org/acme-dns.sh | 17 +++ services/letsencrypt.jean-cloud.org/deploy.sh | 8 ++ .../letsencrypt.jean-cloud.org/deploy_bind.sh | 20 +++ services/letsencrypt.jean-cloud.org/run.sh | 8 ++ .../letsencrypt.jean-cloud.org/run_bind.sh | 40 ++++++ services/ns1.jean-cloud.org/deploy.sh | 4 + .../ns1.jean-cloud.org/helper_functions.sh | 62 +++++--- services/services.txt | 3 +- .../velov.jean-cloud.net/docker-compose.yml | 2 +- 18 files changed, 177 insertions(+), 233 deletions(-) create mode 100755 provisioning/roles/deploy_all/files/bin/resolvable.sh create mode 100644 provisioning/roles/jean-cloud-common/files/certs.priv.pub delete mode 100755 provisioning/services_nougaro.yml delete mode 100755 provisioning/services_vandamme.yml delete mode 100644 services/_ssh/docker-compose.yml create mode 100755 services/letsencrypt.jean-cloud.org/acme-dns.sh create mode 100755 services/letsencrypt.jean-cloud.org/deploy.sh create mode 100755 services/letsencrypt.jean-cloud.org/deploy_bind.sh create mode 100755 services/letsencrypt.jean-cloud.org/run.sh create mode 100755 services/letsencrypt.jean-cloud.org/run_bind.sh diff --git a/provisioning/roles/deploy_all/files/bin/deploy_service.sh b/provisioning/roles/deploy_all/files/bin/deploy_service.sh index f22694d..639773d 100755 --- a/provisioning/roles/deploy_all/files/bin/deploy_service.sh +++ b/provisioning/roles/deploy_all/files/bin/deploy_service.sh @@ -4,6 +4,8 @@ set -euo pipefail +[ ! -f /data/mounted ] && die "/data is not mounted" + noreload=false deploy=true if [ "$#" -ge 2 ] && [ "$2" = noreload ] ; then diff --git a/provisioning/roles/deploy_all/files/bin/gen_env.sh b/provisioning/roles/deploy_all/files/bin/gen_env.sh index e0f75a3..8ed3047 100755 --- a/provisioning/roles/deploy_all/files/bin/gen_env.sh +++ b/provisioning/roles/deploy_all/files/bin/gen_env.sh @@ -14,6 +14,7 @@ nginx_conf_path='$proxy_dir/sites-enabled' new_nginx_conf_path='$proxy_dir/new-sites-enabled' certs_path='$certs_path' dummy_cert_path='$certs_path/dummy' +servicefile=/docker/services.txt EOF for dir in /docker/* ; do diff --git a/provisioning/roles/deploy_all/files/bin/resolvable.sh b/provisioning/roles/deploy_all/files/bin/resolvable.sh new file mode 100755 index 0000000..f8086bf --- /dev/null +++ b/provisioning/roles/deploy_all/files/bin/resolvable.sh @@ -0,0 +1,14 @@ +#/bin/bash + +# Read domains form stdin and echo the ones resolved successfully + +server="" +if [ "$#" -ge 1 ] && [ -n "$1" ] ; then + server="$1" +fi + +while read domain; do + host "$domain" $server &>/dev/null + [ "$?" -eq 0 ] && echo "$domain" +done +exit 0 diff --git a/provisioning/roles/deploy_all/files/bin/template.sh b/provisioning/roles/deploy_all/files/bin/template.sh index ad1b800..f4fac51 100755 --- a/provisioning/roles/deploy_all/files/bin/template.sh +++ b/provisioning/roles/deploy_all/files/bin/template.sh @@ -7,5 +7,6 @@ fi if [ -f "$1" ] ; then bash -c 'set -a && . '"$1"' && envsubst "$(cat '"$1"' | grep -o ^.*= | sed "s/=//" | sed "s/^/$/")"' else + echo "No env file found, no modifications made." >&2 cat /dev/stdin fi diff --git a/provisioning/roles/jean-cloud-common/files/certs.priv.pub b/provisioning/roles/jean-cloud-common/files/certs.priv.pub new file mode 100644 index 0000000..e42bc79 --- /dev/null +++ b/provisioning/roles/jean-cloud-common/files/certs.priv.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYnqsFJpr2wF3WD9QO81SIZJ1Ah7qRM+F3mOgGItYD1qjO4/0Hx8HCzebZkega3wP6Pn5ML/CANXzGnniib5khzjuecF2RU1yNqO40BmuuJ1T0AtdHLuLc6RXmpBmE2NFmSxTEAMN1/kWXoo2gwiBTFOqSnIElfRoHCxwAZFZN/wuEnZKuMV4xw8NWkkrmruSBdkOMHvZshtdzxkvi61FI/7MOWFdmiFy2YypmPUEx3h3ujykQcgVpZ7RYz8seh5v7j5P2N/xYJYEatMN7che31d6u1Nazd0kfV/9jAQNXxsKgSZdnxJpoKDjXe6DLLwQ3gybOT70ehzdG77SK/bi6KnVYrH5E/2lehHOGgKpdskt0pEJ8d0rM5hIgnyCX+S6S7H8c8Uca+UVc+/g59qW1XRyYmr/o4zljmkZA4Zx0IEk3MQYCFkT9JZQTjrgSCI01ggHxwOoQhpiJVnoUZNa3j3DMTi86A3H4XD5ByIK8k7t5MFzk6sDWYDi25CVa/HM= root@raku.jean-cloud.org diff --git a/provisioning/roles/jean-cloud-common/tasks/main.yml b/provisioning/roles/jean-cloud-common/tasks/main.yml index a5f5daf..c290aa8 100644 --- a/provisioning/roles/jean-cloud-common/tasks/main.yml +++ b/provisioning/roles/jean-cloud-common/tasks/main.yml @@ -24,6 +24,21 @@ when: inventory_hostname in groups["shlago"] +# Account for deploying SSL certs +- name: Add certs user + ansible.builtin.user: + name: certs + shell: /bin/bash + home: /data/letsencrypt.jean-cloud.org + +- name: Set authorized key, removing all the authorized keys already set + ansible.posix.authorized_key: + user: certs + key: "{{ lookup('file', 'certs.pub') }}" + state: present + exclusive: true + +# - name: Show last changed password for security copy: dest: /etc/profile.d/user_last_passwd.sh @@ -77,6 +92,7 @@ "max-file": "3" } } + #TODO add this to /etc/docker/daemon.json #{ # "iptables": false diff --git a/provisioning/services_nougaro.yml b/provisioning/services_nougaro.yml deleted file mode 100755 index 88d342b..0000000 --- a/provisioning/services_nougaro.yml +++ /dev/null @@ -1,64 +0,0 @@ -# TODO ansible secrets -# Oma-Radio host - -- name: Deploy specific services - hosts: nougaro.jean-cloud.net - become: no - roles: - #- role: docker-network-setup - - # The proxy docker stack must be the first to be deployed - - role: prepare-nginx - - - role: deploy - service_name: proxy - state: started - monitored: false - - - role: deploy - service_name: nsslave.jean-cloud.net - state: started - monitored: false - - - role: deploy - service_name: registry.oma-radio.fr - state: started - - - role: deploy - service_name: wordpress.inurbe.fr - state: started - - - role: deploy - service_name: compagnienouvelle.fr - state: started - - - role: deploy - service_name: icecast.oma-radio.fr - state: started - monitored: false - remote_docker_login_user: oma - remote_docker_login_pass: KkK8Aavmm4cN6nBM - remote_docker_login_registry: http://registry.oma-radio.fr - - - role: deploy - service_name: soundbase.oma-radio.fr - state: started - monitored: false - - - role: deploy - service_name: paj.oma-radio.fr - state: started - monitored: false - remote_docker_login_user: oma - remote_docker_login_pass: KkK8Aavmm4cN6nBM - remote_docker_login_registry: http://registry.oma-radio.fr - - #- role: deploy - # service_name: radionimaitre.oma-radio.fr - # state: started - # monitored: false - # remote_docker_login_user: oma - # remote_docker_login_pass: KkK8Aavmm4cN6nBM - # remote_docker_login_registry: http://registry.oma-radio.fr - - - role: restart-nginx diff --git a/provisioning/services_vandamme.yml b/provisioning/services_vandamme.yml deleted file mode 100755 index 5182fb6..0000000 --- a/provisioning/services_vandamme.yml +++ /dev/null @@ -1,132 +0,0 @@ -# The host have: -# - /data -> every data -# - /docker -> deployed docker-compose files - -- name: Deploy specific services - hosts: vandamme.jean-cloud.net - become: yes - roles: - #- role: docker-network-setup - - # The proxy docker stack must be the first to be deployed - - role: prepare-nginx - - - role: deploy - service_name: proxy - state: started - monitored: false - - - role: deploy - service_name: meta-morpho.se - state: started - - - role: deploy - service_name: mailer.jean-cloud.net - state: started - remote_docker_login_user: jean-cloud - remote_docker_login_pass: KaJefxXiNr327EfG4suYD2PM4tYF5Jy8AhMYntfdjVhX - monitored: false - - - role: deploy - service_name: static.jean-cloud.net - state: started - - - - role: deploy - service_name: ssh - state: started - monitored: false - - #- role: deploy - # service_name: myrrdel.jean-cloud.net - # state: started - - - role: deploy - service_name: collectif-arthadie.fr - state: started - - #- role: deploy - # service_name: karna.jean-cloud.net - # state: started - - - role: deploy - service_name: oma-radio.fr - state: started - - - role: deploy - service_name: rpnow.jean-cloud.net - state: started - - - role: deploy - service_name: ns.jean-cloud.org - state: started - monitored: false - - - role: deploy - service_name: gmx-webmail.jean-cloud.net - state: started - - - role: deploy - service_name: registry.jean-cloud.net - state: started - - - role: deploy - service_name: inurbe.fr - state: started - - - role: deploy - service_name: feteducourt.jean-cloud.net - state: started - remote_docker_login_user: jean-cloud - remote_docker_login_pass: KaJefxXiNr327EfG4suYD2PM4tYF5Jy8AhMYntfdjVhX - - - role: deploy - service_name: feteducourt2020.jean-cloud.net - state: started - remote_docker_login_user: jean-cloud - remote_docker_login_pass: KaJefxXiNr327EfG4suYD2PM4tYF5Jy8AhMYntfdjVhX - - - role: deploy - service_name: leida.fr - state: started - - - role: deploy - service_name: lalis.fr - state: started - - - role: deploy - service_name: amaglio.fr - state: started - - - role: deploy - service_name: velov.jean-cloud.net - state: started - - - role: deploy - service_name: cousinades.jean-cloud.net - state: started - monitored: false # web cant pass basic auth yet - - - role: deploy - service_name: cousinades2.jean-cloud.net - state: started - monitored: false # web cant pass basic auth yet - - - - role: deploy - service_name: nuage.jean-cloud.net - state: started - - - role: deploy - service_name: git.jean-cloud.net - state: started - - - role: deploy - service_name: wiki-cgr.jean-cloud.net - state: started - - - role: deploy - service_name: jean-cloud.net - state: started - - - role: restart-nginx diff --git a/services/_ssh/docker-compose.yml b/services/_ssh/docker-compose.yml deleted file mode 100644 index e014065..0000000 --- a/services/_ssh/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' -services: - sshd: - image: atmoz/sftp - volumes: - - /data/ssh/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key - - /data/ssh/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key - - /data/leida.fr:/home/leida/sftp - - /data/lalis.fr:/home/lalis/sftp - - /data/oma-radio.fr:/home/oma/sftp - - /data/collectif-arthadie.fr/wordpress:/home/collectifarthadie/sftp - - /data/ssh/users.conf:/etc/sftp/users.conf:ro - ports: - - '2222:22' - diff --git a/services/letsencrypt.jean-cloud.org/acme-dns.sh b/services/letsencrypt.jean-cloud.org/acme-dns.sh new file mode 100755 index 0000000..1c6a903 --- /dev/null +++ b/services/letsencrypt.jean-cloud.org/acme-dns.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ "$#" -ne 1 ] ; then + echo "Usage: $0 " >&2 + exit 1 +fi + +service="$1" + +nginxfile="/docker/$service/nginx_server.conf" +if [ -f "$nginxfile" ] ; then + nginxdomains="$(extract_domain_nginx_conf.sh "$nginxfile" | template.sh "/docker/$service/.env")" + domains="$(echo "$service $nginxdomains" | tr ' ' '\n' | sort -u | resolvable.sh ns.jean-cloud.org | sed -z -e 's/\n$//' -e 's/\n/ -d /g' )" + [ -z "$domains" ] && exit 0 + echo "--------------- -d $domains" + certbot certonly --config-dir "$DATA_DIR/certs" --work-dir "$tmp/work" --logs-dir "$tmp/logs" --agree-tos -m contact@jean-cloud.org -n --cert-name "$service" --dns-rfc2136 --dns-rfc2136-credentials "$DATA_DIR/rfc2136.ini" -d $domains +fi diff --git a/services/letsencrypt.jean-cloud.org/deploy.sh b/services/letsencrypt.jean-cloud.org/deploy.sh new file mode 100755 index 0000000..b0055e3 --- /dev/null +++ b/services/letsencrypt.jean-cloud.org/deploy.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +# For some variables +. /etc/jeancloud.env + +apt install -y python3-certbot-dns-rfc2136 diff --git a/services/letsencrypt.jean-cloud.org/deploy_bind.sh b/services/letsencrypt.jean-cloud.org/deploy_bind.sh new file mode 100755 index 0000000..da19d12 --- /dev/null +++ b/services/letsencrypt.jean-cloud.org/deploy_bind.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +. /etc/jeancloud.env + +[ ! -f "$DATA_DIR/rfc2136.ini" ] && echo "$0 Missing files" && exit 1 + +while read line ; do + read -r service target < <(echo "$line") + echo "---- $service $target ----" + nginxfile="/docker/$service/nginx_server.conf" + if [ -f "$nginxfile" ] ; then + nginxdomains="$(extract_domain_nginx_conf.sh "$nginxfile" | template.sh "/docker/$service/.env")" + domains="$(echo "$service $nginxdomains" | tr ' ' '\n' | sort -u | sed -z 's/\n/ -d /')" + echo "$domains" + certbot certonly -m contact@jean-cloud.org -n --cert-name "$service" --dns-rfc2136 --dns-rfc2136-credentials "$DATA_DIR/rfc2136.ini" -d $domains + fi + +done < "$servicefile" diff --git a/services/letsencrypt.jean-cloud.org/run.sh b/services/letsencrypt.jean-cloud.org/run.sh new file mode 100755 index 0000000..37a0f3c --- /dev/null +++ b/services/letsencrypt.jean-cloud.org/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +. driglibash-base +here="$(where)" + +sudo -u bind bash -c "$here/run_bind.sh $@" diff --git a/services/letsencrypt.jean-cloud.org/run_bind.sh b/services/letsencrypt.jean-cloud.org/run_bind.sh new file mode 100755 index 0000000..a3ee486 --- /dev/null +++ b/services/letsencrypt.jean-cloud.org/run_bind.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euo pipefail + +. driglibash-base +here="$(where)" + +# For some variables +. /etc/jeancloud.env +. "$here/.env" + +# Test secret presence +[ ! -f "$DATA_DIR/rfc2136.ini" ] && echo "$0 Missing file '$DATA_DIR/rfc2136.ini'" && exit 1 + +export tmp="$(mktemp -d)" +mkdir -p "$tmp/{work,logs}" + +# If there is some args, populate a fake service file +if [ "$#" -ge 1 ] && [ -n "$1" ] ; then + servicefile="$(mktemp)" + for service in "$@" ; do + echo "$service _" >> "$servicefile" + done +fi + +# For each service, read all possible domains +while read line ; do + read -r service target < <(echo "$line") + + # removo dummy cert + dummy_cert.sh "$service" remove + + [ -d "$DATA_DIR/certs/live/$service" ] && echo "Already exists, thats a job for renew : $service" && continue + + # acme + "$here/acme-dns.sh" "$service" + + # Replace dummy cert if letsencrypt failed + [ "$?" -ne 0 ] && dummy_cert.sh "$servic" remove +done < "$servicefile" diff --git a/services/ns1.jean-cloud.org/deploy.sh b/services/ns1.jean-cloud.org/deploy.sh index 88c16c7..fe45540 100755 --- a/services/ns1.jean-cloud.org/deploy.sh +++ b/services/ns1.jean-cloud.org/deploy.sh @@ -29,6 +29,7 @@ secondary_ips="37.65.119.74" # NS name default_dns_name="shlago.jean-cloud.org." +CAA_RR='CAA 0 issue "letsencrypt.org;validationmethods=dns-01"' run () { if [ "$#" -ne 1 ] ; then @@ -39,6 +40,9 @@ run () { primary_ips="$primary_ips;$(fakeresolve_ip_list raku)" secondary_ips="$secondary_ips;$(fakeresolve_ip_list shlago)" + line_in_file "primary_ips=\"$primary_ips\"" "$DOCKER_DIR/.env" + line_in_file "secondary_ips=\"$secondary_ips\"" "$DOCKER_DIR/.env" + if [ "$1" = "primary" ] ; then create_primary_files else diff --git a/services/ns1.jean-cloud.org/helper_functions.sh b/services/ns1.jean-cloud.org/helper_functions.sh index 4dfa37c..f9287cc 100644 --- a/services/ns1.jean-cloud.org/helper_functions.sh +++ b/services/ns1.jean-cloud.org/helper_functions.sh @@ -34,8 +34,8 @@ prepare () { } restart () { - echo 'Restart bind9' - systemctl restart bind9 + echo 'Restart named' + systemctl restart named } # Function that simulate a DNS resolve by reading bind zone file @@ -81,6 +81,12 @@ addbindline () { # Only append if db file exists [ ! -f "$bindfile" ] && return 0 + # BTW allow ACME DNS update + token="#JC-ACME $domain" + acme_dns="grant letsencrypt.key. name _acme-challenge.$name. TXT;" + sed -i "s/\([[:space:]]*\)$token/\1$acme_dns\n\1$token/" "$debian_bind_confdir/named.conf.local" + + if [ -z "$shortname" ] ; then # CNAME are forbiden for empty shortnames, so we must resolve the target IPs while read line ; do @@ -89,8 +95,6 @@ addbindline () { else line_in_file "$shortname IN CNAME $target." "$bindfile" fi - - #XXX Add CAA records } list_template_db_files () { @@ -102,6 +106,7 @@ create_primary_files () { # Compact the default SOA SOA="$(grep -o '^[^;]*' SOA | sed -z -e 's/[[:space:]]\{2,\}/ /g' -e 's/\n/\\n/')" + cat "$debian_bind_confdir/template.named.conf" | template.sh "$DOCKER_DIR/.env" > "$debian_bind_confdir/named.conf" for file in $(list_template_db_files) ; do domain="$(basename "$file" | sed 's/template.db.//')" @@ -109,27 +114,44 @@ create_primary_files () { # Set the default SOA if needed sed "s/^;JC_AUTOSOA$/$SOA/" "$file" > "$new_db_file" - + + # Set serial + serial="$(date '+%s')" + sed -i "s/\(@ IN SOA [^(]*( \)[0-9]\+/\1$serial/" "$new_db_file" + # If no NS record in the db file if [ -z "$(grep '[^;].*IN.*NS' "$new_db_file")" ] ; then echo "@ IN NS $default_dns_name" >> "$new_db_file" fi - - cat >> "$debian_bind_confdir/named.conf.local" <> "$debian_bind_confdir/named.conf.local" <<-EOF + zone "$domain" { + # Zone file + type master; + file "$new_db_file"; + + # Secondary conf + # https://kb.isc.org/docs/aa-00723 + #allow-update { !{!{$secondary_ips};any;}; key update-key; }; + allow-transfer { $secondary_ips }; + also-notify { $secondary_ips }; + notify yes; + + # DNSSEC + dnssec-policy default; + inline-signing yes; + key-directory "$DATA_DIR/keys"; + + # ACME autorizations + update-policy { + #JC-ACME $domain + }; + + }; + EOF + + done echo 'Find every used domain and add them to bind db' diff --git a/services/services.txt b/services/services.txt index 55b9098..8c93f00 100644 --- a/services/services.txt +++ b/services/services.txt @@ -15,6 +15,7 @@ gypsylyonfestival.com max.jean-cloud.org inurbe.fr max.jean-cloud.org jean-cloud.net shlago.jean-cloud.org leida.fr vandamme.jean-cloud.org +letsencrypt.jean-cloud.org max.jean-cloud.org lexicographe.jean-cloud.net shlago.jean-cloud.org metamorphosemagazine.fr shlago.jean-cloud.org nc-backup.jean-cloud.net raku.jean-cloud.org @@ -28,6 +29,6 @@ radiodemo.oma-radio.fr tetede.jean-cloud.org radionimaitre.oma-radio.fr tetede.jean-cloud.org raplacgr.jean-cloud.net tetede.jean-cloud.org rpnow.jean-cloud.net vandamme.jean-cloud.org -_ssh vandamme.jean-cloud.org +sftp.jean-cloud.net max.jean-cloud.org velov.jean-cloud.net shlago.jean-cloud.org wiki-cgr.jean-cloud.net vandamme.jean-cloud.org diff --git a/services/velov.jean-cloud.net/docker-compose.yml b/services/velov.jean-cloud.net/docker-compose.yml index 661b2d8..b86ffc9 100755 --- a/services/velov.jean-cloud.net/docker-compose.yml +++ b/services/velov.jean-cloud.net/docker-compose.yml @@ -3,7 +3,7 @@ services: app: image: php:7.2-fpm-alpine volumes: - - /data/velov.jean-cloud.net:/usr/src/app + - $HTTP_DIR:/usr/src/app restart: unless-stopped networks: default: