certs and stuff

provisioning/roles/deploy_all/files/bin/deploy_service.sh

@@ -4,6 +4,8 @@
 
 set -euo pipefail
 
+[ ! -f /data/mounted ] && die "/data is not mounted"
+
 noreload=false
 deploy=true
 if [ "$#" -ge 2 ] && [ "$2" = noreload ] ; then

provisioning/roles/deploy_all/files/bin/gen_env.sh

@@ -14,6 +14,7 @@ nginx_conf_path='$proxy_dir/sites-enabled'
 new_nginx_conf_path='$proxy_dir/new-sites-enabled'
 certs_path='$certs_path'
 dummy_cert_path='$certs_path/dummy'
+servicefile=/docker/services.txt
 EOF
 	
 for dir in /docker/* ; do

provisioning/roles/deploy_all/files/bin/resolvable.sh

@@ -0,0 +1,14 @@
+#/bin/bash
+
+# Read domains form stdin and echo the ones resolved successfully
+
+server=""
+if [ "$#" -ge 1 ] && [ -n "$1" ] ; then
+	server="$1"
+fi	
+
+while read domain; do
+	host "$domain" $server &>/dev/null
+	[ "$?" -eq 0 ] && echo "$domain"
+done
+exit 0

provisioning/roles/deploy_all/files/bin/template.sh

@@ -7,5 +7,6 @@ fi
 if [ -f "$1" ] ; then
 	bash -c 'set -a && . '"$1"' &&  envsubst "$(cat '"$1"' | grep -o ^.*= | sed "s/=//" | sed "s/^/$/")"'
 else
+	echo "No env file found, no modifications made." >&2
 	cat /dev/stdin
 fi

provisioning/roles/jean-cloud-common/files/certs.priv.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYnqsFJpr2wF3WD9QO81SIZJ1Ah7qRM+F3mOgGItYD1qjO4/0Hx8HCzebZkega3wP6Pn5ML/CANXzGnniib5khzjuecF2RU1yNqO40BmuuJ1T0AtdHLuLc6RXmpBmE2NFmSxTEAMN1/kWXoo2gwiBTFOqSnIElfRoHCxwAZFZN/wuEnZKuMV4xw8NWkkrmruSBdkOMHvZshtdzxkvi61FI/7MOWFdmiFy2YypmPUEx3h3ujykQcgVpZ7RYz8seh5v7j5P2N/xYJYEatMN7che31d6u1Nazd0kfV/9jAQNXxsKgSZdnxJpoKDjXe6DLLwQ3gybOT70ehzdG77SK/bi6KnVYrH5E/2lehHOGgKpdskt0pEJ8d0rM5hIgnyCX+S6S7H8c8Uca+UVc+/g59qW1XRyYmr/o4zljmkZA4Zx0IEk3MQYCFkT9JZQTjrgSCI01ggHxwOoQhpiJVnoUZNa3j3DMTi86A3H4XD5ByIK8k7t5MFzk6sDWYDi25CVa/HM= root@raku.jean-cloud.org

provisioning/roles/jean-cloud-common/tasks/main.yml

@@ -24,6 +24,21 @@
   when: inventory_hostname in groups["shlago"]
 
 
+# Account for deploying SSL certs
+- name: Add certs user
+  ansible.builtin.user:
+    name: certs
+    shell: /bin/bash
+    home: /data/letsencrypt.jean-cloud.org
+
+- name: Set authorized key, removing all the authorized keys already set
+  ansible.posix.authorized_key:
+    user: certs
+    key: "{{ lookup('file', 'certs.pub') }}"
+    state: present
+    exclusive: true
+
+#
 - name: Show last changed password for security
   copy:
       dest: /etc/profile.d/user_last_passwd.sh
@@ -77,6 +92,7 @@
              "max-file": "3"    
              }
          }
+
 #TODO add this to /etc/docker/daemon.json
 #{
 #        "iptables": false

provisioning/services_nougaro.yml

@@ -1,64 +0,0 @@
-# TODO ansible secrets
-# Oma-Radio host
-
-- name: Deploy specific services
-  hosts: nougaro.jean-cloud.net
-  become: no
-  roles:
-    #- role: docker-network-setup
-
-    # The proxy docker stack must be the first to be deployed
-    - role: prepare-nginx
-
-    - role: deploy
-      service_name: proxy
-      state: started
-      monitored: false
-
-    - role: deploy
-      service_name: nsslave.jean-cloud.net
-      state: started
-      monitored: false
-      
-    - role: deploy
-      service_name: registry.oma-radio.fr
-      state: started
-
-    - role: deploy
-      service_name: wordpress.inurbe.fr
-      state: started
-
-    - role: deploy
-      service_name: compagnienouvelle.fr
-      state: started
-
-    - role: deploy
-      service_name: icecast.oma-radio.fr
-      state: started
-      monitored: false
-      remote_docker_login_user: oma
-      remote_docker_login_pass: KkK8Aavmm4cN6nBM
-      remote_docker_login_registry: http://registry.oma-radio.fr
-
-    - role: deploy
-      service_name: soundbase.oma-radio.fr
-      state: started
-      monitored: false
-
-    - role: deploy
-      service_name: paj.oma-radio.fr
-      state: started
-      monitored: false
-      remote_docker_login_user: oma
-      remote_docker_login_pass: KkK8Aavmm4cN6nBM
-      remote_docker_login_registry: http://registry.oma-radio.fr
-
-        #- role: deploy
-        #  service_name: radionimaitre.oma-radio.fr
-        #  state: started
-        #  monitored: false
-        #  remote_docker_login_user: oma
-        #  remote_docker_login_pass: KkK8Aavmm4cN6nBM
-        #  remote_docker_login_registry: http://registry.oma-radio.fr
-
-    - role: restart-nginx

provisioning/services_vandamme.yml

@@ -1,132 +0,0 @@
-# The host have:
-# - /data -> every data
-# - /docker -> deployed docker-compose files
-
-- name: Deploy specific services
-  hosts: vandamme.jean-cloud.net
-  become: yes
-  roles:
-    #- role: docker-network-setup
-
-    # The proxy docker stack must be the first to be deployed
-    - role: prepare-nginx
-
-    - role: deploy
-      service_name: proxy
-      state: started
-      monitored: false
-
-    - role: deploy
-      service_name: meta-morpho.se
-      state: started
-
-    - role: deploy
-      service_name: mailer.jean-cloud.net
-      state: started
-      remote_docker_login_user: jean-cloud
-      remote_docker_login_pass: KaJefxXiNr327EfG4suYD2PM4tYF5Jy8AhMYntfdjVhX
-      monitored: false
-
-    - role: deploy
-      service_name: static.jean-cloud.net
-      state: started
-
-
-    - role: deploy
-      service_name: ssh
-      state: started
-      monitored: false
-
-      #- role: deploy
-      #  service_name: myrrdel.jean-cloud.net
-      #  state: started
-
-    - role: deploy
-      service_name: collectif-arthadie.fr
-      state: started
-
-      #- role: deploy
-      #  service_name: karna.jean-cloud.net
-      #  state: started
-
-    - role: deploy
-      service_name: oma-radio.fr
-      state: started
-
-    - role: deploy
-      service_name: rpnow.jean-cloud.net
-      state: started
-
-    - role: deploy
-      service_name: ns.jean-cloud.org
-      state: started
-      monitored: false
-
-    - role: deploy
-      service_name: gmx-webmail.jean-cloud.net
-      state: started
-
-    - role: deploy
-      service_name: registry.jean-cloud.net
-      state: started
-
-    - role: deploy
-      service_name: inurbe.fr
-      state: started
-
-    - role: deploy
-      service_name: feteducourt.jean-cloud.net
-      state: started
-      remote_docker_login_user: jean-cloud
-      remote_docker_login_pass: KaJefxXiNr327EfG4suYD2PM4tYF5Jy8AhMYntfdjVhX
-
-    - role: deploy
-      service_name: feteducourt2020.jean-cloud.net
-      state: started
-      remote_docker_login_user: jean-cloud
-      remote_docker_login_pass: KaJefxXiNr327EfG4suYD2PM4tYF5Jy8AhMYntfdjVhX
-
-    - role: deploy
-      service_name: leida.fr
-      state: started
-
-    - role: deploy
-      service_name: lalis.fr
-      state: started
-
-    - role: deploy
-      service_name: amaglio.fr
-      state: started
-
-    - role: deploy
-      service_name: velov.jean-cloud.net
-      state: started
-
-    - role: deploy
-      service_name: cousinades.jean-cloud.net
-      state: started
-      monitored: false # web cant pass basic auth yet
-
-    - role: deploy
-      service_name: cousinades2.jean-cloud.net
-      state: started
-      monitored: false # web cant pass basic auth yet
-
-
-    - role: deploy
-      service_name: nuage.jean-cloud.net
-      state: started
-
-    - role: deploy
-      service_name: git.jean-cloud.net
-      state: started
-
-    - role: deploy
-      service_name: wiki-cgr.jean-cloud.net
-      state: started
-    
-    - role: deploy
-      service_name: jean-cloud.net
-      state: started
-
-    - role: restart-nginx

services/_ssh/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3'
-services:
-    sshd:
-        image: atmoz/sftp
-        volumes:
-            - /data/ssh/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key
-            - /data/ssh/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key
-            - /data/leida.fr:/home/leida/sftp
-            - /data/lalis.fr:/home/lalis/sftp
-            - /data/oma-radio.fr:/home/oma/sftp
-            - /data/collectif-arthadie.fr/wordpress:/home/collectifarthadie/sftp
-            - /data/ssh/users.conf:/etc/sftp/users.conf:ro
-        ports:
-            - '2222:22'
-

services/letsencrypt.jean-cloud.org/acme-dns.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ "$#" -ne 1 ] ; then
+	echo "Usage: $0 <service_name>" >&2
+	exit 1
+fi
+
+service="$1"
+
+nginxfile="/docker/$service/nginx_server.conf"
+if [ -f "$nginxfile" ] ; then    
+	nginxdomains="$(extract_domain_nginx_conf.sh "$nginxfile" | template.sh "/docker/$service/.env")"
+	domains="$(echo "$service $nginxdomains" | tr ' ' '\n' | sort -u | resolvable.sh ns.jean-cloud.org | sed -z -e 's/\n$//' -e 's/\n/ -d /g' )"
+	[ -z "$domains" ] && exit 0
+	echo "--------------- -d $domains"
+	certbot certonly --config-dir "$DATA_DIR/certs" --work-dir "$tmp/work" --logs-dir "$tmp/logs" --agree-tos -m contact@jean-cloud.org -n --cert-name "$service" --dns-rfc2136 --dns-rfc2136-credentials "$DATA_DIR/rfc2136.ini" -d $domains
+fi

services/letsencrypt.jean-cloud.org/deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# For some variables
+. /etc/jeancloud.env
+
+apt install -y python3-certbot-dns-rfc2136

services/letsencrypt.jean-cloud.org/deploy_bind.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+. /etc/jeancloud.env
+
+[ ! -f "$DATA_DIR/rfc2136.ini" ] && echo "$0 Missing files" && exit 1
+
+while read line ; do
+	read -r service target < <(echo "$line")
+	echo "---- $service $target ----"
+	nginxfile="/docker/$service/nginx_server.conf"
+	if [ -f "$nginxfile" ] ; then    
+		nginxdomains="$(extract_domain_nginx_conf.sh "$nginxfile" | template.sh "/docker/$service/.env")"
+		domains="$(echo "$service $nginxdomains" | tr ' ' '\n' | sort -u | sed -z 's/\n/ -d /')"
+		echo "$domains"
+		certbot certonly -m contact@jean-cloud.org -n --cert-name "$service" --dns-rfc2136 --dns-rfc2136-credentials "$DATA_DIR/rfc2136.ini" -d $domains
+    fi
+
+done < "$servicefile"

services/letsencrypt.jean-cloud.org/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euo pipefail
+
+. driglibash-base
+here="$(where)"
+
+sudo -u bind bash -c "$here/run_bind.sh $@"

services/letsencrypt.jean-cloud.org/run_bind.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+. driglibash-base
+here="$(where)"
+
+# For some variables
+. /etc/jeancloud.env
+. "$here/.env"
+
+# Test secret presence
+[ ! -f "$DATA_DIR/rfc2136.ini" ] && echo "$0 Missing file '$DATA_DIR/rfc2136.ini'" && exit 1
+
+export tmp="$(mktemp -d)"
+mkdir -p "$tmp/{work,logs}"
+
+# If there is some args, populate a fake service file
+if [ "$#" -ge 1 ] && [ -n "$1" ] ; then
+	servicefile="$(mktemp)"
+	for service in "$@" ; do
+		echo "$service _" >> "$servicefile"
+	done
+fi
+
+# For each service, read all possible domains
+while read line ; do
+	read -r service target < <(echo "$line")
+
+	# removo dummy cert
+	dummy_cert.sh "$service" remove
+
+	[ -d "$DATA_DIR/certs/live/$service" ] && echo "Already exists, thats a job for renew : $service" && continue
+
+	# acme
+	"$here/acme-dns.sh" "$service"
+
+	# Replace dummy cert if letsencrypt failed
+	[ "$?" -ne 0 ] && dummy_cert.sh "$servic" remove
+done < "$servicefile"

services/ns1.jean-cloud.org/deploy.sh

@@ -29,6 +29,7 @@ secondary_ips="37.65.119.74"
 # NS name
 default_dns_name="shlago.jean-cloud.org."
 
+CAA_RR='CAA 0 issue "letsencrypt.org;validationmethods=dns-01"'
 
 run () {
 	if [ "$#" -ne 1 ] ; then
@@ -39,6 +40,9 @@ run () {
 	primary_ips="$primary_ips;$(fakeresolve_ip_list raku)"
 	secondary_ips="$secondary_ips;$(fakeresolve_ip_list shlago)"
 
+	line_in_file "primary_ips=\"$primary_ips\"" "$DOCKER_DIR/.env"
+	line_in_file "secondary_ips=\"$secondary_ips\"" "$DOCKER_DIR/.env"
+
 	if [ "$1" = "primary" ] ; then
 		create_primary_files
 	else

services/ns1.jean-cloud.org/helper_functions.sh

@@ -34,8 +34,8 @@ prepare () {
 }
 
 restart () {
-	echo 'Restart bind9'
+	echo 'Restart named'
-	systemctl restart bind9
+	systemctl restart named
 }
 
 # Function that simulate a DNS resolve by reading bind zone file
@@ -81,6 +81,12 @@ addbindline () {
 	# Only append if db file exists
 	[ ! -f "$bindfile" ] && return 0
 
+	# BTW allow ACME DNS update
+	token="#JC-ACME $domain"
+	acme_dns="grant letsencrypt.key. name _acme-challenge.$name. TXT;"
+	sed -i "s/\([[:space:]]*\)$token/\1$acme_dns\n\1$token/" "$debian_bind_confdir/named.conf.local"
+
+
 	if [ -z "$shortname" ] ; then
 		# CNAME are forbiden for empty shortnames, so we must resolve the target IPs
 		while read line ; do
@@ -89,8 +95,6 @@ addbindline () {
 	else
 		line_in_file "$shortname IN CNAME $target." "$bindfile"
 	fi
-
-	#XXX Add CAA records
 }
 
 list_template_db_files () {
@@ -102,6 +106,7 @@ create_primary_files () {
 	# Compact the default SOA
 	SOA="$(grep -o '^[^;]*' SOA | sed -z -e 's/[[:space:]]\{2,\}/ /g' -e 's/\n/\\n/')"
 	
+	cat "$debian_bind_confdir/template.named.conf" | template.sh "$DOCKER_DIR/.env" > "$debian_bind_confdir/named.conf"
 	
 	for file in $(list_template_db_files) ; do
 		domain="$(basename "$file" | sed 's/template.db.//')"
@@ -110,26 +115,43 @@ create_primary_files () {
 		# Set the default SOA if needed
 		sed "s/^;JC_AUTOSOA$/$SOA/" "$file" > "$new_db_file"
 
+		# Set serial
+		serial="$(date '+%s')"
+		sed -i "s/\(@ IN SOA [^(]*( \)[0-9]\+/\1$serial/"  "$new_db_file"
+
 		# If no NS record in the db file
 		if [ -z "$(grep '[^;].*IN.*NS' "$new_db_file")" ] ; then
 			echo "@ IN NS $default_dns_name" >> "$new_db_file"
 		fi
 
-		cat  >> "$debian_bind_confdir/named.conf.local" <<EOF
+		# Populate named.conf.local
-zone "$domain" {
+		cat  >> "$debian_bind_confdir/named.conf.local" <<-EOF
-	# https://kb.isc.org/docs/aa-00723
+			zone "$domain" {
-	#allow-update { !{!{$secondary_ips};any;}; key update-key; };
+				# Zone file
-	allow-transfer { $secondary_ips };
+			    type master;
-	also-notify { $secondary_ips };
+			    file "$new_db_file";
-	notify yes;
+
-    type master;
+				# Secondary conf
-    file "$new_db_file";
+			    # https://kb.isc.org/docs/aa-00723
-	dnssec-policy default;
+			    #allow-update { !{!{$secondary_ips};any;}; key update-key; };
-	inline-signing yes;
+			    allow-transfer { $secondary_ips };
-	key-directory "$DATA_DIR/keys";
+			    also-notify { $secondary_ips };
+			    notify yes;
+
+				# DNSSEC
+			    dnssec-policy default;
+			    inline-signing yes;
+			    key-directory "$DATA_DIR/keys";
+
+				# ACME autorizations
+		        update-policy {    
+		    	    #JC-ACME $domain  
+		    	};    
+
+			};
+		EOF
+
 	
-};
-EOF
 	done
 	
 	echo 'Find every used domain and add them to bind db'

services/services.txt

@@ -15,6 +15,7 @@ gypsylyonfestival.com max.jean-cloud.org
 inurbe.fr max.jean-cloud.org
 jean-cloud.net shlago.jean-cloud.org
 leida.fr vandamme.jean-cloud.org
+letsencrypt.jean-cloud.org max.jean-cloud.org
 lexicographe.jean-cloud.net shlago.jean-cloud.org
 metamorphosemagazine.fr shlago.jean-cloud.org
 nc-backup.jean-cloud.net raku.jean-cloud.org
@@ -28,6 +29,6 @@ radiodemo.oma-radio.fr tetede.jean-cloud.org
 radionimaitre.oma-radio.fr tetede.jean-cloud.org
 raplacgr.jean-cloud.net tetede.jean-cloud.org
 rpnow.jean-cloud.net vandamme.jean-cloud.org
-_ssh vandamme.jean-cloud.org
+sftp.jean-cloud.net max.jean-cloud.org
 velov.jean-cloud.net shlago.jean-cloud.org
 wiki-cgr.jean-cloud.net vandamme.jean-cloud.org

services/velov.jean-cloud.net/docker-compose.yml

@@ -3,7 +3,7 @@ services:
   app:
     image: php:7.2-fpm-alpine
     volumes:
-      - /data/velov.jean-cloud.net:/usr/src/app
+      - $HTTP_DIR:/usr/src/app
     restart: unless-stopped
     networks:
       default: