Pre-release: Improved `install_ssl` resilency. (#4807)

+ Possible certs renewal on re-run.

gl-install.sh

@@ -85,6 +85,7 @@ main() {
   ACCESS_LOG_DEST=/var/log/nginx
   NGINX_FILES_DEST=/etc/greenlight/nginx
   ASSETS_DEST=/var/www/greenlight-default/assets
+  EXIT_CODE=0
 
   # Eager checks and assertions.
   check_root
@@ -151,6 +152,8 @@ main() {
 
   apt-get auto-remove -y
   say "DONE ^^"
+
+  return $EXIT_CODE
 }
 
 check_env() {
@@ -181,7 +184,7 @@ check_env() {
   fi
 
   # Possible conflicts on setup.
-  if [ ! -f /etc/nginx/sites-enabled/greenlight ]; then
+  if [ ! -f /etc/nginx/sites-available/greenlight ]; then
     # Conflict detection of existent nginx on the system not installed by this script (possible collision with other applications).
     if dpkg -s nginx 1> /dev/null 2>&1; then
       say "Nginx is already installed on this system by another mean, this deployment may impact your workload!"
@@ -207,6 +210,11 @@ err() {
   exit 1
 }
 
+warn() {
+  say "$1" >&2
+  EXIT_CODE=1
+}
+
 check_root() {
   if [ $EUID != 0 ]; then err "You must run this command as root."; fi
 }
@@ -468,30 +476,54 @@ install_greenlight_v3(){
 }
 
 install_ssl() {
-  # Already installed.
-  if [ -f /etc/nginx/sites-available/greenlight ]; then
-    # Upgrade logic goes here.
-    return 0
-  fi
-
-  # Fresh installation.
-  # Assertions
+  # Assertions for fresh installations
+  if [ ! -f /etc/nginx/sites-available/greenlight ]; then
     if [ -d "/etc/letsencrypt/live/$HOST" ]; then
       err "Unable to generate certificates for $HOST, /etc/letsencrypt/live/$HOST/ already exists."
     fi
+  fi
 
   if [ -n "$PROVIDED_CERTIFICATE" ]; then
     if [ ! -f /local/certs/fullchain.pem ] || [ ! -f /local/certs/privkey.pem ]; then
       err "Unable to find your provided certificate files in /local/certs, Have you placed the full chain and private key for your certificate as expected?"
     fi
+
+    # Detecting generated certs and possible conflicts.
+    if [ -f /etc/letsencrypt/live/$HOST/fullchain.pem ]; then
+      if [ ! "$(readlink -e /etc/letsencrypt/live/$HOST/fullchain.pem)" == /local/certs/fullchain.pem ]; then
+        err "fullchain.pem was probably generated and not provided by this script, exiting to avoid conflict."
+      fi
+    fi
+
+    if [ -f /etc/letsencrypt/live/$HOST/privkey.pem ]; then
+      if [ ! "$(readlink -e /etc/letsencrypt/live/$HOST/privkey.pem)" == /local/certs/privkey.pem ]; then
+        err "privkey.pem was probably generated and not provided by this script, exiting to avoid conflict."
+      fi
+    fi
   else
+    # Detecting provided certs and possible conflicts.
+    if [ -f /etc/letsencrypt/live/$HOST/fullchain.pem ]; then
+      if [[ ! "$(readlink -e /etc/letsencrypt/live/$HOST/fullchain.pem)" =~ ^/etc/letsencrypt/archive/$HOST.*/fullchain.*\.pem$ ]]; then
+        err "fullchain.pem was probably provided and not generated by this script, exiting to avoid conflict."
+      fi
+    fi
+
+    if [ -f /etc/letsencrypt/live/$HOST/privkey.pem ]; then
+      if [[ ! "$(readlink -e /etc/letsencrypt/live/$HOST/privkey.pem)" =~ ^/etc/letsencrypt/archive/$HOST.*/privkey.*\.pem$ ]]; then
+        err "privkey.pem was probably provided and not generated by this script, exiting to avoid conflict."
+      fi
+    fi
+
     need_pkg certbot
   fi
 
   need_pkg nginx
   mkdir -p /etc/nginx/ssl $ACCESS_LOG_DEST $NGINX_FILES_DEST $ASSETS_DEST
 
+  cp -v /etc/nginx/sites-available/greenlight /etc/nginx/sites-available/greenlight.old # Preserve older config for admins.
+
   # HTTP only
+  # Updating HTTP config.
   cat <<HERE > /etc/nginx/sites-available/greenlight
 server_tokens off;
 server {
@@ -513,45 +545,58 @@ server {
 HERE
 
   if [ ! -f /etc/nginx/sites-enabled/greenlight ]; then
-    ln -s /etc/nginx/sites-available/greenlight /etc/nginx/sites-enabled/greenlight
+    ln -sf /etc/nginx/sites-available/greenlight /etc/nginx/sites-enabled/greenlight # Activate greenlight nginx config.
   fi
 
   if [ -f /etc/nginx/sites-enabled/default ]; then
-    rm -v /etc/nginx/sites-enabled/default
+    rm -v /etc/nginx/sites-enabled/default # Remove nginx default config.
   fi
 
-  nginx -qs reload || err "Unable to configure nginx - if following the official guides then please contact the maintainers."
+  # Validating the config.
+  nginx -qt || warn "Unable to configure nginx - if following the official guides then please contact the maintainers."
+  systemctl restart nginx
 
 # Enabling HTTPS.
-  cp -v /etc/nginx/sites-available/greenlight /etc/nginx/sites-available/greenlight.http # Preserve used HTTP config for admins.
+  cp -v /etc/nginx/sites-available/greenlight /etc/nginx/sites-available/greenlight.http # Preserve valid HTTP config for admins.
 
   if [ -n "$PROVIDED_CERTIFICATE" ]; then
+      say "Providing SSL certificates for $HOST..."
       mkdir -p "/etc/letsencrypt/live/$HOST" && say "Created $HOST live directory"
-      ln -s /local/certs/fullchain.pem "/etc/letsencrypt/live/$HOST/fullchain.pem" && say "fullchain.pem found and placed"
-      ln -s /local/certs/privkey.pem "/etc/letsencrypt/live/$HOST/privkey.pem" && say "privkey.pem found and placed"
+      ln -sf /local/certs/fullchain.pem "/etc/letsencrypt/live/$HOST/fullchain.pem" && say "fullchain.pem found and placed"
+      ln -sf /local/certs/privkey.pem "/etc/letsencrypt/live/$HOST/privkey.pem" && say "privkey.pem found and placed"
   else
     # Auto generate a standalone SSL x509 certificate publicly signed by Let's encrypt for this domain $HOST.
-    say "Generating SSL certificates for $HOST..."
     say "Rehearsal phase..."
-    if ! certbot --dry-run --email "$EMAIL" --agree-tos --rsa-key-size 4096 -w $ASSETS_DEST \
+    if certbot --staging --email "$EMAIL" --agree-tos --rsa-key-size 4096 -w $ASSETS_DEST \
         -d "$HOST" --deploy-hook "systemctl reload nginx" $LETS_ENCRYPT_OPTIONS certonly; then
-      err "Let's Encrypt SSL (dry-run) request for $HOST did not succeed - exiting"
-    fi
+      say "Generating SSL certificates for $HOST..."
       say "Rehearsal passed, ready to issue production certificates!"
       say "Issuing production certificates..."
-    if ! certbot --email "$EMAIL" --agree-tos --rsa-key-size 4096 -w $ASSETS_DEST \
+      if certbot --force-renewal --email "$EMAIL" --agree-tos --rsa-key-size 4096 -w $ASSETS_DEST \
           -d "$HOST" --deploy-hook "systemctl reload nginx" $LETS_ENCRYPT_OPTIONS certonly; then
-      err "Let's Encrypt SSL request for $HOST did not succeed - exiting"
-    fi
         say "Production SSL certificates has been generated!"
+      else
+        warn "Something went wrong when generating production certificates"
+      fi
+    else
+      warn "Unable to pass the rehearsal phase, avoided generating production certificates to keep rates under limit."
+    fi
   fi
 
-  say "Configuring nginx with SSL enabled..."
+  say "Generating DH key exchange parameters..."
+  cp -v /etc/nginx/ssl/dhp-4096.pem /etc/nginx/ssl/dhp-4096.pem.old
 
-  if [ ! -f /etc/nginx/ssl/dhp-4096.pem ]; then
-    openssl dhparam -dsaparam  -out /etc/nginx/ssl/dhp-4096.pem 4096
+  # For security reasons upon upgrade, the Diffie Hellman key exchange parameters will rotate.
+  if ! openssl dhparam -dsaparam  -out /etc/nginx/ssl/dhp-4096.pem 4096; then
+    warn "Unable to generate DH key exchange parameters - rolling back..."
+    mv -v /etc/nginx/ssl/dhp-4096.pem.old /etc/nginx/ssl/dhp-4096.pem || warn "Unable to generate new DH key exchage parameters nor to recover."
+  else
+    say "DH key exchange parameters was generated!"
   fi
 
+  say "Configuring nginx with SSL enabled..."
+
+  # Updating HTTPS config.
   cat <<HERE > /etc/nginx/sites-available/greenlight
 server_tokens off;
 
@@ -593,15 +638,32 @@ server {
 
 HERE
 
-  if ! nginx -qs reload; then
+  # Validating new config
+  if ! nginx -qt; then
+    # Rollback logic
+    warn "Something went wrong configuring nginx - attempting to recover..."
+
     mv -v /etc/nginx/sites-available/greenlight /etc/nginx/sites-available/greenlight.https # Preserve used HTTPS config for admins.
+
+    if mv -v /etc/nginx/sites-available/greenlight.old /etc/nginx/sites-available/greenlight; then
+      warn "Fallen back to previous configuration!"
+    else
       cp -v /etc/nginx/sites-available/greenlight.http /etc/nginx/sites-available/greenlight # Preserve used HTTP config for admins while falling back to HTTP. 
-    nginx -qs reload
+      warn "No previous configuration was found - fallen back to http configuration!"
+    fi
     
-    err "Unable to configure nginx with certificates - if following the official guides then please contact the maintainers."
+    systemctl restart nginx
+
+    warn "Unable to configure nginx with certificates, retry to resolve."
+
+    return 1
   fi
 
   say "Nginx was configured successuflly with SSL enabled!"
+  systemctl restart nginx && say "Nginx is UP!"
+
+  return 0
+
 }
 
 # Given a container name as $1, this function will check if there's a match for that name in the list of running docker containers on the system.