feature: TOASTER_WEBMAIL_PROXY=nginx (#595)

- feat: as an alternative to haproxy, use the installed nginx as the
reverse proxy
  - nginx: support TOASTER_WEBMAIL_PROXY=nginx
  - webmail: add https proxy config, for TOASTER_WEBMAIL_PROXY=nginx
- previously, haproxy sent proxy_protocol v2. Eventually, everything
should use Forwarded (RFC 7239), for now we use the lowest common
denominator: X-Forwarded-For
- fix: newsyslog.conf.d files must end with .conf
- fix(mail_dmarc): install .ini file in correct location
- data: standardize dirs in /data
  - mt: stage_setup_tls populates /data/etc/tls
  - dovecot: /data/etc/ssl -> /data/etc/tls
  - haproxy: /data/ssl.d -> /data/etc/tls.d
  - webmail: acme document root /data -> /data/htdocs
- letsencrypt: install acme.sh from ports
- letsencrypt: updates for etc/tls, add webmail deploy script
- mua: for testing, find or install curl, fixes #594
- mysql: use jail_is_running (noop)
- roundcube: configure SA user prefs plugin
- mt: added stage_enable_newsyslog, used in nginx, php, haraka, mongodb,
redis, rspamd, spamassassin
- geoip: switch updater to geoipupdate, fixes #591 

Checklist:
- [x] docs up-to-date

include/mua.sh

@@ -40,22 +40,26 @@ test_imap_openssl()
 EOF
 }
 
-test_imap_curl()
+get_curl_cmd()
 {
- local _test_uri
- _test_uri="imaps://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
-
  if [ -x /usr/local/bin/curl ]; then
- curl -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+ echo "curl"
  elif [ -x "$STAGE_MNT/usr/local/bin/curl" ]; then
- _test_uri="imaps://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@localhost/"
- stage_exec curl -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+ echo "stage_exec curl"
  else
- pkg install -y curl
- curl -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+ pkg install -qy curl
+ echo "curl"
  fi
 }
 
+test_imap_curl()
+{
+ local _test_uri
+ _test_uri="imaps://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+ _curl_cmd="$(get_curl_cmd)"
+ $_curl_cmd -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+}
+
 test_imap()
 {
  echo "testing IMAP AUTH as $MUA_TEST_USER"
@@ -90,9 +94,11 @@ test_pop3_empty()
 
 test_pop3()
 {
+ local _test_uri
+ _test_uri="pop3s://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+ _curl_cmd="$(get_curl_cmd)"
  # shellcheck disable=2001
- curl -k -v --login-options 'AUTH=PLAIN' \
- "pop3s://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+ $_curl_cmd -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
 }
 
 uriencode() {

include/nginx.sh

@@ -37,8 +37,10 @@ install_nginx()
 
 install_nginx_newsyslog()
 {
+ stage_enable_newsyslog
+
  tell_status "enabling nginx log file rotation"
- tee "$STAGE_MNT/etc/newsyslog.conf.d/nginx" <<EO_NG_NSL
+ tee "$STAGE_MNT/etc/newsyslog.conf.d/nginx.conf" <<EO_NG_NSL
 # rotate nightly (default)
 /var/log/nginx/*.log root:wheel 644 7 * @T00 BCGX /var/run/nginx.pid 30
 
@@ -69,10 +71,20 @@ configure_nginx_server_d()
  fi
 
  # most calls get enclosing server block
- local _prefix='server {
- listen 80 proxy_protocol;
- listen [::]:80 proxy_protocol;
+ local _prefix
+ if [ "$TOASTER_WEBMAIL_PROXY" = "haproxy" ]; then
+ _prefix='server {
+ listen 80;
+ listen [::]:80;
+'
+ else
+ # nginx can't send proxy protocol AND route URIs at the same time
+ _prefix='server {
+ listen 80;
+ listen [::]:80;
 '
+ fi
+
  local _suffix='location ~ /\.ht {
  deny all;
  }
@@ -136,8 +148,10 @@ http {
  keepalive_timeout 65;
 
  set_real_ip_from $(get_jail_ip haproxy);
+ set_real_ip_from $(get_jail_ip webmail);
  set_real_ip_from $(get_jail_ip6 haproxy);
- real_ip_header proxy_protocol;
+ set_real_ip_from $(get_jail_ip6 webmail);
+ real_ip_header X-Forwarded-For;
  real_ip_recursive on;
  client_max_body_size 25m;
 

include/php.sh

@@ -29,8 +29,10 @@ install_php()
 }
 
 install_php_newsyslog() {
+ stage_enable_newsyslog
+
  tell_status "enabling PHP-FPM log file rotation"
- tee "$STAGE_MNT/etc/newsyslog.conf.d/php-fpm" <<EO_FPM_NSL
+ tee "$STAGE_MNT/etc/newsyslog.conf.d/php-fpm.conf" <<EO_FPM_NSL
 # rotate the file after it reaches 1M
 /var/log/php-fpm.log 600 7 1024 * BCX /var/run/php-fpm.pid 30
 EO_FPM_NSL

include/shell.sh

@@ -51,7 +51,7 @@ export HISTIGNORE="&:[bf]g:exit"
 
 shopt -s histappend
 shopt -s cdspell
-set -o vi
+#set -o vi
 
 if [[ $- == *i* ]]
 then

mail-toaster.sh

@@ -88,7 +88,10 @@ export TOASTER_PKG_BRANCH="latest"
 export TOASTER_USE_TMPFS="0"
 export TOASTER_VPOPMAIL_CLEAR="1"
 export TOASTER_VPOPMAIL_EXT="0"
+export TOASTER_WEBMAIL_PROXY="haproxy"
 export CLAMAV_FANGFRISCH="0"
+export GEOIP_UPDATER="geoipupdate"
+export MAXMIND_ACCOUNT_ID=""
 export MAXMIND_LICENSE_KEY=""
 export ROUNDCUBE_SQL="0"
 export ROUNDCUBE_DEFAULT_HOST=""
@@ -188,6 +191,7 @@ export TOASTER_USE_TMPFS=${TOASTER_USE_TMPFS:="0"}
 export TOASTER_VPOPMAIL_CLEAR=${TOASTER_VPOPMAIL_CLEAR:="1"}
 export TOASTER_VPOPMAIL_EXT=${TOASTER_VPOPMAIL_EXT:="0"}
 export TOASTER_VQADMIN=${TOASTER_VQADMIN:="0"}
+export TOASTER_WEBMAIL_PROXY=${TOASTER_WEBMAIL_PROXY:="haproxy"}
 export CLAMAV_FANGFRISCH=${CLAMAV_FANGFRISCH:="0"}
 export CLAMAV_UNOFFICIAL=${CLAMAV_UNOFFICIAL:="0"}
 export ROUNDCUBE_SQL=${ROUNDCUBE_SQL:="$TOASTER_MYSQL"}
@@ -1012,6 +1016,38 @@ stage_fbsd_package()
  echo "done"
 }
 
+stage_setup_tls()
+{
+ # static TLS certificates (installed at deploy)
+ if [ ! -f "$STAGE_MNT/etc/ssl/certs/${TOASTER_MAIL_DOMAIN}.pem" ]; then
+ tell_status "installing TLS certificate"
+ cp /etc/ssl/certs/server.crt "$STAGE_MNT/etc/ssl/certs/${TOASTER_MAIL_DOMAIN}.pem"
+ cp /etc/ssl/private/server.key "$STAGE_MNT/etc/ssl/private/${TOASTER_MAIL_DOMAIN}.pem"
+ fi
+
+ # dynamic TLS certs, kept up-to-date by acme.sh or certbot
+ if [ ! -f "$STAGE_MNT/data/etc/tls/certs" ]; then
+ # shellcheck disable=SC2174
+ mkdir -m 0644 -p "$STAGE_MNT/data/etc/tls/certs"
+ cp /etc/ssl/certs/server.crt "$STAGE_MNT/data/etc/tls/certs/${TOASTER_MAIL_DOMAIN}.pem"
+ fi
+
+ if [ ! -f "$STAGE_MNT/data/etc/tls/private" ]; then
+ # shellcheck disable=SC2174
+ mkdir -m 0640 -p "$STAGE_MNT/data/etc/tls/private"
+ cp /etc/ssl/private/server.key "$STAGE_MNT/data/etc/tls/private/${TOASTER_MAIL_DOMAIN}.pem"
+ fi
+}
+
+stage_enable_newsyslog()
+{
+ tell_status "enabling newsyslog"
+ sysrc -f "$STAGE_MNT/etc/rc.conf" newsyslog_enable=YES
+ sed -i.bak \
+ -e '/^#0.*newsyslog/ s/^#0/0/' \
+ "$STAGE_MNT/etc/crontab"
+}
+
 unmount_data()
 {
  # $1 is ZFS fs (eg: /data/mysql)
@@ -1348,7 +1384,8 @@ assure_jail()
  fi
 }
 
-preserve_file() {
+preserve_file()
+{
  local _jail_name=$1
  local _file_path=$2
 

provision/base.sh

@@ -125,7 +125,7 @@ do
 done
 
 # packages to be updated automatically
-auto_upgrade="curl expat libxml2 pkg sudo vim-tiny"
+auto_upgrade="curl expat libxml2 pkg sudo unbound vim-tiny"
 
 # add packages with:
 # sysrc -f /usr/local/etc/periodic/daily/auto_security_upgrades auto_upgrade+=" $NEW"
@@ -137,17 +137,15 @@ done
 EO_PKG_SECURITY
 }
 
-configure_ssl_dirs()
+configure_tls_dirs()
 {
  if [ ! -d "$BASE_MNT/etc/ssl/certs" ]; then
- mkdir "$BASE_MNT/etc/ssl/certs"
+ mkdir -m 0644 "$BASE_MNT/etc/ssl/certs"
  fi
 
  if [ ! -d "$BASE_MNT/etc/ssl/private" ]; then
- mkdir "$BASE_MNT/etc/ssl/private"
+ mkdir -m 0640 "$BASE_MNT/etc/ssl/private"
  fi
-
- chmod o-r "$BASE_MNT/etc/ssl/private"
 }
 
 configure_tls_dhparams()
@@ -215,7 +213,7 @@ configure_base()
  update_motd=NO
 
  configure_pkg_latest "$BASE_MNT"
- configure_ssl_dirs
+ configure_tls_dirs
  configure_tls_dhparams
  disable_cron_jobs
  enable_security_periodic

provision/dovecot.sh

@@ -112,13 +112,13 @@ protocol lmtp {
 }
 
 # default TLS certificate (no SNI)
-ssl_cert = </data/etc/ssl/certs/dovecot.pem
-ssl_key = </data/etc/ssl/private/dovecot.pem
+ssl_cert = </data/etc/tls/certs/dovecot.pem
+ssl_key = </data/etc/tls/private/dovecot.pem
 
 # example TLS SNI (see https://wiki.dovecot.org/SSL/DovecotConfiguration)
 #local_name mail.example.com {
-# ssl_cert = </data/etc/ssl/certs/mail.example.com.pem
-# ssl_key = </data/etc/ssl/private/mail.example.com.pem
+# ssl_cert = </data/etc/tls/certs/mail.example.com.pem
+# ssl_key = </data/etc/tls/private/mail.example.com.pem
 #}
 
 # dovecot 2.3+ supports a ssl_dh file
@@ -322,7 +322,7 @@ configure_tls_certs()
  "$_localconf"
  fi
 
- local _ssldir="$ZFS_DATA_MNT/dovecot/etc/ssl"
+ local _ssldir="$ZFS_DATA_MNT/dovecot/etc/tls"
  if [ ! -d "$_ssldir/certs" ]; then
  # shellcheck disable=SC2174
  mkdir -m 644 -p "$_ssldir/certs"
@@ -351,8 +351,8 @@ configure_postfix_with_sasl()
  stage_exec postconf -e 'smtpd_sasl_path = private/auth'
  stage_exec postconf -e 'smtpd_sasl_auth_enable = yes'
  stage_exec postconf -e 'smtpd_recipient_restrictions = permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination'
- stage_exec postconf -e "smtpd_tls_cert_file = /data/etc/ssl/certs/$TOASTER_HOSTNAME.pem"
- stage_exec postconf -e "smtpd_tls_key_file = /data/etc/ssl/private/$TOASTER_HOSTNAME.pem"
+ stage_exec postconf -e "smtpd_tls_cert_file = /data/etc/tls/certs/$TOASTER_HOSTNAME.pem"
+ stage_exec postconf -e "smtpd_tls_key_file = /data/etc/tls/private/$TOASTER_HOSTNAME.pem"
  stage_exec postconf -e 'smtp_tls_security_level = may'
 
  for _s in 512 1024 2048; do