From 3377c4833ec13da3cfb078182fd83c3dd191a8fd Mon Sep 17 00:00:00 2001 From: Douglas Kerr Date: Tue, 25 Jul 2023 10:07:00 -0700 Subject: [PATCH 1/2] Improve FTP & Ubuntu compatibility for Release 1.2 - Fix FTP masquerade configuration to work on all systems, not just AWS. - Document firewall configuration. - Fix incompatibilites so code works on Unbuntu 18.04 & 20.04 as well as 16.04. - Update relase notes. --- CommunityView/confcvserver/confcvserver.sh | 65 ++++- .../confcvserver/cvserver_example.conf | 27 +++ .../confcvserver/test/testConfcvserver.sh | 229 ++++++++++++++++++ CommunityView/confcvserver/utils.sh | 27 +++ CommunityView/doc/InstallCVDedicated.md | 99 ++++++-- CommunityView/doc/ReleaseNotes.md | 26 +- CommunityView/src/communityview.py | 9 +- 7 files changed, 441 insertions(+), 41 deletions(-) diff --git a/CommunityView/confcvserver/confcvserver.sh b/CommunityView/confcvserver/confcvserver.sh index 2307a4a..51aad1f 100644 --- a/CommunityView/confcvserver/confcvserver.sh +++ b/CommunityView/confcvserver/confcvserver.sh @@ -126,7 +126,7 @@ editsiteconf() { if grep -E '[[:space:]]*' $1 > /dev/null then local ar - ar='/[[:space:]]*/,/[:space:]*<\/Directory>/' + ar='/[[:space:]]*/,/[[:space:]]*<\/Directory>/' sed -i -r "${ar}c\\$block" "$1" else sed -i -r "/[[:space:]]*<\/VirtualHost>/i\\\n$block\n" "$1" @@ -142,9 +142,14 @@ editsiteconf() { # name: value # value may contain spaces and is not followed by a comment. # If the name is not found in the file, append the name-value pair -# to the end of the config file +# to the end of the config file. +# +# When the -r option is used, if the name is found in the file, +# the line containing the name is removed. If the name is not found, +# nothing is changed # # usage: editnpconf filename name value +# or: editnpconf filename -r name # editnpconf() { local cf="$1" @@ -153,6 +158,7 @@ editnpconf() { if [ $# -ne 3 ] then echo "usage: editnpconf filename name value" + echo " or: editnpconf filename -r name" return 1 fi if [ ! -e "$cf" ] @@ -161,6 +167,18 @@ editnpconf() { return 1 fi + # if it's the -r (remove) option, remove the line with the name + if [ "$nm" = "-r" ] + then + nm="$3" + if grep -E "^[[:space:]]*$nm[[:space:]]+" "$cf" > /dev/null + then + sed -i -r "/^([[:space:]]*)$nm([[:space:]]+).+$/d" \ + "$cf" + fi + return + fi + # If the name is found, replace the value while preserving indentation # and spacing. # Otherwise, append the name-value pair to the end of the file @@ -193,6 +211,36 @@ editcrontab() { echo "$tab$2" | crontab - } +# set up the appropriate FTP masquerade address in proftpd conf file +# +# usage: set_up_ftp_masquerade proftpd_config_file +# +set_up_ftp_masquerade() { + local proftpcf="$1" + local masq=`get_config $confile masquerade` + if [ "$masq" = "" ] # no masquerade spec + then + local extip + if extip=`get_external_ip` + then + editnpconf "$proftpcf" MasqueradeAddress $extip + else + echo "Cannot determine external IP address--set masquerade \c" + echo "variable in $confile" + return 1 + fi + elif [ "$masq" = "localif" ] # remove name & let server use local i/f's ip + then + editnpconf "$proftpcf" -r MasqueradeAddress + elif is_ip_addr_form "$masq" + then + editnpconf "$proftpcf" MasqueradeAddress "$masq" + else + echo "masquerade value is not an ip address" + return 1 + fi +} + # take the config information and build the server # configure() { @@ -353,15 +401,10 @@ configure() { # limit the upload user's group (==username) to the html subdir of /var/www editnpconf $cf DefaultRoot "~/html $up_user" # set the passive port range; must agree w/ firewall rules for this server - editnpconf $cf PassivePorts "60000 60999" - # if we're running in an AWS EC2 instance, get the public IP address + editnpconf $cf PassivePorts "60000 60099" + # set up the proftpd MasqueradeAddress spec # so proftpd can tell the client how to do passive mode - local pubip - if pubip=`curl -s -m 4 \ - http://169.254.169.254/latest/meta-data/public-ipv4` - then # sucess, we're on AWS; add the Masquerade line - editnpconf $cf MasqueradeAddress "$pubip" - fi + set_up_ftp_masquerade $cf # turn off the log files so the root fs will not fill up editnpconf $cf SystemLog none @@ -376,7 +419,7 @@ configure() { task="installing Python and its imaging library" echo "***** $task" | tee /dev/tty - install "python python-imaging" + install "python python-pil" task="installing and configuring CommunityView server" echo "***** $task" | tee /dev/tty diff --git a/CommunityView/confcvserver/cvserver_example.conf b/CommunityView/confcvserver/cvserver_example.conf index 68b4154..2d43cac 100644 --- a/CommunityView/confcvserver/cvserver_example.conf +++ b/CommunityView/confcvserver/cvserver_example.conf @@ -34,6 +34,33 @@ timezone=America/Los_Angeles up_user=upload_user_name up_pass=upload_password +# Specify an IP address for the FTP server to tell the client to connect to +# when setting up a file transfer. +# +# Files are uploaded via passive FTP, which means the client establishes the +# file transfer data connetion, rather than the server as in active FTP. In +# passive FTP, the server sends the IP address the client must connect to for +# file transfer on the control connection. If the server is behind a NAT +# firewall, the server must know its external (outside the firewall) address so +# it can send the correct IP address. This external address is the +# "masquerade" address. +# +# If the value "localif" is specified instead of an IP address, the server will +# use the IP address of the local interface that the FTP request is received +# on. +# +# Examples: +# +# masquerade=1.2.3.4 +# +# masquerade=localif +# +# If no masquerade value is specified, the installation script attempts to +# determine the external IP address of the server and specifies that as the +# masquerade address to the FTP server. This is usually a good choice. +# +# masquerade=your_external_ip + # Specify the number of days of images this server should retain retain_days=21 diff --git a/CommunityView/confcvserver/test/testConfcvserver.sh b/CommunityView/confcvserver/test/testConfcvserver.sh index bb01e2d..feef055 100644 --- a/CommunityView/confcvserver/test/testConfcvserver.sh +++ b/CommunityView/confcvserver/test/testConfcvserver.sh @@ -63,6 +63,7 @@ test_editnpconf_name_value_editing() { editnpconf $tcf ExistingDirectiveWithSpecialChars '~/foo bar' # excpected conf file after editing + local o="" o=${o}'# Test conf file\n' o=${o}'# DefaultRoot <- this should not get edited\n' o=${o}'# next line is indented and contains multiple spaces\n' @@ -88,6 +89,48 @@ test_editnpconf_name_value_editing() { fi } +test_editnpconf_name_removal() { + # initial conf file before editing + local i="" + i=${i}'# Test conf file\n' + i=${i}'# DefaultRoot <- this should not get edited\n' + i=${i}'# next line is indented and contains multiple spaces\n' + i=${i}' PassivePorts 100 200\n' + i=${i}'\tNameSurroundedByTabs\t100 200\n' + i=${i}'ExistingDirectiveWithSpecialChars ~\n' + i=${i}'MasqueradeAddress blah\n' + i=${i}'# End of initial conf file\n' + local tcf=unit_test_temp_conf_file2 + local tof=unit_test_temp_orig_file2 + /bin/echo -ne "$i" > $tcf + /bin/echo -ne "$i" > $tof # for debugging + + editnpconf $tcf -r MasqueradeAddress + editnpconf $tcf -r NonextantName + + # excpected conf file after editing + local o="" + o=${o}'# Test conf file\n' + o=${o}'# DefaultRoot <- this should not get edited\n' + o=${o}'# next line is indented and contains multiple spaces\n' + o=${o}' PassivePorts 100 200\n' + o=${o}'\tNameSurroundedByTabs\t100 200\n' + o=${o}'ExistingDirectiveWithSpecialChars ~\n' + o=${o}'# End of initial conf file\n' + local tef=unit_test_temp_expctd_file2 + /bin/echo -ne "$o" > $tef + + local diffs=`diff $tef $tcf` + local status=$? + if [ $status -ne 0 ] + then + fail "Output differs from expected:" + echo "$diffs" + else + rm $tef $tcf $tof + fi +} + # # Test the editcrontab function for proper operation # @@ -181,4 +224,190 @@ test_editcrontab_bad_args() { _restore_crontab } +test_is_ip_addr_form() { + local goodaddr="\ + 127.0.0.1 \ + 111.111.111.111 \ + 1.2.3.4 \ + 01.02.03.04 \ + 999.999.999.999 \ + " + local badaddr="\ + 1.2.3 \ + 1.2.3.4.5 \ + 1.2..3 \ + 1.a.b.3 \ + 1 \ + 1/4 \ + 1/2/3/4 \ + 1.4 \ + 1.2a.3.4 \ + "" \ + 1..2 \ + a \ + " + + local addr + for addr in $goodaddr + do + assertTrue "is_ip_addr_form fails on good addr \"$addr\"" \ + "is_ip_addr_form $addr" + done + for addr in $badaddr + do + assertFalse "is_ip_addr_form fails on bad addr \"$addr\"" \ + "is_ip_addr_form $addr" + done +} + +_proftpd_testconf() { + local i="" + i=${i}'# Test conf file\n' + i=${i}'# DefaultRoot <- this should not get edited\n' + i=${i}'# next line is indented and contains multiple spaces\n' + i=${i}' PassivePorts 100 200\n' + i=${i}'\tNameSurroundedByTabs\t100 200\n' + i=${i}'ExistingDirectiveWithSpecialChars ~\n' + i=${i}'# End of initial conf file\n' + /bin/echo -ne "$i" +} + +test_set_up_ftp_masquerade_nospec() { + local tcf=unit_test_temp_conf_file3 + local tof=unit_test_temp_orig_file3 + local tef=unit_test_temp_expctd_file3 + local tcvcf=unit_test_temp_cvconf_file3 + + # initial conf file before editing + _proftpd_testconf > $tcf + cat $tcf > $tof # for debugging + + # empty cvserver.conf file + echo "" > $tcvcf + + confile=$tcvcf + set_up_ftp_masquerade $tcf + + # excpected conf file after editing + _proftpd_testconf > $tef + echo MasqueradeAddress `get_external_ip` >> $tef + + local diffs=`diff $tef $tcf` + local status=$? + if [ $status -ne 0 ] + then + fail "Output differs from expected:" + echo "$diffs" + else + rm $tef $tcf $tof $tcvcf + fi +} + +test_set_up_ftp_masquerade_localif() { + local tcf=unit_test_temp_conf_file4 + local tof=unit_test_temp_orig_file4 + local tef=unit_test_temp_expctd_file4 + local tcvcf=unit_test_temp_cvconf_file4 + + # initial conf file before editing + _proftpd_testconf > $tcf + echo "MasqueradeAddress 1.1.1.1" >> $tcf + cat $tcf > $tof # for debugging + + # cvserver.conf file with masquerade value + echo "masquerade=localif" > $tcvcf + + confile=$tcvcf + set_up_ftp_masquerade $tcf + + # excpected conf file after editing + _proftpd_testconf > $tef + + local diffs=`diff $tef $tcf` + local status=$? + if [ $status -ne 0 ] + then + fail "Output differs from expected:" + echo "$diffs" + else + rm $tef $tcf $tof $tcvcf + fi +} + +test_set_up_ftp_masquerade_badip() { + local tcf=unit_test_temp_conf_file5 + local tof=unit_test_temp_orig_file5 + local tef=unit_test_temp_expctd_file5 + local tcvcf=unit_test_temp_cvconf_file5 + + # initial conf file before editing + _proftpd_testconf > $tcf + cat $tcf > $tof # for debugging + + # cvserver.conf file with masquerade value + echo "masquerade=0.0.0.0.0" > $tcvcf + + confile=$tcvcf + if set_up_ftp_masquerade $tcf > /dev/null + then + echo "set_up_ftp_masquerade returned success on bad IP address" + return 1 + fi + + # excpected conf file after editing + _proftpd_testconf > $tef + + local diffs=`diff $tef $tcf` + local status=$? + if [ $status -ne 0 ] + then + fail "Output differs from expected:" + echo "$diffs" + else + rm $tef $tcf $tof $tcvcf + fi +} + +test_set_up_ftp_masquerade_ipspec() { + local tcf=unit_test_temp_conf_file6 + local tof=unit_test_temp_orig_file6 + local tef=unit_test_temp_expctd_file6 + local tcvcf=unit_test_temp_cvconf_file6 + + # initial conf file before editing + _proftpd_testconf > $tcf + cat $tcf > $tof # for debugging + + # cvserver.conf file with masquerade value + echo "masquerade=1.2.3.4" > $tcvcf + + confile=$tcvcf + set_up_ftp_masquerade $tcf + + # excpected conf file after editing + _proftpd_testconf > $tef + echo "MasqueradeAddress 1.2.3.4" >> $tef + + local diffs=`diff $tef $tcf` + local status=$? + if [ $status -ne 0 ] + then + fail "Output differs from expected:" + echo "$diffs" + else + rm $tef $tcf $tof $tcvcf + fi +} + +test_get_external_ip() { + local result + result=`get_external_ip` + local status=$? + assertTrue "get_external_ip returns failure status" "$status" + assertNotNull "get_external_ip outputs empty string" "$result" + local ip=`wget -q -O - https://api.ipify.org` + assertEquals "get_external_ip returns wrong addr" "$result" "$ip" +} + + . `which shunit2` diff --git a/CommunityView/confcvserver/utils.sh b/CommunityView/confcvserver/utils.sh index b43931b..51ff22f 100644 --- a/CommunityView/confcvserver/utils.sh +++ b/CommunityView/confcvserver/utils.sh @@ -83,3 +83,30 @@ create_dir() { done } +# get our external (outside any firewall) IP address and print to +# stdout. If we can't get the address, return failure status +# +get_external_ip() { + if curl -s -m 4 \ + http://169.254.169.254/latest/meta-data/public-ipv4 # AWS + then + : # no action + elif wget -q -O - https://api.ipify.org # generic + then + : # no action + else + return 1 + fi +} + +# return true if the argument is in the form of an IPv4 address. +# Only checks for proper form--does not check for improper numbers, e.g., +# 999.0.0.0. +# +# usage: is_ip_addr_form argument +# +is_ip_addr_form() { + echo "$1" | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' \ + > /dev/null +} + diff --git a/CommunityView/doc/InstallCVDedicated.md b/CommunityView/doc/InstallCVDedicated.md index 96ddb7d..db8e49f 100644 --- a/CommunityView/doc/InstallCVDedicated.md +++ b/CommunityView/doc/InstallCVDedicated.md @@ -4,10 +4,18 @@ This document describes the steps to install Neighborhood Guard's CommunityView software on a dedicated server, either physical or virtual. It assumes -you have root access to the server. This process was developed for Ubuntu -16.04 LTS and has been tested on dedicated, virtual x86 machines and on Amazon +you have root access to the server. + +This process was developed on Ubuntu +16.04 LTS and has been tested and run extensively +on dedicated, virtual and physical x86 machines and on Amazon Web Services (AWS) EC2 virtual machines loaded with Amazon's Ubuntu Server 16.04 LTS AMD64 20180814 AMI (ami-51537029). +It has also been tested on Ubuntu 18.04 LTS and 20.04 LTS. + +CommunityView does not yet run on Ubuntu 22.04 LTS. Ubuntu 22.04 +switches to Python 3 as the standard Python and there are multiple +compatibility issues with CommunityView to be resolved. If you are installing on a shared-hosting server, please see [Installing CommunityView on a Shared-Hosting Server](InstallCommunityView.md). @@ -15,21 +23,23 @@ If you are installing on a shared-hosting server, please see ### Overview Broadly speaking, the installation process consists of the following steps: -1. Install Ubuntu Server 16.04 LTS on your (virtual or physical) machine. +1. Install Ubuntu Server 16.04, 18.04 or 20.04 +on your (virtual or physical) machine. 2. Configure the machine with appropriate disk storage. -3. Download the CommunityView software. -4. Create and edit the `cvserver.conf` file, which provides the configuration +3. Configure the firewall. +4. Download the CommunityView software. +5. Create and edit the `cvserver.conf` file, which provides the configuration information for the installation. -5. Run the `confcvserver.sh` script to install, configure and run all +6. Run the `confcvserver.sh` script to install, configure and run all necessary software for a ComunityView server. -### 1. Install Ubuntu Server 16.04 LTS +### 1. Install Ubuntu Server 16.04, 18.04 or 20.04 There are many tutorials and guides to installing the Ubuntu server software -on the Web. Search for "install ubuntu server 16.04" using your favorite -search engine to find them. The official Ubuntu tutorial is here: [https://tutorials.ubuntu.com/tutorial/tutorial-install-ubuntu-server-1604](https://tutorials.ubuntu.com/tutorial/tutorial-install-ubuntu-server-1604). +on the Web. Search for "install ubuntu server" using your favorite +search engine to find them. -After installing a fresh Ubuntu Server 16.04 (or any Linux system), +After installing a Ubuntu Server 16.04, 18.04 or 20.04, it's a good idea to bring the system up to date with the latest system software. To do this, log into the server and execute the following two commands: @@ -82,7 +92,24 @@ A simple way to make this change permanent is to edit `/etc/rc.local`, and add the command to the end of the file. This will cause the command to be executed each time the system is rebooted. -### 3. Download the CommunityView Software +### 3. Configure the Firewall + +The exact steps to configure the firewall protecting the CommunityView server +depend on the hosting environment in which the server is running, e.g., AWS, +DreamHost, etc. Please consult the documentation for your hosting environment +for the steps involved. + +The following ports must be open for their respective protocols in order +for the CommunityView server to operate. + +| Port | Protocol | Description | +| ----------:|:--------:| ----------- | +| 21|TCP | FTP control connection | +| 22|TCP | Secure Shell (SSH) | +| 80|TCP | CommunityView website (Apache) +| 60000-60099|TCP | FTP data connection range| + +### 4. Download the CommunityView Software Log into the server and use the following command to download a ZIP archive of the CommunityView @@ -101,7 +128,7 @@ Then, extract the files in the downloaded ZIP archive with this command: This will create the `communityview-master` directory in the current directory which contains the installation files. -### 4. Create and Edit the `cvserver.conf` File +### 5. Create and Edit the `cvserver.conf` File Change to the directory that will contain the `cvserver.conf` file: @@ -156,6 +183,30 @@ CommunityView server. If the FTP user name is "ng_user" and the password is up_user=ng_user up_pass=secretcode +#### FTP Masquerade Address + +Image files are uploaded by `ftp_upload` +using passive FTP, which means the client establishes the +file transfer data connection, rather than the server as in active FTP. In +passive FTP, the server sends the IP address the client must connect to for +file transfer on the control connection. If the server is behind a NAT +firewall, the server must know its external (outside the firewall) address so +it can send the correct IP address. This external address is the +"masquerade" address. + +If the value "localif" is specified instead of an IP address, the server will +use the IP address of the local interface that the FTP request is received +on. Examples: + + masquerade=1.2.3.4 + masquerade=localif + +If no masquerade value is specified, the installation script attempts to +determine the external IP address of the server and specifies that as the +masquerade address to the FTP server. It is usually a good choice to +not specify a masquerade value and let the installation software determine +this for the FTP server. + #### Number of Days of Images to Retain on the CommunityView As noted above, `retain_days` sets the number of days of @@ -206,7 +257,7 @@ navigation page, but otherwise does not matter. When you have finished editing the cvserver.conf file, write it out to the `confcvserver` directory. -### Run the `confcvserver.sh` Script +### 6. Run the `confcvserver.sh` Script Once you have edited the `cvserver.conf` file to reflect your setup, run the `confcvserver.sh` script as root by issuing the following command: @@ -217,21 +268,19 @@ This will install and configure all the software required to implement your CommunityView server. At the end of the installation process, the script will create a private key -for the upload user account (the account -named in the `up_user` configuration line above), -if one does not already exist. -The installation script will place the key file in the current directory -and print an informational message to that effect. -If your upload machine authenticates itself to the CommunityView server -via Public Key authentication, use this key as the upload machine's -private key. If not, you can ignore this message. - -After running the script, if you find you need to change any of the -configuration items, simply edit the configuration file and run the script +for the upload user account (the account named in the `up_user` configuration +line above), if one does not already exist. The installation script will place +the key file in the current directory and print an informational message to +that effect. This key is needed by the `cktunnel` and `starttunnel` scripts +that are part of the remote access mechanism included in the `FTP_Upload` +software. It can also be used for ssh login access to the `up_user` account. + +After running the installation script, if you find you need to change any of +the configuration items, simply edit the configuration file and run the script again. If the script encounters errors, examine the error messages, edit the cvserver.conf file as appropriate, then re-run the `confcvserver.sh` script. If you cannot determine the cause of an error and are a member of Neighborhood Guard, please contact one of the -Neighborhood Guard board members for assistance. \ No newline at end of file +Neighborhood Guard board members for assistance. diff --git a/CommunityView/doc/ReleaseNotes.md b/CommunityView/doc/ReleaseNotes.md index 09a628e..be6fe0e 100644 --- a/CommunityView/doc/ReleaseNotes.md +++ b/CommunityView/doc/ReleaseNotes.md @@ -1,5 +1,30 @@ # Release Notes for CommunityView # +## v1.2.0 - 2023/07/25 +_Doug Kerr_ + +### Changes + +- CommunityView now runs on Ubuntu 16.04, 18.04 and 20.04. +- Improve FTP masquerade. Should automatically select proper masquerade +address for passive FTP on any system. Formerly only worked correctly on AWS. +- Document FTP masquerade configuration and required firewall settings in +dedicated-installation document. +- Reduce passive-FTP port range to 60000-60099. +- Fix incompatibilities with Ubuntu 18.04 and 20.04. + +### To Do + +* Update CommunityView to run on Ubuntu 22.04. +* Make CommunityView remove oldest images if disk space nearly full. +* Move sequence-page navigation to top of page. +* Add UI link to source code. +* Add graceful shutdown. + +### Known Issues + +* The `Next day` links in day pages are sometimes incorrectly grayed out. + ## v1.1.0 - 2020/03/09 _Doug Kerr_ @@ -13,7 +38,6 @@ _Doug Kerr_ * Move sequence-page navigation to top of page. * Add UI link to source code. * Add graceful shutdown. -* Update communityview.py to use config file ### Known Issues diff --git a/CommunityView/src/communityview.py b/CommunityView/src/communityview.py index 8abdb8e..41225b4 100644 --- a/CommunityView/src/communityview.py +++ b/CommunityView/src/communityview.py @@ -30,10 +30,11 @@ import os -import Image -import ImageChops -import ImageOps -import ImageDraw +#import Image +from PIL import Image +from PIL import ImageChops +from PIL import ImageOps +from PIL import ImageDraw import shutil import datetime import re From 5bdc8b47c8e87817f63e1472cebe9c17f114057d Mon Sep 17 00:00:00 2001 From: Douglas Kerr Date: Tue, 25 Jul 2023 10:37:41 -0700 Subject: [PATCH 2/2] Update version strings for Release 1.2 --- CommunityView/confcvserver/confcvserver.sh | 2 +- CommunityView/src/communityview.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CommunityView/confcvserver/confcvserver.sh b/CommunityView/confcvserver/confcvserver.sh index 51aad1f..8096b15 100644 --- a/CommunityView/confcvserver/confcvserver.sh +++ b/CommunityView/confcvserver/confcvserver.sh @@ -28,7 +28,7 @@ # CommunityView software. # version of the confcvserver software -version="1.1.0" +version="1.2.0" . ./utils.sh #. ./confui.sh diff --git a/CommunityView/src/communityview.py b/CommunityView/src/communityview.py index 41225b4..954f9d1 100644 --- a/CommunityView/src/communityview.py +++ b/CommunityView/src/communityview.py @@ -26,7 +26,7 @@ # # ################################################################################ -version_string = "1.1.0" +version_string = "1.2.0" import os