diff --git a/CommunityView/confcvserver/confcvserver.sh b/CommunityView/confcvserver/confcvserver.sh
index 2307a4a..8096b15 100644
--- a/CommunityView/confcvserver/confcvserver.sh
+++ b/CommunityView/confcvserver/confcvserver.sh
@@ -28,7 +28,7 @@
# CommunityView software.
# version of the confcvserver software
-version="1.1.0"
+version="1.2.0"
. ./utils.sh
#. ./confui.sh
@@ -126,7 +126,7 @@ editsiteconf() {
if grep -E '[[:space:]]*' $1 > /dev/null
then
local ar
- ar='/[[:space:]]*/,/[:space:]*<\/Directory>/'
+ ar='/[[:space:]]*/,/[[:space:]]*<\/Directory>/'
sed -i -r "${ar}c\\$block" "$1"
else
sed -i -r "/[[:space:]]*<\/VirtualHost>/i\\\n$block\n" "$1"
@@ -142,9 +142,14 @@ editsiteconf() {
# name: value
# value may contain spaces and is not followed by a comment.
# If the name is not found in the file, append the name-value pair
-# to the end of the config file
+# to the end of the config file.
+#
+# When the -r option is used, if the name is found in the file,
+# the line containing the name is removed. If the name is not found,
+# nothing is changed
#
# usage: editnpconf filename name value
+# or: editnpconf filename -r name
#
editnpconf() {
local cf="$1"
@@ -153,6 +158,7 @@ editnpconf() {
if [ $# -ne 3 ]
then
echo "usage: editnpconf filename name value"
+ echo " or: editnpconf filename -r name"
return 1
fi
if [ ! -e "$cf" ]
@@ -161,6 +167,18 @@ editnpconf() {
return 1
fi
+ # if it's the -r (remove) option, remove the line with the name
+ if [ "$nm" = "-r" ]
+ then
+ nm="$3"
+ if grep -E "^[[:space:]]*$nm[[:space:]]+" "$cf" > /dev/null
+ then
+ sed -i -r "/^([[:space:]]*)$nm([[:space:]]+).+$/d" \
+ "$cf"
+ fi
+ return
+ fi
+
# If the name is found, replace the value while preserving indentation
# and spacing.
# Otherwise, append the name-value pair to the end of the file
@@ -193,6 +211,36 @@ editcrontab() {
echo "$tab$2" | crontab -
}
+# set up the appropriate FTP masquerade address in proftpd conf file
+#
+# usage: set_up_ftp_masquerade proftpd_config_file
+#
+set_up_ftp_masquerade() {
+ local proftpcf="$1"
+ local masq=`get_config $confile masquerade`
+ if [ "$masq" = "" ] # no masquerade spec
+ then
+ local extip
+ if extip=`get_external_ip`
+ then
+ editnpconf "$proftpcf" MasqueradeAddress $extip
+ else
+ echo "Cannot determine external IP address--set masquerade \c"
+ echo "variable in $confile"
+ return 1
+ fi
+ elif [ "$masq" = "localif" ] # remove name & let server use local i/f's ip
+ then
+ editnpconf "$proftpcf" -r MasqueradeAddress
+ elif is_ip_addr_form "$masq"
+ then
+ editnpconf "$proftpcf" MasqueradeAddress "$masq"
+ else
+ echo "masquerade value is not an ip address"
+ return 1
+ fi
+}
+
# take the config information and build the server
#
configure() {
@@ -353,15 +401,10 @@ configure() {
# limit the upload user's group (==username) to the html subdir of /var/www
editnpconf $cf DefaultRoot "~/html $up_user"
# set the passive port range; must agree w/ firewall rules for this server
- editnpconf $cf PassivePorts "60000 60999"
- # if we're running in an AWS EC2 instance, get the public IP address
+ editnpconf $cf PassivePorts "60000 60099"
+ # set up the proftpd MasqueradeAddress spec
# so proftpd can tell the client how to do passive mode
- local pubip
- if pubip=`curl -s -m 4 \
- http://169.254.169.254/latest/meta-data/public-ipv4`
- then # sucess, we're on AWS; add the Masquerade line
- editnpconf $cf MasqueradeAddress "$pubip"
- fi
+ set_up_ftp_masquerade $cf
# turn off the log files so the root fs will not fill up
editnpconf $cf SystemLog none
@@ -376,7 +419,7 @@ configure() {
task="installing Python and its imaging library"
echo "***** $task" | tee /dev/tty
- install "python python-imaging"
+ install "python python-pil"
task="installing and configuring CommunityView server"
echo "***** $task" | tee /dev/tty
diff --git a/CommunityView/confcvserver/cvserver_example.conf b/CommunityView/confcvserver/cvserver_example.conf
index 68b4154..2d43cac 100644
--- a/CommunityView/confcvserver/cvserver_example.conf
+++ b/CommunityView/confcvserver/cvserver_example.conf
@@ -34,6 +34,33 @@ timezone=America/Los_Angeles
up_user=upload_user_name
up_pass=upload_password
+# Specify an IP address for the FTP server to tell the client to connect to
+# when setting up a file transfer.
+#
+# Files are uploaded via passive FTP, which means the client establishes the
+# file transfer data connetion, rather than the server as in active FTP. In
+# passive FTP, the server sends the IP address the client must connect to for
+# file transfer on the control connection. If the server is behind a NAT
+# firewall, the server must know its external (outside the firewall) address so
+# it can send the correct IP address. This external address is the
+# "masquerade" address.
+#
+# If the value "localif" is specified instead of an IP address, the server will
+# use the IP address of the local interface that the FTP request is received
+# on.
+#
+# Examples:
+#
+# masquerade=1.2.3.4
+#
+# masquerade=localif
+#
+# If no masquerade value is specified, the installation script attempts to
+# determine the external IP address of the server and specifies that as the
+# masquerade address to the FTP server. This is usually a good choice.
+#
+# masquerade=your_external_ip
+
# Specify the number of days of images this server should retain
retain_days=21
diff --git a/CommunityView/confcvserver/test/testConfcvserver.sh b/CommunityView/confcvserver/test/testConfcvserver.sh
index bb01e2d..feef055 100644
--- a/CommunityView/confcvserver/test/testConfcvserver.sh
+++ b/CommunityView/confcvserver/test/testConfcvserver.sh
@@ -63,6 +63,7 @@ test_editnpconf_name_value_editing() {
editnpconf $tcf ExistingDirectiveWithSpecialChars '~/foo bar'
# excpected conf file after editing
+ local o=""
o=${o}'# Test conf file\n'
o=${o}'# DefaultRoot <- this should not get edited\n'
o=${o}'# next line is indented and contains multiple spaces\n'
@@ -88,6 +89,48 @@ test_editnpconf_name_value_editing() {
fi
}
+test_editnpconf_name_removal() {
+ # initial conf file before editing
+ local i=""
+ i=${i}'# Test conf file\n'
+ i=${i}'# DefaultRoot <- this should not get edited\n'
+ i=${i}'# next line is indented and contains multiple spaces\n'
+ i=${i}' PassivePorts 100 200\n'
+ i=${i}'\tNameSurroundedByTabs\t100 200\n'
+ i=${i}'ExistingDirectiveWithSpecialChars ~\n'
+ i=${i}'MasqueradeAddress blah\n'
+ i=${i}'# End of initial conf file\n'
+ local tcf=unit_test_temp_conf_file2
+ local tof=unit_test_temp_orig_file2
+ /bin/echo -ne "$i" > $tcf
+ /bin/echo -ne "$i" > $tof # for debugging
+
+ editnpconf $tcf -r MasqueradeAddress
+ editnpconf $tcf -r NonextantName
+
+ # excpected conf file after editing
+ local o=""
+ o=${o}'# Test conf file\n'
+ o=${o}'# DefaultRoot <- this should not get edited\n'
+ o=${o}'# next line is indented and contains multiple spaces\n'
+ o=${o}' PassivePorts 100 200\n'
+ o=${o}'\tNameSurroundedByTabs\t100 200\n'
+ o=${o}'ExistingDirectiveWithSpecialChars ~\n'
+ o=${o}'# End of initial conf file\n'
+ local tef=unit_test_temp_expctd_file2
+ /bin/echo -ne "$o" > $tef
+
+ local diffs=`diff $tef $tcf`
+ local status=$?
+ if [ $status -ne 0 ]
+ then
+ fail "Output differs from expected:"
+ echo "$diffs"
+ else
+ rm $tef $tcf $tof
+ fi
+}
+
#
# Test the editcrontab function for proper operation
#
@@ -181,4 +224,190 @@ test_editcrontab_bad_args() {
_restore_crontab
}
+test_is_ip_addr_form() {
+ local goodaddr="\
+ 127.0.0.1 \
+ 111.111.111.111 \
+ 1.2.3.4 \
+ 01.02.03.04 \
+ 999.999.999.999 \
+ "
+ local badaddr="\
+ 1.2.3 \
+ 1.2.3.4.5 \
+ 1.2..3 \
+ 1.a.b.3 \
+ 1 \
+ 1/4 \
+ 1/2/3/4 \
+ 1.4 \
+ 1.2a.3.4 \
+ "" \
+ 1..2 \
+ a \
+ "
+
+ local addr
+ for addr in $goodaddr
+ do
+ assertTrue "is_ip_addr_form fails on good addr \"$addr\"" \
+ "is_ip_addr_form $addr"
+ done
+ for addr in $badaddr
+ do
+ assertFalse "is_ip_addr_form fails on bad addr \"$addr\"" \
+ "is_ip_addr_form $addr"
+ done
+}
+
+_proftpd_testconf() {
+ local i=""
+ i=${i}'# Test conf file\n'
+ i=${i}'# DefaultRoot <- this should not get edited\n'
+ i=${i}'# next line is indented and contains multiple spaces\n'
+ i=${i}' PassivePorts 100 200\n'
+ i=${i}'\tNameSurroundedByTabs\t100 200\n'
+ i=${i}'ExistingDirectiveWithSpecialChars ~\n'
+ i=${i}'# End of initial conf file\n'
+ /bin/echo -ne "$i"
+}
+
+test_set_up_ftp_masquerade_nospec() {
+ local tcf=unit_test_temp_conf_file3
+ local tof=unit_test_temp_orig_file3
+ local tef=unit_test_temp_expctd_file3
+ local tcvcf=unit_test_temp_cvconf_file3
+
+ # initial conf file before editing
+ _proftpd_testconf > $tcf
+ cat $tcf > $tof # for debugging
+
+ # empty cvserver.conf file
+ echo "" > $tcvcf
+
+ confile=$tcvcf
+ set_up_ftp_masquerade $tcf
+
+ # excpected conf file after editing
+ _proftpd_testconf > $tef
+ echo MasqueradeAddress `get_external_ip` >> $tef
+
+ local diffs=`diff $tef $tcf`
+ local status=$?
+ if [ $status -ne 0 ]
+ then
+ fail "Output differs from expected:"
+ echo "$diffs"
+ else
+ rm $tef $tcf $tof $tcvcf
+ fi
+}
+
+test_set_up_ftp_masquerade_localif() {
+ local tcf=unit_test_temp_conf_file4
+ local tof=unit_test_temp_orig_file4
+ local tef=unit_test_temp_expctd_file4
+ local tcvcf=unit_test_temp_cvconf_file4
+
+ # initial conf file before editing
+ _proftpd_testconf > $tcf
+ echo "MasqueradeAddress 1.1.1.1" >> $tcf
+ cat $tcf > $tof # for debugging
+
+ # cvserver.conf file with masquerade value
+ echo "masquerade=localif" > $tcvcf
+
+ confile=$tcvcf
+ set_up_ftp_masquerade $tcf
+
+ # excpected conf file after editing
+ _proftpd_testconf > $tef
+
+ local diffs=`diff $tef $tcf`
+ local status=$?
+ if [ $status -ne 0 ]
+ then
+ fail "Output differs from expected:"
+ echo "$diffs"
+ else
+ rm $tef $tcf $tof $tcvcf
+ fi
+}
+
+test_set_up_ftp_masquerade_badip() {
+ local tcf=unit_test_temp_conf_file5
+ local tof=unit_test_temp_orig_file5
+ local tef=unit_test_temp_expctd_file5
+ local tcvcf=unit_test_temp_cvconf_file5
+
+ # initial conf file before editing
+ _proftpd_testconf > $tcf
+ cat $tcf > $tof # for debugging
+
+ # cvserver.conf file with masquerade value
+ echo "masquerade=0.0.0.0.0" > $tcvcf
+
+ confile=$tcvcf
+ if set_up_ftp_masquerade $tcf > /dev/null
+ then
+ echo "set_up_ftp_masquerade returned success on bad IP address"
+ return 1
+ fi
+
+ # excpected conf file after editing
+ _proftpd_testconf > $tef
+
+ local diffs=`diff $tef $tcf`
+ local status=$?
+ if [ $status -ne 0 ]
+ then
+ fail "Output differs from expected:"
+ echo "$diffs"
+ else
+ rm $tef $tcf $tof $tcvcf
+ fi
+}
+
+test_set_up_ftp_masquerade_ipspec() {
+ local tcf=unit_test_temp_conf_file6
+ local tof=unit_test_temp_orig_file6
+ local tef=unit_test_temp_expctd_file6
+ local tcvcf=unit_test_temp_cvconf_file6
+
+ # initial conf file before editing
+ _proftpd_testconf > $tcf
+ cat $tcf > $tof # for debugging
+
+ # cvserver.conf file with masquerade value
+ echo "masquerade=1.2.3.4" > $tcvcf
+
+ confile=$tcvcf
+ set_up_ftp_masquerade $tcf
+
+ # excpected conf file after editing
+ _proftpd_testconf > $tef
+ echo "MasqueradeAddress 1.2.3.4" >> $tef
+
+ local diffs=`diff $tef $tcf`
+ local status=$?
+ if [ $status -ne 0 ]
+ then
+ fail "Output differs from expected:"
+ echo "$diffs"
+ else
+ rm $tef $tcf $tof $tcvcf
+ fi
+}
+
+test_get_external_ip() {
+ local result
+ result=`get_external_ip`
+ local status=$?
+ assertTrue "get_external_ip returns failure status" "$status"
+ assertNotNull "get_external_ip outputs empty string" "$result"
+ local ip=`wget -q -O - https://api.ipify.org`
+ assertEquals "get_external_ip returns wrong addr" "$result" "$ip"
+}
+
+
. `which shunit2`
diff --git a/CommunityView/confcvserver/utils.sh b/CommunityView/confcvserver/utils.sh
index b43931b..51ff22f 100644
--- a/CommunityView/confcvserver/utils.sh
+++ b/CommunityView/confcvserver/utils.sh
@@ -83,3 +83,30 @@ create_dir() {
done
}
+# get our external (outside any firewall) IP address and print to
+# stdout. If we can't get the address, return failure status
+#
+get_external_ip() {
+ if curl -s -m 4 \
+ http://169.254.169.254/latest/meta-data/public-ipv4 # AWS
+ then
+ : # no action
+ elif wget -q -O - https://api.ipify.org # generic
+ then
+ : # no action
+ else
+ return 1
+ fi
+}
+
+# return true if the argument is in the form of an IPv4 address.
+# Only checks for proper form--does not check for improper numbers, e.g.,
+# 999.0.0.0.
+#
+# usage: is_ip_addr_form argument
+#
+is_ip_addr_form() {
+ echo "$1" | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' \
+ > /dev/null
+}
+
diff --git a/CommunityView/doc/InstallCVDedicated.md b/CommunityView/doc/InstallCVDedicated.md
index 96ddb7d..db8e49f 100644
--- a/CommunityView/doc/InstallCVDedicated.md
+++ b/CommunityView/doc/InstallCVDedicated.md
@@ -4,10 +4,18 @@
This document describes the steps to install Neighborhood Guard's CommunityView
software on a dedicated server, either physical or virtual. It assumes
-you have root access to the server. This process was developed for Ubuntu
-16.04 LTS and has been tested on dedicated, virtual x86 machines and on Amazon
+you have root access to the server.
+
+This process was developed on Ubuntu
+16.04 LTS and has been tested and run extensively
+on dedicated, virtual and physical x86 machines and on Amazon
Web Services (AWS) EC2 virtual machines loaded with Amazon's
Ubuntu Server 16.04 LTS AMD64 20180814 AMI (ami-51537029).
+It has also been tested on Ubuntu 18.04 LTS and 20.04 LTS.
+
+CommunityView does not yet run on Ubuntu 22.04 LTS. Ubuntu 22.04
+switches to Python 3 as the standard Python and there are multiple
+compatibility issues with CommunityView to be resolved.
If you are installing on a shared-hosting server, please see
[Installing CommunityView on a Shared-Hosting Server](InstallCommunityView.md).
@@ -15,21 +23,23 @@ If you are installing on a shared-hosting server, please see
### Overview
Broadly speaking, the installation process consists of the following steps:
-1. Install Ubuntu Server 16.04 LTS on your (virtual or physical) machine.
+1. Install Ubuntu Server 16.04, 18.04 or 20.04
+on your (virtual or physical) machine.
2. Configure the machine with appropriate disk storage.
-3. Download the CommunityView software.
-4. Create and edit the `cvserver.conf` file, which provides the configuration
+3. Configure the firewall.
+4. Download the CommunityView software.
+5. Create and edit the `cvserver.conf` file, which provides the configuration
information for the installation.
-5. Run the `confcvserver.sh` script to install, configure and run all
+6. Run the `confcvserver.sh` script to install, configure and run all
necessary software for a ComunityView server.
-### 1. Install Ubuntu Server 16.04 LTS
+### 1. Install Ubuntu Server 16.04, 18.04 or 20.04
There are many tutorials and guides to installing the Ubuntu server software
-on the Web. Search for "install ubuntu server 16.04" using your favorite
-search engine to find them. The official Ubuntu tutorial is here: [https://tutorials.ubuntu.com/tutorial/tutorial-install-ubuntu-server-1604](https://tutorials.ubuntu.com/tutorial/tutorial-install-ubuntu-server-1604).
+on the Web. Search for "install ubuntu server" using your favorite
+search engine to find them.
-After installing a fresh Ubuntu Server 16.04 (or any Linux system),
+After installing a Ubuntu Server 16.04, 18.04 or 20.04,
it's a good idea
to bring the system up to date with the latest system software.
To do this, log into the server and execute the following two commands:
@@ -82,7 +92,24 @@ A simple way to make this change permanent is to edit
`/etc/rc.local`, and add the command to the end of the file.
This will cause the command to be executed each time the system is rebooted.
-### 3. Download the CommunityView Software
+### 3. Configure the Firewall
+
+The exact steps to configure the firewall protecting the CommunityView server
+depend on the hosting environment in which the server is running, e.g., AWS,
+DreamHost, etc. Please consult the documentation for your hosting environment
+for the steps involved.
+
+The following ports must be open for their respective protocols in order
+for the CommunityView server to operate.
+
+| Port | Protocol | Description |
+| ----------:|:--------:| ----------- |
+| 21|TCP | FTP control connection |
+| 22|TCP | Secure Shell (SSH) |
+| 80|TCP | CommunityView website (Apache)
+| 60000-60099|TCP | FTP data connection range|
+
+### 4. Download the CommunityView Software
Log into the server and
use the following command to download a ZIP archive of the CommunityView
@@ -101,7 +128,7 @@ Then, extract the files in the downloaded ZIP archive with this command:
This will create the `communityview-master` directory in the current
directory which contains the installation files.
-### 4. Create and Edit the `cvserver.conf` File
+### 5. Create and Edit the `cvserver.conf` File
Change to the directory that will contain the `cvserver.conf` file:
@@ -156,6 +183,30 @@ CommunityView server. If the FTP user name is "ng_user" and the password is
up_user=ng_user
up_pass=secretcode
+#### FTP Masquerade Address
+
+Image files are uploaded by `ftp_upload`
+using passive FTP, which means the client establishes the
+file transfer data connection, rather than the server as in active FTP. In
+passive FTP, the server sends the IP address the client must connect to for
+file transfer on the control connection. If the server is behind a NAT
+firewall, the server must know its external (outside the firewall) address so
+it can send the correct IP address. This external address is the
+"masquerade" address.
+
+If the value "localif" is specified instead of an IP address, the server will
+use the IP address of the local interface that the FTP request is received
+on. Examples:
+
+ masquerade=1.2.3.4
+ masquerade=localif
+
+If no masquerade value is specified, the installation script attempts to
+determine the external IP address of the server and specifies that as the
+masquerade address to the FTP server. It is usually a good choice to
+not specify a masquerade value and let the installation software determine
+this for the FTP server.
+
#### Number of Days of Images to Retain on the CommunityView
As noted above, `retain_days` sets the number of days of
@@ -206,7 +257,7 @@ navigation page, but otherwise does not matter.
When you have finished editing the cvserver.conf file, write it out to the
`confcvserver` directory.
-### Run the `confcvserver.sh` Script
+### 6. Run the `confcvserver.sh` Script
Once you have edited the `cvserver.conf` file to reflect your setup,
run the `confcvserver.sh` script as root by issuing the following command:
@@ -217,21 +268,19 @@ This will install and configure all the software required
to implement your CommunityView server.
At the end of the installation process, the script will create a private key
-for the upload user account (the account
-named in the `up_user` configuration line above),
-if one does not already exist.
-The installation script will place the key file in the current directory
-and print an informational message to that effect.
-If your upload machine authenticates itself to the CommunityView server
-via Public Key authentication, use this key as the upload machine's
-private key. If not, you can ignore this message.
-
-After running the script, if you find you need to change any of the
-configuration items, simply edit the configuration file and run the script
+for the upload user account (the account named in the `up_user` configuration
+line above), if one does not already exist. The installation script will place
+the key file in the current directory and print an informational message to
+that effect. This key is needed by the `cktunnel` and `starttunnel` scripts
+that are part of the remote access mechanism included in the `FTP_Upload`
+software. It can also be used for ssh login access to the `up_user` account.
+
+After running the installation script, if you find you need to change any of
+the configuration items, simply edit the configuration file and run the script
again.
If the script encounters errors, examine the error messages, edit the
cvserver.conf file as appropriate, then re-run the `confcvserver.sh`
script. If you cannot determine the cause of an
error and are a member of Neighborhood Guard, please contact one of the
-Neighborhood Guard board members for assistance.
\ No newline at end of file
+Neighborhood Guard board members for assistance.
diff --git a/CommunityView/doc/ReleaseNotes.md b/CommunityView/doc/ReleaseNotes.md
index 09a628e..be6fe0e 100644
--- a/CommunityView/doc/ReleaseNotes.md
+++ b/CommunityView/doc/ReleaseNotes.md
@@ -1,5 +1,30 @@
# Release Notes for CommunityView #
+## v1.2.0 - 2023/07/25
+_Doug Kerr_
+
+### Changes
+
+- CommunityView now runs on Ubuntu 16.04, 18.04 and 20.04.
+- Improve FTP masquerade. Should automatically select proper masquerade
+address for passive FTP on any system. Formerly only worked correctly on AWS.
+- Document FTP masquerade configuration and required firewall settings in
+dedicated-installation document.
+- Reduce passive-FTP port range to 60000-60099.
+- Fix incompatibilities with Ubuntu 18.04 and 20.04.
+
+### To Do
+
+* Update CommunityView to run on Ubuntu 22.04.
+* Make CommunityView remove oldest images if disk space nearly full.
+* Move sequence-page navigation to top of page.
+* Add UI link to source code.
+* Add graceful shutdown.
+
+### Known Issues
+
+* The `Next day` links in day pages are sometimes incorrectly grayed out.
+
## v1.1.0 - 2020/03/09
_Doug Kerr_
@@ -13,7 +38,6 @@ _Doug Kerr_
* Move sequence-page navigation to top of page.
* Add UI link to source code.
* Add graceful shutdown.
-* Update communityview.py to use config file
### Known Issues
diff --git a/CommunityView/src/communityview.py b/CommunityView/src/communityview.py
index 8abdb8e..954f9d1 100644
--- a/CommunityView/src/communityview.py
+++ b/CommunityView/src/communityview.py
@@ -26,14 +26,15 @@
# #
################################################################################
-version_string = "1.1.0"
+version_string = "1.2.0"
import os
-import Image
-import ImageChops
-import ImageOps
-import ImageDraw
+#import Image
+from PIL import Image
+from PIL import ImageChops
+from PIL import ImageOps
+from PIL import ImageDraw
import shutil
import datetime
import re