Improve FTP & Ubuntu compatibility for Release 1.2

- Fix FTP masquerade configuration to work on all systems, not just AWS. - Document firewall configuration. - Fix incompatibilites so code works on Unbuntu 18.04 & 20.04 as well as 16.04. - Update relase notes.
NeighborhoodGuard · Jul 25, 2023 · 3377c48 · 3377c48
1 parent b92cdbc
commit 3377c48
Show file tree

Hide file tree

Showing 7 changed files with 441 additions and 41 deletions.
diff --git a/CommunityView/confcvserver/confcvserver.sh b/CommunityView/confcvserver/confcvserver.sh
@@ -126,7 +126,7 @@ editsiteconf() {
     if grep -E '[[:space:]]*<Directory +/var/www/>' $1 > /dev/null
     then
     local ar
-    ar='/[[:space:]]*<Directory +\/var\/www\/>/,/[:space:]*<\/Directory>/'
+    ar='/[[:space:]]*<Directory +\/var\/www\/>/,/[[:space:]]*<\/Directory>/'
         sed -i -r "${ar}c\\$block" "$1"
     else
         sed -i -r "/[[:space:]]*<\/VirtualHost>/i\\\n$block\n" "$1"
@@ -142,9 +142,14 @@ editsiteconf() {
 #    name: value
 # value may contain spaces and is not followed by a comment.
 # If the name is not found in the file, append the name-value pair
-# to the end of the config file
+# to the end of the config file.
+#
+# When the -r option is used, if the name is found in the file,
+# the line containing the name is removed. If the name is not found,
+# nothing is changed
 #
 # usage: editnpconf filename name value
+#    or: editnpconf filename -r name
 #
 editnpconf() {
     local cf="$1"
@@ -153,6 +158,7 @@ editnpconf() {
     if [ $# -ne 3 ]
     then
         echo "usage: editnpconf filename name value"
+        echo "   or: editnpconf filename -r name"
         return 1
     fi
     if [ ! -e "$cf" ]
@@ -161,6 +167,18 @@ editnpconf() {
         return 1
     fi
 
+    # if it's the -r (remove) option, remove the line with the name
+    if [ "$nm" = "-r" ]
+    then
+        nm="$3"
+        if grep -E "^[[:space:]]*$nm[[:space:]]+" "$cf" > /dev/null
+        then
+            sed -i -r "/^([[:space:]]*)$nm([[:space:]]+).+$/d" \
+                "$cf"
+        fi
+        return
+    fi
+
     # If the name is found, replace the value while preserving indentation
     # and spacing.  
     # Otherwise, append the name-value pair to the end of the file
@@ -193,6 +211,36 @@ editcrontab() {
     echo "$tab$2" | crontab -
 }
 
+# set up the appropriate FTP masquerade address in proftpd conf file
+#
+# usage: set_up_ftp_masquerade proftpd_config_file
+#
+set_up_ftp_masquerade() {
+    local proftpcf="$1"
+    local masq=`get_config $confile masquerade`
+    if [ "$masq" = "" ]   # no masquerade spec
+    then
+        local extip
+        if extip=`get_external_ip`
+        then
+            editnpconf "$proftpcf" MasqueradeAddress $extip
+        else
+            echo "Cannot determine external IP address--set masquerade \c"
+            echo "variable in $confile"
+            return 1
+        fi
+    elif [ "$masq" = "localif" ]  # remove name & let server use local i/f's ip
+    then
+        editnpconf "$proftpcf" -r MasqueradeAddress
+    elif is_ip_addr_form "$masq"
+    then
+        editnpconf "$proftpcf" MasqueradeAddress "$masq"
+    else
+        echo "masquerade value is not an ip address"
+        return 1
+    fi
+}
+
 # take the config information and build the server
 #
 configure() {
@@ -353,15 +401,10 @@ configure() {
     # limit the upload user's group (==username) to the html subdir of /var/www
     editnpconf $cf DefaultRoot "~/html $up_user"
     # set the passive port range; must agree w/ firewall rules for this server
-    editnpconf $cf PassivePorts "60000 60999"
-    # if we're running in an AWS EC2 instance, get the public IP address
+    editnpconf $cf PassivePorts "60000 60099"
+    # set up the proftpd MasqueradeAddress spec
     # so proftpd can tell the client how to do passive mode
-    local pubip
-    if pubip=`curl -s -m 4 \
-        http://169.254.169.254/latest/meta-data/public-ipv4`
-    then    # sucess, we're on AWS; add the Masquerade line
-        editnpconf $cf MasqueradeAddress "$pubip"
-    fi
+    set_up_ftp_masquerade $cf 
 
     # turn off the log files so the root fs will not fill up
     editnpconf $cf SystemLog none
@@ -376,7 +419,7 @@ configure() {
 
     task="installing Python and its imaging library"
     echo "***** $task" | tee /dev/tty
-    install "python python-imaging"
+    install "python python-pil"
 
     task="installing and configuring CommunityView server"
     echo "***** $task" | tee /dev/tty

diff --git a/CommunityView/confcvserver/cvserver_example.conf b/CommunityView/confcvserver/cvserver_example.conf
@@ -34,6 +34,33 @@ timezone=America/Los_Angeles
 up_user=upload_user_name
 up_pass=upload_password
 
+# Specify an IP address for the FTP server to tell the client to connect to
+# when setting up a file transfer.
+#
+# Files are uploaded via passive FTP, which means the client establishes the
+# file transfer data connetion, rather than the server as in active FTP.  In
+# passive FTP, the server sends the IP address the client must connect to for
+# file transfer on the control connection.  If the server is behind a NAT
+# firewall, the server must know its external (outside the firewall) address so
+# it can send the correct IP address.  This external address is the
+# "masquerade" address.
+#
+# If the value "localif" is specified instead of an IP address, the server will
+# use the IP address of the local interface that the FTP request is received
+# on.
+#
+# Examples:
+#
+#   masquerade=1.2.3.4
+#
+#   masquerade=localif
+#
+# If no masquerade value is specified, the installation script attempts to
+# determine the external IP address of the server and specifies that as the
+# masquerade address to the FTP server.  This is usually a good choice.
+#
+# masquerade=your_external_ip
+
 # Specify the number of days of images this server should retain
 
 retain_days=21

diff --git a/CommunityView/confcvserver/test/testConfcvserver.sh b/CommunityView/confcvserver/test/testConfcvserver.sh
@@ -63,6 +63,7 @@ test_editnpconf_name_value_editing() {
     editnpconf $tcf ExistingDirectiveWithSpecialChars '~/foo bar'
 
     # excpected conf file after editing
+    local o=""
     o=${o}'# Test conf file\n'
     o=${o}'# DefaultRoot <- this should not get edited\n'
     o=${o}'# next line is indented and contains multiple spaces\n'
@@ -88,6 +89,48 @@ test_editnpconf_name_value_editing() {
     fi
 }
 
+test_editnpconf_name_removal() {
+    # initial conf file before editing
+    local i=""
+    i=${i}'# Test conf file\n'
+    i=${i}'# DefaultRoot <- this should not get edited\n'
+    i=${i}'# next line is indented and contains multiple spaces\n'
+    i=${i}'    PassivePorts    100    200\n'
+    i=${i}'\tNameSurroundedByTabs\t100    200\n'
+    i=${i}'ExistingDirectiveWithSpecialChars ~\n'
+    i=${i}'MasqueradeAddress blah\n'
+    i=${i}'# End of initial conf file\n'
+    local tcf=unit_test_temp_conf_file2
+    local tof=unit_test_temp_orig_file2
+    /bin/echo -ne "$i" > $tcf
+    /bin/echo -ne "$i" > $tof    # for debugging
+
+    editnpconf $tcf -r MasqueradeAddress
+    editnpconf $tcf -r NonextantName
+
+    # excpected conf file after editing
+    local o=""
+    o=${o}'# Test conf file\n'
+    o=${o}'# DefaultRoot <- this should not get edited\n'
+    o=${o}'# next line is indented and contains multiple spaces\n'
+    o=${o}'    PassivePorts    100    200\n'
+    o=${o}'\tNameSurroundedByTabs\t100    200\n'
+    o=${o}'ExistingDirectiveWithSpecialChars ~\n'
+    o=${o}'# End of initial conf file\n'
+    local tef=unit_test_temp_expctd_file2
+    /bin/echo -ne "$o" > $tef
+
+    local diffs=`diff $tef $tcf`
+    local status=$?
+    if [ $status -ne 0 ]
+    then
+        fail "Output differs from expected:"
+        echo "$diffs"
+    else
+        rm $tef $tcf $tof
+    fi
+}
+
 #
 # Test the editcrontab function for proper operation
 #
@@ -181,4 +224,190 @@ test_editcrontab_bad_args() {
     _restore_crontab
 }
 
+test_is_ip_addr_form() {
+    local goodaddr="\
+        127.0.0.1 \
+        111.111.111.111 \
+        1.2.3.4 \
+        01.02.03.04 \
+        999.999.999.999 \
+        "
+    local badaddr="\
+        1.2.3 \
+        1.2.3.4.5 \
+        1.2..3 \
+        1.a.b.3 \
+        1 \
+        1/4 \
+        1/2/3/4 \
+        1.4 \
+        1.2a.3.4 \
+        "" \
+        1..2 \
+        a \
+        "
+
+    local addr
+    for addr in $goodaddr
+    do
+        assertTrue "is_ip_addr_form fails on good addr \"$addr\"" \
+            "is_ip_addr_form $addr"
+    done
+    for addr in $badaddr
+    do
+        assertFalse "is_ip_addr_form fails on bad addr \"$addr\"" \
+            "is_ip_addr_form $addr"
+    done
+}
+
+_proftpd_testconf() {
+    local i=""
+    i=${i}'# Test conf file\n'
+    i=${i}'# DefaultRoot <- this should not get edited\n'
+    i=${i}'# next line is indented and contains multiple spaces\n'
+    i=${i}'    PassivePorts    100    200\n'
+    i=${i}'\tNameSurroundedByTabs\t100    200\n'
+    i=${i}'ExistingDirectiveWithSpecialChars ~\n'
+    i=${i}'# End of initial conf file\n'
+    /bin/echo -ne "$i"
+}
+
+test_set_up_ftp_masquerade_nospec() {
+    local tcf=unit_test_temp_conf_file3
+    local tof=unit_test_temp_orig_file3
+    local tef=unit_test_temp_expctd_file3
+    local tcvcf=unit_test_temp_cvconf_file3
+
+    # initial conf file before editing
+    _proftpd_testconf > $tcf
+    cat $tcf > $tof    # for debugging
+
+    # empty cvserver.conf file
+    echo "" > $tcvcf
+
+    confile=$tcvcf
+    set_up_ftp_masquerade $tcf
+
+    # excpected conf file after editing
+    _proftpd_testconf > $tef
+    echo MasqueradeAddress `get_external_ip` >> $tef
+
+    local diffs=`diff $tef $tcf`
+    local status=$?
+    if [ $status -ne 0 ]
+    then
+        fail "Output differs from expected:"
+        echo "$diffs"
+    else
+        rm $tef $tcf $tof $tcvcf
+    fi
+}
+
+test_set_up_ftp_masquerade_localif() {
+    local tcf=unit_test_temp_conf_file4
+    local tof=unit_test_temp_orig_file4
+    local tef=unit_test_temp_expctd_file4
+    local tcvcf=unit_test_temp_cvconf_file4
+
+    # initial conf file before editing
+    _proftpd_testconf > $tcf
+    echo "MasqueradeAddress 1.1.1.1" >> $tcf
+    cat $tcf > $tof    # for debugging
+
+    # cvserver.conf file with masquerade value
+    echo "masquerade=localif" > $tcvcf
+
+    confile=$tcvcf
+    set_up_ftp_masquerade $tcf
+
+    # excpected conf file after editing
+    _proftpd_testconf > $tef
+
+    local diffs=`diff $tef $tcf`
+    local status=$?
+    if [ $status -ne 0 ]
+    then
+        fail "Output differs from expected:"
+        echo "$diffs"
+    else
+        rm $tef $tcf $tof $tcvcf
+    fi
+}
+
+test_set_up_ftp_masquerade_badip() {
+    local tcf=unit_test_temp_conf_file5
+    local tof=unit_test_temp_orig_file5
+    local tef=unit_test_temp_expctd_file5
+    local tcvcf=unit_test_temp_cvconf_file5
+
+    # initial conf file before editing
+    _proftpd_testconf > $tcf
+    cat $tcf > $tof    # for debugging
+
+    # cvserver.conf file with masquerade value
+    echo "masquerade=0.0.0.0.0" > $tcvcf
+
+    confile=$tcvcf
+    if set_up_ftp_masquerade $tcf > /dev/null
+    then
+        echo "set_up_ftp_masquerade returned success on bad IP address"
+        return 1
+    fi
+
+    # excpected conf file after editing
+    _proftpd_testconf > $tef
+
+    local diffs=`diff $tef $tcf`
+    local status=$?
+    if [ $status -ne 0 ]
+    then
+        fail "Output differs from expected:"
+        echo "$diffs"
+    else
+        rm $tef $tcf $tof $tcvcf
+    fi
+}
+
+test_set_up_ftp_masquerade_ipspec() {
+    local tcf=unit_test_temp_conf_file6
+    local tof=unit_test_temp_orig_file6
+    local tef=unit_test_temp_expctd_file6
+    local tcvcf=unit_test_temp_cvconf_file6
+
+    # initial conf file before editing
+    _proftpd_testconf > $tcf
+    cat $tcf > $tof    # for debugging
+
+    # cvserver.conf file with masquerade value
+    echo "masquerade=1.2.3.4" > $tcvcf
+
+    confile=$tcvcf
+    set_up_ftp_masquerade $tcf
+
+    # excpected conf file after editing
+    _proftpd_testconf > $tef
+    echo "MasqueradeAddress 1.2.3.4" >> $tef
+
+    local diffs=`diff $tef $tcf`
+    local status=$?
+    if [ $status -ne 0 ]
+    then
+        fail "Output differs from expected:"
+        echo "$diffs"
+    else
+        rm $tef $tcf $tof $tcvcf
+    fi
+}
+
+test_get_external_ip() {
+    local result
+    result=`get_external_ip`
+    local status=$?
+    assertTrue "get_external_ip returns failure status" "$status"
+    assertNotNull "get_external_ip outputs empty string" "$result"
+    local ip=`wget -q -O - https://api.ipify.org`
+    assertEquals "get_external_ip returns wrong addr" "$result" "$ip"
+}
+
+
 . `which shunit2`