polscan_html

#!/bin/bash

# polscan_html: Process polscan results into static HTML

# Copyright (C) 2015-2016  Lars Windolf <lars.windolf@gmx.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

BASE=$(dirname "$0")
BASE=$(readlink -e "$BASE")
if [ "$BASE" == "/usr/bin" ]; then
	CONF_DIR=/etc/polscan/
	LIB_DIR=/usr/lib/polscan/
else
	CONF_DIR=${BASE}/etc/
	LIB_DIR=${BASE}/lib
fi

DATE=$1		# Allow passing date

source "$LIB_DIR/polscan-common.inc"

# Prepare output dir
if [ ! -d "$RESULT_DIR" ]; then
	echo "ERROR: No results found in '$RESULT_DIR'"
	exit 1
fi
if [ ! -d "$JSON_DIR" ]; then
	mkdir -p "$JSON_DIR" || exit 1
fi
cd "$RESULT_DIR" || exit 1

# Determine host list from input result dir
HOST_LIST=$(cd "$RESULT_DIR" && ls)

load_host_groups

###############################################################################
# Helper method to write JSON, solves the issue of forbidden final comma in
# array lists...
#
# $1	result type 'list' or 'hash'
# $2	file name
# $3	root key
###############################################################################
write_json() {
	result_type=$1
	name=$2
	root=$3

	echo "Writing $name.json..."
	(
		echo '{"date": "'$DATE'", "'$root'":'
		if [ "$result_type" == 'list' ]; then
			echo '['
		else
			echo '{'
		fi
		cat |\
		sed 's/[^[:print:]]//g' |\
		sed 's/\(\\[^bfnrt"\\]\)/\\\1/g' |\
		sed '/^$/d' |\
		sed '$s/,$//' 	# Strip last comma in lists
		if [ "$result_type" == 'list' ]; then
			echo ']'
		else
			echo '}'
		fi
		echo -n "}"
	) | sed -e 's/,\]/]/g' >"${JSON_DIR}/${name}.json.tmp"
	mv "${JSON_DIR}/${name}.json.tmp" "${JSON_DIR}/${name}.json"

        if [ "$RESULT_JSON_GZIP" == "1" ]; then
		gzip -9 "${JSON_DIR}/${name}.json"
		mv "${JSON_DIR}/${name}.json.gz" "${JSON_DIR}/${name}.jsongz"
	fi
}

# Do regroup results by group and policy
declare -A policies
declare -A pgroups
declare -A pall
declare -A hosts	# contains host states (FIXME: wrong name 'hosts')
declare -A host_names
declare -A inventory_types
declare -A netedge_types

# Produce inventory indices
echo "Compiling inventories..."
while read host group severity details; do
	inventory=${details/#|||/}
	inventory=${inventory/|||*/}
	message=${details/*|||/}
	echo -n '{"host": "'$host'", "values": "'"${message//\"/\'}"'"},' >> "$JSON_DIR/$group ${inventory}.json.tmp"
	inventory_types["$group_${inventory// /_}"]="$group $inventory"
done < <(
	ls | xargs -n1 egrep -H "^[a-zA-Z0-9_-][a-zA-Z0-9_-]* INVENTORY " |\
	sed 's/:/ /'
)

for key in ${!inventory_types[*]}; do
	name=${inventory_types[$key]}
	cat "$JSON_DIR/${name}.json.tmp" | write_json list "inventory $name" results
	rm "$JSON_DIR/${name}.json.tmp"
done

# Produce netegde indices
echo "Compiling network edge maps..."
while read host group severity json; do
	netedgemap=${json/#|||/}
	netedgemap=${netedgemap/|||*/}
	json=${json/*|||/}
	echo -n "${json}," >> "$JSON_DIR/${netedgemap}.json.tmp"
	netedge_types["${netedgemap// /_}"]="$netedgemap"
done < <(
	ls | xargs -n1 egrep -H "^[a-zA-Z0-9_-][a-zA-Z0-9_-]* NETEDGE " |\
	sed 's/:/ /'
)

for key in ${!netedge_types[*]}; do
	name=${netedge_types[$key]}
	cat "$JSON_DIR/${name}.json.tmp" | write_json list "netedge $name" results
	rm "$JSON_DIR/${name}.json.tmp"
done

# Produce finding indices
groups=$(
	egrep -h "^[a-zA-Z0-9_-][a-zA-Z0-9_-]* (OK|WARNING|FAILED) " ./* |\
	awk '{print $1}' | sort -u
)
echo "Found finding groups:" $groups
echo "Aggregating all results ..."
i=0
findings=0
TMPFILE=$(mktemp || exit 1)
while read host group severity details; do
	policy=${details/#|||/}
	policy=${policy/|||*/}
	message=${details/*|||/}
	case $severity in
		OK|WARNING|FAILED)
			echo '{"host": "'$host'", "group": "'$group'", "severity": "'$severity'", "policy": "'$policy'", "message": "'"${message//\"/\'}"'"},' >>$TMPFILE
			;;
	esac
	case $severity in
		WARNING)
			pall[warn]=$(( ${pall[warn]} + 1))
			pgroups[warn_$group]=$(( ${pgroups[warn_$group]} + 1 ))
			policies[warn_$policy]=$(( ${policies[warn_$policy]} + 1 ))
			hosts[warn_$host]=$(( ${hosts[warn_$host]} + 1 ))
			;;
		FAILED)
			pall[fail]=$(( ${pall[fail]} + 1))
			pgroups[fail_$group]=$(( ${pgroups[fail_$group]} + 1 ))
			policies[fail_$policy]=$(( ${policies[fail_$policy]} + 1 ))
			hosts[fail_$host]=$(( ${hosts[fail_$host]} + 1 ))
			;;
		OK)
			hosts[ok_$host]=$(( ${hosts[ok_$host]} + 1 ))
			;;
	esac

	# Update summary every 50 hosts
	host_names[$host]=1
	if [ $i -eq 50 ]; then
		i=0
		echo "   $(date +%H:%M:%S) ${#host_names[*]} Hosts, $findings Findings (${pall[fail]} Failed ${pall[warn]} Warnings)"
	fi
	i=$(($i + 1))
	findings=$(($findings + 1))
done < <(
	ls | xargs -I {} -n1 sed '/^[^ ][^ ]* OK /d; s/^/{} /;' {}
)
cat $TMPFILE | write_json list all results
rm $TMPFILE

# Per group results
for g in $groups; do
	(
		ls | xargs -n1 egrep -H "^$g (OK|WARNING|FAILED) " |\
		sed 's/:/ /' |\
		while read host group severity details; do
			policy=${details/#|||/}
			policy=${policy/|||*/}
			message=${details/*|||/}
			echo -n '{"host": "'$host'", "severity": "'$severity'", "policy": "'$policy'", "message": "'"${message//\"/\'}"'"},'
		done
	) | write_json list $g 'group": "'$g'", "WARNING": '${pgroups[warn_$g]-0}', "FAILED": '${pgroups[fail_$g]-0}', "results'
done

# Statistics
(
	for key in ${!inventory_types[*]}; do
		name=${inventory_types[$key]}
		echo -n '{"inventory": "'"$name"'"},'
	done
	for key in ${!netedge_types[*]}; do
		name=${netedge_types[$key]}
		echo -n '{"netedge": "'"$name"'"},'
	done

	for g in $groups; do
		echo "- $g" >&2
		count=$(ls | xargs -n1 grep "^$g (OK|WARNING|FAILED) " | wc -l)
		echo -n '{"group": "'$g'", "WARNING": '${pgroups[warn_$g]-0}', "FAILED": '${pgroups[fail_$g]-0}', "count": '${count-0}'},'
	done
	for s in $(
		get_scanners remote | sed "s/^/scanners\//"
		get_scanners standalone | sed "s/^/standalone\//"
	); do
		echo "- $s" >&2
		group=$(get_policy_info "$LIB_DIR/$s" group)
		policy=$(get_policy_info "$LIB_DIR/$s" name)
		description=$(get_policy_info "$LIB_DIR/$s" description)
		tags=$(get_policy_info "$LIB_DIR/$s" tags)
		solution=$(get_policy_info "$LIB_DIR/$s" solution)
		solution_cmd=$(get_policy_info "$LIB_DIR/$s" solution-cmd)
		references=$(get_policy_info "$LIB_DIR/$s" 'source' | xargs)
		echo -n '{"policy": "'$policy'", "parent": "'$group'", "description": "'${description//\"/\\\"}'", "solution": "'${solution//\"/\\\"}'", "solution_cmd": "'${solution_cmd//\"/\\\"}'", "tags": [ "'${tags// /\",\"}'" ], "references": [ "'${references// /\",\"}'" ], "WARNING": '${policies[warn_$policy]-0}', "FAILED": '${policies[fail_$policy]-0}'},'
	done
) | write_json list overview 'WARNING": '${pall[warn]-0}', "FAILED": '${pall[fail]-0}', "hostCount": '$(echo $HOST_LIST | wc -w)', "overview'

# Host infos
(
	for h in *; do
		echo -n "\"$h\": {\"WARNING\": ${hosts["warn_$h"]-0}, \"FAILED\": ${hosts["fail_$h"]-0}},"
	done
) | write_json hash hosts results
(
	for h in ${!host_groups[*]}; do
		tmp=${host_groups[$h]% }
		tmp=${tmp// /\",\"}
		echo -n "\"$h\": [\"$tmp\"],"
	done
) | write_json hash host_groups results

# 30 days per-policy histogram
declare -A histogram
declare -A histogram_ids
declare -A histogram_labels

histogram_ids['all']='all'
i=0
while [ $i -le 30 ]; do
	day=$(date -d "$i days ago" +%Y/%m/%d)
	if [ -d "$RESULT_BASE_DIR/$day" ]; then
		histogram_labels[$i]=$day
		echo $day
		while read nr group severity policy; do
			histogram_ids["group_${group// /_}"]=$group
			histogram_ids["policy_${policy// /_}"]="${group}:::${policy}"
			histogram["policy_${policy// /_}_${severity}_${day}"]=$nr
			histogram["group_${group// /_}_${severity}_${day}"]=$(( $nr + ${histogram["group_${group// /_}_${severity}_${day}"]-0} ))
			histogram["all_${severity}_${day}"]=$(( $nr + ${histogram["all_${severity}_${day}"]-0} ))
		done < <(egrep -r '(FAILED|WARNING) \|\|\|' "$RESULT_BASE_DIR/$day" |sed 's/.*:\([^ ]*\) \([^ ]*\) |||\(.*\)|||.*/\1 \2 \3/' | sort | uniq -c)
	fi
	i=$(( $i + 1 ))
done

labels=$(
	for l in ${!histogram_labels[*]}
	do
		echo "\\\"${histogram_labels[$l]}\\\","
	done | sort | xargs
)

(
for p in ${!histogram_ids[*]}; do
	echo -n ',{"id": "'${histogram_ids[$p]}'",'
	for severity in FAILED WARNING; do
		echo -n ' "'$severity'": ['
		i=30
		while [ $i -ge 0 ]; do
			day=$(date -d "$i days ago" +%Y/%m/%d)
			if [ -d "$RESULT_BASE_DIR/$day" ]; then
				echo -n "${histogram[${p}_${severity}_$day]-0}, "
			fi
			i=$(( $i - 1 ))
		done
		echo -n "],"
	done
	echo "}"
done
) | sed "1s/^,//;s/\],\}/\]\}/;s/, \]/\]/g" | write_json list histogram "labels\": [${labels}], \"histogram"

if [ -d "$RESULT_BASE_DIR/$ONE_DAY_AGO/" ]; then
	# FIXME: Eliminate all result files that are only
	# in one result set and those with a "Polscan" failure
	# inside as diffing those produces false solved/new
	# counts.
        diff=$(diff -u <(cd "$RESULT_BASE_DIR/$ONE_DAY_AGO" && grep -F "|||" * | sort) <(cd "$RESULT_BASE_DIR/$DATE" && grep -F "|||" * | sort) | grep "^[-+]" | egrep -v " (INVENTORY|OK) [|][|][|]")
        new_today=$(echo "$diff" | grep "^+[^+]")
        solved_today=$(echo "$diff" | grep "^-[^-]")

	(
		echo "$new_today" | sed "s/^+//" |\
		sed 's/:/ /' | grep -v "^$" |\
		while read host group severity details; do
			policy=${details/#|||/}
			policy=${policy/|||*/}
			message=${details/*|||/}
			regex="-${host}:${group} ${severity} \|\|\|${policy}\|\|\|"
			if [[ ! $solved_today =~ $regex ]]; then
				echo -n '{"host": "'$host'", "group": "'$group'", "severity": "'$severity'", "policy": "'$policy'", "message": "'"${message//\"/\'}"'"},'
			fi
		done
	) | write_json list new results

	(
		echo "$solved_today" | sed "s/^-//" |\
		sed 's/:/ /' | grep -v "^$" |\
		while read host group severity details; do
			policy=${details/#|||/}
			policy=${policy/|||*/}
			message=${details/*|||/}
			regex="\+${host}:${group} ${severity} \|\|\|${policy}\|\|\|"
			if [[ ! $new_today =~ $regex ]]; then
				echo -n '{"host": "'$host'", "group": "'$group'", "severity": "'$severity'", "policy": "'$policy'", "message": "'"${message//\"/\'}"'"},'
			fi
		done
	) | write_json list solved results
fi

# Update month calendar
(
	i=1
	while [ $i -le 31 ]; do
		if [ -d "$(dirname $RESULT_DIR)/$(printf "%02d" $i)" ]; then
			echo "$i,"
		fi
		i=$((i + 1))
	done
) | write_json list "../calendar" days

if [ $LATEST == 1 ]; then
	# Link latest results
	rm "$RESULT_BASE_DIR/latest"
	cd $RESULT_BASE_DIR && ln -s "json/$DATE" latest
fi

echo "JSON Done."