WiP: Testbench: add performance-testing tools

.travis.yml

@@ -28,16 +28,22 @@ install:
   - sh autogen.sh
 
 before_script:
+  - sudo add-apt-repository ppa:adiscon/v8-stable -y
+  - sudo apt-get update -qq
+  - sudo apt-get -qq --force-yes install valgrind libfastjson
   - # note: valgrind is only available on Linux
   - if [ "$CC" == "gcc" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then export ENA_VG="--enable-valgrind"; fi
   - if [ "$CC" == "clang" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then export SANITIZER="-fsanitize=address" ; fi
   - if [ "$CC" == "clang" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then export CFLAGS="$CFLAGS -fsanitize=address" ; fi
   - ./CI/check_codestyle.sh
-  - ./configure $ENA_VG
+  - ./configure --enable-performance-testbench $ENA_VG
 
 script:
   - if [ "$CC" == "clang" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then  scan-build-3.6 --status-bugs make check TESTS="" && make clean ; fi
   - make
+  - valgrind --tool=callgrind ls
   - export VERBOSE=1
   - make check
+  - cat tests/test-suite.log
+  - cat tests/perf_to_string.trs
   - make distcheck

configure.ac

@@ -80,6 +80,23 @@ if test "$enable_valgrind" = "yes"; then
 	AC_DEFINE(VALGRIND, 1, [valgrind enabled])
 fi
 
+AC_ARG_ENABLE(performance_testbench,
+        [AS_HELP_STRING([--enable-performance-testbench],[enable running the performance testbench @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_performance_testbench="yes" ;;
+          no) enable_performance_testbench="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-performance-testbench) ;;
+         esac],
+        [enable_performance_testbench="no"]
+)
+if test "$enable_performance_testbench" = "yes"; then
+        AC_CHECK_PROG(VALGRIND_CALLGRIND, [valgrind], [valgrind], [no])
+	if test "x$VALGRIND_CALLGRIND" = "xno"; then
+		AC_MSG_ERROR([--enable-performance-testbench given, but valgrind is not present on system])
+	fi
+fi
+AM_CONDITIONAL(ENABLE_PERFORMANCE_TESTBENCH, test x$enable_performance_testbench = xyes)
+
 # Checks for programs.
 
 # Checks for libraries.

tests/Makefile.am

@@ -60,6 +60,13 @@ check_PROGRAMS += test_object_object_add_exFormatted
 test_object_object_add_exFormatted_SOURCES = test_object_object_add_ex.c parse_flags.c parse_flags.h
 test_object_object_add_exFormatted_CPPFLAGS = -DTEST_FORMATTED
 
+# performancebaseline tests
+if ENABLE_PERFORMANCE_TESTBENCH
+check_PROGRAMS += perf_to_string
+TESTS += perf_to_string.test
+endif
+
+
 EXTRA_DIST=
 EXTRA_DIST += $(TESTS)
 EXTRA_DIST += test-defs.sh
@@ -90,6 +97,7 @@ EXTRA_DIST += test_object_object_add_exFormatted_pretty.expected
 EXTRA_DIST += test_object_object_add_exFormatted_spaced.expected
 EXTRA_DIST += test_many_subobj.expected
 EXTRA_DIST += test_obj_obj_get_ex-null.expected
+EXTRA_DIST += perf_to_string.c
 
 testsubdir=testSubDir
 TESTS_ENVIRONMENT       = top_builddir=$(top_builddir)

tests/perf_to_string.c

@@ -0,0 +1,51 @@
+/* libfastjson testbench tool
+ * This program can be used to test the performance of the _to_string()
+ * subsystem. It is meant to be used together with a profiler.
+ *
+ * Copyright (c) 2016 Adiscon GmbH
+ * Rainer Gerhards <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See COPYING for details.
+ *
+ */
+#include "config.h"
+
+#include "../json.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define NUM_ITERATIONS 100000
+
+int
+main(int __attribute__((unused)) argc, char __attribute__((unused)) **argv)
+{
+	int i;
+	struct fjson_object *json;
+
+	json = fjson_object_new_object();
+	fjson_object_object_add(json, "string1", fjson_object_new_string("This is a test"));
+	fjson_object_object_add(json, "string2", fjson_object_new_string("This is a "
+		"loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo"
+		"ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo"
+		"ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"
+		"test"));
+	fjson_object_object_add(json, "string3", fjson_object_new_string("This is\n a test öäü"));
+	fjson_object_object_add(json, "int1", fjson_object_new_int(4711));
+
+
+	for(i = 0 ; i < NUM_ITERATIONS ; ++i) {
+		const char *dummy = fjson_object_to_json_string(json);
+		if(dummy == NULL) {
+			fprintf(stderr, "dummy has received no output!");
+			exit(1);
+		}
+		if(i == 0)
+			printf("%s\n", dummy);
+	}
+
+	fjson_object_put(json);
+	return 0;
+}

tests/perf_to_string.test

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Common definitions
+#if test -z "$srcdir"; then
+    #srcdir="${0%/*}"
+    #test "$srcdir" = "$0" && srcdir=.
+    #test -z "$srcdir" && srcdir=.
+#fi
+#. "$srcdir/test-defs.sh"
+
+# first, get baseline from system-installed library
+rm -f callgrind.out &> /dev/null
+valgrind --tool=callgrind --callgrind-out-file=callgrind.out .libs/perf_to_string
+if [ $? -ne 0 ]; then
+    echo "Error obtaining performance baseline, assuming system is not able to"
+    echo "support this test - skipping"
+    rm callgrind.out
+    exit 77
+fi
+export RETCODE=$?
+echo return code $RETCODE
+MAXCYCLES=$(grep < callgrind.out "^totals:" | tr -dc '0-9')
+echo "system installed libfastjson requires $MAXCYCLES cycles"
+# give 5% allowence to care for exec env differences between runs
+MAXCYCLES=$((MAXCYCLES * 105))
+MAXCYCLES=$((MAXCYCLES / 100))
+
+ls -l ../.libs/libfastjson.so
+rm -f callgrind.out &> /dev/null
+LD_PRELOAD="../.libs/libfastjson.so.4" valgrind --tool=callgrind --callgrind-out-file=callgrind.out .libs/perf_to_string
+CYCLES=$(grep < callgrind.out "^totals:" | tr -dc '0-9')
+echo "libfastjson being testes requires $CYCLES cycles"
+
+if [ $CYCLES -gt $MAXCYCLES ]; then
+    echo
+    echo performance issue: we need $CYCLES cycles, but upper bound is $MAXCYCLES
+    exit 1
+fi
+rm callgrind.out