Artifact eval (#147)

* fixed benchmarks

* added artifact eval folder

* added AE readme and tc script

* change tc script

* fixed bench and cost model

* Reformat Python scripts with IntelliJ

* Detect viaduct binary inside Docker

* Add Makefile for compiling benchmarks

* Copy artifact evaluation into the Docker image

* Add an all target to Makefile

* Enable Bash completion

* Move Viaduct Bash completion file

* Revert "Bump z3-turnkey from 4.8.9 to 4.8.10 (#144)"

This reverts commit 974283d.

* Revert "Bump z3-turnkey from 4.8.7.1 to 4.8.9"

This reverts commit 0abc4f2.

* Compile erased benchmarks separately in Makefile

* hit a performance cliff (thanks Z3), reverted to dumber cost model that doesn't scale replication and commitments linearly to # of participants

* more artifact eval instructions

* Sort benchmarks by name in Makefile

* Add Python script to run benchmarks (only RQ2 for now)

* Change generated reports directory

* Refactor benchmark script slightly

* Improve documentation of benchmarking script

* Add script for running Viaduct programs (RQ3)

* RQ4 script

* Update RQ3 script to produce a report

* Cleanup benchmarking script

* Add IntelliJ config for Python modules

* Remove old benchmarking scripts

* Provide a way to change input files per benchmark

* Fix host names in compiled HhiScore

* Fix units in RQ3 report

* Include source code in the final Docker image

* Print report file location after RQ4

* Rename README.txt to README.md

* IntelliJ: don't autoformat Markdown files

* Update Docker instructions

* Include tmux in the Docker image

* Update instructions for viaduct run

* instructions

* Update Tmux instructions

* Minor README fixes

* Terminate benchmarks when a run fails

* Docker ignore tar archives

* Parse viaduct run logs from file instead of storing them in memeory

* Small updates to README

* Add instructions to increase memory limit for Docker

* Suppress viaduct run output for RQ3

* Better handling of file descriptors with viaduct run

* Ignore DS_Store

* Better logging during RQ3

* Update instructions for settraffic

* Fix some bugs in the settraffic script

* settraffic: remove LAN setup; it's just the default

* Remove the NETWORK column from RQ3 results

* Add figure references to section titles

* Properly erase all benchmarks

* Add MedianErased

* Add KmeansUnrolledErased

* Count number of lines and annotations in rq2

* Add simple examples so people can provide input manually

* Add a Pipfile

Co-authored-by: Cosku Acay <[email protected]>

.dockerignore

@@ -1,10 +1,18 @@
+# Operating systems
+**/.DS_Store
+
 # Editors
 .idea
 .vscode
+**/*.iml
 
 # Git
 .git
 
 # Gradle
 .gradle
 **/build
+
+# Archives
+*.tar
+*.tar.gz

.gitignore

@@ -86,7 +86,3 @@ bin/*
 !lib/*
 
 .*.sw?
-
-*.py
-*.mpc
-!viaduct_bench.py

Dockerfile

@@ -1,6 +1,6 @@
-ARG JDK_VERSION=11.0.3
+ARG JDK_VERSION=11.0.10
 
-# Stage 1 (the build container)
+# Build stage
 FROM openjdk:${JDK_VERSION}-jdk-slim AS builder
 CMD ["/bin/bash"]
 WORKDIR /app
@@ -17,7 +17,7 @@ COPY compiler/*.gradle.kts compiler/
 COPY runtime/*.gradle.kts runtime/
 COPY shared/*.gradle.kts shared/
 COPY test-utilities/*.gradle.kts test-utilities/
-RUN ./gradlew assemble
+RUN ./gradlew build
 
 ## Build the app
 COPY cli cli
@@ -28,16 +28,50 @@ COPY test-utilities test-utilities
 RUN ./gradlew :cli:installDist
 
 
-# Stage 2 (the distribution container)
-FROM openjdk:${JDK_VERSION}-jre-slim
+# Collect source files
+FROM busybox AS source
+WORKDIR /root/source
+
+## Copy source code and remove build files
+COPY --from=builder /app .
+RUN find . -name build -exec rm -rf {} +
+RUN rm -r ./.gradle
+COPY viaduct .
+
+
+# Distribution stage
+#FROM openjdk:${JDK_VERSION}-jre-slim
+FROM openjdk:${JDK_VERSION}-jdk-slim
+CMD ["/bin/bash"]
 WORKDIR /root
 
-## Copy example programs for testing
-COPY compiler/tests/should-pass examples
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash-completion \
+    iproute2 \
+    less \
+    make \
+    nano \
+    python3 \
+    tmux \
+    && rm -rf /var/lib/apt/lists/*
 
+## Enable Bash completion
+RUN echo source /etc/profile.d/bash_completion.sh >> .bashrc
+
+## Copy application binary
 COPY --from=builder /app/cli/build/install /usr/local/
 RUN ["ln", "-s", "/usr/local/cli/bin/cli", "/usr/local/bin/viaduct" ]
 
 ## Add command-line completion (i.e. tab to autocomplete)
-RUN _VIADUCT_COMPLETE=bash viaduct > viaduct-completion.sh
-RUN echo source viaduct-completion.sh >> .bashrc
+RUN _VIADUCT_COMPLETE=bash viaduct > /etc/profile.d/viaduct_completion.sh
+RUN echo source /etc/profile.d/viaduct_completion.sh >> .bashrc
+
+## Copy artifact evaluation
+COPY artifact-eval ./
+
+## Copy Gradle cache to make build self contained
+# COPY --from=builder /root/.gradle/caches /root/.gradle/caches
+# COPY --from=builder /root/.gradle/wrapper /root/.gradle/wrapper
+
+## Copy application code to allow building from source
+COPY --from=source /root/source source

artifact-eval/Makefile

@@ -0,0 +1,35 @@
+VIADUCT := $(if $(wildcard ../viaduct), ../viaduct, viaduct)
+
+BENCHMARK_DIR := benchmarks
+BENCHMARKS := $(sort $(wildcard $(BENCHMARK_DIR)/*.via))
+ERASED_BENCHMARKS := $(sort $(wildcard $(BENCHMARK_DIR)/*Erased.via))
+ANNOTATED_BENCHMARKS := $(filter-out $(ERASED_BENCHMARKS), $(BENCHMARKS))
+
+BUILD_DIR ?= build
+LAN_BUILD_DIR := $(BUILD_DIR)/lan
+WAN_BUILD_DIR := $(BUILD_DIR)/wan
+
+.PHONY: all
+all: lan wan erased
+
+.PHONY: lan wan erased
+lan: $(subst $(BENCHMARK_DIR), $(LAN_BUILD_DIR), $(ANNOTATED_BENCHMARKS))
+wan: $(subst $(BENCHMARK_DIR), $(WAN_BUILD_DIR), $(ANNOTATED_BENCHMARKS))
+erased: $(subst $(BENCHMARK_DIR), $(LAN_BUILD_DIR), $(ERASED_BENCHMARKS))
+
+$(LAN_BUILD_DIR)/%.via: $(BENCHMARK_DIR)/%.via
+	@mkdir -p $(dir $@)
+	$(VIADUCT) -v compile $< -o $@
+
+$(WAN_BUILD_DIR)/%.via: $(BENCHMARK_DIR)/%.via
+	@mkdir -p $(dir $@)
+	$(VIADUCT) -v compile $< -o $@ --wancost
+
+.PHONY: clean
+clean:
+	rm -rf $(LAN_BUILD_DIR)
+	rm -rf $(WAN_BUILD_DIR)
+
+# Printing variables
+print-%:
+	@echo $($*)

artifact-eval/Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+
+[requires]
+python_version = "3.7"