diff --git a/examples/wordcount/Dockerfile b/examples/wordcount/Dockerfile index deb4393c..3f7e2669 100644 --- a/examples/wordcount/Dockerfile +++ b/examples/wordcount/Dockerfile @@ -1,34 +1,14 @@ -FROM openjdk:8-jdk +FROM flink:1.8.2-scala_2.12 # Prepare environment -ENV FLINK_HOME=/opt/flink ENV MAVEN_HOME=/opt/maven -ENV HADOOP_HOME=/opt/hadoop -ENV PATH=$FLINK_HOME/bin:$HADOOP_HOME/bin:$MAVEN_HOME/bin:$PATH - -COPY . /code - -# Configure Flink version -ENV FLINK_VERSION=1.8.2 \ - HADOOP_SCALA_VARIANT=scala_2.12 +ENV PATH=$MAVEN_HOME/bin:$PATH # Install dependencies RUN set -ex; \ - apt-get update; \ - apt-get -y install libsnappy1v5; \ - apt-get -y install netcat net-tools; \ - apt-get -y install gettext-base; \ - rm -rf /var/lib/apt/lists/* - -# Grab gosu for easy step-down from root -ENV GOSU_VERSION 1.11 -RUN set -ex; \ - wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \ - wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \ - export GNUPGHOME="$(mktemp -d)"; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - chmod +x /usr/local/bin/gosu; \ - gosu nobody true + apt-get update \ + && apt-get -y install gettext-base openjdk-8-jdk-headless \ + && rm -rf /var/lib/apt/lists/* # Install Maven ENV MAVEN_VERSION 3.6.1 @@ -38,39 +18,13 @@ RUN \ mv apache-maven-$MAVEN_VERSION $MAVEN_HOME; \ rm apache-maven-$MAVEN_VERSION-bin.tar.gz +# Build application jar +COPY . /code WORKDIR /code - RUN \ - mvn package; \ - mkdir -p /opt/flink/flink-web-upload; \ - cp flink-conf.yaml /usr/local/; \ - cp /code/target/*.jar /opt/flink/flink-web-upload/ - -RUN groupadd --system --gid=9999 flink && \ - useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink -WORKDIR $FLINK_HOME - -ENV FLINK_URL_FILE_PATH=flink/flink-${FLINK_VERSION}/flink-${FLINK_VERSION}-bin-${HADOOP_SCALA_VARIANT}.tgz -ENV FLINK_TGZ_URL=https://archive.apache.org/dist/$FLINK_URL_FILE_PATH - -# Install Flink -RUN set -ex; \ - wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ - \ - tar -xf flink.tgz --strip-components=1; \ - rm flink.tgz; \ - \ - cp ./opt/flink-s3-fs-presto-${FLINK_VERSION}.jar ./lib/;\ - cp ./opt/flink-s3-fs-hadoop-${FLINK_VERSION}.jar ./lib/;\ - \ - chown -R flink:flink .; - -# Needed on OpenShift for the entrypoint script to work -RUN chmod -R 777 /opt/flink + JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 mvn package \ + && ln -s /code/target $FLINK_HOME/flink-web-upload -# control script expects manifest.yaml at this location -RUN chown -R flink:flink /var -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] -EXPOSE 6123 8081 -CMD ["local"] +COPY docker-entrypoint.sh /flinkk8soperator-entrypoint.sh +ENTRYPOINT ["/flinkk8soperator-entrypoint.sh"] +CMD ["help"] diff --git a/examples/wordcount/docker-entrypoint.sh b/examples/wordcount/docker-entrypoint.sh index 79f4df3f..db30823c 100755 --- a/examples/wordcount/docker-entrypoint.sh +++ b/examples/wordcount/docker-entrypoint.sh @@ -1,45 +1,10 @@ #!/bin/sh -drop_privs_cmd() { - if [ $(id -u) != 0 ]; then - # Don't need to drop privs if EUID != 0 - return - elif [ -x /sbin/su-exec ]; then - # Alpine - echo su-exec - else - # Others - echo gosu flink - fi -} - -# Add in extra configs set by the operator +# Map config from FlinkK8sOperator to base container +# https://github.com/lyft/flinkk8soperator/issues/135 +# https://github.com/docker-flink/docker-flink/pull/91 if [ -n "$OPERATOR_FLINK_CONFIG" ]; then - echo "$OPERATOR_FLINK_CONFIG" >> "/usr/local/flink-conf.yaml" -fi - -envsubst < /usr/local/flink-conf.yaml > $FLINK_HOME/conf/flink-conf.yaml - -COMMAND=$@ - -if [ $# -lt 1 ]; then - COMMAND="local" -fi - -if [ "$COMMAND" = "help" ]; then - echo "Usage: $(basename "$0") (jobmanager|taskmanager|local|help)" - exit 0 -elif [ "$FLINK_DEPLOYMENT_TYPE" = "jobmanager" ]; then - echo "Starting Job Manager" - echo "config file: " && grep '^[^\n#]' "$FLINK_HOME/conf/flink-conf.yaml" - exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground -elif [ "$FLINK_DEPLOYMENT_TYPE" = "taskmanager" ]; then - echo "Starting Task Manager" - echo "config file: " && grep '^[^\n#]' "$FLINK_HOME/conf/flink-conf.yaml" - exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground -elif [ "$COMMAND" = "local" ]; then - echo "Starting local cluster" - exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground local + export FLINK_PROPERTIES="`echo \"${OPERATOR_FLINK_CONFIG}\" | envsubst`" fi -exec "$@" +exec /docker-entrypoint.sh "$@" diff --git a/examples/wordcount/flink-conf.yaml b/examples/wordcount/flink-conf.yaml deleted file mode 100644 index 7593df8d..00000000 --- a/examples/wordcount/flink-conf.yaml +++ /dev/null @@ -1,31 +0,0 @@ -jobmanager.web.submit.enable: true -jobmanager.web.log.path: /var/log/jobmanager/current - -taskmanager.log.path: /var/log/taskmanager/current -taskmanager.exit-on-fatal-akka-error: true -taskmanager.network.memory.min: 27748736 -taskmanager.network.memory.max: 2147483648 -taskmanager.network.memory.fraction: 0.1 - -web.upload.dir: /opt/flink - -# Akka config -akka.framesize: 20MB -parallelism.default: 1 - -# State backend config -state.backend: rocksdb -state.checkpoints.num-retained: 4 - -# Restart strategy -restart-strategy: fixed-delay -restart-strategy.fixed-delay.delay: 0s -restart-strategy.fixed-delay.attempts: 2147483647 - -# These parameters control how often TaskManagers try to connect to a JobManager. -# These values are set a bit lower than the defaults to make recovery and cluster restarts -# a bit faster -taskmanager.maxRegistrationDuration: Inf -taskmanager.initial-registration-pause: 500 ms -taskmanager.max-registration-pause: 5 s -taskmanager.refused-registration-pause: 5 s diff --git a/examples/wordcount/flink-operator-custom-resource.yaml b/examples/wordcount/flink-operator-custom-resource.yaml index ddc8314f..e250a388 100644 --- a/examples/wordcount/flink-operator-custom-resource.yaml +++ b/examples/wordcount/flink-operator-custom-resource.yaml @@ -8,23 +8,27 @@ metadata: environment: development spec: image: docker.io/lyft/wordcount-operator-example:{sha} + deleteMode: None flinkConfig: taskmanager.heap.size: 200 + taskmanager.network.memory.fraction: 0.1 + taskmanager.network.memory.min: 10m state.backend.fs.checkpointdir: file:///checkpoints/flink/checkpoints state.checkpoints.dir: file:///checkpoints/flink/externalized-checkpoints state.savepoints.dir: file:///checkpoints/flink/savepoints + web.upload.dir: /opt/flink jobManagerConfig: resources: requests: memory: "200Mi" - cpu: "0.2" + cpu: "0.1" replicas: 1 taskManagerConfig: taskSlots: 2 resources: requests: memory: "200Mi" - cpu: "0.2" + cpu: "0.1" flinkVersion: "1.8" jarName: "wordcount-operator-example-1.0.0-SNAPSHOT.jar" parallelism: 3