From b163af1c0d0a4af3e32a85a8863ccc4f984091a9 Mon Sep 17 00:00:00 2001 From: Gunnar Schaefer Date: Thu, 11 Jan 2018 14:00:44 -0800 Subject: [PATCH] Substantial cleanup --- .dockerignore | 19 ++--- CONTRIBUTING.md => .github/CONTRIBUTING.md | 0 .../PULL_REQUEST_TEMPLATE.md | 0 .gitignore | 8 +- .travis.yml | 2 +- Dockerfile | 8 +- {docker => bin}/build_info.sh | 0 {docker => bin}/dev+mongo.sh | 0 entrypoint.sh => bin/entrypoint.sh | 0 bootstrap.sample.json | 29 ------- docker/README.md | 61 --------------- docker/bootstrap-accounts.sh | 28 ------- docker/bootstrap-data.sh | 77 ------------------- docker/pymongo-cli.py | 5 -- docker/unit.json => nginx-unit.json | 0 TESTING.md => tests/README.md | 0 .../{run-tests-docker.sh => docker-tests.sh} | 4 +- tests/bin/{run-tests-ubuntu.sh => tests.sh} | 0 18 files changed, 14 insertions(+), 227 deletions(-) rename CONTRIBUTING.md => .github/CONTRIBUTING.md (100%) rename PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md (100%) rename {docker => bin}/build_info.sh (100%) rename {docker => bin}/dev+mongo.sh (100%) rename entrypoint.sh => bin/entrypoint.sh (100%) delete mode 100644 bootstrap.sample.json delete mode 100644 docker/README.md delete mode 100755 docker/bootstrap-accounts.sh delete mode 100755 docker/bootstrap-data.sh delete mode 100644 docker/pymongo-cli.py rename docker/unit.json => nginx-unit.json (100%) rename TESTING.md => tests/README.md (100%) rename tests/bin/{run-tests-docker.sh => docker-tests.sh} (96%) rename tests/bin/{run-tests-ubuntu.sh => tests.sh} (100%) diff --git a/.dockerignore b/.dockerignore index 3e75f5950..4fe073583 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,8 @@ .git +**/.DS_Store +**/.*sw[op] +**/*.py[co] +**/*.egg-info -# --- Below Should mirror .gitignore, without leading '/' --- # -.DS_Store -.*sw[op] -*.py[co] -*.egg-info -persistent -runtime -bootstrap.json -.cache -.coverage* -coverage.xml -htmlcov -node_modules +/.coverage* +/htmlcov diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE.md diff --git a/.gitignore b/.gitignore index 14982fa1d..9dd251cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,6 @@ .*sw[op] *.py[co] *.egg-info -/persistent -/runtime -bootstrap.json -.cache + /.coverage* -coverage.xml /htmlcov -node_modules/ -/bin/accesslog.csv diff --git a/.travis.yml b/.travis.yml index ff8d7c6e6..9dcb50df2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ script: - docker build -t core:dist --target dist --build-arg VCS_BRANCH="$TRAVIS_BRANCH" --build-arg VCS_COMMIT="$TRAVIS_COMMIT" . - docker build -t core:testing --target dev . - docker save -o "$DOCKER_DIR/image.tar" $(docker history -q core:build | grep -v '') $(docker history -q core:dist | grep -v '') - - ./tests/bin/run-tests-docker.sh --image core:testing + - ./tests/bin/docker-tests.sh --image core:testing after_success: - if [ "$TRAVIS_TAG" ]; then diff --git a/Dockerfile b/Dockerfile index 380ed1606..8d3ef864a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ VOLUME /data/persistent WORKDIR /src/core ENV SCITRAN_PERSISTENT_DATA_PATH=/data/persistent -COPY docker/unit.json /var/local/unit/conf.json +COPY nginx-unit.json /var/local/unit/conf.json COPY requirements.txt requirements.txt RUN pip install -r requirements.txt @@ -33,9 +33,9 @@ RUN pip install -e . ARG VCS_BRANCH=NULL ARG VCS_COMMIT=NULL -RUN docker/build_info.sh $VCS_BRANCH $VCS_COMMIT | tee /version.json +RUN ./bin/build_info.sh $VCS_BRANCH $VCS_COMMIT | tee /version.json -ENTRYPOINT ["./entrypoint.sh"] +ENTRYPOINT ["./bin/entrypoint.sh"] CMD ["unitd", "--control", "*:8080", "--no-daemon", "--log", "/dev/stdout"] @@ -47,4 +47,4 @@ RUN apk add --no-cache mongodb RUN pip install -r tests/requirements.txt -CMD ["./docker/dev+mongo.sh"] +CMD ["./bin/dev+mongo.sh"] diff --git a/docker/build_info.sh b/bin/build_info.sh similarity index 100% rename from docker/build_info.sh rename to bin/build_info.sh diff --git a/docker/dev+mongo.sh b/bin/dev+mongo.sh similarity index 100% rename from docker/dev+mongo.sh rename to bin/dev+mongo.sh diff --git a/entrypoint.sh b/bin/entrypoint.sh similarity index 100% rename from entrypoint.sh rename to bin/entrypoint.sh diff --git a/bootstrap.sample.json b/bootstrap.sample.json deleted file mode 100644 index e85943b67..000000000 --- a/bootstrap.sample.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "groups": [ - { - "_id": "unknown", - "name": "Unknown", - "roles": [ - { - "access": "admin", - "_id": "user1@example.com" - } - ] - } - ], - "users": [ - { - "_id": "user1@example.com", - "email": "user1@example.com", - "firstname": "First", - "lastname": "User", - "root": true - } - ], - "drones": [ - { - "_id": "local", - "type": "engine" - } - ] -} diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 725f65ddd..000000000 --- a/docker/README.md +++ /dev/null @@ -1,61 +0,0 @@ - - -## Examples -Following the examples below will server up scitran/core with uwsgi on port 8080 -with auto-reload enabled. This will not utilize HTTPS, thus is meant only for -development. - -The below examples do not account for complexities of docker volumes, and -preserving their contents across container instances. - - -``` -# Build Example: - docker build -t scitran-core . - -# Run Example: - # First start mongodb - docker run --name some-mongo -d mongo - - # Then startup scitran-core, attaching to linked mongo. - docker run \ - --name scitran-core \ - -e "SCITRAN_PERSISTENT_DB_URI=mongodb://some-mongo:27017/scitran" \ - -e "SCITRAN_CORE_INSECURE=true" \ - -e "SCITRAN_CORE_DRONE_SECRET=change-me" \ - -e "SCITRAN_SITE_API_URL=http://localhost:8080/api" \ - -v $(pwd)/persistent/data:/var/scitran/data \ - -v $(pwd):/var/scitran/code/api \ - --link some-mongo \ - -p 0.0.0.0:8080:8080 \ - scitran-core \ - uwsgi \ - --ini /var/scitran/config/uwsgi-config.ini \ - --http 0.0.0.0:8080 \ - --http-keepalive \ - --python-autoreload 1 - - -# Bootstrap Account Example: - docker run \ - -e "SCITRAN_SITE_API_URL=http://scitran-core:8080/api" \ - -e "SCITRAN_CORE_DRONE_SECRET=change-me" \ - --link scitran-core \ - --rm \ - -v /dev/bali.prod/docker/uwsgi/bootstrap-dev.json:/accounts.json \ - scitran-core \ - /var/scitran/code/api/docker/bootstrap-accounts.sh \ - /accounts.json - - -# Bootstrap Data Example: - docker run \ - -e "SCITRAN_SITE_API_URL=http://scitran-core:8080/api" \ - -e "SCITRAN_CORE_DRONE_SECRET=change-me" \ - -e "PRE_RUNAS_CMD=/var/scitran/code/api/docker/bootstrap-data.sh" \ - --link scitran-core \ - --volumes-from scitran-core \ - --rm \ - scitran-core \ - echo "Data bootstrap complete." -``` diff --git a/docker/bootstrap-accounts.sh b/docker/bootstrap-accounts.sh deleted file mode 100755 index e8aab4c1b..000000000 --- a/docker/bootstrap-accounts.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e -set -x - -echo "IN BOOTSTRAP ACCOUNTS" - -( - -# Parse input parameters... -# -# bootstrap account file -bootstrap_user_file=${1:-'/var/scitran/code/api/bootstrap.json.sample'} - - -# Move to API folder for relative path assumptions later on -# -cd /var/scitran/code/api - -# Export PYTHONPATH for python script later on. -# -export PYTHONPATH=. - - -# Bootstrap Users -./bin/load_users_drone_secret.py --insecure --secret "${SCITRAN_CORE_DRONE_SECRET}" "${SCITRAN_SITE_API_URL}" "${bootstrap_user_file}" - - -) diff --git a/docker/bootstrap-data.sh b/docker/bootstrap-data.sh deleted file mode 100755 index 82ced6bc3..000000000 --- a/docker/bootstrap-data.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -set -e -set -x - -echo "IN BOOTSTRAP DATA" - -( - -# Parse input parameters... -# -# See if we're pulling latest data no matter what, ignoring cache. -# Default = N -GET_LATEST_DATA=${1:-N} - - -# Hard code some other vars important for bootstrapping -# - -# Set the commit hash or tag or branch desired for scitran/testdata. -# Branch name should only be used for testing convenience. -# -# When changing scitran/testdata, merge that change to master first, -# then reference that resulting commit hash here. -bootstrap_data_label=7d5c3608ff360d6ae28aab0ef262e6781c4ae8d6 - - -# Same as bootstrap_data_label above, except for scitran/reaper. -bootstrap_reaper_label=2.0.0-beta.0 - - -# Move to API folder for relative path assumptions later on -# -cd /var/scitran/code/api - -# Export PYTHONPATH for python script later on. -# -export PYTHONPATH=. - - -# Bootstrap data - -# Compare hash of source test data to most recent download. Remove local copy to force re-download if they are different. -TESTDATA_URL="https://github.com/scitran/testdata/archive/${bootstrap_data_label}.tar.gz" -TESTDATA_VERSION=$(curl -sLI ${TESTDATA_URL} | grep ETag | tail -n 1 | cut -f 2 -d '"') - -# use hidden -TESTDATA_DIR=$SCITRAN_PERSISTENT_PATH/testdata - -if [ ! -d "$TESTDATA_DIR" ] || [ ! -d "$TESTDATA_DIR/download" ] || [ ! -f "$TESTDATA_DIR/.testdata_version" ]; then - echo "Downloading testdata to $TESTDATA_DIR" - - # Remove old contents, as they may not be forward compatible. - rm -rf "$TESTDATA_DIR" - mkdir -p "$TESTDATA_DIR/download" - curl -L $TESTDATA_URL | tar xz -C "$TESTDATA_DIR/download" --strip-components 1 -else - if [ "$TESTDATA_VERSION" != "$(cat $TESTDATA_DIR/.testdata_version)" ]; then - echo "Testdata out of date; downloading" - - # Remove old contents, as they may not be forward compatible. - rm -rf "$TESTDATA_DIR" - mkdir -p "$TESTDATA_DIR/download" - curl -L $TESTDATA_URL | tar xz -C "$TESTDATA_DIR/download" --strip-components 1 - else - echo "Testdata up to date" - fi -fi -builtin echo "$TESTDATA_VERSION" > "$TESTDATA_DIR/.testdata_version" - -# pull reaper module -pip install "git+https://github.com/scitran/reaper.git@${bootstrap_reaper_label}" - - -## load the test data in -folder_sniper --yes --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" "$TESTDATA_DIR/download" $SCITRAN_SITE_API_URL - -) diff --git a/docker/pymongo-cli.py b/docker/pymongo-cli.py deleted file mode 100644 index 4dd9937ef..000000000 --- a/docker/pymongo-cli.py +++ /dev/null @@ -1,5 +0,0 @@ -import os -import pymongo -from bson import ObjectId -db_uri = os.getenv('SCITRAN_PERSISTENT_DB_URI') -db = pymongo.MongoClient(db_uri).get_default_database() diff --git a/docker/unit.json b/nginx-unit.json similarity index 100% rename from docker/unit.json rename to nginx-unit.json diff --git a/TESTING.md b/tests/README.md similarity index 100% rename from TESTING.md rename to tests/README.md diff --git a/tests/bin/run-tests-docker.sh b/tests/bin/docker-tests.sh similarity index 96% rename from tests/bin/run-tests-docker.sh rename to tests/bin/docker-tests.sh index 88f400ea4..f9e957cc0 100755 --- a/tests/bin/run-tests-docker.sh +++ b/tests/bin/docker-tests.sh @@ -17,7 +17,7 @@ Options: -B, --no-build Skip rebuilding default Docker image --image IMAGE Use custom Docker image - -- TEST_ARGS Arguments passed to tests/bin/run-tests-ubuntu.sh + -- TEST_ARGS Arguments passed to tests/bin/tests.sh " @@ -89,7 +89,7 @@ main() { --env SCITRAN_PERSISTENT_DB_URI=mongodb://core-test-service:27017/scitran \ --env SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://core-test-service:27017/logs \ scitran/core:testing \ - tests/bin/run-tests-ubuntu.sh $TEST_ARGS + tests/bin/tests.sh $TEST_ARGS } diff --git a/tests/bin/run-tests-ubuntu.sh b/tests/bin/tests.sh similarity index 100% rename from tests/bin/run-tests-ubuntu.sh rename to tests/bin/tests.sh