From d43a3790e5a171025686e6a5c49f7e66e87d065c Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Tue, 12 Feb 2019 13:38:42 +1000 Subject: [PATCH 01/11] Add base images for Python 2.7.15, 3.7.2 and CLI/CKAN variants. --- Makefile | 47 +++++++++++++++++++++++++++++++ images/python/Dockerfile | 31 ++++++++++++++++++++ images/python/cli-ckan/Dockerfile | 10 +++++++ images/python/cli/Dockerfile | 7 +++++ 4 files changed, 95 insertions(+) create mode 100644 images/python/Dockerfile create mode 100644 images/python/cli-ckan/Dockerfile create mode 100644 images/python/cli/Dockerfile diff --git a/Makefile b/Makefile index fa7a6270d9..e7ed1a95c7 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,12 @@ BRANCH_NAME := # Docker Build Context docker_build = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(LAGOON_VERSION) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) -t $(CI_BUILD_TAG)/$(1) -f $(2) $(3) +# Build a Python docker image. Expects as arguments: +# 1. Python version +# 2. Location of Dockerfile +# 3. Path of Docker Build context +docker_build_python = docker build $(DOCKER_BUILD_PARAMS) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) --build-arg PYTHON_VERSION=$(1) -t $(CI_BUILD_TAG)/python:$(2) -f $(3) $(4) + # Build a PHP docker image. Expects as arguments: # 1. PHP version # 2. PHP version and type of image (ie 7.0-fpm, 7.0-cli etc) @@ -178,6 +184,47 @@ build/curator: build/commons images/curator/Dockerfile build/oc-build-deploy-dind: build/oc images/oc-build-deploy-dind build/athenapdf-service: images/athenapdf-service/Dockerfile + +####### +####### Python Images +####### +####### Python Images are alpine linux based Python images. + +pythonimages := python__2.7.15 \ + python__3.7.2 \ + python__2.7.15-cli \ + python__3.7.2-cli \ + python__2.7.15-cli-ckan + +build-pythonimages = $(foreach image,$(pythonimages),build/$(image)) + +# Define the make recepie for all base images +$(build-pythonimages): build/commons + $(eval clean = $(subst build/python__,,$@)) + $(eval version = $(word 1,$(subst -, ,$(clean)))) + $(eval type = $(word 2,$(subst -, ,$(clean)))) + $(eval subtype = $(word 3,$(subst -, ,$(clean)))) +# this fills variables only if $type is existing, if not they are just empty + $(eval type_dash = $(if $(type),-$(type))) + $(eval type_slash = $(if $(type),/$(type))) +# if there is a subtype, add it. If not, just keep what we already had + $(eval type_dash = $(if $(subtype),-$(type)-$(subtype),$(type_dash))) + $(eval type_slash = $(if $(subtype),/$(type)-$(subtype),$(type_slash))) +# Call the docker build + $(call docker_build_python,$(version),$(version)$(type_dash),images/python$(type_slash)/Dockerfile,images/python$(type_slash)) +# Touch an empty file which make itself is using to understand when the image has been last build + touch $@ + +base-images += $(pythonimages) +s3-images += python + +build/python__2.7.15 build/python__3.7.2: images/commons +build/python__2.7.15-cli: build/python__2.7.15 +build/python__3.7.2-cli: build/python__3.7.2 +build/python__2.7.15-cli-ckan: build/python__2.7.15-cli +build/python__3.7.2-cli-ckan: build/python__3.7.2-cli + + ####### ####### PHP Images ####### diff --git a/images/python/Dockerfile b/images/python/Dockerfile new file mode 100644 index 0000000000..f7099b1bee --- /dev/null +++ b/images/python/Dockerfile @@ -0,0 +1,31 @@ +ARG PYTHON_VERSION +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/commons as commons +FROM python:${PYTHON_VERSION}-alpine + +LABEL maintainer="amazee.io" +ENV LAGOON=python + +# Copy commons files +COPY --from=commons /lagoon /lagoon +COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/ +COPY --from=commons /sbin/tini /sbin/ +COPY --from=commons /home /home + +RUN chmod g+w /etc/passwd \ + && mkdir -p /home + +ENV TMPDIR=/tmp \ + TMP=/tmp \ + HOME=/home \ + # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV` + ENV=/home/.bashrc \ + # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV` + BASH_ENV=/home/.bashrc + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache python-dev \ + py2-pip \ + py2-virtualenv + diff --git a/images/python/cli-ckan/Dockerfile b/images/python/cli-ckan/Dockerfile new file mode 100644 index 0000000000..33982c004e --- /dev/null +++ b/images/python/cli-ckan/Dockerfile @@ -0,0 +1,10 @@ +ARG PYTHON_VERSION +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION}-cli + +RUN mkdir -p /app/ckan/default \ + && fix-permissions /app/ckan/default + +RUN virtualenv --no-site-packages /app/ckan/default \ + && . /app/ckan/default/bin/activate \ + && pip install setuptools==20.4 diff --git a/images/python/cli/Dockerfile b/images/python/cli/Dockerfile new file mode 100644 index 0000000000..89d2abe016 --- /dev/null +++ b/images/python/cli/Dockerfile @@ -0,0 +1,7 @@ +ARG PYTHON_VERSION +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION} + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache libpq git From c84fbfb6a011e4460d0ecf69af51b2ca12525097 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Tue, 12 Feb 2019 13:41:04 +1000 Subject: [PATCH 02/11] No CKAN support for Py3 yet. --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index e7ed1a95c7..27710528fe 100644 --- a/Makefile +++ b/Makefile @@ -222,7 +222,6 @@ build/python__2.7.15 build/python__3.7.2: images/commons build/python__2.7.15-cli: build/python__2.7.15 build/python__3.7.2-cli: build/python__3.7.2 build/python__2.7.15-cli-ckan: build/python__2.7.15-cli -build/python__3.7.2-cli-ckan: build/python__3.7.2-cli ####### From 013e8c735a9cd45a9c5e4a8072d8c728a0f555d4 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 13 Feb 2019 08:53:11 +1000 Subject: [PATCH 03/11] Base solr support. --- Makefile | 27 +- images/solr-ckan/Dockerfile | 10 + images/solr-ckan/solr5.5/conf/schema.xml | 188 ++ images/solr-ckan/solr5.5/conf/solrconfig.xml | 1800 ++++++++++++++++++ images/solr-ckan/solr6.6/conf/schema.xml | 188 ++ images/solr-ckan/solr6.6/conf/solrconfig.xml | 1494 +++++++++++++++ images/solr-ckan/solr7.5/conf/schema.xml | 188 ++ images/solr-ckan/solr7.5/conf/solrconfig.xml | 343 ++++ images/solr-ckan/solr7.5/solrconfig.xml | 1413 ++++++++++++++ 9 files changed, 5637 insertions(+), 14 deletions(-) create mode 100644 images/solr-ckan/Dockerfile create mode 100644 images/solr-ckan/solr5.5/conf/schema.xml create mode 100644 images/solr-ckan/solr5.5/conf/solrconfig.xml create mode 100644 images/solr-ckan/solr6.6/conf/schema.xml create mode 100644 images/solr-ckan/solr6.6/conf/solrconfig.xml create mode 100644 images/solr-ckan/solr7.5/conf/schema.xml create mode 100644 images/solr-ckan/solr7.5/conf/solrconfig.xml create mode 100644 images/solr-ckan/solr7.5/solrconfig.xml diff --git a/Makefile b/Makefile index 27710528fe..893263e47a 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,7 @@ images := oc \ mariadb-galera \ mariadb-galera-drupal \ postgres \ + postgres-ckan \ postgres-drupal \ oc-build-deploy-dind \ commons \ @@ -162,6 +163,7 @@ build/mariadb-drupal: build/mariadb images/mariadb-drupal/Dockerfile build/mariadb-galera: build/commons images/mariadb-galera/Dockerfile build/mariadb-galera-drupal: build/mariadb-galera images/mariadb-galera-drupal/Dockerfile build/postgres: build/commons images/postgres/Dockerfile +build/postgres-ckan: build/postgres images/postgres-ckan/Dockerfile build/postgres-drupal: build/postgres images/postgres-drupal/Dockerfile build/commons: images/commons/Dockerfile build/nginx: build/commons images/nginx/Dockerfile @@ -192,9 +194,7 @@ build/athenapdf-service: images/athenapdf-service/Dockerfile pythonimages := python__2.7.15 \ python__3.7.2 \ - python__2.7.15-cli \ - python__3.7.2-cli \ - python__2.7.15-cli-ckan + python__2.7.15-ckan build-pythonimages = $(foreach image,$(pythonimages),build/$(image)) @@ -203,15 +203,10 @@ $(build-pythonimages): build/commons $(eval clean = $(subst build/python__,,$@)) $(eval version = $(word 1,$(subst -, ,$(clean)))) $(eval type = $(word 2,$(subst -, ,$(clean)))) - $(eval subtype = $(word 3,$(subst -, ,$(clean)))) # this fills variables only if $type is existing, if not they are just empty $(eval type_dash = $(if $(type),-$(type))) - $(eval type_slash = $(if $(type),/$(type))) -# if there is a subtype, add it. If not, just keep what we already had - $(eval type_dash = $(if $(subtype),-$(type)-$(subtype),$(type_dash))) - $(eval type_slash = $(if $(subtype),/$(type)-$(subtype),$(type_slash))) # Call the docker build - $(call docker_build_python,$(version),$(version)$(type_dash),images/python$(type_slash)/Dockerfile,images/python$(type_slash)) + $(call docker_build_python,$(version),$(version)$(type_dash),images/python$(type_dash)/Dockerfile,images/python$(type_dash)) # Touch an empty file which make itself is using to understand when the image has been last build touch $@ @@ -219,9 +214,7 @@ base-images += $(pythonimages) s3-images += python build/python__2.7.15 build/python__3.7.2: images/commons -build/python__2.7.15-cli: build/python__2.7.15 -build/python__3.7.2-cli: build/python__3.7.2 -build/python__2.7.15-cli-ckan: build/python__2.7.15-cli +build/python__2.7.15-ckan: build/python__2.7.15 ####### @@ -290,7 +283,10 @@ solrimages := solr__5.5 \ solr__7.5 \ solr__5.5-drupal \ solr__6.6-drupal \ - solr__7.5-drupal + solr__7.5-drupal \ + solr__5.5-ckan \ + solr__6.6-ckan \ + solr__7.5-ckan build-solrimages = $(foreach image,$(solrimages),build/$(image)) @@ -314,6 +310,9 @@ build/solr__5.5 build/solr__6.6 build/solr__7.5: images/commons build/solr__5.5-drupal: build/solr__5.5 build/solr__6.6-drupal: build/solr__6.6 build/solr__7.5-drupal: build/solr__7.5 +build/solr__5.5-ckan: build/solr__5.5 +build/solr__6.6-ckan: build/solr__6.6 +build/solr__7.5-ckan: build/solr__7.5 ####### ####### Node Images @@ -522,7 +521,7 @@ $(run-rest-tests): minishift build/node__6-builder build/node__8-builder build/o IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest) IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter) -tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/php__7.3-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift +tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/php__7.3-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/postgres-ckan build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift $(eval testname = $(subst tests/,,$@)) IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest) drush-alias IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter) diff --git a/images/solr-ckan/Dockerfile b/images/solr-ckan/Dockerfile new file mode 100644 index 0000000000..be3563bcd4 --- /dev/null +++ b/images/solr-ckan/Dockerfile @@ -0,0 +1,10 @@ +ARG SOLR_MAJ_MIN_VERSION +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/solr:${SOLR_MAJ_MIN_VERSION} +ARG SOLR_MAJ_MIN_VERSION + +COPY solr${SOLR_MAJ_MIN_VERSION} /solr-conf + +RUN precreate-core ckan /solr-conf + +CMD ["solr-foreground"] diff --git a/images/solr-ckan/solr5.5/conf/schema.xml b/images/solr-ckan/solr5.5/conf/schema.xml new file mode 100644 index 0000000000..8e5018a2e2 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/schema.xml @@ -0,0 +1,188 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +index_id +text + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/solr-ckan/solr5.5/conf/solrconfig.xml b/images/solr-ckan/solr5.5/conf/solrconfig.xml new file mode 100644 index 0000000000..a0549e1146 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/solrconfig.xml @@ -0,0 +1,1800 @@ + + + + + + + + + ${solr.abortOnConfigurationError:true} + + + ${solr.luceneMatchVersion:LUCENE_50} + + + + + + + + + + + + + + + + + + + + + /var/solr/${solr.core.name} + + + + + + + + + + + + + + + + + + + + + + + + 32 + + + + + + + + + + 4 + + + ${solr.lock.type:none} + + + + + + false + + + true + + + + + 1 + + 0 + + + + + + true + + + + + + + + + + + + + + + + ${solr.autoCommit.MaxDocs:10000} + ${solr.autoCommit.MaxTime:120000} + + + + + ${solr.autoSoftCommit.MaxDocs:2000} + ${solr.autoSoftCommit.MaxTime:10000} + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + 1024 + + + -1 + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + solr rocks010 + + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + json + true + text + + + + + + + + {!xport} + xsort + false + + + + query + + + + + + + + + + + + + + + + + edismax + content + explicit + true + 0.01 + + ${solr.pinkPony.timeAllowed:-1} + *:* + + + false + + true + false + + 1 + + + spellcheck + elevator + + + + + + + content + 1 + 1 + 3 + 15 + 20 + false + + ${solr.mlt.timeAllowed:2000} + + + + + + + content + explicit + true + + + + + + + + text + + + + + + + _src_ + + true + + + + + + + + + + + + + + + text + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + true + + + + + + + ${solr.replication.master:false} + commit + startup + ${solr.replication.confFiles:schema.xml,mapping-ISOLatin1Accent.txt,protwords.txt,stopwords.txt,synonyms.txt,elevate.xml} + + + ${solr.replication.slave:false} + ${solr.replication.masterUrl:http://localhost:8983/solr}/replication + ${solr.replication.pollInterval:00:00:60} + + + + + + + true + json + true + + + + + + + + + + default + wordbreak + false + false + 1 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + mySuggester + FuzzyLookupFactory + DocumentDictionaryFactory + cat + price + string + + + + + + true + 10 + + + suggest + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + terms + + + + + + + string + elevate.xml + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + 5 + + + + + + + + + + + + + *:* + + + + + + + + + + + + textSpell + + + + default + spell + spellchecker + true + + + + + + diff --git a/images/solr-ckan/solr6.6/conf/schema.xml b/images/solr-ckan/solr6.6/conf/schema.xml new file mode 100644 index 0000000000..8e5018a2e2 --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/schema.xml @@ -0,0 +1,188 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +index_id +text + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/solr-ckan/solr6.6/conf/solrconfig.xml b/images/solr-ckan/solr6.6/conf/solrconfig.xml new file mode 100644 index 0000000000..12f7fb9966 --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/solrconfig.xml @@ -0,0 +1,1494 @@ + + + +]> + + + + + + + ${solr.abortOnConfigurationError:true} + + + ${solr.luceneMatchVersion:LUCENE_60} + + + + + + + + + + + + + + + + + + + + /var/solr/${solr.core.name} + + + + + + + + + ${solr.hdfs.home:} + + ${solr.hdfs.confdir:} + + ${solr.hdfs.blockcache.enabled:true} + + ${solr.hdfs.blockcache.global:true} + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4 + + + + + + + ${solr.lock.type:none} + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + + ${solr.autoCommit.MaxDocs:10000} + ${solr.autoCommit.MaxTime:120000} + false + + + + + ${solr.autoSoftCommit.MaxDocs:2000} + ${solr.autoSoftCommit.MaxTime:10000} + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + + + edismax + content + explicit + true + 0.01 + + ${solr.selectSearchHandler.timeAllowed:-1} + *:* + + + false + + true + false + + 1 + + + spellcheck + elevator + + + + + + + explicit + json + true + text + + + + + + + + + + content + 1 + 1 + 3 + 15 + 20 + false + + ${solr.mlt.timeAllowed:2000} + + + + + + + content + explicit + true + + + + + + text + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + explicit + true + + + + + + + ${solr.replication.master:false} + commit + startup + ${solr.replication.confFiles:schema.xml,mapping-ISOLatin1Accent.txt,protwords.txt,stopwords.txt,synonyms.txt,elevate.xml} + + + ${solr.replication.slave:false} + ${solr.replication.masterUrl:http://localhost:8983/solr}/replication + ${solr.replication.pollInterval:00:00:60} + + + + + + + true + json + true + + + + + + + + &spellcheck; + + + + + spell + + default + wordbreak + on + false + false + 1 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + mySuggester + FuzzyLookupFactory + DocumentDictionaryFactory + cat + price + string + + + + + + true + 10 + + + suggest + + + + + + + + + text + true + + + tvComponent + + + + + + + + + + true + false + + + terms + + + + + + + false + false + + false + + true + false + + 1 + + + terms + spellcheck + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + + + + + + + + + application/json; charset=UTF-8 + + + + + + + + + 5 + + + + + + + + + + + + + + + diff --git a/images/solr-ckan/solr7.5/conf/schema.xml b/images/solr-ckan/solr7.5/conf/schema.xml new file mode 100644 index 0000000000..8e5018a2e2 --- /dev/null +++ b/images/solr-ckan/solr7.5/conf/schema.xml @@ -0,0 +1,188 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +index_id +text + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/solr-ckan/solr7.5/conf/solrconfig.xml b/images/solr-ckan/solr7.5/conf/solrconfig.xml new file mode 100644 index 0000000000..655ffffa7f --- /dev/null +++ b/images/solr-ckan/solr7.5/conf/solrconfig.xml @@ -0,0 +1,343 @@ + + + + + + 7.5.0 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + ${solr.lock.type:native} + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + ${solr.autoCommit.maxTime:15000} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + 1024 + + + + + true + 20 + 200 + + + + + + + + + + false + 2 + + + + + + + + + + + + + + explicit + 10 + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + + + _text_ + + + + + + + + add-unknown-fields-to-the-schema + + + + + + + + true + ignored_ + _text_ + + + + + + + + + + + explicit + true + + + + + + + text_general + + + default + _text_ + solr.DirectSolrSpellChecker + internal + 0.5 + 2 + 1 + 5 + 4 + 0.01 + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + true + false + + + + terms + + + + + + + string + elevate.xml + + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + 70 + 0.5 + [-\w ,/\n\"']{20,200} + + + + + + ]]> + ]]> + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + + 10 + .,!? + + + + + + WORD + en + US + + + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + text/plain; charset=UTF-8 + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + 5 + + + diff --git a/images/solr-ckan/solr7.5/solrconfig.xml b/images/solr-ckan/solr7.5/solrconfig.xml new file mode 100644 index 0000000000..8f668c5b76 --- /dev/null +++ b/images/solr-ckan/solr7.5/solrconfig.xml @@ -0,0 +1,1413 @@ + + + + + +]> + + + + + + + ${solr.abortOnConfigurationError:true} + + + ${solr.luceneMatchVersion:LUCENE_70} + + + + + + + + + + + + + + + + + + + + + + + + /var/solr/${solr.core.name} + + + + + + + + + ${solr.hdfs.home:} + + ${solr.hdfs.confdir:} + + ${solr.hdfs.blockcache.enabled:true} + + ${solr.hdfs.blockcache.global:true} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:none} + + + + + + + + + + + + + true + + + &index; + + + + + + + + + ${solr.ulog.dir:} + + + + + ${solr.autoCommit.MaxDocs:-1} + ${solr.autoCommit.MaxTime:15000} + false + + + + + + ${solr.autoSoftCommit.MaxDocs:-1} + ${solr.autoSoftCommit.MaxTime:-1} + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + lucene + id + explicit + true + + ${solr.selectSearchHandler.timeAllowed:-1} + + + false + + true + false + + 1 + + + spellcheck + elevator + + + + + + + explicit + json + true + text + + + + + + + + + + content + 1 + 1 + 3 + 15 + 20 + false + + ${solr.mlt.timeAllowed:2000} + + + + + + + content + explicit + true + + + + + + text + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + explicit + true + + + + + + + ${solr.replication.master:false} + commit + startup + ${solr.replication.confFiles:schema.xml,mapping-ISOLatin1Accent.txt,protwords.txt,stopwords.txt,synonyms.txt,elevate.xml} + + + ${solr.replication.slave:false} + ${solr.replication.masterUrl:http://localhost:8983/solr}/replication + ${solr.replication.pollInterval:00:00:60} + + + + + + + true + json + true + + + + + + + + &spellcheck; + + + + + spell + + default + wordbreak + on + false + false + 1 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + &extra; + + + + true + und + 10 + + + suggest + + + + + + + + + text + true + + + tvComponent + + + + + + + + + + true + false + + + terms + + + + + + + false + false + false + true + false + 1 + false + 10 + + + terms + spellcheck + suggest + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + + + + + + + + + + + + + + + From 18c1d02ffa6e2aa2dbf1dbb376cc2226aec29f82 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 13 Feb 2019 08:59:16 +1000 Subject: [PATCH 04/11] More complete solr config, remove 7.5 support. --- Makefile | 4 +- images/solr-ckan/solr5.5/conf/elevate.xml | 27 + .../solr5.5/conf/mapping-ISOLatin1Accent.txt | 14 + images/solr-ckan/solr5.5/conf/protwords.txt | 7 + .../solr5.5/conf/solrconfig_extra.xml | 80 + .../solr5.5/conf/solrcore.properties | 20 + images/solr-ckan/solr5.5/conf/stopwords.txt | 4 + images/solr-ckan/solr5.5/conf/synonyms.txt | 3 + images/solr-ckan/solr6.6/conf/elevate.xml | 27 + .../solr6.6/conf/mapping-ISOLatin1Accent.txt | 14 + images/solr-ckan/solr6.6/conf/protwords.txt | 7 + .../solr6.6/conf/solrconfig_spellcheck.xml | 89 ++ .../solr6.6/conf/solrcore.properties | 20 + images/solr-ckan/solr6.6/conf/stopwords.txt | 4 + images/solr-ckan/solr6.6/conf/synonyms.txt | 3 + images/solr-ckan/solr7.5/conf/schema.xml | 188 --- images/solr-ckan/solr7.5/conf/solrconfig.xml | 343 ---- images/solr-ckan/solr7.5/solrconfig.xml | 1413 ----------------- 18 files changed, 320 insertions(+), 1947 deletions(-) create mode 100644 images/solr-ckan/solr5.5/conf/elevate.xml create mode 100644 images/solr-ckan/solr5.5/conf/mapping-ISOLatin1Accent.txt create mode 100644 images/solr-ckan/solr5.5/conf/protwords.txt create mode 100644 images/solr-ckan/solr5.5/conf/solrconfig_extra.xml create mode 100644 images/solr-ckan/solr5.5/conf/solrcore.properties create mode 100644 images/solr-ckan/solr5.5/conf/stopwords.txt create mode 100644 images/solr-ckan/solr5.5/conf/synonyms.txt create mode 100644 images/solr-ckan/solr6.6/conf/elevate.xml create mode 100644 images/solr-ckan/solr6.6/conf/mapping-ISOLatin1Accent.txt create mode 100644 images/solr-ckan/solr6.6/conf/protwords.txt create mode 100644 images/solr-ckan/solr6.6/conf/solrconfig_spellcheck.xml create mode 100644 images/solr-ckan/solr6.6/conf/solrcore.properties create mode 100644 images/solr-ckan/solr6.6/conf/stopwords.txt create mode 100644 images/solr-ckan/solr6.6/conf/synonyms.txt delete mode 100644 images/solr-ckan/solr7.5/conf/schema.xml delete mode 100644 images/solr-ckan/solr7.5/conf/solrconfig.xml delete mode 100644 images/solr-ckan/solr7.5/solrconfig.xml diff --git a/Makefile b/Makefile index 893263e47a..76132ae1c2 100644 --- a/Makefile +++ b/Makefile @@ -285,8 +285,7 @@ solrimages := solr__5.5 \ solr__6.6-drupal \ solr__7.5-drupal \ solr__5.5-ckan \ - solr__6.6-ckan \ - solr__7.5-ckan + solr__6.6-ckan build-solrimages = $(foreach image,$(solrimages),build/$(image)) @@ -312,7 +311,6 @@ build/solr__6.6-drupal: build/solr__6.6 build/solr__7.5-drupal: build/solr__7.5 build/solr__5.5-ckan: build/solr__5.5 build/solr__6.6-ckan: build/solr__6.6 -build/solr__7.5-ckan: build/solr__7.5 ####### ####### Node Images diff --git a/images/solr-ckan/solr5.5/conf/elevate.xml b/images/solr-ckan/solr5.5/conf/elevate.xml new file mode 100644 index 0000000000..193a0e727a --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/elevate.xml @@ -0,0 +1,27 @@ + + + + + + + + + + diff --git a/images/solr-ckan/solr5.5/conf/mapping-ISOLatin1Accent.txt b/images/solr-ckan/solr5.5/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 0000000000..b92d03c550 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,14 @@ +# This file contains character mappings for the default fulltext field type. +# The source characters (on the left) will be replaced by the respective target +# characters before any other processing takes place. +# Lines starting with a pound character # are ignored. +# +# For sensible defaults, use the mapping-ISOLatin1Accent.txt file distributed +# with the example application of your Solr version. +# +# Examples: +# "À" => "A" +# "\u00c4" => "A" +# "\u00c4" => "\u0041" +# "æ" => "ae" +# "\n" => " " diff --git a/images/solr-ckan/solr5.5/conf/protwords.txt b/images/solr-ckan/solr5.5/conf/protwords.txt new file mode 100644 index 0000000000..cda8581497 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/protwords.txt @@ -0,0 +1,7 @@ +#----------------------------------------------------------------------- +# This file blocks words from being operated on by the stemmer and word delimiter. +& +< +> +' +" diff --git a/images/solr-ckan/solr5.5/conf/solrconfig_extra.xml b/images/solr-ckan/solr5.5/conf/solrconfig_extra.xml new file mode 100644 index 0000000000..c5bc3acfb5 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/solrconfig_extra.xml @@ -0,0 +1,80 @@ + + + +textSpell + + + + + + default + spell + spellchecker + true + + + + + + + + + + + + + + + diff --git a/images/solr-ckan/solr5.5/conf/solrcore.properties b/images/solr-ckan/solr5.5/conf/solrcore.properties new file mode 100644 index 0000000000..3a2433f676 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/solrcore.properties @@ -0,0 +1,20 @@ +# Defines Solr properties for this specific core. +solr.replication.master=false +solr.replication.slave=false +solr.replication.pollInterval=00:00:60 +solr.replication.masterUrl=http://localhost:8983/solr +solr.replication.confFiles=schema.xml,mapping-ISOLatin1Accent.txt,protwords.txt,stopwords.txt,synonyms.txt,elevate.xml +solr.mlt.timeAllowed=2000 +# You should not set your luceneMatchVersion to anything lower than your Solr +# Version. +solr.luceneMatchVersion=5.0 +solr.pinkPony.timeAllowed=-1 +# autoCommit after 10000 docs +solr.autoCommit.MaxDocs=10000 +# autoCommit after 2 minutes +solr.autoCommit.MaxTime=120000 +# autoSoftCommit after 2000 docs +solr.autoSoftCommit.MaxDocs=2000 +# autoSoftCommit after 10 seconds +solr.autoSoftCommit.MaxTime=10000 +solr.install.dir=../../.. diff --git a/images/solr-ckan/solr5.5/conf/stopwords.txt b/images/solr-ckan/solr5.5/conf/stopwords.txt new file mode 100644 index 0000000000..d7f243e48a --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/stopwords.txt @@ -0,0 +1,4 @@ +# Contains words which shouldn't be indexed for fulltext fields, e.g., because +# they're too common. For documentation of the format, see +# http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.StopFilterFactory +# (Lines starting with a pound character # are ignored.) diff --git a/images/solr-ckan/solr5.5/conf/synonyms.txt b/images/solr-ckan/solr5.5/conf/synonyms.txt new file mode 100644 index 0000000000..7d22eea6d6 --- /dev/null +++ b/images/solr-ckan/solr5.5/conf/synonyms.txt @@ -0,0 +1,3 @@ +# Contains synonyms to use for your index. For the format used, see +# http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.SynonymFilterFactory +# (Lines starting with a pound character # are ignored.) diff --git a/images/solr-ckan/solr6.6/conf/elevate.xml b/images/solr-ckan/solr6.6/conf/elevate.xml new file mode 100644 index 0000000000..193a0e727a --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/elevate.xml @@ -0,0 +1,27 @@ + + + + + + + + + + diff --git a/images/solr-ckan/solr6.6/conf/mapping-ISOLatin1Accent.txt b/images/solr-ckan/solr6.6/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 0000000000..b92d03c550 --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,14 @@ +# This file contains character mappings for the default fulltext field type. +# The source characters (on the left) will be replaced by the respective target +# characters before any other processing takes place. +# Lines starting with a pound character # are ignored. +# +# For sensible defaults, use the mapping-ISOLatin1Accent.txt file distributed +# with the example application of your Solr version. +# +# Examples: +# "À" => "A" +# "\u00c4" => "A" +# "\u00c4" => "\u0041" +# "æ" => "ae" +# "\n" => " " diff --git a/images/solr-ckan/solr6.6/conf/protwords.txt b/images/solr-ckan/solr6.6/conf/protwords.txt new file mode 100644 index 0000000000..cda8581497 --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/protwords.txt @@ -0,0 +1,7 @@ +#----------------------------------------------------------------------- +# This file blocks words from being operated on by the stemmer and word delimiter. +& +< +> +' +" diff --git a/images/solr-ckan/solr6.6/conf/solrconfig_spellcheck.xml b/images/solr-ckan/solr6.6/conf/solrconfig_spellcheck.xml new file mode 100644 index 0000000000..5c9d7ad79e --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/solrconfig_spellcheck.xml @@ -0,0 +1,89 @@ + + + + textSpell + + + + + + + default + spell + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + solr.WordBreakSolrSpellChecker + name + true + true + 10 + + + + + + + + + + + diff --git a/images/solr-ckan/solr6.6/conf/solrcore.properties b/images/solr-ckan/solr6.6/conf/solrcore.properties new file mode 100644 index 0000000000..d7d045b0fd --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/solrcore.properties @@ -0,0 +1,20 @@ +# Defines Solr properties for this specific core. +solr.replication.master=false +solr.replication.slave=false +solr.replication.pollInterval=00:00:60 +solr.replication.masterUrl=http://localhost:8983/solr +solr.replication.confFiles=schema.xml,mapping-ISOLatin1Accent.txt,protwords.txt,stopwords.txt,synonyms.txt,elevate.xml +solr.mlt.timeAllowed=2000 +# You should not set your luceneMatchVersion to anything lower than your Solr +# Version. +solr.luceneMatchVersion=6.0 +solr.selectSearchHandler.timeAllowed=-1 +# autoCommit after 10000 docs +solr.autoCommit.MaxDocs=10000 +# autoCommit after 2 minutes +solr.autoCommit.MaxTime=120000 +# autoSoftCommit after 2000 docs +solr.autoSoftCommit.MaxDocs=2000 +# autoSoftCommit after 10 seconds +solr.autoSoftCommit.MaxTime=10000 +solr.install.dir=../../.. diff --git a/images/solr-ckan/solr6.6/conf/stopwords.txt b/images/solr-ckan/solr6.6/conf/stopwords.txt new file mode 100644 index 0000000000..d7f243e48a --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/stopwords.txt @@ -0,0 +1,4 @@ +# Contains words which shouldn't be indexed for fulltext fields, e.g., because +# they're too common. For documentation of the format, see +# http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.StopFilterFactory +# (Lines starting with a pound character # are ignored.) diff --git a/images/solr-ckan/solr6.6/conf/synonyms.txt b/images/solr-ckan/solr6.6/conf/synonyms.txt new file mode 100644 index 0000000000..7d22eea6d6 --- /dev/null +++ b/images/solr-ckan/solr6.6/conf/synonyms.txt @@ -0,0 +1,3 @@ +# Contains synonyms to use for your index. For the format used, see +# http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.SynonymFilterFactory +# (Lines starting with a pound character # are ignored.) diff --git a/images/solr-ckan/solr7.5/conf/schema.xml b/images/solr-ckan/solr7.5/conf/schema.xml deleted file mode 100644 index 8e5018a2e2..0000000000 --- a/images/solr-ckan/solr7.5/conf/schema.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -index_id -text - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/images/solr-ckan/solr7.5/conf/solrconfig.xml b/images/solr-ckan/solr7.5/conf/solrconfig.xml deleted file mode 100644 index 655ffffa7f..0000000000 --- a/images/solr-ckan/solr7.5/conf/solrconfig.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - 7.5.0 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - ${solr.lock.type:native} - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - ${solr.autoCommit.maxTime:15000} - false - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - 1024 - - - - - true - 20 - 200 - - - - - - - - - - false - 2 - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - - - _text_ - - - - - - - - add-unknown-fields-to-the-schema - - - - - - - - true - ignored_ - _text_ - - - - - - - - - - - explicit - true - - - - - - - text_general - - - default - _text_ - solr.DirectSolrSpellChecker - internal - 0.5 - 2 - 1 - 5 - 4 - 0.01 - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - true - false - - - - terms - - - - - - - string - elevate.xml - - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - 70 - 0.5 - [-\w ,/\n\"']{20,200} - - - - - - ]]> - ]]> - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - text/plain; charset=UTF-8 - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - 5 - - - diff --git a/images/solr-ckan/solr7.5/solrconfig.xml b/images/solr-ckan/solr7.5/solrconfig.xml deleted file mode 100644 index 8f668c5b76..0000000000 --- a/images/solr-ckan/solr7.5/solrconfig.xml +++ /dev/null @@ -1,1413 +0,0 @@ - - - - - -]> - - - - - - - ${solr.abortOnConfigurationError:true} - - - ${solr.luceneMatchVersion:LUCENE_70} - - - - - - - - - - - - - - - - - - - - - - - - /var/solr/${solr.core.name} - - - - - - - - - ${solr.hdfs.home:} - - ${solr.hdfs.confdir:} - - ${solr.hdfs.blockcache.enabled:true} - - ${solr.hdfs.blockcache.global:true} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:none} - - - - - - - - - - - - - true - - - &index; - - - - - - - - - ${solr.ulog.dir:} - - - - - ${solr.autoCommit.MaxDocs:-1} - ${solr.autoCommit.MaxTime:15000} - false - - - - - - ${solr.autoSoftCommit.MaxDocs:-1} - ${solr.autoSoftCommit.MaxTime:-1} - - - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - static firstSearcher warming in solrconfig.xml - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - lucene - id - explicit - true - - ${solr.selectSearchHandler.timeAllowed:-1} - - - false - - true - false - - 1 - - - spellcheck - elevator - - - - - - - explicit - json - true - text - - - - - - - - - - content - 1 - 1 - 3 - 15 - 20 - false - - ${solr.mlt.timeAllowed:2000} - - - - - - - content - explicit - true - - - - - - text - - - - - - - true - ignored_ - - - true - links - ignored_ - - - - - - - - - - - - - - - explicit - true - - - - - - - ${solr.replication.master:false} - commit - startup - ${solr.replication.confFiles:schema.xml,mapping-ISOLatin1Accent.txt,protwords.txt,stopwords.txt,synonyms.txt,elevate.xml} - - - ${solr.replication.slave:false} - ${solr.replication.masterUrl:http://localhost:8983/solr}/replication - ${solr.replication.pollInterval:00:00:60} - - - - - - - true - json - true - - - - - - - - &spellcheck; - - - - - spell - - default - wordbreak - on - false - false - 1 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - &extra; - - - - true - und - 10 - - - suggest - - - - - - - - - text - true - - - tvComponent - - - - - - - - - - true - false - - - terms - - - - - - - false - false - false - true - false - 1 - false - 10 - - - terms - spellcheck - suggest - - - - - - - string - elevate.xml - - - - - - explicit - text - - - elevator - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 5 - - - - - - - - - - - - - - - From 8ee487e6b8c4da41dbf862ec786e4f3a93fc5252 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 13 Feb 2019 09:01:49 +1000 Subject: [PATCH 05/11] Remove need for cli python images. Add CKAN specific python image. Add CKAN specific postgres image. --- images/postgres-ckan/Dockerfile | 10 ++++++++++ images/{python/cli-ckan => python-ckan}/Dockerfile | 11 ++++++++++- images/python/80-shell-timeout.sh | 7 +++++++ images/python/Dockerfile | 7 +++++-- 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 images/postgres-ckan/Dockerfile rename images/{python/cli-ckan => python-ckan}/Dockerfile (55%) create mode 100644 images/python/80-shell-timeout.sh diff --git a/images/postgres-ckan/Dockerfile b/images/postgres-ckan/Dockerfile new file mode 100644 index 0000000000..2c4eda7949 --- /dev/null +++ b/images/postgres-ckan/Dockerfile @@ -0,0 +1,10 @@ +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/postgres + +# change log_min_error_statement and log_min_messages from `error` to `log` as drupal is prone to cause some errors which are all logged (yes `log` is a less verbose mode than `error`) +RUN sed -i "s/#log_min_error_statement = error/log_min_error_statement = log/" /usr/local/share/postgresql/postgresql.conf.sample \ + && sed -i "s/#log_min_messages = warning/log_min_messages = log/" /usr/local/share/postgresql/postgresql.conf.sample + +ENV POSTGRES_PASSWORD=ckan \ + POSTGRES_USER=ckan \ + POSTGRES_DB=ckan diff --git a/images/python/cli-ckan/Dockerfile b/images/python-ckan/Dockerfile similarity index 55% rename from images/python/cli-ckan/Dockerfile rename to images/python-ckan/Dockerfile index 33982c004e..c281e03761 100644 --- a/images/python/cli-ckan/Dockerfile +++ b/images/python-ckan/Dockerfile @@ -1,6 +1,15 @@ ARG PYTHON_VERSION ARG IMAGE_REPO -FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION}-cli +FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION} + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache git \ + libpq \ + postgresql-dev \ + gcc \ + musl-dev \ + libmagic RUN mkdir -p /app/ckan/default \ && fix-permissions /app/ckan/default diff --git a/images/python/80-shell-timeout.sh b/images/python/80-shell-timeout.sh new file mode 100644 index 0000000000..fdc02f389e --- /dev/null +++ b/images/python/80-shell-timeout.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# If we are running within kubernetes, set a shell timeout of 10mins. +# We do that so old shells are closed and we can idle the cli container +if [ $KUBERNETES_PORT ]; then + TMOUT=600 +fi \ No newline at end of file diff --git a/images/python/Dockerfile b/images/python/Dockerfile index f7099b1bee..9f9aaf60a8 100644 --- a/images/python/Dockerfile +++ b/images/python/Dockerfile @@ -26,6 +26,9 @@ ENV TMPDIR=/tmp \ RUN apk update \ && apk upgrade \ && apk add --no-cache python-dev \ - py2-pip \ - py2-virtualenv + py2-pip \ + py2-virtualenv +# Make sure shells are not running forever +COPY 80-shell-timeout.sh /lagoon/entrypoints/ +RUN echo "source /lagoon/entrypoints/80-shell-timeout.sh" >> /home/.bashrc From 0ec48b207ce3e585f14ddd7ef592d2756a72a7c4 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 13 Feb 2019 12:08:04 +1000 Subject: [PATCH 06/11] Add r/o user for datastore. --- images/postgres-ckan/90-datastore-user.sh | 7 +++++++ images/postgres-ckan/Dockerfile | 2 ++ 2 files changed, 9 insertions(+) create mode 100755 images/postgres-ckan/90-datastore-user.sh diff --git a/images/postgres-ckan/90-datastore-user.sh b/images/postgres-ckan/90-datastore-user.sh new file mode 100755 index 0000000000..d12ea97269 --- /dev/null +++ b/images/postgres-ckan/90-datastore-user.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE USER ckan_datastore with encrypted password 'ckan'; + GRANT ALL PRIVILEGES ON DATABASE ckan TO ckan_datastore; +EOSQL diff --git a/images/postgres-ckan/Dockerfile b/images/postgres-ckan/Dockerfile index 2c4eda7949..a20a80827b 100644 --- a/images/postgres-ckan/Dockerfile +++ b/images/postgres-ckan/Dockerfile @@ -8,3 +8,5 @@ RUN sed -i "s/#log_min_error_statement = error/log_min_error_statement = log/" / ENV POSTGRES_PASSWORD=ckan \ POSTGRES_USER=ckan \ POSTGRES_DB=ckan + +COPY 90-datastore-user.sh /docker-entrypoint-initdb.d/ From db8251f61c5d5872c78ef0ee26232ee8e75d9754 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Wed, 13 Feb 2019 13:04:41 +1000 Subject: [PATCH 07/11] Add required tooling for datapusher support. --- images/python-ckan/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/images/python-ckan/Dockerfile b/images/python-ckan/Dockerfile index c281e03761..6549686e86 100644 --- a/images/python-ckan/Dockerfile +++ b/images/python-ckan/Dockerfile @@ -9,7 +9,10 @@ RUN apk update \ postgresql-dev \ gcc \ musl-dev \ - libmagic + libmagic \ + libxslt-dev \ + libxml2-dev \ + libffi-dev RUN mkdir -p /app/ckan/default \ && fix-permissions /app/ckan/default From 33844daa2b1be460c10169a0e6164be9a04ecaf1 Mon Sep 17 00:00:00 2001 From: Stuart Rowlands Date: Tue, 19 Feb 2019 17:19:55 +1000 Subject: [PATCH 08/11] Add separate image for datapusher support. --- Makefile | 4 ++- images/python-ckandatapusher/Dockerfile | 34 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 images/python-ckandatapusher/Dockerfile diff --git a/Makefile b/Makefile index 76132ae1c2..e3a44af002 100644 --- a/Makefile +++ b/Makefile @@ -194,7 +194,8 @@ build/athenapdf-service: images/athenapdf-service/Dockerfile pythonimages := python__2.7.15 \ python__3.7.2 \ - python__2.7.15-ckan + python__2.7.15-ckan \ + python__2.7.15-ckandatapusher build-pythonimages = $(foreach image,$(pythonimages),build/$(image)) @@ -215,6 +216,7 @@ s3-images += python build/python__2.7.15 build/python__3.7.2: images/commons build/python__2.7.15-ckan: build/python__2.7.15 +build/python__2.7.15-ckandatapusher: build/python__2.7.15 ####### diff --git a/images/python-ckandatapusher/Dockerfile b/images/python-ckandatapusher/Dockerfile new file mode 100644 index 0000000000..c7b589dd7f --- /dev/null +++ b/images/python-ckandatapusher/Dockerfile @@ -0,0 +1,34 @@ +ARG PYTHON_VERSION +ARG IMAGE_REPO +FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION} + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache git \ + libpq \ + postgresql-dev \ + gcc \ + musl-dev \ + libmagic \ + libxslt-dev \ + libxml2-dev \ + libffi-dev \ + pcre-dev + +RUN virtualenv /app/ckan/datapusher + +RUN mkdir -p /app/ckan/datapusher/src \ + && mkdir -p /etc/ckan \ + && fix-permissions /app/ckan \ + && ln -s /app/ckan /usr/lib/ckan \ + && . /app/ckan/datapusher/bin/activate \ + && pip install uwsgi \ + && cd /app/ckan/datapusher/src \ + && git clone -b 0.0.14 https://github.com/ckan/datapusher.git \ + && cd datapusher \ + && /app/ckan/datapusher/bin/pip install -r requirements.txt \ + && /app/ckan/datapusher/bin/python setup.py develop \ + && cp deployment/datapusher.wsgi /etc/ckan/ \ + && cp deployment/datapusher_settings.py /etc/ckan/ + +ENTRYPOINT . /app/ckan/datapusher/bin/activate && uwsgi --http :8800 --wsgi-file /etc/ckan/datapusher.wsgi From fe84302aeed74fbcc47f1f72ec6ba6c827b8d688 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Mon, 18 Mar 2019 14:55:14 -0500 Subject: [PATCH 09/11] use minor and not patch versions. --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index e3a44af002..bd612576ca 100644 --- a/Makefile +++ b/Makefile @@ -192,10 +192,10 @@ build/athenapdf-service: images/athenapdf-service/Dockerfile ####### ####### Python Images are alpine linux based Python images. -pythonimages := python__2.7.15 \ - python__3.7.2 \ - python__2.7.15-ckan \ - python__2.7.15-ckandatapusher +pythonimages := python__2.7 \ + python__3.7 \ + python__2.7-ckan \ + python__2.7-ckandatapusher build-pythonimages = $(foreach image,$(pythonimages),build/$(image)) @@ -214,9 +214,9 @@ $(build-pythonimages): build/commons base-images += $(pythonimages) s3-images += python -build/python__2.7.15 build/python__3.7.2: images/commons -build/python__2.7.15-ckan: build/python__2.7.15 -build/python__2.7.15-ckandatapusher: build/python__2.7.15 +build/python__2.7 build/python__3.7: images/commons +build/python__2.7-ckan: build/python__2.7 +build/python__2.7-ckandatapusher: build/python__2.7 ####### From c7ed84c570549add272b9a27c08182bf3daf7bf9 Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Wed, 20 Mar 2019 13:10:21 -0500 Subject: [PATCH 10/11] update docker entrypoints --- images/python-ckandatapusher/Dockerfile | 2 +- images/python/Dockerfile | 3 +++ images/python/cli/Dockerfile | 7 ------- 3 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 images/python/cli/Dockerfile diff --git a/images/python-ckandatapusher/Dockerfile b/images/python-ckandatapusher/Dockerfile index c7b589dd7f..88f560f59d 100644 --- a/images/python-ckandatapusher/Dockerfile +++ b/images/python-ckandatapusher/Dockerfile @@ -31,4 +31,4 @@ RUN mkdir -p /app/ckan/datapusher/src \ && cp deployment/datapusher.wsgi /etc/ckan/ \ && cp deployment/datapusher_settings.py /etc/ckan/ -ENTRYPOINT . /app/ckan/datapusher/bin/activate && uwsgi --http :8800 --wsgi-file /etc/ckan/datapusher.wsgi +CMD ["sh", "-c", ". /app/ckan/datapusher/bin/activate && uwsgi --http :8800 --wsgi-file /etc/ckan/datapusher.wsgi"] diff --git a/images/python/Dockerfile b/images/python/Dockerfile index 9f9aaf60a8..33c8e39ce2 100644 --- a/images/python/Dockerfile +++ b/images/python/Dockerfile @@ -32,3 +32,6 @@ RUN apk update \ # Make sure shells are not running forever COPY 80-shell-timeout.sh /lagoon/entrypoints/ RUN echo "source /lagoon/entrypoints/80-shell-timeout.sh" >> /home/.bashrc + +ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] +CMD ["python"] \ No newline at end of file diff --git a/images/python/cli/Dockerfile b/images/python/cli/Dockerfile deleted file mode 100644 index 89d2abe016..0000000000 --- a/images/python/cli/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG PYTHON_VERSION -ARG IMAGE_REPO -FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION} - -RUN apk update \ - && apk upgrade \ - && apk add --no-cache libpq git From e17fa6d3d50bd7b2be4d7c8dc4b73f9a2eb4b03e Mon Sep 17 00:00:00 2001 From: Schnitzel Date: Thu, 21 Mar 2019 11:15:55 -0500 Subject: [PATCH 11/11] update to new variables --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bd612576ca..074e4b8f6a 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ docker_build = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$( # 1. Python version # 2. Location of Dockerfile # 3. Path of Docker Build context -docker_build_python = docker build $(DOCKER_BUILD_PARAMS) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) --build-arg PYTHON_VERSION=$(1) -t $(CI_BUILD_TAG)/python:$(2) -f $(3) $(4) +docker_build_python = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(LAGOON_VERSION) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) --build-arg PYTHON_VERSION=$(1) -t $(CI_BUILD_TAG)/python:$(2) -f $(3) $(4) # Build a PHP docker image. Expects as arguments: # 1. PHP version @@ -211,7 +211,7 @@ $(build-pythonimages): build/commons # Touch an empty file which make itself is using to understand when the image has been last build touch $@ -base-images += $(pythonimages) +base-images-with-versions += $(pythonimages) s3-images += python build/python__2.7 build/python__3.7: images/commons