From cae2b5b7c73c722d916faf94289299e2b11e544e Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Fri, 14 May 2021 16:59:22 +0200 Subject: [PATCH] Feature/193 (#196) * #193 chore: added oauth * #193 chore: install social login after pip upgrade * #193 chore: update & fix dependencies and implemented twitter login * #193 chore: moved the social keys and secrets to a k8 secret * #193 chore: small refactor of scidash dockerfile * #193 chore: updated k8s scidash yaml --- .dockerignore | 3 + requirements.in | 29 ++-- requirements.txt | 130 +++--------------- scidash/account/static/css/main.css | 3 +- .../account/templates/registration/login.html | 31 ++--- scidash/main/settings.py | 42 +++++- scidash/main/urls.py | 5 +- service/deployment/docker-compose.yml | 9 +- service/docker/Dockerfile-scidash | 12 +- service/k8s/scidash.yaml | 19 +++ service/scripts/install-backend.sh | 12 +- 11 files changed, 140 insertions(+), 155 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8bb6a6a4..6e5eccd6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,5 @@ venv +tmp **/*/node_modules +pygeppetto-django +static diff --git a/requirements.in b/requirements.in index f7392fab..87a504d5 100644 --- a/requirements.in +++ b/requirements.in @@ -1,21 +1,26 @@ -django==1.11.23 -psycopg2==2.7.7 -channels==2.1.2 +django==1.11.29 +psycopg2-binary==2.8.6 +channels==2.3.1 djangorestframework==3.7.1 drf-writable-nested django-filter==1.1.0 -djangorestframework-jwt -django-extensions +django-timezone-field==3.1 +# djangorestframework-jwt +django-extensions==2.1.6 django-dotenv rest-framework-cache django-material -celery[redis] -django-celery-beat -django-celery-results +celery[redis]==4.2.2 +django-celery-beat==1.4.0 +django-celery-results==1.0.4 django-db-logger -git+git://github.com/scidash/neuronunit@metacell#egg=neuronunit -git+git://github.com/MetaCell/scidash-api.git@master#egg=scidash_api -wheel==0.33.6 -sentry-sdk==1.0.0 +neo==0.5.2 +pynn==0.9.5 +Jinja2==2.11.3 +git+git://github.com/scidash/neuronunit@metacell +git+git://github.com/MetaCell/scidash-api.git@master +wheel +sentry-sdk django-ckeditor==5.9.0 django-admin-sortable2==0.7.5 +social-auth-app-django==4.0.0 # will be installed through install script install-backend diff --git a/requirements.txt b/requirements.txt index 0e823f8d..87a504d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,112 +1,26 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile requirements.in -# -amqp==2.4.2 # via kombu -asgiref==2.3.2 # via channels -async-timeout==3.0.1 # via asgiref -attrs==19.1.0 # via automat, twisted -autobahn==19.3.3 # via daphne -automat==0.7.0 # via twisted -billiard==3.5.0.5 # via celery +django==1.11.29 +psycopg2-binary==2.8.6 +channels==2.3.1 +djangorestframework==3.7.1 +drf-writable-nested +django-filter==1.1.0 +django-timezone-field==3.1 +# djangorestframework-jwt +django-extensions==2.1.6 +django-dotenv +rest-framework-cache +django-material celery[redis]==4.2.2 -channels==2.1.2 -constantly==15.1.0 # via twisted -daphne==2.2.5 # via channels -dask[complete]==2.9.0 # via distributed -deap==1.3.0 -decorator==4.4.1 # via ipyparallel, ipython, networkx, traitlets, validators -defusedxml==0.6.0 # via nbconvert -distributed==2.9.0 # via dask -django-admin-sortable2==0.7.5 django-celery-beat==1.4.0 django-celery-results==1.0.4 -django-ckeditor==5.9.0 -django-db-logger==0.1.7 -django-dotenv==1.4.2 -django-extensions==2.1.6 -django-filter==1.1.0 -django-material==1.5.2 -django-timezone-field==3.0 # via django-celery-beat -django==1.11.23 -djangorestframework-jwt==1.11.0 -djangorestframework==3.7.1 -drf-writable-nested==0.5.1 -git+git://github.com/scidash/neuronunit@metacell#egg=neuronunit -git+git://github.com/MetaCell/scidash-api.git@master#egg=scidash_api -hyperlink==18.0.0 # via twisted -idna==2.8 # via hyperlink, requests -igor==0.3 -imageio==2.6.1 # via scikit-image -importlib-metadata==1.3.0 # via jsonschema -incremental==17.5.0 # via twisted -ipykernel==5.1.3 # via ipyparallel -ipyparallel==6.2.4 -ipython-genutils==0.2.0 # via ipyparallel, nbformat, traitlets -ipython==7.10.2 # via ipykernel, ipyparallel -jedi==0.15.2 # via ipython -jinja2==2.10.3 # via allensdk, bokeh, nbconvert -jsonschema==3.2.0 # via nbformat -jupyter-client==5.3.4 # via ipykernel, ipyparallel -jupyter-core==4.6.1 # via jupyter-client, nbconvert, nbformat -kiwisolver==1.1.0 # via matplotlib -kombu==4.3.0 # via celery -lazyarray==0.3.3 # via pynn -libneuroml==0.2.47 -llvmlite==0.30.0 # via numba -lmfit==1.0.0 -locket==0.2.0 # via partd -lxml==4.4.2 # via libneuroml -markupsafe==1.1.1 # via jinja2 -matplotlib==3.1.2 # via allensdk, scikit-image -mistune==0.8.4 # via nbconvert -more-itertools==8.0.2 # via zipp -msgpack==0.6.2 # via distributed -nbconvert==5.6.1 -nbformat==4.4.0 # via nbconvert -neo==0.5.2 # via elephant, pynn -networkx==2.4 # via scikit-image -neuromllite==0.1.9 -numba==0.46.0 -numpy==1.18.0 # via allensdk, bokeh, dask, deap, efel, elephant, h5py, imageio, lazyarray, lmfit, matplotlib, neo, numba, pandas, patsy, pynn, pynrrd, pywavelets, scipy, statsmodels -packaging==19.2 # via bokeh -pandas==0.25.3 # via allensdk, dask, statsmodels -pandocfilters==1.4.2 # via nbconvert -parso==0.5.2 # via jedi -partd==1.1.0 # via dask -patsy==0.5.1 # via statsmodels -pexpect==4.7.0 # via ipython -pickleshare==0.7.5 # via ipython -pillow==6.2.1 # via bokeh, imageio, scikit-image -prompt-toolkit==3.0.2 # via ipython -psutil==5.6.7 # via distributed -psycopg2==2.7.7 -pyhamcrest==1.9.0 # via twisted -pyjwt==1.7.1 # via djangorestframework-jwt -pylems==0.4.9.3 -git+https://github.com/rgerkin/pyneuroml@master#egg=pyneuroml-9999 +django-db-logger +neo==0.5.2 pynn==0.9.5 -pynrrd==0.4.1 # via allensdk -pyparsing==2.4.6 # via matplotlib, packaging -pyrsistent==0.15.6 # via jsonschema -python-crontab==2.3.6 # via django-celery-beat -python-dateutil==2.8.0 # via python-crontab -pytz==2018.9 # via celery, django, django-timezone-field -redis==2.10.6 # via celery -rest-framework-cache==0.1 -sentry-sdk==1.0.0 -six==1.12.0 # via autobahn, automat, django-extensions, pyhamcrest, python-dateutil, txaio -twisted==18.9.0 # via daphne -txaio==18.8.1 # via autobahn -vine==1.3.0 # via amqp -wcwidth==0.1.7 # via prompt-toolkit -webencodings==0.5.1 # via bleach -wheel==0.33.6 -zict==1.0.0 # via distributed -zipp==0.6.0 # via importlib-metadata -zope.interface==4.6.0 # via twisted - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +Jinja2==2.11.3 +git+git://github.com/scidash/neuronunit@metacell +git+git://github.com/MetaCell/scidash-api.git@master +wheel +sentry-sdk +django-ckeditor==5.9.0 +django-admin-sortable2==0.7.5 +social-auth-app-django==4.0.0 # will be installed through install script install-backend diff --git a/scidash/account/static/css/main.css b/scidash/account/static/css/main.css index ab45a818..3bc5fe47 100644 --- a/scidash/account/static/css/main.css +++ b/scidash/account/static/css/main.css @@ -68,10 +68,9 @@ body { .login-container { position: fixed !important; - top: 50%; + top: 40%; left: 50%; transform: translate(-50%, -50%) !important; - width: 250px; } .password-reset-container { diff --git a/scidash/account/templates/registration/login.html b/scidash/account/templates/registration/login.html index 7318c296..ab31db88 100644 --- a/scidash/account/templates/registration/login.html +++ b/scidash/account/templates/registration/login.html @@ -8,25 +8,22 @@
SCIDASH LOGIN

-
-
- {% csrf_token %} - {% form form=form %}{% endform %} - - -
-
-
-
- - + + - - + + -
-
- Go home + + + + Go home
{% endblock %} diff --git a/scidash/main/settings.py b/scidash/main/settings.py index 188ad4bc..337c2016 100644 --- a/scidash/main/settings.py +++ b/scidash/main/settings.py @@ -71,6 +71,8 @@ 'django_db_logger', 'ckeditor', 'adminsortable2', + # Add the following django-allauth apps + 'social_django', ] SCIDASH_APPS = [ @@ -94,12 +96,48 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] +SOCIAL_AUTH_POSTGRES_JSONFIELD = True +SOCIAL_AUTH_URL_NAMESPACE = 'social' + +# see https://python-social-auth.readthedocs.io/en/latest/backends/index.html +# for configation of social backends + +def get_secret(secret): + sec_path = os.getenv('SECRETS_PATH','/etc/secrets') + with open(os.path.join(sec_path, secret)) as fh: + return fh.read() + +# GOOGLE +# https://python-social-auth.readthedocs.io/en/latest/backends/google.html +# see https://developers.google.com/identity/protocols/oauth2?csw=1#Registering +# to get google client id (key) and secret +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = get_secret('SOCIAL_AUTH_GOOGLE_OAUTH2_KEY') +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = get_secret('SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET') + +# TWITTER +# https://python-social-auth.readthedocs.io/en/latest/backends/twitter.html +SOCIAL_AUTH_TWITTER_KEY = get_secret('SOCIAL_AUTH_TWITTER_KEY') +SOCIAL_AUTH_TWITTER_SECRET = get_secret('SOCIAL_AUTH_TWITTER_SECRET') + +# GITHUB +# https://python-social-auth.readthedocs.io/en/latest/backends/github.html +SOCIAL_AUTH_GITHUB_KEY = get_secret('SOCIAL_AUTH_GITHUB_KEY') +SOCIAL_AUTH_GITHUB_SECRET = get_secret('SOCIAL_AUTH_GITHUB_SECRET') + +AUTHENTICATION_BACKENDS = ( + 'social_core.backends.open_id.OpenIdAuth', + 'social_core.backends.google.GoogleOAuth2', + 'social_core.backends.twitter.TwitterOAuth', + 'social_core.backends.github.GithubOAuth2', + 'django.contrib.auth.backends.ModelBackend', +) + REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. 'DEFAULT_AUTHENTICATION_CLASSES': [ 'scidash.account.auth.CsrfExemptSessionAuthentication', - 'rest_framework_jwt.authentication.JSONWebTokenAuthentication' + # 'rest_framework_jwt.authentication.JSONWebTokenAuthentication' ], 'DEFAULT_FILTER_BACKENDS': [ 'django_filters.rest_framework.DjangoFilterBackend' @@ -118,6 +156,8 @@ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', + 'social_django.context_processors.backends', + 'social_django.context_processors.login_redirect', 'django.contrib.messages.context_processors.messages', ], }, diff --git a/scidash/main/urls.py b/scidash/main/urls.py index f508b92b..11ef9151 100644 --- a/scidash/main/urls.py +++ b/scidash/main/urls.py @@ -19,7 +19,7 @@ from django.contrib.auth import views as auth_views from rest_framework.routers import DefaultRouter from rest_framework_cache.registry import cache_registry -from rest_framework_jwt.views import obtain_jwt_token +# from rest_framework_jwt.views import obtain_jwt_token from scidash.account.api.views import CheckIsLoggedView, \ UserViewSet, \ @@ -65,7 +65,8 @@ urlpatterns = [ url(r'^admin/', admin.site.urls), - url(r'^api/login/$', obtain_jwt_token), + # url(r'^api/login/$', obtain_jwt_token), + url('', include('social_django.urls', namespace='social')), url(r'^data/', include('scidash.general.urls')), url(r'^api/date-range/$', DateRangeView.as_view(), name='date-range-view'), url(r'^api/', include(router.urls)), diff --git a/service/deployment/docker-compose.yml b/service/deployment/docker-compose.yml index dc0cc0bd..d27614b1 100644 --- a/service/deployment/docker-compose.yml +++ b/service/deployment/docker-compose.yml @@ -1,13 +1,17 @@ -version: '2' +version: '3' services: scidash-redis: image: redis + ports: + - 6379:6379 expose: - 6379 scidash-postgres: image: metacell/scidash_db:latest container_name: scidash_db + ports: + - 5432:5432 expose: - 5432 volumes: @@ -17,6 +21,8 @@ services: container_name: scidash_virgo volumes: - geppettoTmp-volume:/opt/virgo/geppettoTmp + ports: + - 8080:8080 expose: - 8080 mem_reservation: 5120m @@ -33,6 +39,7 @@ services: - SYS_ADMIN volumes: - geppettoTmp-volume:/opt/virgo/geppettoTmp + - ./secrets:/etc/secrets ports: - 8000:8000 depends_on: diff --git a/service/docker/Dockerfile-scidash b/service/docker/Dockerfile-scidash index 7b656377..a677180d 100644 --- a/service/docker/Dockerfile-scidash +++ b/service/docker/Dockerfile-scidash @@ -17,11 +17,16 @@ ARG defaultBranch=development ENV SERVER_HOME $APP_DIR/virgo-tomcat-server +RUN cat /etc/passwd RUN useradd -ms /bin/bash developer ENV HOME /home/developer +RUN cat /etc/passwd WORKDIR $ROOT +RUN echo 'kernel.unprivileged_userns_clone=1' > /etc/sysctl.d/userns.conf +RUN mkdir -p /etc/secrets + # INSTALLING REQUIREMENTS RUN apt-get update RUN apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ @@ -29,7 +34,7 @@ RUN apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libc libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 \ libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 \ - lsb-release xdg-utils wget curl llvm python3-tk + lsb-release xdg-utils wget curl llvm python3-tk RUN curl -sL https://deb.nodesource.com/setup_10.x | bash RUN apt-get update && apt-get -y install nodejs RUN curl https://www.npmjs.com/install.sh | sh @@ -52,7 +57,6 @@ RUN git config --global user.email "scidash@metacell.us" RUN git config --global user.name "SciDash" WORKDIR $APP_DIR/scidash -RUN virtualenv venv-py -p python3.6 WORKDIR $APP_DIR/scidash RUN make ARGS="-b $targetBranch" install-backend-with-env @@ -67,8 +71,4 @@ RUN npm run build-dev-noTest WORKDIR $APP_DIR/scidash RUN cp ./service/dotenv/scidash_env .env -USER root -RUN echo 'kernel.unprivileged_userns_clone=1' > /etc/sysctl.d/userns.conf - -USER developer CMD ./service/scripts/run.sh diff --git a/service/k8s/scidash.yaml b/service/k8s/scidash.yaml index ee569690..50be73d3 100644 --- a/service/k8s/scidash.yaml +++ b/service/k8s/scidash.yaml @@ -1,4 +1,17 @@ apiVersion: v1 +kind: Secret +metadata: + name: scidash +type: Opaque +stringData: + SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: key + SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET: secret + SOCIAL_AUTH_TWITTER_KEY: key + SOCIAL_AUTH_TWITTER_SECRET: secret + SOCIAL_AUTH_GITHUB_KEY: key + SOCIAL_AUTH_GITHUB_SECRET: secret +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: scidash-geppettotmp @@ -42,6 +55,9 @@ spec: volumeMounts: - name: scidash-geppettotmp mountPath: /opt/virgo/geppettoTmp + - name: secrets + mountPath: "/etc/secrets" + readOnly: true affinity: podAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -56,6 +72,9 @@ spec: - name: scidash-geppettotmp persistentVolumeClaim: claimName: scidash-geppettotmp + - name: secrets + secret: + secretName: scidash --- apiVersion: v1 kind: Service diff --git a/service/scripts/install-backend.sh b/service/scripts/install-backend.sh index 1c77f26b..ad9f051f 100755 --- a/service/scripts/install-backend.sh +++ b/service/scripts/install-backend.sh @@ -29,14 +29,14 @@ done set -- "${POSITIONAL[@]}" # restore positional parameters if [ "$virtualenv" = true ] ; then - virtualenv -p python3.6 ./venv; - source ./venv/bin/activate; + python -m venv venv + source ./venv/bin/activate fi -python -m pip install pip==9.0.3 -pip install -r requirements.txt; -# pip uninstall -y sciunit -# pip install git+git://github.com/ddelpiano/sciunit@4.0.0#egg=sciunit +# python -m pip install pip==9.0.3 +python3 -m pip install --upgrade pip +pip install Jinja2==2.11.3 --no-cache-dir +pip install -r requirements.txt --no-cache-dir git ls-remote --heads --tags $pygeppetto_django_repo | grep -E 'refs/(heads|tags)/'$pygeppetto_branch > /dev/null