diff --git a/.circleci/config.yml b/.circleci/config.yml index 98a022bbf..53fe0079c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: steps: - checkout - restore_cache: - key: codalab-{{ .Branch }}-{{ checksum "codalab/requirements/common.txt" }} + key: codalab-{{ .Branch }}-{{ checksum "codalab/requirements/requirements.txt" }} - run: name: CHOWN Python Library Dirs command: sudo chown -R $(whoami) /usr/local/ @@ -23,9 +23,9 @@ jobs: command: sudo apt-get update --allow-releaseinfo-change && sudo apt-get install libmemcached-dev --fix-missing - run: name: PIP Install Requirements - command: pip install -r codalab/requirements/common.txt + command: pip install -r codalab/requirements/requirements.txt - save_cache: - key: codalab-{{ .Branch }}-{{ checksum "codalab/requirements/common.txt" }} + key: codalab-{{ .Branch }}-{{ checksum "codalab/requirements/requirements.txt" }} paths: - "~/.cache/pip" - run: diff --git a/.env_production_sample b/.env_production_sample deleted file mode 100644 index b53ecbfac..000000000 --- a/.env_production_sample +++ /dev/null @@ -1,140 +0,0 @@ -# ---------------------------------------------------------------------------- -# Submission processing -# ---------------------------------------------------------------------------- -SUBMISSION_TEMP_DIR=/tmp/codalab - - -# ---------------------------------------------------------------------------- -# Storage -# ---------------------------------------------------------------------------- - -# Uncomment to use AWS -DEFAULT_FILE_STORAGE=storages.backends.s3boto3.S3Boto3Storage -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_STORAGE_BUCKET_NAME=public -AWS_STORAGE_PRIVATE_BUCKET_NAME=private -AWS_S3_CALLING_FORMAT=boto.s3.connection.OrdinaryCallingFormat -AWS_S3_HOST=s3.amazonaws.com -AWS_QUERYSTRING_AUTH=False -S3DIRECT_REGION=us-west-2 -# ^^Set the S3DIRECT_REGION to the AWS region of your storage buckets - -# Uncomment to use Minio -#DEFAULT_FILE_STORAGE=storages.backends.s3boto3.S3Boto3Storage -#AWS_ACCESS_KEY_ID= -#AWS_SECRET_ACCESS_KEY= -#AWS_STORAGE_BUCKET_NAME=public -#AWS_STORAGE_PRIVATE_BUCKET_NAME=private -#AWS_S3_CALLING_FORMAT=boto.s3.connection.OrdinaryCallingFormat -#AWS_S3_HOST=minio.yoursite.org -#AWS_QUERYSTRING_AUTH=False -#S3DIRECT_REGION=us-east-1 -#S3_USE_SIGV4=True - -# Uncomment to use Azure -#DEFAULT_FILE_STORAGE=codalab.azure_storage.AzureStorage -#AZURE_ACCOUNT_NAME= -#AZURE_ACCOUNT_KEY= -#AZURE_CONTAINER=public -# Only set these if bundle storage key is different from normal account keys -# BUNDLE_AZURE_ACCOUNT_NAME= -# BUNDLE_AZURE_ACCOUNT_KEY= -BUNDLE_AZURE_CONTAINER=bundles - -# ---------------------------------------------------------------------------- -# Database -# ---------------------------------------------------------------------------- - -# Used engine (mysql, postgresql, sqlite3, memory) -DB_ENGINE=postgresql - -# Connection parameters -DB_HOST=postgres -DB_PORT=5432 -DB_NAME=codalab_website -DB_USER=root -DB_PASSWORD=password - -# Path where DB files will be mapped -DB_DATA_PATH=./var/data/postgres - - -# ---------------------------------------------------------------------------- -# Caching -# ---------------------------------------------------------------------------- -MEMCACHED_PORT=11211 - - -# ---------------------------------------------------------------------------- -# RabbitMQ and management -# ---------------------------------------------------------------------------- -BROKER_URL=pyamqp://guest:guest@rabbit:5671// -BROKER_USE_SSL=True -RABBITMQ_DEFAULT_USER=guest -RABBITMQ_DEFAULT_PASS=guest -RABBITMQ_HOST=rabbit -RABBITMQ_PORT=5671 -RABBITMQ_MANAGEMENT_PORT=15672 -FLOWER_BASIC_AUTH=root:password -FLOWER_PORT=5555 - - -# ========================================================================= -# Email -# ========================================================================= -EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend -EMAIL_HOST=smtp.sendgrid.net -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= -EMAIL_PORT=587 -EMAIL_USE_TLS=True -DEFAULT_FROM_EMAIL=CodaLab -SERVER_EMAIL=noreply@codalab.org - - -# ---------------------------------------------------------------------------- -# Django/nginx -# ---------------------------------------------------------------------------- -DJANGO_SECRET_KEY=change-me-to-a-secret -DJANGO_PORT=8000 -# Make sure debug is off on production -#DEBUG= -# These admins will be emailed when there are errors -ADMINS=Name,you@example.com;OtherName,other@example.com - -NGINX_PORT=80 -SSL_PORT=443 - -# Put SSL certificates in ./certs/ and they are mapped to /app/certs in the container -SSL_CERTIFICATE=/app/certs/certificate.crt -SSL_CERTIFICATE_KEY=/app/certs/certificate.pem -# Allowed hosts separated by space -SSL_ALLOWED_HOSTS=yourhost.com - - -# Set this to your actual domain, like example.com -CODALAB_SITE_DOMAIN=localhost - -# How many site workers (submission result processors)? -WEB_CONCURRENCY=16 - -# Google Analytics code (leave empty to disable Google's user tracking) -GOOGLE_ANALYTICS= - - -# ---------------------------------------------------------------------------- -# ChaHub -# ---------------------------------------------------------------------------- -# Be sure to include trailing slash on URL -#CHAHUB_API_URL=https://chahub.org/api/v1/ -#CHAHUB_API_KEY=some-secret-key -#SOCIAL_AUTH_CHAHUB_BASE_URL=https://chahub.org - - -# ---------------------------------------------------------------------------- -# Logging -# ---------------------------------------------------------------------------- -# Make sure LOGGING_DIR doesn't end with a slash -LOGGING_DIR=./var/logs -DJANGO_LOG_LEVEL=info diff --git a/.env_sample b/.env_sample index 0457accc0..8c377ec53 100644 --- a/.env_sample +++ b/.env_sample @@ -103,24 +103,43 @@ FLOWER_PORT=5555 # ---------------------------------------------------------------------------- DJANGO_SECRET_KEY=change-me-to-a-secret DJANGO_PORT=8000 -DEBUG=True NGINX_PORT=80 - SSL_PORT=443 -#SSL_CERTIFICATE= -#SSL_CERTIFICATE_KEY= -# Allowed hosts separated by space + +# Make sure debug is off on production +DEBUG=True +# These admins will be emailed when there are errors +#ADMINS=Name,you@example.com;OtherName,other@example.com + +# Put SSL certificates in ./certs/ and they are mapped to /app/certs in the container +#SSL_CERTIFICATE=/app/certs/certificate.crt +#SSL_CERTIFICATE_KEY=/app/certs/certificate.pem +# Allowed hosts separated by space (e.g. example.com) SSL_ALLOWED_HOSTS= # Set this to your actual domain, like example.com CODALAB_SITE_DOMAIN=localhost # How many site workers (submission result processors)? +# A higher value, e.g. 16, is advised for a production environment WEB_CONCURRENCY=2 # Google Analytics code (leave empty to disable Google's user tracking) GOOGLE_ANALYTICS= +# ========================================================================= +# Email +# ========================================================================= +# This section can be left commented for a testing environment +#EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +#EMAIL_HOST=smtp.sendgrid.net +#EMAIL_HOST_USER= +#EMAIL_HOST_PASSWORD= +#EMAIL_PORT=587 +#EMAIL_USE_TLS=True +#DEFAULT_FROM_EMAIL=CodaLab +#SERVER_EMAIL=noreply@codalab.org + # ---------------------------------------------------------------------------- # Logging # ---------------------------------------------------------------------------- diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..872ba94e9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/codalab/requirements" + schedule: + interval: "daily" diff --git a/.github/workflows/run-pytest.yaml b/.github/workflows/run-pytest.yaml index e69de29bb..e63fb19ab 100644 --- a/.github/workflows/run-pytest.yaml +++ b/.github/workflows/run-pytest.yaml @@ -0,0 +1,83 @@ +name: Run Pytests +on: + push: + branches: + - master + - develop + pull_request: +jobs: + run-tests: + services: + # Label used to access the service container + postgres: + # Docker Hub image + # use the same version as docker-compose.yml + image: "postgres:12.6-alpine" + # Provide the password for postgres + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: password + POSTGRES_USER: root + POSTGRES_DB: codalab_website + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + memcached: + image: memcached + ports: + - 11211:11211 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8.3 + - name: Cache pip + uses: actions/cache@v2 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('common.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: Add required host mappings + run: | + echo "127.0.0.1 memcached" | sudo tee --append /etc/hosts + - name: Install dependencies + run: | + sudo apt-get install libxml2-dev libxslt-dev python-dev + sudo apt-get update --allow-releaseinfo-change && sudo apt-get install libmemcached-dev --fix-missing + python -m pip install --upgrade pip + pip install flake8 pytest pytest-cov pytest-dependency pytest-django + pip install -r ./codalab/requirements/requirements.txt + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --show-source + - name: Run Tests + env: + DB_ENGINE: "postgresql" + DB_HOST: 127.0.0.1 + DB_PORT: "5432" + DB_NAME: "codalab_website" + DB_USER: "root" + DB_PASSWORD: "password" + CHAHUB_API_URL: "http://localhost/test/" + CHAHUB_API_KEY: "some-secret-key" + MEMCACHED_PORT: 11211 + working-directory: ./codalab + run: | + #pytest --cov=./ --cov-report=xml + py.test --cov=./ --cov-report=xml + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v3 + diff --git a/Dockerfile b/Dockerfile index ae0bd2b73..5f87d7fc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,6 @@ RUN curl -sL https://deb.nodesource.com/setup_4.x | bash - RUN apt-get update && apt-get install -y npm netcat nodejs python-dev libmemcached-dev RUN pip install --upgrade pip # make things faster, hopefully - -COPY codalab/requirements/common.txt requirements.txt -RUN pip install -r requirements.txt +RUN pip install -r codalab/requirements/requirements.txt WORKDIR /app/codalab diff --git a/README.md b/README.md index 463726446..0ee3dad9b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ ![CodaLab logo](codalab/apps/web/static/img/codalab-logo-fullcolor-positive.png) [![Circle CI](https://circleci.com/gh/codalab/codalab-competitions.svg?style=shield)](https://circleci.com/gh/codalab/codalab-competitions) +[![codecov](https://codecov.io/gh/codalab/codalab-competitions/branch/develop/graph/badge.svg)](https://codecov.io/gh/codalab/codalab-competitions) @@ -59,14 +60,8 @@ http://www.opensource.org/licenses/apache2.0.php ``` @article{codalab_competitions, - author = {Adrien Pavao and - Isabelle Guyon and - Anne-Catherine Letournel and - Xavier Baró and - Hugo Escalante and - Sergio Escalera and - Tyler Thomas and - Zhen Xu}, + author = {Adrien Pavao and Isabelle Guyon and Anne-Catherine Letournel and Xavier Baró and + Hugo Escalante and Sergio Escalera and Tyler Thomas and Zhen Xu}, title = {CodaLab Competitions: An open source platform to organize scientific challenges}, url = {https://hal.inria.fr/hal-03629462v1}, year = {2022}, diff --git a/codalab/apps/api/routers.py b/codalab/apps/api/routers.py index 71e02ed82..77414e255 100644 --- a/codalab/apps/api/routers.py +++ b/codalab/apps/api/routers.py @@ -1,5 +1,6 @@ from apps.api.views import competition_views as views from apps.api.views import storage_views as storage_views +from apps.api.views import admin_views as admin_views from django.conf.urls import url from rest_framework import routers @@ -40,4 +41,9 @@ # Storage Analytics url(r'^storage/analytics', storage_views.GetExistingStorageAnalytics.as_view(), name="existing_storage_analytics"), url(r'^storage/usage-history', storage_views.GetStorageUsageHistory.as_view(), name="storage_usage_history"), + # Admin + url(r'^admin/competitions/list', admin_views.GetCompetitions.as_view(), name="competitions"), + url(r'^admin/competitions/update', admin_views.UpdateCompetitions.as_view(), name="update_competitions"), + url(r'^admin/competition/(?P\d+)/apply_upper_bound_limit', admin_views.ApplyUpperBoundLimit.as_view(), name="apply_upper_bound_limit"), + url(r'^admin/competitions/default_upper_bound_limit', admin_views.GetDefaultUpperBoundLimit.as_view(), name="get_default_upper_bound_limit") ) diff --git a/codalab/apps/api/views/admin_views.py b/codalab/apps/api/views/admin_views.py new file mode 100644 index 000000000..e29e9be39 --- /dev/null +++ b/codalab/apps/api/views/admin_views.py @@ -0,0 +1,138 @@ +from rest_framework import (permissions, status, views) +from rest_framework.decorators import permission_classes +from rest_framework.exceptions import PermissionDenied +from rest_framework.response import Response + +import logging + +from django.db import transaction +from django.conf import settings + +from apps.web.models import Competition, CompetitionPhase + +logger = logging.getLogger(__name__) + + +@permission_classes((permissions.IsAuthenticated,)) +class GetCompetitions(views.APIView): + """ + Gets the competitions + """ + def get(self, request, *args, **kwargs): + if not self.request.user.is_staff: + raise PermissionDenied(detail="Admin only") + + competitions = [] + for competition in list(Competition.objects.order_by('id').all()): + competitions.append({ + 'id': competition.id, + 'title': competition.title, + 'creator': competition.creator.username + " (" + competition.creator.email + ")", + 'start_date': competition.start_date, + 'end_date': competition.end_date, + 'upper_bound_max_submission_size': competition.upper_bound_max_submission_size, + 'max_submission_sizes': [phase.max_submission_size for phase in competition.phases.all().order_by('start_date')] + }) + + return Response(competitions, status=status.HTTP_200_OK) + + +@permission_classes((permissions.IsAuthenticated,)) +class UpdateCompetitions(views.APIView): + """ + Update competitions in batch + body template: + { + competitions: [ + { + id: 0, + : + } + ] + } + """ + def post(self, request, *args, **kwargs): + if not self.request.user.is_staff: + raise PermissionDenied(detail="Admin only") + + competitions_to_update = request.data['competitions'] + if not competitions_to_update: + return Response("No competitions to update", status=status.HTTP_204_NO_CONTENT) + + with transaction.atomic(): + for competition in competitions_to_update: + try: + competition_in_db = Competition.objects.select_for_update().get(pk=competition['id']) + for attribute, value in competition.items(): + if attribute != 'id': + logger.debug("Updating competition %d", competition_in_db.id) + setattr(competition_in_db, attribute, value) + competition_in_db.save() + except Competition.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + ids = [comp['id'] for comp in competitions_to_update] + competitions_updated = [] + for comp_id, competition in Competition.objects.in_bulk(ids).items(): + competitions_updated.append({ + 'id': comp_id, + 'title': competition.title, + 'creator': competition.creator.username, + 'start_date': competition.start_date, + 'end_date': competition.end_date, + 'upper_bound_max_submission_size': competition.upper_bound_max_submission_size, + 'max_submission_sizes': [phase.max_submission_size for phase in competition.phases.all().order_by('start_date')] + }) + + return Response(competitions_updated, status=status.HTTP_200_OK) + + +@permission_classes((permissions.IsAuthenticated,)) +class ApplyUpperBoundLimit(views.APIView): + """ + Update the max submission size for all phases of the tompetition + """ + def patch(self, request, *args, **kwargs): + if not self.request.user.is_staff: + raise PermissionDenied(detail="Admin only") + + competition_id = self.kwargs.get('competition_id') + try: + competition = Competition.objects.get(pk=competition_id) + except Competition.DoesNotExist: + return Response("Competition not found or is not accessible", status=status.HTTP_404_NOT_FOUND) + + with transaction.atomic(): + for phase in competition.phases.all().order_by('start_date'): + try: + phase_in_db = CompetitionPhase.objects.select_for_update().get(pk=phase.id) + phase_in_db.max_submission_size = competition.upper_bound_max_submission_size + phase_in_db.save() + except CompetitionPhase.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + competition_updated_in_db = Competition.objects.get(pk=competition_id) + competition_updated = { + 'id': competition_updated_in_db.id, + 'title': competition_updated_in_db.title, + 'creator': competition_updated_in_db.creator.username, + 'start_date': competition_updated_in_db.start_date, + 'end_date': competition_updated_in_db.end_date, + 'upper_bound_max_submission_size': competition_updated_in_db.upper_bound_max_submission_size, + 'max_submission_sizes': [phase.max_submission_size for phase in competition_updated_in_db.phases.all().order_by('start_date')] + } + return Response(competition_updated, status=status.HTTP_200_OK) + + +@permission_classes((permissions.IsAuthenticated,)) +class GetDefaultUpperBoundLimit(views.APIView): + """ + Gets the default upper bound max submission size + """ + def get(self, request, *args, **kwargs): + if not self.request.user.is_staff: + raise PermissionDenied(detail="Admin only") + + default_upper_bound_max_submission_size = settings.DEFAULT_UPPER_BOUND_MAX_SUBMISSION_SIZE_MB + + return Response(default_upper_bound_max_submission_size, status=status.HTTP_200_OK) diff --git a/codalab/apps/api/views/competition_views.py b/codalab/apps/api/views/competition_views.py index 34d336574..bea82c7ba 100644 --- a/codalab/apps/api/views/competition_views.py +++ b/codalab/apps/api/views/competition_views.py @@ -42,7 +42,7 @@ class CompetitionCreatorAdminPermission(BasePermission): def has_object_permission(self, request, view, obj): if request.user.id is obj.creator.id or request.user.id in obj.admins.all().values_list('id', flat=True): return True - if request.user.id in obj.participants.all().values_list('id', flat=True) and reqest.method in SAFE_METHODS: + if request.user.id in obj.participants.all().values_list('id', flat=True) and request.method in SAFE_METHODS: return True diff --git a/codalab/apps/health/templates/health/storage.html b/codalab/apps/health/templates/health/storage.html index eebcde654..fd193a9d1 100644 --- a/codalab/apps/health/templates/health/storage.html +++ b/codalab/apps/health/templates/health/storage.html @@ -164,7 +164,9 @@

User distribution

{% block extra_scripts %} - + diff --git a/codalab/apps/web/models.py b/codalab/apps/web/models.py index 19e0ae6e5..d19f39db8 100644 --- a/codalab/apps/web/models.py +++ b/codalab/apps/web/models.py @@ -288,7 +288,7 @@ class Competition(ChaHubSaveMixin, models.Model): hide_chart = models.BooleanField(default=False, verbose_name="Hide Chart") allow_organizer_teams = models.BooleanField(default=False, verbose_name="Allow Organizer Teams") upper_bound_max_submission_size = models.PositiveIntegerField( - default=300, + default=settings.DEFAULT_UPPER_BOUND_MAX_SUBMISSION_SIZE_MB, validators=[ MinValueValidator(0) ], @@ -899,7 +899,7 @@ class CompetitionPhase(models.Model): color = models.CharField(max_length=24, choices=COLOR_CHOICES, blank=True, null=True) max_submission_size = models.PositiveIntegerField( - default=300, + default=settings.DEFAULT_UPPER_BOUND_MAX_SUBMISSION_SIZE_MB, validators=[ MinValueValidator(0), ], diff --git a/codalab/apps/web/static/css/admin_competitions_manager.css b/codalab/apps/web/static/css/admin_competitions_manager.css new file mode 100755 index 000000000..a5dc00235 --- /dev/null +++ b/codalab/apps/web/static/css/admin_competitions_manager.css @@ -0,0 +1,15 @@ +.admin-competitions-manager-app-container { + display: flex; + flex-direction: column; + width: 100%; + height: 100%; + background-color: #f7f7f7; +} + +.admin-competitions-manager-footer-row { + display: flex; + flex-direction: row; + width: 100%; + justify-content: flex-end; + margin: 24px 0px; +} diff --git a/codalab/apps/web/templates/base.html b/codalab/apps/web/templates/base.html index 322d13c58..97c518d87 100644 --- a/codalab/apps/web/templates/base.html +++ b/codalab/apps/web/templates/base.html @@ -28,9 +28,13 @@ {% else %} - + {% endif %} - + diff --git a/codalab/apps/web/templates/web/admin_competitions_manager.html b/codalab/apps/web/templates/web/admin_competitions_manager.html index 912015cbc..490688e39 100644 --- a/codalab/apps/web/templates/web/admin_competitions_manager.html +++ b/codalab/apps/web/templates/web/admin_competitions_manager.html @@ -4,154 +4,336 @@ {% load codalab_tags %} {% block page_title %}Admin Competitions Manager{% endblock page_title %} -{% block head_title %}Admin Competitions Manager{% endblock %} -{% block content %} -
-
-
- {% if formset %} - - - - - - - - - - - {{ formset.management_form }} - {% for form in formset %} - {% csrf_token %} - {% for hidden in form.hidden_fields %} - {{ hidden }} - {% endfor %} - - - - - - - - - - {% endfor %} -
IDTitleOrganizerStart dateEnd dateUpper bound of the max submission size (in MB)
- - {{ form.id.value }}{{ form.title }}{{ form.creator }}{{ form.instance.start_date|date:"M d, Y" }} - {% if form.instance.end_date %} - {{ form.instance.end_date|date:"M d, Y" }} - {% else %} - No end date - {% endif %} - {{ form.upper_bound_max_submission_size }}
- - {% else %} -

There are no competitions.

- {% endif %} -
-
-
- - + + + - window.addEventListener('load', function () { - selectAllCheckbox = document.querySelector('input[id="select-all-checkbox"]'); - competitionCheckboxes = document.querySelectorAll('td input[type="checkbox"]'); - competitionUpperBoundMaxSubmissionSizeInputs = document.querySelectorAll('td input[id$="upper_bound_max_submission_size"]'); + {% csrf_token %} + - - + }) + {% endblock %} diff --git a/codalab/apps/web/templates/web/highlights.html b/codalab/apps/web/templates/web/highlights.html index ad97426a1..f5faf34ae 100644 --- a/codalab/apps/web/templates/web/highlights.html +++ b/codalab/apps/web/templates/web/highlights.html @@ -9,9 +9,12 @@ Codalab - - + +