diff --git a/.env_example b/.env_example index ebe7f699..653025ea 100644 --- a/.env_example +++ b/.env_example @@ -10,11 +10,11 @@ CONSUMER_SECRET=secret WEB_PORT=8080 REDIS_URL=redis://redis:6379/0 -CELERY_QUERIES=query +CELERY_QUERIES=check-solution FLOWER_SECRET_KEY=123 FLOWER_PORT=5555 FLOWER_AUTH=user:user FLOWER_PREFIX=/ -MONGODB_CACHE_SIZE=1 \ No newline at end of file +MONGODB_CACHE_SIZE=1 diff --git a/.github/workflows/selenium_tests.yml b/.github/workflows/selenium_tests.yml new file mode 100644 index 00000000..a31e1253 --- /dev/null +++ b/.github/workflows/selenium_tests.yml @@ -0,0 +1,45 @@ +name: Selenium_tests + +on: [push] + +jobs: + check_tests: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Build docker-compose with docker-compose-selenium (tests) + run: | + cp .env_example .env + cp app/VERSION_example.json app/VERSION.json + docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build + + - name: Run docker-compose with docker-compose-selenium (tests) + run: | + docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d + chmod +x tests/scripts/docker_check_tests.sh + ./tests/scripts/docker_check_tests.sh + + +# jobs: +# check_tests: +# runs-on: ubuntu-20.04 + +# steps: +# - uses: actions/checkout@v2 + +# - name: Simplify docker-compose file name +# run: mv docker-compose-tests.yml docker-compose.yml + +# - name: Build system images (non-pulling) +# run: | +# # build base image +# docker build -f Dockerfile_base -t osll/slides-base:20230202 . + +# - name: Run docker-compose tests +# run: | +# cp .env_example .env +# docker-compose up -d +# chmod +x tests/scripts/docker_check_tests.sh +# ./tests/scripts/docker_check_tests.sh diff --git a/Dockerfile_selenium b/Dockerfile_selenium new file mode 100644 index 00000000..d9b03e09 --- /dev/null +++ b/Dockerfile_selenium @@ -0,0 +1,17 @@ +FROM selenium/standalone-chrome:121.0-chromedriver-121.0-grid-4.18.0-20240220 + +ENV LOGIN="" +ENV PASSWORD="" + +WORKDIR /usr/src/project + +USER root +RUN apt-get update && \ + apt-get install -y python3 python3-pip && \ + rm -rf /var/lib/apt/lists/* + +COPY tests/requirements.txt tests/requirements.txt +RUN pip install -r tests/requirements.txt +COPY tests ./tests + +ENTRYPOINT python3 tests/main.py --login ${LOGIN} --password ${PASSWORD} diff --git a/app/VERSION_example.json b/app/VERSION_example.json new file mode 100644 index 00000000..af2c707d --- /dev/null +++ b/app/VERSION_example.json @@ -0,0 +1,6 @@ +{ +"commit": "", +"message": "", +"date": "", +"version": "" +} diff --git a/app/main/checks/report_checks/needed_headers_check.py b/app/main/checks/report_checks/needed_headers_check.py index cc9621ad..b481f908 100644 --- a/app/main/checks/report_checks/needed_headers_check.py +++ b/app/main/checks/report_checks/needed_headers_check.py @@ -23,8 +23,8 @@ def late_init(self): self.headers = self.file.make_chapters(self.file_type['report_type']) self.headers_page = self.file.find_header_page(self.file_type['report_type']) self.chapters_str = self.file.show_chapters(self.file_type['report_type']) - self.headers_main = self.file.get_main_headers() # TODO: change + self.headers_main = self.file.get_main_headers(self.file_type['report_type']) if self.headers_main == "Задание 1": self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"] elif self.headers_main == "Задание 2": diff --git a/app/main/reports/document_uploader.py b/app/main/reports/document_uploader.py index 31cae722..d0653fae 100644 --- a/app/main/reports/document_uploader.py +++ b/app/main/reports/document_uploader.py @@ -48,3 +48,6 @@ def find_literature_page(self, work_type): @abstractmethod def show_chapters(self, work_type): pass + + def get_main_headers(self): + pass \ No newline at end of file diff --git a/app/main/reports/docx_uploader/docx_uploader.py b/app/main/reports/docx_uploader/docx_uploader.py index 8660fd04..18d901b1 100644 --- a/app/main/reports/docx_uploader/docx_uploader.py +++ b/app/main/reports/docx_uploader/docx_uploader.py @@ -18,6 +18,7 @@ def __init__(self): self.inline_shapes = [] self.core_properties = None self.headers = [] + self.headers_main = '' self.file = None self.special_paragraph_indices = {} self.headers_page = 0 @@ -90,6 +91,12 @@ def make_headers(self, work_type): break self.headers = headers return self.headers + + def get_main_headers(self, work_type): #this method helps to avoid mistake in "needed_headers_check" (because of structure checks for md) + if not self.headers_main: + if work_type == 'VKR': + self.headers_main = self.make_headers(work_type)[1]['name'] + return self.headers_main def __make_table(self, tables): for i in range(len(tables)): diff --git a/app/main/reports/md_uploader/md_uploader.py b/app/main/reports/md_uploader/md_uploader.py index cbf63710..ce55fb44 100644 --- a/app/main/reports/md_uploader/md_uploader.py +++ b/app/main/reports/md_uploader/md_uploader.py @@ -95,7 +95,7 @@ def page_counter(self): # we need this just to find a last page and make link to self.page_count = 5 return self.page_count - def get_main_headers(self): + def get_main_headers(self, work_type): if not self.headers_main: header_main_regex = "

(.*?)<\/h1>" self.headers_main = re.findall(header_main_regex, self.html_text)[0] diff --git a/app/nlp/stemming.py b/app/nlp/stemming.py index 0fce45e2..784f3fe9 100644 --- a/app/nlp/stemming.py +++ b/app/nlp/stemming.py @@ -1,12 +1,8 @@ import itertools - -import nltk from nltk.corpus import stopwords from nltk.tokenize import word_tokenize, sent_tokenize from pymorphy2 import MorphAnalyzer -nltk.download('stopwords') -nltk.download('punkt') MORPH_ANALYZER = MorphAnalyzer() TASKS = 'задачи:' diff --git a/app/tasks.py b/app/tasks.py index 1bee7882..095a3870 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -1,10 +1,11 @@ -import configparser +from configparser import ConfigParser import os from os.path import join, exists from celery import Celery +from celery.signals import worker_ready -import passback_grades +from passback_grades import run_passback from db import db_methods from db.db_types import Check from main.checker import check @@ -12,7 +13,7 @@ from main.check_packs import BASE_PACKS from root_logger import get_root_logger -config = configparser.ConfigParser() +config = ConfigParser() config.read('app/config.ini') TASK_RETRY_COUNTDOWN = 60 # default = 3 * 60 @@ -33,6 +34,16 @@ celery.conf.timezone = 'Europe/Moscow' # todo: get from env +@worker_ready.connect +def at_start(sender, **k): + from nltk import download + download('stopwords') + download('punkt') + + from language_tool_python.download_lt import download_lt + download_lt() + + @celery.task(name="create_task", queue='check-solution', bind=True) def create_task(self, check_info): check_obj = Check(check_info) @@ -67,12 +78,11 @@ def create_task(self, check_info): self.retry(countdown=TASK_RETRY_COUNTDOWN) # Retry the task, adding it to the back of the queue. +@celery.task(name="passback-task", queue='passback-grade') +def passback_task(): + return run_passback() + + def remove_files(filepaths): for filepath in filepaths: if exists(filepath): os.remove(filepath) - - -@celery.task(name="passback-task", queue='passback-grade') -def passback_task(): - print('Run passback') - return passback_grades.run_passback() diff --git a/app/templates/criteria_pack.html b/app/templates/criteria_pack.html index b0c312a7..e923e565 100644 --- a/app/templates/criteria_pack.html +++ b/app/templates/criteria_pack.html @@ -46,7 +46,7 @@