466 criteria pack for md test

app/main/check_packs/pack_config.py

@@ -22,14 +22,13 @@
     ["banned_words_in_literature"],
     ["page_counter"],
     ["image_share_check"],
-    # ["headers_at_page_top_check", {"headers": ["Приложение А Исходный код программы"]}],
+    ["headers_at_page_top_check", {"headers": ["Приложение А Исходный код программы"]}],
     ["headers_at_page_top_check"],
     ["lr_sections_check"],
     ["style_check"],
     ["short_sections_check"],
     ["banned_words_check"],
     ["right_words_check"],
-    ["banned_words_in_literature"],
     ["literature_references"],
     ["image_references"],
     ["table_references"],

app/main/checks/report_checks/banned_words_in_literature.py

@@ -13,10 +13,12 @@ def __init__(self, file_info, banned_words=["wikipedia"]):
         self.literature_header = []
         self.banned_words = [morph.normal_forms(word)[0] for word in banned_words]
         self.name_pattern = r'список[ \t]*(использованных|использованной|)[ \t]*(источников|литературы)'
+        self.md_name_pattern = r'<h2>список[ \t]*(использованных|использованной|)[ \t]*(источников|литературы)</h2>'
 
     def late_init_vkr(self):
         self.literature_header = self.file.find_literature_vkr(self.file_type['report_type'])
-        self.headers_page = self.file.find_header_page(self.file_type['report_type'])
+        # self.headers_page = self.file.find_header_page(self.file_type['report_type'])
+        self.lit_page = self.file.find_literature_page(self.file_type['report_type'])
 
     def check(self):
         if self.file.page_counter() < 4:
@@ -52,7 +54,7 @@ def check(self):
             for i in sorted(detected_words_dict.keys()):
                 result_str += f"Абзац {i}: {detected_words_dict[i]}.<br>"
             return answer(False, f'Есть запрещенные слова в списке источников '
-                                 f'{self.format_page_link([self.headers_page])}:<br><br>{result_str}')
+                                 f'{self.format_page_link([self.lit_page])}:<br><br>{result_str}')
         return answer(True, f"Пройдена!")
 
     def find_banned_words(self, list_of_literature):
@@ -83,6 +85,6 @@ def start_of_literature_chapter(self, ):
         start_index = 0
         for i in range(len(self.file.paragraphs)):
             text_string = self.file.paragraphs[i].to_string().lower().split('\n')[1]
-            if re.fullmatch(self.name_pattern, text_string):
+            if re.fullmatch(f'{self.name_pattern}|{self.md_name_pattern}', text_string):
                 start_index = i
         return start_index

app/main/checks/report_checks/image_references.py

@@ -63,7 +63,10 @@ def check(self):
     def search_references(self):
         array_of_references = set()
         for i in range(0, self.last_child_number):
-            detected_references = re.findall(r'[Рр]ис\. [\d .,]+', self.file.paragraphs[i].paragraph_text)
+            if isinstance(self.file.paragraphs[i], str):
+                detected_references = re.findall(r'[Рр]ис\. [\d .,]+', self.file.paragraphs[i])
+            else:    
+                detected_references = re.findall(r'[Рр]ис\. [\d .,]+', self.file.paragraphs[i].paragraph_text)
             if detected_references:
                 for reference in detected_references:
                     for one_part in re.split(r'[Рр]ис\.|,| ', reference):

app/main/checks/report_checks/literature_references.py

@@ -12,6 +12,7 @@ def __init__(self, file_info, min_ref=1, max_ref=1000):
         self.headers = []
         self.literature_header = []
         self.name_pattern = r'список[ \t]*(использованных|использованной|)[ \t]*(источников|литературы)'
+        self.md_name_pattern = r"<h2>(Список использованных источников|Список использованной литературы)<\/h2>"
         self.min_ref = min_ref
         self.max_ref = max_ref
 
@@ -77,7 +78,10 @@ def check(self):
     def search_references(self, start_par):
         array_of_references = set()
         for i in range(0, start_par):
-            detected_references = re.findall(r'\[[\d \-,]+\]', self.file.paragraphs[i].to_string().split('\n')[1])
+            if isinstance(self.file.paragraphs[i], str):
+                detected_references = re.findall(r'\[[\d \-,]+\]', self.file.paragraphs[i])
+            else:    
+                detected_references = re.findall(r'\[[\d \-,]+\]', self.file.paragraphs[i].to_string().split('\n')[1])
             if detected_references:
                 for reference in detected_references:
                     for one_part in re.split(r'[\[\],]', reference):
@@ -86,16 +90,22 @@ def search_references(self, start_par):
                             for k in range(int(start), int(end) + 1):
                                 array_of_references.add(k)
                         elif one_part != '':
-                            array_of_references.add(int(one_part))
+                            array_of_references.add(int(one_part))               
         return array_of_references
 
     def find_start_paragraph(self):
         start_index = 0
         for i in range(len(self.file.paragraphs)):
-            text_string = self.file.paragraphs[i].to_string().lower().split('\n')[1]
-            if re.fullmatch(self.name_pattern, text_string):
-                start_index = i
-                break
+            if isinstance(self.file.paragraphs[i], str):
+                text_string = self.file.paragraphs[i].lower()
+                if re.fullmatch(self.md_name_pattern, text_string):
+                    start_index = i
+                    break
+            else:    
+                text_string = self.file.paragraphs[i].to_string().lower().split('\n')[1]
+                if re.fullmatch(self.name_pattern, text_string):
+                    start_index = i
+                    break
         return start_index
 
     def count_sources_vkr(self, header):
@@ -142,4 +152,4 @@ def search_literature_start_pdf(self):
             if re.search('приложение а[\n .]', lowercase_str):
                 end_page = i
                 break
-        return start_page, end_page
+        return start_page, end_page

app/main/checks/report_checks/section_component.py

@@ -2,10 +2,10 @@
 
 
 class ReportSectionComponent(BaseReportCriterion):
-    description = "Проверка наличия необходимых компонент указанного раздела"
+    description = "Проверка наличия необходимых компонентов указанного раздела"
     id = 'report_section_component'
 
-    def __init__(self, file_info, chapter='Введение', patterns=('цель', 'задачи', 'объект', 'предмет')):
+    def __init__(self, file_info, chapter='Введение', patterns=('цель', 'задач', 'объект', 'предмет')):
         super().__init__(file_info)
         self.intro = {}
         self.chapter = chapter

app/main/checks/report_checks/table_references.py

@@ -63,7 +63,10 @@ def check(self):
     def search_references(self):
         array_of_references = set()
         for i in range(0, self.last_child_number):
-            detected_references = re.findall(r'таблиц[аеыу][\d .]+', self.file.paragraphs[i].paragraph_text)
+            if  isinstance(self.file.paragraphs[i], str):
+                detected_references = re.findall(r'таблиц[аеыу][\d .]+', self.file.paragraphs[i])
+            else:    
+                detected_references = re.findall(r'таблиц[аеыу][\d .]+', self.file.paragraphs[i].paragraph_text)
             if detected_references:
                 for reference in detected_references:
                     for one_part in re.split(r'таблиц[аеыу]| ', reference):

app/main/parser.py

@@ -5,6 +5,7 @@
 
 from main.presentations import PresentationPPTX
 from main.reports.docx_uploader import DocxUploader
+from main.reports.md_uploader import MdUpload
 from utils import convert_to
 
 logger = logging.getLogger('root_logger')
@@ -19,15 +20,24 @@ def parse(filepath, pdf_filepath):
                 logger.info(f"Презентация {filepath} старого формата. Временно преобразована в pptx для обработки.")
                 new_filepath = convert_to(filepath, target_format='pptx')
             file_object = PresentationPPTX(new_filepath)
-        elif tmp_filepath.endswith(('.doc', '.odt', '.docx')):
+        elif tmp_filepath.endswith(('.doc', '.odt', '.docx', )):
             new_filepath = filepath
             if tmp_filepath.endswith(('.doc', '.odt')):
                 logger.info(f"Отчёт {filepath} старого формата. Временно преобразован в docx для обработки.")
                 new_filepath = convert_to(filepath, target_format='docx')
+
             docx = DocxUploader()
             docx.upload(new_filepath, pdf_filepath)
             docx.parse()
             file_object = docx
+
+        elif tmp_filepath.endswith('.md' ):
+            new_filepath = filepath
+            doc = MdUpload(new_filepath)
+            md_text = doc.upload()
+            doc.parse(md_text)
+            file_object = doc
+
         else:
             raise ValueError("Файл с недопустимым именем или недопустимого формата: " + filepath)
         # Если была конвертация, то удаляем временный файл.

app/main/reports/README.md

@@ -65,12 +65,10 @@ Proof-of-concept парсинг файлов `.docx` с выводом стру
 ```bash
 $ python3 -m app.main.mse22.pdf_document text_from_pages --filename path_to_file
 ```
-
 ## `MD`
 
 Парсинг файлов `.md` с выводом структуры файла в текстовом виде в stdout.
 
 ```bash
 $ python3 -m app.main.reports.md_uploader md_parser --mdfile path_to_md_file
 ```
-

app/main/reports/document_uploader.py

@@ -0,0 +1,31 @@
+from abc import ABC, abstractmethod
+
+class DocumentUploader(ABC):
+
+    @abstractmethod
+    def upload(self):
+        pass
+
+    @abstractmethod
+    def parse(self):
+        pass
+
+    @abstractmethod
+    def parse_effective_styles(self):
+        pass
+
+    @abstractmethod
+    def page_counter(self):
+        pass
+
+    @abstractmethod
+    def make_chapters(self, work_type):
+        pass
+
+    @abstractmethod
+    def find_header_page(self, work_type):
+        pass
+
+    @abstractmethod
+    def find_literature_vkr(self, work_type):
+        pass

app/main/reports/docx_uploader/docx_uploader.py

@@ -9,9 +9,10 @@
 from .style import Style
 from .table import Table, Cell
 from ..pdf_document.pdf_document_manager import PdfDocumentManager
+from ..document_uploader import DocumentUploader
 
 
-class DocxUploader:
+class DocxUploader(DocumentUploader):
     def __init__(self):
         self.inline_shapes = []
         self.core_properties = None
@@ -27,6 +28,7 @@ def __init__(self):
         self.first_lines = []
         self.literature_header = []
         self.headers_page = 0
+        self.literature_page = 0
 
     def upload(self, file, pdf_filepath=''):
         self.file = docx.Document(file)
@@ -122,6 +124,16 @@ def find_header_page(self, work_type):
                         self.headers_page = header["page"]
                     break
         return self.headers_page
+
+    def find_literature_page(self, work_type):
+        if not self.literature_page:
+            for k, v in self.pdf_file.text_on_page.items():
+                line = v[:40] if len(v) > 21 else v
+                if re.search('СПИСОК ИСПОЛЬЗОВАННЫХ ИСТОЧНИКОВ', line.strip()):
+                    break
+                self.literature_page += 1
+        self.literature_page += 1
+        return self.literature_page
 
     def find_literature_vkr(self, work_type):
         if not self.literature_header: