' in line) ^ ('' in line)):
+ in_code_block = not in_code_block
+ if options[VERBOSE]:
+ if in_code_block:
+ print("Detected start of a codeblock, not registering titles")
+ else:
+ print("Detected end of codeblock, registering titles again")
+
+ # only split up if current line is in a fully non-os-specific section
+ if in_if_statement == 0:
+
+ title_level = check_for_title(line, in_code_block, curr_dirs, options)
+
+ # line is a title with a maximum depth of 4
+ if title_level > 0:
+ if after_first_title:
+
+ # write text of previous file
+ if previous_contained_if:
+ paragraphs_os_text[title] = current_paragraph
+ if options[VERBOSE]:
+ print("Saved os-specific chunk with temporary title: " + title + "\n")
+ else:
+ paragraphs_os_free_text[title] = current_paragraph
+ if options[VERBOSE]:
+ print("Saved generic chunk with title: " + title + "\n")
+
+ # write metadata of previous file
+ paragraphs_metadata[title] = write_metadata(main_title, title, link_list, last_title_level, last_dir, options[SOURCE_DIRECTORY] + '/' + main_title + '.md')
+
+ # make a new title
+ title = make_valid_title(line[title_level + 1:-1])
+
+ # create an entry for the file in the paragraphs text dictionary
+ current_paragraph = ""
+
+ after_first_title = True
+ subtitle_order.append(title)
+
+ # reset link_list
+ link_list = []
+
+ previous_contained_if = False
+
+ # line is not a title
+ elif after_first_title:
+ line, link_list = replace_markdown_markers(line, link_list, in_code_block, main_title, is_linux_tutorial)
+ if line != "\n":
+ current_paragraph += line
+
+ # keep track of title level and directory to write to metadata upon discovering a new subtitle
+ if title_level > 0:
+ last_title_level = title_level
+ last_dir = curr_dirs[last_title_level]
+ else:
+ previous_contained_if = True
+ line, link_list = replace_markdown_markers(line, link_list, in_code_block, main_title, is_linux_tutorial)
+ if line != "\n":
+ current_paragraph += line
+
+ # write dictionaries for the last file
+ if previous_contained_if:
+ paragraphs_os_text[title] = current_paragraph
+ if options[VERBOSE]:
+ print("Saved os-specific chunk with temporary title: " + title + "\n")
+ else:
+ paragraphs_os_free_text[title] = current_paragraph
+ if options[VERBOSE]:
+ print("Saved generic chunk with title: " + title + "\n")
+ paragraphs_metadata[title] = write_metadata(main_title, title, link_list, last_title_level, curr_dirs[last_title_level], options[SOURCE_DIRECTORY] + '/' + main_title + '.md')
+
+ return paragraphs_os_text, paragraphs_os_free_text, paragraphs_metadata, subtitle_order
+
+
+def split_on_paragraphs(file, main_title, options, is_linux_tutorial, current_paragraph_number=-1, OS=GENERIC):
+ """
+ Function that splits the text into smaller sections based on the paragraph structure and makes them into two dictionaries containing text and metadata
+
+ :param file: the filepath of the file to be split
+ :param main_title: the main title of the file
+ :param options: dictionary containing the options given by the user
+ :param is_linux_tutorial: boolean indicating whether the current file is part of the linux tutorial
+ :param current_paragraph_number: number of the paragraph that is being split, only applicable when splitting an os-specific paragraph
+ :param OS: the OS of the file to be split, only applicable when splitting an os-specific paragraph
+ :return paragraphs_text: dictionary containing the split sections of text
+ :return paragraphs_metadata: dictionary containing the metadata of each split section of text
+ :return subtitle_order: list containing all encountered subtitles in order of appearance
+ """
+
+ if options[VERBOSE]:
+ print("Splitting on paragraphs\n")
+
+ # start of assuming we are not in a code_block
+ in_code_block = False
+
+ # define initial dictionaries
+ paragraphs_os_free_text = {}
+ paragraphs_os_text = {}
+ paragraphs_metadata = {}
+
+ # variable to keep track of the current paragraph
+ current_paragraph = ""
+
+ # list to keep track of links in the text
+ link_list = []
+
+ # list to keep track of the order of the subtitles
+ subtitle_order = []
+
+ # variable to keep track of how many if-statements deep the current line is
+ in_if_statement = 0
+
+ # variable to indicate that previous section was one with if-statements
+ previous_contained_if = False
+
+ # variable to indicate that the previous line was part of a list
+ in_list = False
+
+ # paragraph number to add to title
+ paragraph_number = 1
+
+ # metadata title
+ metadata_title = main_title
+
+ # define metadata data if split occurs on paragraphs and last_title and title_level are known (will be replaced later on in the process)
+ if current_paragraph_number != -1:
+ last_title_level = 4
+ last_dir = "PLACEHOLDER"
+
+ # list to keep track of most recent directories on each title level
+ curr_dirs = [main_title for _ in range(options[MAX_TITLE_DEPTH] + 1)]
+
+ with open(file, 'r') as readfile:
+
+ # Create two independent iterators from the original file iterator (needed to check for lists)
+ current_line, next_line = tee(readfile)
+
+ # Advance the next_line iterator by one step, so it is always one step ahead
+ next(next_line, None)
+
+ # Process the lines
+ for line, nxt in zip_longest(current_line, next_line, fillvalue=""):
+
+ # detect if-statements starting or ending on the current line
+ in_if_statement += len(re.findall(IF_MANGLED_PATTERNS[IF], line)) - len(
+ re.findall(IF_MANGLED_PATTERNS[ENDIF], line))
+
+ # detect whether the current line is in a list
+ if re.search(r'^(\s*)([*+-]|\d+\.|[a-zA-Z]\.)\s+.*$', line): # beginning of a list entry
+ in_list = True
+ if options[VERBOSE]:
+ print("First line of new list entry found, not starting new paragraphs: " + line[:-1])
+ elif re.search(r'^\s{2,}.+$', line) and in_list: # middle of a list entry
+ pass
+ elif re.search(r'^(\s*)([*+-]|\d+\.|[a-zA-Z]\.)\s+.*$|^\s{2,}.+$|^\n', nxt) and in_list: # line(s) between list entries
+ pass
+ elif re.search(r'^(\s*)([*+-]|\d+\.|[a-zA-Z]\.)\s+.*$', nxt):
+ in_list = True
+ elif in_list:
+ if options[VERBOSE]:
+ print("List ended, starting new paragraphs again")
+ in_list = False
+ else:
+ in_list = False
+
+ # detect codeblocks to make sure titles aren't detected in them
+ if '```' in line or (('' in line) ^ ('A line with style
", [], False, "", "A line with style", []), + # Bot comment + ("", [], False, "", "Something about the following table", []), + # non-Bot comment + ("", [], False, "", "", []), + # something else with <> + ("A line with an example where you should put