From 1551ab07c7b6210cd0669f9b8f40d82295c4c745 Mon Sep 17 00:00:00 2001 From: Abhilasha Narendra Date: Thu, 22 Jul 2021 17:35:31 +0530 Subject: [PATCH 01/21] Delete migration_toolkit_v54.0.0_documentation.pdf Former-commit-id: e298e303a0e8f710e90f66357437f50474bd8c14 From b7790d7760762eccaf8800c275a56ddad6ae4d89 Mon Sep 17 00:00:00 2001 From: George Song Date: Fri, 11 Jun 2021 17:34:26 -0700 Subject: [PATCH 02/21] fix: remove erroneously placed PDFs Former-commit-id: 8daa20434606343ee52bb03a682cfbf3a1977ad1 From 757c90c9b948d899ae5c40ad5a6630b5a10090df Mon Sep 17 00:00:00 2001 From: George Song Date: Sat, 12 Jun 2021 01:04:38 -0700 Subject: [PATCH 03/21] fix: remove blank PDFs Former-commit-id: bec5a11645732a8041bfa82b52f2ccc8142f6c96 From fe900c6e678641244f1ea58efae9ab67e8612d83 Mon Sep 17 00:00:00 2001 From: George Song Date: Sat, 12 Jun 2021 00:24:13 -0700 Subject: [PATCH 04/21] feat: add script to compare PDFs Former-commit-id: b650dafb40a9dcafcfa8d45a2395d42e6126d47f --- scripts/pdf/compare_pdfs.py | 67 +++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 scripts/pdf/compare_pdfs.py diff --git a/scripts/pdf/compare_pdfs.py b/scripts/pdf/compare_pdfs.py new file mode 100755 index 00000000000..bcf1e6297b5 --- /dev/null +++ b/scripts/pdf/compare_pdfs.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import argparse +import multiprocessing +import shlex +import subprocess +from itertools import zip_longest +from pathlib import Path + + +def main(args): + pdfs = sorted( + args.dir1.glob("**/*.pdf"), key=lambda x: x.stat().st_size, reverse=True + ) + args.removed_pdfs = args.output_dir / "00_removed_pdfs.txt" + args.removed_pdfs.unlink(missing_ok=True) + with multiprocessing.Pool(int(multiprocessing.cpu_count() * 2 / 3)) as pool: + pool.starmap(diff_pdfs, zip_longest(pdfs, [], fillvalue=args), chunksize=1) + + print(f'See removed PDFs in "{args.removed_pdfs}"') + + +def diff_pdfs(pdf1, args): + pdf2 = args.dir2 / pdf1.name + + if not pdf2.is_file(): + print(f"☠️ {pdf1.name}") + with open(args.removed_pdfs, "a") as of: + print(pdf1.name, file=of) + return + + diff = args.output_dir / pdf1.name + cmd = shlex.split(f"diff-pdf -s -m -g --output-diff={diff} {pdf1} {pdf2}") + print(f"👷‍♀️ {pdf1.name}") + subprocess.run(cmd, capture_output=True) + print(f"✅ {pdf1.name}") + + +def cli(): + parser = argparse.ArgumentParser( + description=( + "compare PDFs from dir1 and dir2 " + "(files present in dir2 but not in dir1 are ignored)" + ) + ) + parser.add_argument("dir1", type=path_type, help="first directory of PDFs") + parser.add_argument("dir2", type=path_type, help="second directory of PDFs") + parser.add_argument( + "output_dir", type=output_type, help="directory to store diff PDFs" + ) + return parser.parse_args() + + +def path_type(arg): + p = Path(arg) + if not p.is_dir(): + raise argparse.ArgumentTypeError(f'"{arg}" is not a valid directory') + return p + + +def output_type(arg): + p = Path(arg) + p.mkdir(parents=True, exist_ok=True) + return p + + +if __name__ == "__main__": + main(cli()) From cf893a4bd5d8b5292013740fc61da9ca842a5035 Mon Sep 17 00:00:00 2001 From: George Song Date: Thu, 22 Apr 2021 11:31:13 -0700 Subject: [PATCH 05/21] fix: remove unnecessary import Former-commit-id: e72c21d70000ec67a44f48eab742647ca427fdeb --- scripts/pdf/generate_pdf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index f9082851c7b..56d5214d9d6 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 - import os import re import sys from datetime import datetime -from itertools import chain from pathlib import Path from subprocess import run From e8f64b04d2f1cb74b60d9c911757942d578bd496 Mon Sep 17 00:00:00 2001 From: George Song Date: Fri, 4 Jun 2021 16:02:40 -0700 Subject: [PATCH 06/21] refactor: make the script more Pythonic * Use `pathlib.Path` instead of `os.*` * Use dataclass for well, a data Class Former-commit-id: 54ae4f40e5a7517d1cb0393ae4e53e8755879333 --- scripts/pdf/generate_pdf.py | 101 ++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 56d5214d9d6..01fa47ab640 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import os import re import sys +from dataclasses import dataclass from datetime import datetime from pathlib import Path from subprocess import run @@ -19,36 +19,31 @@ # import code; code.interact(local=dict(globals(), **locals())) +@dataclass class ToCItem: - def __init__(self, filename, chapter): - self.filename = filename - self.chapter = chapter - self.title = "" - self.anchor = "" + filename: Path + chapter: str + title: str = "" + anchor: str = "" -def putIndexFirst(e): - return e.filename.replace("index.mdx", "00_index.mdx") +def putIndexFirst(path): + filename = str(path) + return filename.replace("index.mdx", "00_index.mdx") -def putIndexFirst2(e): - return e.replace("index.mdx", "00_index.mdx") - - -def getFilename(file): - return file.filename - - -def filterList(filename): - if ".png" in filename or "images" in filename or ".DS_Store" in filename: - return False - else: +def filterList(path): + if path.is_dir() and not path.match("*images*"): + return True + elif path.suffix in [".mdx", ".md"]: return True + else: + return False def getTitle(dirName): - indexPath = dirName + "/index.mdx" - if os.path.exists(indexPath): + indexPath = dirName / "index.mdx" + if indexPath.exists(): indexFile = open(indexPath, "r") for line in indexFile.readlines(): if "title: " in line: @@ -61,24 +56,21 @@ def stripQuotes(str): def getListOfFiles(dirName, parentChapter): - # create a list of file and sub directories - # names in the given directory - listOfFiles = list(filter(filterList, os.listdir(dirName))) - listOfFiles.sort(key=putIndexFirst2) + # create a list of file and sub directories names in the given directory + listOfFiles = list(filter(filterList, dirName.iterdir())) + listOfFiles.sort(key=putIndexFirst) allFiles = list() chapter = 0 # Iterate over all the entries for entry in listOfFiles: - # Create full path - fullPath = os.path.join(dirName, entry) # If entry is a directory then get the list of files in this directory - if os.path.isdir(fullPath): + if entry.is_dir(): allFiles = allFiles + getListOfFiles( - fullPath, parentChapter + str(chapter) + "." + entry, f"{parentChapter}{str(chapter)}." ) - elif ".mdx" in entry or ".md" in entry: - allFiles.append(ToCItem(fullPath, parentChapter + str(chapter))) + else: + allFiles.append(ToCItem(entry, parentChapter + str(chapter))) chapter += 1 return allFiles @@ -87,8 +79,7 @@ def getListOfFiles(dirName, parentChapter): def main(): dirName = "" try: - dirName = sys.argv[1] - dirName = re.sub(r"\/$", "", dirName) + dirName = Path(sys.argv[1]) except BaseException: print("directory not passed in") print("if running from yarn use `yarn build-pdf directory/path/here`") @@ -102,32 +93,30 @@ def main(): except BaseException: pass - splitDirName = dirName.split("/") - product = splitDirName[2] - version = splitDirName[3] + product = dirName.parts[2] + version = dirName.parts[3] fullProductPdf = True guide = None - if len(splitDirName) > 4: + if len(dirName.parts) > 4: fullProductPdf = False - guide = splitDirName[4] + guide = dirName.parts[4] _file_prefix = f"{dirName}/{product}_v{version}" _doc_prefix = f"{_file_prefix}_documentation" - mdxFilePath = f"{_doc_prefix}.mdx" - htmlFilePath = f"{_doc_prefix}.html" - coverFilePath = f"{_doc_prefix}_cover.html" - pdfFilePath = f"{_file_prefix}" "_{}documentation.pdf".format( - guide + "_" if guide else "" + mdxFilePath = Path(f"{_doc_prefix}.mdx") + htmlFilePath = Path(f"{_doc_prefix}.html") + coverFilePath = Path(f"{_doc_prefix}_cover.html") + pdfFilePath = Path( + f"{_file_prefix}" "_{}documentation.pdf".format(guide + "_" if guide else "") ) print(f"{ANSI_BLUE}building {pdfFilePath}{ANSI_STOP}") - if not os.path.exists(dirName): + if not dirName.exists(): raise Exception("directory does not exist") - if os.path.exists(mdxFilePath): - os.remove(mdxFilePath) + mdxFilePath.unlink(missing_ok=True) # Get the list of all files in directory tree at given path listOfFiles = getListOfFiles(dirName, "") @@ -147,15 +136,13 @@ def main(): if tag and len(elem.anchor) == 0: elem.anchor = tag.group(1) - resourceSearchPaths = {dirName} + resourceSearchPaths = {str(dirName)} # Print the files with open(mdxFilePath, "w") as fp: for elem in listOfFiles: g = open(elem.filename, "r") - - baseImagePath = os.path.split(elem.filename)[0] - resourceSearchPaths.add(baseImagePath) + resourceSearchPaths.add(elem.filename.parent) frontmatterCount = 2 for line in g.readlines(): @@ -193,14 +180,14 @@ def main(): "--self-contained", "--highlight-style=tango", f"--css={BASE_DIR / 'pdf-styles.css'}", - f"--resource-path={':'.join(resourceSearchPaths)}", + f"--resource-path={':'.join((str(p) for p in resourceSearchPaths))}", f"--output={htmlFilePath}", ] ) output.check_returncode() - if not os.path.exists(htmlFilePath): - os.remove(mdxFilePath) + if not htmlFilePath.exists(): + mdxFilePath.unlink() raise Exception( f"\033[91m html file failed to generate for {mdxFilePath} \033[0m" ) @@ -272,10 +259,10 @@ def main(): if openPdf: run(["open", pdfFilePath]) - os.remove(mdxFilePath) + mdxFilePath.unlink() if not html: - os.remove(htmlFilePath) - os.remove(coverFilePath) + htmlFilePath.unlink() + coverFilePath.unlink() if __name__ == "__main__": From 123b80475750055248755a776a98db15b1c87c6c Mon Sep 17 00:00:00 2001 From: George Song Date: Tue, 8 Jun 2021 20:50:02 -0700 Subject: [PATCH 07/21] refactor: build proper CLI using argparse * Rearrange file in preparation of future refactoring Former-commit-id: 2c0f5bd28dc1749663ebebc64fed4d2eea29f2af --- scripts/pdf/generate_pdf.py | 183 ++++++++++++++++++++---------------- 1 file changed, 100 insertions(+), 83 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 01fa47ab640..004ee2f659b 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 +import argparse import re -import sys from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -15,89 +15,14 @@ ANSI_RED = "\033[31m" -# magic snippet for inline repl -# import code; code.interact(local=dict(globals(), **locals())) - - -@dataclass -class ToCItem: - filename: Path - chapter: str - title: str = "" - anchor: str = "" - - -def putIndexFirst(path): - filename = str(path) - return filename.replace("index.mdx", "00_index.mdx") - - -def filterList(path): - if path.is_dir() and not path.match("*images*"): - return True - elif path.suffix in [".mdx", ".md"]: - return True - else: - return False - - -def getTitle(dirName): - indexPath = dirName / "index.mdx" - if indexPath.exists(): - indexFile = open(indexPath, "r") - for line in indexFile.readlines(): - if "title: " in line: - return stripQuotes(line.replace("title: ", "")) - return None - - -def stripQuotes(str): - return str.strip().strip("'").strip('"') - - -def getListOfFiles(dirName, parentChapter): - # create a list of file and sub directories names in the given directory - listOfFiles = list(filter(filterList, dirName.iterdir())) - listOfFiles.sort(key=putIndexFirst) - allFiles = list() - chapter = 0 - - # Iterate over all the entries - for entry in listOfFiles: - # If entry is a directory then get the list of files in this directory - if entry.is_dir(): - allFiles = allFiles + getListOfFiles( - entry, f"{parentChapter}{str(chapter)}." - ) - else: - allFiles.append(ToCItem(entry, parentChapter + str(chapter))) - - chapter += 1 - return allFiles - - -def main(): - dirName = "" - try: - dirName = Path(sys.argv[1]) - except BaseException: - print("directory not passed in") - print("if running from yarn use `yarn build-pdf directory/path/here`") - sys.exit(1) - - openPdf = False - html = False - try: - html = sys.argv[2] == "--html" - openPdf = sys.argv[2] == "--open" - except BaseException: - pass - +def main(args): + dirName = args.doc_path product = dirName.parts[2] version = dirName.parts[3] fullProductPdf = True guide = None + # GS 2021-06-04: I don't think this if condition is possible if len(dirName.parts) > 4: fullProductPdf = False guide = dirName.parts[4] @@ -192,7 +117,7 @@ def main(): f"\033[91m html file failed to generate for {mdxFilePath} \033[0m" ) - if html: + if args.generate_html_only: run(["open", htmlFilePath]) else: print("generating cover page") @@ -256,14 +181,106 @@ def main(): ) output.check_returncode() - if openPdf: + if args.open_pdf: run(["open", pdfFilePath]) mdxFilePath.unlink() - if not html: + if not args.generate_html_only: htmlFilePath.unlink() coverFilePath.unlink() +def putIndexFirst(path): + filename = str(path) + return filename.replace("index.mdx", "00_index.mdx") + + +def filterList(path): + if path.is_dir() and not path.match("*images*"): + return True + elif path.suffix in [".mdx", ".md"]: + return True + else: + return False + + +def getTitle(dirName): + indexPath = dirName / "index.mdx" + if indexPath.exists(): + indexFile = open(indexPath, "r") + for line in indexFile.readlines(): + if "title: " in line: + return stripQuotes(line.replace("title: ", "")) + return None + + +def stripQuotes(str): + return str.strip().strip("'").strip('"') + + +def getListOfFiles(dirName, parentChapter): + # create a list of file and sub directories names in the given directory + listOfFiles = list(filter(filterList, dirName.iterdir())) + listOfFiles.sort(key=putIndexFirst) + allFiles = list() + chapter = 0 + + # Iterate over all the entries + for entry in listOfFiles: + # If entry is a directory then get the list of files in this directory + if entry.is_dir(): + allFiles = allFiles + getListOfFiles( + entry, f"{parentChapter}{str(chapter)}." + ) + else: + allFiles.append(ToCItem(entry, parentChapter + str(chapter))) + + chapter += 1 + return allFiles + + +@dataclass +class ToCItem: + filename: Path + chapter: str + title: str = "" + anchor: str = "" + + +def cli(): + parser = argparse.ArgumentParser( + description=( + "generate a PDF based on documentation files from a product directory" + ) + ) + parser.add_argument( + "doc_path", + metavar="doc-path", + type=path_type, + help="directory path for a specific product's documentation", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--html", + dest="generate_html_only", + action="store_true", + help="generate the intermediate HTML instead of PDF", + ) + group.add_argument( + "--open", + dest="open_pdf", + action="store_true", + help="open the PDF after generation", + ) + return parser.parse_args() + + +def path_type(arg): + p = Path(arg) + if not (p.exists() and p.is_dir()): + raise argparse.ArgumentTypeError(f'"{arg}" is not a valid directory') + return p + + if __name__ == "__main__": - main() + main(cli()) From 7616546890411c19023a2028b6fe3e175655f5ec Mon Sep 17 00:00:00 2001 From: George Song Date: Thu, 10 Jun 2021 15:07:10 -0700 Subject: [PATCH 08/21] =?UTF-8?q?refactor:=20simplify=20and=20python-ify?= =?UTF-8?q?=20the=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove impossible code paths * Remove unneeded variable creations * 🐍 case variable names Former-commit-id: 41e799a5bd9c5f8e4f017d2050082b01bd945a96 --- scripts/pdf/generate_pdf.py | 169 ++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 94 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 004ee2f659b..76d2245aefd 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import argparse import re +from argparse import ArgumentParser, ArgumentTypeError from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -16,41 +16,28 @@ def main(args): - dirName = args.doc_path - product = dirName.parts[2] - version = dirName.parts[3] - - fullProductPdf = True - guide = None - # GS 2021-06-04: I don't think this if condition is possible - if len(dirName.parts) > 4: - fullProductPdf = False - guide = dirName.parts[4] - - _file_prefix = f"{dirName}/{product}_v{version}" - _doc_prefix = f"{_file_prefix}_documentation" - mdxFilePath = Path(f"{_doc_prefix}.mdx") - htmlFilePath = Path(f"{_doc_prefix}.html") - coverFilePath = Path(f"{_doc_prefix}_cover.html") - pdfFilePath = Path( - f"{_file_prefix}" "_{}documentation.pdf".format(guide + "_" if guide else "") - ) + doc_path = args.doc_path + product = doc_path.parts[2] + version = doc_path.parts[3] + + _file_prefix = f"{product}_v{version}_documentation" - print(f"{ANSI_BLUE}building {pdfFilePath}{ANSI_STOP}") + mdx_file = doc_path / f"{_file_prefix}.mdx" + html_file = doc_path / f"{_file_prefix}.html" + cover_file = doc_path / f"{_file_prefix}_cover.html" + pdf_file = doc_path / f"{_file_prefix}.pdf" - if not dirName.exists(): - raise Exception("directory does not exist") + print(f"{ANSI_BLUE}building {pdf_file}{ANSI_STOP}") - mdxFilePath.unlink(missing_ok=True) + mdx_file.unlink(missing_ok=True) # Get the list of all files in directory tree at given path - listOfFiles = getListOfFiles(dirName, "") - if len(listOfFiles) == 0: - raise Exception(f"no files in {dirName}") - if fullProductPdf: - listOfFiles.pop(0) # remove base product index page, which are empty + files = list_files(doc_path, "") + if len(files) == 0: + raise Exception(f"no files in {doc_path}") + files.pop(0) # remove base product index page, which are empty - for elem in listOfFiles: + for elem in files: g = open(elem.filename, "r") for line in g.readlines(): if "title: " in line: @@ -61,75 +48,71 @@ def main(args): if tag and len(elem.anchor) == 0: elem.anchor = tag.group(1) - resourceSearchPaths = {str(dirName)} + resource_search_paths = {doc_path} # Print the files - with open(mdxFilePath, "w") as fp: - for elem in listOfFiles: + with open(mdx_file, "w") as fp: + for elem in files: g = open(elem.filename, "r") - resourceSearchPaths.add(elem.filename.parent) + resource_search_paths.add(elem.filename.parent) - frontmatterCount = 2 + front_matter_count = 2 for line in g.readlines(): - newLine = line + new_line = line if line[0:2] == "# ": - newLine = "##" + line + new_line = "##" + line if line[0:3] == "## ": - newLine = "#" + line + new_line = "#" + line if "toctree" in line: - frontmatterCount = 3 - if frontmatterCount == 0: - fp.write(newLine) + front_matter_count = 3 + if front_matter_count == 0: + fp.write(new_line) if "title: " in line: - newLine = ( + new_line = ( re.sub(r"\.0", "", elem.chapter) + (" " * 10) - + stripQuotes(line[7:]) + + strip_quotes(line[7:]) + "\n" ) - fp.write(newLine) - if "---" in line and frontmatterCount > 0: - frontmatterCount -= 1 - fp.write(newLine) + fp.write(new_line) + if "---" in line and front_matter_count > 0: + front_matter_count -= 1 + fp.write(new_line) fp.write("\n") - title = getTitle(dirName) or product + title = get_title(doc_path) or product print("generating docs html") output = run( [ "pandoc", - mdxFilePath, + mdx_file, "--from=gfm", "--self-contained", "--highlight-style=tango", f"--css={BASE_DIR / 'pdf-styles.css'}", - f"--resource-path={':'.join((str(p) for p in resourceSearchPaths))}", - f"--output={htmlFilePath}", + f"--resource-path={':'.join((str(p) for p in resource_search_paths))}", + f"--output={html_file}", ] ) output.check_returncode() - if not htmlFilePath.exists(): - mdxFilePath.unlink() - raise Exception( - f"\033[91m html file failed to generate for {mdxFilePath} \033[0m" - ) + if not html_file.exists(): + mdx_file.unlink() + raise Exception(f"\033[91m html file failed to generate for {mdx_file} \033[0m") if args.generate_html_only: - run(["open", htmlFilePath]) + run(["open", html_file]) else: print("generating cover page") - with open(BASE_DIR / "cover.html") as source, open( - coverFilePath, "w" - ) as output: + with open(BASE_DIR / "cover.html") as source, open(cover_file, "w") as output: data = source.read() data = data.replace("[PRODUCT]", title) data = data.replace("[VERSION]", version) output.write(data) - headerFooterCommon = [ + header_footer_common = [ "--header-font-name", "Signika", "--header-font-size", @@ -147,7 +130,7 @@ def main(args): "All rights reserved.", ] - headerFooterOptions = [ + header_footer_options = [ "--header-right", "[doctitle]", "--footer-right", @@ -166,36 +149,36 @@ def main(args): "15mm", "--margin-bottom", "15mm", - *headerFooterCommon, - coverFilePath, + *header_footer_common, + cover_file, "--footer-right", f"Built at {datetime.utcnow().replace(microsecond=0).isoformat()}", "toc", "--xsl-style-sheet", "scripts/pdf/toc-style.xsl", - *headerFooterOptions, - htmlFilePath, - *headerFooterOptions, - pdfFilePath, + *header_footer_options, + html_file, + *header_footer_options, + pdf_file, ] ) output.check_returncode() if args.open_pdf: - run(["open", pdfFilePath]) + run(["open", pdf_file]) - mdxFilePath.unlink() + mdx_file.unlink() if not args.generate_html_only: - htmlFilePath.unlink() - coverFilePath.unlink() + html_file.unlink() + cover_file.unlink() -def putIndexFirst(path): +def put_index_first(path): filename = str(path) return filename.replace("index.mdx", "00_index.mdx") -def filterList(path): +def filter_path(path): if path.is_dir() and not path.match("*images*"): return True elif path.suffix in [".mdx", ".md"]: @@ -204,43 +187,41 @@ def filterList(path): return False -def getTitle(dirName): - indexPath = dirName / "index.mdx" - if indexPath.exists(): - indexFile = open(indexPath, "r") - for line in indexFile.readlines(): +def get_title(doc_path): + index_path = doc_path / "index.mdx" + if index_path.exists(): + index_file = open(index_path, "r") + for line in index_file.readlines(): if "title: " in line: - return stripQuotes(line.replace("title: ", "")) + return strip_quotes(line.replace("title: ", "")) return None -def stripQuotes(str): +def strip_quotes(str): return str.strip().strip("'").strip('"') -def getListOfFiles(dirName, parentChapter): +def list_files(doc_path, parent_chapter): # create a list of file and sub directories names in the given directory - listOfFiles = list(filter(filterList, dirName.iterdir())) - listOfFiles.sort(key=putIndexFirst) - allFiles = list() + files = list(filter(filter_path, doc_path.iterdir())) + files.sort(key=put_index_first) + all_files = list() chapter = 0 # Iterate over all the entries - for entry in listOfFiles: + for entry in files: # If entry is a directory then get the list of files in this directory if entry.is_dir(): - allFiles = allFiles + getListOfFiles( - entry, f"{parentChapter}{str(chapter)}." - ) + all_files += list_files(entry, f"{parent_chapter}{str(chapter)}.") else: - allFiles.append(ToCItem(entry, parentChapter + str(chapter))) + all_files.append(TocItem(entry, parent_chapter + str(chapter))) chapter += 1 - return allFiles + return all_files @dataclass -class ToCItem: +class TocItem: filename: Path chapter: str title: str = "" @@ -248,7 +229,7 @@ class ToCItem: def cli(): - parser = argparse.ArgumentParser( + parser = ArgumentParser( description=( "generate a PDF based on documentation files from a product directory" ) @@ -278,7 +259,7 @@ def cli(): def path_type(arg): p = Path(arg) if not (p.exists() and p.is_dir()): - raise argparse.ArgumentTypeError(f'"{arg}" is not a valid directory') + raise ArgumentTypeError(f'"{arg}" is not a valid directory') return p From ef49778bed73236707eb112801a608ed439114aa Mon Sep 17 00:00:00 2001 From: George Song Date: Fri, 11 Jun 2021 19:18:58 -0700 Subject: [PATCH 09/21] fix: don't ignore top-level index file if it's not empty * Re-write chapter numbering logic to be more expressive Former-commit-id: eb18f6919346e800892f2e3506acc098a3bda386 --- scripts/pdf/generate_pdf.py | 97 +++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 76d2245aefd..ed48e506da3 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -5,37 +5,22 @@ from datetime import datetime from pathlib import Path from subprocess import run +from typing import List BASE_DIR = Path(__file__).resolve().parent ANSI_STOP = "\033[0m" ANSI_BLUE = "\033[34m" -ANSI_GREEN = "\033[32m" -ANSI_YELLOW = "\033[33m" -ANSI_RED = "\033[31m" def main(args): - doc_path = args.doc_path - product = doc_path.parts[2] - version = doc_path.parts[3] - - _file_prefix = f"{product}_v{version}_documentation" - - mdx_file = doc_path / f"{_file_prefix}.mdx" - html_file = doc_path / f"{_file_prefix}.html" - cover_file = doc_path / f"{_file_prefix}_cover.html" - pdf_file = doc_path / f"{_file_prefix}.pdf" + doc_path, product, version, mdx_file, html_file, cover_file, pdf_file = setup(args) print(f"{ANSI_BLUE}building {pdf_file}{ANSI_STOP}") - mdx_file.unlink(missing_ok=True) - - # Get the list of all files in directory tree at given path - files = list_files(doc_path, "") + files = list_files(doc_path) if len(files) == 0: raise Exception(f"no files in {doc_path}") - files.pop(0) # remove base product index page, which are empty for elem in files: g = open(elem.filename, "r") @@ -173,20 +158,64 @@ def main(args): cover_file.unlink() -def put_index_first(path): - filename = str(path) - return filename.replace("index.mdx", "00_index.mdx") +def setup(args): + doc_path = args.doc_path + product = doc_path.parts[2] + version = doc_path.parts[3] + + _file_prefix = f"{product}_v{version}_documentation" + + mdx_file = doc_path / f"{_file_prefix}.mdx" + html_file = doc_path / f"{_file_prefix}.html" + cover_file = doc_path / f"{_file_prefix}_cover.html" + pdf_file = doc_path / f"{_file_prefix}.pdf" + + for f in [mdx_file, html_file, cover_file]: + f.unlink(missing_ok=True) + + return doc_path, product, version, mdx_file, html_file, cover_file, pdf_file + + +def list_files(doc_path, chapter=None): + chapter = [1] if chapter is None else chapter + all_files = [] + directory_contents = sorted( + filter(filter_path, doc_path.iterdir()), key=put_index_first + ) + + for i, entry in enumerate(directory_contents): + if entry.is_dir(): + chapter = chapter if i == 0 else advance_chapter(chapter) + all_files += list_files(entry, chapter) + else: + chapter = [*chapter, 0] if i == 0 else advance_chapter(chapter) + all_files.append(TocItem(filename=entry, chapter=chapter)) + + return all_files + + +def advance_chapter(chapter): + return [*chapter[:-1], chapter[-1] + 1] def filter_path(path): if path.is_dir() and not path.match("*images*"): return True elif path.suffix in [".mdx", ".md"]: - return True + with open(path) as f: + content = re.sub( + "^---$.*?^---$", "", f.read(), flags=re.DOTALL | re.MULTILINE + ).strip() + return False if content == "" else True else: return False +def put_index_first(path): + filename = str(path) + return filename.replace("index.mdx", "00_index.mdx") + + def get_title(doc_path): index_path = doc_path / "index.mdx" if index_path.exists(): @@ -201,32 +230,16 @@ def strip_quotes(str): return str.strip().strip("'").strip('"') -def list_files(doc_path, parent_chapter): - # create a list of file and sub directories names in the given directory - files = list(filter(filter_path, doc_path.iterdir())) - files.sort(key=put_index_first) - all_files = list() - chapter = 0 - - # Iterate over all the entries - for entry in files: - # If entry is a directory then get the list of files in this directory - if entry.is_dir(): - all_files += list_files(entry, f"{parent_chapter}{str(chapter)}.") - else: - all_files.append(TocItem(entry, parent_chapter + str(chapter))) - - chapter += 1 - return all_files - - @dataclass class TocItem: filename: Path - chapter: str + chapter: List[int] title: str = "" anchor: str = "" + def __post_init__(self): + self.chapter = ".".join((str(c) for c in self.chapter if c > 0)) + def cli(): parser = ArgumentParser( From dc4adb9266e40cf566d4fdad8d620b4d8111b8ec Mon Sep 17 00:00:00 2001 From: George Song Date: Fri, 11 Jun 2021 19:52:44 -0700 Subject: [PATCH 10/21] fix: use context to ensure files are closed properly Former-commit-id: b1f95632898b38794baab0a7e48aa81475c52081 --- scripts/pdf/generate_pdf.py | 75 ++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index ed48e506da3..96126dacd54 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -22,49 +22,49 @@ def main(args): if len(files) == 0: raise Exception(f"no files in {doc_path}") + pattern = re.compile('div id="(.*?)" class="registered_link"') for elem in files: - g = open(elem.filename, "r") - for line in g.readlines(): - if "title: " in line: - elem.title = line[7:].replace('"', "").replace("\n", "") - pattern = re.compile('div id="(.*?)" class="registered_link"') + with open(elem.filename, "r") as g: + for line in g: + if "title: " in line: + elem.title = line[7:].replace('"', "").replace("\n", "") - tag = pattern.search(line) - if tag and len(elem.anchor) == 0: - elem.anchor = tag.group(1) + tag = pattern.search(line) + if tag and len(elem.anchor) == 0: + elem.anchor = tag.group(1) resource_search_paths = {doc_path} # Print the files with open(mdx_file, "w") as fp: for elem in files: - g = open(elem.filename, "r") - resource_search_paths.add(elem.filename.parent) - - front_matter_count = 2 - for line in g.readlines(): - new_line = line - - if line[0:2] == "# ": - new_line = "##" + line - if line[0:3] == "## ": - new_line = "#" + line - if "toctree" in line: - front_matter_count = 3 - if front_matter_count == 0: - fp.write(new_line) - if "title: " in line: - new_line = ( - re.sub(r"\.0", "", elem.chapter) - + (" " * 10) - + strip_quotes(line[7:]) - + "\n" - ) - fp.write(new_line) - if "---" in line and front_matter_count > 0: - front_matter_count -= 1 - fp.write(new_line) - fp.write("\n") + with open(elem.filename, "r") as g: + resource_search_paths.add(elem.filename.parent) + + front_matter_count = 2 + for line in g: + new_line = line + + if line[0:2] == "# ": + new_line = "##" + line + if line[0:3] == "## ": + new_line = "#" + line + if "toctree" in line: + front_matter_count = 3 + if front_matter_count == 0: + fp.write(new_line) + if "title: " in line: + new_line = ( + re.sub(r"\.0", "", elem.chapter) + + (" " * 10) + + strip_quotes(line[7:]) + + "\n" + ) + fp.write(new_line) + if "---" in line and front_matter_count > 0: + front_matter_count -= 1 + fp.write(new_line) + fp.write("\n") title = get_title(doc_path) or product @@ -219,9 +219,8 @@ def put_index_first(path): def get_title(doc_path): index_path = doc_path / "index.mdx" if index_path.exists(): - index_file = open(index_path, "r") - for line in index_file.readlines(): - if "title: " in line: + with open(index_path) as index_file: + for line in (line for line in index_file if "title: " in line): return strip_quotes(line.replace("title: ", "")) return None From c5052d7c43f11d592a5d54b0f54840dd9d376ba5 Mon Sep 17 00:00:00 2001 From: George Song Date: Sat, 12 Jun 2021 01:00:27 -0700 Subject: [PATCH 11/21] fix: skip StubCards index files Former-commit-id: 2111ad64e9e5af296283164206b223c7c943b5e6 --- scripts/pdf/generate_pdf.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 96126dacd54..feb0169e253 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import os import re from argparse import ArgumentParser, ArgumentTypeError from dataclasses import dataclass @@ -9,18 +10,20 @@ BASE_DIR = Path(__file__).resolve().parent -ANSI_STOP = "\033[0m" ANSI_BLUE = "\033[34m" +ANSI_STOP = "\033[0m" +ANSI_YELLOW = "\033[33m" def main(args): doc_path, product, version, mdx_file, html_file, cover_file, pdf_file = setup(args) - print(f"{ANSI_BLUE}building {pdf_file}{ANSI_STOP}") - files = list_files(doc_path) if len(files) == 0: - raise Exception(f"no files in {doc_path}") + print(f"{ANSI_YELLOW}skipping {pdf_file}{ANSI_STOP}") + return + + print(f"{ANSI_BLUE}building {pdf_file}{ANSI_STOP}") pattern = re.compile('div id="(.*?)" class="registered_link"') for elem in files: @@ -91,11 +94,10 @@ def main(args): run(["open", html_file]) else: print("generating cover page") - with open(BASE_DIR / "cover.html") as source, open(cover_file, "w") as output: - data = source.read() - data = data.replace("[PRODUCT]", title) - data = data.replace("[VERSION]", version) - output.write(data) + data = (BASE_DIR / "cover.html").read_text() + data = data.replace("[PRODUCT]", title) + data = data.replace("[VERSION]", version) + cover_file.write_text(data) header_footer_common = [ "--header-font-name", @@ -202,11 +204,15 @@ def filter_path(path): if path.is_dir() and not path.match("*images*"): return True elif path.suffix in [".mdx", ".md"]: - with open(path) as f: - content = re.sub( - "^---$.*?^---$", "", f.read(), flags=re.DOTALL | re.MULTILINE - ).strip() - return False if content == "" else True + content = re.sub( + "^---$.*?^---$", "", path.read_text(), flags=re.DOTALL | re.MULTILINE + ).strip() + + no_content = content == "" or ( + len(content.split(os.linesep)) == 1 and "StubCards" in content + ) + + return False if no_content else True else: return False From fee04719d4d0d39b1bf00fc5f94dd891ef3c23e1 Mon Sep 17 00:00:00 2001 From: George Song Date: Wed, 16 Jun 2021 13:46:30 -0700 Subject: [PATCH 12/21] test: add tests for generate_pdf.list_files function Former-commit-id: 25f0dfc223095533baf3495cf583ef4f66745f49 --- .gitignore | 3 +- pytest.ini | 4 +++ requirements.in | 2 ++ requirements.txt | 33 +++++++++++++++++++ scripts/pdf/__init__.py | 0 scripts/pdf/__tests__/__init__.py | 0 .../empty_root_index_1/1/1.1.mdx | 5 +++ .../empty_root_index_1/1/1.2/1.2.1.mdx | 5 +++ .../empty_root_index_1/1/1.2/index.mdx | 5 +++ .../empty_root_index_1/1/index.mdx | 5 +++ .../empty_root_index_1/2/2.1.mdx | 5 +++ .../empty_root_index_1/2/index.mdx | 5 +++ .../empty_root_index_1/3.mdx | 5 +++ .../empty_root_index_1/expected.py | 13 ++++++++ .../empty_root_index_1/index.mdx | 3 ++ .../frontmatter_only/expected.py | 1 + .../frontmatter_only/index.mdx | 3 ++ .../index_only/expected.py | 5 +++ .../index_only/index.mdx | 5 +++ .../stubcards/expected.py | 1 + .../list_files_test_cases/stubcards/index.mdx | 5 +++ .../__tests__/test_generate_pdf_list_files.py | 31 +++++++++++++++++ scripts/pdf/generate_pdf.py | 8 ++--- 23 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 pytest.ini create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 scripts/pdf/__init__.py create mode 100644 scripts/pdf/__tests__/__init__.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/1.2.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/2.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/3.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_only/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/stubcards/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/stubcards/index.mdx create mode 100644 scripts/pdf/__tests__/test_generate_pdf_list_files.py diff --git a/.gitignore b/.gitignore index 2df3774fd5b..373a0eb8c9b 100644 --- a/.gitignore +++ b/.gitignore @@ -74,9 +74,10 @@ yarn-error.log __pycache__ # Project specific +.project +advocacy_docs/kubernetes/cloud_native_postgresql/*.md.in dev-sources.json product_docs/content/ product_docs/content_build/ static/nginx_redirects.generated temp_kubernetes/ -advocacy_docs/kubernetes/cloud_native_postgresql/*.md.in diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..107c2431f97 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +minversion = 6.0 +testpaths = + scripts diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000000..d84f59f5def --- /dev/null +++ b/requirements.in @@ -0,0 +1,2 @@ +pip-tools +pytest diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000000..60eea22a50d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,33 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile +# +attrs==21.2.0 + # via pytest +click==8.0.1 + # via pip-tools +iniconfig==1.1.1 + # via pytest +packaging==20.9 + # via pytest +pep517==0.10.0 + # via pip-tools +pip-tools==6.1.0 + # via -r requirements.in +pluggy==0.13.1 + # via pytest +py==1.10.0 + # via pytest +pyparsing==2.4.7 + # via packaging +pytest==6.2.4 + # via -r requirements.in +toml==0.10.2 + # via + # pep517 + # pytest + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/scripts/pdf/__init__.py b/scripts/pdf/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/pdf/__tests__/__init__.py b/scripts/pdf/__tests__/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/1.2.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/1.2.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/1.2.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/2.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/2.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/2.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/3.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/3.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/3.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py new file mode 100644 index 00000000000..4390bed020e --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py @@ -0,0 +1,13 @@ +from pathlib import Path + +from ....generate_pdf import TocItem + +expected = [ + TocItem(filename=Path(__file__).parent / "1/index.mdx", chapter=[1]), + TocItem(filename=Path(__file__).parent / "1/1.1.mdx", chapter=[1, 1]), + TocItem(filename=Path(__file__).parent / "1/1.2/index.mdx", chapter=[1, 2]), + TocItem(filename=Path(__file__).parent / "1/1.2/1.2.1.mdx", chapter=[1, 2, 1]), + TocItem(filename=Path(__file__).parent / "2/index.mdx", chapter=[2]), + TocItem(filename=Path(__file__).parent / "2/2.1.mdx", chapter=[2, 1]), + TocItem(filename=Path(__file__).parent / "3.mdx", chapter=[3]), +] diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/index.mdx new file mode 100644 index 00000000000..d7f0f976790 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/index.mdx @@ -0,0 +1,3 @@ +--- +title: 'Title' +--- diff --git a/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/expected.py b/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/expected.py new file mode 100644 index 00000000000..76a24b8e28c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/expected.py @@ -0,0 +1 @@ +expected = [] diff --git a/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/index.mdx new file mode 100644 index 00000000000..d7f0f976790 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/index.mdx @@ -0,0 +1,3 @@ +--- +title: 'Title' +--- diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py b/scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py new file mode 100644 index 00000000000..f1c074ebce3 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py @@ -0,0 +1,5 @@ +from pathlib import Path + +from ....generate_pdf import TocItem + +expected = [TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1])] diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_only/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_only/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_only/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/stubcards/expected.py b/scripts/pdf/__tests__/list_files_test_cases/stubcards/expected.py new file mode 100644 index 00000000000..76a24b8e28c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/stubcards/expected.py @@ -0,0 +1 @@ +expected = [] diff --git a/scripts/pdf/__tests__/list_files_test_cases/stubcards/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/stubcards/index.mdx new file mode 100644 index 00000000000..58ec2e80c29 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/stubcards/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + + diff --git a/scripts/pdf/__tests__/test_generate_pdf_list_files.py b/scripts/pdf/__tests__/test_generate_pdf_list_files.py new file mode 100644 index 00000000000..f50771c3077 --- /dev/null +++ b/scripts/pdf/__tests__/test_generate_pdf_list_files.py @@ -0,0 +1,31 @@ +import importlib +from collections import namedtuple +from pathlib import Path +from tempfile import mkdtemp + +import pytest + +from ..generate_pdf import list_files + +base = Path(__file__).parent +base_package = ".".join(__name__.split(".")[:-1]) +Scenario = namedtuple("Scenario", ("path", "expected_module")) + + +def test_empty(): + tmp_path = Path(mkdtemp()) + assert list_files(tmp_path) == [] + tmp_path.rmdir() + + +@pytest.fixture(params=(base / "list_files_test_cases").iterdir()) +def test_case(request): + expected_module = ".".join( + ["", *str(request.param.relative_to(base)).split("/"), "expected"] + ) + return Scenario(request.param, expected_module) + + +def test_list_files(test_case): + expected = importlib.import_module(test_case.expected_module, base_package) + assert list_files(test_case.path) == expected.expected diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index feb0169e253..5bcf5730b6f 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -186,12 +186,12 @@ def list_files(doc_path, chapter=None): ) for i, entry in enumerate(directory_contents): - if entry.is_dir(): - chapter = chapter if i == 0 else advance_chapter(chapter) - all_files += list_files(entry, chapter) - else: + if entry.is_file(): chapter = [*chapter, 0] if i == 0 else advance_chapter(chapter) all_files.append(TocItem(filename=entry, chapter=chapter)) + else: + chapter = chapter if i == 0 else advance_chapter(chapter) + all_files += list_files(entry, chapter) return all_files From 3b191b0b7464447ae7d853842e8fde3744daf288 Mon Sep 17 00:00:00 2001 From: George Song Date: Mon, 21 Jun 2021 15:21:27 -0700 Subject: [PATCH 13/21] feat: number top level peer chapters to index.mdx properly Addresses #1290. Former-commit-id: b52fabdf5e4195f4417cd2271e2ee167a956cda1 --- .../empty_root_index_2/1.mdx | 5 +++++ .../empty_root_index_2/2/2.1.mdx | 5 +++++ .../empty_root_index_2/2/index.mdx | 5 +++++ .../empty_root_index_2/3.mdx | 5 +++++ .../empty_root_index_2/expected.py | 10 ++++++++++ .../empty_root_index_2/index.mdx | 3 +++ .../index_and_peers_1/1.mdx | 5 +++++ .../index_and_peers_1/2/2.1.mdx | 5 +++++ .../index_and_peers_1/2/index.mdx | 5 +++++ .../index_and_peers_1/3.mdx | 5 +++++ .../index_and_peers_1/expected.py | 11 +++++++++++ .../index_and_peers_1/index.mdx | 5 +++++ .../index_and_peers_2/1/1.1.mdx | 5 +++++ .../index_and_peers_2/1/index.mdx | 5 +++++ .../index_and_peers_2/2.mdx | 5 +++++ .../index_and_peers_2/3/3.1.mdx | 5 +++++ .../index_and_peers_2/3/index.mdx | 5 +++++ .../index_and_peers_2/expected.py | 12 ++++++++++++ .../index_and_peers_2/index.mdx | 5 +++++ scripts/pdf/generate_pdf.py | 16 ++++++++++++---- 20 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/2.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/3.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/2.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/3.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/1.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/2.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/3.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/2.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/2.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/2.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/3.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/3.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/3.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py new file mode 100644 index 00000000000..9701a1a4723 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py @@ -0,0 +1,10 @@ +from pathlib import Path + +from ....generate_pdf import TocItem + +expected = [ + TocItem(filename=Path(__file__).parent / "1.mdx", chapter=[1]), + TocItem(filename=Path(__file__).parent / "2/index.mdx", chapter=[2]), + TocItem(filename=Path(__file__).parent / "2/2.1.mdx", chapter=[2, 1]), + TocItem(filename=Path(__file__).parent / "3.mdx", chapter=[3]), +] diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/index.mdx new file mode 100644 index 00000000000..d7f0f976790 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/index.mdx @@ -0,0 +1,3 @@ +--- +title: 'Title' +--- diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/1.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/2.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/2.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/2.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/3.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/3.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/3.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py new file mode 100644 index 00000000000..ab6cd1cf92b --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py @@ -0,0 +1,11 @@ +from pathlib import Path + +from ....generate_pdf import TocItem + +expected = [ + TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1]), + TocItem(filename=Path(__file__).parent / "1.mdx", chapter=[2]), + TocItem(filename=Path(__file__).parent / "2/index.mdx", chapter=[3]), + TocItem(filename=Path(__file__).parent / "2/2.1.mdx", chapter=[3, 1]), + TocItem(filename=Path(__file__).parent / "3.mdx", chapter=[4]), +] diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/1.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/1.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/1.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/2.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/2.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/2.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/3.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/3.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/3.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py new file mode 100644 index 00000000000..7f931649948 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py @@ -0,0 +1,12 @@ +from pathlib import Path + +from ....generate_pdf import TocItem + +expected = [ + TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1]), + TocItem(filename=Path(__file__).parent / "1/index.mdx", chapter=[2]), + TocItem(filename=Path(__file__).parent / "1/1.1.mdx", chapter=[2, 1]), + TocItem(filename=Path(__file__).parent / "2.mdx", chapter=[3]), + TocItem(filename=Path(__file__).parent / "3/index.mdx", chapter=[4]), + TocItem(filename=Path(__file__).parent / "3/3.1.mdx", chapter=[4, 1]), +] diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 5bcf5730b6f..53706f7d330 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -178,7 +178,7 @@ def setup(args): return doc_path, product, version, mdx_file, html_file, cover_file, pdf_file -def list_files(doc_path, chapter=None): +def list_files(doc_path, chapter=None, is_root_dir=True): chapter = [1] if chapter is None else chapter all_files = [] directory_contents = sorted( @@ -186,12 +186,20 @@ def list_files(doc_path, chapter=None): ) for i, entry in enumerate(directory_contents): + chapter = ( + advance_chapter( + chapter[:-1] if is_root_dir and len(chapter) > 1 else chapter + ) + if i != 0 + else [*chapter, 0] + if entry.is_file() + else chapter + ) + if entry.is_file(): - chapter = [*chapter, 0] if i == 0 else advance_chapter(chapter) all_files.append(TocItem(filename=entry, chapter=chapter)) else: - chapter = chapter if i == 0 else advance_chapter(chapter) - all_files += list_files(entry, chapter) + all_files += list_files(entry, chapter, is_root_dir=False) return all_files From 79023d43b5e4f78aa93f6aa8351828e9445400ce Mon Sep 17 00:00:00 2001 From: George Song Date: Mon, 21 Jun 2021 17:52:47 -0700 Subject: [PATCH 14/21] feat: add option to generate raw PDF text for comparison Former-commit-id: affd4d95a6e15c226bfb5c4b5f8f21653c28af5e --- requirements.in | 1 + requirements.txt | 17 +++++++++------ scripts/pdf/compare_pdfs.py | 41 +++++++++++++++++++++++++++++++------ 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/requirements.in b/requirements.in index d84f59f5def..2a18e84a1fb 100644 --- a/requirements.in +++ b/requirements.in @@ -1,2 +1,3 @@ +pdftotext pip-tools pytest diff --git a/requirements.txt b/requirements.txt index 60eea22a50d..ae91e756ca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,11 +10,13 @@ click==8.0.1 # via pip-tools iniconfig==1.1.1 # via pytest -packaging==20.9 +packaging==21.0 # via pytest -pep517==0.10.0 +pdftotext==2.1.6 + # via -r requirements.in +pep517==0.11.0 # via pip-tools -pip-tools==6.1.0 +pip-tools==6.2.0 # via -r requirements.in pluggy==0.13.1 # via pytest @@ -25,9 +27,12 @@ pyparsing==2.4.7 pytest==6.2.4 # via -r requirements.in toml==0.10.2 - # via - # pep517 - # pytest + # via pytest +tomli==1.0.4 + # via pep517 +wheel==0.36.2 + # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip +# setuptools diff --git a/scripts/pdf/compare_pdfs.py b/scripts/pdf/compare_pdfs.py index bcf1e6297b5..f960091b715 100755 --- a/scripts/pdf/compare_pdfs.py +++ b/scripts/pdf/compare_pdfs.py @@ -6,6 +6,8 @@ from itertools import zip_longest from pathlib import Path +import pdftotext + def main(args): pdfs = sorted( @@ -14,12 +16,13 @@ def main(args): args.removed_pdfs = args.output_dir / "00_removed_pdfs.txt" args.removed_pdfs.unlink(missing_ok=True) with multiprocessing.Pool(int(multiprocessing.cpu_count() * 2 / 3)) as pool: - pool.starmap(diff_pdfs, zip_longest(pdfs, [], fillvalue=args), chunksize=1) + pool.starmap(process_pdf, zip_longest(pdfs, [], fillvalue=args), chunksize=1) - print(f'See removed PDFs in "{args.removed_pdfs}"') + if args.removed_pdfs.exists(): + print(f'See removed PDFs in "{args.removed_pdfs}"') -def diff_pdfs(pdf1, args): +def process_pdf(pdf1, args): pdf2 = args.dir2 / pdf1.name if not pdf2.is_file(): @@ -28,11 +31,28 @@ def diff_pdfs(pdf1, args): print(pdf1.name, file=of) return + print(f"👷‍♀️ {pdf1.name}") + if args.should_output_text: + generate_pdf_text(pdf1, pdf2, args) + else: + diff_pdfs(pdf1, pdf2, args) + print(f"✅ {pdf1.name}") + + +def generate_pdf_text(pdf1, pdf2, args): + for i, pdf in enumerate([pdf1, pdf2], start=1): + output = args.output_dir / f"dir{i}" / f"{pdf.stem}.txt" + with open(pdf, "rb") as f: + content = pdftotext.PDF(f) + output.parent.mkdir(exist_ok=True) + with open(output, "w+") as of: + of.write("\n".join(content)) + + +def diff_pdfs(pdf1, pdf2, args): diff = args.output_dir / pdf1.name cmd = shlex.split(f"diff-pdf -s -m -g --output-diff={diff} {pdf1} {pdf2}") - print(f"👷‍♀️ {pdf1.name}") subprocess.run(cmd, capture_output=True) - print(f"✅ {pdf1.name}") def cli(): @@ -45,7 +65,16 @@ def cli(): parser.add_argument("dir1", type=path_type, help="first directory of PDFs") parser.add_argument("dir2", type=path_type, help="second directory of PDFs") parser.add_argument( - "output_dir", type=output_type, help="directory to store diff PDFs" + "output_dir", type=output_type, help="directory to store visual PDF diffs" + ) + parser.add_argument( + "--text", + dest="should_output_text", + action="store_true", + help=( + "generate raw text (instead of visual diffs) " + "from source PDFs for manual diff-ing" + ), ) return parser.parse_args() From 810b15fdad9986d81d620d4a0a42b028e136abef Mon Sep 17 00:00:00 2001 From: Josh Heyer <63653723+josh-heyer@users.noreply.github.com> Date: Tue, 22 Jun 2021 21:59:42 +0000 Subject: [PATCH 15/21] test: add case for empty chapter index.mdx Former-commit-id: 07f594c560201f65acf8e31812465168d7c35cea --- .../list_files_test_cases/empty_sub_index/1/0.mdx | 5 +++++ .../list_files_test_cases/empty_sub_index/1/1.1.mdx | 5 +++++ .../list_files_test_cases/empty_sub_index/1/1.2.mdx | 5 +++++ .../empty_sub_index/1/index.mdx | 3 +++ .../list_files_test_cases/empty_sub_index/2.mdx | 5 +++++ .../list_files_test_cases/empty_sub_index/3/3.1.mdx | 5 +++++ .../empty_sub_index/3/index.mdx | 5 +++++ .../empty_sub_index/expected.py | 13 +++++++++++++ .../list_files_test_cases/empty_sub_index/index.mdx | 5 +++++ 9 files changed, 51 insertions(+) create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/0.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.2.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/2.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/3.1.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/index.mdx create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py create mode 100644 scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/0.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/0.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/0.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.2.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.2.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.2.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/index.mdx new file mode 100644 index 00000000000..d7f0f976790 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/index.mdx @@ -0,0 +1,3 @@ +--- +title: 'Title' +--- diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/2.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/2.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/2.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/3.1.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/3.1.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/3.1.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py new file mode 100644 index 00000000000..ba675a92746 --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py @@ -0,0 +1,13 @@ +from pathlib import Path + +from ....generate_pdf import TocItem + +expected = [ + TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1]), + TocItem(filename=Path(__file__).parent / "1/0.mdx", chapter=[2]), + TocItem(filename=Path(__file__).parent / "1/1.1.mdx", chapter=[2, 1]), + TocItem(filename=Path(__file__).parent / "1/1.2.mdx", chapter=[2, 2]), + TocItem(filename=Path(__file__).parent / "2.mdx", chapter=[3]), + TocItem(filename=Path(__file__).parent / "3/index.mdx", chapter=[4]), + TocItem(filename=Path(__file__).parent / "3/3.1.mdx", chapter=[4, 1]), +] diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/index.mdx b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/index.mdx new file mode 100644 index 00000000000..42bda5c006c --- /dev/null +++ b/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/index.mdx @@ -0,0 +1,5 @@ +--- +title: 'Title' +--- + +# Title From ec985b8a7f5b3663f04b774bb006c14cd9d1160d Mon Sep 17 00:00:00 2001 From: George Song Date: Tue, 29 Jun 2021 11:53:25 -0700 Subject: [PATCH 16/21] test: rearrange test files Former-commit-id: 73b14da0fde46b50e497f75d2f804a9f5383fd2a --- scripts/pdf/__tests__/list_files/__init__.py | 0 .../test_cases}/empty_root_index_1/1/1.1.mdx | 0 .../test_cases}/empty_root_index_1/1/1.2/1.2.1.mdx | 0 .../test_cases}/empty_root_index_1/1/1.2/index.mdx | 0 .../test_cases}/empty_root_index_1/1/index.mdx | 0 .../test_cases}/empty_root_index_1/2/2.1.mdx | 0 .../test_cases}/empty_root_index_1/2/index.mdx | 0 .../test_cases}/empty_root_index_1/3.mdx | 0 .../test_cases}/empty_root_index_1/expected.py | 2 +- .../test_cases}/empty_root_index_1/index.mdx | 0 .../test_cases}/empty_root_index_2/1.mdx | 0 .../test_cases}/empty_root_index_2/2/2.1.mdx | 0 .../test_cases}/empty_root_index_2/2/index.mdx | 0 .../test_cases}/empty_root_index_2/3.mdx | 0 .../test_cases}/empty_root_index_2/expected.py | 2 +- .../test_cases}/empty_root_index_2/index.mdx | 0 .../test_cases}/empty_sub_index/1/0.mdx | 0 .../test_cases}/empty_sub_index/1/1.1.mdx | 0 .../test_cases}/empty_sub_index/1/1.2.mdx | 0 .../test_cases}/empty_sub_index/1/index.mdx | 0 .../test_cases}/empty_sub_index/2.mdx | 0 .../test_cases}/empty_sub_index/3/3.1.mdx | 0 .../test_cases}/empty_sub_index/3/index.mdx | 0 .../test_cases}/empty_sub_index/expected.py | 2 +- .../test_cases}/empty_sub_index/index.mdx | 0 .../test_cases}/frontmatter_only/expected.py | 0 .../test_cases}/frontmatter_only/index.mdx | 0 .../test_cases}/index_and_peers_1/1.mdx | 0 .../test_cases}/index_and_peers_1/2/2.1.mdx | 0 .../test_cases}/index_and_peers_1/2/index.mdx | 0 .../test_cases}/index_and_peers_1/3.mdx | 0 .../test_cases}/index_and_peers_1/expected.py | 2 +- .../test_cases}/index_and_peers_1/index.mdx | 0 .../test_cases}/index_and_peers_2/1/1.1.mdx | 0 .../test_cases}/index_and_peers_2/1/index.mdx | 0 .../test_cases}/index_and_peers_2/2.mdx | 0 .../test_cases}/index_and_peers_2/3/3.1.mdx | 0 .../test_cases}/index_and_peers_2/3/index.mdx | 0 .../test_cases}/index_and_peers_2/expected.py | 2 +- .../test_cases}/index_and_peers_2/index.mdx | 0 .../test_cases}/index_only/expected.py | 2 +- .../test_cases}/index_only/index.mdx | 0 .../test_cases}/stubcards/expected.py | 0 .../test_cases}/stubcards/index.mdx | 0 .../{ => list_files}/test_generate_pdf_list_files.py | 4 ++-- 45 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 scripts/pdf/__tests__/list_files/__init__.py rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/1/1.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/1/1.2/1.2.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/1/1.2/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/1/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/2/2.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/2/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/3.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/expected.py (93%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_1/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_2/1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_2/2/2.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_2/2/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_2/3.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_2/expected.py (89%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_root_index_2/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/1/0.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/1/1.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/1/1.2.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/1/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/2.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/3/3.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/3/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/expected.py (93%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/empty_sub_index/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/frontmatter_only/expected.py (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/frontmatter_only/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_1/1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_1/2/2.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_1/2/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_1/3.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_1/expected.py (91%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_1/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/1/1.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/1/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/2.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/3/3.1.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/3/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/expected.py (92%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_and_peers_2/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_only/expected.py (73%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/index_only/index.mdx (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/stubcards/expected.py (100%) rename scripts/pdf/__tests__/{list_files_test_cases => list_files/test_cases}/stubcards/index.mdx (100%) rename scripts/pdf/__tests__/{ => list_files}/test_generate_pdf_list_files.py (87%) diff --git a/scripts/pdf/__tests__/list_files/__init__.py b/scripts/pdf/__tests__/list_files/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/1.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/1.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/1.2.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/1.2/1.2.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/1.2.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/1.2/1.2.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/1.2/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/1.2/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/1.2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/1/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/1/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/2.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/2/2.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/2.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/2/2.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/2/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/2/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/3.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/3.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/3.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/3.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/expected.py similarity index 93% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/expected.py index 4390bed020e..adc6443a7b9 100644 --- a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/expected.py +++ b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/expected.py @@ -1,6 +1,6 @@ from pathlib import Path -from ....generate_pdf import TocItem +from .....generate_pdf import TocItem expected = [ TocItem(filename=Path(__file__).parent / "1/index.mdx", chapter=[1]), diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_1/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_1/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/2.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/2/2.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/2.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/2/2.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/2/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/2/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/3.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/3.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/3.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/3.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/expected.py similarity index 89% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/expected.py index 9701a1a4723..8a57e85441c 100644 --- a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/expected.py +++ b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/expected.py @@ -1,6 +1,6 @@ from pathlib import Path -from ....generate_pdf import TocItem +from .....generate_pdf import TocItem expected = [ TocItem(filename=Path(__file__).parent / "1.mdx", chapter=[1]), diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_root_index_2/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_root_index_2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/0.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/0.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/0.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/0.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/1.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/1.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.2.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/1.2.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/1.2.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/1.2.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/1/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/1/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/2.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/2.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/2.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/2.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/3.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/3/3.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/3.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/3/3.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/3/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/3/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/3/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/expected.py similarity index 93% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/expected.py index ba675a92746..6def05f7ae5 100644 --- a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/expected.py +++ b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/expected.py @@ -1,6 +1,6 @@ from pathlib import Path -from ....generate_pdf import TocItem +from .....generate_pdf import TocItem expected = [ TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1]), diff --git a/scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/empty_sub_index/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/empty_sub_index/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/expected.py b/scripts/pdf/__tests__/list_files/test_cases/frontmatter_only/expected.py similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/frontmatter_only/expected.py diff --git a/scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/frontmatter_only/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/frontmatter_only/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/frontmatter_only/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/1.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/2.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/2/2.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/2.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/2/2.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/2/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/2/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/3.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/3.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/3.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/3.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/expected.py similarity index 91% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/expected.py index ab6cd1cf92b..f873406abcc 100644 --- a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/expected.py +++ b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/expected.py @@ -1,6 +1,6 @@ from pathlib import Path -from ....generate_pdf import TocItem +from .....generate_pdf import TocItem expected = [ TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1]), diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_1/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_1/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/1.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/1/1.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/1.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/1/1.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/1/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/1/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/1/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/2.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/2.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/2.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/2.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/3.1.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/3/3.1.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/3.1.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/3/3.1.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/3/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/3/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/3/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/expected.py similarity index 92% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/expected.py index 7f931649948..d05a204dc86 100644 --- a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/expected.py +++ b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/expected.py @@ -1,6 +1,6 @@ from pathlib import Path -from ....generate_pdf import TocItem +from .....generate_pdf import TocItem expected = [ TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1]), diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_and_peers_2/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_and_peers_2/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py b/scripts/pdf/__tests__/list_files/test_cases/index_only/expected.py similarity index 73% rename from scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/index_only/expected.py index f1c074ebce3..e4f075667be 100644 --- a/scripts/pdf/__tests__/list_files_test_cases/index_only/expected.py +++ b/scripts/pdf/__tests__/list_files/test_cases/index_only/expected.py @@ -1,5 +1,5 @@ from pathlib import Path -from ....generate_pdf import TocItem +from .....generate_pdf import TocItem expected = [TocItem(filename=Path(__file__).parent / "index.mdx", chapter=[1])] diff --git a/scripts/pdf/__tests__/list_files_test_cases/index_only/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/index_only/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/index_only/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/index_only/index.mdx diff --git a/scripts/pdf/__tests__/list_files_test_cases/stubcards/expected.py b/scripts/pdf/__tests__/list_files/test_cases/stubcards/expected.py similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/stubcards/expected.py rename to scripts/pdf/__tests__/list_files/test_cases/stubcards/expected.py diff --git a/scripts/pdf/__tests__/list_files_test_cases/stubcards/index.mdx b/scripts/pdf/__tests__/list_files/test_cases/stubcards/index.mdx similarity index 100% rename from scripts/pdf/__tests__/list_files_test_cases/stubcards/index.mdx rename to scripts/pdf/__tests__/list_files/test_cases/stubcards/index.mdx diff --git a/scripts/pdf/__tests__/test_generate_pdf_list_files.py b/scripts/pdf/__tests__/list_files/test_generate_pdf_list_files.py similarity index 87% rename from scripts/pdf/__tests__/test_generate_pdf_list_files.py rename to scripts/pdf/__tests__/list_files/test_generate_pdf_list_files.py index f50771c3077..39c21fbf17c 100644 --- a/scripts/pdf/__tests__/test_generate_pdf_list_files.py +++ b/scripts/pdf/__tests__/list_files/test_generate_pdf_list_files.py @@ -5,7 +5,7 @@ import pytest -from ..generate_pdf import list_files +from ...generate_pdf import list_files base = Path(__file__).parent base_package = ".".join(__name__.split(".")[:-1]) @@ -18,7 +18,7 @@ def test_empty(): tmp_path.rmdir() -@pytest.fixture(params=(base / "list_files_test_cases").iterdir()) +@pytest.fixture(params=(base / "test_cases").iterdir()) def test_case(request): expected_module = ".".join( ["", *str(request.param.relative_to(base)).split("/"), "expected"] From df1fef0a96f37a38489f4c6f09bbbad2d170fced Mon Sep 17 00:00:00 2001 From: George Song Date: Wed, 23 Jun 2021 19:10:30 -0700 Subject: [PATCH 17/21] refactor: remove unnecessary code TocItem.title and anchor are never used. Former-commit-id: ddd4fc2ae169b512e75b663a252a14b2def879d7 --- scripts/pdf/generate_pdf.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 53706f7d330..3e61e62ac5f 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -25,17 +25,6 @@ def main(args): print(f"{ANSI_BLUE}building {pdf_file}{ANSI_STOP}") - pattern = re.compile('div id="(.*?)" class="registered_link"') - for elem in files: - with open(elem.filename, "r") as g: - for line in g: - if "title: " in line: - elem.title = line[7:].replace('"', "").replace("\n", "") - - tag = pattern.search(line) - if tag and len(elem.anchor) == 0: - elem.anchor = tag.group(1) - resource_search_paths = {doc_path} # Print the files @@ -247,8 +236,6 @@ def strip_quotes(str): class TocItem: filename: Path chapter: List[int] - title: str = "" - anchor: str = "" def __post_init__(self): self.chapter = ".".join((str(c) for c in self.chapter if c > 0)) From 64df02269acfdae52dfafba9c124cc8613f25045 Mon Sep 17 00:00:00 2001 From: George Song Date: Tue, 29 Jun 2021 11:55:22 -0700 Subject: [PATCH 18/21] test: combine_mdx Former-commit-id: f3296040019f7462e6e0caf2aaf4bfc74b161d0f --- scripts/pdf/__tests__/combine_mdx/__init__.py | 0 .../pdf/__tests__/combine_mdx/expected.mdx | 29 +++++++++++++++ scripts/pdf/__tests__/combine_mdx/src/1.mdx | 37 +++++++++++++++++++ scripts/pdf/__tests__/combine_mdx/src/2.mdx | 15 ++++++++ .../test_generate_pdf_combin_mdx.py | 15 ++++++++ 5 files changed, 96 insertions(+) create mode 100644 scripts/pdf/__tests__/combine_mdx/__init__.py create mode 100644 scripts/pdf/__tests__/combine_mdx/expected.mdx create mode 100644 scripts/pdf/__tests__/combine_mdx/src/1.mdx create mode 100644 scripts/pdf/__tests__/combine_mdx/src/2.mdx create mode 100644 scripts/pdf/__tests__/combine_mdx/test_generate_pdf_combin_mdx.py diff --git a/scripts/pdf/__tests__/combine_mdx/__init__.py b/scripts/pdf/__tests__/combine_mdx/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/pdf/__tests__/combine_mdx/expected.mdx b/scripts/pdf/__tests__/combine_mdx/expected.mdx new file mode 100644 index 00000000000..1b822ee2768 --- /dev/null +++ b/scripts/pdf/__tests__/combine_mdx/expected.mdx @@ -0,0 +1,29 @@ +--- +1          Title for 1.mdx +--- + +1.mdx content + +## Heading 1 + +### Heading 2 + +### Heading 3 + +#### Heading 4 + + +--- +2          Title for 2.mdx +--- + +2.mdx content + +## Heading 1 + +### Heading 2 + +### Heading 3 + +#### Heading 4 + diff --git a/scripts/pdf/__tests__/combine_mdx/src/1.mdx b/scripts/pdf/__tests__/combine_mdx/src/1.mdx new file mode 100644 index 00000000000..a2908e8eea5 --- /dev/null +++ b/scripts/pdf/__tests__/combine_mdx/src/1.mdx @@ -0,0 +1,37 @@ +--- +Something to be ignored +title: Title for 1.mdx +Some other thing to be ignored +--- + +1.mdx content + +# Heading 1 + +## Heading 2 + +### Heading 3 + +#### Heading 4 + +
+ introduction alter_directory alter_index alter_procedure alter_profile + alter_queue alter_queue_table alter_role_identified_by + alter_role_managing_database_link_and_dbms_rls_privileges alter_sequence + alter_session alter_table alter_trigger alter_tablespace + alter_user_identified_by alter_user_role_profile_management_clauses call + comment commit create_database create_public_database_link create_directory + create_function create_index create_materialized_view create_package + create_package_body create_procedure create_profile create_queue + create_queue_table create_role create_schema create_sequence create_synonym + create_table create_table_as create_trigger create_type create_type_body + create_user create_user_role_profile_management_clauses create_view delete + drop_public_database_link drop_directory drop_function drop_index drop_package + drop_procedure drop_profile drop_queue drop_queue_table drop_synonym drop_role + drop_sequence drop_table drop_tablespace drop_trigger drop_type drop_user + drop_view exec grant insert lock revoke rollback rollback_to_savepoint + savepoint select set_constraints set_role set_transaction truncate update + conclusion +
+ +This is ignored as well. diff --git a/scripts/pdf/__tests__/combine_mdx/src/2.mdx b/scripts/pdf/__tests__/combine_mdx/src/2.mdx new file mode 100644 index 00000000000..ee89500e8bf --- /dev/null +++ b/scripts/pdf/__tests__/combine_mdx/src/2.mdx @@ -0,0 +1,15 @@ +--- +Something to be ignored +title: Title for 2.mdx +Some other thing to be ignored +--- + +2.mdx content + +# Heading 1 + +## Heading 2 + +### Heading 3 + +#### Heading 4 diff --git a/scripts/pdf/__tests__/combine_mdx/test_generate_pdf_combin_mdx.py b/scripts/pdf/__tests__/combine_mdx/test_generate_pdf_combin_mdx.py new file mode 100644 index 00000000000..1f53700d478 --- /dev/null +++ b/scripts/pdf/__tests__/combine_mdx/test_generate_pdf_combin_mdx.py @@ -0,0 +1,15 @@ +from io import StringIO +from pathlib import Path + +from ...generate_pdf import TocItem, combine_mdx + +base = Path(__file__).parent + + +def test_combine_mdx(): + src = base / "src" + files = [TocItem(p, [i]) for i, p in enumerate((src).glob("**/*.mdx"), start=1)] + output = StringIO() + resource_search_paths = combine_mdx(files, output) + assert output.getvalue() == (base / "expected.mdx").read_text() + assert resource_search_paths == {src} From b194d10c51f86bf06adbbb8eb5e784825452959b Mon Sep 17 00:00:00 2001 From: George Song Date: Thu, 24 Jun 2021 08:25:01 -0700 Subject: [PATCH 19/21] refactor: abstract combine_mdx into its own function Former-commit-id: 81fea9918dbe5da789ab06fc488da6f93f749a84 --- scripts/pdf/generate_pdf.py | 88 ++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index 3e61e62ac5f..e48d52b2b05 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import re from argparse import ArgumentParser, ArgumentTypeError from dataclasses import dataclass from datetime import datetime @@ -24,40 +23,10 @@ def main(args): return print(f"{ANSI_BLUE}building {pdf_file}{ANSI_STOP}") + with open(mdx_file, "w") as output: + resource_search_paths = {doc_path, *combine_mdx(files, output)} - resource_search_paths = {doc_path} - - # Print the files - with open(mdx_file, "w") as fp: - for elem in files: - with open(elem.filename, "r") as g: - resource_search_paths.add(elem.filename.parent) - - front_matter_count = 2 - for line in g: - new_line = line - - if line[0:2] == "# ": - new_line = "##" + line - if line[0:3] == "## ": - new_line = "#" + line - if "toctree" in line: - front_matter_count = 3 - if front_matter_count == 0: - fp.write(new_line) - if "title: " in line: - new_line = ( - re.sub(r"\.0", "", elem.chapter) - + (" " * 10) - + strip_quotes(line[7:]) - + "\n" - ) - fp.write(new_line) - if "---" in line and front_matter_count > 0: - front_matter_count -= 1 - fp.write(new_line) - fp.write("\n") - + # TODO: Pick up refactoring from this point title = get_title(doc_path) or product print("generating docs html") @@ -193,6 +162,44 @@ def list_files(doc_path, chapter=None, is_root_dir=True): return all_files +def combine_mdx(files, output): + resource_search_paths = set() + + for item in files: + resource_search_paths.add(item.filename.parent) + front_matter, content = parse_mdx(item.filename) + write_front_matter(front_matter, item.chapter, output) + write_content(content, output) + + return resource_search_paths + + +def write_front_matter(front_matter, chapter, output): + title_marker = "title: " + + print("---", file=output) + + for line in [ + line for line in front_matter.split(os.linesep) if title_marker in line + ]: + title = strip_quotes(line.replace(title_marker, "")) + print(f'{chapter}{" " * 10}{title}', file=output) + + print("---", file=output) + output.write(os.linesep) + + +def write_content(content, output): + lines = [] + for line in content.split(os.linesep): + if "toctree" in line: + break + line = f"#{line}" if (line.startswith("# ") or line.startswith("## ")) else line + lines.append(line + os.linesep) + output.writelines(lines) + output.write(os.linesep) + + def advance_chapter(chapter): return [*chapter[:-1], chapter[-1] + 1] @@ -201,19 +208,20 @@ def filter_path(path): if path.is_dir() and not path.match("*images*"): return True elif path.suffix in [".mdx", ".md"]: - content = re.sub( - "^---$.*?^---$", "", path.read_text(), flags=re.DOTALL | re.MULTILINE - ).strip() - + _, content = parse_mdx(path) no_content = content == "" or ( len(content.split(os.linesep)) == 1 and "StubCards" in content ) - - return False if no_content else True + return not no_content else: return False +def parse_mdx(mdx_file): + front_matter, _, content = mdx_file.read_text().partition("---")[2].partition("---") + return front_matter.strip(), content.strip() + + def put_index_first(path): filename = str(path) return filename.replace("index.mdx", "00_index.mdx") From bc143a5cc31869f7b39ee5396630570f0c781735 Mon Sep 17 00:00:00 2001 From: George Song Date: Mon, 19 Jul 2021 12:28:30 -0700 Subject: [PATCH 20/21] feat: indent all headings Former-commit-id: 3fc4a3bb16b76bce0a6ad561f64c8af730934ff2 --- scripts/pdf/generate_pdf.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/pdf/generate_pdf.py b/scripts/pdf/generate_pdf.py index e48d52b2b05..b6e49ff4c32 100755 --- a/scripts/pdf/generate_pdf.py +++ b/scripts/pdf/generate_pdf.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import re from argparse import ArgumentParser, ArgumentTypeError from dataclasses import dataclass from datetime import datetime @@ -189,13 +190,17 @@ def write_front_matter(front_matter, chapter, output): output.write(os.linesep) +RE_MD_HEADER = re.compile(r"^#+\s") + + def write_content(content, output): lines = [] for line in content.split(os.linesep): if "toctree" in line: break - line = f"#{line}" if (line.startswith("# ") or line.startswith("## ")) else line - lines.append(line + os.linesep) + # TODO: Use a more robust mechanism such as remark to handle increasing + # heading depth + lines.append(RE_MD_HEADER.sub(r"#\g<0>", line) + os.linesep) output.writelines(lines) output.write(os.linesep) From 7d436e2bc1f947883a6182f0c9466cde68304f73 Mon Sep 17 00:00:00 2001 From: josh-heyer Date: Thu, 22 Jul 2021 20:38:52 +0000 Subject: [PATCH 21/21] New PDFs generated by Github Actions Former-commit-id: e8638183bdf92eb499b905922acb2e147686be4b