From 5b28d9e3078a60649077b00e6771c2a8fbd1a220 Mon Sep 17 00:00:00 2001 From: silasprincipe <53846571+silasprincipe@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:03:41 +0200 Subject: [PATCH] Add notebooks folder and actions --- .github/scripts/render_and_screenshot.py | 116 ++++++++++++++++++ .../workflows/render_and_update_readme.yml | 42 +++++++ .gitignore | 14 +++ notebooks/README.md | 14 +++ 4 files changed, 186 insertions(+) create mode 100644 .github/scripts/render_and_screenshot.py create mode 100644 .github/workflows/render_and_update_readme.yml create mode 100644 .gitignore create mode 100644 notebooks/README.md diff --git a/.github/scripts/render_and_screenshot.py b/.github/scripts/render_and_screenshot.py new file mode 100644 index 0000000..c9f9a9e --- /dev/null +++ b/.github/scripts/render_and_screenshot.py @@ -0,0 +1,116 @@ +import os +import subprocess +import nbformat +from nbconvert import HTMLExporter +from selenium import webdriver +from webdriver_manager.chrome import ChromeDriverManager +from selenium.webdriver.chrome.options import Options +import time +import re + +# Directories +notebooks_dir = "notebooks" +screenshots_dir = os.path.join(notebooks_dir, "screenshots") + +# Ensure screenshots directory exists +os.makedirs(screenshots_dir, exist_ok=True) + +# Function to render Jupyter Notebook or Quarto file +def render_file(file_path): + if file_path.endswith(".ipynb"): + html_exporter = HTMLExporter() + with open(file_path, "r", encoding="utf-8") as f: + notebook_content = nbformat.read(f, as_version=4) + body, _ = html_exporter.from_notebook_node(notebook_content) + return body + elif file_path.endswith(".qmd"): + subprocess.run(["quarto", "render", file_path, "--no-execute"]) + #subprocess.run(["quarto", "render", file_path]) + html_path = file_path.replace(".qmd", ".html") + with open(html_path, "r", encoding="utf-8") as f: + body = f.read() + os.remove(html_path) # Clean up + return body + +# Function to capture screenshot of HTML content +def capture_screenshot(html_content, output_path): + # options = webdriver.ChromeOptions() + # options.add_argument("--headless") + # options.add_argument("--no-sandbox") + # options.add_argument("--disable-dev-shm-usage") + + # driver = webdriver.Chrome(ChromeDriverManager().install(), options=options) + # Set up Chrome options + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--disable-dev-shm-usage") + + # Ensure the WebDriver is passed the options only once + driver = webdriver.Chrome(options=chrome_options) + + with open("temp.html", "w") as f: + f.write(html_content) + + driver.get(f"file://{os.path.abspath('temp.html')}") + time.sleep(2) # Allow time for rendering + driver.set_window_size(1200, 2000) + driver.save_screenshot(output_path) + driver.quit() + os.remove("temp.html") + +# Function to update README.md +def update_readme(section, file_name, file_path, screenshot_path): + readme_path = os.path.join(notebooks_dir, "README.md") + title = file_name.split(".")[0] # Get title from file name + title = title.replace("_", " ").title() # Replace underscores with spaces and capitalize + screenshot_rel_path = os.path.relpath(screenshot_path, notebooks_dir) + file_rel_path = os.path.relpath(file_path, notebooks_dir) + + # Load the README.md content + with open(readme_path, "r") as f: + content = f.read() + + # Define the pattern to match an existing entry + pattern = re.compile(rf"(## {title}\n\[.*\]\({re.escape(file_rel_path)}\)\n!\[.*\]\(.*\))", re.MULTILINE) + + # Create the new entry text + new_entry = f"## {title}\n[{file_name}]({file_rel_path})\n![{title}]({screenshot_rel_path})\n" + + if pattern.search(content): + # If the entry exists, update it with the new screenshot path + content = pattern.sub(new_entry, content) + else: + # If no entry exists, add a new one under the correct section + if section == "Python": + content = re.sub(r"(# Python\n)", rf"\1{new_entry}\n", content) + elif section == "R": + content = re.sub(r"(# R\n)", rf"\1{new_entry}\n", content) + + # Write the updated content back to README.md + with open(readme_path, "w") as f: + f.write(content) + +# Main function to process all new files +def process_notebooks(): + for root, _, files in os.walk(notebooks_dir): + for file in files: + if file.endswith(".ipynb") or file.endswith(".qmd"): + file_path = os.path.join(root, file) + section = "Python" if "Python" in root else "R" + + print(f"Processing {file_path}...") + + # Render file to HTML + html_content = render_file(file_path) + + # Capture screenshot + screenshot_name = file.replace(".ipynb", ".png").replace(".qmd", ".png") + screenshot_path = os.path.join(screenshots_dir, screenshot_name) + capture_screenshot(html_content, screenshot_path) + + # Update README + update_readme(section, file, file_path, screenshot_path) + +# Run the process +process_notebooks() diff --git a/.github/workflows/render_and_update_readme.yml b/.github/workflows/render_and_update_readme.yml new file mode 100644 index 0000000..e30bfc0 --- /dev/null +++ b/.github/workflows/render_and_update_readme.yml @@ -0,0 +1,42 @@ +name: Render Notebooks and Update README + +on: + push: + paths: + - 'notebooks/R/**' + - 'notebooks/Python/**' + pull_request: + paths: + - 'notebooks/R/**' + - 'notebooks/Python/**' + +jobs: + render_and_update: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install necessary dependencies + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + pip install nbconvert selenium webdriver-manager quarto-cli + + # Render the added/changed files + - name: Render and take screenshots + run: | + python .github/scripts/render_and_screenshot.py + + # Commit changes (screenshots and updated README) + - name: Commit and push changes + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + git add notebooks/screenshots README.md + git commit -m "Auto update README with new files and screenshots" + git push diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d108ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.DS_Store +.Rhistory +*.Rproj +shapes +.Rproj.user +*_storr +shapes_errors +*.sqlite +*.sqlite-journal +*.rds +dev.R +protectedseas.zip +data/ +.venv/ \ No newline at end of file diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 0000000..d3ba6a2 --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,14 @@ +# Notebooks + +Here we present some notebooks covering several topics as OBIS data access, data cleaning, environmental information extraction, and data visualization. + +# R + + + +# Python + + + +------ +_Created by the OBIS secretariat_ \ No newline at end of file