diff --git a/.github/workflows/build-offline.yml b/.github/workflows/build-offline.yml new file mode 100644 index 00000000000..ec0d410e620 --- /dev/null +++ b/.github/workflows/build-offline.yml @@ -0,0 +1,59 @@ +name: Build offline Hedy +on: + # Can be run on-demand + workflow_dispatch: {} + + # Runs when 'deploy to prod' runs + workflow_run: + workflows: ["Deploy to hedy.org"] + types: [requested] + branches: + - 'main' + +jobs: + build: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + cache: 'pip' + - name: Set up NodeJS 18 + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'npm' + - name: 'Install npx' + run: npm install -g npx + - run: pip install -r requirements.txt + name: 'Python requirements' + - run: doit run _offline + - name: Smoke test the build + run: cd dist && offlinehedy/run-hedy-server --smoketest + + - uses: fregante/daily-version-action@v2 + name: Create tag if necessary + id: daily-version + + - name: Create zip file + # Because we're on Windows + run: | + cd dist/offlinehedy && Compress-Archive -Path . -Destination ../../offlinehedy-${{ steps.daily-version.outputs.version }}.zip + + + - if: steps.daily-version.outputs.created + name: Create Release + uses: shogo82148/actions-create-release@v1 + id: create_release + with: + tag_name: ${{ steps.daily-version.outputs.version }} + generate_release_notes: true + + - name: Upload Assets + if: steps.daily-version.outputs.created + uses: shogo82148/actions-upload-release-asset@v1 + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: '*.zip' diff --git a/.gitignore b/.gitignore index 32bf9d22a30..a4ed89ad544 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,6 @@ MANIFEST # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest -*.spec # Installer logs pip-log.txt @@ -179,4 +178,5 @@ grammars-Total/*.lark .test-cache .doit.db -.doit.db.* \ No newline at end of file +.doit.db.* +dist/ diff --git a/OFFLINE_README.txt b/OFFLINE_README.txt new file mode 100644 index 00000000000..2a5250b2957 --- /dev/null +++ b/OFFLINE_README.txt @@ -0,0 +1,56 @@ + _ _ _ + | | | | | | + | |__| | ___ __| |_ _ + | __ |/ _ \/ _` | | | | + | | | | __/ (_| | |_| | + |_| |_|\___|\__,_|\__, | + __/ | + o f f l i n e |___/ + +Welcome to offline Hedy! You can use offline Hedy to run Hedy on your own +computer, and it can be used by anyone on the same network. + + +For you +======= + +When you first start up offline Hedy on a Windows computer, two things can +happen: + +- Windows Firewall will ask you whether or not to allow network connections. + You should click "Allow". +- Windows Defender may say that Hedy is a dangerous program and ask you whether + or not it should be run. You should click "More Info" and then "Run Anyway". + +You can create a teacher account for yourself by visiting the following link +and then clicking "Create Account". + + http://localhost/invite/newteacher + +You can also use one of the built-in accounts, which is named "teacher1" +with password "123456". + + +For students +============ + +When Hedy starts up, it will print a web address made from numbers. Your students +should type this address into the address bar of their browser. + +The address will look something like this, but it will be different on every +computer. It can also change every time your computer starts up: + + http://192.168.31.13/ + + +Upgrading to a newer version +============================ + +All your data and your student's data is stored in the file `database.json` +that you will see in the program's directory. When you download a newer version +of Offline Hedy it will come with its own empty database. To keep all programs +around, you can copy over the `database.json` file from the old version to the +new version of Hedy. + +To keep it simple, you can also start fresh and use the newer version only for a +different class or different year. \ No newline at end of file diff --git a/app.py b/app.py index 69c7a419e71..1e5653124df 100644 --- a/app.py +++ b/app.py @@ -371,9 +371,14 @@ def before_request_https(): Compress(app) Commonmark(app) -parse_logger = s3_logger.S3ParseLogger.from_env_vars() -querylog.LOG_QUEUE.set_transmitter( - aws_helpers.s3_querylog_transmitter_from_env()) + +# We don't need to log in offline mode +if utils.is_offline_mode(): + parse_logger = s3_logger.NullLogger() +else: + parse_logger = s3_logger.S3ParseLogger.from_env_vars() + querylog.LOG_QUEUE.set_transmitter( + aws_helpers.s3_querylog_transmitter_from_env()) @app.before_request @@ -2710,14 +2715,91 @@ def split_at(n, xs): return xs[:n], xs[n:] +def on_offline_mode(): + """Prepare for running in offline mode.""" + # We are running in a standalone build made using pyinstaller. + # cd to the directory that has the data files, disable debug mode, and + # use port 80 (unless overridden). + # There will be a standard teacher invite code that everyone can use + # by going to `http://localhost/invite/newteacher`. + os.chdir(utils.offline_data_dir()) + config['port'] = int(os.environ.get('PORT', 80)) + if not os.getenv('TEACHER_INVITE_CODES'): + os.environ['TEACHER_INVITE_CODES'] = 'newteacher' + utils.set_debug_mode(False) + + # Disable logging, so Werkzeug doesn't log all requests and tell users with big red + # letters they're running a non-production server. + # from werkzeug import serving + # def do_nothing(*args, **kwargs): pass + # serving.WSGIRequestHandler.log_request = do_nothing + log = logging.getLogger('werkzeug') + log.setLevel(logging.ERROR) + + # Get our IP addresses so we can print a helpful hint + import socket + ip_addresses = [addr[4][0] for addr in socket.getaddrinfo( + socket.gethostname(), None, socket.AF_INET, socket.SOCK_STREAM)] + ip_addresses = [i for i in ip_addresses if i != '127.0.0.1'] + + from colorama import colorama_text, Fore, Back, Style + g = Fore.GREEN + lines = [ + ('', ''), + ('', ''), + (g, r' _ _ _ '), + (g, r'| | | | | | '), + (g, r'| |__| | ___ __| |_ _ '), + (g, r'| __ |/ _ \/ _` | | | |'), + (g, r'| | | | __/ (_| | |_| |'), + (g, r'|_| |_|\___|\__,_|\__, |'), + (g, r' __/ |'), + (g, r' o f f l i n e |___/ '), + ('', ''), + ('', 'Use a web browser to visit the following website:'), + ('', ''), + *[(Fore.BLUE, f' http://{ip}/') for ip in ip_addresses], + ('', ''), + ('', ''), + ] + # This is necessary to make ANSI color codes work on Windows. + # Init and deinit so we don't mess with Werkzeug's use of this library later on. + with colorama_text(): + for style, text in lines: + print(Back.WHITE + Fore.BLACK + ''.ljust(10) + style + text.ljust(60) + Style.RESET_ALL) + + # We have this option for testing the offline build. A lot of modules read + # files upon import, and those happen before the offline build 'cd' we do + # here and need to be written to use __file__. During the offline build, + # we want to run the actual code to see that nobody added file accesses that + # crash, but we don't actually want to start the server. + smoke_test = '--smoketest' in sys.argv + if smoke_test: + sys.exit(0) + + if __name__ == '__main__': # Start the server on a developer machine. Flask is initialized in DEBUG mode, so it # hot-reloads files. We also flip our own internal "debug mode" flag to True, so our # own file loading routines also hot-reload. - utils.set_debug_mode(not os.getenv('NO_DEBUG_MODE')) + no_debug_mode_requested = os.getenv('NO_DEBUG_MODE') + utils.set_debug_mode(not no_debug_mode_requested) - # For local debugging, fetch all static files on every request - app.config['SEND_FILE_MAX_AGE_DEFAULT'] = None + if utils.is_offline_mode(): + on_offline_mode() + + # Set some default environment variables for development mode + env_defaults = dict( + BASE_URL=f"http://localhost:{config['port']}/", + ADMIN_USER="admin", + ) + for key, value in env_defaults.items(): + if key not in os.environ: + os.environ[key] = value + + if utils.is_debug_mode(): + # For local debugging, fetch all static files on every request + app.config['SEND_FILE_MAX_AGE_DEFAULT'] = None # If we are running in a Python debugger, don't use flasks reload mode. It creates # subprocesses which make debugging harder. @@ -2732,9 +2814,12 @@ def split_at(n, xs): start_snapshot = tracemalloc.take_snapshot() on_server_start() - logger.debug('app starting in debug mode') + debug = utils.is_debug_mode() and not (is_in_debugger or profile_memory) + if debug: + logger.debug('app starting in debug mode') + # Threaded option enables multiple instances for multiple user access support - app.run(threaded=True, debug=not is_in_debugger and not profile_memory, + app.run(threaded=True, debug=debug, port=config['port'], host="0.0.0.0") # See `Procfile` for how the server is started on Heroku. diff --git a/app.spec b/app.spec new file mode 100644 index 00000000000..f6060c002f3 --- /dev/null +++ b/app.spec @@ -0,0 +1,75 @@ +#---------------------------------------------------------- +# PyInstaller configuration file +# +# This file controls how we build a standalone distribution of +# Hedy that can run in environments where Internet access might +# be spotty. +#---------------------------------------------------------- +# -*- mode: python ; coding: utf-8 -*- +from os import path +import sys + +dirname = 'offlinehedy' +appname = 'run-hedy-server' + +# Find the venv directory. We need to be able to pass this to +# pyinstaller, otherwise it will not bundle the libraries we installed +# from the venv. +venv_dir = [p for p in sys.path if 'site-packages' in p][0] + + +data_files = [ + # Files + ('README.md', '.'), + ('static_babel_content.json', '.'), + + # Folders + ('content', 'content'), + ('grammars', 'grammars'), + ('grammars-Total', 'grammars-Total'), + ('prefixes', 'prefixes'), + ('static', 'static'), + ('templates', 'templates'), + ('translations', 'translations'), +] + +a = Analysis( + ['app.py'], + pathex=[venv_dir], + binaries=[], + datas=data_files, + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name=appname, + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=False, + upx_exclude=[], + name=dirname, +) diff --git a/dodo.py b/dodo.py index 0b3f202330c..8074974ffc4 100644 --- a/dodo.py +++ b/dodo.py @@ -24,9 +24,25 @@ from os import path from glob import glob import sys +import platform from doit.tools import LongRunning +if os.getenv('GITHUB_ACTION') and platform.system() == 'Windows': + # Add MSYS2 to the path, so we can use commands like 'bash' and 'cp' and 'mv'. + # https://github.com/actions/runner-images/blob/win22/20240204.1/images/windows/Windows2022-Readme.md + print('Detected a Windows GitHub runner. Adding MSYS2 to the PATH.') + msys_dir = 'C:\\msys64\\usr\\bin' + os.environ['PATH'] = msys_dir + ';' + os.environ['PATH'] + + # We need to explicitly invoke bash from this directory, otherwise + # it will pick up a bash that requires WSL to run, which is not installed. + # And npx must be invoked like this as well. + npx = 'npx.cmd' + bash = f'{msys_dir}\\bash.exe' +else: + npx = 'npx' + bash = 'bash' # The current Python interpreter, use to run other Python scripts as well python3 = sys.executable @@ -99,7 +115,7 @@ def task_tailwind(): task_dep=['npm'], title=lambda _: 'Generate Tailwind CSS', actions=[ - [script], + [bash, script], ], targets=[target], @@ -194,10 +210,10 @@ def task_typescript(): # Use tsc to do type checking of the .ts files, but don't actually emit. # We will bundle using `esbuild`, which will properly handle including the `tw-elements` # library (which is ESM-only) from otherwise CommonJS packages. - ['npx', 'tsc', '--noEmit'], + [npx, 'tsc', '--noEmit'], # Then bundle JavaScript into a single bundle - ['npx', 'esbuild', 'static/js/index.ts', + [npx, 'esbuild', 'static/js/index.ts', '--bundle', '--sourcemap', '--target=es2017', '--global-name=hedyApp', '--platform=browser', '--outfile=static/js/appbundle.js'], @@ -238,7 +254,7 @@ def task_prefixes(): *glob('prefixes/*.py'), ], actions=[ - [script], + [bash, script], ], targets=[ 'static/js/pythonPrefixes.ts' @@ -264,7 +280,7 @@ def task_lezer_parsers(): ], task_dep=['npm'], actions=[ - [script], + [bash, script], ], targets=lezer_files, ) @@ -357,6 +373,25 @@ def task_devdb(): ) +def task__offline(): + """Build the offline Hedy distribution.""" + + return dict( + title=lambda _: 'Build offline Hedy', + task_dep=['backend', 'frontend'], + actions=[ + 'pyinstaller -y app.spec', + # We copy this here instead of in the 'spec' file so that we can rename + # the file (spec file copies cannot do that). + 'cp data-for-testing.json dist/offlinehedy/database.json', + 'cp OFFLINE_README.txt dist/offlinehedy/README.txt', + # There are some research papers in the distribution that take up a lot + # of space. + 'rm -rf dist/offlinehedy/_internal/content/research/*', + ], + ) + + ###################################################################################### # Below this line are helpers for the task definitions # diff --git a/hedy_content.py b/hedy_content.py index fcf10dbeff0..56b2fecc5ee 100644 --- a/hedy_content.py +++ b/hedy_content.py @@ -1,5 +1,6 @@ import logging import os +from os import path import static_babel_content @@ -404,9 +405,16 @@ ] } +# We must find our data relative to this .py file. This will give the +# correct answer both for when Hedy is run as a webserver on Heroku, as well +# as when it has been bundled using pyinstaller. +data_root = path.dirname(__file__) + +content_dir = path.join(data_root, 'content') +translations_dir = path.join(data_root, 'translations') RESEARCH = {} -for paper in sorted(os.listdir('content/research'), +for paper in sorted(os.listdir(f'{content_dir}/research'), key=lambda x: int(x.split("_")[-1][:-4]), reverse=True): # An_approach_to_describing_the_semantics_of_Hedy_2022.pdf -> 2022, An @@ -415,16 +423,17 @@ name = name[-4:] + ". " + name[:-5] RESEARCH[name] = paper + # load all available languages in dict # list_translations of babel does about the same, but without territories. languages = {} -if not os.path.isdir('translations'): +if not os.path.isdir(translations_dir): # should not be possible, but if it's moved someday, EN would still be working. ALL_LANGUAGES['en'] = 'English' ALL_KEYWORD_LANGUAGES['en'] = 'EN' -for folder in os.listdir('translations'): - locale_dir = os.path.join('translations', folder, 'LC_MESSAGES') +for folder in os.listdir(translations_dir): + locale_dir = os.path.join(translations_dir, folder, 'LC_MESSAGES') if not os.path.isdir(locale_dir): continue if filter(lambda x: x.endswith('.mo'), os.listdir(locale_dir)): @@ -434,13 +443,13 @@ for lang in sorted(languages): ALL_LANGUAGES[lang] = languages[lang] - if os.path.exists('./grammars/keywords-' + lang + '.lark'): + if os.path.exists(path.join(data_root, './grammars/keywords-' + lang + '.lark')): ALL_KEYWORD_LANGUAGES[lang] = lang[0:2].upper() # first two characters # Load and cache all keyword yamls KEYWORDS = {} for lang in ALL_KEYWORD_LANGUAGES.keys(): - KEYWORDS[lang] = YamlFile.for_file(f'content/keywords/{lang}.yaml').to_dict() + KEYWORDS[lang] = YamlFile.for_file(f'{content_dir}/keywords/{lang}.yaml').to_dict() for k, v in KEYWORDS[lang].items(): if isinstance(v, str) and "|" in v: # when we have several options, pick the first one as default @@ -466,7 +475,7 @@ def file(self): class Commands(StructuredDataFile): def __init__(self, language): self.language = language - super().__init__(f'content/cheatsheets/{self.language}.yaml') + super().__init__(f'{content_dir}/cheatsheets/{self.language}.yaml') def get_commands_for_level(self, level, keyword_lang): return deep_translate_keywords(self.file.get(int(level), {}), keyword_lang) @@ -505,7 +514,7 @@ def get_commands_for_level(self, level, keyword_lang): class Adventures(StructuredDataFile): def __init__(self, language): self.language = language - super().__init__(f'content/adventures/{self.language}.yaml') + super().__init__(f'{content_dir}/adventures/{self.language}.yaml') def get_adventure_keyname_name_levels(self): return {aid: {adv['name']: list(adv['levels'].keys())} for aid, adv in self.file.get('adventures', {}).items()} @@ -574,7 +583,7 @@ def get_adventure(self): class ParsonsProblem(StructuredDataFile): def __init__(self, language): self.language = language - super().__init__(f'content/parsons/{self.language}.yaml') + super().__init__(f'{content_dir}/parsons/{self.language}.yaml') def get_highest_exercise_level(self, level): return max(int(lnum) for lnum in self.file.get('levels', {}).get(level, {}).keys()) @@ -589,7 +598,7 @@ def get_parsons_data_for_level_exercise(self, level, excercise, keyword_lang="en class Quizzes(StructuredDataFile): def __init__(self, language): self.language = language - super().__init__(f'content/quizzes/{self.language}.yaml') + super().__init__(f'{content_dir}/quizzes/{self.language}.yaml') def get_highest_question_level(self, level): return max(int(k) for k in self.file.get('levels', {}).get(level, {})) @@ -611,7 +620,7 @@ class Tutorials(StructuredDataFile): # action on server start def __init__(self, language): self.language = language - super().__init__(f'content/tutorials/{self.language}.yaml') + super().__init__(f'{content_dir}/tutorials/{self.language}.yaml') def get_tutorial_for_level(self, level, keyword_lang="en"): if level not in ["intro", "teacher"]: @@ -632,7 +641,7 @@ def get_tutorial_for_level(self, level, keyword_lang): class Slides(StructuredDataFile): def __init__(self, language): self.language = language - super().__init__(f'content/slides/{self.language}.yaml') + super().__init__(f'{content_dir}/slides/{self.language}.yaml') def get_slides_for_level(self, level, keyword_lang="en"): return deep_translate_keywords(self.file.get('levels', {}).get(level), keyword_lang) @@ -651,7 +660,7 @@ def get_tags(self): class Tags(StructuredDataFile): def __init__(self, language): self.language = language - super().__init__(f'content/tags/{self.language}.yaml') + super().__init__(f'{content_dir}/tags/{self.language}.yaml') def get_tags_names(self): return {tid: tags['items'] for tid, tags in self.file.get('tags', {}).items()} diff --git a/requirements.txt b/requirements.txt index 49f936a6440..cbeb3516f09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,3 +33,4 @@ pytest-xdist==3.3.1 email-validator==2.1.0.post1 doit==0.36.0 doit_watch>=0.1.0 +pyinstaller==6.3.0 diff --git a/static/images/teacherfeedback/Fkids1.JPG b/static/images/teacherfeedback/Fkids1.JPG index 2d66dd2e3dc..73b28553965 100644 Binary files a/static/images/teacherfeedback/Fkids1.JPG and b/static/images/teacherfeedback/Fkids1.JPG differ diff --git a/static/images/teacherfeedback/Fkids2.JPG b/static/images/teacherfeedback/Fkids2.JPG index e85974848f3..ecb4e17976a 100644 Binary files a/static/images/teacherfeedback/Fkids2.JPG and b/static/images/teacherfeedback/Fkids2.JPG differ diff --git a/static_babel_content.py b/static_babel_content.py index d73b028df4e..a25c87a3260 100644 --- a/static_babel_content.py +++ b/static_babel_content.py @@ -1,6 +1,7 @@ +from os import path import json -with open("static_babel_content.json") as f: +with open(path.join(path.dirname(__file__), 'static_babel_content.json')) as f: data = json.load(f) diff --git a/utils.py b/utils.py index d77ed3e81a0..0bf5325d136 100644 --- a/utils.py +++ b/utils.py @@ -6,11 +6,13 @@ import time import functools import os +from os import path import re import string import random import uuid import unicodedata +import sys import traceback from email_validator import EmailNotValidError, validate_email @@ -23,21 +25,23 @@ IS_WINDOWS = os.name == 'nt' +prefixes_dir = path.join(path.dirname(__file__), 'prefixes') + # Define code that will be used if some turtle command is present -with open('prefixes/turtle.py', encoding='utf-8') as f: +with open(f'{prefixes_dir}/turtle.py', encoding='utf-8') as f: TURTLE_PREFIX_CODE = f.read() # Preamble that will be used for non-Turtle programs # numerals list generated from: https://replit.com/@mevrHermans/multilangnumerals -with open('prefixes/normal.py', encoding='utf-8') as f: +with open(f'{prefixes_dir}/normal.py', encoding='utf-8') as f: NORMAL_PREFIX_CODE = f.read() # Define code that will be used if a pressed command is used -with open('prefixes/pygame.py', encoding='utf-8') as f: +with open(f'{prefixes_dir}/pygame.py', encoding='utf-8') as f: PYGAME_PREFIX_CODE = f.read() # Define code that will be used if music code is used -with open('prefixes/music.py', encoding='utf-8') as f: +with open(f'{prefixes_dir}/music.py', encoding='utf-8') as f: MUSIC_PREFIX_CODE = f.read() @@ -93,6 +97,20 @@ def is_debug_mode(): return DEBUG_MODE +def is_offline_mode(): + """Return whether or not we're in offline mode. + + Offline mode is a special build of Hedy that teachers can download and run + on their own computers. + """ + return getattr(sys, 'frozen', False) and offline_data_dir() is not None + + +def offline_data_dir(): + """Return the data directory in offline mode.""" + return getattr(sys, '_MEIPASS') + + def set_debug_mode(debug_mode): """Switch debug mode to given value.""" global DEBUG_MODE diff --git a/website/database.py b/website/database.py index 58832dc1212..32ee6af7cab 100644 --- a/website/database.py +++ b/website/database.py @@ -2,13 +2,21 @@ import operator import itertools from datetime import date, timedelta +import sys +from os import path from utils import timems, times from . import dynamo, auth from . import querylog -storage = dynamo.AwsDynamoStorage.from_env() or dynamo.MemoryStorage("dev_database.json") +is_offline = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') +if is_offline: + # Offline mode. Store data 1 directory upwards from `_internal` + storage = dynamo.MemoryStorage(path.join(sys._MEIPASS, "..", "database.json")) +else: + # Production or dev: use environment variables or dev storage + storage = dynamo.AwsDynamoStorage.from_env() or dynamo.MemoryStorage("dev_database.json") USERS = dynamo.Table(storage, "users", "username", indexes=[ dynamo.Index("email"), diff --git a/website/log_queue.py b/website/log_queue.py index 71f2e661432..c41120b68b4 100644 --- a/website/log_queue.py +++ b/website/log_queue.py @@ -134,9 +134,6 @@ def transmit_now(self, max_time=None): def _save_records(self, timestamp, records): if self.transmitter: return self.transmitter(timestamp, records) - else: - count = len(records) - logger.warning(f"No querylog transmitter configured, {count} records dropped") def _write_thread(self): """Background thread which will wake up every batch_window_s seconds