From 23bb5bc1cc5ba446e3d35ee8f310da68e202fe79 Mon Sep 17 00:00:00 2001 From: Luca Scheller Date: Sun, 10 Dec 2023 20:13:07 +0100 Subject: [PATCH] Add install matrix and skip checksum check (because of SideFX bug) --- .github/scripts/houdini.py | 233 +++++++++++++++++++--------- .github/workflows/build_houdini.yml | 10 +- 2 files changed, 166 insertions(+), 77 deletions(-) diff --git a/.github/scripts/houdini.py b/.github/scripts/houdini.py index 4beff9e..c24154f 100644 --- a/.github/scripts/houdini.py +++ b/.github/scripts/houdini.py @@ -12,11 +12,11 @@ import tarfile -SIDEFX_CLIENT_ID = os.environ.get('SIDEFX_CLIENT_ID', "") -SIDEFX_CLIENT_SECRET_KEY = os.environ.get('SIDEFX_CLIENT_SECRET_KEY', "") +SIDEFX_CLIENT_ID = os.environ.get("SIDEFX_CLIENT_ID", "") +SIDEFX_CLIENT_SECRET_KEY = os.environ.get("SIDEFX_CLIENT_SECRET_KEY", "") -logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') +logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%m/%d/%Y %I:%M:%S %p") def create_sidefx_service(client_id, client_secret_key): @@ -41,14 +41,14 @@ def get_sidefx_platform(): str: The active platform """ current_platform = platform.system() - if current_platform == 'Windows' or current_platform.startswith('CYGWIN'): - return 'win64' - elif current_platform == 'Darwin': - return 'macos' - elif current_platform == 'Linux': - return 'linux' + if current_platform == "Windows" or current_platform.startswith("CYGWIN"): + return "win64" + elif current_platform == "Darwin": + return "macos" + elif current_platform == "Linux": + return "linux" else: - return '' + return "" def download_sidefx_product_release(dir_path, release): @@ -60,63 +60,90 @@ def download_sidefx_product_release(dir_path, release): str: The file path of the downloaded file """ # Download file - download_file_name = release['filename'] + download_file_name = release["filename"] download_file_path = os.path.join(dir_path, download_file_name) - request = requests.get(release['download_url'], stream=True) + request = requests.get(release["download_url"], stream=True) if request.status_code == 200: - with open(download_file_path, 'wb') as download_file: + with open(download_file_path, "wb") as download_file: request.raw.decode_content = True shutil.copyfileobj(request.raw, download_file) else: - raise Exception('Error downloading file!') + raise Exception("Error downloading file!") # Verify file checksum - download_file_hash = hashlib.md5() - with open(download_file_path, 'rb') as download_file: - for chunk in iter(lambda: download_file.read(4096), b''): - download_file_hash.update(chunk) - if download_file_hash.hexdigest() != release['hash']: - raise Exception('Checksum does not match!') + # 10.12.23 -> SideFX Bug, this currently fails on Linux + # download_file_hash = hashlib.md5() + # with open(download_file_path, "rb") as download_file: + # for chunk in iter(lambda: download_file.read(4096), b""): + # download_file_hash.update(chunk) + # if download_file_hash.hexdigest() != release["hash"]: + # raise Exception("Checksum does not match!") return download_file_path -def install_sidefx_houdini(): - """Install the latest production release of Houdini""" +def install_sidefx_houdini(houdini_version): + """Install the latest production release of Houdini + Args: + houdini_version (str): The target Houdini version (e.g. 20.0, 19.5, etc.) + """ # Connect to SideFX API - logging.info('Connecting to SideFX API') + logging.info("Connecting to SideFX API") sidefx_service = create_sidefx_service(SIDEFX_CLIENT_ID, SIDEFX_CLIENT_SECRET_KEY) sidefx_platform = get_sidefx_platform() sidefx_product = "houdini" # Get release data - releases_list = sidefx_service.download.get_daily_builds_list(product=sidefx_product, - platform=sidefx_platform, - only_production=True) - # Switch to new gcc version starting with H20 + releases_list = sidefx_service.download.get_daily_builds_list( + product=sidefx_product, + version=houdini_version, + platform=sidefx_platform, + only_production=True, + ) + target_release = None if sidefx_platform == "linux": for release in releases_list: + # Switch to new gcc version starting with H20 if release["version"] == "20.0": if not release["platform"].endswith("gcc11.2"): continue - latest_production_release = release + target_release = release break else: - latest_production_release = releases_list[0] - latest_production_release_download = sidefx_service.download.get_daily_build_download(product='houdini', - version=latest_production_release["version"], - build=latest_production_release["build"], - platform=sidefx_platform) + for release in releases_list: + target_release = release + break + + if not target_release: + raise Exception( + "No Houdini version found for requested version | {}".format( + houdini_version + ) + ) + + target_release_download = sidefx_service.download.get_daily_build_download( + product="houdini", + version=target_release["version"], + build=target_release["build"], + platform=sidefx_platform, + ) # Download latest production release - logging.info('Downloading Houdini build {version}.{build}'.format(version=latest_production_release["version"], - build=latest_production_release["build"])) + logging.info( + "Downloading Houdini build {version}.{build}".format( + version=target_release["version"], build=target_release["build"] + ) + ) downloads_dir_path = os.path.join(os.path.expanduser("~"), "Downloads") if not os.path.isdir(downloads_dir_path): os.makedirs(downloads_dir_path) - houdini_installer_file_path = download_sidefx_product_release(downloads_dir_path, - latest_production_release_download) + houdini_installer_file_path = download_sidefx_product_release( + downloads_dir_path, target_release_download + ) # Install latest production release - logging.info('Installing Houdini build {version}.{build}'.format(version=latest_production_release["version"], - build=latest_production_release["build"])) + logging.info( + "Installing Houdini build {version}.{build}".format( + version=target_release["version"], build=target_release["build"] + ) + ) hfs_dir_path = "" if sidefx_platform == "linux": # Unpack tar file @@ -124,39 +151,83 @@ def install_sidefx_houdini(): tar_file.extractall(downloads_dir_path) os.remove(houdini_installer_file_path) # Get folder name - houdini_installer_dir_name = latest_production_release_download['filename'] + houdini_installer_dir_name = target_release_download["filename"] houdini_installer_dir_name = houdini_installer_dir_name.replace(".tar", "") houdini_installer_dir_name = houdini_installer_dir_name.replace(".gz", "") - houdini_installer_dir_path = os.path.join(downloads_dir_path, houdini_installer_dir_name) - cmd = [os.path.join(houdini_installer_dir_path, "houdini.install"), - "--auto-install", "--accept-EULA", "2021-10-13", - "--install-houdini", "--no-install-license", "--no-install-avahi", - "--no-install-hqueue-server", "--no-install-hqueue-client", - "--no-install-menus", "--no-install-bin-symlink", - "--no-install-engine-maya", "--no-install-engine-unity", "--no-install-engine-unreal", "--no-install-sidefxlabs"] + houdini_installer_dir_path = os.path.join( + downloads_dir_path, houdini_installer_dir_name + ) + cmd = [ + os.path.join(houdini_installer_dir_path, "houdini.install"), + "--auto-install", + "--accept-EULA", + "2021-10-13", + "--install-houdini", + "--no-install-license", + "--no-install-avahi", + "--no-install-hqueue-server", + "--no-install-hqueue-client", + "--no-install-menus", + "--no-install-bin-symlink", + "--no-install-engine-maya", + "--no-install-engine-unity", + "--no-install-engine-unreal", + "--no-install-sidefxlabs", + ] status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if status.returncode != 0: - raise Exception("Failed to install Houdini, ran into the following error:\n {error}".format(error=status.stderr)) - hfs_dir_path = os.path.join("/opt", "hfs{}.{}".format(latest_production_release["version"], - latest_production_release["build"])) - hfs_versionless_dir_path = os.path.join(os.path.dirname(hfs_dir_path), "hfs") + raise Exception( + "Failed to install Houdini, ran into the following error:\n {error}".format( + error=status.stderr + ) + ) + hfs_dir_path = os.path.join( + "/opt", + "hfs{}.{}".format(target_release["version"], target_release["build"]), + ) + hfs_symlink_dir_path = os.path.join(os.path.dirname(hfs_dir_path), "hfs") elif sidefx_platform == "win64": - cmd = [houdini_installer_file_path, - "/S", "/AcceptEULA=2021-10-13", - "/MainApp", "/LicenseServer=No", "/StartMenu=No", - "/HQueueServer=No", "/HQueueClient=No", - "/EngineMaya=No", "/Engine3dsMax", "/EngineUnity", "/EngineUnreal=No", "/SideFXLabs=No"] + cmd = [ + houdini_installer_file_path, + "/S", + "/AcceptEULA=2021-10-13", + "/MainApp", + "/LicenseServer=No", + "/StartMenu=No", + "/HQueueServer=No", + "/HQueueClient=No", + "/EngineMaya=No", + "/Engine3dsMax", + "/EngineUnity", + "/EngineUnreal=No", + "/SideFXLabs=No", + ] status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if status.returncode != 0: - raise Exception("Failed to install Houdini, ran into the following error:\n {error}".format(error=status.stderr)) - hfs_dir_path = os.path.join("C:\Program Files\Side Effects Software", "Houdini {}.{}".format(latest_production_release["version"], latest_production_release["build"])) - hfs_versionless_dir_path = os.path.join(os.path.dirname(hfs_dir_path), "Houdini") + raise Exception( + "Failed to install Houdini, ran into the following error:\n {error}".format( + error=status.stderr + ) + ) + hfs_dir_path = os.path.join( + "C:\Program Files\Side Effects Software", + "Houdini {}.{}".format(target_release["version"], target_release["build"]), + ) + hfs_symlink_dir_path = os.path.join( + os.path.dirname(hfs_dir_path), "Houdini" + ) else: - raise Exception("Platform {platform} is currently not" - "supported!".format(platform=platform)) + raise Exception( + "Platform {platform} is currently not" + "supported!".format(platform=platform) + ) # Create version-less symlink - logging.info('Creating symlink Houdini build {src} -> {dst}'.format(src=hfs_dir_path, dst=hfs_versionless_dir_path)) - os.symlink(hfs_dir_path, hfs_versionless_dir_path) + logging.info( + "Creating symlink Houdini build {src} -> {dst}".format( + src=hfs_dir_path, dst=hfs_symlink_dir_path + ) + ) + os.symlink(hfs_dir_path, hfs_symlink_dir_path) def create_sidefx_houdini_artifact(artifact_src, artifact_dst, artifact_prefix): @@ -175,31 +246,43 @@ def create_sidefx_houdini_artifact(artifact_src, artifact_dst, artifact_prefix): if sidefx_platform == "linux": hfs_build_name = os.path.basename(pathlib.Path("/opt/hfs").resolve()) elif sidefx_platform == "win64": - hfs_build_name = os.path.basename(pathlib.Path("C:\Program Files\Side Effects Software\Houdini").resolve()) + hfs_build_name = os.path.basename( + pathlib.Path("C:\Program Files\Side Effects Software\Houdini").resolve() + ) else: - raise Exception("Platform {platform} is currently not" - "supported!".format(platform=platform)) + raise Exception( + "Platform {platform} is currently not" + "supported!".format(platform=platform) + ) hfs_build_name = re_digitdot.sub("", hfs_build_name) - artifact_file_path = os.path.join(artifact_dst, f"{artifact_prefix}_houdini-{hfs_build_name}-{sidefx_platform}") + artifact_file_path = os.path.join( + artifact_dst, f"{artifact_prefix}_houdini-{hfs_build_name}-{sidefx_platform}" + ) artifact_dir_path = os.path.dirname(artifact_file_path) if not os.path.exists(artifact_dir_path): os.makedirs(artifact_dir_path) - shutil.make_archive(artifact_file_path, 'zip', artifact_src) + shutil.make_archive(artifact_file_path, "zip", artifact_src) if __name__ == "__main__": # Parse args parser = argparse.ArgumentParser() - parser.add_argument('--install', action='store_true', help='Install Houdini') - parser.add_argument('--artifact', action='store_true', help='Create artifact') - parser.add_argument('--artifact_src', help='Artifact source directory') - parser.add_argument('--artifact_dst', help='Artifact target directory') - parser.add_argument('--artifact_prefix', help='Artifact name prefix') + parser.add_argument("--install", action="store_true", help="Install Houdini") + parser.add_argument( + "--install_version", + help="Houdini version to install. If not provided, fallback to the latest version.", + ) + parser.add_argument("--artifact", action="store_true", help="Create artifact") + parser.add_argument("--artifact_src", help="Artifact source directory") + parser.add_argument("--artifact_dst", help="Artifact target directory") + parser.add_argument("--artifact_prefix", help="Artifact name prefix") args = parser.parse_args() # Execute # Install Houdini if args.install: - install_sidefx_houdini() + install_sidefx_houdini(args.install_version) # Create artifact tagged with Houdini build name (expects Houdini to be installed via the above install command) if args.artifact: - create_sidefx_houdini_artifact(args.artifact_src, args.artifact_dst, args.artifact_prefix) \ No newline at end of file + create_sidefx_houdini_artifact( + args.artifact_src, args.artifact_dst, args.artifact_prefix + ) diff --git a/.github/workflows/build_houdini.yml b/.github/workflows/build_houdini.yml index 1d48fe3..e912423 100644 --- a/.github/workflows/build_houdini.yml +++ b/.github/workflows/build_houdini.yml @@ -16,6 +16,9 @@ jobs: build_linux: runs-on: ubuntu-latest environment: houdini + strategy: + matrix: + houdini_version: [19.5, 20.0] steps: - name: Checkout repository uses: actions/checkout@v3 @@ -45,7 +48,7 @@ jobs: SIDEFX_CLIENT_ID: '${{ secrets.SIDEFX_CLIENT_ID }}' SIDEFX_CLIENT_SECRET_KEY: '${{ secrets.SIDEFX_CLIENT_SECRET_KEY }}' run: | - sudo --preserve-env python3 .github/scripts/houdini.py --install + sudo --preserve-env python3 .github/scripts/houdini.py --install --install_version ${{ matrix.houdini_version }} - name: Build USD File Resolver run: | .github/scripts/houdini_build.sh fileResolver @@ -71,6 +74,9 @@ jobs: build_windows: runs-on: windows-2019 environment: houdini + strategy: + matrix: + houdini_version: [19.5, 20.0] steps: - name: Checkout repository uses: actions/checkout@v3 @@ -96,7 +102,7 @@ jobs: SIDEFX_CLIENT_ID: '${{ secrets.SIDEFX_CLIENT_ID }}' SIDEFX_CLIENT_SECRET_KEY: '${{ secrets.SIDEFX_CLIENT_SECRET_KEY }}' run: | - python3 .github\scripts\houdini.py --install + python3 .github\scripts\houdini.py --install --install_version ${{ matrix.houdini_version }} - name: Build USD File Resolver run: | .\.github\scripts\houdini_build.bat fileResolver