From fd1ab5820999e2a6c6b5df301ff6c2075c1a6c9e Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 01:30:54 +0300 Subject: [PATCH 01/18] update to the frame interpolation pipeline, there is some minor issue with creating go api bindings because of openapi json sceme having a null option. --- runner/app/main.py | 18 ++- runner/app/pipelines/frame_interpolation.py | 114 +++++++++++++- runner/app/pipelines/image_to_video.py | 6 +- runner/app/pipelines/upscale.py | 4 +- runner/app/pipelines/utils/__init__.py | 4 + runner/app/pipelines/utils/utils.py | 160 ++++++++++++++++++++ runner/app/routes/frame_interpolation.py | 120 +++++++++++++++ runner/dl_checkpoints.sh | 3 + runner/gen_openapi.py | 9 +- runner/openapi.json | 151 ++++++++++++++++-- runner/requirements.txt | 3 + 11 files changed, 566 insertions(+), 26 deletions(-) create mode 100644 runner/app/routes/frame_interpolation.py diff --git a/runner/app/main.py b/runner/app/main.py index 6f511420..c4dd6b8e 100644 --- a/runner/app/main.py +++ b/runner/app/main.py @@ -1,5 +1,7 @@ import logging import os +import sys +import cv2 from contextlib import asynccontextmanager from app.routes import health @@ -15,8 +17,8 @@ async def lifespan(app: FastAPI): app.include_router(health.router) - pipeline = os.environ["PIPELINE"] - model_id = os.environ["MODEL_ID"] + pipeline = os.environ.get("PIPELINE", "") # Default to + model_id = os.environ.get("MODEL_ID", "") # Provide a default if necessary app.pipeline = load_pipeline(pipeline, model_id) app.include_router(load_route(pipeline)) @@ -46,8 +48,10 @@ def load_pipeline(pipeline: str, model_id: str) -> any: from app.pipelines.audio_to_text import AudioToTextPipeline return AudioToTextPipeline(model_id) - case "frame-interpolation": - raise NotImplementedError("frame-interpolation pipeline not implemented") + case "FILMPipeline": + from app.pipelines.frame_interpolation import FILMPipeline + + return FILMPipeline(model_id) case "upscale": from app.pipelines.upscale import UpscalePipeline @@ -76,8 +80,10 @@ def load_route(pipeline: str) -> any: from app.routes import audio_to_text return audio_to_text.router - case "frame-interpolation": - raise NotImplementedError("frame-interpolation pipeline not implemented") + case "FILMPipeline": + from app.routes import frame_interpolation + + return frame_interpolation.router case "upscale": from app.routes import upscale diff --git a/runner/app/pipelines/frame_interpolation.py b/runner/app/pipelines/frame_interpolation.py index 396c8067..755afee7 100644 --- a/runner/app/pipelines/frame_interpolation.py +++ b/runner/app/pipelines/frame_interpolation.py @@ -1,5 +1,113 @@ -from app.pipelines.base import Pipeline +import torch +from torchvision.transforms import v2 +from tqdm import tqdm +import bisect +import numpy as np +from app.pipelines.utils.utils import get_model_dir -class FrameInterpolationPipeline(Pipeline): - pass +class FILMPipeline: + model: torch.jit.ScriptModule + + def __init__(self, model_id: str): + self.model_id = model_id + model_dir = get_model_dir() # Get the directory where models are stored + model_path = f"{model_dir}/{model_id}" # Construct the full path to the model file + + self.model = torch.jit.load(model_path, map_location="cpu") + self.model.eval() + + def to(self, *args, **kwargs): + self.model = self.model.to(*args, **kwargs) + return self + + @property + def device(self) -> torch.device: + # Checking device for ScriptModule requires checking one of its parameters + params = self.model.parameters() + return next(params).device + + @property + def dtype(self) -> torch.dtype: + # Checking device for ScriptModule requires checking one of its parameters + params = self.model.parameters() + return next(params).dtype + + def __call__( + self, + reader, + writer, + inter_frames: int = 2, + ): + transforms = v2.Compose( + [ + v2.ToDtype(torch.uint8, scale=True), + ] + ) + + writer.open() + + while True: + frame_1 = reader.get_frame() + # If the first frame read is None then there are no more frames + if frame_1 is None: + break + + frame_2 = reader.get_frame() + # If the second frame read is None there there is a final frame + if frame_2 is None: + writer.write_frame(transforms(frame_1)) + break + + # frame_1 and frame_2 must be tensors with n c h w format + frame_1 = frame_1.unsqueeze(0) + frame_2 = frame_2.unsqueeze(0) + + frames = inference( + self.model, frame_1, frame_2, inter_frames, self.device, self.dtype + ) + + frames = [transforms(frame.detach().cpu()) for frame in frames] + for frame in frames: + writer.write_frame(frame) + + writer.close() + + +def inference( + model, img_batch_1, img_batch_2, inter_frames, device, dtype +) -> torch.Tensor: + results = [img_batch_1, img_batch_2] + + idxes = [0, inter_frames + 1] + remains = list(range(1, inter_frames + 1)) + + splits = torch.linspace(0, 1, inter_frames + 2) + + for _ in tqdm(range(len(remains)), "Generating in-between frames"): + starts = splits[idxes[:-1]] + ends = splits[idxes[1:]] + distances = ( + (splits[None, remains] - starts[:, None]) + / (ends[:, None] - starts[:, None]) + - 0.5 + ).abs() + matrix = torch.argmin(distances).item() + start_i, step = np.unravel_index(matrix, distances.shape) + end_i = start_i + 1 + + x0 = results[start_i].to(device=device, dtype=dtype) + x1 = results[end_i].to(device=device, dtype=dtype) + + dt = x0.new_full((1, 1), (splits[remains[step]] - splits[idxes[start_i]])) / ( + splits[idxes[end_i]] - splits[idxes[start_i]] + ) + + with torch.no_grad(): + prediction = model(x0, x1, dt) + insert_position = bisect.bisect_left(idxes, remains[step]) + idxes.insert(insert_position, remains[step]) + results.insert(insert_position, prediction.clamp(0, 1).float()) + del remains[step] + + return results \ No newline at end of file diff --git a/runner/app/pipelines/image_to_video.py b/runner/app/pipelines/image_to_video.py index f605cb2f..988c075b 100644 --- a/runner/app/pipelines/image_to_video.py +++ b/runner/app/pipelines/image_to_video.py @@ -6,10 +6,14 @@ import PIL import torch from app.pipelines.base import Pipeline -from app.pipelines.utils import SafetyChecker, get_model_dir, get_torch_device from diffusers import StableVideoDiffusionPipeline from huggingface_hub import file_download from PIL import ImageFile +from app.pipelines.utils import ( + SafetyChecker, + get_model_dir, + get_torch_device +) ImageFile.LOAD_TRUNCATED_IMAGES = True diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 0cba865e..6fd3cefe 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -87,7 +87,7 @@ def __init__(self, model_id: str): elif deepcache_enabled: logger.warning( "DeepCache is not supported for Lightning or Turbo models. " - "TextToImagePipeline will NOT be optimized with DeepCache for %s", + "UpscalingPiepline will NOT be optimized with DeepCache for %s", model_id, ) @@ -112,7 +112,7 @@ def __call__( ] if num_inference_steps is None or num_inference_steps < 1: - del kwargs["num_inference_steps"] + kwargs.pop("num_inference_steps", None) output = self.ldm(prompt, image=image, **kwargs) diff --git a/runner/app/pipelines/utils/__init__.py b/runner/app/pipelines/utils/__init__.py index 844b86e9..2c083d20 100644 --- a/runner/app/pipelines/utils/__init__.py +++ b/runner/app/pipelines/utils/__init__.py @@ -9,4 +9,8 @@ is_turbo_model, split_prompt, validate_torch_device, + frames_compactor, + video_shredder, + DirectoryReader, + DirectoryWriter ) diff --git a/runner/app/pipelines/utils/utils.py b/runner/app/pipelines/utils/utils.py index dbc44d48..5c9ad6a5 100644 --- a/runner/app/pipelines/utils/utils.py +++ b/runner/app/pipelines/utils/utils.py @@ -5,11 +5,18 @@ import re from pathlib import Path from typing import Optional +import glob +import tempfile +from io import BytesIO +from typing import List, Union import numpy as np import torch from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker from PIL import Image +from torchvision.transforms import v2 +import cv2 +from torchaudio.io import StreamWriter from torch import dtype as TorchDtype from transformers import CLIPFeatureExtractor @@ -111,6 +118,108 @@ def split_prompt( return prompt_dict +def frames_compactor( + frames: Union[List[np.ndarray], List[torch.Tensor]], + output_path: str, + fps: float, + codec: str = "MJPEG", + is_directory: bool = False, + width: int = None, + height: int = None +) -> None: + """ + Generate a video from a list of frames. Frames can be from a directory or in-memory. + + Args: + frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. + output_path (str): Path to save the output video file. + fps (float): Frames per second for the video. + codec (str): Codec used for video compression (default is "XVID"). + is_directory (bool): If True, treat `frames` as a directory path containing image files. + width (int): Width of the video. Must be provided if `frames` are in-memory. + height (int): Height of the video. Must be provided if `frames` are in-memory. + + Returns: + None + """ + if is_directory: + # Read frames from a directory + frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] + else: + # Convert torch tensors to numpy arrays if necessary + if isinstance(frames[0], torch.Tensor): + frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] + + # Ensure frames are numpy arrays and are uint8 type + frames = [frame.astype(np.uint8) for frame in frames] + + # Check if frames are consistent + if not frames: + raise ValueError("No frames to process.") + + if width is None or height is None: + # Use dimensions of the first frame if not provided + height, width = frames[0].shape[:2] + + # Define the codec and create VideoWriter object + fourcc = cv2.VideoWriter_fourcc(*codec) + video_writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) + + # Write frames to the video file + for frame in frames: + # Ensure each frame has the correct size + if frame.shape[1] != width or frame.shape[0] != height: + frame = cv2.resize(frame, (width, height)) + video_writer.write(frame) + + # Release the video writer + video_writer.release() + +def video_shredder(video_data, is_file_path=True) -> np.ndarray: + """ + Extract frames from a video file or in-memory video data and return them as a NumPy array. + + Args: + video_data (str or BytesIO): Path to the input video file or in-memory video data. + is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). + + Returns: + np.ndarray: Array of frames with shape (num_frames, height, width, channels). + """ + if is_file_path: + # Handle file-based video input + video_capture = cv2.VideoCapture(video_data) + else: + # Handle in-memory video input + # Create a temporary file to store in-memory video data + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: + temp_file.write(video_data.getvalue()) + temp_file_path = temp_file.name + + # Open the temporary video file + video_capture = cv2.VideoCapture(temp_file_path) + + if not video_capture.isOpened(): + raise ValueError("Error opening video data") + + frames = [] + success, frame = video_capture.read() + + while success: + frames.append(frame) + success, frame = video_capture.read() + + video_capture.release() + + # Delete the temporary file if it was created + if not is_file_path: + os.remove(temp_file_path) + + # Convert list of frames to a NumPy array + frames_array = np.array(frames) + print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") + + return frames_array class SafetyChecker: """Checks images for unsafe or inappropriate content using a pretrained model. @@ -167,3 +276,54 @@ def check_nsfw_images( clip_input=safety_checker_input.pixel_values.to(self._dtype), ) return images, has_nsfw_concept + +class DirectoryReader: + def __init__(self, dir: str): + self.paths = sorted( + glob.glob(os.path.join(dir, "*")), + key=lambda x: int(os.path.basename(x).split(".")[0]), + ) + self.nb_frames = len(self.paths) + self.idx = 0 + + assert self.nb_frames > 0, "no frames found in directory" + + first_img = Image.open(self.paths[0]) + self.height = first_img.height + self.width = first_img.width + + def get_resolution(self): + return self.height, self.width + + def reset(self): + self.idx = 0 # Reset the index counter to 0 + + def get_frame(self): + if self.idx >= self.nb_frames: + return None + + path = self.paths[self.idx] + self.idx += 1 + + img = Image.open(path) + transforms = v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]) + + return transforms(img) + +class DirectoryWriter: + def __init__(self, dir: str): + self.dir = dir + self.idx = 0 + + def open(self): + return + + def close(self): + return + + def write_frame(self, frame: torch.Tensor): + path = f"{self.dir}/{self.idx}.png" + self.idx += 1 + + transforms = v2.Compose([v2.ToPILImage()]) + transforms(frame.squeeze(0)).save(path) \ No newline at end of file diff --git a/runner/app/routes/frame_interpolation.py b/runner/app/routes/frame_interpolation.py new file mode 100644 index 00000000..a1ae75bc --- /dev/null +++ b/runner/app/routes/frame_interpolation.py @@ -0,0 +1,120 @@ +# app/routes/film_interpolate.py + +import logging +import os +import torch +import glob +from typing import Annotated, Optional +from fastapi import APIRouter, Depends, File, Form, UploadFile, status +from fastapi.responses import JSONResponse +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from PIL import Image, ImageFile + +from app.dependencies import get_pipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils.utils import DirectoryReader, DirectoryWriter, get_torch_device, get_model_dir +from app.routes.util import HTTPError, ImageResponse, http_error, image_to_data_url + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +router = APIRouter() + +logger = logging.getLogger(__name__) + +RESPONSES = { + status.HTTP_400_BAD_REQUEST: {"model": HTTPError}, + status.HTTP_401_UNAUTHORIZED: {"model": HTTPError}, + status.HTTP_500_INTERNAL_SERVER_ERROR: {"model": HTTPError}, +} + +@router.post("/frame_interpolation", response_model=ImageResponse, responses=RESPONSES) +@router.post( + "/frame_interpolation/", + response_model=ImageResponse, + responses=RESPONSES, + include_in_schema=False, +) +async def frame_interpolation( + model_id: Annotated[str, Form()], + image1: Annotated[Optional[UploadFile], File()]=None, + image2: Annotated[Optional[UploadFile], File()]=None, + image_dir: Annotated[Optional[str], Form()]="", + inter_frames: Annotated[int, Form()] = 2, + token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False)), +): + auth_token = os.environ.get("AUTH_TOKEN") + if auth_token: + if not token or token.credentials != auth_token: + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + headers={"WWW-Authenticate": "Bearer"}, + content=http_error("Invalid bearer token"), + ) + + + # Initialize FILMPipeline + film_pipeline = FILMPipeline(model_id) + film_pipeline.to(device=get_torch_device(),dtype=torch.float16) + + # Prepare directories for input and output + temp_input_dir = "temp_input" + temp_output_dir = "temp_output" + os.makedirs(temp_input_dir, exist_ok=True) + os.makedirs(temp_output_dir, exist_ok=True) + + try: + if os.path.isdir(image_dir): + if image1 and image2: + logger.info("Both directory and individual images provided. Directory will be used, and images will be ignored.") + reader = DirectoryReader(image_dir) + else: + if not (image1 and image2): + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content=http_error("Either a directory or two images must be provided."), + ) + + image1_path = os.path.join(temp_input_dir, "0.png") + image2_path = os.path.join(temp_input_dir, "1.png") + + with open(image1_path, "wb") as f: + f.write(await image1.read()) + with open(image2_path, "wb") as f: + f.write(await image2.read()) + + reader = DirectoryReader(temp_input_dir) + + writer = DirectoryWriter(temp_output_dir) + # Perform interpolation + film_pipeline(reader, writer, inter_frames=inter_frames) + + writer.close() + reader.reset() + + # Collect output frames + output_frames = [] + for frame_path in sorted(glob.glob(os.path.join(temp_output_dir, "*.png"))): + frame = Image.open(frame_path) + output_frames.append(frame) + + output_images = [{"url": image_to_data_url(frame),"seed":0, "nsfw":False} for frame in output_frames] + + except Exception as e: + logger.error(f"FILMPipeline error: {e}") + logger.exception(e) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content=http_error("FILMPipeline error"), + ) + + finally: + # Clean up temporary directories + for file_path in glob.glob(os.path.join(temp_input_dir, "*")): + os.remove(file_path) + os.rmdir(temp_input_dir) + + for file_path in glob.glob(os.path.join(temp_output_dir, "*")): + os.remove(file_path) + os.rmdir(temp_output_dir) + + return {"images": output_images} diff --git a/runner/dl_checkpoints.sh b/runner/dl_checkpoints.sh index 822590d4..1528a60b 100755 --- a/runner/dl_checkpoints.sh +++ b/runner/dl_checkpoints.sh @@ -59,6 +59,9 @@ function download_all_models() { # Download image-to-video models. huggingface-cli download stabilityai/stable-video-diffusion-img2vid-xt --include "*.fp16.safetensors" "*.json" --cache-dir models + + #Download frame-interpolation model. + wget -O models/film_net_fp16.pt https://github.com/dajes/frame-interpolation-pytorch/releases/download/v1.0.2/film_net_fp16.pt } # Enable HF transfer acceleration. diff --git a/runner/gen_openapi.py b/runner/gen_openapi.py index 7fde5ee3..198102db 100644 --- a/runner/gen_openapi.py +++ b/runner/gen_openapi.py @@ -11,6 +11,7 @@ image_to_image, image_to_video, text_to_image, + frame_interpolation, upscale, ) from fastapi.openapi.utils import get_openapi @@ -68,7 +69,7 @@ def translate_to_gateway(openapi): openapi["components"]["schemas"]["VideoResponse"]["title"] = "VideoResponse" return openapi - + def write_openapi(fname, entrypoint="runner"): """Write OpenAPI schema to file. @@ -83,8 +84,10 @@ def write_openapi(fname, entrypoint="runner"): app.include_router(text_to_image.router) app.include_router(image_to_image.router) app.include_router(image_to_video.router) - app.include_router(upscale.router) app.include_router(audio_to_text.router) + app.include_router(frame_interpolation.router) + app.include_router(upscale.router) + use_route_names_as_operation_ids(app) @@ -144,3 +147,5 @@ def write_openapi(fname, entrypoint="runner"): args = parser.parse_args() write_openapi(f"openapi.{args.type.lower()}", args.entrypoint) + + diff --git a/runner/openapi.json b/runner/openapi.json index 4345e565..c103d2f6 100644 --- a/runner/openapi.json +++ b/runner/openapi.json @@ -249,15 +249,15 @@ ] } }, - "/upscale": { + "/audio-to-text": { "post": { - "summary": "Upscale", - "operationId": "upscale", + "summary": "Audio To Text", + "operationId": "audio_to_text", "requestBody": { "content": { "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/Body_upscale_upscale_post" + "$ref": "#/components/schemas/Body_audio_to_text_audio_to_text_post" } } }, @@ -269,7 +269,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ImageResponse" + "$ref": "#/components/schemas/TextResponse" } } } @@ -294,6 +294,16 @@ } } }, + "413": { + "description": "Request Entity Too Large", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, "500": { "description": "Internal Server Error", "content": { @@ -322,15 +332,15 @@ ] } }, - "/audio-to-text": { + "/frame_interpolation": { "post": { - "summary": "Audio To Text", - "operationId": "audio_to_text", + "summary": "Frame Interpolation", + "operationId": "frame_interpolation", "requestBody": { "content": { "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/Body_audio_to_text_audio_to_text_post" + "$ref": "#/components/schemas/Body_frame_interpolation_frame_interpolation_post" } } }, @@ -342,7 +352,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TextResponse" + "$ref": "#/components/schemas/ImageResponse" } } } @@ -367,8 +377,71 @@ } } }, - "413": { - "description": "Request Entity Too Large", + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + }, + "security": [ + { + "HTTPBearer": [] + } + ] + } + }, + "/upscale": { + "post": { + "summary": "Upscale", + "operationId": "upscale", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_upscale_upscale_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImageResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPError" + } + } + } + }, + "401": { + "description": "Unauthorized", "content": { "application/json": { "schema": { @@ -440,6 +513,60 @@ ], "title": "Body_audio_to_text_audio_to_text_post" }, + "Body_frame_interpolation_frame_interpolation_post": { + "properties": { + "model_id": { + "type": "string", + "title": "Model Id" + }, + "image1": { + "anyOf": [ + { + "type": "string", + "format": "binary" + }, + { + "type": "null" + } + ], + "title": "Image1" + }, + "image2": { + "anyOf": [ + { + "type": "string", + "format": "binary" + }, + { + "type": "null" + } + ], + "title": "Image2" + }, + "image_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Image Dir", + "default": "" + }, + "inter_frames": { + "type": "integer", + "title": "Inter Frames", + "default": 2 + } + }, + "type": "object", + "required": [ + "model_id" + ], + "title": "Body_frame_interpolation_frame_interpolation_post" + }, "Body_image_to_image_image_to_image_post": { "properties": { "prompt": { diff --git a/runner/requirements.txt b/runner/requirements.txt index 3852800d..0a75fd8a 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -6,6 +6,8 @@ pydantic==2.7.2 Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 +setuptools +torch huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 @@ -17,3 +19,4 @@ numpy==1.26.4 av==12.1.0 sentencepiece== 0.2.0 protobuf==5.27.2 +opencv-python==4.10.0.84 From 65d2c5390698a225a1a0c45be05bae5d509b20b6 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 11:51:56 +0300 Subject: [PATCH 02/18] minor changes to requirements --- runner/requirements.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/runner/requirements.txt b/runner/requirements.txt index 0a75fd8a..c02b564e 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -1,13 +1,15 @@ diffusers==0.29.2 accelerate==0.30.1 -transformers==4.41.1 +transformers==4.43.1 fastapi==0.111.0 pydantic==2.7.2 Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 -setuptools -torch +setuptools==71.1.0 +torch==2.4.0+cu121 +torchaudio==2.4.0+cu121 +torchvision==0.19.0+cu121 huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 From 796796dfee1bcec784e65147288969eec9fda70a Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 12:18:07 +0300 Subject: [PATCH 03/18] update to requrements to fetch from --index-url --- runner/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/requirements.txt b/runner/requirements.txt index c02b564e..82877c24 100644 --- a/runner/requirements.txt +++ b/runner/requirements.txt @@ -7,9 +7,9 @@ Pillow==10.3.0 python-multipart==0.0.9 uvicorn==0.30.0 setuptools==71.1.0 -torch==2.4.0+cu121 -torchaudio==2.4.0+cu121 -torchvision==0.19.0+cu121 +torch --index-url https://download.pytorch.org/whl/cu121 +torchvision --index-url https://download.pytorch.org/whl/cu121 +torchaudio --index-url https://download.pytorch.org/whl/cu121 huggingface_hub==0.23.2 xformers==0.0.23 triton>=2.1.0 From b5eb66df646a293d5a842b7cb5aa27437f2f4699 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 14:12:57 +0300 Subject: [PATCH 04/18] simple patch to solve the go api bindings issue --- runner/app/routes/frame_interpolation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/app/routes/frame_interpolation.py b/runner/app/routes/frame_interpolation.py index a1ae75bc..6abf6a66 100644 --- a/runner/app/routes/frame_interpolation.py +++ b/runner/app/routes/frame_interpolation.py @@ -36,9 +36,9 @@ ) async def frame_interpolation( model_id: Annotated[str, Form()], - image1: Annotated[Optional[UploadFile], File()]=None, - image2: Annotated[Optional[UploadFile], File()]=None, - image_dir: Annotated[Optional[str], Form()]="", + image1: Annotated[UploadFile, File()]=None, + image2: Annotated[UploadFile, File()]=None, + image_dir: Annotated[str, Form()]="", inter_frames: Annotated[int, Form()] = 2, token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False)), ): From 522ca4f3d31f7e694b38e3079deafb7cb01f6532 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 14:47:44 +0300 Subject: [PATCH 05/18] checking if it works in my system --- runner/openapi.json | 31 +----- worker/runner.gen.go | 230 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 209 insertions(+), 52 deletions(-) diff --git a/runner/openapi.json b/runner/openapi.json index c103d2f6..2e2bb4e7 100644 --- a/runner/openapi.json +++ b/runner/openapi.json @@ -520,38 +520,17 @@ "title": "Model Id" }, "image1": { - "anyOf": [ - { - "type": "string", - "format": "binary" - }, - { - "type": "null" - } - ], + "type": "string", + "format": "binary", "title": "Image1" }, "image2": { - "anyOf": [ - { - "type": "string", - "format": "binary" - }, - { - "type": "null" - } - ], + "type": "string", + "format": "binary", "title": "Image2" }, "image_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "title": "Image Dir", "default": "" }, diff --git a/worker/runner.gen.go b/worker/runner.gen.go index 0dbe8036..788a7e82 100644 --- a/worker/runner.gen.go +++ b/worker/runner.gen.go @@ -37,6 +37,15 @@ type BodyAudioToTextAudioToTextPost struct { ModelId *string `json:"model_id,omitempty"` } +// BodyFrameInterpolationFrameInterpolationPost defines model for Body_frame_interpolation_frame_interpolation_post. +type BodyFrameInterpolationFrameInterpolationPost struct { + Image1 *openapi_types.File `json:"image1,omitempty"` + Image2 *openapi_types.File `json:"image2,omitempty"` + ImageDir *string `json:"image_dir,omitempty"` + InterFrames *int `json:"inter_frames,omitempty"` + ModelId string `json:"model_id"` +} + // BodyImageToImageImageToImagePost defines model for Body_image_to_image_image_to_image_post. type BodyImageToImageImageToImagePost struct { GuidanceScale *float32 `json:"guidance_scale,omitempty"` @@ -155,6 +164,9 @@ type Chunk struct { // AudioToTextMultipartRequestBody defines body for AudioToText for multipart/form-data ContentType. type AudioToTextMultipartRequestBody = BodyAudioToTextAudioToTextPost +// FrameInterpolationMultipartRequestBody defines body for FrameInterpolation for multipart/form-data ContentType. +type FrameInterpolationMultipartRequestBody = BodyFrameInterpolationFrameInterpolationPost + // ImageToImageMultipartRequestBody defines body for ImageToImage for multipart/form-data ContentType. type ImageToImageMultipartRequestBody = BodyImageToImageImageToImagePost @@ -305,6 +317,9 @@ type ClientInterface interface { // AudioToTextWithBody request with any body AudioToTextWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // FrameInterpolationWithBody request with any body + FrameInterpolationWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // Health request Health(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -335,6 +350,18 @@ func (c *Client) AudioToTextWithBody(ctx context.Context, contentType string, bo return c.Client.Do(req) } +func (c *Client) FrameInterpolationWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewFrameInterpolationRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) Health(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewHealthRequest(c.Server) if err != nil { @@ -436,6 +463,35 @@ func NewAudioToTextRequestWithBody(server string, contentType string, body io.Re return req, nil } +// NewFrameInterpolationRequestWithBody generates requests for FrameInterpolation with any type of body +func NewFrameInterpolationRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/frame_interpolation") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewHealthRequest generates requests for Health func NewHealthRequest(server string) (*http.Request, error) { var err error @@ -636,6 +692,9 @@ type ClientWithResponsesInterface interface { // AudioToTextWithBodyWithResponse request with any body AudioToTextWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AudioToTextResponse, error) + // FrameInterpolationWithBodyWithResponse request with any body + FrameInterpolationWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*FrameInterpolationResponse, error) + // HealthWithResponse request HealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*HealthResponse, error) @@ -681,6 +740,32 @@ func (r AudioToTextResponse) StatusCode() int { return 0 } +type FrameInterpolationResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *ImageResponse + JSON400 *HTTPError + JSON401 *HTTPError + JSON422 *HTTPValidationError + JSON500 *HTTPError +} + +// Status returns HTTPResponse.Status +func (r FrameInterpolationResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r FrameInterpolationResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type HealthResponse struct { Body []byte HTTPResponse *http.Response @@ -816,6 +901,15 @@ func (c *ClientWithResponses) AudioToTextWithBodyWithResponse(ctx context.Contex return ParseAudioToTextResponse(rsp) } +// FrameInterpolationWithBodyWithResponse request with arbitrary body returning *FrameInterpolationResponse +func (c *ClientWithResponses) FrameInterpolationWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*FrameInterpolationResponse, error) { + rsp, err := c.FrameInterpolationWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseFrameInterpolationResponse(rsp) +} + // HealthWithResponse request returning *HealthResponse func (c *ClientWithResponses) HealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*HealthResponse, error) { rsp, err := c.Health(ctx, reqEditors...) @@ -930,6 +1024,60 @@ func ParseAudioToTextResponse(rsp *http.Response) (*AudioToTextResponse, error) return response, nil } +// ParseFrameInterpolationResponse parses an HTTP response from a FrameInterpolationWithResponse call +func ParseFrameInterpolationResponse(rsp *http.Response) (*FrameInterpolationResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &FrameInterpolationResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest ImageResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 422: + var dest HTTPValidationError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON422 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest HTTPError + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseHealthResponse parses an HTTP response from a HealthWithResponse call func ParseHealthResponse(rsp *http.Response) (*HealthResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -1177,6 +1325,9 @@ type ServerInterface interface { // Audio To Text // (POST /audio-to-text) AudioToText(w http.ResponseWriter, r *http.Request) + // Frame Interpolation + // (POST /frame_interpolation) + FrameInterpolation(w http.ResponseWriter, r *http.Request) // Health // (GET /health) Health(w http.ResponseWriter, r *http.Request) @@ -1204,6 +1355,12 @@ func (_ Unimplemented) AudioToText(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotImplemented) } +// Frame Interpolation +// (POST /frame_interpolation) +func (_ Unimplemented) FrameInterpolation(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + // Health // (GET /health) func (_ Unimplemented) Health(w http.ResponseWriter, r *http.Request) { @@ -1260,6 +1417,23 @@ func (siw *ServerInterfaceWrapper) AudioToText(w http.ResponseWriter, r *http.Re handler.ServeHTTP(w, r.WithContext(ctx)) } +// FrameInterpolation operation middleware +func (siw *ServerInterfaceWrapper) FrameInterpolation(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + ctx = context.WithValue(ctx, HTTPBearerScopes, []string{}) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.FrameInterpolation(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r.WithContext(ctx)) +} + // Health operation middleware func (siw *ServerInterfaceWrapper) Health(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -1459,6 +1633,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/audio-to-text", wrapper.AudioToText) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/frame_interpolation", wrapper.FrameInterpolation) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/health", wrapper.Health) }) @@ -1481,32 +1658,33 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xZ227bOBN+FYL/f+nEhzabhe+SbLcNtoegdrsXRWAw0thmK5FaHtJ6A7/7gkNZomQp", - "cpDGC2R9Zcsaznxz+IZD+o5GMs2kAGE0Hd9RHS0hZfj17OrylVJSue+ZkhkowwHfpHrhPgw3CdAxfacX", - "tEfNKnMP2iguFnS97lEFf1muIKbjL7jkulcsKXQX6+TNV4gMXffouYxXM2ZjLmdGzgz8MLWnTGqzDQpl", - "3Je5VCkzdExvuGBqRQOrKLIFtUdTGUMy47FbHsOc2cStD1a+cwLkMu7006MIPN3Nm7Yw8JQtwIn6L7XH", - "5kAsLI+ZiGCmI+YgBC6dHp+UyF7ncmSCcgUEYdMbUA4CWrk/pJco0hBSj/AeLMMQC6oh3YgekageFbBg", - "ht/CLFMyzUyrjve5HLnyck2qbOpzoGcZqCaFw0CfTQk6qMkVqC2tXBhYePdQrZiDAoyZgUxXlQ4GNbUb", - "YTJB4SalJbjNyna/NJuDWc2iJUTfKpaNslCanqAYuUCxQs2NlAkwgXoA4tDixD03gdNGgViYZcXY4PjX", - "wNZGYqscatTLNl75sq1zcAcqdbLwlscg64/NLJzXUvdLCef3lkQtgS+W1So6OQ3WvfHvm5Y+hqmP4lQq", - "DZdidmOjb2DqSoaj01CLkyTnKFnRFhJAcg0zZhezlsIYjAICOGFyZhekvUa6OTU6eTil9k6T7zyuhWI4", - "GL0sLf2J77dX1ijSwYz28m5jhs2wsRefzVz416qzK/enJ8+qnT6sITbmriHRb6bTq5ZBMAbDeOK+/V/B", - "nI7p//rlONnPZ8l+MezVAebLA2ClrRYgn1nCY+Y6SSckbiDVXdjq+tYllt+8pgIIU4qt0IcQbV1BE25g", - "iVlebIqgilcbZmy1KumHP2i4/6FA0+BZbgylgQb7yK2PoDMpNLSwU+8csXcQcxbGyY82TXHaaj06zHUV", - "VgNub2kLr9Dz7yEZ3rvnR3VXq5JQ7pNKOud8izLaa0REgWceeINHU/hh2hMRLa34tnsiUDxMxIVfX09E", - "j7pzRuigg9HpofFCOajAu4oTLU5OJWb3iinmHXmqI0o5M+0wJf3HTw8nz+3wUExFDxyDcqdqNV2t2YbC", - "7tx7EhlV2MvE6sOcjr/cbcXqbgvidUDktzJCMw1Url+9gNYtg5P/oRRFzGTqfu2ivvPDm8olg0jtsN99", - "dnNje5ubK5bW9psHbjz19rY5V3nFHRtRbj50qYK3wSHfabcc2a2tOjspaMPSLHQ1wD0t3ndAN6GgMxY4", - "4TFugUc6RVZxs5q4OHrkbnA5B6ZAFXd+yEH/U6FkaUxG104HF3PpKa0jxTMszjE9E4RlWcJ9tRIjibKC", - "nF2SjGeQcOGTsSlqfgsZgHLvP1oh0NAtKO11DY6HxwMXLZmBYBmnY/oCf+rRjJklwu7jzdmRkUeb0G/O", - "Gy4tCOIy3tzzTWWeDxdB0MbNvLjLSmFA4KrUJoZnTJm+O5gcxcyw8g60qxx3u9hbV3PoOiH+4IsNvRoN", - "BjVcQVD7X7ULz66gKnsz2q5mbGKjCLSe24SUYj368idCKEf4BvvnLCYffT683eF+7H4SzJqlVPxviNHw", - "8MV+DOfOklfCcLMiUynJW6YWPuqj0U8FsXWW2YZTipDivHOyr+RfCgNKsIRMQN2CIuWhcNOicK8Mm9OX", - "6/V1j2qbpkytNswmU0mQ225pf4mHH5wqoaEX+LMRfULOhaevXSm3Dp3KIaI3OBa6DlfcmTS3OBxV8onl", - "iXvcDhene+5y1ZPjoc21t7lDh3loh/H/RE2lP3PVSIk3op2kxHlyX6Rsv7PdMymrU/SBlAdSPgEpPbWQ", - "lG7G3mGjDE7291LycTN39e7gsB0emPdMmOeKu7Yb5v8XtVPuUy7wtDtg499XB+YdmPdMmLdh0dqvcmo0", - "LqpaKq7VLhJpY3Ih09QKblbkNTPwna1o/vcWXubpcb8fK2Dp0cK/PU7y5ceRW07X1+t/AgAA///2pVcb", - "EigAAA==", + "H4sIAAAAAAAC/+xZWW/bOhb+KwRnHp14aTMZ+C1Jt2C6BI3beSgCg5GObbYSqUtSaX0D//cLHsoSJVOR", + "jTS+QK6fvOgs31m+w0X3NJJpJgUIo+n4nupoASnDr2dXl6+Vksp+z5TMQBkO+CTVc/thuEmAjukHPac9", + "apaZ/aGN4mJOV6seVfBHzhXEdPwNVW56pUppu9STt98hMnTVo+cyXk5ZHnM5NXJq4Jdp/MqkNpugUMZ+", + "mUmVMkPH9JYLppbU84oiG1B7NJUxJFMeW/UYZixPrL6n+cEKkMu4M06Hwot0u2ja0jBTLIUpFwZUJhNm", + "uBTB/8Ip4Smbw/DhnFw6mUBSUHu0hfaoVXsac9WaU9Qlr7gKqtvwXKi6ZmHkGbAy5I2TKW1YzTmoZll3", + "rmSp3CzmTjVpK6xLj5HFl8bPcDnnOY+ZiGCqI2bheFk5PT6pUL4t5Mg1ypUQRJ7eusSgly0q217YB7AM", + "fSyuyN2IHsHAHhUwZ4bfwTRTMs1Mq42PhRy5cnIhU3nqaqCnGaiQwaFnL08JBqjJFagNq14nolkxAwWY", + "MwNZvauHg0HD7FqYXKNwyGgFbq3ZHpdmMzDLabSA6EfNs1E5VK6vUYxcoFhp5lbKBJhAOwA1Ol3b3yFw", + "2igQc7OoORsc/9fztZbYaIcGE7N1VK5tm3zcgkqdLLzjMcjmzzALZ43S/aeC86alUAvg80W9i05OPb13", + "7nlI9TFMfRSnUolD7DaPfoBpGhmOTn0rVpKco2TNmk8AyTVMWT6ftjTGwBvsH60wOcvnpL1Hujk1Otmd", + "UnunyU8eN1IxHIxeVp7+j883NRsU6WBGe3u3MSPPcLCXnw9sMP6O7uyq/enJsxqnuw3EYO0ChX43mVy1", + "7PBjMIwn9tu/FczomP6rX50T+sUhoV/u4psAC3UPWOWrBchXlvAYN06dkLiBVHdha9pbVVheOUslEKYU", + "W2IMPtqmgRBuYIlZXKyboI5XG2byelfST/+j/vqHAqF9aLUwVA4C/pFbn0FnUmhoYafeOmMfIObMz5Pb", + "2oTytDF6tF/rOqwAbudpA6/Qs58+GT7a34+arrlKfLkvKunc9ucoo51FRORF5oAHIprAL9NeiGiRix/b", + "FwLF/UJcOP1mIXrUHiD9AC2MzgiNEypAedHVgmgJciKxuldMMRfIUx1Rqj3TFrukf/jp4eS5HR7KXdGO", + "26AiqEZP13s20Nida08ioxp7mVh+mtHxt/uNXN1vQLzxiPxeRugmQOXmnRpo3bJxcn9UooiZTOy/XdS3", + "cThXhaSXqS3Wu69239g+5qrbmjJROy48zfG2Plc1rnjCC1Hh3g+phjcQkJu0G4FsN1atnxS0YWnmh+rh", + "npTPO6AbX9A684JwGDfAI52iXHGzvLZ5dMjtxuUcmAJVXuYiB91fpZGFMRldrfCebSYdpXWkeIbNOaZn", + "grAsS7jrVmIkUbkgZ5ck4xkkXLhirJua30EGoOzzz7kQ6OgOlHa2BsfD44HNlsxAsIzTMX2Bf/VoxswC", + "YffxSvTIyKN16tfnDVsWBHEZry9wJ7Koh80gaGP3vLjKSmFAoFaaJ4ZnTJm+PZgcxcyw6nK7qx23u7Fd", + "1WtoJyH+4ZoNoxoNBg1cXlL737VNz7agamsz+q5X7DqPItB6liekEuvRl78RQrWFD/g/ZzH57Orh/A73", + "4/eLYLlZSMX/hBgdD1/sx3ERLHktDDdLMpGSvGdq7rI+Gv1WEBtnmU04lQgpzzsn+yo+XsQLlpBrUHeg", + "SHUoXI8oXCv94fTtZnXTozpPU6aWa2aTiSTIbavaD9ypt08GXCIua7JPOyB2eguw52FRP4AdpkX7tDgQ", + "dVeiItFInWlI1wXeVeAhEAIEdVcZ9Am73r8s2bbnV35oBUSMBk9xdkNSXnGG5w5SrThgPPHE2eI9x2HO", + "HObMM5kz7sXxRLorkgYp8QVGJynx+LcvUra/YtkzKeuH3gMpD6R8AlI6aiEp7ZF4i4XSu4h7kJKPOyLX", + "r/oOy+GBec+Eeba5G6th8Xq3nXJfCoGnXQGDb5sPzDsw75kwb82ildOyZjQq1T2Vt+AXicxjciHTNBfc", + "LMlbZuAnW9LibTTevetxvx8rYOnR3D09Tgr148iq09XN6q8AAAD///BmUXaaLQAA", } // GetSwagger returns the content of the embedded swagger specification file From ef155aa2219f625eba187b50dbd550b1ae81737e Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Fri, 26 Jul 2024 18:22:36 +0300 Subject: [PATCH 06/18] test-examples for frame-interpolation --- .../app/test_examples/continuous-creation.py | 154 +++++++++++++++++ .../generate-interpolate-upscale.py | 159 ++++++++++++++++++ .../test_examples/test-film-interpolate.py | 45 +++++ runner/app/test_examples/test-optimized.py | 69 ++++++++ .../test_examples/utility-func-compactor.py | 89 ++++++++++ .../app/test_examples/utility-function-vid.py | 62 +++++++ runner/uvicorn.env | 4 + 7 files changed, 582 insertions(+) create mode 100644 runner/app/test_examples/continuous-creation.py create mode 100644 runner/app/test_examples/generate-interpolate-upscale.py create mode 100644 runner/app/test_examples/test-film-interpolate.py create mode 100644 runner/app/test_examples/test-optimized.py create mode 100644 runner/app/test_examples/utility-func-compactor.py create mode 100644 runner/app/test_examples/utility-function-vid.py create mode 100644 runner/uvicorn.env diff --git a/runner/app/test_examples/continuous-creation.py b/runner/app/test_examples/continuous-creation.py new file mode 100644 index 00000000..54473930 --- /dev/null +++ b/runner/app/test_examples/continuous-creation.py @@ -0,0 +1,154 @@ +import torch +import numpy as np +import os +import shutil +import time +import sys +sys.path.append('C://Users//ganes//ai-worker//runner') +from PIL import PngImagePlugin, Image +from diffusers import StableVideoDiffusionPipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter + +# Increase the max text chunk size for PNG images +PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) + +def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): + for attempt in range(retries): + try: + shutil.move(src_file_path, dst_file_path) + return + except PermissionError: + print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") + time.sleep(delay) + raise PermissionError(f"Failed to move file after {retries} attempts.") + +def get_last_file_sorted_by_name(directory: str) -> str: + """ + Get the last file in the directory when sorted by filename. + + Args: + directory (str): Path to the directory. + + Returns: + str: Path to the last file in the sorted list, or None if directory is empty. + """ + try: + # List all files in the directory + files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] + + if not files: + print("No files found in the directory.") + return None + + # Sort files by name + files.sort() + + # Get the last file in the sorted list + last_file = files[-1] + + return os.path.join(directory, last_file) + + except Exception as e: + print(f"An error occurred: {e}") + return None + +def main(): + # Initialize pipelines + repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" + svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( + repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" + ) + svd_xt_pipeline.enable_model_cpu_offload() + + film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) + + # Load initial input image + image_path = "G:/ai-models/models/gif_frames/donut_motion.png" + image = Image.open(image_path) + + fps = 24.0 + inter_frames = 4 + rounds = 2 # Number of rounds of generation and interpolation + base_output_dir = "G:/ai-models/models" + + all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") + os.makedirs(all_frames_dir, exist_ok=True) + + last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") + + for round_num in range(1, rounds + 1): + svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") + os.makedirs(svd_xt_output_dir, exist_ok=True) + + # Generate frames using SVD pipeline + generator = torch.manual_seed(42) + frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] + + # Save SVD frames to directory + film_writer = DirectoryWriter(svd_xt_output_dir) + for idx, frame in enumerate(frames): + film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) + + # Read saved frames for interpolation + film_reader = DirectoryReader(svd_xt_output_dir) + height, width = film_reader.get_resolution() + + # Interpolate frames using FILM pipeline + film_pipeline(film_reader, film_writer, inter_frames=inter_frames) + + # Close reader and writer + film_writer.close() + film_reader.reset() + + # Deleting the SVD generated images. + for i in range(len(frames)): + os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) + print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") + + # Save the last frame separately for the next round + last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) + if last_frame_path: + shutil.copy2(last_frame_path, last_frame_for_next_round) + else: + print("No frames found to copy.") + + # Move all interpolated frames to a common directory with a unique naming scheme + for file_name in sorted(os.listdir(svd_xt_output_dir)): + src_file_path = os.path.join(svd_xt_output_dir, file_name) + dst_file_name = f"round_{round_num:03d}_frame_{file_name}" + dst_file_path = os.path.join(all_frames_dir, dst_file_name) + + move_file_with_retry(src_file_path, dst_file_path) + + # Clean up the source directory after moving frames + for file_name in os.listdir(svd_xt_output_dir): + os.remove(os.path.join(svd_xt_output_dir, file_name)) + os.rmdir(svd_xt_output_dir) + + # Ensure all operations on last frame are complete before opening it again + time.sleep(1) # Small delay to ensure file system operations are complete + + # Prepare for next round + image = Image.open(last_frame_for_next_round) + + # Compile all interpolated frames from all rounds into a final video + video_output_dir = "G:/ai-models/models/video_out" + os.makedirs(video_output_dir, exist_ok=True) + + film_output_path = os.path.join(video_output_dir, "output.avi") + frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) + + # Clean up all frames in the directories after video generation + for file_name in os.listdir(all_frames_dir): + file_path = os.path.join(all_frames_dir, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + os.rmdir(all_frames_dir) + + print(f"All frames deleted from directories.") + print(f"Video generated at: {film_output_path}") + return film_output_path + +if __name__ == "__main__": + main() diff --git a/runner/app/test_examples/generate-interpolate-upscale.py b/runner/app/test_examples/generate-interpolate-upscale.py new file mode 100644 index 00000000..69954a0a --- /dev/null +++ b/runner/app/test_examples/generate-interpolate-upscale.py @@ -0,0 +1,159 @@ +import logging +import os +os.environ["MODEL_DIR"] = "G://ai-models//models" +import shutil +import time +import sys +sys.path.append('C://Users//ganes//ai-worker//runner') +from typing import List, Optional, Tuple +import PIL +import torch +from diffusers import StableVideoDiffusionPipeline +from PIL import PngImagePlugin, Image, ImageFile + +from app.pipelines.base import Pipeline +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.upscale import UpscalePipeline +from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter, SafetyChecker, get_model_dir, get_torch_device, is_lightning_model, is_turbo_model +from huggingface_hub import file_download + +# Increase the max text chunk size for PNG images +PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) +ImageFile.LOAD_TRUNCATED_IMAGES = True + +logger = logging.getLogger(__name__) + +# Helper function to move files with retry mechanism +def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): + for attempt in range(retries): + try: + shutil.move(src_file_path, dst_file_path) + return + except PermissionError: + print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") + time.sleep(delay) + raise PermissionError(f"Failed to move file after {retries} attempts.") + +# Helper function to get the last file in a directory sorted by filename +def get_last_file_sorted_by_name(directory: str) -> str: + try: + files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] + if not files: + print("No files found in the directory.") + return None + files.sort() + last_file = files[-1] + return os.path.join(directory, last_file) + except Exception as e: + print(f"An error occurred: {e}") + return None + +def main(): + # Initialize SVD and FILM pipelines + repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" + svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( + repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" + ) + svd_xt_pipeline.enable_model_cpu_offload() + + film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) + + # Load initial input image + image_path = "G:/ai-models/models/gif_frames/donut_motion.png" + image = Image.open(image_path) + + fps = 24.0 + inter_frames = 4 + rounds = 2 # Number of rounds of generation and interpolation + base_output_dir = "G:/ai-models/models" + + all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") + os.makedirs(all_frames_dir, exist_ok=True) + + last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") + + for round_num in range(1, rounds + 1): + svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") + os.makedirs(svd_xt_output_dir, exist_ok=True) + + # Generate frames using SVD pipeline + generator = torch.manual_seed(42) + frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] + + # Save SVD frames to directory + film_writer = DirectoryWriter(svd_xt_output_dir) + for idx, frame in enumerate(frames): + film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) + + # Read saved frames for interpolation + film_reader = DirectoryReader(svd_xt_output_dir) + height, width = film_reader.get_resolution() + + # Interpolate frames using FILM pipeline + film_pipeline(film_reader, film_writer, inter_frames=inter_frames) + + # Close reader and writer + film_writer.close() + film_reader.reset() + + # Deleting the SVD generated images. + for i in range(len(frames)): + os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) + print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") + + # Save the last frame separately for the next round + last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) + if last_frame_path: + shutil.copy2(last_frame_path, last_frame_for_next_round) + else: + print("No frames found to copy.") + + # Initialize Upscale pipeline and Upscale the last frame before passing to the next round + upscale_pipeline = UpscalePipeline("stabilityai/stable-diffusion-x4-upscaler", torch_dtype=torch.float16) + upscale_pipeline.enable_model_cpu_offload() + upscale_pipeline.sfast_enabled() + upscaled_image, _ = upscale_pipeline("", image=Image.open(last_frame_for_next_round),) + print('Upscaling of the seed image before next round.') + print(upscaled_image[0].shape) + exit + upscaled_image[0].save(last_frame_for_next_round) + + # Move all interpolated frames to a common directory with a unique naming scheme + for file_name in sorted(os.listdir(svd_xt_output_dir)): + src_file_path = os.path.join(svd_xt_output_dir, file_name) + dst_file_name = f"round_{round_num:03d}_frame_{file_name}" + dst_file_path = os.path.join(all_frames_dir, dst_file_name) + + move_file_with_retry(src_file_path, dst_file_path) + + # Clean up the source directory after moving frames + for file_name in os.listdir(svd_xt_output_dir): + os.remove(os.path.join(svd_xt_output_dir, file_name)) + os.rmdir(svd_xt_output_dir) + + # Ensure all operations on last frame are complete before opening it again + time.sleep(1) # Small delay to ensure file system operations are complete + + # Prepare for next round + image = Image.open(last_frame_for_next_round) + + # Compile all interpolated frames from all rounds into a final video + video_output_dir = "G:/ai-models/models/video_out" + os.makedirs(video_output_dir, exist_ok=True) + + film_output_path = os.path.join(video_output_dir, "output.avi") + frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) + + # Clean up all frames in the directories after video generation + for file_name in os.listdir(all_frames_dir): + file_path = os.path.join(all_frames_dir, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + os.rmdir(all_frames_dir) + + print(f"All frames deleted from directories.") + print(f"Video generated at: {film_output_path}") + return film_output_path + +if __name__ == "__main__": + main() diff --git a/runner/app/test_examples/test-film-interpolate.py b/runner/app/test_examples/test-film-interpolate.py new file mode 100644 index 00000000..195a3a4b --- /dev/null +++ b/runner/app/test_examples/test-film-interpolate.py @@ -0,0 +1,45 @@ +import os +import requests + +# Define the URL of the FastAPI application +URL = "http://localhost:8000/FILMPipeline" + +# Test with two images +def test_with_two_images(): + image1_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_74.png" + image2_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_36.png" + + with open(image1_path, "rb") as image1, open(image2_path, "rb") as image2: + files = { + "image1": ("image1.png", image1, "image/png"), + "image2": ("image2.png", image2, "image/png"), + } + data = { + "inter_frames": 2, + "model_id": "film_net_fp16.pt" + } + response = requests.post(URL, files=files, data=data) + + print("Test with two images") + print(response.status_code) + print(response.json()) + +# Test with a directory of images +def test_with_image_directory(): + image_dir = "path/to/image_directory" + + data = { + "inter_frames": 2, + "model_path": "path/to/film_net_fp16.pt", + "image_dir": image_dir + } + response = requests.post(URL, data=data) + + print("Test with image directory") + print(response.status_code) + print(response.json()) + +if __name__ == "__main__": + # Ensure that the FastAPI server is running before executing these tests + test_with_two_images() + test_with_image_directory() diff --git a/runner/app/test_examples/test-optimized.py b/runner/app/test_examples/test-optimized.py new file mode 100644 index 00000000..db200286 --- /dev/null +++ b/runner/app/test_examples/test-optimized.py @@ -0,0 +1,69 @@ +import torch +import numpy as np +import os +import sys +sys.path.append('C://Users//ganes//ai-worker//runner') +from diffusers import StableVideoDiffusionPipeline +from PIL import PngImagePlugin, Image +from app.pipelines.frame_interpolation import FILMPipeline +from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter + +# Increase the max text chunk size for PNG images +PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) + +def main(): + # Initialize pipelines + repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" + svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( + repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" + ) + svd_xt_pipeline.enable_model_cpu_offload() + + film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) + + # Load input image + image_path = "G:/ai-models/models/gif_frames/rocket.png" + image = Image.open(image_path) + + # Generate frames using SVD pipeline + generator = torch.manual_seed(42) + frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] + + fps = 24.0 + inter_frames = 2 + svd_xt_output_dir = "G:/ai-models/models/svd_xt_output" + video_output_dir = "G:/ai-models/models/video_out" + + # Save SVD frames to directory + film_writer = DirectoryWriter(svd_xt_output_dir) + for frame in frames: + film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) + + # Read saved frames for interpolation + film_reader = DirectoryReader(svd_xt_output_dir) + height, width = film_reader.get_resolution() + + # Interpolate frames using FILM pipeline + film_pipeline(film_reader, film_writer, inter_frames=inter_frames) + + # Delete original SVD frames since interpolated frames are also in the same directory. + for i in range(len(frames)): + os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) + print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") + + # Compile interpolated frames into a video + film_output_path = os.path.join(video_output_dir, "output.avi") + frames_compactor(frames=svd_xt_output_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) + + # Clean up all frames in the directory after video generation + for file_name in os.listdir(svd_xt_output_dir): + file_path = os.path.join(svd_xt_output_dir, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + print(f"All frames deleted from directory: {svd_xt_output_dir}") + + print(f"Video generated at: {film_output_path}") + return film_output_path + +if __name__ == "__main__": + main() diff --git a/runner/app/test_examples/utility-func-compactor.py b/runner/app/test_examples/utility-func-compactor.py new file mode 100644 index 00000000..6dac4309 --- /dev/null +++ b/runner/app/test_examples/utility-func-compactor.py @@ -0,0 +1,89 @@ +import subprocess +import numpy as np +import torch +import os +from typing import List, Union +from pathlib import Path +import cv2 + +def generate_video_from_frames( + frames: Union[List[np.ndarray], List[torch.Tensor]], + output_path: str, + fps: float, + codec: str = "MJPEG", + is_directory: bool = False, + width: int = None, + height: int = None +) -> None: + """ + Generate a video from a list of frames. Frames can be from a directory or in-memory. + + Args: + frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. + output_path (str): Path to save the output video file. + fps (float): Frames per second for the video. + codec (str): Codec used for video compression (default is "MJPEG"). + is_directory (bool): If True, treat `frames` as a directory path containing image files. + width (int): Width of the video. Must be provided if `frames` are in-memory. + height (int): Height of the video. Must be provided if `frames` are in-memory. + + Returns: + None + """ + if is_directory: + # Read frames from a directory + frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] + else: + # Convert torch tensors to numpy arrays if necessary + if isinstance(frames[0], torch.Tensor): + frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] + + # Ensure frames are numpy arrays and are uint8 type + frames = [frame.astype(np.uint8) for frame in frames] + + # Check if frames are consistent + if not frames: + raise ValueError("No frames to process.") + + if width is None or height is None: + # Use dimensions of the first frame if not provided + height, width = frames[0].shape[:2] + + # Write frames to a temporary directory + temp_dir = Path("temp_frames") + temp_dir.mkdir(exist_ok=True) + for i, frame in enumerate(frames): + cv2.imwrite(str(temp_dir / f"frame_{i:05d}.png"), frame) + + # Build ffmpeg command + ffmpeg_cmd = [ + 'ffmpeg', '-y', '-framerate', str(fps), + '-i', str(temp_dir / 'frame_%05d.png'), + '-c:v', codec, '-pix_fmt', 'yuv420p', + output_path + ] + + # Run ffmpeg command + subprocess.run(ffmpeg_cmd, check=True) + + # Clean up temporary frames + for file in temp_dir.glob("*.png"): + file.unlink() + temp_dir.rmdir() + + print(f"Video saved to {output_path}") + +# Example usage +if __name__ == "__main__": + # Example with in-memory frames (as np.ndarray) + # Assuming `in_memory_frames` is a list of numpy arrays + + # Example with frames from a directory + frames_directory = "G:/ai-models/models/svd_xt_output" + generate_video_from_frames( + frames=frames_directory, + output_path="G:/ai-models/models/video_out/output.mp4", + fps=24.0, + codec="mpeg4", + is_directory=True + ) diff --git a/runner/app/test_examples/utility-function-vid.py b/runner/app/test_examples/utility-function-vid.py new file mode 100644 index 00000000..adf194cc --- /dev/null +++ b/runner/app/test_examples/utility-function-vid.py @@ -0,0 +1,62 @@ +import cv2 +import numpy as np +import tempfile +import os +from io import BytesIO + +def extract_frames_from_video(video_data, is_file_path=True) -> np.ndarray: + """ + Extract frames from a video file or in-memory video data and return them as a NumPy array. + + Args: + video_data (str or BytesIO): Path to the input video file or in-memory video data. + is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). + + Returns: + np.ndarray: Array of frames with shape (num_frames, height, width, channels). + """ + if is_file_path: + # Handle file-based video input + video_capture = cv2.VideoCapture(video_data) + else: + # Handle in-memory video input + # Create a temporary file to store in-memory video data + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: + temp_file.write(video_data.getvalue()) + temp_file_path = temp_file.name + + # Open the temporary video file + video_capture = cv2.VideoCapture(temp_file_path) + + if not video_capture.isOpened(): + raise ValueError("Error opening video data") + + frames = [] + success, frame = video_capture.read() + + while success: + frames.append(frame) + success, frame = video_capture.read() + + video_capture.release() + + # Delete the temporary file if it was created + if not is_file_path: + os.remove(temp_file_path) + + # Convert list of frames to a NumPy array + frames_array = np.array(frames) + print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") + + return frames_array + +# Example usage +if __name__ == "__main__": + # File path example + video_file_path = "C:/Users/ganes/Desktop/Generated videos/output.mp4" + + + # In-memory video example + with open(video_file_path, "rb") as f: + video_data = BytesIO(f.read()) + frames_array_from_memory = extract_frames_from_video(video_data, is_file_path=False) diff --git a/runner/uvicorn.env b/runner/uvicorn.env new file mode 100644 index 00000000..77869fee --- /dev/null +++ b/runner/uvicorn.env @@ -0,0 +1,4 @@ +# myenvfile.env +MODEL_ID="" +MODEL_DIR="" +PIPELINE=FILMPipeline From 950cdf9beabc8c0e612ec7e952a2d39b666bfb1e Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 20:32:38 +0300 Subject: [PATCH 07/18] update to sfast optimization to i2i and t2i and upscale pipelines --- runner/app/pipelines/image_to_image.py | 30 +++++++++++++++++++---- runner/app/pipelines/text_to_image.py | 33 ++++++++++++++++++++------ runner/app/pipelines/upscale.py | 30 +++++++++++++++++++---- 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/runner/app/pipelines/image_to_image.py b/runner/app/pipelines/image_to_image.py index 4080c919..d5c821a5 100644 --- a/runner/app/pipelines/image_to_image.py +++ b/runner/app/pipelines/image_to_image.py @@ -1,6 +1,7 @@ import logging import os from enum import Enum +import time from typing import List, Optional, Tuple import PIL @@ -30,6 +31,7 @@ logger = logging.getLogger(__name__) +SFAST_WARMUP_ITERATIONS = 2 # Model warm-up iterations when SFAST is enabled. class ModelName(Enum): """Enumeration mapping model names to their corresponding IDs.""" @@ -142,11 +144,29 @@ def __init__(self, model_id: str): # Warm-up the pipeline. # TODO: Not yet supported for ImageToImagePipeline. if os.getenv("SFAST_WARMUP", "true").lower() == "true": - logger.warning( - "The 'SFAST_WARMUP' flag is not yet supported for the " - "ImageToImagePipeline and will be ignored. As a result the first " - "call may be slow if 'SFAST' is enabled." - ) + warmup_kwargs = { + "prompt":"A warmed up pipeline is a happy pipeline a short poem by ricksta", + "image": PIL.Image.new("RGB", (576, 1024)), + "strength": 0.8, + "negative_prompt": "No blurry or weird artifacts", + "num_images_per_prompt":4, + } + + logger.info("Warming up ImageToImagePipeline pipeline...") + total_time = 0 + for ii in range(SFAST_WARMUP_ITERATIONS): + t = time.time() + try: + self.ldm(**warmup_kwargs).images + except Exception as e: + logger.error(f"ImageToImagePipeline warmup error: {e}") + raise e + iteration_time = time.time() - t + total_time += iteration_time + logger.info( + "Warmup iteration %s took %s seconds", ii + 1, iteration_time + ) + logger.info("Total warmup time: %s seconds", total_time) if deepcache_enabled and not ( is_lightning_model(model_id) or is_turbo_model(model_id) diff --git a/runner/app/pipelines/text_to_image.py b/runner/app/pipelines/text_to_image.py index e2d6c692..85f37cdb 100644 --- a/runner/app/pipelines/text_to_image.py +++ b/runner/app/pipelines/text_to_image.py @@ -1,6 +1,7 @@ import logging import os from enum import Enum +import time from typing import List, Optional, Tuple import PIL @@ -26,6 +27,7 @@ logger = logging.getLogger(__name__) +SFAST_WARMUP_ITERATIONS = 2 # Model warm-up iterations when SFAST is enabled. class ModelName(Enum): """Enumeration mapping model names to their corresponding IDs.""" @@ -151,14 +153,31 @@ def __init__(self, model_id: str): self.ldm = compile_model(self.ldm) - # Warm-up the pipeline. - # TODO: Not yet supported for TextToImagePipeline. if os.getenv("SFAST_WARMUP", "true").lower() == "true": - logger.warning( - "The 'SFAST_WARMUP' flag is not yet supported for the " - "TextToImagePipeline and will be ignored. As a result the first " - "call may be slow if 'SFAST' is enabled." - ) + # Retrieve default model params. + # TODO: Retrieve defaults from Pydantic class in route. + warmup_kwargs = { + "prompt": "A happy pipe in the line looking at the wall with words sfast", + "num_images_per_prompt": 4, + "negative_prompt": "No blurry or weird artifacts", + } + + logger.info("Warming up TextToImagePipeline pipeline...") + total_time = 0 + for ii in range(SFAST_WARMUP_ITERATIONS): + t = time.time() + try: + self.ldm(**warmup_kwargs).images + except Exception as e: + # FIXME: When out of memory, pipeline is corrupted. + logger.error(f"TextToImagePipeline warmup error: {e}") + raise e + iteration_time = time.time() - t + total_time += iteration_time + logger.info( + "Warmup iteration %s took %s seconds", ii + 1, iteration_time + ) + logger.info("Total warmup time: %s seconds", total_time) if deepcache_enabled and not ( is_lightning_model(model_id) or is_turbo_model(model_id) diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 6fd3cefe..b743434b 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -1,5 +1,6 @@ import logging import os +import time from typing import List, Optional, Tuple import PIL @@ -21,6 +22,7 @@ logger = logging.getLogger(__name__) +SFAST_WARMUP_ITERATIONS = 2 # Model warm-up iterations when SFAST is enabled. class UpscalePipeline(Pipeline): def __init__(self, model_id: str): @@ -68,11 +70,29 @@ def __init__(self, model_id: str): # Warm-up the pipeline. # TODO: Not yet supported for UpscalePipeline. if os.getenv("SFAST_WARMUP", "true").lower() == "true": - logger.warning( - "The 'SFAST_WARMUP' flag is not yet supported for the " - "UpscalePipeline and will be ignored. As a result the first " - "call may be slow if 'SFAST' is enabled." - ) + # Retrieve default model params. + # TODO: Retrieve defaults from Pydantic class in route. + warmup_kwargs = { + "prompt": "Upscaling the pipeline with sfast enabled", + "image": PIL.Image.new("RGB", (576, 1024)), + } + + logger.info("Warming up ImageToVideoPipeline pipeline...") + total_time = 0 + for ii in range(SFAST_WARMUP_ITERATIONS): + t = time.time() + try: + self.ldm(**warmup_kwargs).images + except Exception as e: + # FIXME: When out of memory, pipeline is corrupted. + logger.error(f"ImageToVideoPipeline warmup error: {e}") + raise e + iteration_time = time.time() - t + total_time += iteration_time + logger.info( + "Warmup iteration %s took %s seconds", ii + 1, iteration_time + ) + logger.info("Total warmup time: %s seconds", total_time) if deepcache_enabled and not ( is_lightning_model(model_id) or is_turbo_model(model_id) From b48692b6499dc93a50eca8081939654bc69ecb5d Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:07:09 +0300 Subject: [PATCH 08/18] changes to extra files --- runner/app/.gitignore/base64-image.py | 62 +++++++++++++++++++ .../continuous-creation.py | 0 .../generate-interpolate-upscale.py | 0 .../test-film-interpolate.py | 0 .../test-optimized.py | 0 .../utility-func-compactor.py | 0 .../utility-function-vid.py | 0 runner/app/pipelines/optim/sfast.py | 2 +- runner/uvicorn.env | 4 -- 9 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 runner/app/.gitignore/base64-image.py rename runner/app/{test_examples => .gitignore}/continuous-creation.py (100%) rename runner/app/{test_examples => .gitignore}/generate-interpolate-upscale.py (100%) rename runner/app/{test_examples => .gitignore}/test-film-interpolate.py (100%) rename runner/app/{test_examples => .gitignore}/test-optimized.py (100%) rename runner/app/{test_examples => .gitignore}/utility-func-compactor.py (100%) rename runner/app/{test_examples => .gitignore}/utility-function-vid.py (100%) delete mode 100644 runner/uvicorn.env diff --git a/runner/app/.gitignore/base64-image.py b/runner/app/.gitignore/base64-image.py new file mode 100644 index 00000000..9116fdbe --- /dev/null +++ b/runner/app/.gitignore/base64-image.py @@ -0,0 +1,62 @@ +import json +import base64 +from PIL import Image, UnidentifiedImageError +from io import BytesIO +import os + +def extract_base64_string(data_url): + # Remove the 'data:image/png;base64,' prefix + base64_str = data_url.split(',', 1)[1] + return base64_str + +def add_padding(base64_string): + # Add padding if necessary + missing_padding = len(base64_string) % 4 + if missing_padding: + base64_string += '=' * (4 - missing_padding) + return base64_string + +def convert_base64_to_image(base64_string, output_path): + try: + # Add padding to the base64 string + base64_string = add_padding(base64_string) + + # Decode the base64 string to bytes + image_data = base64.b64decode(base64_string) + + # Convert bytes to an image + image = Image.open(BytesIO(image_data)) + + # Save the image to a file + image.save(output_path) + print(f"Image saved successfully to {output_path}") + except (base64.binascii.Error, UnidentifiedImageError) as e: + print(f"Failed to decode and save image: {e}") + +def extract_and_convert_images(json_file_path, output_dir): + try: + # Read the JSON file + with open(json_file_path, 'r') as file: + data = json.load(file) + + # Create the output directory if it doesn't exist + os.makedirs(output_dir, exist_ok=True) + + # Extract the base64 strings from the URLs + if 'images' in data: + for idx, image_info in enumerate(data['images']): + if 'url' in image_info: + data_url = image_info['url'] + base64_string = extract_base64_string(data_url) + + output_image_path = os.path.join(output_dir, f'image_{idx}.png') + convert_base64_to_image(base64_string, output_image_path) + else: + print("Invalid JSON schema or missing 'url' field") + except json.JSONDecodeError as e: + print(f"Failed to parse JSON file: {e}") + +# Example usage +json_file_path = 'G:/ai-models/models/response_1722196176417.json' # Path to your JSON file +output_image_path = 'G:/ai-models/models/output_image.jpg' # Path to save the output image +extract_and_convert_images(json_file_path, output_image_path) diff --git a/runner/app/test_examples/continuous-creation.py b/runner/app/.gitignore/continuous-creation.py similarity index 100% rename from runner/app/test_examples/continuous-creation.py rename to runner/app/.gitignore/continuous-creation.py diff --git a/runner/app/test_examples/generate-interpolate-upscale.py b/runner/app/.gitignore/generate-interpolate-upscale.py similarity index 100% rename from runner/app/test_examples/generate-interpolate-upscale.py rename to runner/app/.gitignore/generate-interpolate-upscale.py diff --git a/runner/app/test_examples/test-film-interpolate.py b/runner/app/.gitignore/test-film-interpolate.py similarity index 100% rename from runner/app/test_examples/test-film-interpolate.py rename to runner/app/.gitignore/test-film-interpolate.py diff --git a/runner/app/test_examples/test-optimized.py b/runner/app/.gitignore/test-optimized.py similarity index 100% rename from runner/app/test_examples/test-optimized.py rename to runner/app/.gitignore/test-optimized.py diff --git a/runner/app/test_examples/utility-func-compactor.py b/runner/app/.gitignore/utility-func-compactor.py similarity index 100% rename from runner/app/test_examples/utility-func-compactor.py rename to runner/app/.gitignore/utility-func-compactor.py diff --git a/runner/app/test_examples/utility-function-vid.py b/runner/app/.gitignore/utility-function-vid.py similarity index 100% rename from runner/app/test_examples/utility-function-vid.py rename to runner/app/.gitignore/utility-function-vid.py diff --git a/runner/app/pipelines/optim/sfast.py b/runner/app/pipelines/optim/sfast.py index c449aadb..001a2b53 100644 --- a/runner/app/pipelines/optim/sfast.py +++ b/runner/app/pipelines/optim/sfast.py @@ -1,6 +1,6 @@ """This module provides a function to enable StableFast optimization for the pipeline. -For more information, see the DeepCache project on GitHub: https://github.com/chengzeyi/stable-fast +For more information, see the Stable Fast project on GitHub: https://github.com/chengzeyi/stable-fast """ import logging diff --git a/runner/uvicorn.env b/runner/uvicorn.env deleted file mode 100644 index 77869fee..00000000 --- a/runner/uvicorn.env +++ /dev/null @@ -1,4 +0,0 @@ -# myenvfile.env -MODEL_ID="" -MODEL_DIR="" -PIPELINE=FILMPipeline From 2d25c46eecc26f177127d764835f78734bc161ec Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:18:03 +0300 Subject: [PATCH 09/18] added git ignore to the files to remove unnecessary files --- .gitignore | 1 + runner/app/{.gitignore => tests-examples}/base64-image.py | 0 runner/app/{.gitignore => tests-examples}/continuous-creation.py | 0 .../generate-interpolate-upscale.py | 0 .../app/{.gitignore => tests-examples}/test-film-interpolate.py | 0 runner/app/{.gitignore => tests-examples}/test-optimized.py | 0 .../app/{.gitignore => tests-examples}/utility-func-compactor.py | 0 .../app/{.gitignore => tests-examples}/utility-function-vid.py | 0 8 files changed, 1 insertion(+) rename runner/app/{.gitignore => tests-examples}/base64-image.py (100%) rename runner/app/{.gitignore => tests-examples}/continuous-creation.py (100%) rename runner/app/{.gitignore => tests-examples}/generate-interpolate-upscale.py (100%) rename runner/app/{.gitignore => tests-examples}/test-film-interpolate.py (100%) rename runner/app/{.gitignore => tests-examples}/test-optimized.py (100%) rename runner/app/{.gitignore => tests-examples}/utility-func-compactor.py (100%) rename runner/app/{.gitignore => tests-examples}/utility-function-vid.py (100%) diff --git a/.gitignore b/.gitignore index 0a9429b0..1832d101 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ output aiModels.json models checkpoints +C:/Users/ganes/ai-worker/runner/app/tests-examples # IDE .vscode diff --git a/runner/app/.gitignore/base64-image.py b/runner/app/tests-examples/base64-image.py similarity index 100% rename from runner/app/.gitignore/base64-image.py rename to runner/app/tests-examples/base64-image.py diff --git a/runner/app/.gitignore/continuous-creation.py b/runner/app/tests-examples/continuous-creation.py similarity index 100% rename from runner/app/.gitignore/continuous-creation.py rename to runner/app/tests-examples/continuous-creation.py diff --git a/runner/app/.gitignore/generate-interpolate-upscale.py b/runner/app/tests-examples/generate-interpolate-upscale.py similarity index 100% rename from runner/app/.gitignore/generate-interpolate-upscale.py rename to runner/app/tests-examples/generate-interpolate-upscale.py diff --git a/runner/app/.gitignore/test-film-interpolate.py b/runner/app/tests-examples/test-film-interpolate.py similarity index 100% rename from runner/app/.gitignore/test-film-interpolate.py rename to runner/app/tests-examples/test-film-interpolate.py diff --git a/runner/app/.gitignore/test-optimized.py b/runner/app/tests-examples/test-optimized.py similarity index 100% rename from runner/app/.gitignore/test-optimized.py rename to runner/app/tests-examples/test-optimized.py diff --git a/runner/app/.gitignore/utility-func-compactor.py b/runner/app/tests-examples/utility-func-compactor.py similarity index 100% rename from runner/app/.gitignore/utility-func-compactor.py rename to runner/app/tests-examples/utility-func-compactor.py diff --git a/runner/app/.gitignore/utility-function-vid.py b/runner/app/tests-examples/utility-function-vid.py similarity index 100% rename from runner/app/.gitignore/utility-function-vid.py rename to runner/app/tests-examples/utility-function-vid.py From eb5ae468328a709025902aaecc42577ee327cefb Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:22:25 +0300 Subject: [PATCH 10/18] files not removed checking again --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1832d101..abd3b40b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ output aiModels.json models checkpoints -C:/Users/ganes/ai-worker/runner/app/tests-examples +tests-examples # IDE .vscode From e9b39650e680368e6bd3db3ff58004962b7fc7a5 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:25:13 +0300 Subject: [PATCH 11/18] still in test phase --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index abd3b40b..73953d48 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ cache __pycache__ input output +runner\app\tests-examples # AI Configuration files and model storage folders. aiModels.json From 19261b6ecfff1a280f07c0dd607e4beb4881dbb3 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Sun, 28 Jul 2024 23:32:40 +0300 Subject: [PATCH 12/18] test-test --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 73953d48..05c24bab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,12 @@ cache __pycache__ input output -runner\app\tests-examples +runner/app/tests-examples # AI Configuration files and model storage folders. aiModels.json models checkpoints -tests-examples # IDE .vscode From cb6498e499bf095c37d698b4edcaaaea9b60bbef Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sun, 28 Jul 2024 23:34:27 +0300 Subject: [PATCH 13/18] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 05c24bab..0a9429b0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ cache __pycache__ input output -runner/app/tests-examples # AI Configuration files and model storage folders. aiModels.json From b91da520e5c32034118a69e8dba7a8aac48a2344 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sun, 28 Jul 2024 23:34:53 +0300 Subject: [PATCH 14/18] Delete runner/app/tests-examples directory --- runner/app/tests-examples/base64-image.py | 62 ------- .../app/tests-examples/continuous-creation.py | 154 ----------------- .../generate-interpolate-upscale.py | 159 ------------------ .../tests-examples/test-film-interpolate.py | 45 ----- runner/app/tests-examples/test-optimized.py | 69 -------- .../tests-examples/utility-func-compactor.py | 89 ---------- .../tests-examples/utility-function-vid.py | 62 ------- 7 files changed, 640 deletions(-) delete mode 100644 runner/app/tests-examples/base64-image.py delete mode 100644 runner/app/tests-examples/continuous-creation.py delete mode 100644 runner/app/tests-examples/generate-interpolate-upscale.py delete mode 100644 runner/app/tests-examples/test-film-interpolate.py delete mode 100644 runner/app/tests-examples/test-optimized.py delete mode 100644 runner/app/tests-examples/utility-func-compactor.py delete mode 100644 runner/app/tests-examples/utility-function-vid.py diff --git a/runner/app/tests-examples/base64-image.py b/runner/app/tests-examples/base64-image.py deleted file mode 100644 index 9116fdbe..00000000 --- a/runner/app/tests-examples/base64-image.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import base64 -from PIL import Image, UnidentifiedImageError -from io import BytesIO -import os - -def extract_base64_string(data_url): - # Remove the 'data:image/png;base64,' prefix - base64_str = data_url.split(',', 1)[1] - return base64_str - -def add_padding(base64_string): - # Add padding if necessary - missing_padding = len(base64_string) % 4 - if missing_padding: - base64_string += '=' * (4 - missing_padding) - return base64_string - -def convert_base64_to_image(base64_string, output_path): - try: - # Add padding to the base64 string - base64_string = add_padding(base64_string) - - # Decode the base64 string to bytes - image_data = base64.b64decode(base64_string) - - # Convert bytes to an image - image = Image.open(BytesIO(image_data)) - - # Save the image to a file - image.save(output_path) - print(f"Image saved successfully to {output_path}") - except (base64.binascii.Error, UnidentifiedImageError) as e: - print(f"Failed to decode and save image: {e}") - -def extract_and_convert_images(json_file_path, output_dir): - try: - # Read the JSON file - with open(json_file_path, 'r') as file: - data = json.load(file) - - # Create the output directory if it doesn't exist - os.makedirs(output_dir, exist_ok=True) - - # Extract the base64 strings from the URLs - if 'images' in data: - for idx, image_info in enumerate(data['images']): - if 'url' in image_info: - data_url = image_info['url'] - base64_string = extract_base64_string(data_url) - - output_image_path = os.path.join(output_dir, f'image_{idx}.png') - convert_base64_to_image(base64_string, output_image_path) - else: - print("Invalid JSON schema or missing 'url' field") - except json.JSONDecodeError as e: - print(f"Failed to parse JSON file: {e}") - -# Example usage -json_file_path = 'G:/ai-models/models/response_1722196176417.json' # Path to your JSON file -output_image_path = 'G:/ai-models/models/output_image.jpg' # Path to save the output image -extract_and_convert_images(json_file_path, output_image_path) diff --git a/runner/app/tests-examples/continuous-creation.py b/runner/app/tests-examples/continuous-creation.py deleted file mode 100644 index 54473930..00000000 --- a/runner/app/tests-examples/continuous-creation.py +++ /dev/null @@ -1,154 +0,0 @@ -import torch -import numpy as np -import os -import shutil -import time -import sys -sys.path.append('C://Users//ganes//ai-worker//runner') -from PIL import PngImagePlugin, Image -from diffusers import StableVideoDiffusionPipeline -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter - -# Increase the max text chunk size for PNG images -PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) - -def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): - for attempt in range(retries): - try: - shutil.move(src_file_path, dst_file_path) - return - except PermissionError: - print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") - time.sleep(delay) - raise PermissionError(f"Failed to move file after {retries} attempts.") - -def get_last_file_sorted_by_name(directory: str) -> str: - """ - Get the last file in the directory when sorted by filename. - - Args: - directory (str): Path to the directory. - - Returns: - str: Path to the last file in the sorted list, or None if directory is empty. - """ - try: - # List all files in the directory - files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] - - if not files: - print("No files found in the directory.") - return None - - # Sort files by name - files.sort() - - # Get the last file in the sorted list - last_file = files[-1] - - return os.path.join(directory, last_file) - - except Exception as e: - print(f"An error occurred: {e}") - return None - -def main(): - # Initialize pipelines - repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" - svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( - repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" - ) - svd_xt_pipeline.enable_model_cpu_offload() - - film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) - - # Load initial input image - image_path = "G:/ai-models/models/gif_frames/donut_motion.png" - image = Image.open(image_path) - - fps = 24.0 - inter_frames = 4 - rounds = 2 # Number of rounds of generation and interpolation - base_output_dir = "G:/ai-models/models" - - all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") - os.makedirs(all_frames_dir, exist_ok=True) - - last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") - - for round_num in range(1, rounds + 1): - svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") - os.makedirs(svd_xt_output_dir, exist_ok=True) - - # Generate frames using SVD pipeline - generator = torch.manual_seed(42) - frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] - - # Save SVD frames to directory - film_writer = DirectoryWriter(svd_xt_output_dir) - for idx, frame in enumerate(frames): - film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) - - # Read saved frames for interpolation - film_reader = DirectoryReader(svd_xt_output_dir) - height, width = film_reader.get_resolution() - - # Interpolate frames using FILM pipeline - film_pipeline(film_reader, film_writer, inter_frames=inter_frames) - - # Close reader and writer - film_writer.close() - film_reader.reset() - - # Deleting the SVD generated images. - for i in range(len(frames)): - os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) - print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") - - # Save the last frame separately for the next round - last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) - if last_frame_path: - shutil.copy2(last_frame_path, last_frame_for_next_round) - else: - print("No frames found to copy.") - - # Move all interpolated frames to a common directory with a unique naming scheme - for file_name in sorted(os.listdir(svd_xt_output_dir)): - src_file_path = os.path.join(svd_xt_output_dir, file_name) - dst_file_name = f"round_{round_num:03d}_frame_{file_name}" - dst_file_path = os.path.join(all_frames_dir, dst_file_name) - - move_file_with_retry(src_file_path, dst_file_path) - - # Clean up the source directory after moving frames - for file_name in os.listdir(svd_xt_output_dir): - os.remove(os.path.join(svd_xt_output_dir, file_name)) - os.rmdir(svd_xt_output_dir) - - # Ensure all operations on last frame are complete before opening it again - time.sleep(1) # Small delay to ensure file system operations are complete - - # Prepare for next round - image = Image.open(last_frame_for_next_round) - - # Compile all interpolated frames from all rounds into a final video - video_output_dir = "G:/ai-models/models/video_out" - os.makedirs(video_output_dir, exist_ok=True) - - film_output_path = os.path.join(video_output_dir, "output.avi") - frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) - - # Clean up all frames in the directories after video generation - for file_name in os.listdir(all_frames_dir): - file_path = os.path.join(all_frames_dir, file_name) - if os.path.isfile(file_path): - os.remove(file_path) - os.rmdir(all_frames_dir) - - print(f"All frames deleted from directories.") - print(f"Video generated at: {film_output_path}") - return film_output_path - -if __name__ == "__main__": - main() diff --git a/runner/app/tests-examples/generate-interpolate-upscale.py b/runner/app/tests-examples/generate-interpolate-upscale.py deleted file mode 100644 index 69954a0a..00000000 --- a/runner/app/tests-examples/generate-interpolate-upscale.py +++ /dev/null @@ -1,159 +0,0 @@ -import logging -import os -os.environ["MODEL_DIR"] = "G://ai-models//models" -import shutil -import time -import sys -sys.path.append('C://Users//ganes//ai-worker//runner') -from typing import List, Optional, Tuple -import PIL -import torch -from diffusers import StableVideoDiffusionPipeline -from PIL import PngImagePlugin, Image, ImageFile - -from app.pipelines.base import Pipeline -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.upscale import UpscalePipeline -from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter, SafetyChecker, get_model_dir, get_torch_device, is_lightning_model, is_turbo_model -from huggingface_hub import file_download - -# Increase the max text chunk size for PNG images -PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) -ImageFile.LOAD_TRUNCATED_IMAGES = True - -logger = logging.getLogger(__name__) - -# Helper function to move files with retry mechanism -def move_file_with_retry(src_file_path: str, dst_file_path: str, retries: int = 5, delay: float = 1.0): - for attempt in range(retries): - try: - shutil.move(src_file_path, dst_file_path) - return - except PermissionError: - print(f"Attempt {attempt + 1} failed: File is in use. Retrying in {delay} seconds...") - time.sleep(delay) - raise PermissionError(f"Failed to move file after {retries} attempts.") - -# Helper function to get the last file in a directory sorted by filename -def get_last_file_sorted_by_name(directory: str) -> str: - try: - files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] - if not files: - print("No files found in the directory.") - return None - files.sort() - last_file = files[-1] - return os.path.join(directory, last_file) - except Exception as e: - print(f"An error occurred: {e}") - return None - -def main(): - # Initialize SVD and FILM pipelines - repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" - svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( - repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" - ) - svd_xt_pipeline.enable_model_cpu_offload() - - film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) - - # Load initial input image - image_path = "G:/ai-models/models/gif_frames/donut_motion.png" - image = Image.open(image_path) - - fps = 24.0 - inter_frames = 4 - rounds = 2 # Number of rounds of generation and interpolation - base_output_dir = "G:/ai-models/models" - - all_frames_dir = os.path.join(base_output_dir, "all_interpolated_frames") - os.makedirs(all_frames_dir, exist_ok=True) - - last_frame_for_next_round = os.path.join(base_output_dir, "last_frame_for_next_round.png") - - for round_num in range(1, rounds + 1): - svd_xt_output_dir = os.path.join(base_output_dir, f"svd_xt_output_round_{round_num}") - os.makedirs(svd_xt_output_dir, exist_ok=True) - - # Generate frames using SVD pipeline - generator = torch.manual_seed(42) - frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] - - # Save SVD frames to directory - film_writer = DirectoryWriter(svd_xt_output_dir) - for idx, frame in enumerate(frames): - film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) - - # Read saved frames for interpolation - film_reader = DirectoryReader(svd_xt_output_dir) - height, width = film_reader.get_resolution() - - # Interpolate frames using FILM pipeline - film_pipeline(film_reader, film_writer, inter_frames=inter_frames) - - # Close reader and writer - film_writer.close() - film_reader.reset() - - # Deleting the SVD generated images. - for i in range(len(frames)): - os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) - print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") - - # Save the last frame separately for the next round - last_frame_path = get_last_file_sorted_by_name(svd_xt_output_dir) - if last_frame_path: - shutil.copy2(last_frame_path, last_frame_for_next_round) - else: - print("No frames found to copy.") - - # Initialize Upscale pipeline and Upscale the last frame before passing to the next round - upscale_pipeline = UpscalePipeline("stabilityai/stable-diffusion-x4-upscaler", torch_dtype=torch.float16) - upscale_pipeline.enable_model_cpu_offload() - upscale_pipeline.sfast_enabled() - upscaled_image, _ = upscale_pipeline("", image=Image.open(last_frame_for_next_round),) - print('Upscaling of the seed image before next round.') - print(upscaled_image[0].shape) - exit - upscaled_image[0].save(last_frame_for_next_round) - - # Move all interpolated frames to a common directory with a unique naming scheme - for file_name in sorted(os.listdir(svd_xt_output_dir)): - src_file_path = os.path.join(svd_xt_output_dir, file_name) - dst_file_name = f"round_{round_num:03d}_frame_{file_name}" - dst_file_path = os.path.join(all_frames_dir, dst_file_name) - - move_file_with_retry(src_file_path, dst_file_path) - - # Clean up the source directory after moving frames - for file_name in os.listdir(svd_xt_output_dir): - os.remove(os.path.join(svd_xt_output_dir, file_name)) - os.rmdir(svd_xt_output_dir) - - # Ensure all operations on last frame are complete before opening it again - time.sleep(1) # Small delay to ensure file system operations are complete - - # Prepare for next round - image = Image.open(last_frame_for_next_round) - - # Compile all interpolated frames from all rounds into a final video - video_output_dir = "G:/ai-models/models/video_out" - os.makedirs(video_output_dir, exist_ok=True) - - film_output_path = os.path.join(video_output_dir, "output.avi") - frames_compactor(frames=all_frames_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) - - # Clean up all frames in the directories after video generation - for file_name in os.listdir(all_frames_dir): - file_path = os.path.join(all_frames_dir, file_name) - if os.path.isfile(file_path): - os.remove(file_path) - os.rmdir(all_frames_dir) - - print(f"All frames deleted from directories.") - print(f"Video generated at: {film_output_path}") - return film_output_path - -if __name__ == "__main__": - main() diff --git a/runner/app/tests-examples/test-film-interpolate.py b/runner/app/tests-examples/test-film-interpolate.py deleted file mode 100644 index 195a3a4b..00000000 --- a/runner/app/tests-examples/test-film-interpolate.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import requests - -# Define the URL of the FastAPI application -URL = "http://localhost:8000/FILMPipeline" - -# Test with two images -def test_with_two_images(): - image1_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_74.png" - image2_path = "G:/ai-models/models/all_interpolated_frames/round_003_frame_36.png" - - with open(image1_path, "rb") as image1, open(image2_path, "rb") as image2: - files = { - "image1": ("image1.png", image1, "image/png"), - "image2": ("image2.png", image2, "image/png"), - } - data = { - "inter_frames": 2, - "model_id": "film_net_fp16.pt" - } - response = requests.post(URL, files=files, data=data) - - print("Test with two images") - print(response.status_code) - print(response.json()) - -# Test with a directory of images -def test_with_image_directory(): - image_dir = "path/to/image_directory" - - data = { - "inter_frames": 2, - "model_path": "path/to/film_net_fp16.pt", - "image_dir": image_dir - } - response = requests.post(URL, data=data) - - print("Test with image directory") - print(response.status_code) - print(response.json()) - -if __name__ == "__main__": - # Ensure that the FastAPI server is running before executing these tests - test_with_two_images() - test_with_image_directory() diff --git a/runner/app/tests-examples/test-optimized.py b/runner/app/tests-examples/test-optimized.py deleted file mode 100644 index db200286..00000000 --- a/runner/app/tests-examples/test-optimized.py +++ /dev/null @@ -1,69 +0,0 @@ -import torch -import numpy as np -import os -import sys -sys.path.append('C://Users//ganes//ai-worker//runner') -from diffusers import StableVideoDiffusionPipeline -from PIL import PngImagePlugin, Image -from app.pipelines.frame_interpolation import FILMPipeline -from app.pipelines.utils import DirectoryReader, frames_compactor, DirectoryWriter - -# Increase the max text chunk size for PNG images -PngImagePlugin.MAX_TEXT_CHUNK = 100 * (1024**2) - -def main(): - # Initialize pipelines - repo_id = "stabilityai/stable-video-diffusion-img2vid-xt" - svd_xt_pipeline = StableVideoDiffusionPipeline.from_pretrained( - repo_id, torch_dtype=torch.float16, variant="fp16", cache_dir="G:/ai-models/models" - ) - svd_xt_pipeline.enable_model_cpu_offload() - - film_pipeline = FILMPipeline("G:/ai-models/models/film_net_fp16.pt").to(device="cuda", dtype=torch.float16) - - # Load input image - image_path = "G:/ai-models/models/gif_frames/rocket.png" - image = Image.open(image_path) - - # Generate frames using SVD pipeline - generator = torch.manual_seed(42) - frames = svd_xt_pipeline(image, decode_chunk_size=8, generator=generator, output_type="np").frames[0] - - fps = 24.0 - inter_frames = 2 - svd_xt_output_dir = "G:/ai-models/models/svd_xt_output" - video_output_dir = "G:/ai-models/models/video_out" - - # Save SVD frames to directory - film_writer = DirectoryWriter(svd_xt_output_dir) - for frame in frames: - film_writer.write_frame(torch.tensor(frame).permute(2, 0, 1)) - - # Read saved frames for interpolation - film_reader = DirectoryReader(svd_xt_output_dir) - height, width = film_reader.get_resolution() - - # Interpolate frames using FILM pipeline - film_pipeline(film_reader, film_writer, inter_frames=inter_frames) - - # Delete original SVD frames since interpolated frames are also in the same directory. - for i in range(len(frames)): - os.remove(os.path.join(svd_xt_output_dir, f"{i}.png")) - print(f"Deleted the first {len(frames)} frames in directory: {svd_xt_output_dir}") - - # Compile interpolated frames into a video - film_output_path = os.path.join(video_output_dir, "output.avi") - frames_compactor(frames=svd_xt_output_dir, output_path=film_output_path, fps=fps, codec="MJPG", is_directory=True) - - # Clean up all frames in the directory after video generation - for file_name in os.listdir(svd_xt_output_dir): - file_path = os.path.join(svd_xt_output_dir, file_name) - if os.path.isfile(file_path): - os.remove(file_path) - print(f"All frames deleted from directory: {svd_xt_output_dir}") - - print(f"Video generated at: {film_output_path}") - return film_output_path - -if __name__ == "__main__": - main() diff --git a/runner/app/tests-examples/utility-func-compactor.py b/runner/app/tests-examples/utility-func-compactor.py deleted file mode 100644 index 6dac4309..00000000 --- a/runner/app/tests-examples/utility-func-compactor.py +++ /dev/null @@ -1,89 +0,0 @@ -import subprocess -import numpy as np -import torch -import os -from typing import List, Union -from pathlib import Path -import cv2 - -def generate_video_from_frames( - frames: Union[List[np.ndarray], List[torch.Tensor]], - output_path: str, - fps: float, - codec: str = "MJPEG", - is_directory: bool = False, - width: int = None, - height: int = None -) -> None: - """ - Generate a video from a list of frames. Frames can be from a directory or in-memory. - - Args: - frames (List[np.ndarray] | List[torch.Tensor]): List of frames as NumPy arrays or PyTorch tensors. - output_path (str): Path to save the output video file. - fps (float): Frames per second for the video. - codec (str): Codec used for video compression (default is "MJPEG"). - is_directory (bool): If True, treat `frames` as a directory path containing image files. - width (int): Width of the video. Must be provided if `frames` are in-memory. - height (int): Height of the video. Must be provided if `frames` are in-memory. - - Returns: - None - """ - if is_directory: - # Read frames from a directory - frames = [cv2.imread(os.path.join(frames, file)) for file in sorted(os.listdir(frames))] - else: - # Convert torch tensors to numpy arrays if necessary - if isinstance(frames[0], torch.Tensor): - frames = [frame.permute(1, 2, 0).cpu().numpy() for frame in frames] - - # Ensure frames are numpy arrays and are uint8 type - frames = [frame.astype(np.uint8) for frame in frames] - - # Check if frames are consistent - if not frames: - raise ValueError("No frames to process.") - - if width is None or height is None: - # Use dimensions of the first frame if not provided - height, width = frames[0].shape[:2] - - # Write frames to a temporary directory - temp_dir = Path("temp_frames") - temp_dir.mkdir(exist_ok=True) - for i, frame in enumerate(frames): - cv2.imwrite(str(temp_dir / f"frame_{i:05d}.png"), frame) - - # Build ffmpeg command - ffmpeg_cmd = [ - 'ffmpeg', '-y', '-framerate', str(fps), - '-i', str(temp_dir / 'frame_%05d.png'), - '-c:v', codec, '-pix_fmt', 'yuv420p', - output_path - ] - - # Run ffmpeg command - subprocess.run(ffmpeg_cmd, check=True) - - # Clean up temporary frames - for file in temp_dir.glob("*.png"): - file.unlink() - temp_dir.rmdir() - - print(f"Video saved to {output_path}") - -# Example usage -if __name__ == "__main__": - # Example with in-memory frames (as np.ndarray) - # Assuming `in_memory_frames` is a list of numpy arrays - - # Example with frames from a directory - frames_directory = "G:/ai-models/models/svd_xt_output" - generate_video_from_frames( - frames=frames_directory, - output_path="G:/ai-models/models/video_out/output.mp4", - fps=24.0, - codec="mpeg4", - is_directory=True - ) diff --git a/runner/app/tests-examples/utility-function-vid.py b/runner/app/tests-examples/utility-function-vid.py deleted file mode 100644 index adf194cc..00000000 --- a/runner/app/tests-examples/utility-function-vid.py +++ /dev/null @@ -1,62 +0,0 @@ -import cv2 -import numpy as np -import tempfile -import os -from io import BytesIO - -def extract_frames_from_video(video_data, is_file_path=True) -> np.ndarray: - """ - Extract frames from a video file or in-memory video data and return them as a NumPy array. - - Args: - video_data (str or BytesIO): Path to the input video file or in-memory video data. - is_file_path (bool): Indicates if video_data is a file path (True) or in-memory data (False). - - Returns: - np.ndarray: Array of frames with shape (num_frames, height, width, channels). - """ - if is_file_path: - # Handle file-based video input - video_capture = cv2.VideoCapture(video_data) - else: - # Handle in-memory video input - # Create a temporary file to store in-memory video data - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as temp_file: - temp_file.write(video_data.getvalue()) - temp_file_path = temp_file.name - - # Open the temporary video file - video_capture = cv2.VideoCapture(temp_file_path) - - if not video_capture.isOpened(): - raise ValueError("Error opening video data") - - frames = [] - success, frame = video_capture.read() - - while success: - frames.append(frame) - success, frame = video_capture.read() - - video_capture.release() - - # Delete the temporary file if it was created - if not is_file_path: - os.remove(temp_file_path) - - # Convert list of frames to a NumPy array - frames_array = np.array(frames) - print(f"Extracted {frames_array.shape[0]} frames from video in shape of {frames_array.shape}") - - return frames_array - -# Example usage -if __name__ == "__main__": - # File path example - video_file_path = "C:/Users/ganes/Desktop/Generated videos/output.mp4" - - - # In-memory video example - with open(video_file_path, "rb") as f: - video_data = BytesIO(f.read()) - frames_array_from_memory = extract_frames_from_video(video_data, is_file_path=False) From 12a925eec6b197dda8c6f11e97d368088ba88bc5 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Mon, 29 Jul 2024 16:41:33 +0300 Subject: [PATCH 15/18] update to directory reader as it now reads almost any naming convention --- runner/app/pipelines/utils/utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/runner/app/pipelines/utils/utils.py b/runner/app/pipelines/utils/utils.py index 5c9ad6a5..f2deeae8 100644 --- a/runner/app/pipelines/utils/utils.py +++ b/runner/app/pipelines/utils/utils.py @@ -277,11 +277,21 @@ def check_nsfw_images( ) return images, has_nsfw_concept +def natural_sort_key(s): + """ + Sort in a natural order, separating strings into a list of strings and integers. + This handles leading zeros and case insensitivity. + """ + return [ + int(text) if text.isdigit() else text.lower() + for text in re.split(r'([0-9]+)', os.path.basename(s)) + ] + class DirectoryReader: def __init__(self, dir: str): self.paths = sorted( glob.glob(os.path.join(dir, "*")), - key=lambda x: int(os.path.basename(x).split(".")[0]), + key=natural_sort_key ) self.nb_frames = len(self.paths) self.idx = 0 @@ -306,9 +316,9 @@ def get_frame(self): self.idx += 1 img = Image.open(path) - transforms = v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]) + transforms = v2.Compose([v2.ToTensor(), v2.ConvertImageDtype(torch.float32)]) - return transforms(img) + return transforms(img) class DirectoryWriter: def __init__(self, dir: str): From cd45ee0c34bd352e3d75be130d77d90fe10fc8f5 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Tue, 30 Jul 2024 13:20:24 +0300 Subject: [PATCH 16/18] changing files similar to main to make easy to merge --- ...e-ai-runner.yaml => ai-runner-docker.yaml} | 10 +++---- .../trigger-upstream-openapi-sync.yaml | 30 +++++++++++++++++++ ...-push.yaml => validate-openapi-on-pr.yaml} | 4 +-- 3 files changed, 37 insertions(+), 7 deletions(-) rename .github/workflows/{docker-create-ai-runner.yaml => ai-runner-docker.yaml} (88%) create mode 100644 .github/workflows/trigger-upstream-openapi-sync.yaml rename .github/workflows/{validate-openapi-on-push.yaml => validate-openapi-on-pr.yaml} (97%) diff --git a/.github/workflows/docker-create-ai-runner.yaml b/.github/workflows/ai-runner-docker.yaml similarity index 88% rename from .github/workflows/docker-create-ai-runner.yaml rename to .github/workflows/ai-runner-docker.yaml index d71aa858..161eb5cf 100644 --- a/.github/workflows/docker-create-ai-runner.yaml +++ b/.github/workflows/ai-runner-docker.yaml @@ -1,7 +1,7 @@ name: Build Docker image for ai-runner on: - check_run: + pull_request: paths: - "runner/**" - "!runner/.devcontainer/**" @@ -13,7 +13,6 @@ on: paths: - "runner/**" - "!runner/.devcontainer/**" - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -22,6 +21,7 @@ concurrency: jobs: docker: name: Docker image generation + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository permissions: packages: write contents: read @@ -49,7 +49,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - jjassonn69/ai-runner + livepeer/ai-runner tags: | type=sha type=ref,event=pr @@ -73,5 +73,5 @@ jobs: tags: ${{ steps.meta.outputs.tags }} file: "Dockerfile" labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=jjassonn69/build:cache - cache-to: type=registry,ref=jjassonn69/build:cache,mode=max + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max \ No newline at end of file diff --git a/.github/workflows/trigger-upstream-openapi-sync.yaml b/.github/workflows/trigger-upstream-openapi-sync.yaml new file mode 100644 index 00000000..5c7bdeb1 --- /dev/null +++ b/.github/workflows/trigger-upstream-openapi-sync.yaml @@ -0,0 +1,30 @@ +name: Trigger upstream OpenAPI sync + +on: + push: + paths: + - "runner/openapi.json" + workflow_dispatch: + +jobs: + trigger-upstream-openapi-sync: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Trigger docs AI OpenAPI spec update + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.DOCS_TRIGGER_PAT }} + repository: livepeer/docs + event-type: update-ai-openapi + client-payload: '{"sha": "${{ github.sha }}"}' + + - name: Trigger SDK generation + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.SDKS_TRIGGER_PAT }} + repository: livepeer/livepeer-ai-sdks + event-type: update-ai-openapi + client-payload: '{"sha": "${{ github.sha }}"}' \ No newline at end of file diff --git a/.github/workflows/validate-openapi-on-push.yaml b/.github/workflows/validate-openapi-on-pr.yaml similarity index 97% rename from .github/workflows/validate-openapi-on-push.yaml rename to .github/workflows/validate-openapi-on-pr.yaml index 886d24e6..08cc39e4 100644 --- a/.github/workflows/validate-openapi-on-push.yaml +++ b/.github/workflows/validate-openapi-on-pr.yaml @@ -1,7 +1,7 @@ name: Check OpenAPI spec and Golang bindings on: - check_run: + pull_request: jobs: check-openapi-and-bindings: @@ -41,4 +41,4 @@ jobs: if ! git diff --exit-code; then echo "::error::Go bindings have changed. Please run 'make' at the root of the repository and commit the changes." exit 1 - fi + fi \ No newline at end of file From 088571894314e6f8885a315a21b4b8a118d77ae3 Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Tue, 30 Jul 2024 14:40:38 +0300 Subject: [PATCH 17/18] update to upscale warmup params to fix OOM --- runner/app/pipelines/upscale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index b743434b..3d7c4e39 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -74,7 +74,7 @@ def __init__(self, model_id: str): # TODO: Retrieve defaults from Pydantic class in route. warmup_kwargs = { "prompt": "Upscaling the pipeline with sfast enabled", - "image": PIL.Image.new("RGB", (576, 1024)), + "image": PIL.Image.new("RGB", (400, 400)), # anything higher than this size cause the model to OOM } logger.info("Warming up ImageToVideoPipeline pipeline...") From 3de64b37f95b4c7eead7e70c97a4835e6ded44de Mon Sep 17 00:00:00 2001 From: Jason Stone Date: Wed, 31 Jul 2024 07:05:55 +0300 Subject: [PATCH 18/18] naming wrong in the info section for error msg --- runner/app/pipelines/upscale.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/app/pipelines/upscale.py b/runner/app/pipelines/upscale.py index 3d7c4e39..174b0d14 100644 --- a/runner/app/pipelines/upscale.py +++ b/runner/app/pipelines/upscale.py @@ -77,7 +77,7 @@ def __init__(self, model_id: str): "image": PIL.Image.new("RGB", (400, 400)), # anything higher than this size cause the model to OOM } - logger.info("Warming up ImageToVideoPipeline pipeline...") + logger.info("Warming up Upscale pipeline...") total_time = 0 for ii in range(SFAST_WARMUP_ITERATIONS): t = time.time() @@ -85,7 +85,7 @@ def __init__(self, model_id: str): self.ldm(**warmup_kwargs).images except Exception as e: # FIXME: When out of memory, pipeline is corrupted. - logger.error(f"ImageToVideoPipeline warmup error: {e}") + logger.error(f"Upscale pipeline warmup error: {e}") raise e iteration_time = time.time() - t total_time += iteration_time