enable input_mode

app-controlnet.py

@@ -1,322 +0,0 @@
-import asyncio
-import json
-import logging
-import traceback
-from pydantic import BaseModel
-
-from fastapi import FastAPI, WebSocket, HTTPException, WebSocketDisconnect
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import (
-    StreamingResponse,
-    JSONResponse,
-    HTMLResponse,
-    FileResponse,
-)
-
-from diffusers import AutoencoderTiny, ControlNetModel
-from latent_consistency_controlnet import LatentConsistencyModelPipeline_controlnet
-from compel import Compel
-import torch
-
-from canny_gpu import SobelOperator
-
-# from controlnet_aux import OpenposeDetector
-# import cv2
-
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    pass
-from PIL import Image
-import numpy as np
-import gradio as gr
-import io
-import uuid
-import os
-import time
-import psutil
-
-
-MAX_QUEUE_SIZE = int(os.environ.get("MAX_QUEUE_SIZE", 0))
-TIMEOUT = float(os.environ.get("TIMEOUT", 0))
-SAFETY_CHECKER = os.environ.get("SAFETY_CHECKER", None)
-TORCH_COMPILE = os.environ.get("TORCH_COMPILE", None)
-WIDTH = 512
-HEIGHT = 512
-# disable tiny autoencoder for better quality speed tradeoff
-USE_TINY_AUTOENCODER = True
-
-# check if MPS is available OSX only M1/M2/M3 chips
-mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
-xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
-device = torch.device(
-    "cuda" if torch.cuda.is_available() else "xpu" if xpu_available else "cpu"
-)
-
-# change to torch.float16 to save GPU memory
-torch_dtype = torch.float16
-
-print(f"TIMEOUT: {TIMEOUT}")
-print(f"SAFETY_CHECKER: {SAFETY_CHECKER}")
-print(f"MAX_QUEUE_SIZE: {MAX_QUEUE_SIZE}")
-print(f"device: {device}")
-
-if mps_available:
-    device = torch.device("mps")
-    device = "cpu"
-    torch_dtype = torch.float32
-
-controlnet_canny = ControlNetModel.from_pretrained(
-    "lllyasviel/control_v11p_sd15_canny", torch_dtype=torch_dtype
-).to(device)
-
-canny_torch = SobelOperator(device=device)
-# controlnet_pose = ControlNetModel.from_pretrained(
-#     "lllyasviel/control_v11p_sd15_openpose", torch_dtype=torch_dtype
-# ).to(device)
-# controlnet_depth = ControlNetModel.from_pretrained(
-#     "lllyasviel/control_v11f1p_sd15_depth", torch_dtype=torch_dtype
-# ).to(device)
-
-
-# pose_processor = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
-
-if SAFETY_CHECKER == "True":
-    pipe = LatentConsistencyModelPipeline_controlnet.from_pretrained(
-        "SimianLuo/LCM_Dreamshaper_v7",
-        controlnet=controlnet_canny,
-        scheduler=None,
-    )
-else:
-    pipe = LatentConsistencyModelPipeline_controlnet.from_pretrained(
-        "SimianLuo/LCM_Dreamshaper_v7",
-        safety_checker=None,
-        controlnet=controlnet_canny,
-        scheduler=None,
-    )
-
-if USE_TINY_AUTOENCODER:
-    pipe.vae = AutoencoderTiny.from_pretrained(
-        "madebyollin/taesd", torch_dtype=torch_dtype, use_safetensors=True
-    )
-pipe.set_progress_bar_config(disable=True)
-pipe.to(device=device, dtype=torch_dtype).to(device)
-pipe.unet.to(memory_format=torch.channels_last)
-
-if psutil.virtual_memory().total < 64 * 1024**3:
-    pipe.enable_attention_slicing()
-
-compel_proc = Compel(
-    tokenizer=pipe.tokenizer,
-    text_encoder=pipe.text_encoder,
-    truncate_long_prompts=False,
-)
-if TORCH_COMPILE:
-    pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
-    pipe.vae = torch.compile(pipe.vae, mode="reduce-overhead", fullgraph=True)
-
-    pipe(
-        prompt="warmup",
-        image=[Image.new("RGB", (768, 768))],
-        control_image=[Image.new("RGB", (768, 768))],
-    )
-
-
-user_queue_map = {}
-
-
-class InputParams(BaseModel):
-    seed: int = 2159232
-    prompt: str
-    guidance_scale: float = 8.0
-    strength: float = 0.5
-    steps: int = 4
-    lcm_steps: int = 50
-    width: int = WIDTH
-    height: int = HEIGHT
-    controlnet_scale: float = 0.8
-    controlnet_start: float = 0.0
-    controlnet_end: float = 1.0
-    canny_low_threshold: float = 0.31
-    canny_high_threshold: float = 0.78
-    debug_canny: bool = False
-
-
-def predict(
-    input_image: Image.Image, params: InputParams, prompt_embeds: torch.Tensor = None
-):
-    generator = torch.manual_seed(params.seed)
-
-    control_image = canny_torch(
-        input_image, params.canny_low_threshold, params.canny_high_threshold
-    )
-    results = pipe(
-        control_image=control_image,
-        prompt_embeds=prompt_embeds,
-        generator=generator,
-        image=input_image,
-        strength=params.strength,
-        num_inference_steps=params.steps,
-        guidance_scale=params.guidance_scale,
-        width=params.width,
-        height=params.height,
-        lcm_origin_steps=params.lcm_steps,
-        output_type="pil",
-        controlnet_conditioning_scale=params.controlnet_scale,
-        control_guidance_start=params.controlnet_start,
-        control_guidance_end=params.controlnet_end,
-    )
-    nsfw_content_detected = (
-        results.nsfw_content_detected[0]
-        if "nsfw_content_detected" in results
-        else False
-    )
-    if nsfw_content_detected:
-        return None
-    result_image = results.images[0]
-    if params.debug_canny:
-        # paste control_image on top of result_image
-        w0, h0 = (200, 200)
-        control_image = control_image.resize((w0, h0))
-        w1, h1 = result_image.size
-        result_image.paste(control_image, (w1 - w0, h1 - h0))
-
-    return result_image
-
-
-app = FastAPI()
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.websocket("/ws")
-async def websocket_endpoint(websocket: WebSocket):
-    await websocket.accept()
-    if MAX_QUEUE_SIZE > 0 and len(user_queue_map) >= MAX_QUEUE_SIZE:
-        print("Server is full")
-        await websocket.send_json({"status": "error", "message": "Server is full"})
-        await websocket.close()
-        return
-
-    try:
-        uid = str(uuid.uuid4())
-        print(f"New user connected: {uid}")
-        await websocket.send_json(
-            {"status": "success", "message": "Connected", "userId": uid}
-        )
-        user_queue_map[uid] = {"queue": asyncio.Queue()}
-        await websocket.send_json(
-            {"status": "start", "message": "Start Streaming", "userId": uid}
-        )
-        await handle_websocket_data(websocket, uid)
-    except WebSocketDisconnect as e:
-        logging.error(f"WebSocket Error: {e}, {uid}")
-        traceback.print_exc()
-    finally:
-        print(f"User disconnected: {uid}")
-        queue_value = user_queue_map.pop(uid, None)
-        queue = queue_value.get("queue", None)
-        if queue:
-            while not queue.empty():
-                try:
-                    queue.get_nowait()
-                except asyncio.QueueEmpty:
-                    continue
-
-
-@app.get("/queue_size")
-async def get_queue_size():
-    queue_size = len(user_queue_map)
-    return JSONResponse({"queue_size": queue_size})
-
-
-@app.get("/stream/{user_id}")
-async def stream(user_id: uuid.UUID):
-    uid = str(user_id)
-    try:
-        user_queue = user_queue_map[uid]
-        queue = user_queue["queue"]
-
-        async def generate():
-            last_prompt: str = None
-            prompt_embeds: torch.Tensor = None
-            while True:
-                data = await queue.get()
-                input_image = data["image"]
-                params = data["params"]
-                if input_image is None:
-                    continue
-                # avoid recalculate prompt embeds
-                if last_prompt != params.prompt:
-                    print("new prompt")
-                    prompt_embeds = compel_proc(params.prompt)
-                    last_prompt = params.prompt
-
-                image = predict(
-                    input_image,
-                    params,
-                    prompt_embeds,
-                )
-                if image is None:
-                    continue
-                frame_data = io.BytesIO()
-                image.save(frame_data, format="JPEG")
-                frame_data = frame_data.getvalue()
-                if frame_data is not None and len(frame_data) > 0:
-                    yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + frame_data + b"\r\n"
-
-                await asyncio.sleep(1.0 / 120.0)
-
-        return StreamingResponse(
-            generate(), media_type="multipart/x-mixed-replace;boundary=frame"
-        )
-    except Exception as e:
-        logging.error(f"Streaming Error: {e}, {user_queue_map}")
-        traceback.print_exc()
-        return HTTPException(status_code=404, detail="User not found")
-
-
-async def handle_websocket_data(websocket: WebSocket, user_id: uuid.UUID):
-    uid = str(user_id)
-    user_queue = user_queue_map[uid]
-    queue = user_queue["queue"]
-    if not queue:
-        return HTTPException(status_code=404, detail="User not found")
-    last_time = time.time()
-    try:
-        while True:
-            data = await websocket.receive_bytes()
-            params = await websocket.receive_json()
-            params = InputParams(**params)
-            pil_image = Image.open(io.BytesIO(data))
-
-            while not queue.empty():
-                try:
-                    queue.get_nowait()
-                except asyncio.QueueEmpty:
-                    continue
-            await queue.put({"image": pil_image, "params": params})
-            if TIMEOUT > 0 and time.time() - last_time > TIMEOUT:
-                await websocket.send_json(
-                    {
-                        "status": "timeout",
-                        "message": "Your session has ended",
-                        "userId": uid,
-                    }
-                )
-                await websocket.close()
-                return
-
-    except Exception as e:
-        logging.error(f"Error: {e}")
-        traceback.print_exc()
-
-
-@app.get("/", response_class=HTMLResponse)
-async def root():
-    return FileResponse("./static/controlnet.html")

app-txt2img.py

@@ -1,255 +0,0 @@
-import asyncio
-import json
-import logging
-import traceback
-from pydantic import BaseModel
-
-from fastapi import FastAPI, WebSocket, HTTPException, WebSocketDisconnect
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import (
-    StreamingResponse,
-    JSONResponse,
-    HTMLResponse,
-    FileResponse,
-)
-
-from diffusers import DiffusionPipeline, AutoencoderTiny
-from compel import Compel
-import torch
-
-try:
-    import intel_extension_for_pytorch as ipex
-except:
-    pass
-from PIL import Image
-import numpy as np
-import gradio as gr
-import io
-import uuid
-import os
-import time
-import psutil
-
-
-MAX_QUEUE_SIZE = int(os.environ.get("MAX_QUEUE_SIZE", 0))
-TIMEOUT = float(os.environ.get("TIMEOUT", 0))
-SAFETY_CHECKER = os.environ.get("SAFETY_CHECKER", None)
-TORCH_COMPILE = os.environ.get("TORCH_COMPILE", None)
-
-WIDTH = 768
-HEIGHT = 768
-# disable tiny autoencoder for better quality speed tradeoff
-USE_TINY_AUTOENCODER = False
-
-# check if MPS is available OSX only M1/M2/M3 chips
-mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
-xpu_available = hasattr(torch, "xpu") and torch.xpu.is_available()
-device = torch.device(
-    "cuda" if torch.cuda.is_available() else "xpu" if xpu_available else "cpu"
-)
-torch_device = device
-# change to torch.float16 to save GPU memory
-torch_dtype = torch.float32
-
-print(f"TIMEOUT: {TIMEOUT}")
-print(f"SAFETY_CHECKER: {SAFETY_CHECKER}")
-print(f"MAX_QUEUE_SIZE: {MAX_QUEUE_SIZE}")
-print(f"device: {device}")
-
-if mps_available:
-    device = torch.device("mps")
-    torch_device = "cpu"
-    torch_dtype = torch.float32
-
-if SAFETY_CHECKER == "True":
-    pipe = DiffusionPipeline.from_pretrained(
-        "SimianLuo/LCM_Dreamshaper_v7",
-    )
-else:
-    pipe = DiffusionPipeline.from_pretrained(
-        "SimianLuo/LCM_Dreamshaper_v7",
-        safety_checker=None,
-    )
-if USE_TINY_AUTOENCODER:
-    pipe.vae = AutoencoderTiny.from_pretrained(
-        "madebyollin/taesd", torch_dtype=torch_dtype, use_safetensors=True
-    )
-pipe.set_progress_bar_config(disable=True)
-pipe.to(device=torch_device, dtype=torch_dtype).to(device)
-pipe.unet.to(memory_format=torch.channels_last)
-
-# check if computer has less than 64GB of RAM using sys or os
-if psutil.virtual_memory().total < 64 * 1024**3:
-    pipe.enable_attention_slicing()
-
-if TORCH_COMPILE:
-    pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
-    pipe.vae = torch.compile(pipe.vae, mode="reduce-overhead", fullgraph=True)
-
-    pipe(prompt="warmup", num_inference_steps=1, guidance_scale=8.0)
-
-compel_proc = Compel(
-    tokenizer=pipe.tokenizer,
-    text_encoder=pipe.text_encoder,
-    truncate_long_prompts=False,
-)
-user_queue_map = {}
-
-
-class InputParams(BaseModel):
-    seed: int = 2159232
-    prompt: str
-    guidance_scale: float = 8.0
-    strength: float = 0.5
-    steps: int = 4
-    lcm_steps: int = 50
-    width: int = WIDTH
-    height: int = HEIGHT
-
-
-def predict(params: InputParams):
-    generator = torch.manual_seed(params.seed)
-    prompt_embeds = compel_proc(params.prompt)
-    # Can be set to 1~50 steps. LCM support fast inference even <= 4 steps. Recommend: 1~8 steps.
-    results = pipe(
-        prompt_embeds=prompt_embeds,
-        generator=generator,
-        num_inference_steps=params.steps,
-        guidance_scale=params.guidance_scale,
-        width=params.width,
-        height=params.height,
-        original_inference_steps=params.lcm_steps,
-        output_type="pil",
-    )
-    nsfw_content_detected = (
-        results.nsfw_content_detected[0]
-        if "nsfw_content_detected" in results
-        else False
-    )
-    if nsfw_content_detected:
-        return None
-    return results.images[0]
-
-
-app = FastAPI()
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.websocket("/ws")
-async def websocket_endpoint(websocket: WebSocket):
-    await websocket.accept()
-    if MAX_QUEUE_SIZE > 0 and len(user_queue_map) >= MAX_QUEUE_SIZE:
-        print("Server is full")
-        await websocket.send_json({"status": "error", "message": "Server is full"})
-        await websocket.close()
-        return
-
-    try:
-        uid = str(uuid.uuid4())
-        print(f"New user connected: {uid}")
-        await websocket.send_json(
-            {"status": "success", "message": "Connected", "userId": uid}
-        )
-        user_queue_map[uid] = {
-            "queue": asyncio.Queue(),
-        }
-        await websocket.send_json(
-            {"status": "start", "message": "Start Streaming", "userId": uid}
-        )
-        await handle_websocket_data(websocket, uid)
-    except WebSocketDisconnect as e:
-        logging.error(f"WebSocket Error: {e}, {uid}")
-        traceback.print_exc()
-    finally:
-        print(f"User disconnected: {uid}")
-        queue_value = user_queue_map.pop(uid, None)
-        queue = queue_value.get("queue", None)
-        if queue:
-            while not queue.empty():
-                try:
-                    queue.get_nowait()
-                except asyncio.QueueEmpty:
-                    continue
-
-
-@app.get("/queue_size")
-async def get_queue_size():
-    queue_size = len(user_queue_map)
-    return JSONResponse({"queue_size": queue_size})
-
-
-@app.get("/stream/{user_id}")
-async def stream(user_id: uuid.UUID):
-    uid = str(user_id)
-    try:
-        user_queue = user_queue_map[uid]
-        queue = user_queue["queue"]
-
-        async def generate():
-            while True:
-                params = await queue.get()
-                if params is None:
-                    continue
-
-                image = predict(params)
-                if image is None:
-                    continue
-                frame_data = io.BytesIO()
-                image.save(frame_data, format="JPEG")
-                frame_data = frame_data.getvalue()
-                if frame_data is not None and len(frame_data) > 0:
-                    yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + frame_data + b"\r\n"
-
-                await asyncio.sleep(1.0 / 120.0)
-
-        return StreamingResponse(
-            generate(), media_type="multipart/x-mixed-replace;boundary=frame"
-        )
-    except Exception as e:
-        logging.error(f"Streaming Error: {e}, {user_queue_map}")
-        traceback.print_exc()
-        return HTTPException(status_code=404, detail="User not found")
-
-
-async def handle_websocket_data(websocket: WebSocket, user_id: uuid.UUID):
-    uid = str(user_id)
-    user_queue = user_queue_map[uid]
-    queue = user_queue["queue"]
-    if not queue:
-        return HTTPException(status_code=404, detail="User not found")
-    last_time = time.time()
-    try:
-        while True:
-            params = await websocket.receive_json()
-            params = InputParams(**params)
-            while not queue.empty():
-                try:
-                    queue.get_nowait()
-                except asyncio.QueueEmpty:
-                    continue
-            await queue.put(params)
-            if TIMEOUT > 0 and time.time() - last_time > TIMEOUT:
-                await websocket.send_json(
-                    {
-                        "status": "timeout",
-                        "message": "Your session has ended",
-                        "userId": uid,
-                    }
-                )
-                await websocket.close()
-                return
-
-    except Exception as e:
-        logging.error(f"Error: {e}")
-        traceback.print_exc()
-
-
-@app.get("/", response_class=HTMLResponse)
-async def root():
-    return FileResponse("./static/txt2img.html")

app_init.py

@@ -2,6 +2,7 @@ from fastapi import FastAPI, WebSocket, HTTPException, WebSocketDisconnect
 from fastapi.responses import StreamingResponse, JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
+from fastapi import Request
 
 import logging
 import traceback
@@ -11,8 +12,8 @@ import uuid
 from asyncio import Event, sleep
 import time
 from PIL import Image
-import io
 from types import SimpleNamespace
+from util import pil_to_frame, is_firefox
 
 
 def init_app(app: FastAPI, user_data_events: UserDataEventMap, args: Args, pipeline):
@@ -23,7 +24,6 @@ def init_app(app: FastAPI, user_data_events: UserDataEventMap, args: Args, pipel
         allow_methods=["*"],
         allow_headers=["*"],
     )
-    print("Init app", app)
 
     @app.websocket("/ws")
     async def websocket_endpoint(websocket: WebSocket):
@@ -41,7 +41,6 @@ def init_app(app: FastAPI, user_data_events: UserDataEventMap, args: Args, pipel
                 {"status": "success", "message": "Connected", "userId": uid}
             )
             user_data_events[uid] = UserDataEvent()
-            print(f"User data events: {user_data_events}")
             await websocket.send_json(
                 {"status": "start", "message": "Start Streaming", "userId": uid}
             )
@@ -59,31 +58,27 @@ def init_app(app: FastAPI, user_data_events: UserDataEventMap, args: Args, pipel
         return JSONResponse({"queue_size": queue_size})
 
     @app.get("/stream/{user_id}")
-    async def stream(user_id: uuid.UUID):
+    async def stream(user_id: uuid.UUID, request: Request):
         uid = str(user_id)
         try:
 
             async def generate():
-                last_prompt: str = None
                 while True:
                     data = await user_data_events[uid].wait_for_data()
                     params = data["params"]
-                    # input_image = data["image"]
-                    # if input_image is None:
-                    # continue
                     image = pipeline.predict(params)
                     if image is None:
                         continue
-                    frame_data = io.BytesIO()
-                    image.save(frame_data, format="JPEG")
-                    frame_data = frame_data.getvalue()
-                    if frame_data is not None and len(frame_data) > 0:
-                        yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + frame_data + b"\r\n"
-
-                    await sleep(1.0 / 120.0)
+                    frame = pil_to_frame(image)
+                    yield frame
+                    # https://bugs.chromium.org/p/chromium/issues/detail?id=1250396
+                    if not is_firefox(request.headers["user-agent"]):
+                        yield frame
 
             return StreamingResponse(
-                generate(), media_type="multipart/x-mixed-replace;boundary=frame"
+                generate(),
+                media_type="multipart/x-mixed-replace;boundary=frame",
+                headers={"Cache-Control": "no-cache"},
             )
         except Exception as e:
             logging.error(f"Streaming Error: {e}, {user_data_events}")
@@ -99,8 +94,9 @@ def init_app(app: FastAPI, user_data_events: UserDataEventMap, args: Args, pipel
             while True:
                 params = await websocket.receive_json()
                 params = pipeline.InputParams(**params)
+                info = pipeline.Info()
                 params = SimpleNamespace(**params.dict())
-                if hasattr(params, "image"):
+                if info.input_mode == "image":
                     image_data = await websocket.receive_bytes()
                     pil_image = Image.open(io.BytesIO(image_data))
                     params.image = pil_image
@@ -125,6 +121,12 @@ def init_app(app: FastAPI, user_data_events: UserDataEventMap, args: Args, pipel
     async def settings():
         info = pipeline.Info.schema()
         input_params = pipeline.InputParams.schema()
-        return JSONResponse({"info": info, "input_params": input_params})
+        return JSONResponse(
+            {
+                "info": info,
+                "input_params": input_params,
+                "max_queue_size": args.max_queue_size,
+            }
+        )
 
     app.mount("/", StaticFiles(directory="public", html=True), name="public")

frontend/src/lib/components/ImagePlayer.svelte

@@ -3,7 +3,10 @@
   import { onFrameChangeStore } from '$lib/mediaStream';
   import { PUBLIC_BASE_URL } from '$env/static/public';
 
-  $: streamId = $lcmLiveState.streamId;
+  $: streamId = $lcmLiveState?.streamId;
+  $: {
+    console.log('streamId', streamId);
+  }
 </script>
 
 <div class="relative overflow-hidden rounded-lg border border-slate-300">
@@ -14,19 +17,6 @@
     <div class="aspect-square w-full rounded-lg" />
   {/if}
   <div class="absolute left-0 top-0 aspect-square w-1/4">
-    <div class="relative z-10 aspect-square w-full object-cover">
-      <slot />
-    </div>
-    <svg
-      xmlns="http://www.w3.org/2000/svg"
-      viewBox="0 0 448 448"
-      width="100"
-      class="absolute top-0 z-0 w-full p-4 opacity-20"
-    >
-      <path
-        fill="currentColor"
-        d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z"
-      />
-    </svg>
+    <slot />
   </div>
 </div>

frontend/src/lib/components/InputRange.svelte

@@ -8,7 +8,7 @@
   });
 </script>
 
-<div class="grid grid-cols-4 items-center gap-3">
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
   <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
   <input
     class="col-span-2 h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-300 dark:bg-gray-500"

frontend/src/lib/components/PipelineOptions.svelte

@@ -6,9 +6,9 @@
   import SeedInput from './SeedInput.svelte';
   import TextArea from './TextArea.svelte';
   import Checkbox from './Checkbox.svelte';
+  import { pipelineValues } from '$lib/store';
 
   export let pipelineParams: FieldProps[];
-  export let pipelineValues = {} as any;
 
   $: advanceOptions = pipelineParams?.filter((e) => e?.hide == true);
   $: featuredOptions = pipelineParams?.filter((e) => e?.hide !== true);
@@ -18,13 +18,13 @@
   {#if featuredOptions}
     {#each featuredOptions as params}
       {#if params.field === FieldType.range}
-        <InputRange {params} bind:value={pipelineValues[params.id]}></InputRange>
+        <InputRange {params} bind:value={$pipelineValues[params.id]}></InputRange>
       {:else if params.field === FieldType.seed}
-        <SeedInput bind:value={pipelineValues[params.id]}></SeedInput>
+        <SeedInput bind:value={$pipelineValues[params.id]}></SeedInput>
       {:else if params.field === FieldType.textarea}
-        <TextArea {params} bind:value={pipelineValues[params.id]}></TextArea>
+        <TextArea {params} bind:value={$pipelineValues[params.id]}></TextArea>
       {:else if params.field === FieldType.checkbox}
-        <Checkbox {params} bind:value={pipelineValues[params.id]}></Checkbox>
+        <Checkbox {params} bind:value={$pipelineValues[params.id]}></Checkbox>
       {/if}
     {/each}
   {/if}
@@ -32,17 +32,19 @@
 
 <details open>
   <summary class="cursor-pointer font-medium">Advanced Options</summary>
-  <div class="grid grid-cols-1 items-center gap-3 sm:grid-cols-2">
+  <div
+    class="grid grid-cols-1 items-center gap-3 {pipelineValues.length > 5 ? 'sm:grid-cols-2' : ''}"
+  >
     {#if advanceOptions}
       {#each advanceOptions as params}
         {#if params.field === FieldType.range}
-          <InputRange {params} bind:value={pipelineValues[params.id]}></InputRange>
+          <InputRange {params} bind:value={$pipelineValues[params.id]}></InputRange>
         {:else if params.field === FieldType.seed}
-          <SeedInput bind:value={pipelineValues[params.id]}></SeedInput>
+          <SeedInput bind:value={$pipelineValues[params.id]}></SeedInput>
         {:else if params.field === FieldType.textarea}
-          <TextArea {params} bind:value={pipelineValues[params.id]}></TextArea>
+          <TextArea {params} bind:value={$pipelineValues[params.id]}></TextArea>
         {:else if params.field === FieldType.checkbox}
-          <Checkbox {params} bind:value={pipelineValues[params.id]}></Checkbox>
+          <Checkbox {params} bind:value={$pipelineValues[params.id]}></Checkbox>
         {/if}
       {/each}
     {/if}

frontend/src/lib/components/VideoInput.svelte

@@ -62,12 +62,25 @@
   }
 </script>
 
-<video
-  class="aspect-square w-full object-cover"
-  bind:this={videoEl}
-  playsinline
-  autoplay
-  muted
-  loop
-  use:srcObject={mediaStream}
-></video>
+<div class="relative z-10 aspect-square w-full object-cover">
+  <video
+    class="aspect-square w-full object-cover"
+    bind:this={videoEl}
+    playsinline
+    autoplay
+    muted
+    loop
+    use:srcObject={mediaStream}
+  ></video>
+</div>
+<svg
+  xmlns="http://www.w3.org/2000/svg"
+  viewBox="0 0 448 448"
+  width="100"
+  class="absolute top-0 z-0 w-full p-4 opacity-20"
+>
+  <path
+    fill="currentColor"
+    d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z"
+  />
+</svg>

frontend/src/lib/lcmLive.ts

@@ -1,5 +1,5 @@
 import { writable } from 'svelte/store';
-import { PUBLIC_BASE_URL, PUBLIC_WSS_URL } from '$env/static/public';
+import { PUBLIC_WSS_URL } from '$env/static/public';
 
 export const isStreaming = writable(false);
 export const isLCMRunning = writable(false);
@@ -26,55 +26,75 @@ export const lcmLiveState = writable(initialState);
 let websocket: WebSocket | null = null;
 export const lcmLiveActions = {
     async start() {
+        return new Promise((resolve, reject) => {
 
-        isLCMRunning.set(true);
-        try {
-            const websocketURL = PUBLIC_WSS_URL ? PUBLIC_WSS_URL : `${window.location.protocol === "https:" ? "wss" : "ws"
-                }:${window.location.host}/ws`;
+            try {
+                const websocketURL = PUBLIC_WSS_URL ? PUBLIC_WSS_URL : `${window.location.protocol === "https:" ? "wss" : "ws"
+                    }:${window.location.host}/ws`;
 
-            websocket = new WebSocket(websocketURL);
-            websocket.onopen = () => {
-                console.log("Connected to websocket");
-            };
-            websocket.onclose = () => {
+                websocket = new WebSocket(websocketURL);
+                websocket.onopen = () => {
+                    console.log("Connected to websocket");
+                };
+                websocket.onclose = () => {
+                    lcmLiveState.update((state) => ({
+                        ...state,
+                        status: LCMLiveStatus.DISCONNECTED
+                    }));
+                    console.log("Disconnected from websocket");
+                    isLCMRunning.set(false);
+                };
+                websocket.onerror = (err) => {
+                    console.error(err);
+                };
+                websocket.onmessage = (event) => {
+                    const data = JSON.parse(event.data);
+                    console.log("WS: ", data);
+                    switch (data.status) {
+                        case "success":
+                            break;
+                        case "start":
+                            const streamId = data.userId;
+                            lcmLiveState.update((state) => ({
+                                ...state,
+                                status: LCMLiveStatus.CONNECTED,
+                                streamId: streamId,
+                            }));
+                            isLCMRunning.set(true);
+                            resolve(streamId);
+                            break;
+                        case "timeout":
+                            console.log("timeout");
+                            isLCMRunning.set(false);
+                            lcmLiveState.update((state) => ({
+                                ...state,
+                                status: LCMLiveStatus.DISCONNECTED,
+                                streamId: null,
+                            }));
+                            reject("timeout");
+                        case "error":
+                            console.log(data.message);
+                            isLCMRunning.set(false);
+                            lcmLiveState.update((state) => ({
+                                ...state,
+                                status: LCMLiveStatus.DISCONNECTED,
+                                streamId: null,
+                            }));
+                            reject(data.message);
+                    }
+                };
+
+            } catch (err) {
+                console.error(err);
+                isLCMRunning.set(false);
                 lcmLiveState.update((state) => ({
                     ...state,
-                    status: LCMLiveStatus.DISCONNECTED
+                    status: LCMLiveStatus.DISCONNECTED,
+                    streamId: null,
                 }));
-                console.log("Disconnected from websocket");
-                isLCMRunning.set(false);
-            };
-            websocket.onerror = (err) => {
-                console.error(err);
-            };
-            websocket.onmessage = (event) => {
-                const data = JSON.parse(event.data);
-                console.log("WS: ", data);
-                switch (data.status) {
-                    case "success":
-                        break;
-                    case "start":
-                        const streamId = data.userId;
-                        lcmLiveState.update((state) => ({
-                            ...state,
-                            status: LCMLiveStatus.CONNECTED,
-                            streamId: streamId,
-                        }));
-                        break;
-                    case "timeout":
-                        console.log("timeout");
-                    case "error":
-                        console.log(data.message);
-                        isLCMRunning.set(false);
-                }
-            };
-            lcmLiveState.update((state) => ({
-                ...state,
-            }));
-        } catch (err) {
-            console.error(err);
-            isLCMRunning.set(false);
-        }
+                reject(err);
+            }
+        });
     },
     send(data: Blob | { [key: string]: any }) {
         if (websocket && websocket.readyState === WebSocket.OPEN) {

frontend/src/lib/store.ts

@@ -0,0 +1,4 @@
+
+import { writable, type Writable } from 'svelte/store';
+
+export const pipelineValues = writable({});

frontend/src/lib/types.ts

@@ -4,6 +4,11 @@ export const enum FieldType {
     textarea = "textarea",
     checkbox = "checkbox",
 }
+export const enum PipelineMode {
+    image = "image",
+    video = "video",
+    text = "text",
+}
 
 export interface FieldProps {
     default: number | string;
@@ -19,5 +24,7 @@ export interface FieldProps {
 export interface PipelineInfo {
     name: string;
     description: string;
-    mode: string;
+    input_mode: {
+        default: PipelineMode;
+    }
 }

frontend/src/routes/+page.svelte

@@ -2,6 +2,7 @@
   import { onMount } from 'svelte';
   import { PUBLIC_BASE_URL } from '$env/static/public';
   import type { FieldProps, PipelineInfo } from '$lib/types';
+  import { PipelineMode } from '$lib/types';
   import ImagePlayer from '$lib/components/ImagePlayer.svelte';
   import VideoInput from '$lib/components/VideoInput.svelte';
   import Button from '$lib/components/Button.svelte';
@@ -14,10 +15,12 @@
     isMediaStreaming,
     onFrameChangeStore
   } from '$lib/mediaStream';
+  import { pipelineValues } from '$lib/store';
 
   let pipelineParams: FieldProps[];
   let pipelineInfo: PipelineInfo;
-  let pipelineValues = {};
+  let isImageMode: boolean = false;
+  let maxQueueSize: number = 0;
 
   onMount(() => {
     getSettings();
@@ -27,89 +30,73 @@
     const settings = await fetch(`${PUBLIC_BASE_URL}/settings`).then((r) => r.json());
     pipelineParams = Object.values(settings.input_params.properties);
     pipelineInfo = settings.info.properties;
+    isImageMode = pipelineInfo.input_mode.default === PipelineMode.image;
+    maxQueueSize = settings.max_queue_size;
     pipelineParams = pipelineParams.filter((e) => e?.disabled !== true);
     console.log('PARAMS', pipelineParams);
     console.log('SETTINGS', pipelineInfo);
   }
+  console.log('isImageMode', isImageMode);
 
-  // $: {
-  //   console.log('isLCMRunning', $isLCMRunning);
-  // }
-  // $: {
-  //   console.log('lcmLiveState', $lcmLiveState);
-  // }
-  // $: {
-  //   console.log('mediaStreamState', $mediaStreamState);
-  // }
-  // $: if ($lcmLiveState.status === LCMLiveStatus.CONNECTED) {
-  //   lcmLiveActions.send(pipelineValues);
-  // }
-  onFrameChangeStore.subscribe(async (frame) => {
-    if ($lcmLiveState.status === LCMLiveStatus.CONNECTED) {
-      lcmLiveActions.send(pipelineValues);
-      lcmLiveActions.send(frame.blob);
+  // send Webcam stream to LCM if image mode
+  $: {
+    if (isImageMode && $lcmLiveState.status === LCMLiveStatus.CONNECTED) {
+      lcmLiveActions.send($pipelineValues);
+      lcmLiveActions.send($onFrameChangeStore.blob);
     }
-  });
-  let startBt: Button;
-  let stopBt: Button;
-  let snapShotBt: Button;
+  }
 
+  // send Webcam stream to LCM
+  $: {
+    if ($lcmLiveState.status === LCMLiveStatus.CONNECTED) {
+      lcmLiveActions.send($pipelineValues);
+    }
+  }
   async function toggleLcmLive() {
     if (!$isLCMRunning) {
-      await mediaStreamActions.enumerateDevices();
-      await mediaStreamActions.start();
-      lcmLiveActions.start();
+      if (isImageMode) {
+        await mediaStreamActions.enumerateDevices();
+        await mediaStreamActions.start();
+      }
+      await lcmLiveActions.start();
     } else {
-      mediaStreamActions.stop();
+      if (isImageMode) {
+        mediaStreamActions.stop();
+      }
       lcmLiveActions.stop();
     }
   }
-  async function startLcmLive() {
-    try {
-      $isLCMRunning = true;
-      // const res = await lcmLive.start();
-      $isLCMRunning = false;
-      // if (res.status === "timeout")
-      // toggleMessage("success")
-    } catch (err) {
-      console.log(err);
-      // toggleMessage("error")
-      $isLCMRunning = false;
-    }
-  }
-  async function stopLcmLive() {
-    // await lcmLive.stop();
-    $isLCMRunning = false;
-  }
 </script>
 
 <div class="fixed right-2 top-2 max-w-xs rounded-lg p-4 text-center text-sm font-bold" id="error" />
 <main class="container mx-auto flex max-w-4xl flex-col gap-3 px-4 py-4">
   <article class="flex- mx-auto max-w-xl text-center">
     <h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
-    <p class="text-sm">
+    <p class="py-2 text-sm">
       This demo showcases
       <a
-        href="https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7"
+        href="https://huggingface.co/blog/lcm_lora"
         target="_blank"
-        class="text-blue-500 underline hover:no-underline">LCM</a
+        class="text-blue-500 underline hover:no-underline">LCM LoRA</a
       >
       Image to Image pipeline using
       <a
-        href="https://github.com/huggingface/diffusers/tree/main/examples/community#latent-consistency-pipeline"
+        href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
         target="_blank"
         class="text-blue-500 underline hover:no-underline">Diffusers</a
       > with a MJPEG stream server.
     </p>
-    <p class="text-sm">
-      There are <span id="queue_size" class="font-bold">0</span> user(s) sharing the same GPU,
-      affecting real-time performance. Maximum queue size is 4.
-      <a
-        href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
-        target="_blank"
-        class="text-blue-500 underline hover:no-underline">Duplicate</a
-      > and run it on your own GPU.
-    </p>
+    {#if maxQueueSize > 0}
+      <p class="text-sm">
+        There are <span id="queue_size" class="font-bold">0</span> user(s) sharing the same GPU,
+        affecting real-time performance. Maximum queue size is {maxQueueSize}.
+        <a
+          href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
+          target="_blank"
+          class="text-blue-500 underline hover:no-underline">Duplicate</a
+        > and run it on your own GPU.
+      </p>
+    {/if}
   </article>
   {#if pipelineParams}
     <header>
@@ -122,7 +109,7 @@
         > syntax.
       </p>
     </header>
-    <PipelineOptions {pipelineParams} bind:pipelineValues></PipelineOptions>
+    <PipelineOptions {pipelineParams}></PipelineOptions>
     <div class="flex gap-3">
       <Button on:click={toggleLcmLive}>
         {#if $isLCMRunning}
@@ -135,7 +122,9 @@
     </div>
 
     <ImagePlayer>
-      <VideoInput></VideoInput>
+      {#if isImageMode}
+        <VideoInput></VideoInput>
+      {/if}
     </ImagePlayer>
   {:else}
     <!-- loading -->

pipelines/controlnet.py

@@ -28,6 +28,7 @@ class Pipeline:
     class Info(BaseModel):
         name: str = "txt2img"
         description: str = "Generates an image from a text prompt"
+        input_mode: str = "image"
 
     class InputParams(BaseModel):
         prompt: str = Field(
@@ -125,7 +126,6 @@ class Pipeline:
             hide=True,
             id="debug_canny",
         )
-        image: bool = True
 
     def __init__(self, args: Args, device: torch.device, torch_dtype: torch.dtype):
         controlnet_canny = ControlNetModel.from_pretrained(

pipelines/txt2img.py

@@ -22,6 +22,7 @@ class Pipeline:
     class Info(BaseModel):
         name: str = "txt2img"
         description: str = "Generates an image from a text prompt"
+        input_mode: str = "text"
 
     class InputParams(BaseModel):
         prompt: str = Field(
@@ -52,9 +53,6 @@ class Pipeline:
             hide=True,
             id="guidance_scale",
         )
-        image: bool = Field(
-            True, title="Image", field="checkbox", hide=True, id="image"
-        )
 
     def __init__(self, args: Args, device: torch.device, torch_dtype: torch.dtype):
         if args.safety_checker:

static/controlnet.html

@@ -1,427 +0,0 @@
-<!doctype html>
-<html>
-
-<head>
-    <meta charset="UTF-8">
-    <title>Real-Time Latent Consistency Model ControlNet</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <script
-        src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.1/iframeResizer.contentWindow.min.js"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]/piexif.min.js"></script>
-    <script src="https://cdn.tailwindcss.com"></script>
-    <style type="text/tailwindcss">
-        .button {
-          @apply bg-gray-700 hover:bg-gray-800 text-white font-normal p-2 rounded disabled:bg-gray-300 dark:disabled:bg-gray-700 disabled:cursor-not-allowed dark:disabled:text-black
-        }
-    </style>
-    <script type="module">
-        const getValue = (id) => {
-            const el = document.querySelector(`${id}`)
-            if (el.type === "checkbox")
-                return el.checked;
-            return el.value;
-        }
-        const startBtn = document.querySelector("#start");
-        const stopBtn = document.querySelector("#stop");
-        const videoEl = document.querySelector("#webcam");
-        const imageEl = document.querySelector("#player");
-        const queueSizeEl = document.querySelector("#queue_size");
-        const errorEl = document.querySelector("#error");
-        const snapBtn = document.querySelector("#snap");
-        const webcamsEl = document.querySelector("#webcams");
-
-        function LCMLive(webcamVideo, liveImage) {
-            let websocket;
-
-            async function start() {
-                return new Promise((resolve, reject) => {
-                    const websocketURL = `${window.location.protocol === "https:" ? "wss" : "ws"
-                        }:${window.location.host}/ws`;
-
-                    const socket = new WebSocket(websocketURL);
-                    socket.onopen = () => {
-                        console.log("Connected to websocket");
-                    };
-                    socket.onclose = () => {
-                        console.log("Disconnected from websocket");
-                        stop();
-                        resolve({ "status": "disconnected" });
-                    };
-                    socket.onerror = (err) => {
-                        console.error(err);
-                        reject(err);
-                    };
-                    socket.onmessage = (event) => {
-                        const data = JSON.parse(event.data);
-                        switch (data.status) {
-                            case "success":
-                                break;
-                            case "start":
-                                const userId = data.userId;
-                                initVideoStream(userId);
-                                break;
-                            case "timeout":
-                                stop();
-                                resolve({ "status": "timeout" });
-                            case "error":
-                                stop();
-                                reject(data.message);
-
-                        }
-                    };
-                    websocket = socket;
-                })
-            }
-            function switchCamera() {
-                const constraints = {
-                    audio: false,
-                    video: { width: 1024, height: 1024, deviceId: mediaDevices[webcamsEl.value].deviceId }
-                };
-                navigator.mediaDevices
-                    .getUserMedia(constraints)
-                    .then((mediaStream) => {
-                        webcamVideo.removeEventListener("timeupdate", videoTimeUpdateHandler);
-                        webcamVideo.srcObject = mediaStream;
-                        webcamVideo.onloadedmetadata = () => {
-                            webcamVideo.play();
-                            webcamVideo.addEventListener("timeupdate", videoTimeUpdateHandler);
-                        };
-                    })
-                    .catch((err) => {
-                        console.error(`${err.name}: ${err.message}`);
-                    });
-            }
-
-            async function videoTimeUpdateHandler() {
-                const dimension = getValue("input[name=dimension]:checked");
-                const [WIDTH, HEIGHT] = JSON.parse(dimension);
-
-                const canvas = new OffscreenCanvas(WIDTH, HEIGHT);
-                const videoW = webcamVideo.videoWidth;
-                const videoH = webcamVideo.videoHeight;
-                const aspectRatio = WIDTH / HEIGHT;
-
-                const ctx = canvas.getContext("2d");
-                ctx.drawImage(webcamVideo, videoW / 2 - videoH * aspectRatio / 2, 0, videoH * aspectRatio, videoH, 0, 0, WIDTH, HEIGHT)
-                const blob = await canvas.convertToBlob({ type: "image/jpeg", quality: 1 });
-                websocket.send(blob);
-                websocket.send(JSON.stringify({
-                    "seed": getValue("#seed"),
-                    "prompt": getValue("#prompt"),
-                    "guidance_scale": getValue("#guidance-scale"),
-                    "strength": getValue("#strength"),
-                    "steps": getValue("#steps"),
-                    "lcm_steps": getValue("#lcm_steps"),
-                    "width": WIDTH,
-                    "height": HEIGHT,
-                    "controlnet_scale": getValue("#controlnet_scale"),
-                    "controlnet_start": getValue("#controlnet_start"),
-                    "controlnet_end": getValue("#controlnet_end"),
-                    "canny_low_threshold": getValue("#canny_low_threshold"),
-                    "canny_high_threshold": getValue("#canny_high_threshold"),
-                    "debug_canny": getValue("#debug_canny")
-                }));
-            }
-            let mediaDevices = [];
-            async function initVideoStream(userId) {
-                liveImage.src = `/stream/${userId}`;
-                await navigator.mediaDevices.enumerateDevices()
-                    .then(devices => {
-                        const cameras = devices.filter(device => device.kind === 'videoinput');
-                        mediaDevices = cameras;
-                        webcamsEl.innerHTML = "";
-                        cameras.forEach((camera, index) => {
-                            const option = document.createElement("option");
-                            option.value = index;
-                            option.innerText = camera.label;
-                            webcamsEl.appendChild(option);
-                            option.selected = index === 0;
-                        });
-                        webcamsEl.addEventListener("change", switchCamera);
-                    })
-                    .catch(err => {
-                        console.error(err);
-                    });
-                const constraints = {
-                    audio: false,
-                    video: { width: 1024, height: 1024, deviceId: mediaDevices[0].deviceId }
-                };
-                navigator.mediaDevices
-                    .getUserMedia(constraints)
-                    .then((mediaStream) => {
-                        webcamVideo.srcObject = mediaStream;
-                        webcamVideo.onloadedmetadata = () => {
-                            webcamVideo.play();
-                            webcamVideo.addEventListener("timeupdate", videoTimeUpdateHandler);
-                        };
-                    })
-                    .catch((err) => {
-                        console.error(`${err.name}: ${err.message}`);
-                    });
-            }
-
-
-            async function stop() {
-                websocket.close();
-                navigator.mediaDevices.getUserMedia({ video: true }).then((mediaStream) => {
-                    mediaStream.getTracks().forEach((track) => track.stop());
-                });
-                webcamVideo.removeEventListener("timeupdate", videoTimeUpdateHandler);
-                webcamsEl.removeEventListener("change", switchCamera);
-                webcamVideo.srcObject = null;
-            }
-            return {
-                start,
-                stop
-            }
-        }
-        function toggleMessage(type) {
-            errorEl.hidden = false;
-            errorEl.scrollIntoView();
-            switch (type) {
-                case "error":
-                    errorEl.innerText = "To many users are using the same GPU, please try again later.";
-                    errorEl.classList.toggle("bg-red-300", "text-red-900");
-                    break;
-                case "success":
-                    errorEl.innerText = "Your session has ended, please start a new one.";
-                    errorEl.classList.toggle("bg-green-300", "text-green-900");
-                    break;
-            }
-            setTimeout(() => {
-                errorEl.hidden = true;
-            }, 2000);
-        }
-        function snapImage() {
-            try {
-                const zeroth = {};
-                const exif = {};
-                const gps = {};
-                zeroth[piexif.ImageIFD.Make] = "LCM Image-to-Image ControNet";
-                zeroth[piexif.ImageIFD.ImageDescription] = `prompt: ${getValue("#prompt")} | seed: ${getValue("#seed")} | guidance_scale: ${getValue("#guidance-scale")} | strength: ${getValue("#strength")} | controlnet_start: ${getValue("#controlnet_start")} | controlnet_end: ${getValue("#controlnet_end")} | lcm_steps: ${getValue("#lcm_steps")} | steps: ${getValue("#steps")}`;
-                zeroth[piexif.ImageIFD.Software] = "https://github.com/radames/Real-Time-Latent-Consistency-Model";
-                exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString();
-
-                const exifObj = { "0th": zeroth, "Exif": exif, "GPS": gps };
-                const exifBytes = piexif.dump(exifObj);
-
-                const canvas = document.createElement("canvas");
-                canvas.width = imageEl.naturalWidth;
-                canvas.height = imageEl.naturalHeight;
-                const ctx = canvas.getContext("2d");
-                ctx.drawImage(imageEl, 0, 0);
-                const dataURL = canvas.toDataURL("image/jpeg");
-                const withExif = piexif.insert(exifBytes, dataURL);
-
-                const a = document.createElement("a");
-                a.href = withExif;
-                a.download = `lcm_txt_2_img${Date.now()}.png`;
-                a.click();
-            } catch (err) {
-                console.log(err);
-            }
-        }
-
-
-        const lcmLive = LCMLive(videoEl, imageEl);
-        startBtn.addEventListener("click", async () => {
-            try {
-                startBtn.disabled = true;
-                snapBtn.disabled = false;
-                const res = await lcmLive.start();
-                startBtn.disabled = false;
-                if (res.status === "timeout")
-                    toggleMessage("success")
-            } catch (err) {
-                console.log(err);
-                toggleMessage("error")
-                startBtn.disabled = false;
-            }
-        });
-        stopBtn.addEventListener("click", () => {
-            lcmLive.stop();
-        });
-        window.addEventListener("beforeunload", () => {
-            lcmLive.stop();
-        });
-        snapBtn.addEventListener("click", snapImage);
-        setInterval(() =>
-            fetch("/queue_size")
-                .then((res) => res.json())
-                .then((data) => {
-                    queueSizeEl.innerText = data.queue_size;
-                })
-                .catch((err) => {
-                    console.log(err);
-                })
-            , 5000);
-    </script>
-</head>
-
-<body class="text-black dark:bg-gray-900 dark:text-white">
-    <div class="fixed right-2 top-2 p-4 font-bold text-sm rounded-lg max-w-xs text-center" id="error">
-    </div>
-    <main class="container mx-auto px-4 py-4 max-w-4xl flex flex-col gap-4">
-        <article class="text-center max-w-xl mx-auto">
-            <h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
-            <h2 class="text-2xl font-bold mb-4">ControlNet</h2>
-            <p class="text-sm">
-                This demo showcases
-                <a href="https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7" target="_blank"
-                    class="text-blue-500 underline hover:no-underline">LCM</a> Image to Image pipeline
-                using
-                <a href="https://github.com/huggingface/diffusers/tree/main/examples/community#latent-consistency-pipeline"
-                    target="_blank" class="text-blue-500 underline hover:no-underline">Diffusers</a> with a MJPEG
-                stream server.
-            </p>
-            <p class="text-sm">
-                There are <span id="queue_size" class="font-bold">0</span> user(s) sharing the same GPU, affecting
-                real-time performance. Maximum queue size is 4. <a
-                    href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
-                    target="_blank" class="text-blue-500 underline hover:no-underline">Duplicate</a> and run it on your
-                own GPU.
-            </p>
-        </article>
-        <div>
-            <h2 class="font-medium">Prompt</h2>
-            <p class="text-sm text-gray-500">
-                Change the prompt to generate different images, accepts <a
-                    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md" target="_blank"
-                    class="text-blue-500 underline hover:no-underline">Compel</a> syntax.
-            </p>
-            <div class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center">
-                <textarea type="text" id="prompt" class="font-light w-full px-3 py-2 mx-1  outline-none dark:text-black"
-                    title="Prompt, this is an example, feel free to modify"
-                    placeholder="Add your prompt here...">Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5, cinematic, masterpiece</textarea>
-            </div>
-        </div>
-        <div class="">
-            <details>
-                <summary class="font-medium cursor-pointer">Advanced Options</summary>
-                <div class="grid grid-cols-3 sm:grid-cols-6 items-center gap-3 py-3">
-                    <label for="webcams" class="text-sm font-medium">Camera Options: </label>
-                    <select id="webcams" class="text-sm border-2 border-gray-500 rounded-md font-light dark:text-black">
-                    </select>
-                    <div></div>
-                    <label class="text-sm font-medium " for="steps">Inference Steps
-                    </label>
-                    <input type="range" id="steps" name="steps" min="1" max="20" value="4"
-                        oninput="this.nextElementSibling.value = Number(this.value)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        4</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="lcm_steps">LCM Inference Steps
-                    </label>
-                    <input type="range" id="lcm_steps" name="lcm_steps" min="2" max="60" value="50"
-                        oninput="this.nextElementSibling.value = Number(this.value)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        50</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="guidance-scale">Guidance Scale
-                    </label>
-                    <input type="range" id="guidance-scale" name="guidance-scale" min="0" max="30" step="0.001"
-                        value="8.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        8.0</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="strength">Strength</label>
-                    <input type="range" id="strength" name="strength" min="0.1" max="1" step="0.001" value="0.50"
-                        oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        0.5</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="controlnet_scale">ControlNet Condition Scale</label>
-                    <input type="range" id="controlnet_scale" name="controlnet_scale" min="0.0" max="1" step="0.001"
-                        value="0.80" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        0.8</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="controlnet_start">ControlNet Guidance Start</label>
-                    <input type="range" id="controlnet_start" name="controlnet_start" min="0.0" max="1.0" step="0.001"
-                        value="0.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        0.0</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="controlnet_end">ControlNet Guidance End</label>
-                    <input type="range" id="controlnet_end" name="controlnet_end" min="0.0" max="1.0" step="0.001"
-                        value="1.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        1.0</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="canny_low_threshold">Canny Low Threshold</label>
-                    <input type="range" id="canny_low_threshold" name="canny_low_threshold" min="0.0" max="1.0"
-                        step="0.001" value="0.1"
-                        oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        0.1</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="canny_high_threshold">Canny High Threshold</label>
-                    <input type="range" id="canny_high_threshold" name="canny_high_threshold" min="0.0" max="1.0"
-                        step="0.001" value="0.2"
-                        oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        0.2</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="seed">Seed</label>
-                    <input type="number" id="seed" name="seed" value="299792458"
-                        class="font-light border border-gray-700 text-right rounded-md p-2 dark:text-black">
-                    <button
-                        onclick="document.querySelector('#seed').value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)"
-                        class="button">
-                        Rand
-                    </button>
-                    <!--  -->
-                    <!--  -->
-                    <label class="text-sm font-medium" for="dimension">Image Dimensions</label>
-                    <div class="col-span-2 flex gap-2">
-                        <div class="flex gap-1">
-                            <input type="radio" id="dimension512" name="dimension" value="[512,512]" checked
-                                class="cursor-pointer">
-                            <label for="dimension512" class="text-sm cursor-pointer">512x512</label>
-                        </div>
-                        <div class="flex gap-1">
-                            <input type="radio" id="dimension768" name="dimension" value="[768,768]"
-                                lass="cursor-pointer">
-                            <label for="dimension768" class="text-sm cursor-pointer">768x768</label>
-                        </div>
-                    </div>
-                    <!--  -->
-                    <!--  -->
-                    <label class="text-sm font-medium" for="debug_canny">Debug Canny</label>
-                    <div class="col-span-2 flex gap-2">
-                        <input type="checkbox" id="debug_canny" name="debug_canny" class="cursor-pointer">
-                        <label for="debug_canny" class="text-sm cursor-pointer"></label>
-                    </div>
-                    <div></div>
-                    <!--  -->
-                </div>
-            </details>
-        </div>
-        <div class="flex gap-3">
-            <button id="start" class="button">
-                Start
-            </button>
-            <button id="stop" class="button">
-                Stop
-            </button>
-            <button id="snap" disabled class="button ml-auto">
-                Snapshot
-            </button>
-        </div>
-        <div class="relative rounded-lg border border-slate-300 overflow-hidden">
-            <img id="player" class="w-full aspect-square rounded-lg"
-                src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">
-            <div class="absolute top-0 left-0 w-1/4 aspect-square">
-                <video id="webcam" class="w-full aspect-square relative z-10 object-cover" playsinline autoplay muted
-                    loop></video>
-                <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 448" width="100"
-                    class="w-full p-4 absolute top-0 opacity-20 z-0">
-                    <path fill="currentColor"
-                        d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z" />
-                </svg>
-            </div>
-        </div>
-    </main>
-</body>
-
-</html>

static/txt2img.html

@@ -1,304 +0,0 @@
-<!doctype html>
-<html>
-
-<head>
-    <meta charset="UTF-8">
-    <title>Real-Time Latent Consistency Model</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <script
-        src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.1/iframeResizer.contentWindow.min.js"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]/piexif.min.js"></script>
-    <script src="https://cdn.tailwindcss.com"></script>
-    <style type="text/tailwindcss">
-        .button {
-          @apply bg-gray-700 hover:bg-gray-800 text-white font-normal p-2 rounded disabled:bg-gray-300 dark:disabled:bg-gray-700 disabled:cursor-not-allowed dark:disabled:text-black
-        }
-    </style>
-    <script type="module">
-        const getValue = (id) => {
-            const el = document.querySelector(`${id}`)
-            if (el.type === "checkbox")
-                return el.checked;
-            return el.value;
-        }
-        const startBtn = document.querySelector("#start");
-        const stopBtn = document.querySelector("#stop");
-        const videoEl = document.querySelector("#webcam");
-        const imageEl = document.querySelector("#player");
-        const queueSizeEl = document.querySelector("#queue_size");
-        const errorEl = document.querySelector("#error");
-        const snapBtn = document.querySelector("#snap");
-        const paramsEl = document.querySelector("#params");
-        const promptEl = document.querySelector("#prompt");
-        paramsEl.addEventListener("submit", (e) => e.preventDefault());
-        function LCMLive(promptEl, paramsEl, liveImage) {
-            let websocket;
-
-            async function start() {
-                return new Promise((resolve, reject) => {
-                    const websocketURL = `${window.location.protocol === "https:" ? "wss" : "ws"
-                        }:${window.location.host}/ws`;
-
-                    const socket = new WebSocket(websocketURL);
-                    socket.onopen = () => {
-                        console.log("Connected to websocket");
-                    };
-                    socket.onclose = () => {
-                        console.log("Disconnected from websocket");
-                        stop();
-                        resolve({ "status": "disconnected" });
-                    };
-                    socket.onerror = (err) => {
-                        console.error(err);
-                        reject(err);
-                    };
-                    socket.onmessage = (event) => {
-                        const data = JSON.parse(event.data);
-                        switch (data.status) {
-                            case "success":
-                                break;
-                            case "start":
-                                const userId = data.userId;
-                                initPromptStream(userId);
-                                break;
-                            case "timeout":
-                                stop();
-                                resolve({ "status": "timeout" });
-                            case "error":
-                                stop();
-                                reject(data.message);
-                        }
-                    };
-                    websocket = socket;
-                })
-            }
-
-            async function promptUpdateStream(e) {
-                const dimension = getValue("input[name=dimension]:checked");
-                const [WIDTH, HEIGHT] = JSON.parse(dimension);
-                websocket.send(JSON.stringify({
-                    "seed": getValue("#seed"),
-                    "prompt": getValue("#prompt"),
-                    "guidance_scale": getValue("#guidance-scale"),
-                    "steps": getValue("#steps"),
-                    "lcm_steps": getValue("#lcm_steps"),
-                    "width": WIDTH,
-                    "height": HEIGHT,
-                }));
-            }
-            function debouceInput(fn, delay) {
-                let timer;
-                return function (...args) {
-                    clearTimeout(timer);
-                    timer = setTimeout(() => {
-                        fn(...args);
-                    }, delay);
-                }
-            }
-            const debouncedInput = debouceInput(promptUpdateStream, 200);
-            function initPromptStream(userId) {
-                liveImage.src = `/stream/${userId}`;
-                paramsEl.addEventListener("change", debouncedInput);
-                promptEl.addEventListener("input", debouncedInput);
-            }
-
-            async function stop() {
-                websocket.close();
-                paramsEl.removeEventListener("change", debouncedInput);
-                promptEl.removeEventListener("input", debouncedInput);
-            }
-            return {
-                start,
-                stop
-            }
-        }
-        function toggleMessage(type) {
-            errorEl.hidden = false;
-            errorEl.scrollIntoView();
-            switch (type) {
-                case "error":
-                    errorEl.innerText = "To many users are using the same GPU, please try again later.";
-                    errorEl.classList.toggle("bg-red-300", "text-red-900");
-                    break;
-                case "success":
-                    errorEl.innerText = "Your session has ended, please start a new one.";
-                    errorEl.classList.toggle("bg-green-300", "text-green-900");
-                    break;
-            }
-            setTimeout(() => {
-                errorEl.hidden = true;
-            }, 2000);
-        }
-        function snapImage() {
-            try {
-                const zeroth = {};
-                const exif = {};
-                const gps = {};
-                zeroth[piexif.ImageIFD.Make] = "LCM Text-to-Image";
-                zeroth[piexif.ImageIFD.ImageDescription] = `prompt: ${getValue("#prompt")} | seed: ${getValue("#seed")} | guidance_scale: ${getValue("#guidance-scale")} | lcm_steps: ${getValue("#lcm_steps")} | steps: ${getValue("#steps")}`;
-                zeroth[piexif.ImageIFD.Software] = "https://github.com/radames/Real-Time-Latent-Consistency-Model";
-
-                exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString();
-
-                const exifObj = { "0th": zeroth, "Exif": exif, "GPS": gps };
-                const exifBytes = piexif.dump(exifObj);
-
-                const canvas = document.createElement("canvas");
-                canvas.width = imageEl.naturalWidth;
-                canvas.height = imageEl.naturalHeight;
-                const ctx = canvas.getContext("2d");
-                ctx.drawImage(imageEl, 0, 0);
-                const dataURL = canvas.toDataURL("image/jpeg");
-                const withExif = piexif.insert(exifBytes, dataURL);
-
-                const a = document.createElement("a");
-                a.href = withExif;
-                a.download = `lcm_txt_2_img${Date.now()}.png`;
-                a.click();
-            } catch (err) {
-                console.log(err);
-            }
-        }
-
-
-        const lcmLive = LCMLive(promptEl, paramsEl, imageEl);
-        startBtn.addEventListener("click", async () => {
-            try {
-                startBtn.disabled = true;
-                snapBtn.disabled = false;
-                const res = await lcmLive.start();
-                startBtn.disabled = false;
-                if (res.status === "timeout")
-                    toggleMessage("success")
-            } catch (err) {
-                console.log(err);
-                toggleMessage("error")
-                startBtn.disabled = false;
-            }
-        });
-        stopBtn.addEventListener("click", () => {
-            lcmLive.stop();
-        });
-        window.addEventListener("beforeunload", () => {
-            lcmLive.stop();
-        });
-        snapBtn.addEventListener("click", snapImage);
-        setInterval(() =>
-            fetch("/queue_size")
-                .then((res) => res.json())
-                .then((data) => {
-                    queueSizeEl.innerText = data.queue_size;
-                })
-                .catch((err) => {
-                    console.log(err);
-                })
-            , 5000);
-    </script>
-</head>
-
-<body class="text-black dark:bg-gray-900 dark:text-white">
-    <div class="fixed right-2 top-2 p-4 font-bold text-sm rounded-lg max-w-xs text-center" id="error">
-    </div>  
-    <main class="container mx-auto px-4 py-4 max-w-4xl flex flex-col gap-4">
-        <article class="text-center max-w-xl mx-auto">
-            <h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
-            <h2 class="text-2xl font-bold mb-4">Text to Image</h2>
-            <p class="text-sm">
-                This demo showcases
-                <a href="https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7" target="_blank"
-                    class="text-blue-500 underline hover:no-underline">LCM</a> Text to Image model
-                using
-                <a href="https://github.com/huggingface/diffusers/tree/main/examples/community#latent-consistency-pipeline"
-                    target="_blank" class="text-blue-500 underline hover:no-underline">Diffusers</a> with a MJPEG
-                stream server.
-            </p>
-            <p class="text-sm">
-                There are <span id="queue_size" class="font-bold">0</span> user(s) sharing the same GPU, affecting
-                real-time performance. Maximum queue size is 10. <a
-                    href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
-                    target="_blank" class="text-blue-500 underline hover:no-underline">Duplicate</a> and run it on your
-                own GPU.
-            </p>
-        </article>
-        <div>
-            <h2 class="font-medium">Prompt</h2>
-            <p class="text-sm text-gray-500 dark:text-gray-400">
-                Start your session and type your prompt here, accepts
-                <a href="https://github.com/damian0815/compel/blob/main/doc/syntax.md" target="_blank"
-                    class="text-blue-500 underline hover:no-underline">Compel</a> syntax.
-            </p>
-            <div class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center">
-                <textarea type="text" id="prompt" class="font-light w-full px-3 py-2 mx-1  outline-none dark:text-black"
-                    title=" Start your session and type your prompt here, you can see the result in real-time."
-                    placeholder="Add your prompt here...">Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5, cinematic, masterpiece</textarea>
-            </div>
-
-        </div>
-        <div class="">
-            <details>
-                <summary class="font-medium cursor-pointer">Advanced Options</summary>
-                <form class="grid grid-cols-3 items-center gap-3 py-3" id="params" action="">
-                    <label class="text-sm font-medium" for="dimension">Image Dimensions</label>
-                    <div class="col-span-2 flex gap-2">
-                        <div class="flex gap-1">
-                            <input type="radio" id="dimension512" name="dimension" value="[512,512]" checked
-                                class="cursor-pointer">
-                            <label for="dimension512" class="text-sm cursor-pointer">512x512</label>
-                        </div>
-                        <div class="flex gap-1">
-                            <input type="radio" id="dimension768" name="dimension" value="[768,768]"
-                                lass="cursor-pointer">
-                            <label for="dimension768" class="text-sm cursor-pointer">768x768</label>
-                        </div>
-                    </div>
-                    <!--  -->
-                    <label class="text-sm font-medium " for="steps">Inference Steps
-                    </label>
-                    <input type="range" id="steps" name="steps" min="1" max="20" value="4"
-                        oninput="this.nextElementSibling.value = Number(this.value)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        4</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="lcm_steps">LCM Inference Steps
-                    </label>
-                    <input type="range" id="lcm_steps" name="lcm_steps" min="2" max="60" value="50"
-                        oninput="this.nextElementSibling.value = Number(this.value)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        50</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="guidance-scale">Guidance Scale
-                    </label>
-                    <input type="range" id="guidance-scale" name="guidance-scale" min="0" max="30" step="0.001"
-                        value="8.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)">
-                    <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
-                        8.0</output>
-                    <!--  -->
-                    <label class="text-sm font-medium" for="seed">Seed</label>
-                    <input type="number" id="seed" name="seed" value="299792458"
-                        class="font-light border border-gray-700 text-right rounded-md p-2 dark:text-black">
-                    <button class="button" onclick="document.querySelector('#seed').value = Math.floor(Math.random() * 1000000000); document.querySelector('#params').dispatchEvent(new Event('change'))">
-                        Rand
-                    </button>
-                    <!--  -->
-                </form>
-            </details>
-        </div>
-        <div class="flex gap-3">
-            <button id="start" class="button">
-                Start
-            </button>
-            <button id="stop" class="button">
-                Stop
-            </button>
-            <button id="snap" disabled class="button ml-auto">
-                Snapshot
-            </button>
-        </div>
-        <div class="relative rounded-lg border border-slate-300 overflow-hidden">
-            <img id="player" class="w-full aspect-square rounded-lg"
-                src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">
-        </div>
-    </main>
-</body>
-
-</html>

util.py

@@ -2,6 +2,8 @@ from importlib import import_module
 from types import ModuleType
 from typing import Dict, Any
 from pydantic import BaseModel as PydanticBaseModel, Field
+from PIL import Image
+import io
 
 
 def get_pipeline_class(pipeline_name: str) -> ModuleType:
@@ -16,3 +18,20 @@ def get_pipeline_class(pipeline_name: str) -> ModuleType:
         raise ValueError(f"'Pipeline' class not found in module '{pipeline_name}'.")
 
     return pipeline_class
+
+
+def pil_to_frame(image: Image.Image) -> bytes:
+    frame_data = io.BytesIO()
+    image.save(frame_data, format="JPEG")
+    frame_data = frame_data.getvalue()
+    return (
+        b"--frame\r\n"
+        + b"Content-Type: image/jpeg\r\n"
+        + f"Content-Length: {len(frame_data)}\r\n\r\n".encode()
+        + frame_data
+        + b"\r\n"
+    )
+
+
+def is_firefox(user_agent: str) -> bool:
+    return "Firefox" in user_agent