diff --git a/README.md b/README.md index 453f2fe..3d87495 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,7 @@ This client library is crafted as a lightweight layer atop platform standards li fal.config({ // Can also be auto-configured using environment variables: - // Either a single FAL_KEY or a combination of FAL_KEY_ID and FAL_KEY_SECRET - credentials: 'FAL_KEY_ID:FAL_KEY_SECRET', + credentials: 'FAL_KEY', }); ``` diff --git a/apps/demo-nextjs-app-router/app/camera-turbo/page.tsx b/apps/demo-nextjs-app-router/app/camera-turbo/page.tsx index 9f05f74..1c10e01 100644 --- a/apps/demo-nextjs-app-router/app/camera-turbo/page.tsx +++ b/apps/demo-nextjs-app-router/app/camera-turbo/page.tsx @@ -109,7 +109,7 @@ const useWebcam = ({ type LCMInput = { prompt: string; - image: Uint8Array; + image_bytes: Uint8Array; strength?: number; negative_prompt?: string; seed?: number | null; @@ -121,8 +121,14 @@ type LCMInput = { width?: number; }; +type ImageOutput = { + content: Uint8Array; + width: number; + height: number; +}; + type LCMOutput = { - image: Uint8Array; + images: ImageOutput[]; timings: Record; seed: number; num_inference_steps: number; @@ -137,15 +143,17 @@ export default function WebcamPage() { const previewRef = useRef(null); const { send } = fal.realtime.connect( - 'fal-ai/sd-turbo-real-time-high-fps-msgpack-a10g', + 'fal-ai/fast-turbo-diffusion/image-to-image', { connectionKey: 'camera-turbo-demo', // not throttling the client, handling throttling of the camera itself // and letting all requests through in real-time throttleInterval: 0, onResult(result) { - if (processedImageRef.current && result.image) { - const blob = new Blob([result.image], { type: 'image/jpeg' }); + if (processedImageRef.current && result.images && result.images[0]) { + const blob = new Blob([result.images[0].content], { + type: 'image/jpeg', + }); const url = URL.createObjectURL(blob); processedImageRef.current.src = url; } @@ -158,10 +166,10 @@ export default function WebcamPage() { return; } send({ - prompt: 'a picture of leonardo di caprio, elegant, in a suit, 8k, uhd', - image: data, + prompt: 'a picture of george clooney, elegant, in a suit, 8k, uhd', + image_bytes: data, num_inference_steps: 3, - strength: 0.44, + strength: 0.6, guidance_scale: 1, seed: 6252023, }); diff --git a/apps/demo-nextjs-app-router/app/realtime/page.tsx b/apps/demo-nextjs-app-router/app/realtime/page.tsx index f39bd4c..4c81771 100644 --- a/apps/demo-nextjs-app-router/app/realtime/page.tsx +++ b/apps/demo-nextjs-app-router/app/realtime/page.tsx @@ -2,27 +2,64 @@ /* eslint-disable @next/next/no-img-element */ import * as fal from '@fal-ai/serverless-client'; -import { useState } from 'react'; +import { ChangeEvent, useRef, useState } from 'react'; import { DrawingCanvas } from '../../components/drawing'; fal.config({ proxyUrl: '/api/fal/proxy', }); -const PROMPT = 'a moon in a starry night sky'; +const PROMPT_EXPANDED = + ', beautiful, colorful, highly detailed, best quality, uhd'; + +const PROMPT = 'a moon in the night sky'; + +const defaults = { + model_name: 'runwayml/stable-diffusion-v1-5', + image_size: 'square', + num_inference_steps: 4, + seed: 6252023, +}; export default function RealtimePage() { - const [image, setImage] = useState(null); + const [prompt, setPrompt] = useState(PROMPT); - const { send } = fal.realtime.connect('fal-ai/lcm-sd15-i2i', { - connectionKey: 'realtime-demo', - throttleInterval: 128, - onResult(result) { - if (result.images && result.images[0]) { - setImage(result.images[0].url); - } - }, - }); + const currentDrawing = useRef(null); + const outputCanvasRef = useRef(null); + + const { send } = fal.realtime.connect( + 'fal-ai/fast-lcm-diffusion/image-to-image', + { + connectionKey: 'realtime-demo', + throttleInterval: 128, + onResult(result) { + if (result.images && result.images[0] && result.images[0].content) { + const canvas = outputCanvasRef.current; + const context = canvas?.getContext('2d'); + if (canvas && context) { + const imageBytes: Uint8Array = result.images[0].content; + const blob = new Blob([imageBytes], { type: 'image/png' }); + createImageBitmap(blob) + .then((bitmap) => { + context.drawImage(bitmap, 0, 0); + }) + .catch(console.error); + } + } + }, + } + ); + + const handlePromptChange = (e: ChangeEvent) => { + setPrompt(e.target.value); + if (currentDrawing.current) { + send({ + prompt: e.target.value.trim() + PROMPT_EXPANDED, + image_bytes: currentDrawing.current, + ...defaults, + }); + } + }; return (
@@ -30,31 +67,34 @@ export default function RealtimePage() {

falrealtime

-
-
{PROMPT}
+
+
{ + currentDrawing.current = imageData; send({ - prompt: PROMPT, - image_url: imageData, - sync_mode: true, - seed: 6252023, + prompt: prompt + PROMPT_EXPANDED, + image_bytes: imageData, + ...defaults, }); }} />
-
- {image && ( - {`${PROMPT} - )} +
+
diff --git a/apps/demo-nextjs-app-router/components/drawing.tsx b/apps/demo-nextjs-app-router/components/drawing.tsx index cd34261..7d48c78 100644 --- a/apps/demo-nextjs-app-router/components/drawing.tsx +++ b/apps/demo-nextjs-app-router/components/drawing.tsx @@ -10,14 +10,14 @@ import initialDrawing from './drawingState.json'; export type CanvasChangeEvent = { elements: readonly ExcalidrawElement[]; appState: AppState; - imageData: string; + imageData: Uint8Array; }; export type DrawingCanvasProps = { onCanvasChange: (event: CanvasChangeEvent) => void; }; -async function blobToBase64(blob: Blob): Promise { +export async function blobToBase64(blob: Blob): Promise { const reader = new FileReader(); reader.readAsDataURL(blob); return new Promise((resolve) => { @@ -27,6 +27,11 @@ async function blobToBase64(blob: Blob): Promise { }); } +export async function blobToUint8Array(blob: Blob): Promise { + const buffer = await blob.arrayBuffer(); + return new Uint8Array(buffer); +} + export function DrawingCanvas({ onCanvasChange }: DrawingCanvasProps) { const [ExcalidrawComponent, setExcalidrawComponent] = useState< typeof Excalidraw | null @@ -95,7 +100,7 @@ export function DrawingCanvas({ onCanvasChange }: DrawingCanvasProps) { return { width: 512, height: 512 }; }, }); - const imageData = await blobToBase64(blob); + const imageData = await blobToUint8Array(blob); onCanvasChange({ elements, appState, imageData }); } }, diff --git a/libs/client/package.json b/libs/client/package.json index 5abf3fc..de72bdf 100644 --- a/libs/client/package.json +++ b/libs/client/package.json @@ -1,7 +1,7 @@ { "name": "@fal-ai/serverless-client", "description": "The fal serverless JS/TS client", - "version": "0.9.3", + "version": "0.10.0", "license": "MIT", "repository": { "type": "git", diff --git a/libs/client/src/index.ts b/libs/client/src/index.ts index daed06e..ef2ed21 100644 --- a/libs/client/src/index.ts +++ b/libs/client/src/index.ts @@ -12,3 +12,4 @@ export type { ValidationErrorInfo, WebHookResponse, } from './types'; +export { parseAppId } from './utils'; diff --git a/libs/client/src/realtime.ts b/libs/client/src/realtime.ts index 3bb65bd..cea3621 100644 --- a/libs/client/src/realtime.ts +++ b/libs/client/src/realtime.ts @@ -16,7 +16,7 @@ import uuid from 'uuid-random'; import { TOKEN_EXPIRATION_SECONDS, getTemporaryAuthToken } from './auth'; import { ApiError } from './response'; import { isBrowser } from './runtime'; -import { ensureAppIdFormat, isReact, parseAppId, throttle } from './utils'; +import { ensureAppIdFormat, isReact, throttle } from './utils'; // Define the context interface Context { @@ -78,10 +78,8 @@ function sendMessage(context: Context, event: SendEvent): Context { if (context.websocket && context.websocket.readyState === WebSocket.OPEN) { if (event.message instanceof Uint8Array) { context.websocket.send(event.message); - } else if (shouldSendBinary(event.message)) { - context.websocket.send(encode(event.message)); } else { - context.websocket.send(JSON.stringify(event.message)); + context.websocket.send(encode(event.message)); } return { @@ -248,17 +246,6 @@ type RealtimeUrlParams = { maxBuffering?: number; }; -// This is a list of apps deployed before formal realtime support. Their URLs follow -// a different pattern and will be kept here until we fully sunset them. -const LEGACY_APPS = [ - 'lcm-sd15-i2i', - 'lcm', - 'sdxl-turbo-realtime', - 'sd-turbo-real-time-high-fps-msgpack-a10g', - 'lcm-plexed-sd15-i2i', - 'sd-turbo-real-time-high-fps-msgpack', -]; - function buildRealtimeUrl( app: string, { token, maxBuffering }: RealtimeUrlParams @@ -273,23 +260,11 @@ function buildRealtimeUrl( queryParams.set('max_buffering', maxBuffering.toFixed(0)); } const appId = ensureAppIdFormat(app); - const { alias } = parseAppId(appId); - const suffix = - LEGACY_APPS.includes(alias) || !app.includes('/') ? 'ws' : 'realtime'; - return `wss://fal.run/${appId}/${suffix}?${queryParams.toString()}`; + return `wss://fal.run/${appId}/realtime?${queryParams.toString()}`; } const DEFAULT_THROTTLE_INTERVAL = 128; -function shouldSendBinary(message: any): boolean { - return Object.values(message).some( - (value) => - value instanceof Blob || - value instanceof ArrayBuffer || - value instanceof Uint8Array - ); -} - function isUnauthorizedError(message: any): boolean { // TODO we need better protocol definition with error codes return message['status'] === 'error' && message['error'] === 'Unauthorized';