diff --git a/apps/demo-nextjs-app-router/app/page.module.css b/apps/demo-nextjs-app-router/app/page.module.css deleted file mode 100644 index 8a13e21..0000000 --- a/apps/demo-nextjs-app-router/app/page.module.css +++ /dev/null @@ -1,2 +0,0 @@ -.page { -} diff --git a/apps/demo-nextjs-app-router/app/realtime/page.tsx b/apps/demo-nextjs-app-router/app/realtime/page.tsx new file mode 100644 index 0000000..152f094 --- /dev/null +++ b/apps/demo-nextjs-app-router/app/realtime/page.tsx @@ -0,0 +1,63 @@ +'use client'; + +/* eslint-disable @next/next/no-img-element */ +import * as fal from '@fal-ai/serverless-client'; +import { DrawingCanvas } from '../../components/drawing'; +import { useState } from 'react'; + +fal.config({ + proxyUrl: '/api/fal/proxy', +}); + +const PROMPT = 'a moon in a starry night sky'; + +export default function RealtimePage() { + const [image, setImage] = useState(null); + + const { send } = fal.realtime.connect('110602490-shared-lcm-test', { + connectionKey: 'realtime-demo', + onResult(result) { + if (result.images && result.images[0]) { + setImage(result.images[0].url); + } + }, + }); + + return ( +
+
+

+ falrealtime +

+
+
{PROMPT}
+
+
+
+ { + send({ + prompt: PROMPT, + image_url: imageData, + sync_mode: true, + seed: 6252023, + }); + }} + /> +
+
+
+ {image && ( + {`${PROMPT} + )} +
+
+
+
+
+ ); +} diff --git a/apps/demo-nextjs-app-router/app/whisper/page.tsx b/apps/demo-nextjs-app-router/app/whisper/page.tsx index 3d15c1b..b79c091 100644 --- a/apps/demo-nextjs-app-router/app/whisper/page.tsx +++ b/apps/demo-nextjs-app-router/app/whisper/page.tsx @@ -113,7 +113,7 @@ export default function WhisperDemo() { const result = await fal.subscribe('110602490-whisper', { input: { file_name: 'recording.wav', - url: audioFile, + audio_url: audioFile, }, pollInterval: 1000, logs: true, @@ -128,7 +128,6 @@ export default function WhisperDemo() { }, }); setResult(result); - console.log(result); } catch (error: any) { setError(error); } finally { diff --git a/apps/demo-nextjs-app-router/components/drawing.tsx b/apps/demo-nextjs-app-router/components/drawing.tsx new file mode 100644 index 0000000..cd34261 --- /dev/null +++ b/apps/demo-nextjs-app-router/components/drawing.tsx @@ -0,0 +1,116 @@ +import { type Excalidraw } from '@excalidraw/excalidraw'; +import { ExcalidrawElement } from '@excalidraw/excalidraw/types/element/types'; +import { + AppState, + ExcalidrawImperativeAPI, +} from '@excalidraw/excalidraw/types/types'; +import { useCallback, useEffect, useState } from 'react'; +import initialDrawing from './drawingState.json'; + +export type CanvasChangeEvent = { + elements: readonly ExcalidrawElement[]; + appState: AppState; + imageData: string; +}; + +export type DrawingCanvasProps = { + onCanvasChange: (event: CanvasChangeEvent) => void; +}; + +async function blobToBase64(blob: Blob): Promise { + const reader = new FileReader(); + reader.readAsDataURL(blob); + return new Promise((resolve) => { + reader.onloadend = () => { + resolve(reader.result?.toString() || ''); + }; + }); +} + +export function DrawingCanvas({ onCanvasChange }: DrawingCanvasProps) { + const [ExcalidrawComponent, setExcalidrawComponent] = useState< + typeof Excalidraw | null + >(null); + const [excalidrawAPI, setExcalidrawAPI] = + useState(null); + const [sceneData, setSceneData] = useState(null); + + useEffect(() => { + import('@excalidraw/excalidraw').then((comp) => + setExcalidrawComponent(comp.Excalidraw) + ); + const onResize = () => { + if (excalidrawAPI) { + excalidrawAPI.refresh(); + } + }; + window.addEventListener('resize', onResize); + return () => { + window.removeEventListener('resize', onResize); + }; + }, []); + + const handleCanvasChanges = useCallback( + async (elements: readonly ExcalidrawElement[], appState: AppState) => { + if (!excalidrawAPI || !elements || !elements.length) { + return; + } + const { exportToBlob, convertToExcalidrawElements, serializeAsJSON } = + await import('@excalidraw/excalidraw'); + + const [boundingBoxElement] = convertToExcalidrawElements([ + { + type: 'rectangle', + x: 0, + y: 0, + width: 512, + height: 512, + fillStyle: 'solid', + backgroundColor: 'cyan', + }, + ]); + + const newSceneData = serializeAsJSON( + elements, + appState, + excalidrawAPI.getFiles(), + 'local' + ); + if (newSceneData !== sceneData) { + setSceneData(newSceneData); + const blob = await exportToBlob({ + elements: [boundingBoxElement, ...elements], + appState: { + ...appState, + frameRendering: { + ...(appState.frameRendering || {}), + clip: false, + }, + }, + files: excalidrawAPI.getFiles(), + mimeType: 'image/webp', + quality: 0.5, + exportPadding: 0, + getDimensions: () => { + return { width: 512, height: 512 }; + }, + }); + const imageData = await blobToBase64(blob); + onCanvasChange({ elements, appState, imageData }); + } + }, + [excalidrawAPI, onCanvasChange, sceneData] + ); + + return ( +
+ {ExcalidrawComponent && ( + setExcalidrawAPI(api)} + initialData={{ elements: initialDrawing as ExcalidrawElement[] }} + onChange={handleCanvasChanges} + /> + )} +
+ ); +} diff --git a/apps/demo-nextjs-app-router/components/drawingState.json b/apps/demo-nextjs-app-router/components/drawingState.json new file mode 100644 index 0000000..4fb7b14 --- /dev/null +++ b/apps/demo-nextjs-app-router/components/drawingState.json @@ -0,0 +1,58 @@ +[ + { + "type": "rectangle", + "version": 240, + "versionNonce": 21728473, + "isDeleted": false, + "id": "EnLu91BTRnzWtj7m-l4Id", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -3.3853912353515625, + "y": -2.3741912841796875, + "strokeColor": "#1971c2", + "backgroundColor": "#343a40", + "width": 568.016487121582, + "height": 582.1398010253906, + "seed": 295965933, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1700904828477, + "link": null, + "locked": false + }, + { + "type": "ellipse", + "version": 3545, + "versionNonce": 647409943, + "isDeleted": false, + "id": "F6oN3k42RqfCqlzJLGXXS", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 345.65307998657227, + "y": 81.02682495117188, + "strokeColor": "#f08c00", + "backgroundColor": "#ffec99", + "width": 124.31249999999997, + "height": 113.591796875, + "seed": 23374002, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1700904844024, + "link": null, + "locked": false + } +] diff --git a/libs/client/package.json b/libs/client/package.json index 70b90ae..4c50a30 100644 --- a/libs/client/package.json +++ b/libs/client/package.json @@ -1,7 +1,7 @@ { "name": "@fal-ai/serverless-client", "description": "The fal serverless JS/TS client", - "version": "0.5.4", + "version": "0.6.0", "license": "MIT", "repository": { "type": "git", diff --git a/libs/client/src/config.ts b/libs/client/src/config.ts index 13b83ff..a9ba9b8 100644 --- a/libs/client/src/config.ts +++ b/libs/client/src/config.ts @@ -1,4 +1,8 @@ -import type { RequestMiddleware } from './middleware'; +import { + withProxy, + type RequestMiddleware, + withMiddleware, +} from './middleware'; import type { ResponseHandler } from './response'; import { defaultResponseHandler } from './response'; @@ -7,6 +11,7 @@ export type CredentialsResolver = () => string | undefined; export type Config = { credentials?: undefined | string | CredentialsResolver; host?: string; + proxyUrl?: string; requestMiddleware?: RequestMiddleware; responseHandler?: ResponseHandler; }; @@ -21,7 +26,7 @@ export type RequiredConfig = Required; */ function hasEnvVariables(): boolean { return ( - process && + typeof process !== 'undefined' && process.env && (typeof process.env.FAL_KEY !== 'undefined' || (typeof process.env.FAL_KEY_ID !== 'undefined' && @@ -49,7 +54,7 @@ export const credentialsFromEnv: CredentialsResolver = () => { */ function getDefaultHost(): string { const host = 'gateway.alpha.fal.ai'; - if (process && process.env) { + if (typeof process !== 'undefined' && process.env) { return process.env.FAL_HOST || host; } return host; @@ -71,6 +76,15 @@ let configuration: RequiredConfig; */ export function config(config: Config) { configuration = { ...DEFAULT_CONFIG, ...config } as RequiredConfig; + if (config.proxyUrl) { + configuration = { + ...configuration, + requestMiddleware: withMiddleware( + configuration.requestMiddleware, + withProxy({ targetUrl: config.proxyUrl }) + ), + }; + } } /** @@ -85,3 +99,11 @@ export function getConfig(): RequiredConfig { } return configuration; } + +/** + * @returns the URL of the fal serverless rest api endpoint. + */ +export function getRestApiUrl(): string { + const { host } = getConfig(); + return host.replace('gateway', 'rest'); +} diff --git a/libs/client/src/index.ts b/libs/client/src/index.ts index 75e3f54..e84e921 100644 --- a/libs/client/src/index.ts +++ b/libs/client/src/index.ts @@ -1,10 +1,11 @@ export { config, getConfig } from './config'; -export { storageImpl as storage } from './storage'; export { queue, run, subscribe } from './function'; export { withMiddleware, withProxy } from './middleware'; export type { RequestMiddleware } from './middleware'; +export { realtimeImpl as realtime } from './realtime'; export { ApiError, ValidationError } from './response'; export type { ResponseHandler } from './response'; +export { storageImpl as storage } from './storage'; export type { QueueStatus, ValidationErrorInfo, diff --git a/libs/client/src/realtime.ts b/libs/client/src/realtime.ts new file mode 100644 index 0000000..fd19d3d --- /dev/null +++ b/libs/client/src/realtime.ts @@ -0,0 +1,295 @@ +import { getConfig, getRestApiUrl } from './config'; +import { dispatchRequest } from './request'; +import { ApiError } from './response'; +import { isBrowser } from './runtime'; +import { isReact, throttle } from './utils'; + +/** + * A connection object that allows you to `send` request payloads to a + * realtime endpoint. + */ +export interface RealtimeConnection { + send(input: Input): void; + + close(): void; +} + +type ResultWithRequestId = { + request_id: string; +}; + +/** + * Options for connecting to the realtime endpoint. + */ +export interface RealtimeConnectionHandler { + /** + * The connection key. This is used to reuse the same connection + * across multiple calls to `connect`. This is particularly useful in + * contexts where the connection is established as part of a component + * lifecycle (e.g. React) and the component is re-rendered multiple times. + */ + connectionKey?: string; + + /** + * If `true`, the connection will only be established on the client side. + * This is useful for frameworks that reuse code for both server-side + * rendering and client-side rendering (e.g. Next.js). + * + * This is set to `true` by default when running on React in the server. + * Otherwise, it is set to `false`. + * + * Note that more SSR frameworks might be automatically detected + * in the future. In the meantime, you can set this to `true` when needed. + */ + clientOnly?: boolean; + + /** + * The throtle duration in milliseconds. This is used to throtle the + * calls to the `send` function. Realtime apps usually react to user + * input, which can be very frequesnt (e.g. fast typing or mouse/drag movements). + * + * The default value is `64` milliseconds. + */ + throttleInterval?: number; + + /** + * Callback function that is called when a result is received. + * @param result - The result of the request. + */ + onResult(result: Output & ResultWithRequestId): void; + + /** + * Callback function that is called when an error occurs. + * @param error - The error that occurred. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + onError?(error: ApiError): void; +} + +export interface RealtimeClient { + /** + * Connect to the realtime endpoint. The default implementation uses + * WebSockets to connect to fal function endpoints that support WSS. + * + * @param app the app alias or identifier. + * @param handler the connection handler. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + connect( + app: string, + handler: RealtimeConnectionHandler + ): RealtimeConnection; +} + +function buildRealtimeUrl(app: string): string { + const { host } = getConfig(); + return `wss://${app}.${host}/ws`; +} + +const TOKEN_EXPIRATION_SECONDS = 120; + +/** + * Get a token to connect to the realtime endpoint. + */ +async function getToken(app: string): Promise { + const [_, ...appAlias] = app.split('-'); + const token: string | object = await dispatchRequest( + 'POST', + `https://${getRestApiUrl()}/tokens/`, + { + allowed_apps: [appAlias.join('-')], + token_expiration: 120, + } + ); + // keep this in case the response was wrapped (old versions of the proxy do that) + // should be safe to remove in the future + if (typeof token !== 'string' && token['detail']) { + return token['detail']; + } + return token; +} + +/** + * See https://www.rfc-editor.org/rfc/rfc6455.html#section-7.4.1 + */ +const WebSocketErrorCodes = { + NORMAL_CLOSURE: 1000, + GOING_AWAY: 1001, +}; + +const connectionManager = (() => { + const connections = new Map(); + const tokens = new Map(); + + return { + token(app: string) { + return tokens.get(app); + }, + expireToken(app: string) { + tokens.delete(app); + }, + async refreshToken(app: string) { + const token = await getToken(app); + tokens.set(app, token); + // Very simple token expiration mechanism. + // We should make it more robust in the future. + setTimeout(() => { + tokens.delete(app); + }, TOKEN_EXPIRATION_SECONDS * 0.9 * 1000); + return token; + }, + has(connectionKey: string): boolean { + return connections.has(connectionKey); + }, + get(connectionKey: string): WebSocket | undefined { + return connections.get(connectionKey); + }, + set(connectionKey: string, ws: WebSocket) { + connections.set(connectionKey, ws); + }, + remove(connectionKey: string) { + connections.delete(connectionKey); + }, + }; +})(); + +async function getConnection(app: string, key: string): Promise { + const url = buildRealtimeUrl(app); + + if (connectionManager.has(key)) { + return connectionManager.get(key) as WebSocket; + } + let token = connectionManager.token(app); + if (!token) { + token = await connectionManager.refreshToken(app); + } + const ws = new WebSocket(`${url}?fal_jwt_token=${token}`); + connectionManager.set(key, ws); + return ws; +} + +const noop = () => { + /* No-op */ +}; + +/** + * A no-op connection that does not send any message. + * Useful on the frameworks that reuse code for both ssr and csr (e.g. Next) + * so the call when doing ssr has no side-effects. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const NoOpConnection: RealtimeConnection = { + send: noop, + close: noop, +}; + +/** + * The default implementation of the realtime client. + */ +export const realtimeImpl: RealtimeClient = { + connect( + app: string, + handler: RealtimeConnectionHandler + ): RealtimeConnection { + const { + // if running on React in the server, set clientOnly to true by default + clientOnly = isReact() && !isBrowser(), + connectionKey = crypto.randomUUID(), + throttleInterval = 64, + onError = noop, + onResult, + } = handler; + if (clientOnly && typeof window === 'undefined') { + return NoOpConnection; + } + + const enqueueMessages: Input[] = []; + + let reconnecting = false; + let ws: WebSocket | null = null; + const _send = (input: Input) => { + const requestId = crypto.randomUUID(); + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send( + JSON.stringify({ + request_id: requestId, + ...input, + }) + ); + } else { + enqueueMessages.push(input); + if (!reconnecting) { + reconnecting = true; + reconnect(); + } + } + }; + const send = + throttleInterval > 0 ? throttle(_send, throttleInterval) : _send; + + const reconnect = () => { + if (ws && ws.readyState === WebSocket.OPEN) { + return; + } + getConnection(app, connectionKey) + .then((connection) => { + ws = connection; + ws.onopen = () => { + reconnecting = false; + if (enqueueMessages.length > 0) { + enqueueMessages.forEach((input) => send(input)); + enqueueMessages.length = 0; + } + }; + ws.onclose = (event) => { + connectionManager.remove(connectionKey); + if (event.code !== WebSocketErrorCodes.NORMAL_CLOSURE) { + onError( + new ApiError({ + message: `Error closing the connection: ${event.reason}`, + status: event.code, + }) + ); + } + ws = null; + }; + ws.onerror = (event) => { + // TODO handle errors once server specify them + // if error 401, refresh token and retry + // if error 403, refresh token and retry + connectionManager.expireToken(app); + connectionManager.remove(connectionKey); + ws = null; + // if any of those are failed again, call onError + onError(new ApiError({ message: 'Unknown error', status: 500 })); + }; + ws.onmessage = (event) => { + const data = JSON.parse(event.data); + // Drop messages that are not related to the actual result. + // In the future, we might want to handle other types of messages. + // TODO: specify the fal ws protocol format + if (data.status !== 'error' && data.type !== 'x-fal-message') { + onResult(data); + } + }; + }) + .catch((error) => { + onError( + new ApiError({ message: 'Error opening connection', status: 500 }) + ); + }); + }; + + return { + send, + close() { + if (ws && ws.readyState === WebSocket.CLOSED) { + ws.close( + WebSocketErrorCodes.GOING_AWAY, + 'Client manually closed the connection.' + ); + } + }, + }; + }, +}; diff --git a/libs/client/src/response.ts b/libs/client/src/response.ts index f972d94..3f03a6d 100644 --- a/libs/client/src/response.ts +++ b/libs/client/src/response.ts @@ -56,7 +56,7 @@ export async function defaultResponseHandler( response: Response ): Promise { const { status, statusText } = response; - const contentType = response.headers.get('Content-Type') ?? ""; + const contentType = response.headers.get('Content-Type') ?? ''; if (!response.ok) { if (contentType.includes('application/json')) { const body = await response.json(); diff --git a/libs/client/src/runtime.ts b/libs/client/src/runtime.ts index 40e818a..4d06306 100644 --- a/libs/client/src/runtime.ts +++ b/libs/client/src/runtime.ts @@ -13,9 +13,6 @@ export function getUserAgent(): string { return memoizedUserAgent; } const packageInfo = require('../package.json'); - const os = require('os'); - memoizedUserAgent = `${packageInfo.name}/${ - packageInfo.version - } ${os.platform()}-${os.arch()} ${process.release.name}-${process.version}`; + memoizedUserAgent = `${packageInfo.name}/${packageInfo.version}`; return memoizedUserAgent; } diff --git a/libs/client/src/storage.ts b/libs/client/src/storage.ts index 544bdd4..d150114 100644 --- a/libs/client/src/storage.ts +++ b/libs/client/src/storage.ts @@ -1,4 +1,4 @@ -import { getConfig } from './config'; +import { getConfig, getRestApiUrl } from './config'; import { dispatchRequest } from './request'; /** @@ -50,11 +50,6 @@ type InitiateUploadData = { content_type: string | null; }; -function getRestApiUrl(): string { - const { host } = getConfig(); - return host.replace('gateway', 'rest'); -} - /** * Get the file extension from the content type. This is used to generate * a file name if the file name is not provided. diff --git a/libs/client/src/utils.ts b/libs/client/src/utils.ts index 2819323..a06c785 100644 --- a/libs/client/src/utils.ts +++ b/libs/client/src/utils.ts @@ -15,3 +15,52 @@ export function isValidUrl(url: string) { return false; } } + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function throttle any>( + func: T, + limit: number +): (...funcArgs: Parameters) => ReturnType | void { + let lastFunc: NodeJS.Timeout | null; + let lastRan: number; + + return (...args: Parameters): ReturnType | void => { + if (!lastRan) { + func(...args); + lastRan = Date.now(); + } else { + if (lastFunc) { + clearTimeout(lastFunc); + } + + lastFunc = setTimeout(() => { + if (Date.now() - lastRan >= limit) { + func(...args); + lastRan = Date.now(); + } + }, limit - (Date.now() - lastRan)); + } + }; +} + +let isRunningInReact: boolean | undefined; + +/** + * Not really the most optimal way to detect if we're running in React, + * but the idea here is that we can support multiple rendering engines + * (starting with React), with all their peculiarities, without having + * to add a dependency or creating custom integrations (e.g. custom hooks). + * + * Yes, a bit of magic to make things works out-of-the-box. + * @returns `true` if running in React, `false` otherwise. + */ +export function isReact() { + if (isRunningInReact === undefined) { + const stack = new Error().stack; + isRunningInReact = + !!stack && + (stack.includes('node_modules/react-dom/') || + stack.includes('node_modules/next/')); + } + return isRunningInReact; +} diff --git a/libs/client/tsconfig.lib.json b/libs/client/tsconfig.lib.json index 4324483..d241f70 100644 --- a/libs/client/tsconfig.lib.json +++ b/libs/client/tsconfig.lib.json @@ -3,6 +3,7 @@ "compilerOptions": { "module": "commonjs", "outDir": "../../dist/out-tsc", + "inlineSources": true, "declaration": true, "allowJs": true, "checkJs": false, diff --git a/package-lock.json b/package-lock.json index f958942..337cd7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "devDependencies": { "@commitlint/cli": "^17.0.0", "@commitlint/config-conventional": "^17.0.0", + "@excalidraw/excalidraw": "^0.17.0", "@nrwl/express": "16.10.0", "@nx/cypress": "16.10.0", "@nx/eslint-plugin": "16.10.0", @@ -3073,6 +3074,16 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@excalidraw/excalidraw": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@excalidraw/excalidraw/-/excalidraw-0.17.0.tgz", + "integrity": "sha512-NzP22v5xMqxYW27ZtTHhiGFe7kE8NeBk45aoeM/mDSkXiOXPDH+PcvwzHRN/Ei+Vj/0sTPHxejn8bZyRWKGjXg==", + "dev": true, + "peerDependencies": { + "react": "^17.0.2 || ^18.2.0", + "react-dom": "^17.0.2 || ^18.2.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.13", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", diff --git a/package.json b/package.json index 1db2814..286ddbd 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "devDependencies": { "@commitlint/cli": "^17.0.0", "@commitlint/config-conventional": "^17.0.0", + "@excalidraw/excalidraw": "^0.17.0", "@nrwl/express": "16.10.0", "@nx/cypress": "16.10.0", "@nx/eslint-plugin": "16.10.0",