feat(client): realtime client (#29)

* feat(client): realtime client

* chore: alpha release

* fix: remove os requirement

* fix: check if process is defined

* fix: ws connection key

* fix: outgoing request throttling logic

* chore: 0.6.0.alpha.4 release

* chore: update realtime demo

* chore: update preloaded scene

* feat: auth wip

* fix: compilation issue

* feat: basic auth impl missing error handling

* chore: remove console.log prepare 0.6.0

* fix: remove unsused code

apps/demo-nextjs-app-router/app/realtime/page.tsx

@@ -0,0 +1,63 @@
+'use client';
+
+/* eslint-disable @next/next/no-img-element */
+import * as fal from '@fal-ai/serverless-client';
+import { DrawingCanvas } from '../../components/drawing';
+import { useState } from 'react';
+
+fal.config({
+  proxyUrl: '/api/fal/proxy',
+});
+
+const PROMPT = 'a moon in a starry night sky';
+
+export default function RealtimePage() {
+  const [image, setImage] = useState<string | null>(null);
+
+  const { send } = fal.realtime.connect('110602490-shared-lcm-test', {
+    connectionKey: 'realtime-demo',
+    onResult(result) {
+      if (result.images && result.images[0]) {
+        setImage(result.images[0].url);
+      }
+    },
+  });
+
+  return (
+    <div className="min-h-screen bg-neutral-900 text-neutral-50">
+      <main className="container flex flex-col items-center justify-center w-full flex-1 py-10 space-y-8">
+        <h1 className="text-4xl font-mono mb-8 text-neutral-50">
+          fal<code className="font-light text-pink-600">realtime</code>
+        </h1>
+        <div className="prose text-neutral-400">
+          <blockquote className="italic text-xl">{PROMPT}</blockquote>
+        </div>
+        <div className="flex flex-col md:flex-row space-x-4">
+          <div className="flex-1">
+            <DrawingCanvas
+              onCanvasChange={({ imageData }) => {
+                send({
+                  prompt: PROMPT,
+                  image_url: imageData,
+                  sync_mode: true,
+                  seed: 6252023,
+                });
+              }}
+            />
+          </div>
+          <div className="flex-1">
+            <div className="w-[512px] h-[512px]">
+              {image && (
+                <img
+                  src={image}
+                  alt={`${PROMPT} generated by fal.ai`}
+                  className="object-contain w-full h-full"
+                />
+              )}
+            </div>
+          </div>
+        </div>
+      </main>
+    </div>
+  );
+}

apps/demo-nextjs-app-router/app/whisper/page.tsx

@@ -113,7 +113,7 @@ export default function WhisperDemo() {
       const result = await fal.subscribe('110602490-whisper', {
         input: {
           file_name: 'recording.wav',
-          url: audioFile,
+          audio_url: audioFile,
         },
         pollInterval: 1000,
         logs: true,
@@ -128,7 +128,6 @@ export default function WhisperDemo() {
         },
       });
       setResult(result);
-      console.log(result);
     } catch (error: any) {
       setError(error);
     } finally {

apps/demo-nextjs-app-router/components/drawing.tsx

@@ -0,0 +1,116 @@
+import { type Excalidraw } from '@excalidraw/excalidraw';
+import { ExcalidrawElement } from '@excalidraw/excalidraw/types/element/types';
+import {
+  AppState,
+  ExcalidrawImperativeAPI,
+} from '@excalidraw/excalidraw/types/types';
+import { useCallback, useEffect, useState } from 'react';
+import initialDrawing from './drawingState.json';
+
+export type CanvasChangeEvent = {
+  elements: readonly ExcalidrawElement[];
+  appState: AppState;
+  imageData: string;
+};
+
+export type DrawingCanvasProps = {
+  onCanvasChange: (event: CanvasChangeEvent) => void;
+};
+
+async function blobToBase64(blob: Blob): Promise<string> {
+  const reader = new FileReader();
+  reader.readAsDataURL(blob);
+  return new Promise<string>((resolve) => {
+    reader.onloadend = () => {
+      resolve(reader.result?.toString() || '');
+    };
+  });
+}
+
+export function DrawingCanvas({ onCanvasChange }: DrawingCanvasProps) {
+  const [ExcalidrawComponent, setExcalidrawComponent] = useState<
+    typeof Excalidraw | null
+  >(null);
+  const [excalidrawAPI, setExcalidrawAPI] =
+    useState<ExcalidrawImperativeAPI | null>(null);
+  const [sceneData, setSceneData] = useState<any>(null);
+
+  useEffect(() => {
+    import('@excalidraw/excalidraw').then((comp) =>
+      setExcalidrawComponent(comp.Excalidraw)
+    );
+    const onResize = () => {
+      if (excalidrawAPI) {
+        excalidrawAPI.refresh();
+      }
+    };
+    window.addEventListener('resize', onResize);
+    return () => {
+      window.removeEventListener('resize', onResize);
+    };
+  }, []);
+
+  const handleCanvasChanges = useCallback(
+    async (elements: readonly ExcalidrawElement[], appState: AppState) => {
+      if (!excalidrawAPI || !elements || !elements.length) {
+        return;
+      }
+      const { exportToBlob, convertToExcalidrawElements, serializeAsJSON } =
+        await import('@excalidraw/excalidraw');
+
+      const [boundingBoxElement] = convertToExcalidrawElements([
+        {
+          type: 'rectangle',
+          x: 0,
+          y: 0,
+          width: 512,
+          height: 512,
+          fillStyle: 'solid',
+          backgroundColor: 'cyan',
+        },
+      ]);
+
+      const newSceneData = serializeAsJSON(
+        elements,
+        appState,
+        excalidrawAPI.getFiles(),
+        'local'
+      );
+      if (newSceneData !== sceneData) {
+        setSceneData(newSceneData);
+        const blob = await exportToBlob({
+          elements: [boundingBoxElement, ...elements],
+          appState: {
+            ...appState,
+            frameRendering: {
+              ...(appState.frameRendering || {}),
+              clip: false,
+            },
+          },
+          files: excalidrawAPI.getFiles(),
+          mimeType: 'image/webp',
+          quality: 0.5,
+          exportPadding: 0,
+          getDimensions: () => {
+            return { width: 512, height: 512 };
+          },
+        });
+        const imageData = await blobToBase64(blob);
+        onCanvasChange({ elements, appState, imageData });
+      }
+    },
+    [excalidrawAPI, onCanvasChange, sceneData]
+  );
+
+  return (
+    <div style={{ height: '560px', width: '560px' }}>
+      {ExcalidrawComponent && (
+        <ExcalidrawComponent
+          excalidrawAPI={(api) => setExcalidrawAPI(api)}
+          initialData={{ elements: initialDrawing as ExcalidrawElement[] }}
+          onChange={handleCanvasChanges}
+        />
+      )}
+    </div>
+  );
+}

apps/demo-nextjs-app-router/components/drawingState.json

@@ -0,0 +1,58 @@
+[
+  {
+    "type": "rectangle",
+    "version": 240,
+    "versionNonce": 21728473,
+    "isDeleted": false,
+    "id": "EnLu91BTRnzWtj7m-l4Id",
+    "fillStyle": "solid",
+    "strokeWidth": 2,
+    "strokeStyle": "solid",
+    "roughness": 1,
+    "opacity": 100,
+    "angle": 0,
+    "x": -3.3853912353515625,
+    "y": -2.3741912841796875,
+    "strokeColor": "#1971c2",
+    "backgroundColor": "#343a40",
+    "width": 568.016487121582,
+    "height": 582.1398010253906,
+    "seed": 295965933,
+    "groupIds": [],
+    "frameId": null,
+    "roundness": null,
+    "boundElements": [],
+    "updated": 1700904828477,
+    "link": null,
+    "locked": false
+  },
+  {
+    "type": "ellipse",
+    "version": 3545,
+    "versionNonce": 647409943,
+    "isDeleted": false,
+    "id": "F6oN3k42RqfCqlzJLGXXS",
+    "fillStyle": "solid",
+    "strokeWidth": 1,
+    "strokeStyle": "solid",
+    "roughness": 1,
+    "opacity": 100,
+    "angle": 0,
+    "x": 345.65307998657227,
+    "y": 81.02682495117188,
+    "strokeColor": "#f08c00",
+    "backgroundColor": "#ffec99",
+    "width": 124.31249999999997,
+    "height": 113.591796875,
+    "seed": 23374002,
+    "groupIds": [],
+    "frameId": null,
+    "roundness": {
+      "type": 2
+    },
+    "boundElements": [],
+    "updated": 1700904844024,
+    "link": null,
+    "locked": false
+  }
+]

libs/client/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fal-ai/serverless-client",
   "description": "The fal serverless JS/TS client",
-  "version": "0.5.4",
+  "version": "0.6.0",
   "license": "MIT",
   "repository": {
     "type": "git",

libs/client/src/config.ts

@@ -1,4 +1,8 @@
-import type { RequestMiddleware } from './middleware';
+import {
+  withProxy,
+  type RequestMiddleware,
+  withMiddleware,
+} from './middleware';
 import type { ResponseHandler } from './response';
 import { defaultResponseHandler } from './response';
 
@@ -7,6 +11,7 @@ export type CredentialsResolver = () => string | undefined;
 export type Config = {
   credentials?: undefined | string | CredentialsResolver;
   host?: string;
+  proxyUrl?: string;
   requestMiddleware?: RequestMiddleware;
   responseHandler?: ResponseHandler<any>;
 };
@@ -21,7 +26,7 @@ export type RequiredConfig = Required<Config>;
  */
 function hasEnvVariables(): boolean {
   return (
-    process &&
+    typeof process !== 'undefined' &&
     process.env &&
     (typeof process.env.FAL_KEY !== 'undefined' ||
       (typeof process.env.FAL_KEY_ID !== 'undefined' &&
@@ -49,7 +54,7 @@ export const credentialsFromEnv: CredentialsResolver = () => {
  */
 function getDefaultHost(): string {
   const host = 'gateway.alpha.fal.ai';
-  if (process && process.env) {
+  if (typeof process !== 'undefined' && process.env) {
     return process.env.FAL_HOST || host;
   }
   return host;
@@ -71,6 +76,15 @@ let configuration: RequiredConfig;
  */
 export function config(config: Config) {
   configuration = { ...DEFAULT_CONFIG, ...config } as RequiredConfig;
+  if (config.proxyUrl) {
+    configuration = {
+      ...configuration,
+      requestMiddleware: withMiddleware(
+        configuration.requestMiddleware,
+        withProxy({ targetUrl: config.proxyUrl })
+      ),
+    };
+  }
 }
 
 /**
@@ -85,3 +99,11 @@ export function getConfig(): RequiredConfig {
   }
   return configuration;
 }
+
+/**
+ * @returns the URL of the fal serverless rest api endpoint.
+ */
+export function getRestApiUrl(): string {
+  const { host } = getConfig();
+  return host.replace('gateway', 'rest');
+}

libs/client/src/index.ts

@@ -1,10 +1,11 @@
 export { config, getConfig } from './config';
-export { storageImpl as storage } from './storage';
 export { queue, run, subscribe } from './function';
 export { withMiddleware, withProxy } from './middleware';
 export type { RequestMiddleware } from './middleware';
+export { realtimeImpl as realtime } from './realtime';
 export { ApiError, ValidationError } from './response';
 export type { ResponseHandler } from './response';
+export { storageImpl as storage } from './storage';
 export type {
   QueueStatus,
   ValidationErrorInfo,