feat: realtime 3d demo

apps/demo-nextjs-app-router/app/realtime-3d/page.tsx

@@ -0,0 +1,165 @@
+'use client';
+
+/* eslint-disable @next/next/no-img-element */
+import * as fal from '@fal-ai/serverless-client';
+import { OrbitControls } from '@react-three/drei';
+import { Canvas, useLoader, useThree } from '@react-three/fiber';
+import { useEffect, useRef, useState } from 'react';
+import {
+  Box3,
+  EdgesGeometry,
+  LineBasicMaterial,
+  LineSegments,
+  MeshBasicMaterial,
+  Vector3,
+} from 'three';
+import { OBJLoader } from 'three/examples/jsm/loaders/OBJLoader';
+
+fal.config({
+  proxyUrl: '/api/fal/proxy',
+});
+
+const PROMPT = 'digital art of a spider robot, low poly';
+
+type CanvasSnapshotProps = {
+  onSnapshot: (image: string) => void;
+};
+
+function CanvasSnapshot({ onSnapshot }: CanvasSnapshotProps) {
+  const { gl } = useThree();
+  const lastSnapshotTime = useRef(Date.now());
+  const takeSnapshot = () => {
+    // call it once every 64 milliseconds at most
+    // note: even though the realtime client itself is throttled
+    // this is still necessary to prevent the canvas from being
+    // too slow by exporting way too many images
+    if (Date.now() - lastSnapshotTime.current < 64) {
+      lastSnapshotTime.current = Date.now();
+      return;
+    }
+    const image = gl.domElement.toDataURL('image/webp', 0.7);
+    onSnapshot(image);
+  };
+
+  return <OrbitControls onChange={takeSnapshot} />;
+}
+
+type ModelBoxProps = {
+  modelPath: string;
+};
+
+function ModelBox({ modelPath }: ModelBoxProps) {
+  const model = useLoader(OBJLoader, modelPath);
+  const { camera, gl } = useThree();
+
+  useEffect(() => {
+    // Calculate bounding box
+    const boundingBox = new Box3().setFromObject(model);
+    const size = new Vector3();
+    boundingBox.getSize(size);
+
+    // Adjust model scale
+    const desiredMaxSize = 5;
+    const scaleFactor = desiredMaxSize / size.length();
+    model.scale.setScalar(scaleFactor);
+
+    // Position camera
+    const cameraDistance = size.length() * 2;
+    camera.position.set(0, 0, cameraDistance);
+    camera.lookAt(model.position);
+
+    // TODO proper typeing once cast to Mesh object is fixed
+    model.traverse((child: any) => {
+      if (child.isMesh) {
+        child.material = new MeshBasicMaterial({ color: '#bcbcbc' });
+
+        // Create edges for the mesh
+        const edgeGeometry = new EdgesGeometry(child.geometry);
+        const edgeMaterial = new LineBasicMaterial({
+          color: 'red',
+          linewidth: 4,
+        });
+        const edge = new LineSegments(edgeGeometry, edgeMaterial);
+
+        // Add the edge to the mesh
+        child.add(edge);
+      }
+    });
+
+    gl.setClearColor('#3d3d3d');
+
+    // Update OrbitControls target
+    // Assuming you have orbitControlsRef set up
+    // orbitControlsRef.current.target.copy(boundingBox.getCenter(new Vector3()));
+  }, [model, camera, gl]);
+
+  return <primitive object={model} />;
+}
+
+export default function Lcm3D() {
+  const [image, setImage] = useState<string | null>(null);
+
+  const { send } = fal.realtime.connect('fal-ai/lcm-sd15-i2i', {
+    connectionKey: '3d-demo',
+    throttleInterval: 128,
+    onResult(result) {
+      if (result.images && result.images[0]) {
+        setImage(result.images[0].url);
+      }
+    },
+  });
+
+  return (
+    <div className="min-h-screen bg-neutral-900 text-neutral-50">
+      <main className="container flex flex-col items-center justify-center w-full flex-1 py-10 space-y-8">
+        <h1 className="text-4xl font-mono mb-8 text-neutral-50">
+          fal<code className="font-light text-pink-600">3d</code>
+        </h1>
+        <div className="prose text-neutral-400">
+          <blockquote className="italic text-xl">{PROMPT}</blockquote>
+        </div>
+        <div className="flex flex-col md:flex-row space-x-4">
+          <div className="flex-1 min-w-[512px]">
+            <Canvas gl={{ preserveDrawingBuffer: true }}>
+              <CanvasSnapshot
+                onSnapshot={(image) => {
+                  send({
+                    prompt: PROMPT,
+                    image_url: image,
+                    sync_mode: true,
+                    seed: 6252023,
+                  });
+                }}
+              />
+              <ambientLight intensity={Math.PI / 2} />
+              <spotLight
+                position={[10, 10, 10]}
+                angle={0.15}
+                penumbra={1}
+                decay={0}
+                intensity={Math.PI}
+              />
+              <pointLight
+                position={[-10, -10, -10]}
+                decay={0}
+                intensity={Math.PI}
+              />
+              <ModelBox modelPath="/3d/spiderbot.obj" />
+            </Canvas>
+          </div>
+          <div className="flex-1">
+            <div className="w-[512px] h-[512px]">
+              {image && (
+                <img
+                  src={image}
+                  alt={`${PROMPT} generated by fal.ai`}
+                  className="object-contain w-full h-full"
+                />
+              )}
+            </div>
+          </div>
+        </div>
+      </main>
+    </div>
+  );
+}