feat: expose Overlay3DCollection (#4598)

* feat: expose Overlay3DCollection

* chore: reformat, small rewrites

* chore: add comments

* chore: add more docstrings

* chore: a new docstring

* chore: update api

* chore: remember to remove camera listener

* chore: factor out TextOverlay, some simplification

* fix: add missing file

* chore: refactor to make point sorting self-contained

viewer/api-entry-points/core.ts

@@ -116,4 +116,11 @@ export {
   Image360AnnotationFilterOptions
 } from '../packages/360-images';
 
-export { OverlayCollection, OverlayInfo, Overlay3D, DefaultOverlay3DContentType } from '../packages/3d-overlays';
+export {
+  OverlayCollection,
+  Overlay3DCollection,
+  Overlay3DCollectionOptions,
+  OverlayInfo,
+  Overlay3D,
+  DefaultOverlay3DContentType
+} from '../packages/3d-overlays';

viewer/packages/3d-overlays/index.ts

@@ -1,7 +1,7 @@
 /*!
  * Copyright 2023 Cognite AS
  */
-export { Overlay3DCollection } from './src/Overlay3DCollection';
+export { Overlay3DCollection, Overlay3DCollectionOptions } from './src/Overlay3DCollection';
 export { OverlayPointsObject, OverlayPointsParameters } from './src/OverlayPointsObject';
 export { Overlay3DIcon } from './src/Overlay3DIcon';
 export { Overlay3D } from './src/Overlay3D';

viewer/packages/3d-overlays/src/CameraChangeThrottler.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright 2024 Cognite AS
+ */
+import throttle from 'lodash/throttle';
+import { Camera, Matrix4 } from 'three';
+
+export type Callback = () => void;
+
+export class CameraChangeThrottler {
+  private readonly _prevCameraMatrix: Matrix4 = new Matrix4();
+
+  public call: (camera: Camera, callback: Callback) => void;
+
+  constructor(wait: number = 500) {
+    this.call = throttle((camera: Camera, callback: Callback) => this._call(camera, callback), wait);
+  }
+
+  private _call(camera: Camera, callback: Callback) {
+    if (camera.matrix.equals(this._prevCameraMatrix)) {
+      return;
+    }
+
+    callback();
+
+    this._prevCameraMatrix.copy(camera.matrix);
+  }
+}

viewer/packages/3d-overlays/src/IconOctree.ts

@@ -9,20 +9,21 @@ import pullAll from 'lodash/pullAll';
 import { Node, PointOctant, PointOctree } from 'sparse-octree';
 import { Box3, Matrix4, Vector3 } from 'three';
 import { Overlay3DIcon } from './Overlay3DIcon';
+import { DefaultOverlay3DContentType } from './OverlayCollection';
 
 type NodeMetadata = {
   icon: Overlay3DIcon;
   level: number;
 };
 
-export class IconOctree extends PointOctree<Overlay3DIcon> {
+export class IconOctree<ContentType = DefaultOverlay3DContentType> extends PointOctree<Overlay3DIcon<ContentType>> {
   private readonly _nodeCenters: Map<Node, NodeMetadata>;
 
-  public static getMinimalOctreeBoundsFromIcons(icons: Overlay3DIcon[]): Box3 {
+  public static getMinimalOctreeBoundsFromIcons<ContentType>(icons: Overlay3DIcon<ContentType>[]): Box3 {
     return new Box3().setFromPoints(icons.map(icon => icon.getPosition()));
   }
 
-  constructor(icons: Overlay3DIcon[], bounds: Box3, maxLeafSize: number) {
+  constructor(icons: Overlay3DIcon<ContentType>[], bounds: Box3, maxLeafSize: number) {
     super(bounds.min, bounds.max, 0, maxLeafSize);
     icons.forEach(icon => this.set(icon.getPosition(), icon));
     this.filterEmptyLeaves();

viewer/packages/3d-overlays/src/Overlay3DCollection.ts

@@ -2,20 +2,40 @@
  * Copyright 2023 Cognite AS
  */
 
-import { CanvasTexture, Color, Texture, Object3D, type Camera } from 'three';
+import { CanvasTexture, Color, Texture, Object3D, Camera, Vector2, Raycaster, WebGLRenderer, Scene } from 'three';
 import { Overlay3DIcon } from './Overlay3DIcon';
 import { Overlay3D } from './Overlay3D';
 import { OverlayPointsObject } from './OverlayPointsObject';
 import { IconOctree } from './IconOctree';
 import { DefaultOverlay3DContentType, OverlayCollection, OverlayInfo } from './OverlayCollection';
+import minBy from 'lodash/minBy';
+import { CameraChangeThrottler } from './CameraChangeThrottler';
 
+/**
+ * Constructor options for the Overlay3DCollection
+ */
 export type Overlay3DCollectionOptions = {
+  /**
+   * The texture to display as icons in this collection
+   */
   overlayTexture?: Texture;
+  /**
+   * A texture mask for marking what pixels are transparent in the supplied overlayTexture
+   */
   overlayTextureMask?: Texture;
+  /**
+   * The maximum display size of each icon in pixels
+   */
   maxPointSize?: number;
+  /**
+   * The default color to apply to overlay icons without a color on their own
+   */
   defaultOverlayColor?: Color;
 };
 
+/**
+ * A collection of overlay icons with associated data
+ */
 export class Overlay3DCollection<MetadataType = DefaultOverlay3DContentType>
   extends Object3D
   implements OverlayCollection<MetadataType>
@@ -33,9 +53,11 @@ export class Overlay3DCollection<MetadataType = DefaultOverlay3DContentType>
   private readonly _iconRadius = 0.4;
   private _overlays: Overlay3DIcon<MetadataType>[];
   //@ts-ignore Will be removed when clustering is added.
-  private _octree: IconOctree;
+  private _octree: IconOctree<MetadataType>;
+  private readonly _rayCaster = new Raycaster();
+  private readonly _cameraChangeDebouncer = new CameraChangeThrottler();
 
-  constructor(overlayInfos?: OverlayInfo<MetadataType>[], options?: Overlay3DCollectionOptions) {
+  constructor(overlayInfos: OverlayInfo<MetadataType>[], options?: Overlay3DCollectionOptions) {
     super();
 
     this.defaultOverlayColor = options?.defaultOverlayColor ?? this.defaultOverlayColor;
@@ -46,30 +68,42 @@ export class Overlay3DCollection<MetadataType = DefaultOverlay3DContentType>
       mask: options?.overlayTextureMask ?? (options?.overlayTexture ? undefined : defaultOverlayTextures.mask)
     };
 
-    this._overlayPoints = new OverlayPointsObject(overlayInfos ? overlayInfos.length : this.DefaultMaxPoints, {
-      spriteTexture: this._sharedTextures.color,
-      maskTexture: this._sharedTextures.mask,
-      minPixelSize: this.MinPixelSize,
-      maxPixelSize: options?.maxPointSize ?? this.MaxPixelSize,
-      radius: this._iconRadius
-    });
-
-    this._overlays = this.initializeOverlay3DIcons(overlayInfos ?? []);
+    this._overlayPoints = new OverlayPointsObject(
+      overlayInfos ? overlayInfos.length : this.DefaultMaxPoints,
+      {
+        spriteTexture: this._sharedTextures.color,
+        maskTexture: this._sharedTextures.mask,
+        minPixelSize: this.MinPixelSize,
+        maxPixelSize: options?.maxPointSize ?? this.MaxPixelSize,
+        radius: this._iconRadius
+      },
+      (...args) => this.onBeforeRenderDelegate(...args)
+    );
+
+    this._overlays = this.initializeOverlay3DIcons(overlayInfos);
     this.add(this._overlayPoints);
-
     this.updatePointsObject();
 
     this._octree = this.rebuildOctree();
   }
 
+  /**
+   * Set whether this collection is visible or not
+   */
   setVisibility(visibility: boolean): void {
     this._overlayPoints.visible = visibility;
   }
 
+  /**
+   * Get the overlay icons contained in this collection
+   */
   getOverlays(): Overlay3D<MetadataType>[] {
     return this._overlays;
   }
 
+  /**
+   * Add more overlays into this collection
+   */
   public addOverlays(overlayInfos: OverlayInfo<MetadataType>[]): Overlay3D<MetadataType>[] {
     if (overlayInfos.length + this._overlays.length > this.DefaultMaxPoints)
       throw new Error('Cannot add more than ' + this.DefaultMaxPoints + ' points');
@@ -83,33 +117,59 @@ export class Overlay3DCollection<MetadataType = DefaultOverlay3DContentType>
     return newIcons;
   }
 
-  sortOverlaysRelativeToCamera(camera: Camera): void {
+  private readonly onBeforeRenderDelegate: Object3D['onBeforeRender'] = (
+    _renderer: WebGLRenderer,
+    _scene: Scene,
+    camera: Camera
+  ) => {
+    this._cameraChangeDebouncer.call(camera, () => this.sortOverlaysRelativeToCamera(camera));
+  };
+
+  private sortOverlaysRelativeToCamera(camera: Camera): void {
     this._overlays = this._overlays.sort((a, b) => {
       return b.getPosition().distanceToSquared(camera.position) - a.getPosition().distanceToSquared(camera.position);
     });
-
     this.updatePointsObject();
   }
 
+  /**
+   * Remove the listed overlays from this collection
+   */
   public removeOverlays(overlays: Overlay3D<MetadataType>[]): void {
     this._overlays = this._overlays.filter(overlay => !overlays.includes(overlay));
 
     this.updatePointsObject();
     this._octree = this.rebuildOctree();
   }
 
+  /**
+   * Clean up all icons in this collection
+   */
   public removeAllOverlays(): void {
     this._overlays = [];
 
     this.updatePointsObject();
     this._octree = this.rebuildOctree();
   }
 
-  private rebuildOctree(): IconOctree {
-    const icons = this._overlays as Overlay3DIcon[];
+  /**
+   * Run intersection on icons in this collection. Returns the closest hit
+   */
+  public intersectOverlays(normalizedCoordinates: Vector2, camera: Camera): Overlay3D<MetadataType> | undefined {
+    this._rayCaster.setFromCamera(normalizedCoordinates.clone(), camera);
+
+    const intersections = this._overlays.filter(icon => {
+      return icon.getVisible() && icon.intersect(this._rayCaster.ray) !== null;
+    });
+
+    return minBy(intersections, a => a.getPosition().clone().sub(this._rayCaster.ray.origin).length());
+  }
+
+  private rebuildOctree(): IconOctree<MetadataType> {
+    const icons = this._overlays;
     const octreeBounds = IconOctree.getMinimalOctreeBoundsFromIcons(icons);
 
-    return new IconOctree(icons, octreeBounds, 2);
+    return new IconOctree<MetadataType>(icons, octreeBounds, 2);
   }
 
   private updatePointsObject(): void {
@@ -142,6 +202,9 @@ export class Overlay3DCollection<MetadataType = DefaultOverlay3DContentType>
     });
   }
 
+  /**
+   * Dispose this collection and icons with all associated resources
+   */
   public dispose(): void {
     this._overlays.forEach(overlay => overlay.dispose());
 

viewer/packages/3d-overlays/src/OverlayPointsObject.ts

@@ -13,6 +13,7 @@ import {
   Group,
   LessEqualDepth,
   Matrix4,
+  Object3D,
   Points,
   RawShaderMaterial,
   ShaderMaterial,
@@ -43,9 +44,14 @@ export class OverlayPointsObject extends Group {
   private readonly _colorBuffer: Float32Array;
   private readonly _colorAttribute: BufferAttribute;
   private readonly _points: { frontPoints: Points; backPoints: Points };
+  private readonly _onBeforeRender?: Object3D['onBeforeRender'];
   private _modelTransform: Matrix4;
 
-  constructor(maxNumberOfPoints: number, materialParameters: OverlayPointsParameters) {
+  constructor(
+    maxNumberOfPoints: number,
+    materialParameters: OverlayPointsParameters,
+    onBeforeRender?: Object3D['onBeforeRender']
+  ) {
     super();
     const geometry = new BufferGeometry();
     this._positionBuffer = new Float32Array(maxNumberOfPoints * 3);
@@ -101,6 +107,7 @@ export class OverlayPointsObject extends Group {
     this._geometry = geometry;
     this._frontMaterial = frontMaterial;
     this._points = { frontPoints, backPoints };
+    this._onBeforeRender = onBeforeRender;
   }
 
   public setPoints(points: Vector3[], colors?: Color[]): void {
@@ -163,7 +170,8 @@ export class OverlayPointsObject extends Group {
 
   private initializePoints(geometry: BufferGeometry, frontMaterial: ShaderMaterial): Points {
     const frontPoints = createPoints(geometry, frontMaterial);
-    frontPoints.onBeforeRender = renderer => {
+    frontPoints.onBeforeRender = (renderer, ...rest) => {
+      this._onBeforeRender?.(renderer, ...rest);
       setUniforms(renderer, frontMaterial);
     };