Remote spatial index widgets #2 (#910)

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Not released
 
+- Remote calculation for dynamic spatial index sources [#908](https://github.com/CartoDB/carto-react/pull/908)
+
 ## 3.0.0
 
 ### 3.0.0-alpha.21 (2024-09-26)

packages/react-api/src/types.d.ts

@@ -48,6 +48,7 @@ export type SourceProps = {
   spatialDataColumn?: string;
   spatialFiltersResolution?: number;
   aggregationExp?: string;
+  aggregationResLevel?: number;
   credentials?: Credentials;
   queryParameters?: QueryParameters;
   provider?: Provider;

packages/react-redux/src/slices/cartoSlice.js

@@ -207,6 +207,7 @@ export const createCartoSlice = (initialState) => {
  * @param {number=} data.dataResolution - data resolution for spatial index data.
  * @param {number=} data.spatialFiltersResolution - spatial filters resolution for spatial index data.
  * @param {string=} data.aggregationExp - (optional) for spatial index data.
+ * @param {number=} data.aggregationResLevel - (optional) for spatial index data.
  * @param {string=} data.provider - (optional) type of the data warehouse.
  */
 export const addSource = ({
@@ -224,6 +225,7 @@ export const addSource = ({
   dataResolution,
   spatialFiltersResolution,
   aggregationExp,
+  aggregationResLevel,
   provider
 }) => ({
   type: 'carto/addSource',
@@ -238,6 +240,7 @@ export const addSource = ({
     queryParameters,
     geoColumn,
     dataResolution,
+    aggregationResLevel,
     spatialDataType,
     spatialDataColumn,
     spatialFiltersResolution,

packages/react-widgets/__tests__/models/utils.test.js

@@ -2,7 +2,8 @@ import {
   sourceAndFiltersToSQL,
   wrapModelCall,
   formatOperationColumn,
-  normalizeObjectKeys
+  normalizeObjectKeys,
+  isRemoteCalculationSupported
 } from '../../src/models/utils';
 import { AggregationTypes, Provider, _filtersToSQL } from '@carto/react-core';
 import { MAP_TYPES, API_VERSIONS } from '@carto/react-api';
@@ -41,6 +42,28 @@ const fromLocal = jest.fn();
 const fromRemote = jest.fn();
 
 describe('utils', () => {
+  describe('isRemoteCalculationSupported', () => {
+    test.each([
+      ['v2', V2_SOURCE, false],
+      ['v3', V3_SOURCE, true],
+      ['v3', { ...V3_SOURCE, type: 'tileset' }, false],
+      ['v3/databricks', { ...V3_SOURCE, provider: 'databricks' }, false],
+      ['v3/databricksRest', { ...V3_SOURCE, provider: 'databricksRest' }, true],
+      ['v3/h3/no dataResolution', { ...V3_SOURCE, geoColumn: 'h3' }, false],
+      [
+        'v3/h3/with dataResolution',
+        { ...V3_SOURCE, geoColumn: 'h3', dataResolution: 5 },
+        true
+      ],
+      [
+        'v3/quadbin/with dataResolution',
+        { ...V3_SOURCE, geoColumn: 'quadbin:abc', spatialFiltersResolution: 5 },
+        true
+      ]
+    ])('works correctly for %s', (_, source, expected) => {
+      expect(isRemoteCalculationSupported({ source })).toEqual(expected);
+    });
+  });
   describe('wrapModelCall', () => {
     const cases = [
       // source, global, remoteCalculation, expectedFn

packages/react-widgets/src/hooks/useWidgetFetch.js

@@ -16,6 +16,7 @@ import { DEFAULT_INVALID_COLUMN_ERR } from '../widgets/utils/constants';
 import useCustomCompareEffect from './useCustomCompareEffect';
 import useWidgetSource from './useWidgetSource';
 import { isRemoteCalculationSupported } from '../models/utils';
+import { getSpatialFiltersResolution } from '../models/spatialFiltersResolution';
 
 export const WidgetStateType = {
   Loading: 'loading',
@@ -57,19 +58,6 @@ export function selectGeometryToIntersect(global, viewport, spatialFilter) {
   }
 }
 
-// stolen from deck.gl/modules/carto/src/layers/h3-tileset-2d.ts
-const BIAS = 2;
-export function getHexagonResolution(viewport, tileSize) {
-  // Difference in given tile size compared to deck's internal 512px tile size,
-  // expressed as an offset to the viewport zoom.
-  const zoomOffset = Math.log2(tileSize / 512);
-  const hexagonScaleFactor = (2 / 3) * (viewport.zoom - zoomOffset);
-  const latitudeScaleFactor = Math.log(1 / Math.cos((Math.PI * viewport.latitude) / 180));
-
-  // Clip and bias
-  return Math.max(0, Math.floor(hexagonScaleFactor + latitudeScaleFactor - BIAS));
-}
-
 export default function useWidgetFetch(
   modelFn,
   {
@@ -118,21 +106,15 @@ export default function useWidgetFetch(
       return source;
     }
 
-    if (source.spatialDataType === 'h3') {
-      const hexagonResolution = getHexagonResolution(
-        { zoom: viewState.zoom, latitude: viewState.latitude },
-        source.dataResolution
-      );
-      return {
-        ...source,
-        spatialFiltersResolution: Math.min(source.dataResolution, hexagonResolution)
-      };
-    }
-    if (source.spatialDataType === 'quadbin') {
-      const quadsResolution = Math.floor(viewState.zoom);
+    if (source.spatialDataType === 'h3' || source.spatialDataType === 'quadbin') {
+      const spatialFiltersResolution = getSpatialFiltersResolution({
+        source,
+        viewState,
+        spatialDataType: source.spatialDataType
+      });
       return {
         ...source,
-        spatialFiltersResolution: Math.min(source.dataResolution, quadsResolution)
+        spatialFiltersResolution
       };
     }
     return source;

packages/react-widgets/src/index.d.ts

@@ -25,7 +25,14 @@ export { default as BarWidget } from './widgets/BarWidget';
 export { default as useSourceFilters } from './hooks/useSourceFilters';
 export { default as FeatureSelectionWidget } from './widgets/FeatureSelectionWidget';
 export { default as FeatureSelectionLayer } from './layers/FeatureSelectionLayer';
-export { default as useGeocoderWidgetController, setGeocoderResult } from './hooks/useGeocoderWidgetController';
+export {
+  default as useGeocoderWidgetController,
+  setGeocoderResult
+} from './hooks/useGeocoderWidgetController';
 export { WidgetState, WidgetStateType } from './types';
-export { isRemoteCalculationSupported as _isRemoteCalculationSupported, sourceAndFiltersToSQL as _sourceAndFiltersToSQL, getSqlEscapedSource as _getSqlEscapedSource } from './models/utils';
-
+export {
+  isRemoteCalculationSupported as _isRemoteCalculationSupported,
+  sourceAndFiltersToSQL as _sourceAndFiltersToSQL,
+  getSqlEscapedSource as _getSqlEscapedSource,
+  getSpatialFiltersResolution as _getSpatialFiltersResolution
+} from './models/utils';

packages/react-widgets/src/index.js

@@ -31,3 +31,4 @@ export {
   sourceAndFiltersToSQL as _sourceAndFiltersToSQL,
   getSqlEscapedSource as _getSqlEscapedSource
 } from './models/utils';
+export { getSpatialFiltersResolution as _getSpatialFiltersResolution } from './models/spatialFiltersResolution';

packages/react-widgets/src/models/spatialFiltersResolution.js

@@ -0,0 +1,87 @@
+// stolen from deck.gl/modules/carto/src/layers/h3-tileset-2d.ts
+const BIAS = 2;
+function getHexagonResolution(viewport, tileSize) {
+  // Difference in given tile size compared to deck's internal 512px tile size,
+  // expressed as an offset to the viewport zoom.
+  const zoomOffset = Math.log2(tileSize / 512);
+  const hexagonScaleFactor = (2 / 3) * (viewport.zoom - zoomOffset);
+  const latitudeScaleFactor = Math.log(1 / Math.cos((Math.PI * viewport.latitude) / 180));
+
+  // Clip and bias
+  return Math.max(0, Math.floor(hexagonScaleFactor + latitudeScaleFactor - BIAS));
+}
+
+const maxH3SpatialFiltersResolutions = [
+  [20, 14],
+  [19, 13],
+  [18, 12],
+  [17, 11],
+  [16, 10],
+  [15, 9],
+  [14, 8],
+  [13, 7],
+  [12, 7],
+  [11, 7],
+  [10, 6],
+  [9, 6],
+  [8, 5],
+  [7, 4],
+  [6, 4],
+  [5, 3],
+  [4, 2],
+  [3, 1],
+  [2, 1],
+  [1, 0]
+];
+
+const quadBinZoomMaxOffset = 4;
+
+const DEFAULT_TILE_SIZE = 512;
+const DEFAULT_AGGREGATION_RES_LEVEL_H3 = 4;
+const DEFAULT_AGGREGATION_RES_LEVEL_QUADBIN = 6;
+
+export function getSpatialFiltersResolution({ source, spatialDataType, viewState }) {
+  if (spatialDataType === 'geo') return undefined;
+
+  const currentZoom = viewState.zoom ?? 1;
+
+  const dataResolution = source.dataResolution ?? Number.MAX_VALUE;
+
+  const aggregationResLevel =
+    source.aggregationResLevel ??
+    (spatialDataType === 'h3'
+      ? DEFAULT_AGGREGATION_RES_LEVEL_H3
+      : DEFAULT_AGGREGATION_RES_LEVEL_QUADBIN);
+
+  const aggregationResLevelOffset = Math.max(0, Math.floor(aggregationResLevel));
+
+  const currentZoomInt = Math.ceil(currentZoom);
+  if (spatialDataType === 'h3') {
+    const tileSize = DEFAULT_TILE_SIZE;
+    const maxResolutionForZoom =
+      maxH3SpatialFiltersResolutions.find(([zoom]) => zoom === currentZoomInt)?.[1] ??
+      Math.max(0, currentZoomInt - 3);
+
+    const maxSpatialFiltersResolution = maxResolutionForZoom
+      ? Math.min(dataResolution, maxResolutionForZoom)
+      : dataResolution;
+
+    const hexagonResolution =
+      getHexagonResolution(
+        { zoom: currentZoom, latitude: viewState.latitude },
+        tileSize
+      ) + aggregationResLevelOffset;
+
+    return Math.min(hexagonResolution, maxSpatialFiltersResolution);
+  }
+
+  if (spatialDataType === 'quadbin') {
+    const maxResolutionForZoom = currentZoomInt + quadBinZoomMaxOffset;
+    const maxSpatialFiltersResolution = Math.min(dataResolution, maxResolutionForZoom);
+
+    const quadsResolution = Math.floor(viewState.zoom) + aggregationResLevelOffset;
+    return Math.min(quadsResolution, maxSpatialFiltersResolution);
+  }
+
+  return undefined;
+}

packages/react-widgets/src/models/utils.d.ts

@@ -1,9 +1,21 @@
-import { SourceProps, MAP_TYPES } from "@carto/react-api";
-import { FiltersLogicalOperators, Provider, _FilterTypes, Provider } from "@carto/react-core";
-import { SourceFilters } from "@carto/react-redux";
+import { SourceProps, MAP_TYPES } from '@carto/react-api';
+import { FiltersLogicalOperators, Provider, _FilterTypes } from '@carto/react-core';
+import { SourceFilters, ViewState } from '@carto/react-redux';
 
-export function isRemoteCalculationSupported(prop: { source: SourceProps }): boolean
+export function isRemoteCalculationSupported(prop: { source: SourceProps }): boolean;
 
-export function sourceAndFiltersToSQL(props: { data: string, filters?: SourceFilters, filtersLogicalOperator?: FiltersLogicalOperators, provider: Provider, type: typeof MAP_TYPES }): string
+export function sourceAndFiltersToSQL(props: {
+  data: string;
+  filters?: SourceFilters;
+  filtersLogicalOperator?: FiltersLogicalOperators;
+  provider: Provider;
+  type: typeof MAP_TYPES;
+}): string;
 
-export function getSqlEscapedSource(table: string, provider: Provider): string
+export function getSqlEscapedSource(table: string, provider: Provider): string;
+
+export function getSpatialFiltersResolution(props: {
+  source: SourceProps;
+  spatialDataType: string;
+  viewState: ViewState;
+}): number;

packages/react-widgets/src/models/utils.js

@@ -7,16 +7,30 @@ import {
 import { FullyQualifiedName } from './fqn';
 import { MAP_TYPES, API_VERSIONS } from '@carto/react-api';
 
+export { getSpatialFiltersResolution } from './spatialFiltersResolution';
+
 export function isRemoteCalculationSupported(props) {
   const { source } = props;
 
-  return (
-    source &&
-    source.type !== MAP_TYPES.TILESET &&
-    source.credentials.apiVersion !== API_VERSIONS.V2 &&
-    !(source.geoColumn && getSpatialIndexFromGeoColumn(source.geoColumn)) &&
-    source.provider !== 'databricks'
-  );
+  if (
+    !source ||
+    source.type === MAP_TYPES.TILESET ||
+    source.credentials.apiVersion === API_VERSIONS.V2 ||
+    source.provider === 'databricks'
+  ) {
+    return false;
+  }
+
+  const isDynamicSpatialIndex =
+    source.geoColumn && getSpatialIndexFromGeoColumn(source.geoColumn);
+  if (
+    isDynamicSpatialIndex &&
+    !source.dataResolution &&
+    !source.spatialFiltersResolution
+  ) {
+    return false;
+  }
+  return true;
 }
 
 export function wrapModelCall(props, fromLocal, fromRemote) {