From 5bcb1eceba23128d013c6f4d872a93eaf390d56c Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 12 Jul 2024 13:33:07 +0200 Subject: [PATCH 01/66] feat: simple screenshot button in settings no functional changes though --- src/python_integration/screenshots.ts | 15 +++++++ src/ui/viewer_settings.ts | 13 +++++- src/viewer.ts | 60 +++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index ca5b39497..8392f9011 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -30,6 +30,21 @@ import { Signal } from "#src/util/signal.js"; import { getCachedJson } from "#src/util/trackable.js"; import type { Viewer } from "#src/viewer.js"; +interface ScreenshotResponse { + id: string; + image: string; + imageType: string; + depthData: string | undefined; + width: number; + height: number; +} + +export interface ScreenshotActionState { + viewerState: any; + selectedValues: any; + screenshot: ScreenshotResponse; +} + export class ScreenshotHandler extends RefCounted { sendScreenshotRequested = new Signal<(state: any) => void>(); sendStatisticsRequested = new Signal<(state: any) => void>(); diff --git a/src/ui/viewer_settings.ts b/src/ui/viewer_settings.ts index 8744cb6f5..49c068453 100644 --- a/src/ui/viewer_settings.ts +++ b/src/ui/viewer_settings.ts @@ -125,7 +125,10 @@ export class ViewerSettingsPanel extends SidePanel { ); addCheckbox("Wire frame rendering", viewer.wireFrame); addCheckbox("Enable prefetching", viewer.chunkQueueManager.enablePrefetch); - addCheckbox("Enable adaptive downsampling", viewer.enableAdaptiveDownsampling); + addCheckbox( + "Enable adaptive downsampling", + viewer.enableAdaptiveDownsampling, + ); const addColor = (label: string, value: WatchableValueInterface) => { const labelElement = document.createElement("label"); @@ -137,5 +140,13 @@ export class ViewerSettingsPanel extends SidePanel { addColor("Cross-section background", viewer.crossSectionBackgroundColor); addColor("Projection background", viewer.perspectiveViewBackgroundColor); + + const addButton = (label: string, callback: () => void) => { + const button = document.createElement("button"); + button.textContent = label; + button.addEventListener("click", callback); + scroll.appendChild(button); + }; + addButton("Take screenshot", () => viewer.screenshot()); } } diff --git a/src/viewer.ts b/src/viewer.ts index 73a72f171..4e693dce6 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -70,6 +70,8 @@ import { WatchableDisplayDimensionRenderInfo, } from "#src/navigation_state.js"; import { overlaysOpen } from "#src/overlay.js"; +import type { ScreenshotActionState } from "#src/python_integration/screenshots.js"; +import { ScreenshotHandler } from "#src/python_integration/screenshots.js"; import { allRenderLayerRoles, RenderLayerRole } from "#src/renderlayer.js"; import { StatusMessage } from "#src/status.js"; import { @@ -487,6 +489,12 @@ export class Viewer extends RefCounted implements ViewerState { resetInitiated = new NullarySignal(); + private screenshotHandler = this.registerDisposer( + new ScreenshotHandler(this), + ); + private screenshotId = 0; + private screenshotUrl: string | undefined; + get chunkManager() { return this.dataContext.chunkManager; } @@ -565,6 +573,11 @@ export class Viewer extends RefCounted implements ViewerState { this.display.applyWindowedViewportToElement(element, value); }, this.partialViewport), ); + this.registerDisposer( + this.screenshotHandler.sendScreenshotRequested.add((state) => { + this.saveScreenshot(state); + }), + ); this.registerDisposer(() => removeFromParent(this.element)); @@ -1142,6 +1155,53 @@ export class Viewer extends RefCounted implements ViewerState { this.statisticsDisplayState.location.visible = value; } + screenshot() { + this.screenshotHandler.requestState.value = this.screenshotId.toString(); + this.screenshotId++; + } + + private saveScreenshot(actionState: ScreenshotActionState) { + function binaryStringToUint8Array(binaryString: string) { + const length = binaryString.length; + const bytes = new Uint8Array(length); + for (let i = 0; i < length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; + } + + function base64ToUint8Array(base64: string) { + const binaryString = window.atob(base64); + return binaryStringToUint8Array(binaryString); + } + + const { screenshot } = actionState; + const { image, imageType } = screenshot; + const screenshotImage = new Blob([base64ToUint8Array(image)], { + type: imageType, + }); + if (this.screenshotUrl !== undefined) { + URL.revokeObjectURL(this.screenshotUrl); + } + this.screenshotUrl = URL.createObjectURL(screenshotImage); + + const a = document.createElement("a"); + const url = this.screenshotUrl; + if (url !== undefined) { + let nowtime = new Date().toLocaleString(); + nowtime = nowtime.replace(", ", "-"); + a.href = url; + a.download = `neuroglancer-screenshot-${nowtime}.png`; + document.body.appendChild(a); + try { + a.click(); + } + finally { + document.body.removeChild(a); + } + } + } + get gl() { return this.display.gl; } From 5f296e85c418bc72d5acfe56ccd901bfd9f4180d Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 12 Jul 2024 16:10:54 +0200 Subject: [PATCH 02/66] poc: scalable screenshot --- src/display_context.ts | 19 +++++++-- src/python_integration/screenshots.ts | 4 +- src/ui/viewer_settings.ts | 24 ++++++++++++ src/viewer.ts | 56 ++++++++++++++++++++++----- 4 files changed, 88 insertions(+), 15 deletions(-) diff --git a/src/display_context.ts b/src/display_context.ts index d25d9e9c2..7b785a333 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -221,8 +221,16 @@ export abstract class RenderedPanel extends RefCounted { 0, clippedBottom - clippedTop, )); - viewport.logicalWidth = logicalWidth; - viewport.logicalHeight = logicalHeight; + // TODO this does not work for 2D panels + if (this.context.tempIgnoreCanvasSize) { + viewport.width = logicalWidth * screenToCanvasPixelScaleX; + viewport.height = logicalHeight * screenToCanvasPixelScaleY; + viewport.logicalWidth = logicalWidth * screenToCanvasPixelScaleX; + viewport.logicalHeight = logicalHeight * screenToCanvasPixelScaleY; + } else { + viewport.logicalWidth = logicalWidth; + viewport.logicalHeight = logicalHeight; + } viewport.visibleLeftFraction = (clippedLeft - logicalLeft) / logicalWidth; viewport.visibleTopFraction = (clippedTop - logicalTop) / logicalHeight; viewport.visibleWidthFraction = clippedWidth / logicalWidth; @@ -403,6 +411,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { rootRect: DOMRect | undefined; resizeGeneration = 0; boundsGeneration = -1; + tempIgnoreCanvasSize = false; private framerateMonitor = new FramerateMonitor(); private continuousCameraMotionInProgress = false; @@ -575,8 +584,10 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { const { resizeGeneration } = this; if (this.boundsGeneration === resizeGeneration) return; const { canvas } = this; - canvas.width = canvas.offsetWidth; - canvas.height = canvas.offsetHeight; + if (!this.tempIgnoreCanvasSize) { + canvas.width = canvas.offsetWidth; + canvas.height = canvas.offsetHeight; + } this.canvasRect = canvas.getBoundingClientRect(); this.rootRect = this.container.getBoundingClientRect(); this.boundsGeneration = resizeGeneration; diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index 8392f9011..410fd8103 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -139,12 +139,12 @@ export class ScreenshotHandler extends RefCounted { return; } const { viewer } = this; - if (!viewer.isReady()) { + if (!viewer.isReady() && !viewer.display.tempIgnoreCanvasSize) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); return; } - if (!this.wasAlreadyVisible) { + if (!this.wasAlreadyVisible && !viewer.display.tempIgnoreCanvasSize) { this.throttledSendStatistics(requestState); this.wasAlreadyVisible = true; this.debouncedMaybeSendScreenshot(); diff --git a/src/ui/viewer_settings.ts b/src/ui/viewer_settings.ts index 49c068453..21a637727 100644 --- a/src/ui/viewer_settings.ts +++ b/src/ui/viewer_settings.ts @@ -148,5 +148,29 @@ export class ViewerSettingsPanel extends SidePanel { scroll.appendChild(button); }; addButton("Take screenshot", () => viewer.screenshot()); + + const addIntSlider = ( + label: string, + min: number, + max: number, + callback: (value: number) => void, + ) => { + const labelElement = document.createElement("label"); + labelElement.textContent = label; + const slider = document.createElement("input"); + slider.type = "range"; + slider.min = min.toString(); + slider.max = max.toString(); + slider.value = callback.toString(); + slider.addEventListener("input", () => { + callback(parseInt(slider.value)); + }); + labelElement.appendChild(slider); + slider.value = viewer.screenshotScale.toString(); + scroll.appendChild(labelElement); + }; + addIntSlider("Screenshot resolution scale", 1, 8, (value) => { + viewer.screenshotScale = value; + }); } } diff --git a/src/viewer.ts b/src/viewer.ts index 4e693dce6..c8870a198 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -492,8 +492,9 @@ export class Viewer extends RefCounted implements ViewerState { private screenshotHandler = this.registerDisposer( new ScreenshotHandler(this), ); - private screenshotId = 0; + private screenshotId: number | undefined; private screenshotUrl: string | undefined; + screenshotScale: number = 1; get chunkManager() { return this.dataContext.chunkManager; @@ -576,6 +577,15 @@ export class Viewer extends RefCounted implements ViewerState { this.registerDisposer( this.screenshotHandler.sendScreenshotRequested.add((state) => { this.saveScreenshot(state); + this.resetCanvasSize(); + }), + ); + this.registerDisposer( + this.display.updateFinished.add(() => { + if (this.screenshotId !== undefined) { + this.screenshotHandler.requestState.value = + this.screenshotId.toString(); + } }), ); @@ -1047,7 +1057,7 @@ export class Viewer extends RefCounted implements ViewerState { }); } - this.bindAction("help", () => this.toggleHelpPanel()); + this.bindAction("help", () => this.screenshot()); for (let i = 1; i <= 9; ++i) { this.bindAction(`toggle-layer-${i}`, () => { @@ -1156,8 +1166,37 @@ export class Viewer extends RefCounted implements ViewerState { } screenshot() { - this.screenshotHandler.requestState.value = this.screenshotId.toString(); - this.screenshotId++; + const shouldResize = this.screenshotScale !== 1; + if (shouldResize) { + const oldSize = { + width: this.display.canvas.width, + height: this.display.canvas.height, + }; + const newSize = { + width: Math.round(oldSize.width * this.screenshotScale), + height: Math.round(oldSize.height * this.screenshotScale), + }; + this.display.canvas.width = newSize.width; + this.display.canvas.height = newSize.height; + } + if (this.screenshotId === undefined) { + this.screenshotId = 0; + } else { + this.screenshotId++; + } + this.display.tempIgnoreCanvasSize = true; + if (!shouldResize) { + this.display.scheduleRedraw(); + } else { + ++this.display.resizeGeneration; + this.display.resizeCallback(); + } + } + + private resetCanvasSize() { + this.display.tempIgnoreCanvasSize = false; + ++this.display.resizeGeneration; + this.display.resizeCallback(); } private saveScreenshot(actionState: ScreenshotActionState) { @@ -1176,7 +1215,7 @@ export class Viewer extends RefCounted implements ViewerState { } const { screenshot } = actionState; - const { image, imageType } = screenshot; + const { image, imageType, width, height } = screenshot; const screenshotImage = new Blob([base64ToUint8Array(image)], { type: imageType, }); @@ -1191,14 +1230,13 @@ export class Viewer extends RefCounted implements ViewerState { let nowtime = new Date().toLocaleString(); nowtime = nowtime.replace(", ", "-"); a.href = url; - a.download = `neuroglancer-screenshot-${nowtime}.png`; + a.download = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; document.body.appendChild(a); try { a.click(); - } - finally { + } finally { document.body.removeChild(a); - } + } } } From a11149566dc26edfd760a86e3221d5ef82c07059 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 27 Aug 2024 15:45:22 +0200 Subject: [PATCH 03/66] feat(ui): add camera icon to top bar --- src/ui/screenshot_menu.css | 26 +++++++++++++++++ src/ui/screenshot_menu.ts | 57 ++++++++++++++++++++++++++++++++++++++ src/viewer.ts | 14 ++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/ui/screenshot_menu.css create mode 100644 src/ui/screenshot_menu.ts diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css new file mode 100644 index 000000000..79f980508 --- /dev/null +++ b/src/ui/screenshot_menu.css @@ -0,0 +1,26 @@ +/** + * @license + * Copyright 2018 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + .neuroglancer-screenshot-dialog { + width: 80%; + position: relative; + top: 10%; +} + +.close-button { + position: absolute; + right: 15px; +} diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts new file mode 100644 index 000000000..5608e92c1 --- /dev/null +++ b/src/ui/screenshot_menu.ts @@ -0,0 +1,57 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Overlay } from "#src/overlay.js"; +import "#src/ui/screenshot_menu.css"; + +import type { Viewer } from "#src/viewer.js"; + +export class ScreenshotDialog extends Overlay { + filenameEditor: HTMLInputElement; + saveScreenshotButton: HTMLButtonElement; + closeButton: HTMLButtonElement; + constructor(public viewer: Viewer) { + super(); + + // TODO: this might be better as a menu, not a dialog. + this.content.classList.add("neuroglancer-screenshot-dialog"); + + const buttonClose = (this.closeButton = document.createElement("button")); + buttonClose.classList.add("close-button"); + buttonClose.textContent = "Close"; + this.content.appendChild(buttonClose); + buttonClose.addEventListener("click", () => this.dispose()); + + this.filenameEditor = document.createElement("input"); + this.filenameEditor.type = "text"; + this.filenameEditor.placeholder = "Enter filename..."; + this.content.appendChild(this.filenameEditor); + + const saveScreenshotButton = (this.saveScreenshotButton = + document.createElement("button")); + saveScreenshotButton.textContent = "Download"; + saveScreenshotButton.title = "Download state as a JSON file"; + this.content.appendChild(saveScreenshotButton); + saveScreenshotButton.addEventListener("click", () => this.screenshot()); + } + + screenshot() { + const filename = this.filenameEditor.value; + filename; + //this.viewer.saveScreenshot(filename); + this.dispose(); + } +} diff --git a/src/viewer.ts b/src/viewer.ts index c8870a198..9fcd3073f 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -17,6 +17,7 @@ import "#src/viewer.css"; import "#src/ui/layer_data_sources_tab.js"; import "#src/noselect.css"; +import svg_camera from "ikonate/icons/camera.svg?raw"; import svg_controls_alt from "ikonate/icons/controls-alt.svg?raw"; import svg_layers from "ikonate/icons/layers.svg?raw"; import svg_list from "ikonate/icons/list.svg?raw"; @@ -91,6 +92,7 @@ import { } from "#src/ui/layer_list_panel.js"; import { LayerSidePanelManager } from "#src/ui/layer_side_panel.js"; import { setupPositionDropHandlers } from "#src/ui/position_drag_and_drop.js"; +import { ScreenshotDialog } from "#src/ui/screenshot_menu.js"; import { SelectionDetailsPanel } from "#src/ui/selection_details.js"; import { SidePanelManager } from "#src/ui/side_panel.js"; import { StateEditorDialog } from "#src/ui/state_editor.js"; @@ -878,6 +880,14 @@ export class Viewer extends RefCounted implements ViewerState { topRow.appendChild(button); } + { + const button = makeIcon({ svg: svg_camera, title: "Screenshot" }); + this.registerEventListener(button, "click", () => { + this.showScreenshotDialog(); + }); + topRow.appendChild(button); + } + { const { helpPanelState } = this; const button = this.registerDisposer( @@ -1158,6 +1168,10 @@ export class Viewer extends RefCounted implements ViewerState { new StateEditorDialog(this); } + showScreenshotDialog() { + new ScreenshotDialog(this); + } + showStatistics(value: boolean | undefined = undefined) { if (value === undefined) { value = !this.statisticsDisplayState.location.visible; From 71b4ec73b1523663400b45977a957bfe73cd0220 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 27 Aug 2024 17:14:05 +0200 Subject: [PATCH 04/66] refactor: pull viewer complexity for screenshot into new class --- src/ui/viewer_settings.ts | 6 +-- src/util/screenshot.ts | 104 ++++++++++++++++++++++++++++++++++++++ src/viewer.ts | 99 ++++-------------------------------- 3 files changed, 118 insertions(+), 91 deletions(-) create mode 100644 src/util/screenshot.ts diff --git a/src/ui/viewer_settings.ts b/src/ui/viewer_settings.ts index 21a637727..4fb8f8973 100644 --- a/src/ui/viewer_settings.ts +++ b/src/ui/viewer_settings.ts @@ -147,7 +147,7 @@ export class ViewerSettingsPanel extends SidePanel { button.addEventListener("click", callback); scroll.appendChild(button); }; - addButton("Take screenshot", () => viewer.screenshot()); + addButton("Take screenshot", () => viewer.screenshotHandler.screenshot()); const addIntSlider = ( label: string, @@ -166,11 +166,11 @@ export class ViewerSettingsPanel extends SidePanel { callback(parseInt(slider.value)); }); labelElement.appendChild(slider); - slider.value = viewer.screenshotScale.toString(); + slider.value = viewer.screenshotHandler.screenshotScale.toString(); scroll.appendChild(labelElement); }; addIntSlider("Screenshot resolution scale", 1, 8, (value) => { - viewer.screenshotScale = value; + viewer.screenshotHandler.screenshotScale = value; }); } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts new file mode 100644 index 000000000..3959b47a5 --- /dev/null +++ b/src/util/screenshot.ts @@ -0,0 +1,104 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use viewer file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ScreenshotActionState } from "#src/python_integration/screenshots.js"; +import { RefCounted } from "#src/util/disposable.js"; +import type { Viewer } from "#src/viewer.js"; + +export class ScreenshotFromViewer extends RefCounted { + public screenshotId: number = -1; + private screenshotUrl: string | undefined; + public screenshotScale: number = 1; + + constructor(public viewer: Viewer) { + super(); + this.viewer = viewer; + } + + screenshot() { + const { viewer } = this; + const shouldResize = this.screenshotScale !== 1; + if (shouldResize) { + const oldSize = { + width: viewer.display.canvas.width, + height: viewer.display.canvas.height, + }; + const newSize = { + width: Math.round(oldSize.width * this.screenshotScale), + height: Math.round(oldSize.height * this.screenshotScale), + }; + viewer.display.canvas.width = newSize.width; + viewer.display.canvas.height = newSize.height; + } + this.screenshotId++; + viewer.display.tempIgnoreCanvasSize = true; + if (!shouldResize) { + viewer.display.scheduleRedraw(); + } else { + ++viewer.display.resizeGeneration; + viewer.display.resizeCallback(); + } + } + + resetCanvasSize() { + const { viewer } = this; + viewer.display.tempIgnoreCanvasSize = false; + ++viewer.display.resizeGeneration; + viewer.display.resizeCallback(); + } + + saveScreenshot(actionState: ScreenshotActionState) { + function binaryStringToUint8Array(binaryString: string) { + const length = binaryString.length; + const bytes = new Uint8Array(length); + for (let i = 0; i < length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; + } + + function base64ToUint8Array(base64: string) { + const binaryString = window.atob(base64); + return binaryStringToUint8Array(binaryString); + } + + const { screenshot } = actionState; + const { image, imageType, width, height } = screenshot; + const screenshotImage = new Blob([base64ToUint8Array(image)], { + type: imageType, + }); + if (this.screenshotUrl !== undefined) { + URL.revokeObjectURL(this.screenshotUrl); + } + this.screenshotUrl = URL.createObjectURL(screenshotImage); + + const a = document.createElement("a"); + if (this.screenshotUrl !== undefined) { + let nowtime = new Date().toLocaleString(); + nowtime = nowtime.replace(", ", "-"); + a.href = this.screenshotUrl; + a.download = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; + document.body.appendChild(a); + try { + a.click(); + } finally { + document.body.removeChild(a); + } + } + + this.resetCanvasSize(); + } +} diff --git a/src/viewer.ts b/src/viewer.ts index 9fcd3073f..6ecbb295d 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -71,7 +71,6 @@ import { WatchableDisplayDimensionRenderInfo, } from "#src/navigation_state.js"; import { overlaysOpen } from "#src/overlay.js"; -import type { ScreenshotActionState } from "#src/python_integration/screenshots.js"; import { ScreenshotHandler } from "#src/python_integration/screenshots.js"; import { allRenderLayerRoles, RenderLayerRole } from "#src/renderlayer.js"; import { StatusMessage } from "#src/status.js"; @@ -121,6 +120,7 @@ import { EventActionMap, KeyboardEventBinder, } from "#src/util/keyboard_bindings.js"; +import { ScreenshotFromViewer } from "#src/util/screenshot.js"; import { NullarySignal } from "#src/util/signal.js"; import { CompoundTrackable, @@ -491,12 +491,12 @@ export class Viewer extends RefCounted implements ViewerState { resetInitiated = new NullarySignal(); - private screenshotHandler = this.registerDisposer( + private screenshotActionHandler = this.registerDisposer( new ScreenshotHandler(this), ); - private screenshotId: number | undefined; - private screenshotUrl: string | undefined; - screenshotScale: number = 1; + public screenshotHandler = this.registerDisposer( + new ScreenshotFromViewer(this), + ); get chunkManager() { return this.dataContext.chunkManager; @@ -577,16 +577,16 @@ export class Viewer extends RefCounted implements ViewerState { }, this.partialViewport), ); this.registerDisposer( - this.screenshotHandler.sendScreenshotRequested.add((state) => { - this.saveScreenshot(state); - this.resetCanvasSize(); + this.screenshotActionHandler.sendScreenshotRequested.add((state) => { + this.screenshotHandler.saveScreenshot(state); }), ); + // TODO this is a bit clunky, but it works for now. this.registerDisposer( this.display.updateFinished.add(() => { - if (this.screenshotId !== undefined) { - this.screenshotHandler.requestState.value = - this.screenshotId.toString(); + if (this.screenshotHandler.screenshotId >= 0) { + this.screenshotActionHandler.requestState.value = + this.screenshotHandler.screenshotId.toString(); } }), ); @@ -1067,8 +1067,6 @@ export class Viewer extends RefCounted implements ViewerState { }); } - this.bindAction("help", () => this.screenshot()); - for (let i = 1; i <= 9; ++i) { this.bindAction(`toggle-layer-${i}`, () => { const layer = this.layerManager.getLayerByNonArchivedIndex(i - 1); @@ -1179,81 +1177,6 @@ export class Viewer extends RefCounted implements ViewerState { this.statisticsDisplayState.location.visible = value; } - screenshot() { - const shouldResize = this.screenshotScale !== 1; - if (shouldResize) { - const oldSize = { - width: this.display.canvas.width, - height: this.display.canvas.height, - }; - const newSize = { - width: Math.round(oldSize.width * this.screenshotScale), - height: Math.round(oldSize.height * this.screenshotScale), - }; - this.display.canvas.width = newSize.width; - this.display.canvas.height = newSize.height; - } - if (this.screenshotId === undefined) { - this.screenshotId = 0; - } else { - this.screenshotId++; - } - this.display.tempIgnoreCanvasSize = true; - if (!shouldResize) { - this.display.scheduleRedraw(); - } else { - ++this.display.resizeGeneration; - this.display.resizeCallback(); - } - } - - private resetCanvasSize() { - this.display.tempIgnoreCanvasSize = false; - ++this.display.resizeGeneration; - this.display.resizeCallback(); - } - - private saveScreenshot(actionState: ScreenshotActionState) { - function binaryStringToUint8Array(binaryString: string) { - const length = binaryString.length; - const bytes = new Uint8Array(length); - for (let i = 0; i < length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes; - } - - function base64ToUint8Array(base64: string) { - const binaryString = window.atob(base64); - return binaryStringToUint8Array(binaryString); - } - - const { screenshot } = actionState; - const { image, imageType, width, height } = screenshot; - const screenshotImage = new Blob([base64ToUint8Array(image)], { - type: imageType, - }); - if (this.screenshotUrl !== undefined) { - URL.revokeObjectURL(this.screenshotUrl); - } - this.screenshotUrl = URL.createObjectURL(screenshotImage); - - const a = document.createElement("a"); - const url = this.screenshotUrl; - if (url !== undefined) { - let nowtime = new Date().toLocaleString(); - nowtime = nowtime.replace(", ", "-"); - a.href = url; - a.download = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; - document.body.appendChild(a); - try { - a.click(); - } finally { - document.body.removeChild(a); - } - } - } - get gl() { return this.display.gl; } From 0472c12b91b6c58facbe9811f1f41d475c7f167a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 27 Aug 2024 17:36:07 +0200 Subject: [PATCH 05/66] refactor: clarify variable --- src/display_context.ts | 7 +++---- src/python_integration/screenshots.ts | 4 ++-- src/util/screenshot.ts | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/display_context.ts b/src/display_context.ts index 7b785a333..6c760b38a 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -221,8 +221,7 @@ export abstract class RenderedPanel extends RefCounted { 0, clippedBottom - clippedTop, )); - // TODO this does not work for 2D panels - if (this.context.tempIgnoreCanvasSize) { + if (this.context.inScreenshotMode) { viewport.width = logicalWidth * screenToCanvasPixelScaleX; viewport.height = logicalHeight * screenToCanvasPixelScaleY; viewport.logicalWidth = logicalWidth * screenToCanvasPixelScaleX; @@ -411,7 +410,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { rootRect: DOMRect | undefined; resizeGeneration = 0; boundsGeneration = -1; - tempIgnoreCanvasSize = false; + inScreenshotMode = false; private framerateMonitor = new FramerateMonitor(); private continuousCameraMotionInProgress = false; @@ -584,7 +583,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { const { resizeGeneration } = this; if (this.boundsGeneration === resizeGeneration) return; const { canvas } = this; - if (!this.tempIgnoreCanvasSize) { + if (!this.inScreenshotMode) { canvas.width = canvas.offsetWidth; canvas.height = canvas.offsetHeight; } diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index 410fd8103..e7a710f60 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -139,12 +139,12 @@ export class ScreenshotHandler extends RefCounted { return; } const { viewer } = this; - if (!viewer.isReady() && !viewer.display.tempIgnoreCanvasSize) { + if (!viewer.isReady() && !viewer.display.inScreenshotMode) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); return; } - if (!this.wasAlreadyVisible && !viewer.display.tempIgnoreCanvasSize) { + if (!this.wasAlreadyVisible && !viewer.display.inScreenshotMode) { this.throttledSendStatistics(requestState); this.wasAlreadyVisible = true; this.debouncedMaybeSendScreenshot(); diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 3959b47a5..2c0ca17ba 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -44,7 +44,7 @@ export class ScreenshotFromViewer extends RefCounted { viewer.display.canvas.height = newSize.height; } this.screenshotId++; - viewer.display.tempIgnoreCanvasSize = true; + viewer.display.inScreenshotMode = true; if (!shouldResize) { viewer.display.scheduleRedraw(); } else { @@ -55,7 +55,7 @@ export class ScreenshotFromViewer extends RefCounted { resetCanvasSize() { const { viewer } = this; - viewer.display.tempIgnoreCanvasSize = false; + viewer.display.inScreenshotMode = false; ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } @@ -99,6 +99,6 @@ export class ScreenshotFromViewer extends RefCounted { } } - this.resetCanvasSize(); + //this.resetCanvasSize(); } } From e2b15b9edce747864c5f160543ab2fe94a0653ba Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 10:55:00 +0200 Subject: [PATCH 06/66] refactor: move interface for screenshot to utils --- src/python_integration/screenshots.ts | 19 ++----------------- src/util/screenshot.ts | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index e7a710f60..ca5b39497 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -30,21 +30,6 @@ import { Signal } from "#src/util/signal.js"; import { getCachedJson } from "#src/util/trackable.js"; import type { Viewer } from "#src/viewer.js"; -interface ScreenshotResponse { - id: string; - image: string; - imageType: string; - depthData: string | undefined; - width: number; - height: number; -} - -export interface ScreenshotActionState { - viewerState: any; - selectedValues: any; - screenshot: ScreenshotResponse; -} - export class ScreenshotHandler extends RefCounted { sendScreenshotRequested = new Signal<(state: any) => void>(); sendStatisticsRequested = new Signal<(state: any) => void>(); @@ -139,12 +124,12 @@ export class ScreenshotHandler extends RefCounted { return; } const { viewer } = this; - if (!viewer.isReady() && !viewer.display.inScreenshotMode) { + if (!viewer.isReady()) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); return; } - if (!this.wasAlreadyVisible && !viewer.display.inScreenshotMode) { + if (!this.wasAlreadyVisible) { this.throttledSendStatistics(requestState); this.wasAlreadyVisible = true; this.debouncedMaybeSendScreenshot(); diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 2c0ca17ba..f7655cff2 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -14,10 +14,24 @@ * limitations under the License. */ -import type { ScreenshotActionState } from "#src/python_integration/screenshots.js"; import { RefCounted } from "#src/util/disposable.js"; import type { Viewer } from "#src/viewer.js"; +interface ScreenshotResponse { + id: string; + image: string; + imageType: string; + depthData: string | undefined; + width: number; + height: number; +} + +export interface ScreenshotActionState { + viewerState: any; + selectedValues: any; + screenshot: ScreenshotResponse; +} + export class ScreenshotFromViewer extends RefCounted { public screenshotId: number = -1; private screenshotUrl: string | undefined; @@ -99,6 +113,6 @@ export class ScreenshotFromViewer extends RefCounted { } } - //this.resetCanvasSize(); + this.resetCanvasSize(); } } From 504228adeb78a3e33d026e769f404f085658ee8e Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 10:57:39 +0200 Subject: [PATCH 07/66] fix: revert help menu change for temp testing --- src/viewer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/viewer.ts b/src/viewer.ts index 6ecbb295d..9c3ad6f1b 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -1067,6 +1067,8 @@ export class Viewer extends RefCounted implements ViewerState { }); } + this.bindAction("help", () => this.toggleHelpPanel()); + for (let i = 1; i <= 9; ++i) { this.bindAction(`toggle-layer-${i}`, () => { const layer = this.layerManager.getLayerByNonArchivedIndex(i - 1); From e0b150425a1115df4429215153b45aad0046cb45 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 11:27:15 +0200 Subject: [PATCH 08/66] feat(ui): add screenshot UI elements --- src/display_context.ts | 1 + src/python_integration/screenshots.ts | 2 +- src/ui/screenshot_menu.ts | 73 +++++++++++++++++++-------- src/ui/viewer_settings.ts | 32 ------------ src/util/screenshot.ts | 21 ++++++-- 5 files changed, 72 insertions(+), 57 deletions(-) diff --git a/src/display_context.ts b/src/display_context.ts index 6c760b38a..39fdf1845 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -411,6 +411,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { resizeGeneration = 0; boundsGeneration = -1; inScreenshotMode = false; + forceScreenshot = false; private framerateMonitor = new FramerateMonitor(); private continuousCameraMotionInProgress = false; diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index ca5b39497..1b9b4d515 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -124,7 +124,7 @@ export class ScreenshotHandler extends RefCounted { return; } const { viewer } = this; - if (!viewer.isReady()) { + if (!viewer.isReady() && !viewer.display.forceScreenshot) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); return; diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 5608e92c1..d891fa549 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -20,38 +20,71 @@ import "#src/ui/screenshot_menu.css"; import type { Viewer } from "#src/viewer.js"; export class ScreenshotDialog extends Overlay { - filenameEditor: HTMLInputElement; - saveScreenshotButton: HTMLButtonElement; + nameInput: HTMLInputElement; + saveButton: HTMLButtonElement; closeButton: HTMLButtonElement; + forceScreenshotButton: HTMLButtonElement; constructor(public viewer: Viewer) { super(); // TODO: this might be better as a menu, not a dialog. this.content.classList.add("neuroglancer-screenshot-dialog"); - const buttonClose = (this.closeButton = document.createElement("button")); - buttonClose.classList.add("close-button"); - buttonClose.textContent = "Close"; - this.content.appendChild(buttonClose); - buttonClose.addEventListener("click", () => this.dispose()); + const closeButton = (this.closeButton = document.createElement("button")); + closeButton.classList.add("close-button"); + closeButton.textContent = "Close"; + closeButton.addEventListener("click", () => this.dispose()); - this.filenameEditor = document.createElement("input"); - this.filenameEditor.type = "text"; - this.filenameEditor.placeholder = "Enter filename..."; - this.content.appendChild(this.filenameEditor); + const nameInput = (this.nameInput = document.createElement("input")); + nameInput.type = "text"; + nameInput.placeholder = "Enter filename..."; - const saveScreenshotButton = (this.saveScreenshotButton = + const saveButton = (this.saveButton = document.createElement("button")); + saveButton.textContent = "Take screenshot"; + saveButton.title = + "Take a screenshot of the current view and save it to a png file"; + saveButton.addEventListener("click", () => this.screenshot()); + + const forceScreenshotButton = (this.forceScreenshotButton = document.createElement("button")); - saveScreenshotButton.textContent = "Download"; - saveScreenshotButton.title = "Download state as a JSON file"; - this.content.appendChild(saveScreenshotButton); - saveScreenshotButton.addEventListener("click", () => this.screenshot()); + forceScreenshotButton.textContent = "Force screenshot"; + forceScreenshotButton.title = + "Force a screenshot of the current view and save it to a png file"; + forceScreenshotButton.addEventListener("click", () => { + this.viewer.display.forceScreenshot = true; + }); + + this.content.appendChild(closeButton); + this.content.appendChild(this.createScaleRadioButtons()); + this.content.appendChild(nameInput); + this.content.appendChild(saveButton); + } + + private createScaleRadioButtons() { + const scaleRadioButtons = document.createElement("div"); + scaleRadioButtons.classList.add("scale-radio-buttons"); + const scales = [1, 2, 4]; + for (const scale of scales) { + const label = document.createElement("label"); + const input = document.createElement("input"); + input.type = "radio"; + input.name = "screenshot-scale"; + input.value = scale.toString(); + input.checked = scale === 1; + label.appendChild(input); + label.appendChild(document.createTextNode(`Scale ${scale}x`)); + scaleRadioButtons.appendChild(label); + input.addEventListener("change", () => { + this.viewer.screenshotHandler.screenshotScale = scale; + }); + } + return scaleRadioButtons; } - screenshot() { - const filename = this.filenameEditor.value; - filename; - //this.viewer.saveScreenshot(filename); + private screenshot() { + const filename = this.nameInput.value; + this.viewer.screenshotHandler.screenshot(filename); + this.viewer.display.forceScreenshot = false; this.dispose(); } } diff --git a/src/ui/viewer_settings.ts b/src/ui/viewer_settings.ts index 4fb8f8973..a0c936f94 100644 --- a/src/ui/viewer_settings.ts +++ b/src/ui/viewer_settings.ts @@ -140,37 +140,5 @@ export class ViewerSettingsPanel extends SidePanel { addColor("Cross-section background", viewer.crossSectionBackgroundColor); addColor("Projection background", viewer.perspectiveViewBackgroundColor); - - const addButton = (label: string, callback: () => void) => { - const button = document.createElement("button"); - button.textContent = label; - button.addEventListener("click", callback); - scroll.appendChild(button); - }; - addButton("Take screenshot", () => viewer.screenshotHandler.screenshot()); - - const addIntSlider = ( - label: string, - min: number, - max: number, - callback: (value: number) => void, - ) => { - const labelElement = document.createElement("label"); - labelElement.textContent = label; - const slider = document.createElement("input"); - slider.type = "range"; - slider.min = min.toString(); - slider.max = max.toString(); - slider.value = callback.toString(); - slider.addEventListener("input", () => { - callback(parseInt(slider.value)); - }); - labelElement.appendChild(slider); - slider.value = viewer.screenshotHandler.screenshotScale.toString(); - scroll.appendChild(labelElement); - }; - addIntSlider("Screenshot resolution scale", 1, 8, (value) => { - viewer.screenshotHandler.screenshotScale = value; - }); } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index f7655cff2..4025e9efe 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -36,13 +36,14 @@ export class ScreenshotFromViewer extends RefCounted { public screenshotId: number = -1; private screenshotUrl: string | undefined; public screenshotScale: number = 1; + private filename: string = ""; constructor(public viewer: Viewer) { super(); this.viewer = viewer; } - screenshot() { + screenshot(filename: string = "") { const { viewer } = this; const shouldResize = this.screenshotScale !== 1; if (shouldResize) { @@ -65,6 +66,7 @@ export class ScreenshotFromViewer extends RefCounted { ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } + this.filename = filename; } resetCanvasSize() { @@ -74,6 +76,19 @@ export class ScreenshotFromViewer extends RefCounted { viewer.display.resizeCallback(); } + generateFilename(width: number, height: number): string { + let filename = this.filename; + if (filename.length === 0) { + let nowtime = new Date().toLocaleString(); + nowtime = nowtime.replace(", ", "-"); + filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; + } + if (!filename.endsWith(".png")) { + filename += ".png"; + } + return filename; + } + saveScreenshot(actionState: ScreenshotActionState) { function binaryStringToUint8Array(binaryString: string) { const length = binaryString.length; @@ -101,10 +116,8 @@ export class ScreenshotFromViewer extends RefCounted { const a = document.createElement("a"); if (this.screenshotUrl !== undefined) { - let nowtime = new Date().toLocaleString(); - nowtime = nowtime.replace(", ", "-"); a.href = this.screenshotUrl; - a.download = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; + a.download = this.generateFilename(width, height); document.body.appendChild(a); try { a.click(); From bd52bd7310319caf1cd51ec6781514f2c2585419 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 11:58:38 +0200 Subject: [PATCH 09/66] fix: allow forcing screenshot if viewer not ready --- src/python_integration/screenshots.ts | 2 +- src/ui/screenshot_menu.ts | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index 1b9b4d515..54182aaa2 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -129,7 +129,7 @@ export class ScreenshotHandler extends RefCounted { this.throttledSendStatistics(requestState); return; } - if (!this.wasAlreadyVisible) { + if (!this.wasAlreadyVisible && !viewer.display.forceScreenshot) { this.throttledSendStatistics(requestState); this.wasAlreadyVisible = true; this.debouncedMaybeSendScreenshot(); diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index d891fa549..e96b2f9bb 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -51,13 +51,14 @@ export class ScreenshotDialog extends Overlay { forceScreenshotButton.title = "Force a screenshot of the current view and save it to a png file"; forceScreenshotButton.addEventListener("click", () => { - this.viewer.display.forceScreenshot = true; + this.forceScreenshot(); }); this.content.appendChild(closeButton); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(nameInput); this.content.appendChild(saveButton); + this.content.appendChild(forceScreenshotButton); } private createScaleRadioButtons() { @@ -70,7 +71,7 @@ export class ScreenshotDialog extends Overlay { input.type = "radio"; input.name = "screenshot-scale"; input.value = scale.toString(); - input.checked = scale === 1; + input.checked = scale === this.viewer.screenshotHandler.screenshotScale; label.appendChild(input); label.appendChild(document.createTextNode(`Scale ${scale}x`)); scaleRadioButtons.appendChild(label); @@ -81,10 +82,15 @@ export class ScreenshotDialog extends Overlay { return scaleRadioButtons; } + private forceScreenshot() { + this.viewer.display.forceScreenshot = true; + this.viewer.display.scheduleRedraw(); + this.dispose(); + } + private screenshot() { const filename = this.nameInput.value; this.viewer.screenshotHandler.screenshot(filename); this.viewer.display.forceScreenshot = false; - this.dispose(); } } From 58deb88757732fd0f40d44679554b9376ebba374 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 13:40:09 +0200 Subject: [PATCH 10/66] feat: crop screenshot to view panels --- src/util/screenshot.ts | 125 ++++++++++++++++++++++++++++++++++------- 1 file changed, 105 insertions(+), 20 deletions(-) diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 4025e9efe..2aedd2b47 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { PerspectivePanel } from "#src/perspective_view/panel.js"; +import { SliceViewPanel } from "#src/sliceview/panel.js"; import { RefCounted } from "#src/util/disposable.js"; import type { Viewer } from "#src/viewer.js"; @@ -26,12 +28,65 @@ interface ScreenshotResponse { height: number; } -export interface ScreenshotActionState { +interface ScreenshotActionState { viewerState: any; selectedValues: any; screenshot: ScreenshotResponse; } +interface ScreenshotCanvasViewport { + left: number; + right: number; + top: number; + bottom: number; +} + +async function cropUint8Image( + image: Uint8Array, + crop: ScreenshotCanvasViewport, +): Promise { + const blob = new Blob([image], { type: "image/png" }); + const img = new Image(); + const loadImage = new Promise((resolve, reject) => { + img.onload = () => resolve(img); + img.onerror = (error) => reject(error); + }); + img.src = URL.createObjectURL(blob); + const loadedImg = await loadImage; + + const cropWidth = crop.right - crop.left; + const cropHeight = crop.bottom - crop.top; + const canvas = document.createElement("canvas"); + canvas.width = cropWidth; + canvas.height = cropHeight; + const ctx = canvas.getContext("2d"); + if (!ctx) { + throw new Error("Failed to get canvas context"); + } + + ctx.drawImage( + loadedImg, + crop.left, + crop.top, // Source image origin + cropWidth, + cropHeight, // Crop dimensions from the source image + 0, + 0, // Target canvas origin + cropWidth, + cropHeight, // Target canvas dimensions + ); + + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (blob) { + resolve(blob); + } else { + reject(new Error("Canvas toBlob failed")); + } + }, "image/png"); + }); +} + export class ScreenshotFromViewer extends RefCounted { public screenshotId: number = -1; private screenshotUrl: string | undefined; @@ -89,6 +144,34 @@ export class ScreenshotFromViewer extends RefCounted { return filename; } + calculateRenderLocation(): ScreenshotCanvasViewport { + const panels = this.viewer.display.panels; + const clippedPanel = { + left: Number.POSITIVE_INFINITY, + right: Number.NEGATIVE_INFINITY, + top: Number.POSITIVE_INFINITY, + bottom: Number.NEGATIVE_INFINITY, + }; + for (const panel of panels) { + const isViewPanel = + panel instanceof SliceViewPanel || panel instanceof PerspectivePanel; + if (!isViewPanel) { + continue; + } + const viewport = panel.renderViewport; + const { width, height } = viewport; + const left = panel.canvasRelativeClippedLeft; + const top = panel.canvasRelativeClippedTop; + const right = left + width; + const bottom = top + height; + clippedPanel.left = Math.min(clippedPanel.left, left); + clippedPanel.right = Math.max(clippedPanel.right, right); + clippedPanel.top = Math.min(clippedPanel.top, top); + clippedPanel.bottom = Math.max(clippedPanel.bottom, bottom); + } + return clippedPanel; + } + saveScreenshot(actionState: ScreenshotActionState) { function binaryStringToUint8Array(binaryString: string) { const length = binaryString.length; @@ -105,27 +188,29 @@ export class ScreenshotFromViewer extends RefCounted { } const { screenshot } = actionState; - const { image, imageType, width, height } = screenshot; - const screenshotImage = new Blob([base64ToUint8Array(image)], { - type: imageType, - }); - if (this.screenshotUrl !== undefined) { - URL.revokeObjectURL(this.screenshotUrl); - } - this.screenshotUrl = URL.createObjectURL(screenshotImage); - - const a = document.createElement("a"); - if (this.screenshotUrl !== undefined) { - a.href = this.screenshotUrl; - a.download = this.generateFilename(width, height); - document.body.appendChild(a); - try { - a.click(); - } finally { - document.body.removeChild(a); + const { image } = screenshot; + const fullImage = base64ToUint8Array(image); + const renderLocation = this.calculateRenderLocation(); + cropUint8Image(fullImage, renderLocation).then((croppedImage) => { + if (this.screenshotUrl !== undefined) { + URL.revokeObjectURL(this.screenshotUrl); } - } + this.screenshotUrl = URL.createObjectURL(croppedImage); + const a = document.createElement("a"); + if (this.screenshotUrl !== undefined) { + a.href = this.screenshotUrl; + const width = renderLocation.right - renderLocation.left; + const height = renderLocation.bottom - renderLocation.top; + a.download = this.generateFilename(width, height); + document.body.appendChild(a); + try { + a.click(); + } finally { + document.body.removeChild(a); + } + } + }); this.resetCanvasSize(); } } From 4ef21f4353e578e187f153de1d3f1ac512f46fa1 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 17:52:45 +0200 Subject: [PATCH 11/66] refactor: clean up screenshot code --- src/util/screenshot.ts | 196 +++++++++++++++++++++-------------------- 1 file changed, 99 insertions(+), 97 deletions(-) diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 2aedd2b47..35fc525bf 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type { RenderedPanel } from "#src/display_context.js"; import { PerspectivePanel } from "#src/perspective_view/panel.js"; import { SliceViewPanel } from "#src/sliceview/panel.js"; import { RefCounted } from "#src/util/disposable.js"; @@ -41,41 +42,66 @@ interface ScreenshotCanvasViewport { bottom: number; } -async function cropUint8Image( - image: Uint8Array, - crop: ScreenshotCanvasViewport, -): Promise { - const blob = new Blob([image], { type: "image/png" }); - const img = new Image(); - const loadImage = new Promise((resolve, reject) => { - img.onload = () => resolve(img); - img.onerror = (error) => reject(error); - }); - img.src = URL.createObjectURL(blob); - const loadedImg = await loadImage; +function downloadFileForBlob(blob: Blob, filename: string) { + const a = document.createElement("a"); + const url = URL.createObjectURL(blob); + a.href = url; + a.download = filename; + try { + a.click(); + } finally { + URL.revokeObjectURL(url); + } +} - const cropWidth = crop.right - crop.left; - const cropHeight = crop.bottom - crop.top; - const canvas = document.createElement("canvas"); - canvas.width = cropWidth; - canvas.height = cropHeight; - const ctx = canvas.getContext("2d"); - if (!ctx) { - throw new Error("Failed to get canvas context"); +function generateFilename( + inputFilename: string, + width: number, + height: number, +): string { + let filename = inputFilename; + if (filename.length === 0) { + let nowtime = new Date().toLocaleString(); + nowtime = nowtime.replace(", ", "-"); + filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; + } + if (!filename.endsWith(".png")) { + filename += ".png"; } + return filename; +} - ctx.drawImage( - loadedImg, - crop.left, - crop.top, // Source image origin - cropWidth, - cropHeight, // Crop dimensions from the source image - 0, - 0, // Target canvas origin - cropWidth, - cropHeight, // Target canvas dimensions - ); +function determineViewPanelArea( + panels: Set, +): ScreenshotCanvasViewport { + const clippedPanel = { + left: Number.POSITIVE_INFINITY, + right: Number.NEGATIVE_INFINITY, + top: Number.POSITIVE_INFINITY, + bottom: Number.NEGATIVE_INFINITY, + }; + for (const panel of panels) { + if ( + !(panel instanceof SliceViewPanel) && + !(panel instanceof PerspectivePanel) + ) { + continue; + } + const viewport = panel.renderViewport; + const { width, height } = viewport; + const left = panel.canvasRelativeClippedLeft; + const top = panel.canvasRelativeClippedTop; + const right = left + width; + const bottom = top + height; + clippedPanel.left = Math.min(clippedPanel.left, left); + clippedPanel.right = Math.max(clippedPanel.right, right); + clippedPanel.top = Math.min(clippedPanel.top, top); + clippedPanel.bottom = Math.max(clippedPanel.bottom, bottom); + } + return clippedPanel; +} +function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { return new Promise((resolve, reject) => { canvas.toBlob((blob) => { if (blob) { @@ -83,13 +109,38 @@ async function cropUint8Image( } else { reject(new Error("Canvas toBlob failed")); } - }, "image/png"); + }, type); }); } +async function cropUint8Image( + image: Uint8Array, + crop: ScreenshotCanvasViewport, +): Promise { + const blob = new Blob([image], { type: "image/png" }); + const cropWidth = crop.right - crop.left; + const cropHeight = crop.bottom - crop.top; + const img = await createImageBitmap( + blob, + crop.left, + crop.top, + cropWidth, + cropHeight, + ); + + const canvas = document.createElement("canvas"); + canvas.width = cropWidth; + canvas.height = cropHeight; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Failed to get canvas context"); + ctx.drawImage(img, 0, 0); + + const croppedBlob = await canvasToBlob(canvas, "image/png"); + return croppedBlob; +} + export class ScreenshotFromViewer extends RefCounted { public screenshotId: number = -1; - private screenshotUrl: string | undefined; public screenshotScale: number = 1; private filename: string = ""; @@ -131,48 +182,7 @@ export class ScreenshotFromViewer extends RefCounted { viewer.display.resizeCallback(); } - generateFilename(width: number, height: number): string { - let filename = this.filename; - if (filename.length === 0) { - let nowtime = new Date().toLocaleString(); - nowtime = nowtime.replace(", ", "-"); - filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; - } - if (!filename.endsWith(".png")) { - filename += ".png"; - } - return filename; - } - - calculateRenderLocation(): ScreenshotCanvasViewport { - const panels = this.viewer.display.panels; - const clippedPanel = { - left: Number.POSITIVE_INFINITY, - right: Number.NEGATIVE_INFINITY, - top: Number.POSITIVE_INFINITY, - bottom: Number.NEGATIVE_INFINITY, - }; - for (const panel of panels) { - const isViewPanel = - panel instanceof SliceViewPanel || panel instanceof PerspectivePanel; - if (!isViewPanel) { - continue; - } - const viewport = panel.renderViewport; - const { width, height } = viewport; - const left = panel.canvasRelativeClippedLeft; - const top = panel.canvasRelativeClippedTop; - const right = left + width; - const bottom = top + height; - clippedPanel.left = Math.min(clippedPanel.left, left); - clippedPanel.right = Math.max(clippedPanel.right, right); - clippedPanel.top = Math.min(clippedPanel.top, top); - clippedPanel.bottom = Math.max(clippedPanel.bottom, bottom); - } - return clippedPanel; - } - - saveScreenshot(actionState: ScreenshotActionState) { + async saveScreenshot(actionState: ScreenshotActionState) { function binaryStringToUint8Array(binaryString: string) { const length = binaryString.length; const bytes = new Uint8Array(length); @@ -190,27 +200,19 @@ export class ScreenshotFromViewer extends RefCounted { const { screenshot } = actionState; const { image } = screenshot; const fullImage = base64ToUint8Array(image); - const renderLocation = this.calculateRenderLocation(); - cropUint8Image(fullImage, renderLocation).then((croppedImage) => { - if (this.screenshotUrl !== undefined) { - URL.revokeObjectURL(this.screenshotUrl); - } - this.screenshotUrl = URL.createObjectURL(croppedImage); - - const a = document.createElement("a"); - if (this.screenshotUrl !== undefined) { - a.href = this.screenshotUrl; - const width = renderLocation.right - renderLocation.left; - const height = renderLocation.bottom - renderLocation.top; - a.download = this.generateFilename(width, height); - document.body.appendChild(a); - try { - a.click(); - } finally { - document.body.removeChild(a); - } - } - }); - this.resetCanvasSize(); + const renderLocation = determineViewPanelArea(this.viewer.display.panels); + try { + const croppedImage = await cropUint8Image(fullImage, renderLocation); + const filename = generateFilename( + this.filename, + screenshot.width, + screenshot.height, + ); + downloadFileForBlob(croppedImage, filename); + } catch (error) { + console.error(error); + } finally { + this.resetCanvasSize(); + } } } From ac9fe6995e30047768bf3b8461967222334c44aa Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 17:59:57 +0200 Subject: [PATCH 12/66] refactor: remove atob and btoa to reduce complexity of codebase --- src/util/screenshot.ts | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 35fc525bf..ff9e9bc0d 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -114,14 +114,13 @@ function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { } async function cropUint8Image( - image: Uint8Array, + viewer: Viewer, crop: ScreenshotCanvasViewport, ): Promise { - const blob = new Blob([image], { type: "image/png" }); const cropWidth = crop.right - crop.left; const cropHeight = crop.bottom - crop.top; const img = await createImageBitmap( - blob, + viewer.display.canvas, crop.left, crop.top, cropWidth, @@ -183,30 +182,19 @@ export class ScreenshotFromViewer extends RefCounted { } async saveScreenshot(actionState: ScreenshotActionState) { - function binaryStringToUint8Array(binaryString: string) { - const length = binaryString.length; - const bytes = new Uint8Array(length); - for (let i = 0; i < length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes; - } - - function base64ToUint8Array(base64: string) { - const binaryString = window.atob(base64); - return binaryStringToUint8Array(binaryString); - } - const { screenshot } = actionState; - const { image } = screenshot; - const fullImage = base64ToUint8Array(image); + const { imageType } = screenshot; + if (imageType !== "image/png") { + console.error("Image type is not PNG"); + return; + } const renderLocation = determineViewPanelArea(this.viewer.display.panels); try { - const croppedImage = await cropUint8Image(fullImage, renderLocation); + const croppedImage = await cropUint8Image(this.viewer, renderLocation); const filename = generateFilename( this.filename, - screenshot.width, - screenshot.height, + renderLocation.right - renderLocation.left, + renderLocation.bottom - renderLocation.top, ); downloadFileForBlob(croppedImage, filename); } catch (error) { From 0c219d3c3c681471dc4af2f2d88cc8d41db010dd Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 28 Aug 2024 18:32:36 +0200 Subject: [PATCH 13/66] fix(ui): remove SVG titles in buttons --- src/widget/icon.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/widget/icon.ts b/src/widget/icon.ts index fa74b5976..d13ba5958 100644 --- a/src/widget/icon.ts +++ b/src/widget/icon.ts @@ -28,6 +28,10 @@ export interface MakeHoverIconOptions extends MakeIconOptions { svgHover?: string; } +function removeSVGTitles(element: HTMLElement) { + element.querySelectorAll("title").forEach((title) => title.remove()); +} + export function makeHoverIcon(options: MakeHoverIconOptions): HTMLElement { const element = makeIcon(options); if (options.svgHover) { @@ -58,6 +62,12 @@ export function makeIcon(options: MakeIconOptions): HTMLElement { element.className = "neuroglancer-icon"; if (svg !== undefined) { element.innerHTML = svg; + if ( + element instanceof HTMLDivElement && + element.firstChild instanceof SVGElement + ) { + removeSVGTitles(element); + } } if (options.text !== undefined) { element.appendChild(document.createTextNode(options.text)); From eac9be375b718855c3751abe6d6603474e3ab7b9 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 29 Aug 2024 13:05:30 +0200 Subject: [PATCH 14/66] fix(ui): better icon fix --- src/widget/icon.css | 1 + src/widget/icon.ts | 10 ---------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/widget/icon.css b/src/widget/icon.css index 0e82a0a6b..6a8126333 100644 --- a/src/widget/icon.css +++ b/src/widget/icon.css @@ -42,6 +42,7 @@ stroke-width: 2; stroke-linecap: round; stroke-linejoin: round; + pointer-events: none; } .neuroglancer-icon:hover { diff --git a/src/widget/icon.ts b/src/widget/icon.ts index d13ba5958..fa74b5976 100644 --- a/src/widget/icon.ts +++ b/src/widget/icon.ts @@ -28,10 +28,6 @@ export interface MakeHoverIconOptions extends MakeIconOptions { svgHover?: string; } -function removeSVGTitles(element: HTMLElement) { - element.querySelectorAll("title").forEach((title) => title.remove()); -} - export function makeHoverIcon(options: MakeHoverIconOptions): HTMLElement { const element = makeIcon(options); if (options.svgHover) { @@ -62,12 +58,6 @@ export function makeIcon(options: MakeIconOptions): HTMLElement { element.className = "neuroglancer-icon"; if (svg !== undefined) { element.innerHTML = svg; - if ( - element instanceof HTMLDivElement && - element.firstChild instanceof SVGElement - ) { - removeSVGTitles(element); - } } if (options.text !== undefined) { element.appendChild(document.createTextNode(options.text)); From 17c8a88ce44c658d2e6d4d3000d2bd1fd80a3ee3 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 29 Aug 2024 13:46:16 +0200 Subject: [PATCH 15/66] feat: switch between buttons for save or force screenshot --- src/display_context.ts | 1 + src/ui/screenshot_menu.ts | 55 +++++++++++++++++++++++++++++++++------ src/util/screenshot.ts | 1 + 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/display_context.ts b/src/display_context.ts index 39fdf1845..4965646ad 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -412,6 +412,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { boundsGeneration = -1; inScreenshotMode = false; forceScreenshot = false; + screenshotFinished = new NullarySignal(); private framerateMonitor = new FramerateMonitor(); private continuousCameraMotionInProgress = false; diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index e96b2f9bb..013a804e8 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; @@ -24,10 +25,10 @@ export class ScreenshotDialog extends Overlay { saveButton: HTMLButtonElement; closeButton: HTMLButtonElement; forceScreenshotButton: HTMLButtonElement; + inScreenshotMode: boolean; constructor(public viewer: Viewer) { super(); - // TODO: this might be better as a menu, not a dialog. this.content.classList.add("neuroglancer-screenshot-dialog"); const closeButton = (this.closeButton = document.createElement("button")); @@ -39,12 +40,39 @@ export class ScreenshotDialog extends Overlay { nameInput.type = "text"; nameInput.placeholder = "Enter filename..."; + const saveButton = this.createSaveButton(); + const forceScreenshotButton = this.createForceScreenshotButton(); + + this.content.appendChild(closeButton); + this.content.appendChild(this.createScaleRadioButtons()); + this.content.appendChild(nameInput); + this.inScreenshotMode = this.viewer.display.inScreenshotMode; + + if (this.inScreenshotMode) { + this.content.appendChild(forceScreenshotButton); + } else { + this.content.appendChild(saveButton); + } + + this.registerDisposer( + this.viewer.display.screenshotFinished.add(() => { + this.debouncedShowSaveOrForceScreenshotButton(); + }), + ); + } + + private createSaveButton() { const saveButton = (this.saveButton = document.createElement("button")); saveButton.textContent = "Take screenshot"; saveButton.title = "Take a screenshot of the current view and save it to a png file"; - saveButton.addEventListener("click", () => this.screenshot()); + saveButton.addEventListener("click", () => { + this.screenshot(); + }); + return saveButton; + } + private createForceScreenshotButton() { const forceScreenshotButton = (this.forceScreenshotButton = document.createElement("button")); forceScreenshotButton.textContent = "Force screenshot"; @@ -53,12 +81,7 @@ export class ScreenshotDialog extends Overlay { forceScreenshotButton.addEventListener("click", () => { this.forceScreenshot(); }); - - this.content.appendChild(closeButton); - this.content.appendChild(this.createScaleRadioButtons()); - this.content.appendChild(nameInput); - this.content.appendChild(saveButton); - this.content.appendChild(forceScreenshotButton); + return forceScreenshotButton; } private createScaleRadioButtons() { @@ -85,6 +108,7 @@ export class ScreenshotDialog extends Overlay { private forceScreenshot() { this.viewer.display.forceScreenshot = true; this.viewer.display.scheduleRedraw(); + this.debouncedShowSaveOrForceScreenshotButton(); this.dispose(); } @@ -92,5 +116,20 @@ export class ScreenshotDialog extends Overlay { const filename = this.nameInput.value; this.viewer.screenshotHandler.screenshot(filename); this.viewer.display.forceScreenshot = false; + this.debouncedShowSaveOrForceScreenshotButton(); + } + + private debouncedShowSaveOrForceScreenshotButton = debounce(() => { + this.showSaveOrForceScreenshotButton(); + }, 200); + + private showSaveOrForceScreenshotButton() { + if (this.viewer.display.inScreenshotMode && !this.inScreenshotMode) { + this.inScreenshotMode = true; + this.content.replaceChild(this.forceScreenshotButton, this.saveButton); + } else if (!this.viewer.display.inScreenshotMode && this.inScreenshotMode) { + this.inScreenshotMode = false; + this.content.replaceChild(this.saveButton, this.forceScreenshotButton); + } } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index ff9e9bc0d..b436124c6 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -177,6 +177,7 @@ export class ScreenshotFromViewer extends RefCounted { resetCanvasSize() { const { viewer } = this; viewer.display.inScreenshotMode = false; + viewer.display.screenshotFinished.dispatch(); ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } From d651e5592aea567002ad1687f0812d222cbfd768 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 29 Aug 2024 13:57:01 +0200 Subject: [PATCH 16/66] refactor: remove event listener complexity from viewer --- src/ui/screenshot_menu.ts | 17 ++++++++--------- src/util/screenshot.ts | 13 ++++++++++--- src/viewer.ts | 23 ++--------------------- 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 013a804e8..19eaf26d0 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -21,15 +21,16 @@ import "#src/ui/screenshot_menu.css"; import type { Viewer } from "#src/viewer.js"; export class ScreenshotDialog extends Overlay { - nameInput: HTMLInputElement; - saveButton: HTMLButtonElement; - closeButton: HTMLButtonElement; - forceScreenshotButton: HTMLButtonElement; - inScreenshotMode: boolean; + private nameInput: HTMLInputElement; + private saveButton: HTMLButtonElement; + private closeButton: HTMLButtonElement; + private forceScreenshotButton: HTMLButtonElement; + private inScreenshotMode: boolean; constructor(public viewer: Viewer) { super(); this.content.classList.add("neuroglancer-screenshot-dialog"); + this.inScreenshotMode = this.viewer.display.inScreenshotMode; const closeButton = (this.closeButton = document.createElement("button")); closeButton.classList.add("close-button"); @@ -43,11 +44,9 @@ export class ScreenshotDialog extends Overlay { const saveButton = this.createSaveButton(); const forceScreenshotButton = this.createForceScreenshotButton(); - this.content.appendChild(closeButton); + this.content.appendChild(this.closeButton); this.content.appendChild(this.createScaleRadioButtons()); - this.content.appendChild(nameInput); - this.inScreenshotMode = this.viewer.display.inScreenshotMode; - + this.content.appendChild(this.nameInput); if (this.inScreenshotMode) { this.content.appendChild(forceScreenshotButton); } else { diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index b436124c6..ecd6ff929 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -146,6 +146,13 @@ export class ScreenshotFromViewer extends RefCounted { constructor(public viewer: Viewer) { super(); this.viewer = viewer; + this.registerDisposer( + this.viewer.screenshotActionHandler.sendScreenshotRequested.add( + (state) => { + this.saveScreenshot(state); + }, + ), + ); } screenshot(filename: string = "") { @@ -164,10 +171,10 @@ export class ScreenshotFromViewer extends RefCounted { viewer.display.canvas.height = newSize.height; } this.screenshotId++; + this.viewer.screenshotActionHandler.requestState.value = + this.screenshotId.toString(); viewer.display.inScreenshotMode = true; - if (!shouldResize) { - viewer.display.scheduleRedraw(); - } else { + if (shouldResize) { ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } diff --git a/src/viewer.ts b/src/viewer.ts index 9c3ad6f1b..49d421e77 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -491,12 +491,8 @@ export class Viewer extends RefCounted implements ViewerState { resetInitiated = new NullarySignal(); - private screenshotActionHandler = this.registerDisposer( - new ScreenshotHandler(this), - ); - public screenshotHandler = this.registerDisposer( - new ScreenshotFromViewer(this), - ); + screenshotActionHandler = this.registerDisposer(new ScreenshotHandler(this)); + screenshotHandler = this.registerDisposer(new ScreenshotFromViewer(this)); get chunkManager() { return this.dataContext.chunkManager; @@ -576,21 +572,6 @@ export class Viewer extends RefCounted implements ViewerState { this.display.applyWindowedViewportToElement(element, value); }, this.partialViewport), ); - this.registerDisposer( - this.screenshotActionHandler.sendScreenshotRequested.add((state) => { - this.screenshotHandler.saveScreenshot(state); - }), - ); - // TODO this is a bit clunky, but it works for now. - this.registerDisposer( - this.display.updateFinished.add(() => { - if (this.screenshotHandler.screenshotId >= 0) { - this.screenshotActionHandler.requestState.value = - this.screenshotHandler.screenshotId.toString(); - } - }), - ); - this.registerDisposer(() => removeFromParent(this.element)); this.dataContext = this.registerDisposer(dataContext); From 570650a972487fbcf1ffb6644ac59e45bc70d44a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 30 Aug 2024 15:24:58 +0200 Subject: [PATCH 17/66] feat: first version of screenshot statistics --- src/ui/screenshot_menu.css | 2 - src/ui/screenshot_menu.ts | 75 ++++++++++++++++++++++++++++++++++++++ src/util/screenshot.ts | 18 +++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index 79f980508..611642fd5 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -16,8 +16,6 @@ .neuroglancer-screenshot-dialog { width: 80%; - position: relative; - top: 10%; } .close-button { diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 19eaf26d0..4d47b8ebf 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -18,6 +18,7 @@ import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; +import type { StatisticsActionState } from "#src/util/screenshot.js"; import type { Viewer } from "#src/viewer.js"; export class ScreenshotDialog extends Overlay { @@ -25,6 +26,8 @@ export class ScreenshotDialog extends Overlay { private saveButton: HTMLButtonElement; private closeButton: HTMLButtonElement; private forceScreenshotButton: HTMLButtonElement; + private statisticsTable: HTMLTableElement; + private titleBar: HTMLDivElement; private inScreenshotMode: boolean; constructor(public viewer: Viewer) { super(); @@ -52,12 +55,20 @@ export class ScreenshotDialog extends Overlay { } else { this.content.appendChild(saveButton); } + this.content.appendChild(this.createStatisticsTable()); this.registerDisposer( this.viewer.display.screenshotFinished.add(() => { this.debouncedShowSaveOrForceScreenshotButton(); }), ); + this.registerDisposer( + this.viewer.screenshotActionHandler.sendStatisticsRequested.add( + (actionState) => { + this.populateStatistics(actionState); + }, + ), + ); } private createSaveButton() { @@ -104,6 +115,32 @@ export class ScreenshotDialog extends Overlay { return scaleRadioButtons; } + private createStatisticsTable() { + const titleBar = document.createElement("div"); + this.titleBar = titleBar; + titleBar.classList.add("neuroglancer-screenshot-statistics-title"); + this.content.appendChild(titleBar); + this.statisticsTable = document.createElement("table"); + this.statisticsTable.classList.add( + "neuroglancer-screenshot-statistics-table", + ); + this.statisticsTable.createTHead().insertRow().innerHTML = + "KeyValue"; + this.statisticsTable.title = "Screenshot statistics"; + + this.setTitleBarText(); + this.populateStatistics(undefined); + return titleBar; + } + + private setTitleBarText() { + const titleBarText = this.inScreenshotMode + ? "Screenshot in progress with the following statistics:" + : "Start screenshot mode to see statistics"; + this.titleBar.textContent = titleBarText; + this.titleBar.appendChild(this.statisticsTable); + } + private forceScreenshot() { this.viewer.display.forceScreenshot = true; this.viewer.display.scheduleRedraw(); @@ -118,8 +155,46 @@ export class ScreenshotDialog extends Overlay { this.debouncedShowSaveOrForceScreenshotButton(); } + private populateStatistics(actionState: StatisticsActionState | undefined) { + const nowtime = new Date().toLocaleString().replace(", ", "-"); + let statsRow; + if (actionState === undefined) { + statsRow = { + time: nowtime, + visibleChunksGpuMemory: 0, + visibleChunksTotal: 0, + visibleGpuMemory: 0, + visibleChunksDownloading: 0, + downloadLatency: 0, + }; + } else { + const total = actionState.screenshotStatistics.total; + + statsRow = { + time: nowtime, + visibleChunksGpuMemory: total.visibleChunksGpuMemory, + visibleChunksTotal: total.visibleChunksTotal, + visibleGpuMemory: total.visibleGpuMemory, + visibleChunksDownloading: total.visibleChunksDownloading, + downloadLatency: total.downloadLatency, + }; + while (this.statisticsTable.rows.length > 1) { + this.statisticsTable.deleteRow(1); + } + } + + for (const key in statsRow) { + const row = this.statisticsTable.insertRow(); + const keyCell = row.insertCell(); + keyCell.textContent = key; + const valueCell = row.insertCell(); + valueCell.textContent = String(statsRow[key as keyof typeof statsRow]); + } + } + private debouncedShowSaveOrForceScreenshotButton = debounce(() => { this.showSaveOrForceScreenshotButton(); + this.setTitleBarText(); }, 200); private showSaveOrForceScreenshotButton() { diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index ecd6ff929..2e33a0100 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -35,6 +35,24 @@ interface ScreenshotActionState { screenshot: ScreenshotResponse; } +export interface StatisticsActionState { + viewerState: any; + selectedValues: any; + screenshotStatistics: { + id: string; + chunkSources: any[]; + total: { + downloadLatency: number; + visibleChunksDownloading: number; + visibleChunksFailed: number; + visibleChunksGpuMemory: number; + visibleChunksSystemMemory: number; + visibleChunksTotal: number; + visibleGpuMemory: number; + }; + }; +} + interface ScreenshotCanvasViewport { left: number; right: number; From aed271e32c3877e44ed7a30f0eac14d5eb04d3a8 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 30 Aug 2024 15:49:44 +0200 Subject: [PATCH 18/66] feat: auto force screenshot if no updates --- src/ui/screenshot_menu.ts | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 4d47b8ebf..1046e07e5 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -21,6 +21,14 @@ import "#src/ui/screenshot_menu.css"; import type { StatisticsActionState } from "#src/util/screenshot.js"; import type { Viewer } from "#src/viewer.js"; +// Warn after 5 seconds that the screenshot is likely stuck if no change in GPU chunks +const SCREENSHOT_TIMEOUT = 5000; + +interface screenshotGpuStats { + numVisibleChunks: number; + timestamp: number; +} + export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private saveButton: HTMLButtonElement; @@ -29,6 +37,11 @@ export class ScreenshotDialog extends Overlay { private statisticsTable: HTMLTableElement; private titleBar: HTMLDivElement; private inScreenshotMode: boolean; + private gpuStats: screenshotGpuStats = { + numVisibleChunks: 0, + timestamp: 0, + }; + private lastUpdateTimestamp = 0; constructor(public viewer: Viewer) { super(); @@ -60,6 +73,7 @@ export class ScreenshotDialog extends Overlay { this.registerDisposer( this.viewer.display.screenshotFinished.add(() => { this.debouncedShowSaveOrForceScreenshotButton(); + this.dispose(); }), ); this.registerDisposer( @@ -69,6 +83,11 @@ export class ScreenshotDialog extends Overlay { }, ), ); + this.registerDisposer( + this.viewer.display.updateFinished.add(() => { + this.lastUpdateTimestamp = Date.now(); + }), + ); } private createSaveButton() { @@ -181,6 +200,13 @@ export class ScreenshotDialog extends Overlay { while (this.statisticsTable.rows.length > 1) { this.statisticsTable.deleteRow(1); } + this.checkForStuckScreenshot( + { + numVisibleChunks: total.visibleChunksGpuMemory, + timestamp: Date.now(), + }, + total.visibleChunksTotal, + ); } for (const key in statsRow) { @@ -197,6 +223,35 @@ export class ScreenshotDialog extends Overlay { this.setTitleBarText(); }, 200); + /** + * Check if the screenshot is stuck by comparing the number of visible chunks + * in the GPU with the previous number of visible chunks. If the number of + * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. + */ + private checkForStuckScreenshot( + newStats: screenshotGpuStats, + totalChunks: number, + ) { + const oldStats = this.gpuStats; + if (oldStats.timestamp === 0) { + this.gpuStats = newStats; + return; + } + if (oldStats.numVisibleChunks === newStats.numVisibleChunks) { + if ( + newStats.timestamp - oldStats.timestamp > SCREENSHOT_TIMEOUT && + Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT + ) { + console.warn( + `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${newStats.numVisibleChunks}/${totalChunks}`, + ); + this.forceScreenshotButton.click(); + } + } else { + this.gpuStats = newStats; + } + } + private showSaveOrForceScreenshotButton() { if (this.viewer.display.inScreenshotMode && !this.inScreenshotMode) { this.inScreenshotMode = true; From 3b87beeee0d0a10991e257696db65314bff43cd5 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 30 Aug 2024 19:02:20 +0200 Subject: [PATCH 19/66] refactor: simplify logic for object interaction --- src/display_context.ts | 17 ++- src/perspective_view/panel.ts | 4 + src/python_integration/screenshots.ts | 7 +- src/sliceview/panel.ts | 3 + src/ui/screenshot_menu.ts | 162 +++++++-------------- src/util/screenshot.ts | 193 ++++++++++++++++++++------ src/util/trackable_screenshot_mode.ts | 31 +++++ 7 files changed, 256 insertions(+), 161 deletions(-) create mode 100644 src/util/trackable_screenshot_mode.ts diff --git a/src/display_context.ts b/src/display_context.ts index 4965646ad..000ee46af 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -25,6 +25,11 @@ import { FramerateMonitor } from "#src/util/framerate.js"; import type { mat4 } from "#src/util/geom.js"; import { parseFixedLengthArray, verifyFloat01 } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; +import type { TrackableScreenshotModeValue } from "#src/util/trackable_screenshot_mode.js"; +import { + ScreenshotModes, + trackableScreenshotModeValue, +} from "#src/util/trackable_screenshot_mode.js"; import type { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js"; import type { GL } from "#src/webgl/context.js"; import { initializeWebGL } from "#src/webgl/context.js"; @@ -221,7 +226,7 @@ export abstract class RenderedPanel extends RefCounted { 0, clippedBottom - clippedTop, )); - if (this.context.inScreenshotMode) { + if (this.context.screenshotMode.value !== ScreenshotModes.OFF) { viewport.width = logicalWidth * screenToCanvasPixelScaleX; viewport.height = logicalHeight * screenToCanvasPixelScaleY; viewport.logicalWidth = logicalWidth * screenToCanvasPixelScaleX; @@ -307,6 +312,10 @@ export abstract class RenderedPanel extends RefCounted { return true; } + get isDataPanel() { + return false; + } + // Returns a number that determine the order in which panels are drawn. This is used by CdfPanel // to ensure it is drawn after other panels that update the histogram. // @@ -410,9 +419,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { rootRect: DOMRect | undefined; resizeGeneration = 0; boundsGeneration = -1; - inScreenshotMode = false; - forceScreenshot = false; - screenshotFinished = new NullarySignal(); + screenshotMode: TrackableScreenshotModeValue = trackableScreenshotModeValue(); private framerateMonitor = new FramerateMonitor(); private continuousCameraMotionInProgress = false; @@ -585,7 +592,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { const { resizeGeneration } = this; if (this.boundsGeneration === resizeGeneration) return; const { canvas } = this; - if (!this.inScreenshotMode) { + if (this.screenshotMode.value === ScreenshotModes.OFF) { canvas.width = canvas.offsetWidth; canvas.height = canvas.offsetHeight; } diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 1ab88d5e1..820a5c2ee 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -307,6 +307,10 @@ export class PerspectivePanel extends RenderedDataPanel { ); } + get isDataPanel() { + return true; + } + /** * If boolean value is true, sliceView is shown unconditionally, regardless of the value of * this.viewer.showSliceViews.value. diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index 54182aaa2..7dbc3bf0a 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -28,6 +28,7 @@ import { convertEndian32, Endianness } from "#src/util/endian.js"; import { verifyOptionalString } from "#src/util/json.js"; import { Signal } from "#src/util/signal.js"; import { getCachedJson } from "#src/util/trackable.js"; +import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; export class ScreenshotHandler extends RefCounted { @@ -124,12 +125,14 @@ export class ScreenshotHandler extends RefCounted { return; } const { viewer } = this; - if (!viewer.isReady() && !viewer.display.forceScreenshot) { + const forceScreenshot = + this.viewer.display.screenshotMode.value === ScreenshotModes.FORCE; + if (!viewer.isReady() && !forceScreenshot) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); return; } - if (!this.wasAlreadyVisible && !viewer.display.forceScreenshot) { + if (!this.wasAlreadyVisible && !forceScreenshot) { this.throttledSendStatistics(requestState); this.wasAlreadyVisible = true; this.debouncedMaybeSendScreenshot(); diff --git a/src/sliceview/panel.ts b/src/sliceview/panel.ts index 172ee6f25..fc801e4c8 100644 --- a/src/sliceview/panel.ts +++ b/src/sliceview/panel.ts @@ -129,6 +129,9 @@ export class SliceViewPanel extends RenderedDataPanel { get rpcId() { return this.sliceView.rpcId!; } + get isDataPanel() { + return true; + } private offscreenFramebuffer = this.registerDisposer( new FramebufferConfiguration(this.gl, { diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 1046e07e5..d391aba2a 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -19,15 +19,9 @@ import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; import type { StatisticsActionState } from "#src/util/screenshot.js"; -import type { Viewer } from "#src/viewer.js"; - -// Warn after 5 seconds that the screenshot is likely stuck if no change in GPU chunks -const SCREENSHOT_TIMEOUT = 5000; -interface screenshotGpuStats { - numVisibleChunks: number; - timestamp: number; -} +import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; +import type { Viewer } from "#src/viewer.js"; export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; @@ -36,42 +30,21 @@ export class ScreenshotDialog extends Overlay { private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; private titleBar: HTMLDivElement; - private inScreenshotMode: boolean; - private gpuStats: screenshotGpuStats = { - numVisibleChunks: 0, - timestamp: 0, - }; - private lastUpdateTimestamp = 0; + private screenshotMode: ScreenshotModes; constructor(public viewer: Viewer) { super(); this.content.classList.add("neuroglancer-screenshot-dialog"); - this.inScreenshotMode = this.viewer.display.inScreenshotMode; - - const closeButton = (this.closeButton = document.createElement("button")); - closeButton.classList.add("close-button"); - closeButton.textContent = "Close"; - closeButton.addEventListener("click", () => this.dispose()); - - const nameInput = (this.nameInput = document.createElement("input")); - nameInput.type = "text"; - nameInput.placeholder = "Enter filename..."; - - const saveButton = this.createSaveButton(); - const forceScreenshotButton = this.createForceScreenshotButton(); + this.screenshotMode = this.viewer.display.screenshotMode.value; - this.content.appendChild(this.closeButton); + this.content.appendChild(this.createCloseButton()); this.content.appendChild(this.createScaleRadioButtons()); - this.content.appendChild(this.nameInput); - if (this.inScreenshotMode) { - this.content.appendChild(forceScreenshotButton); - } else { - this.content.appendChild(saveButton); - } + this.content.appendChild(this.createNameInput()); + this.content.appendChild(this.createSaveAndForceScreenshotButtons()); this.content.appendChild(this.createStatisticsTable()); this.registerDisposer( - this.viewer.display.screenshotFinished.add(() => { + this.viewer.screenshotActionHandler.sendScreenshotRequested.add(() => { this.debouncedShowSaveOrForceScreenshotButton(); this.dispose(); }), @@ -83,11 +56,31 @@ export class ScreenshotDialog extends Overlay { }, ), ); - this.registerDisposer( - this.viewer.display.updateFinished.add(() => { - this.lastUpdateTimestamp = Date.now(); - }), - ); + this.closeButton; + } + + private createSaveAndForceScreenshotButtons() { + this.createSaveButton(); + this.createForceScreenshotButton(); + + return this.screenshotMode === ScreenshotModes.OFF + ? this.saveButton + : this.forceScreenshotButton; + } + + private createCloseButton() { + const closeButton = (this.closeButton = document.createElement("button")); + closeButton.classList.add("close-button"); + closeButton.textContent = "Close"; + closeButton.addEventListener("click", () => this.dispose()); + return closeButton; + } + + private createNameInput() { + const nameInput = (this.nameInput = document.createElement("input")); + nameInput.type = "text"; + nameInput.placeholder = "Enter filename..."; + return nameInput; } private createSaveButton() { @@ -123,12 +116,12 @@ export class ScreenshotDialog extends Overlay { input.type = "radio"; input.name = "screenshot-scale"; input.value = scale.toString(); - input.checked = scale === this.viewer.screenshotHandler.screenshotScale; + input.checked = scale === this.screenshotHandler.screenshotScale; label.appendChild(input); label.appendChild(document.createTextNode(`Scale ${scale}x`)); scaleRadioButtons.appendChild(label); input.addEventListener("change", () => { - this.viewer.screenshotHandler.screenshotScale = scale; + this.screenshotHandler.screenshotScale = scale; }); } return scaleRadioButtons; @@ -153,61 +146,32 @@ export class ScreenshotDialog extends Overlay { } private setTitleBarText() { - const titleBarText = this.inScreenshotMode - ? "Screenshot in progress with the following statistics:" - : "Start screenshot mode to see statistics"; + const titleBarText = + this.screenshotMode !== ScreenshotModes.OFF + ? "Screenshot in progress with the following statistics:" + : "Start screenshot mode to see statistics"; this.titleBar.textContent = titleBarText; this.titleBar.appendChild(this.statisticsTable); } private forceScreenshot() { - this.viewer.display.forceScreenshot = true; - this.viewer.display.scheduleRedraw(); + this.screenshotHandler.forceScreenshot(); this.debouncedShowSaveOrForceScreenshotButton(); - this.dispose(); } private screenshot() { const filename = this.nameInput.value; - this.viewer.screenshotHandler.screenshot(filename); - this.viewer.display.forceScreenshot = false; + this.screenshotHandler.screenshot(filename); this.debouncedShowSaveOrForceScreenshotButton(); } private populateStatistics(actionState: StatisticsActionState | undefined) { - const nowtime = new Date().toLocaleString().replace(", ", "-"); - let statsRow; - if (actionState === undefined) { - statsRow = { - time: nowtime, - visibleChunksGpuMemory: 0, - visibleChunksTotal: 0, - visibleGpuMemory: 0, - visibleChunksDownloading: 0, - downloadLatency: 0, - }; - } else { - const total = actionState.screenshotStatistics.total; - - statsRow = { - time: nowtime, - visibleChunksGpuMemory: total.visibleChunksGpuMemory, - visibleChunksTotal: total.visibleChunksTotal, - visibleGpuMemory: total.visibleGpuMemory, - visibleChunksDownloading: total.visibleChunksDownloading, - downloadLatency: total.downloadLatency, - }; + if (actionState !== undefined) { while (this.statisticsTable.rows.length > 1) { this.statisticsTable.deleteRow(1); } - this.checkForStuckScreenshot( - { - numVisibleChunks: total.visibleChunksGpuMemory, - timestamp: Date.now(), - }, - total.visibleChunksTotal, - ); } + const statsRow = this.screenshotHandler.parseStatistics(actionState); for (const key in statsRow) { const row = this.statisticsTable.insertRow(); @@ -223,42 +187,20 @@ export class ScreenshotDialog extends Overlay { this.setTitleBarText(); }, 200); - /** - * Check if the screenshot is stuck by comparing the number of visible chunks - * in the GPU with the previous number of visible chunks. If the number of - * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. - */ - private checkForStuckScreenshot( - newStats: screenshotGpuStats, - totalChunks: number, - ) { - const oldStats = this.gpuStats; - if (oldStats.timestamp === 0) { - this.gpuStats = newStats; + private showSaveOrForceScreenshotButton() { + // Check to see if the global state matches the current state of the dialog + if (this.viewer.display.screenshotMode.value === this.screenshotMode) { return; } - if (oldStats.numVisibleChunks === newStats.numVisibleChunks) { - if ( - newStats.timestamp - oldStats.timestamp > SCREENSHOT_TIMEOUT && - Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT - ) { - console.warn( - `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${newStats.numVisibleChunks}/${totalChunks}`, - ); - this.forceScreenshotButton.click(); - } + if (this.viewer.display.screenshotMode.value === ScreenshotModes.OFF) { + this.content.replaceChild(this.saveButton, this.forceScreenshotButton); } else { - this.gpuStats = newStats; + this.content.replaceChild(this.forceScreenshotButton, this.saveButton); } + this.screenshotMode = this.viewer.display.screenshotMode.value; } - private showSaveOrForceScreenshotButton() { - if (this.viewer.display.inScreenshotMode && !this.inScreenshotMode) { - this.inScreenshotMode = true; - this.content.replaceChild(this.forceScreenshotButton, this.saveButton); - } else if (!this.viewer.display.inScreenshotMode && this.inScreenshotMode) { - this.inScreenshotMode = false; - this.content.replaceChild(this.saveButton, this.forceScreenshotButton); - } + get screenshotHandler() { + return this.viewer.screenshotHandler; } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 2e33a0100..e6033e6b0 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -15,24 +15,29 @@ */ import type { RenderedPanel } from "#src/display_context.js"; -import { PerspectivePanel } from "#src/perspective_view/panel.js"; -import { SliceViewPanel } from "#src/sliceview/panel.js"; import { RefCounted } from "#src/util/disposable.js"; +import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; -interface ScreenshotResponse { - id: string; - image: string; - imageType: string; - depthData: string | undefined; - width: number; - height: number; +// Warn after 5 seconds that the screenshot is likely stuck if no change in GPU chunks +const SCREENSHOT_TIMEOUT = 5000; + +interface screenshotGpuStats { + numVisibleChunks: number; + timestamp: number; } interface ScreenshotActionState { viewerState: any; selectedValues: any; - screenshot: ScreenshotResponse; + screenshot: { + id: string; + image: string; + imageType: string; + depthData: string | undefined; + width: number; + height: number; + }; } export interface StatisticsActionState { @@ -72,23 +77,6 @@ function downloadFileForBlob(blob: Blob, filename: string) { } } -function generateFilename( - inputFilename: string, - width: number, - height: number, -): string { - let filename = inputFilename; - if (filename.length === 0) { - let nowtime = new Date().toLocaleString(); - nowtime = nowtime.replace(", ", "-"); - filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; - } - if (!filename.endsWith(".png")) { - filename += ".png"; - } - return filename; -} - function determineViewPanelArea( panels: Set, ): ScreenshotCanvasViewport { @@ -99,12 +87,7 @@ function determineViewPanelArea( bottom: Number.NEGATIVE_INFINITY, }; for (const panel of panels) { - if ( - !(panel instanceof SliceViewPanel) && - !(panel instanceof PerspectivePanel) - ) { - continue; - } + if (!panel.isDataPanel) continue; const viewport = panel.renderViewport; const { width, height } = viewport; const left = panel.canvasRelativeClippedLeft; @@ -131,7 +114,7 @@ function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { }); } -async function cropUint8Image( +async function cropViewsFromViewer( viewer: Viewer, crop: ScreenshotCanvasViewport, ): Promise { @@ -160,6 +143,11 @@ export class ScreenshotFromViewer extends RefCounted { public screenshotId: number = -1; public screenshotScale: number = 1; private filename: string = ""; + private gpuStats: screenshotGpuStats = { + numVisibleChunks: 0, + timestamp: 0, + }; + private lastUpdateTimestamp = 0; constructor(public viewer: Viewer) { super(); @@ -171,11 +159,38 @@ export class ScreenshotFromViewer extends RefCounted { }, ), ); + this.registerDisposer( + this.viewer.display.updateFinished.add(() => { + this.lastUpdateTimestamp = Date.now(); + }), + ); + this.registerDisposer( + this.viewer.screenshotActionHandler.sendStatisticsRequested.add( + (actionState) => { + this.checkForStuckScreenshot(actionState); + }, + ), + ); + this.registerDisposer( + this.viewer.display.screenshotMode.changed.add(() => { + this.handleScreenshotModeChange(); + }), + ); } screenshot(filename: string = "") { + this.filename = filename; + this.viewer.display.screenshotMode.value = ScreenshotModes.ON; + } + + private startScreenshot() { const { viewer } = this; const shouldResize = this.screenshotScale !== 1; + this.lastUpdateTimestamp = Date.now(); + this.gpuStats = { + numVisibleChunks: 0, + timestamp: 0, + }; if (shouldResize) { const oldSize = { width: viewer.display.canvas.width, @@ -191,18 +206,14 @@ export class ScreenshotFromViewer extends RefCounted { this.screenshotId++; this.viewer.screenshotActionHandler.requestState.value = this.screenshotId.toString(); - viewer.display.inScreenshotMode = true; if (shouldResize) { ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } - this.filename = filename; } resetCanvasSize() { const { viewer } = this; - viewer.display.inScreenshotMode = false; - viewer.display.screenshotFinished.dispatch(); ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } @@ -212,21 +223,115 @@ export class ScreenshotFromViewer extends RefCounted { const { imageType } = screenshot; if (imageType !== "image/png") { console.error("Image type is not PNG"); + this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; return; } - const renderLocation = determineViewPanelArea(this.viewer.display.panels); + const renderingPanelArea = determineViewPanelArea( + this.viewer.display.panels, + ); try { - const croppedImage = await cropUint8Image(this.viewer, renderLocation); - const filename = generateFilename( - this.filename, - renderLocation.right - renderLocation.left, - renderLocation.bottom - renderLocation.top, + const croppedImage = await cropViewsFromViewer( + this.viewer, + renderingPanelArea, + ); + const filename = this.generateFilename( + renderingPanelArea.right - renderingPanelArea.left, + renderingPanelArea.bottom - renderingPanelArea.top, ); downloadFileForBlob(croppedImage, filename); } catch (error) { console.error(error); } finally { + this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; + } + } + + /** + * Check if the screenshot is stuck by comparing the number of visible chunks + * in the GPU with the previous number of visible chunks. If the number of + * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. + */ + private checkForStuckScreenshot(actionState: StatisticsActionState) { + const total = actionState.screenshotStatistics.total; + const newStats = { + numVisibleChunks: total.visibleChunksGpuMemory, + timestamp: Date.now(), + }; + const oldStats = this.gpuStats; + if (oldStats.timestamp === 0) { + this.gpuStats = newStats; + return; + } + if (oldStats.numVisibleChunks === newStats.numVisibleChunks) { + if ( + newStats.timestamp - oldStats.timestamp > SCREENSHOT_TIMEOUT && + Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT + ) { + const totalChunks = total.visibleChunksTotal; + console.warn( + `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${newStats.numVisibleChunks}/${totalChunks}`, + ); + this.forceScreenshot(); + } + } else { + this.gpuStats = newStats; + } + } + + parseStatistics(actionState: StatisticsActionState | undefined) { + const nowtime = new Date().toLocaleString().replace(", ", "-"); + let statsRow; + if (actionState === undefined) { + statsRow = { + time: nowtime, + visibleChunksGpuMemory: 0, + visibleChunksTotal: 0, + visibleGpuMemory: 0, + visibleChunksDownloading: 0, + downloadLatency: 0, + }; + } else { + const total = actionState.screenshotStatistics.total; + + statsRow = { + time: nowtime, + visibleChunksGpuMemory: total.visibleChunksGpuMemory, + visibleChunksTotal: total.visibleChunksTotal, + visibleGpuMemory: total.visibleGpuMemory, + visibleChunksDownloading: total.visibleChunksDownloading, + downloadLatency: total.downloadLatency, + }; + } + return statsRow; + } + + forceScreenshot() { + this.viewer.display.screenshotMode.value = ScreenshotModes.FORCE; + } + + generateFilename(width: number, height: number): string { + let filename = this.filename; + if (filename.length === 0) { + let nowtime = new Date().toLocaleString(); + nowtime = nowtime.replace(", ", "-"); + filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; + } + if (!filename.endsWith(".png")) { + filename += ".png"; + } + return filename; + } + + handleScreenshotModeChange() { + const { viewer } = this; + const { display } = viewer; + const { screenshotMode } = display; + if (screenshotMode.value === ScreenshotModes.OFF) { this.resetCanvasSize(); + } else if (screenshotMode.value === ScreenshotModes.FORCE) { + display.scheduleRedraw(); + } else if (screenshotMode.value === ScreenshotModes.ON) { + this.startScreenshot(); } } } diff --git a/src/util/trackable_screenshot_mode.ts b/src/util/trackable_screenshot_mode.ts new file mode 100644 index 000000000..bfb848401 --- /dev/null +++ b/src/util/trackable_screenshot_mode.ts @@ -0,0 +1,31 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use viewer file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TrackableEnum } from "#src/util/trackable_enum.js"; + +export enum ScreenshotModes { + OFF = 0, // Default mode + ON = 1, // Screenshot modek + FORCE = 2, // Force screenshot mode - used when the screenshot is stuck +} + +export type TrackableScreenshotModeValue = TrackableEnum; + +export function trackableScreenshotModeValue( + initialValue = ScreenshotModes.OFF, +) { + return new TrackableEnum(ScreenshotModes, initialValue); +} From e39740ac0c6cfce4441abb4f600c98f09994f148 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 2 Sep 2024 17:05:38 +0200 Subject: [PATCH 20/66] refactor: clean up screenshot menu --- src/ui/screenshot_menu.css | 2 +- src/ui/screenshot_menu.ts | 125 ++++++++++++++++++------------------- 2 files changed, 63 insertions(+), 64 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index 611642fd5..acf9fb4de 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -18,7 +18,7 @@ width: 80%; } -.close-button { +.neuroglancer-screenshot-close-button { position: absolute; right: 15px; } diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index d391aba2a..f358153c4 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -33,16 +33,35 @@ export class ScreenshotDialog extends Overlay { private screenshotMode: ScreenshotModes; constructor(public viewer: Viewer) { super(); + this.screenshotMode = this.viewer.display.screenshotMode.value; + + this.initializeUI(); + this.setupEventListeners(); + } + private initializeUI() { this.content.classList.add("neuroglancer-screenshot-dialog"); - this.screenshotMode = this.viewer.display.screenshotMode.value; - this.content.appendChild(this.createCloseButton()); + this.closeButton = this.createButton( + "Close", + () => this.dispose(), + "neuroglancer-screenshot-close-button", + ); + this.saveButton = this.createButton("Take screenshot", () => + this.screenshot(), + ); + this.forceScreenshotButton = this.createButton("Force screenshot", () => + this.forceScreenshot(), + ); + + this.content.appendChild(this.closeButton); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(this.createNameInput()); - this.content.appendChild(this.createSaveAndForceScreenshotButtons()); + this.content.appendChild(this.modeDependentScreenshotButton); this.content.appendChild(this.createStatisticsTable()); + } + private setupEventListeners() { this.registerDisposer( this.viewer.screenshotActionHandler.sendScreenshotRequested.add(() => { this.debouncedShowSaveOrForceScreenshotButton(); @@ -56,82 +75,64 @@ export class ScreenshotDialog extends Overlay { }, ), ); - this.closeButton; - } - - private createSaveAndForceScreenshotButtons() { - this.createSaveButton(); - this.createForceScreenshotButton(); - - return this.screenshotMode === ScreenshotModes.OFF - ? this.saveButton - : this.forceScreenshotButton; - } - - private createCloseButton() { - const closeButton = (this.closeButton = document.createElement("button")); - closeButton.classList.add("close-button"); - closeButton.textContent = "Close"; - closeButton.addEventListener("click", () => this.dispose()); - return closeButton; } - private createNameInput() { - const nameInput = (this.nameInput = document.createElement("input")); + private createNameInput(): HTMLInputElement { + const nameInput = document.createElement("input"); nameInput.type = "text"; nameInput.placeholder = "Enter filename..."; - return nameInput; + return (this.nameInput = nameInput); } - private createSaveButton() { - const saveButton = (this.saveButton = document.createElement("button")); - saveButton.textContent = "Take screenshot"; - saveButton.title = - "Take a screenshot of the current view and save it to a png file"; - saveButton.addEventListener("click", () => { - this.screenshot(); - }); - return saveButton; + private get modeDependentScreenshotButton() { + return this.screenshotMode === ScreenshotModes.OFF + ? this.saveButton + : this.forceScreenshotButton; } - private createForceScreenshotButton() { - const forceScreenshotButton = (this.forceScreenshotButton = - document.createElement("button")); - forceScreenshotButton.textContent = "Force screenshot"; - forceScreenshotButton.title = - "Force a screenshot of the current view and save it to a png file"; - forceScreenshotButton.addEventListener("click", () => { - this.forceScreenshot(); - }); - return forceScreenshotButton; + private createButton( + text: string, + onClick: () => void, + cssClass: string = "", + ): HTMLButtonElement { + const button = document.createElement("button"); + button.textContent = text; + button.classList.add("neuroglancer-screenshot-button"); + if (cssClass) button.classList.add(cssClass); + button.addEventListener("click", onClick); + return button; } private createScaleRadioButtons() { const scaleRadioButtons = document.createElement("div"); scaleRadioButtons.classList.add("scale-radio-buttons"); + const scales = [1, 2, 4]; - for (const scale of scales) { + scales.forEach((scale) => { const label = document.createElement("label"); const input = document.createElement("input"); + input.type = "radio"; input.name = "screenshot-scale"; input.value = scale.toString(); input.checked = scale === this.screenshotHandler.screenshotScale; + input.classList.add("neuroglancer-screenshot-scale-radio"); + label.appendChild(input); label.appendChild(document.createTextNode(`Scale ${scale}x`)); scaleRadioButtons.appendChild(label); + input.addEventListener("change", () => { this.screenshotHandler.screenshotScale = scale; }); - } + }); return scaleRadioButtons; } private createStatisticsTable() { - const titleBar = document.createElement("div"); - this.titleBar = titleBar; - titleBar.classList.add("neuroglancer-screenshot-statistics-title"); - this.content.appendChild(titleBar); + this.titleBar = document.createElement("div"); + this.titleBar.classList.add("neuroglancer-screenshot-statistics-title"); + this.statisticsTable = document.createElement("table"); this.statisticsTable.classList.add( "neuroglancer-screenshot-statistics-table", @@ -142,14 +143,14 @@ export class ScreenshotDialog extends Overlay { this.setTitleBarText(); this.populateStatistics(undefined); - return titleBar; + return this.titleBar; } private setTitleBarText() { const titleBarText = - this.screenshotMode !== ScreenshotModes.OFF - ? "Screenshot in progress with the following statistics:" - : "Start screenshot mode to see statistics"; + this.screenshotMode === ScreenshotModes.OFF + ? "Start screenshot mode to see statistics" + : "Screenshot in progress with the following statistics:"; this.titleBar.textContent = titleBarText; this.titleBar.appendChild(this.statisticsTable); } @@ -176,8 +177,8 @@ export class ScreenshotDialog extends Overlay { for (const key in statsRow) { const row = this.statisticsTable.insertRow(); const keyCell = row.insertCell(); - keyCell.textContent = key; const valueCell = row.insertCell(); + keyCell.textContent = key; valueCell.textContent = String(statsRow[key as keyof typeof statsRow]); } } @@ -188,16 +189,14 @@ export class ScreenshotDialog extends Overlay { }, 200); private showSaveOrForceScreenshotButton() { - // Check to see if the global state matches the current state of the dialog - if (this.viewer.display.screenshotMode.value === this.screenshotMode) { - return; - } - if (this.viewer.display.screenshotMode.value === ScreenshotModes.OFF) { - this.content.replaceChild(this.saveButton, this.forceScreenshotButton); - } else { - this.content.replaceChild(this.forceScreenshotButton, this.saveButton); + if (this.viewer.display.screenshotMode.value !== this.screenshotMode) { + if (this.viewer.display.screenshotMode.value === ScreenshotModes.OFF) { + this.content.replaceChild(this.saveButton, this.forceScreenshotButton); + } else { + this.content.replaceChild(this.forceScreenshotButton, this.saveButton); + } + this.screenshotMode = this.viewer.display.screenshotMode.value; } - this.screenshotMode = this.viewer.display.screenshotMode.value; } get screenshotHandler() { From 3d2804b14f59d3346e05b42b690c65a90554adcf Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 2 Sep 2024 18:14:06 +0200 Subject: [PATCH 21/66] feat: add minimal CSS styling for screenshot --- src/ui/screenshot_menu.css | 53 ++++++++++++++++++++++++++++-- src/ui/screenshot_menu.ts | 66 +++++++++++++++++++++++++++----------- 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index acf9fb4de..cfc73f8a4 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -14,11 +14,60 @@ * limitations under the License. */ - .neuroglancer-screenshot-dialog { - width: 80%; + +.neuroglancer-screenshot-dialog{ + width: 60%; +} + +.neuroglancer-screenshot-scale-radio { + display: inline-block; + width: 20px; + margin-right: -2px; +} + +.neuroglancer-screenshot-filename-and-buttons { + margin-bottom: 5px; +} + +.neuroglancer-screenshot-name-input { + width: 50%; + margin-right: 10px; + border: 1px solid #ccc; +} + +.neuroglancer-screenshot-button { + cursor: pointer; } .neuroglancer-screenshot-close-button { position: absolute; right: 15px; } + +.neuroglancer-screenshot-statistics-title { + margin-top: 20px; +} + +.neuroglancer-screenshot-statistics-table { + width: 100%; + border-collapse: collapse; + margin-top: 5px; +} + +.neuroglancer-screenshot-statistics-table th, +.neuroglancer-screenshot-statistics-table td { + padding: 12px 15px; + text-align: left; + border-bottom: 1px solid #ddd; +} + +.neuroglancer-screenshot-statistics-table th { + background-color: #f8f8f8; + font-weight: bold; + color: #555; +} + +.neuroglancer-screenshot-statistics-table td { + background-color: #fff; + color: #333; +} diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index f358153c4..869c43d96 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -29,7 +29,8 @@ export class ScreenshotDialog extends Overlay { private closeButton: HTMLButtonElement; private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; - private titleBar: HTMLDivElement; + private statisticsContainer: HTMLDivElement; + private filenameAndButtonsContainer: HTMLDivElement; private screenshotMode: ScreenshotModes; constructor(public viewer: Viewer) { super(); @@ -53,11 +54,18 @@ export class ScreenshotDialog extends Overlay { this.forceScreenshotButton = this.createButton("Force screenshot", () => this.forceScreenshot(), ); + this.filenameAndButtonsContainer = document.createElement("div"); + this.filenameAndButtonsContainer.classList.add( + "neuroglancer-screenshot-filename-and-buttons", + ); + this.filenameAndButtonsContainer.appendChild(this.createNameInput()); + this.filenameAndButtonsContainer.appendChild( + this.getScreenshotButtonBasedOnMode, + ); this.content.appendChild(this.closeButton); + this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); - this.content.appendChild(this.createNameInput()); - this.content.appendChild(this.modeDependentScreenshotButton); this.content.appendChild(this.createStatisticsTable()); } @@ -81,10 +89,11 @@ export class ScreenshotDialog extends Overlay { const nameInput = document.createElement("input"); nameInput.type = "text"; nameInput.placeholder = "Enter filename..."; + nameInput.classList.add("neuroglancer-screenshot-name-input"); return (this.nameInput = nameInput); } - private get modeDependentScreenshotButton() { + private get getScreenshotButtonBasedOnMode() { return this.screenshotMode === ScreenshotModes.OFF ? this.saveButton : this.forceScreenshotButton; @@ -104,8 +113,12 @@ export class ScreenshotDialog extends Overlay { } private createScaleRadioButtons() { - const scaleRadioButtons = document.createElement("div"); - scaleRadioButtons.classList.add("scale-radio-buttons"); + const scaleMenu = document.createElement("div"); + scaleMenu.classList.add("neuroglancer-screenshot-scale-menu"); + + const scaleLabel = document.createElement("label"); + scaleLabel.textContent = "Screenshot scale factor:"; + scaleMenu.appendChild(scaleLabel); const scales = [1, 2, 4]; scales.forEach((scale) => { @@ -119,40 +132,49 @@ export class ScreenshotDialog extends Overlay { input.classList.add("neuroglancer-screenshot-scale-radio"); label.appendChild(input); - label.appendChild(document.createTextNode(`Scale ${scale}x`)); - scaleRadioButtons.appendChild(label); + label.appendChild(document.createTextNode(`${scale}x`)); + + scaleMenu.appendChild(label); input.addEventListener("change", () => { this.screenshotHandler.screenshotScale = scale; }); }); - return scaleRadioButtons; + return scaleMenu; } private createStatisticsTable() { - this.titleBar = document.createElement("div"); - this.titleBar.classList.add("neuroglancer-screenshot-statistics-title"); + this.statisticsContainer = document.createElement("div"); + this.statisticsContainer.classList.add( + "neuroglancer-screenshot-statistics-title", + ); this.statisticsTable = document.createElement("table"); this.statisticsTable.classList.add( "neuroglancer-screenshot-statistics-table", ); - this.statisticsTable.createTHead().insertRow().innerHTML = - "KeyValue"; this.statisticsTable.title = "Screenshot statistics"; + const headerRow = this.statisticsTable.createTHead().insertRow(); + const keyHeader = document.createElement("th"); + keyHeader.textContent = "Key"; + headerRow.appendChild(keyHeader); + const valueHeader = document.createElement("th"); + valueHeader.textContent = "Value"; + headerRow.appendChild(valueHeader); + this.setTitleBarText(); this.populateStatistics(undefined); - return this.titleBar; + return this.statisticsContainer; } private setTitleBarText() { const titleBarText = this.screenshotMode === ScreenshotModes.OFF - ? "Start screenshot mode to see statistics" + ? "Start a screenshot to update statistics:" : "Screenshot in progress with the following statistics:"; - this.titleBar.textContent = titleBarText; - this.titleBar.appendChild(this.statisticsTable); + this.statisticsContainer.textContent = titleBarText; + this.statisticsContainer.appendChild(this.statisticsTable); } private forceScreenshot() { @@ -191,9 +213,15 @@ export class ScreenshotDialog extends Overlay { private showSaveOrForceScreenshotButton() { if (this.viewer.display.screenshotMode.value !== this.screenshotMode) { if (this.viewer.display.screenshotMode.value === ScreenshotModes.OFF) { - this.content.replaceChild(this.saveButton, this.forceScreenshotButton); + this.filenameAndButtonsContainer.replaceChild( + this.saveButton, + this.forceScreenshotButton, + ); } else { - this.content.replaceChild(this.forceScreenshotButton, this.saveButton); + this.filenameAndButtonsContainer.replaceChild( + this.forceScreenshotButton, + this.saveButton, + ); } this.screenshotMode = this.viewer.display.screenshotMode.value; } From 53b02e52b5ff733e168b794e00cc01ae8630ec54 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 2 Sep 2024 18:34:58 +0200 Subject: [PATCH 22/66] feat(ui): only show stats while in progress --- src/ui/screenshot_menu.ts | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 869c43d96..871d98b2c 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -72,7 +72,7 @@ export class ScreenshotDialog extends Overlay { private setupEventListeners() { this.registerDisposer( this.viewer.screenshotActionHandler.sendScreenshotRequested.add(() => { - this.debouncedShowSaveOrForceScreenshotButton(); + this.debouncedUpdateUIElements(); this.dispose(); }), ); @@ -99,6 +99,14 @@ export class ScreenshotDialog extends Overlay { : this.forceScreenshotButton; } + private updateStatisticsTableDisplayBasedOnMode() { + if (this.screenshotMode === ScreenshotModes.OFF) { + this.statisticsContainer.style.display = "none"; + } else { + this.statisticsContainer.style.display = "block"; + } + } + private createButton( text: string, onClick: () => void, @@ -148,6 +156,9 @@ export class ScreenshotDialog extends Overlay { this.statisticsContainer.classList.add( "neuroglancer-screenshot-statistics-title", ); + const titleBarText = + "Screenshot in progress with the following statistics:"; + this.statisticsContainer.textContent = titleBarText; this.statisticsTable = document.createElement("table"); this.statisticsTable.classList.add( @@ -163,29 +174,21 @@ export class ScreenshotDialog extends Overlay { valueHeader.textContent = "Value"; headerRow.appendChild(valueHeader); - this.setTitleBarText(); this.populateStatistics(undefined); - return this.statisticsContainer; - } - - private setTitleBarText() { - const titleBarText = - this.screenshotMode === ScreenshotModes.OFF - ? "Start a screenshot to update statistics:" - : "Screenshot in progress with the following statistics:"; - this.statisticsContainer.textContent = titleBarText; + this.updateStatisticsTableDisplayBasedOnMode(); this.statisticsContainer.appendChild(this.statisticsTable); + return this.statisticsContainer; } private forceScreenshot() { this.screenshotHandler.forceScreenshot(); - this.debouncedShowSaveOrForceScreenshotButton(); + this.debouncedUpdateUIElements(); } private screenshot() { const filename = this.nameInput.value; this.screenshotHandler.screenshot(filename); - this.debouncedShowSaveOrForceScreenshotButton(); + this.debouncedUpdateUIElements(); } private populateStatistics(actionState: StatisticsActionState | undefined) { @@ -205,9 +208,9 @@ export class ScreenshotDialog extends Overlay { } } - private debouncedShowSaveOrForceScreenshotButton = debounce(() => { + private debouncedUpdateUIElements = debounce(() => { this.showSaveOrForceScreenshotButton(); - this.setTitleBarText(); + this.updateStatisticsTableDisplayBasedOnMode(); }, 200); private showSaveOrForceScreenshotButton() { From 51e01c3ace9dff302b840c385390a09eb3dcc647 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 2 Sep 2024 19:18:56 +0200 Subject: [PATCH 23/66] feat(ui): improve screenshot stats --- src/ui/screenshot_menu.css | 2 +- src/ui/screenshot_menu.ts | 17 +++++++++++------ src/util/screenshot.ts | 24 +++++++++++++----------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index cfc73f8a4..41c8a9ed2 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -45,7 +45,7 @@ } .neuroglancer-screenshot-statistics-title { - margin-top: 20px; + margin-top: 5px; } .neuroglancer-screenshot-statistics-table { diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 871d98b2c..6adddfe5e 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -23,6 +23,13 @@ import type { StatisticsActionState } from "#src/util/screenshot.js"; import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; +const friendlyNameMap = { + time: "Current time", + visibleChunksGpuMemory: "Number of loaded chunks", + visibleGpuMemory: "Visible chunk GPU memory usage", + visibleChunksDownloading: "Number of downloading chunks", +}; + export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private saveButton: HTMLButtonElement; @@ -156,9 +163,6 @@ export class ScreenshotDialog extends Overlay { this.statisticsContainer.classList.add( "neuroglancer-screenshot-statistics-title", ); - const titleBarText = - "Screenshot in progress with the following statistics:"; - this.statisticsContainer.textContent = titleBarText; this.statisticsTable = document.createElement("table"); this.statisticsTable.classList.add( @@ -168,10 +172,10 @@ export class ScreenshotDialog extends Overlay { const headerRow = this.statisticsTable.createTHead().insertRow(); const keyHeader = document.createElement("th"); - keyHeader.textContent = "Key"; + keyHeader.textContent = "Screenshot in progress..."; headerRow.appendChild(keyHeader); const valueHeader = document.createElement("th"); - valueHeader.textContent = "Value"; + valueHeader.textContent = ""; headerRow.appendChild(valueHeader); this.populateStatistics(undefined); @@ -203,7 +207,8 @@ export class ScreenshotDialog extends Overlay { const row = this.statisticsTable.insertRow(); const keyCell = row.insertCell(); const valueCell = row.insertCell(); - keyCell.textContent = key; + keyCell.textContent = + friendlyNameMap[key as keyof typeof friendlyNameMap]; valueCell.textContent = String(statsRow[key as keyof typeof statsRow]); } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index e6033e6b0..7249e759f 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -279,27 +279,29 @@ export class ScreenshotFromViewer extends RefCounted { } parseStatistics(actionState: StatisticsActionState | undefined) { - const nowtime = new Date().toLocaleString().replace(", ", "-"); + const nowtime = new Date().toLocaleTimeString(); let statsRow; if (actionState === undefined) { statsRow = { time: nowtime, - visibleChunksGpuMemory: 0, - visibleChunksTotal: 0, - visibleGpuMemory: 0, - visibleChunksDownloading: 0, - downloadLatency: 0, + visibleChunksGpuMemory: "", + visibleGpuMemory: "", + visibleChunksDownloading: "", }; } else { const total = actionState.screenshotStatistics.total; + const percentLoaded = + (100 * total.visibleChunksGpuMemory) / total.visibleChunksTotal; + const percentGpuUsage = + (100 * total.visibleGpuMemory) / + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value; + const gpuMemoryUsageInMB = total.visibleGpuMemory / 1024 / 1024; statsRow = { time: nowtime, - visibleChunksGpuMemory: total.visibleChunksGpuMemory, - visibleChunksTotal: total.visibleChunksTotal, - visibleGpuMemory: total.visibleGpuMemory, - visibleChunksDownloading: total.visibleChunksDownloading, - downloadLatency: total.downloadLatency, + visibleChunksGpuMemory: `${total.visibleChunksGpuMemory} out of ${total.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, + visibleGpuMemory: `${gpuMemoryUsageInMB}Mb (${percentGpuUsage.toFixed(2)}% of total)`, + visibleChunksDownloading: `${total.visibleChunksDownloading} at ${total.downloadLatency}ms`, }; } return statsRow; From 24b04a96729d859efbc52a957eb6fb9b88b94c9e Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 11:44:18 +0200 Subject: [PATCH 24/66] feat: hide screenshot controls when not in use and update stats with menu closed --- src/ui/screenshot_menu.ts | 74 ++++++++++++++++++---------------- src/util/screenshot.ts | 83 +++++++++++++++++++++++++++------------ 2 files changed, 98 insertions(+), 59 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 6adddfe5e..321bf346e 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -24,10 +24,9 @@ import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; const friendlyNameMap = { - time: "Current time", - visibleChunksGpuMemory: "Number of loaded chunks", - visibleGpuMemory: "Visible chunk GPU memory usage", - visibleChunksDownloading: "Number of downloading chunks", + chunkUsageDescription: "Number of loaded chunks", + gpuMemoryUsageDescription: "Visible chunk GPU memory usage", + downloadSpeedDescription: "Number of downloading chunks", }; export class ScreenshotDialog extends Overlay { @@ -37,6 +36,7 @@ export class ScreenshotDialog extends Overlay { private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; private statisticsContainer: HTMLDivElement; + private scaleSelectContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; private screenshotMode: ScreenshotModes; constructor(public viewer: Viewer) { @@ -61,19 +61,20 @@ export class ScreenshotDialog extends Overlay { this.forceScreenshotButton = this.createButton("Force screenshot", () => this.forceScreenshot(), ); + this.forceScreenshotButton.title = + "Force a screenshot even if the viewer is not ready"; this.filenameAndButtonsContainer = document.createElement("div"); this.filenameAndButtonsContainer.classList.add( "neuroglancer-screenshot-filename-and-buttons", ); this.filenameAndButtonsContainer.appendChild(this.createNameInput()); - this.filenameAndButtonsContainer.appendChild( - this.getScreenshotButtonBasedOnMode, - ); + this.filenameAndButtonsContainer.appendChild(this.saveButton); this.content.appendChild(this.closeButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(this.createStatisticsTable()); + this.updateSetupUIVisibility(); } private setupEventListeners() { @@ -95,17 +96,11 @@ export class ScreenshotDialog extends Overlay { private createNameInput(): HTMLInputElement { const nameInput = document.createElement("input"); nameInput.type = "text"; - nameInput.placeholder = "Enter filename..."; + nameInput.placeholder = "Enter optional filename..."; nameInput.classList.add("neuroglancer-screenshot-name-input"); return (this.nameInput = nameInput); } - private get getScreenshotButtonBasedOnMode() { - return this.screenshotMode === ScreenshotModes.OFF - ? this.saveButton - : this.forceScreenshotButton; - } - private updateStatisticsTableDisplayBasedOnMode() { if (this.screenshotMode === ScreenshotModes.OFF) { this.statisticsContainer.style.display = "none"; @@ -128,7 +123,8 @@ export class ScreenshotDialog extends Overlay { } private createScaleRadioButtons() { - const scaleMenu = document.createElement("div"); + const scaleMenu = (this.scaleSelectContainer = + document.createElement("div")); scaleMenu.classList.add("neuroglancer-screenshot-scale-menu"); const scaleLabel = document.createElement("label"); @@ -163,6 +159,7 @@ export class ScreenshotDialog extends Overlay { this.statisticsContainer.classList.add( "neuroglancer-screenshot-statistics-title", ); + this.statisticsContainer.appendChild(this.forceScreenshotButton); this.statisticsTable = document.createElement("table"); this.statisticsTable.classList.add( @@ -172,13 +169,13 @@ export class ScreenshotDialog extends Overlay { const headerRow = this.statisticsTable.createTHead().insertRow(); const keyHeader = document.createElement("th"); - keyHeader.textContent = "Screenshot in progress..."; + keyHeader.textContent = "Screenshot in progress"; headerRow.appendChild(keyHeader); const valueHeader = document.createElement("th"); valueHeader.textContent = ""; headerRow.appendChild(valueHeader); - this.populateStatistics(undefined); + this.populateStatistics(); this.updateStatisticsTableDisplayBasedOnMode(); this.statisticsContainer.appendChild(this.statisticsTable); return this.statisticsContainer; @@ -195,15 +192,28 @@ export class ScreenshotDialog extends Overlay { this.debouncedUpdateUIElements(); } - private populateStatistics(actionState: StatisticsActionState | undefined) { + private populateStatistics( + actionState: StatisticsActionState | undefined = undefined, + ) { if (actionState !== undefined) { while (this.statisticsTable.rows.length > 1) { this.statisticsTable.deleteRow(1); } } - const statsRow = this.screenshotHandler.parseStatistics(actionState); + const statsRow = this.screenshotHandler.screenshotStatistics; for (const key in statsRow) { + if (key === "timeElapsedString") { + const headerRow = this.statisticsTable.rows[0]; + const keyHeader = headerRow.cells[0]; + const time = statsRow[key]; + if (time === null) { + keyHeader.textContent = "Screenshot in progress (statistics loading)"; + } else { + keyHeader.textContent = `Screenshot in progress for ${statsRow[key]}s`; + } + continue; + } const row = this.statisticsTable.insertRow(); const keyCell = row.insertCell(); const valueCell = row.insertCell(); @@ -214,24 +224,20 @@ export class ScreenshotDialog extends Overlay { } private debouncedUpdateUIElements = debounce(() => { - this.showSaveOrForceScreenshotButton(); + this.updateSetupUIVisibility(); this.updateStatisticsTableDisplayBasedOnMode(); }, 200); - private showSaveOrForceScreenshotButton() { - if (this.viewer.display.screenshotMode.value !== this.screenshotMode) { - if (this.viewer.display.screenshotMode.value === ScreenshotModes.OFF) { - this.filenameAndButtonsContainer.replaceChild( - this.saveButton, - this.forceScreenshotButton, - ); - } else { - this.filenameAndButtonsContainer.replaceChild( - this.forceScreenshotButton, - this.saveButton, - ); - } - this.screenshotMode = this.viewer.display.screenshotMode.value; + private updateSetupUIVisibility() { + this.screenshotMode = this.viewer.display.screenshotMode.value; + if (this.screenshotMode === ScreenshotModes.OFF) { + this.forceScreenshotButton.style.display = "none"; + this.filenameAndButtonsContainer.style.display = "block"; + this.scaleSelectContainer.style.display = "block"; + } else { + this.forceScreenshotButton.style.display = "block"; + this.filenameAndButtonsContainer.style.display = "none"; + this.scaleSelectContainer.style.display = "none"; } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 7249e759f..2d31a59f3 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -58,6 +58,13 @@ export interface StatisticsActionState { }; } +interface UIScreenshotStatistics { + timeElapsedString: string | null; + chunkUsageDescription: string; + gpuMemoryUsageDescription: string; + downloadSpeedDescription: string; +} + interface ScreenshotCanvasViewport { left: number; right: number; @@ -148,6 +155,13 @@ export class ScreenshotFromViewer extends RefCounted { timestamp: 0, }; private lastUpdateTimestamp = 0; + private screenshotStartTime = 0; + private lastSavedStatistics: UIScreenshotStatistics = { + timeElapsedString: null, + chunkUsageDescription: "", + gpuMemoryUsageDescription: "", + downloadSpeedDescription: "", + }; constructor(public viewer: Viewer) { super(); @@ -167,6 +181,7 @@ export class ScreenshotFromViewer extends RefCounted { this.registerDisposer( this.viewer.screenshotActionHandler.sendStatisticsRequested.add( (actionState) => { + this.parseAndSaveStatistics(actionState); this.checkForStuckScreenshot(actionState); }, ), @@ -178,6 +193,10 @@ export class ScreenshotFromViewer extends RefCounted { ); } + get screenshotStatistics(): UIScreenshotStatistics { + return this.lastSavedStatistics; + } + screenshot(filename: string = "") { this.filename = filename; this.viewer.display.screenshotMode.value = ScreenshotModes.ON; @@ -185,8 +204,8 @@ export class ScreenshotFromViewer extends RefCounted { private startScreenshot() { const { viewer } = this; + this.screenshotStartTime = this.lastUpdateTimestamp = Date.now(); const shouldResize = this.screenshotScale !== 1; - this.lastUpdateTimestamp = Date.now(); this.gpuStats = { numVisibleChunks: 0, timestamp: 0, @@ -278,32 +297,40 @@ export class ScreenshotFromViewer extends RefCounted { } } - parseStatistics(actionState: StatisticsActionState | undefined) { - const nowtime = new Date().toLocaleTimeString(); - let statsRow; + parseAndSaveStatistics( + actionState: StatisticsActionState | undefined, + ): UIScreenshotStatistics { if (actionState === undefined) { - statsRow = { - time: nowtime, - visibleChunksGpuMemory: "", - visibleGpuMemory: "", - visibleChunksDownloading: "", - }; - } else { - const total = actionState.screenshotStatistics.total; - - const percentLoaded = - (100 * total.visibleChunksGpuMemory) / total.visibleChunksTotal; - const percentGpuUsage = - (100 * total.visibleGpuMemory) / - this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value; - const gpuMemoryUsageInMB = total.visibleGpuMemory / 1024 / 1024; - statsRow = { - time: nowtime, - visibleChunksGpuMemory: `${total.visibleChunksGpuMemory} out of ${total.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, - visibleGpuMemory: `${gpuMemoryUsageInMB}Mb (${percentGpuUsage.toFixed(2)}% of total)`, - visibleChunksDownloading: `${total.visibleChunksDownloading} at ${total.downloadLatency}ms`, - }; + return this.lastSavedStatistics; } + const nowtime = Date.now(); + const total = actionState.screenshotStatistics.total; + + const percentLoaded = + total.visibleChunksTotal === 0 + ? 0 + : (100 * total.visibleChunksGpuMemory) / total.visibleChunksTotal; + const percentGpuUsage = + (100 * total.visibleGpuMemory) / + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value; + const gpuMemoryUsageInMB = (total.visibleGpuMemory / 1000000).toFixed(0); + const totalMemoryInMB = ( + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value / + 1000000 + ).toFixed(0); + const latency = isNaN(total.downloadLatency) + ? 0 + : total.downloadLatency.toFixed(0); + const passedTimeInSeconds = ( + (nowtime - this.screenshotStartTime) / + 1000 + ).toFixed(0); + const statsRow = (this.lastSavedStatistics = { + timeElapsedString: passedTimeInSeconds, + chunkUsageDescription: `${total.visibleChunksGpuMemory} out of ${total.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, + gpuMemoryUsageDescription: `${gpuMemoryUsageInMB}Mb / ${totalMemoryInMB}Mb (${percentGpuUsage.toFixed(2)}% of total)`, + downloadSpeedDescription: `${total.visibleChunksDownloading} at ${latency}ms latency`, + }); return statsRow; } @@ -330,6 +357,12 @@ export class ScreenshotFromViewer extends RefCounted { const { screenshotMode } = display; if (screenshotMode.value === ScreenshotModes.OFF) { this.resetCanvasSize(); + this.lastSavedStatistics = { + timeElapsedString: null, + chunkUsageDescription: "", + gpuMemoryUsageDescription: "", + downloadSpeedDescription: "", + }; } else if (screenshotMode.value === ScreenshotModes.FORCE) { display.scheduleRedraw(); } else if (screenshotMode.value === ScreenshotModes.ON) { From d11dff21d19f275a67372d3208a05bec79ceb5a7 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 11:59:12 +0200 Subject: [PATCH 25/66] feat(ui): improve table updating --- src/ui/screenshot_menu.ts | 44 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 321bf346e..6149c94e0 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -18,8 +18,6 @@ import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; -import type { StatisticsActionState } from "#src/util/screenshot.js"; - import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; @@ -39,6 +37,7 @@ export class ScreenshotDialog extends Overlay { private scaleSelectContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; private screenshotMode: ScreenshotModes; + private statisticsKeyToCellMap: Map = new Map(); constructor(public viewer: Viewer) { super(); this.screenshotMode = this.viewer.display.screenshotMode.value; @@ -85,11 +84,9 @@ export class ScreenshotDialog extends Overlay { }), ); this.registerDisposer( - this.viewer.screenshotActionHandler.sendStatisticsRequested.add( - (actionState) => { - this.populateStatistics(actionState); - }, - ), + this.viewer.screenshotActionHandler.sendStatisticsRequested.add(() => { + this.populateStatistics(); + }), ); } @@ -175,6 +172,21 @@ export class ScreenshotDialog extends Overlay { valueHeader.textContent = ""; headerRow.appendChild(valueHeader); + // Populate inital table elements with placeholder text + const statsRow = this.screenshotHandler.screenshotStatistics; + for (const key in statsRow) { + if (key === "timeElapsedString") { + continue; + } + const row = this.statisticsTable.insertRow(); + const keyCell = row.insertCell(); + const valueCell = row.insertCell(); + keyCell.textContent = + friendlyNameMap[key as keyof typeof friendlyNameMap]; + valueCell.textContent = "Loading..."; + this.statisticsKeyToCellMap.set(key, valueCell); + } + this.populateStatistics(); this.updateStatisticsTableDisplayBasedOnMode(); this.statisticsContainer.appendChild(this.statisticsTable); @@ -192,14 +204,7 @@ export class ScreenshotDialog extends Overlay { this.debouncedUpdateUIElements(); } - private populateStatistics( - actionState: StatisticsActionState | undefined = undefined, - ) { - if (actionState !== undefined) { - while (this.statisticsTable.rows.length > 1) { - this.statisticsTable.deleteRow(1); - } - } + private populateStatistics() { const statsRow = this.screenshotHandler.screenshotStatistics; for (const key in statsRow) { @@ -214,12 +219,9 @@ export class ScreenshotDialog extends Overlay { } continue; } - const row = this.statisticsTable.insertRow(); - const keyCell = row.insertCell(); - const valueCell = row.insertCell(); - keyCell.textContent = - friendlyNameMap[key as keyof typeof friendlyNameMap]; - valueCell.textContent = String(statsRow[key as keyof typeof statsRow]); + this.statisticsKeyToCellMap.get(key)!.textContent = String( + statsRow[key as keyof typeof statsRow], + ); } } From 6bafc309c5a114141bfeed94847361219c9c3e7e Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 12:46:15 +0200 Subject: [PATCH 26/66] refactor: improve screenshot from viewer code --- src/ui/screenshot_menu.ts | 4 +- src/util/screenshot.ts | 276 +++++++++++++++++++------------------- 2 files changed, 141 insertions(+), 139 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 6149c94e0..da02b41e2 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -61,7 +61,7 @@ export class ScreenshotDialog extends Overlay { this.forceScreenshot(), ); this.forceScreenshotButton.title = - "Force a screenshot even if the viewer is not ready"; + "Force a screenshot of the current view without waiting for all data to be loaded and rendered"; this.filenameAndButtonsContainer = document.createElement("div"); this.filenameAndButtonsContainer.classList.add( "neuroglancer-screenshot-filename-and-buttons", @@ -200,7 +200,7 @@ export class ScreenshotDialog extends Overlay { private screenshot() { const filename = this.nameInput.value; - this.screenshotHandler.screenshot(filename); + this.screenshotHandler.takeScreenshot(filename); this.debouncedUpdateUIElements(); } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 2d31a59f3..40d66d094 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -19,11 +19,10 @@ import { RefCounted } from "#src/util/disposable.js"; import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; -// Warn after 5 seconds that the screenshot is likely stuck if no change in GPU chunks const SCREENSHOT_TIMEOUT = 5000; -interface screenshotGpuStats { - numVisibleChunks: number; +interface ScreenshotLoadStatistics { + numGpuLoadedVisibleChunks: number; timestamp: number; } @@ -65,14 +64,14 @@ interface UIScreenshotStatistics { downloadSpeedDescription: string; } -interface ScreenshotCanvasViewport { +interface ViewportBounds { left: number; right: number; top: number; bottom: number; } -function downloadFileForBlob(blob: Blob, filename: string) { +function saveBlobToFile(blob: Blob, filename: string) { const a = document.createElement("a"); const url = URL.createObjectURL(blob); a.href = url; @@ -84,10 +83,10 @@ function downloadFileForBlob(blob: Blob, filename: string) { } } -function determineViewPanelArea( - panels: Set, -): ScreenshotCanvasViewport { - const clippedPanel = { +function calculateViewportBounds( + panels: ReadonlySet, +): ViewportBounds { + const viewportBounds = { left: Number.POSITIVE_INFINITY, right: Number.NEGATIVE_INFINITY, top: Number.POSITIVE_INFINITY, @@ -97,16 +96,16 @@ function determineViewPanelArea( if (!panel.isDataPanel) continue; const viewport = panel.renderViewport; const { width, height } = viewport; - const left = panel.canvasRelativeClippedLeft; - const top = panel.canvasRelativeClippedTop; - const right = left + width; - const bottom = top + height; - clippedPanel.left = Math.min(clippedPanel.left, left); - clippedPanel.right = Math.max(clippedPanel.right, right); - clippedPanel.top = Math.min(clippedPanel.top, top); - clippedPanel.bottom = Math.max(clippedPanel.bottom, bottom); + const panelLeft = panel.canvasRelativeClippedLeft; + const panelTop = panel.canvasRelativeClippedTop; + const panelRight = panelLeft + width; + const panelBottom = panelTop + height; + viewportBounds.left = Math.min(viewportBounds.left, panelLeft); + viewportBounds.right = Math.max(viewportBounds.right, panelRight); + viewportBounds.top = Math.min(viewportBounds.top, panelTop); + viewportBounds.bottom = Math.max(viewportBounds.bottom, panelBottom); } - return clippedPanel; + return viewportBounds; } function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { @@ -121,28 +120,28 @@ function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { }); } -async function cropViewsFromViewer( +async function extractViewportScreenshot( viewer: Viewer, - crop: ScreenshotCanvasViewport, + viewportBounds: ViewportBounds, ): Promise { - const cropWidth = crop.right - crop.left; - const cropHeight = crop.bottom - crop.top; + const cropWidth = viewportBounds.right - viewportBounds.left; + const cropHeight = viewportBounds.bottom - viewportBounds.top; const img = await createImageBitmap( viewer.display.canvas, - crop.left, - crop.top, + viewportBounds.left, + viewportBounds.top, cropWidth, cropHeight, ); - const canvas = document.createElement("canvas"); - canvas.width = cropWidth; - canvas.height = cropHeight; - const ctx = canvas.getContext("2d"); + const screenshotCanvas = document.createElement("canvas"); + screenshotCanvas.width = cropWidth; + screenshotCanvas.height = cropHeight; + const ctx = screenshotCanvas.getContext("2d"); if (!ctx) throw new Error("Failed to get canvas context"); ctx.drawImage(img, 0, 0); - const croppedBlob = await canvasToBlob(canvas, "image/png"); + const croppedBlob = await canvasToBlob(screenshotCanvas, "image/png"); return croppedBlob; } @@ -150,8 +149,8 @@ export class ScreenshotFromViewer extends RefCounted { public screenshotId: number = -1; public screenshotScale: number = 1; private filename: string = ""; - private gpuStats: screenshotGpuStats = { - numVisibleChunks: 0, + private screenshotLoadStats: ScreenshotLoadStatistics = { + numGpuLoadedVisibleChunks: 0, timestamp: 0, }; private lastUpdateTimestamp = 0; @@ -181,8 +180,8 @@ export class ScreenshotFromViewer extends RefCounted { this.registerDisposer( this.viewer.screenshotActionHandler.sendStatisticsRequested.add( (actionState) => { - this.parseAndSaveStatistics(actionState); - this.checkForStuckScreenshot(actionState); + this.persistStatisticsData(actionState); + this.checkAndHandleStalledScreenshot(actionState); }, ), ); @@ -193,24 +192,30 @@ export class ScreenshotFromViewer extends RefCounted { ); } - get screenshotStatistics(): UIScreenshotStatistics { - return this.lastSavedStatistics; - } - - screenshot(filename: string = "") { + takeScreenshot(filename: string = "") { this.filename = filename; this.viewer.display.screenshotMode.value = ScreenshotModes.ON; } - private startScreenshot() { + forceScreenshot() { + this.viewer.display.screenshotMode.value = ScreenshotModes.FORCE; + } + + get screenshotStatistics(): UIScreenshotStatistics { + return this.lastSavedStatistics; + } + + private handleScreenshotStarted() { const { viewer } = this; + const shouldIncreaseCanvasSize = this.screenshotScale !== 1; + this.screenshotStartTime = this.lastUpdateTimestamp = Date.now(); - const shouldResize = this.screenshotScale !== 1; - this.gpuStats = { - numVisibleChunks: 0, + this.screenshotLoadStats = { + numGpuLoadedVisibleChunks: 0, timestamp: 0, }; - if (shouldResize) { + + if (shouldIncreaseCanvasSize) { const oldSize = { width: viewer.display.canvas.width, height: viewer.display.canvas.height, @@ -222,22 +227,70 @@ export class ScreenshotFromViewer extends RefCounted { viewer.display.canvas.width = newSize.width; viewer.display.canvas.height = newSize.height; } + + // Pass a new screenshot ID to the viewer to trigger a new screenshot. this.screenshotId++; this.viewer.screenshotActionHandler.requestState.value = this.screenshotId.toString(); - if (shouldResize) { + + // Force handling the canvas size change + if (shouldIncreaseCanvasSize) { ++viewer.display.resizeGeneration; viewer.display.resizeCallback(); } } - resetCanvasSize() { - const { viewer } = this; - ++viewer.display.resizeGeneration; - viewer.display.resizeCallback(); + private handleScreenshotModeChange() { + const { display } = this.viewer; + switch (display.screenshotMode.value) { + case ScreenshotModes.OFF: + this.resetCanvasSize(); + this.resetStatistics(); + break; + case ScreenshotModes.FORCE: + display.scheduleRedraw(); + break; + case ScreenshotModes.ON: + this.handleScreenshotStarted(); + break; + } + } + + /** + * Check if the screenshot is stuck by comparing the number of visible chunks + * in the GPU with the previous number of visible chunks. If the number of + * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. + */ + private checkAndHandleStalledScreenshot(actionState: StatisticsActionState) { + const total = actionState.screenshotStatistics.total; + const newStats: ScreenshotLoadStatistics = { + numGpuLoadedVisibleChunks: total.visibleChunksGpuMemory, + timestamp: Date.now(), + }; + if (this.screenshotLoadStats.timestamp === 0) { + this.screenshotLoadStats = newStats; + return; + } + const oldStats = this.screenshotLoadStats; + if ( + oldStats.numGpuLoadedVisibleChunks === newStats.numGpuLoadedVisibleChunks + ) { + if ( + newStats.timestamp - oldStats.timestamp > SCREENSHOT_TIMEOUT && + Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT + ) { + const totalChunks = total.visibleChunksTotal; + console.warn( + `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${newStats.numGpuLoadedVisibleChunks}/${totalChunks}`, + ); + this.forceScreenshot(); + } + } else { + this.screenshotLoadStats = newStats; + } } - async saveScreenshot(actionState: ScreenshotActionState) { + private async saveScreenshot(actionState: ScreenshotActionState) { const { screenshot } = actionState; const { imageType } = screenshot; if (imageType !== "image/png") { @@ -245,11 +298,11 @@ export class ScreenshotFromViewer extends RefCounted { this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; return; } - const renderingPanelArea = determineViewPanelArea( + const renderingPanelArea = calculateViewportBounds( this.viewer.display.panels, ); try { - const croppedImage = await cropViewsFromViewer( + const croppedImage = await extractViewportScreenshot( this.viewer, renderingPanelArea, ); @@ -257,116 +310,65 @@ export class ScreenshotFromViewer extends RefCounted { renderingPanelArea.right - renderingPanelArea.left, renderingPanelArea.bottom - renderingPanelArea.top, ); - downloadFileForBlob(croppedImage, filename); + saveBlobToFile(croppedImage, filename); } catch (error) { - console.error(error); + console.error("Failed to save screenshot:", error); } finally { this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; } } - /** - * Check if the screenshot is stuck by comparing the number of visible chunks - * in the GPU with the previous number of visible chunks. If the number of - * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. - */ - private checkForStuckScreenshot(actionState: StatisticsActionState) { - const total = actionState.screenshotStatistics.total; - const newStats = { - numVisibleChunks: total.visibleChunksGpuMemory, - timestamp: Date.now(), + private resetCanvasSize() { + // Reset the canvas size to the original size + // No need to manually pass the correct sizes, the viewer will handle it + const { viewer } = this; + ++viewer.display.resizeGeneration; + viewer.display.resizeCallback(); + } + + private resetStatistics() { + this.lastSavedStatistics = { + timeElapsedString: null, + chunkUsageDescription: "", + gpuMemoryUsageDescription: "", + downloadSpeedDescription: "", }; - const oldStats = this.gpuStats; - if (oldStats.timestamp === 0) { - this.gpuStats = newStats; - return; - } - if (oldStats.numVisibleChunks === newStats.numVisibleChunks) { - if ( - newStats.timestamp - oldStats.timestamp > SCREENSHOT_TIMEOUT && - Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT - ) { - const totalChunks = total.visibleChunksTotal; - console.warn( - `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${newStats.numVisibleChunks}/${totalChunks}`, - ); - this.forceScreenshot(); - } - } else { - this.gpuStats = newStats; - } } - parseAndSaveStatistics( - actionState: StatisticsActionState | undefined, - ): UIScreenshotStatistics { - if (actionState === undefined) { - return this.lastSavedStatistics; - } + private persistStatisticsData(actionState: StatisticsActionState) { const nowtime = Date.now(); const total = actionState.screenshotStatistics.total; + const maxGpuMemory = + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value; const percentLoaded = total.visibleChunksTotal === 0 ? 0 : (100 * total.visibleChunksGpuMemory) / total.visibleChunksTotal; - const percentGpuUsage = - (100 * total.visibleGpuMemory) / - this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value; - const gpuMemoryUsageInMB = (total.visibleGpuMemory / 1000000).toFixed(0); - const totalMemoryInMB = ( - this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value / - 1000000 - ).toFixed(0); - const latency = isNaN(total.downloadLatency) - ? 0 - : total.downloadLatency.toFixed(0); + const percentGpuUsage = (100 * total.visibleGpuMemory) / maxGpuMemory; + const gpuMemoryUsageInMB = total.visibleGpuMemory / 1000000; + const totalMemoryInMB = maxGpuMemory / 1000000; + const latency = isNaN(total.downloadLatency) ? 0 : total.downloadLatency; const passedTimeInSeconds = ( (nowtime - this.screenshotStartTime) / 1000 ).toFixed(0); - const statsRow = (this.lastSavedStatistics = { + + this.lastSavedStatistics = { timeElapsedString: passedTimeInSeconds, chunkUsageDescription: `${total.visibleChunksGpuMemory} out of ${total.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, - gpuMemoryUsageDescription: `${gpuMemoryUsageInMB}Mb / ${totalMemoryInMB}Mb (${percentGpuUsage.toFixed(2)}% of total)`, - downloadSpeedDescription: `${total.visibleChunksDownloading} at ${latency}ms latency`, - }); - return statsRow; - } - - forceScreenshot() { - this.viewer.display.screenshotMode.value = ScreenshotModes.FORCE; - } - - generateFilename(width: number, height: number): string { - let filename = this.filename; - if (filename.length === 0) { - let nowtime = new Date().toLocaleString(); - nowtime = nowtime.replace(", ", "-"); - filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; - } - if (!filename.endsWith(".png")) { - filename += ".png"; - } - return filename; + gpuMemoryUsageDescription: `${gpuMemoryUsageInMB.toFixed(0)}MB / ${totalMemoryInMB.toFixed(0)}MB (${percentGpuUsage.toFixed(2)}% of total)`, + downloadSpeedDescription: `${total.visibleChunksDownloading} at ${latency.toFixed(0)}ms latency`, + }; } - handleScreenshotModeChange() { - const { viewer } = this; - const { display } = viewer; - const { screenshotMode } = display; - if (screenshotMode.value === ScreenshotModes.OFF) { - this.resetCanvasSize(); - this.lastSavedStatistics = { - timeElapsedString: null, - chunkUsageDescription: "", - gpuMemoryUsageDescription: "", - downloadSpeedDescription: "", - }; - } else if (screenshotMode.value === ScreenshotModes.FORCE) { - display.scheduleRedraw(); - } else if (screenshotMode.value === ScreenshotModes.ON) { - this.startScreenshot(); + private generateFilename(width: number, height: number): string { + if (!this.filename) { + const nowtime = new Date().toLocaleString().replace(", ", "-"); + this.filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; } + return this.filename.endsWith(".png") + ? this.filename + : this.filename + ".png"; } } From 81154b62de5d4fe489720682d52e0deb7312aaa1 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 13:05:24 +0200 Subject: [PATCH 27/66] refactor: small rename for consistency --- src/util/screenshot.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 40d66d094..da98f6646 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -167,16 +167,11 @@ export class ScreenshotFromViewer extends RefCounted { this.viewer = viewer; this.registerDisposer( this.viewer.screenshotActionHandler.sendScreenshotRequested.add( - (state) => { - this.saveScreenshot(state); + (actionState) => { + this.saveScreenshot(actionState); }, ), ); - this.registerDisposer( - this.viewer.display.updateFinished.add(() => { - this.lastUpdateTimestamp = Date.now(); - }), - ); this.registerDisposer( this.viewer.screenshotActionHandler.sendStatisticsRequested.add( (actionState) => { @@ -185,6 +180,11 @@ export class ScreenshotFromViewer extends RefCounted { }, ), ); + this.registerDisposer( + this.viewer.display.updateFinished.add(() => { + this.lastUpdateTimestamp = Date.now(); + }), + ); this.registerDisposer( this.viewer.display.screenshotMode.changed.add(() => { this.handleScreenshotModeChange(); From aa4b419b35eeb755cd85ab857c319b40e7ad4cb0 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 13:22:47 +0200 Subject: [PATCH 28/66] feat: include state log for screenshot replication --- src/util/screenshot.ts | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index da98f6646..2ee60b5b6 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -83,6 +83,18 @@ function saveBlobToFile(blob: Blob, filename: string) { } } +function setExtension(filename: string, extension: string = ".png"): string { + function replaceExtension(filename: string): string { + const lastDot = filename.lastIndexOf("."); + if (lastDot === -1) { + return filename + extension; + } + return `${filename.substring(0, lastDot)}${extension}`; + } + + return filename.endsWith(extension) ? filename : replaceExtension(filename); +} + function calculateViewportBounds( panels: ReadonlySet, ): ViewportBounds { @@ -306,18 +318,30 @@ export class ScreenshotFromViewer extends RefCounted { this.viewer, renderingPanelArea, ); - const filename = this.generateFilename( + this.generateFilename( renderingPanelArea.right - renderingPanelArea.left, renderingPanelArea.bottom - renderingPanelArea.top, ); - saveBlobToFile(croppedImage, filename); + saveBlobToFile(croppedImage, this.filename); } catch (error) { console.error("Failed to save screenshot:", error); } finally { + this.saveScreenshotLog(actionState); this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; } } + private saveScreenshotLog(actionState: ScreenshotActionState) { + const { viewerState } = actionState; + const stateString = JSON.stringify(viewerState); + this.downloadState(stateString); + } + + private downloadState(state: string) { + const blob = new Blob([state], { type: "text/json" }); + saveBlobToFile(blob, setExtension(this.filename, "_state.json")); + } + private resetCanvasSize() { // Reset the canvas size to the original size // No need to manually pass the correct sizes, the viewer will handle it @@ -365,10 +389,9 @@ export class ScreenshotFromViewer extends RefCounted { private generateFilename(width: number, height: number): string { if (!this.filename) { const nowtime = new Date().toLocaleString().replace(", ", "-"); - this.filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}.png`; + this.filename = `neuroglancer-screenshot-w${width}px-h${height}px-at-${nowtime}`; } - return this.filename.endsWith(".png") - ? this.filename - : this.filename + ".png"; + this.filename = setExtension(this.filename); + return this.filename; } } From 2d597883c429a1602855a029432ee429fc2bf948 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 13:35:30 +0200 Subject: [PATCH 29/66] refactor: change trackable screenshot from type to class --- src/display_context.ts | 13 +++++++------ src/python_integration/screenshots.ts | 4 ++-- src/ui/screenshot_menu.ts | 8 ++++---- src/util/screenshot.ts | 16 ++++++++-------- src/util/trackable_screenshot_mode.ts | 12 +++++------- 5 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/display_context.ts b/src/display_context.ts index 000ee46af..f1fbdea8f 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -25,10 +25,9 @@ import { FramerateMonitor } from "#src/util/framerate.js"; import type { mat4 } from "#src/util/geom.js"; import { parseFixedLengthArray, verifyFloat01 } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; -import type { TrackableScreenshotModeValue } from "#src/util/trackable_screenshot_mode.js"; import { - ScreenshotModes, - trackableScreenshotModeValue, + TrackableScreenshotMode, + ScreenshotMode, } from "#src/util/trackable_screenshot_mode.js"; import type { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js"; import type { GL } from "#src/webgl/context.js"; @@ -226,7 +225,7 @@ export abstract class RenderedPanel extends RefCounted { 0, clippedBottom - clippedTop, )); - if (this.context.screenshotMode.value !== ScreenshotModes.OFF) { + if (this.context.screenshotMode.value !== ScreenshotMode.OFF) { viewport.width = logicalWidth * screenToCanvasPixelScaleX; viewport.height = logicalHeight * screenToCanvasPixelScaleY; viewport.logicalWidth = logicalWidth * screenToCanvasPixelScaleX; @@ -419,7 +418,9 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { rootRect: DOMRect | undefined; resizeGeneration = 0; boundsGeneration = -1; - screenshotMode: TrackableScreenshotModeValue = trackableScreenshotModeValue(); + screenshotMode: TrackableScreenshotMode = new TrackableScreenshotMode( + ScreenshotMode.OFF, + ); private framerateMonitor = new FramerateMonitor(); private continuousCameraMotionInProgress = false; @@ -592,7 +593,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter { const { resizeGeneration } = this; if (this.boundsGeneration === resizeGeneration) return; const { canvas } = this; - if (this.screenshotMode.value === ScreenshotModes.OFF) { + if (this.screenshotMode.value === ScreenshotMode.OFF) { canvas.width = canvas.offsetWidth; canvas.height = canvas.offsetHeight; } diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index 7dbc3bf0a..b80c7354c 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -28,7 +28,7 @@ import { convertEndian32, Endianness } from "#src/util/endian.js"; import { verifyOptionalString } from "#src/util/json.js"; import { Signal } from "#src/util/signal.js"; import { getCachedJson } from "#src/util/trackable.js"; -import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; +import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; export class ScreenshotHandler extends RefCounted { @@ -126,7 +126,7 @@ export class ScreenshotHandler extends RefCounted { } const { viewer } = this; const forceScreenshot = - this.viewer.display.screenshotMode.value === ScreenshotModes.FORCE; + this.viewer.display.screenshotMode.value === ScreenshotMode.FORCE; if (!viewer.isReady() && !forceScreenshot) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index da02b41e2..206d0433b 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -18,7 +18,7 @@ import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; -import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; +import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; const friendlyNameMap = { @@ -36,7 +36,7 @@ export class ScreenshotDialog extends Overlay { private statisticsContainer: HTMLDivElement; private scaleSelectContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; - private screenshotMode: ScreenshotModes; + private screenshotMode: ScreenshotMode; private statisticsKeyToCellMap: Map = new Map(); constructor(public viewer: Viewer) { super(); @@ -99,7 +99,7 @@ export class ScreenshotDialog extends Overlay { } private updateStatisticsTableDisplayBasedOnMode() { - if (this.screenshotMode === ScreenshotModes.OFF) { + if (this.screenshotMode === ScreenshotMode.OFF) { this.statisticsContainer.style.display = "none"; } else { this.statisticsContainer.style.display = "block"; @@ -232,7 +232,7 @@ export class ScreenshotDialog extends Overlay { private updateSetupUIVisibility() { this.screenshotMode = this.viewer.display.screenshotMode.value; - if (this.screenshotMode === ScreenshotModes.OFF) { + if (this.screenshotMode === ScreenshotMode.OFF) { this.forceScreenshotButton.style.display = "none"; this.filenameAndButtonsContainer.style.display = "block"; this.scaleSelectContainer.style.display = "block"; diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 2ee60b5b6..e8f679a17 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -16,7 +16,7 @@ import type { RenderedPanel } from "#src/display_context.js"; import { RefCounted } from "#src/util/disposable.js"; -import { ScreenshotModes } from "#src/util/trackable_screenshot_mode.js"; +import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; const SCREENSHOT_TIMEOUT = 5000; @@ -206,11 +206,11 @@ export class ScreenshotFromViewer extends RefCounted { takeScreenshot(filename: string = "") { this.filename = filename; - this.viewer.display.screenshotMode.value = ScreenshotModes.ON; + this.viewer.display.screenshotMode.value = ScreenshotMode.ON; } forceScreenshot() { - this.viewer.display.screenshotMode.value = ScreenshotModes.FORCE; + this.viewer.display.screenshotMode.value = ScreenshotMode.FORCE; } get screenshotStatistics(): UIScreenshotStatistics { @@ -255,14 +255,14 @@ export class ScreenshotFromViewer extends RefCounted { private handleScreenshotModeChange() { const { display } = this.viewer; switch (display.screenshotMode.value) { - case ScreenshotModes.OFF: + case ScreenshotMode.OFF: this.resetCanvasSize(); this.resetStatistics(); break; - case ScreenshotModes.FORCE: + case ScreenshotMode.FORCE: display.scheduleRedraw(); break; - case ScreenshotModes.ON: + case ScreenshotMode.ON: this.handleScreenshotStarted(); break; } @@ -307,7 +307,7 @@ export class ScreenshotFromViewer extends RefCounted { const { imageType } = screenshot; if (imageType !== "image/png") { console.error("Image type is not PNG"); - this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; + this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; return; } const renderingPanelArea = calculateViewportBounds( @@ -327,7 +327,7 @@ export class ScreenshotFromViewer extends RefCounted { console.error("Failed to save screenshot:", error); } finally { this.saveScreenshotLog(actionState); - this.viewer.display.screenshotMode.value = ScreenshotModes.OFF; + this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; } } diff --git a/src/util/trackable_screenshot_mode.ts b/src/util/trackable_screenshot_mode.ts index bfb848401..d82f3d819 100644 --- a/src/util/trackable_screenshot_mode.ts +++ b/src/util/trackable_screenshot_mode.ts @@ -16,16 +16,14 @@ import { TrackableEnum } from "#src/util/trackable_enum.js"; -export enum ScreenshotModes { +export enum ScreenshotMode { OFF = 0, // Default mode ON = 1, // Screenshot modek FORCE = 2, // Force screenshot mode - used when the screenshot is stuck } -export type TrackableScreenshotModeValue = TrackableEnum; - -export function trackableScreenshotModeValue( - initialValue = ScreenshotModes.OFF, -) { - return new TrackableEnum(ScreenshotModes, initialValue); +export class TrackableScreenshotMode extends TrackableEnum { + constructor(value: ScreenshotMode, defaultValue: ScreenshotMode = value) { + super(ScreenshotMode, value, defaultValue); + } } From f40c432649b0f696a81b0561b44fc1e30ba09868 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 13:38:58 +0200 Subject: [PATCH 30/66] refactor: clarify force check for bool --- src/python_integration/screenshots.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index b80c7354c..ac3b0b2d2 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -125,14 +125,14 @@ export class ScreenshotHandler extends RefCounted { return; } const { viewer } = this; - const forceScreenshot = + const shouldForceScreenshot = this.viewer.display.screenshotMode.value === ScreenshotMode.FORCE; - if (!viewer.isReady() && !forceScreenshot) { + if (!viewer.isReady() && !shouldForceScreenshot) { this.wasAlreadyVisible = false; this.throttledSendStatistics(requestState); return; } - if (!this.wasAlreadyVisible && !forceScreenshot) { + if (!this.wasAlreadyVisible && !shouldForceScreenshot) { this.throttledSendStatistics(requestState); this.wasAlreadyVisible = true; this.debouncedMaybeSendScreenshot(); From d666a3e7a07d53c93b1d533d953f2c158bf71e43 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 13:44:10 +0200 Subject: [PATCH 31/66] remove: is DataPanel flag in favour of checking instance --- src/display_context.ts | 4 ---- src/perspective_view/panel.ts | 4 ---- src/sliceview/panel.ts | 3 --- src/util/screenshot.ts | 3 ++- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/display_context.ts b/src/display_context.ts index f1fbdea8f..5fc14335a 100644 --- a/src/display_context.ts +++ b/src/display_context.ts @@ -311,10 +311,6 @@ export abstract class RenderedPanel extends RefCounted { return true; } - get isDataPanel() { - return false; - } - // Returns a number that determine the order in which panels are drawn. This is used by CdfPanel // to ensure it is drawn after other panels that update the histogram. // diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 820a5c2ee..1ab88d5e1 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -307,10 +307,6 @@ export class PerspectivePanel extends RenderedDataPanel { ); } - get isDataPanel() { - return true; - } - /** * If boolean value is true, sliceView is shown unconditionally, regardless of the value of * this.viewer.showSliceViews.value. diff --git a/src/sliceview/panel.ts b/src/sliceview/panel.ts index fc801e4c8..172ee6f25 100644 --- a/src/sliceview/panel.ts +++ b/src/sliceview/panel.ts @@ -129,9 +129,6 @@ export class SliceViewPanel extends RenderedDataPanel { get rpcId() { return this.sliceView.rpcId!; } - get isDataPanel() { - return true; - } private offscreenFramebuffer = this.registerDisposer( new FramebufferConfiguration(this.gl, { diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index e8f679a17..d1f1e00ce 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -15,6 +15,7 @@ */ import type { RenderedPanel } from "#src/display_context.js"; +import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { RefCounted } from "#src/util/disposable.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; @@ -105,7 +106,7 @@ function calculateViewportBounds( bottom: Number.NEGATIVE_INFINITY, }; for (const panel of panels) { - if (!panel.isDataPanel) continue; + if (!(panel instanceof RenderedDataPanel)) continue; const viewport = panel.renderViewport; const { width, height } = viewport; const panelLeft = panel.canvasRelativeClippedLeft; From 7fbbf88acf9769c2924659d57c06424b5fc9e144 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 13:52:27 +0200 Subject: [PATCH 32/66] feat(ui): small UI improvements in screenshot menu --- src/ui/screenshot_menu.css | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index 41c8a9ed2..046147eed 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -1,6 +1,6 @@ /** * @license - * Copyright 2018 Google Inc. + * Copyright 2024 Google Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -23,6 +23,7 @@ display: inline-block; width: 20px; margin-right: -2px; + cursor: pointer; } .neuroglancer-screenshot-filename-and-buttons { @@ -65,9 +66,4 @@ background-color: #f8f8f8; font-weight: bold; color: #555; -} - -.neuroglancer-screenshot-statistics-table td { - background-color: #fff; - color: #333; -} +} \ No newline at end of file From c6dca6c88dfba49064e31c1725741591d7014a26 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 14:15:28 +0200 Subject: [PATCH 33/66] refactor: rename classes --- src/ui/screenshot_menu.ts | 40 ++++++++++++++++++--------------------- src/util/screenshot.ts | 8 ++++---- src/viewer.ts | 7 ++++--- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 206d0433b..80712ff21 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -21,7 +21,7 @@ import "#src/ui/screenshot_menu.css"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; -const friendlyNameMap = { +const statisticsNamesForUI = { chunkUsageDescription: "Number of loaded chunks", gpuMemoryUsageDescription: "Visible chunk GPU memory usage", downloadSpeedDescription: "Number of downloading chunks", @@ -78,13 +78,13 @@ export class ScreenshotDialog extends Overlay { private setupEventListeners() { this.registerDisposer( - this.viewer.screenshotActionHandler.sendScreenshotRequested.add(() => { + this.viewer.screenshotHandler.sendScreenshotRequested.add(() => { this.debouncedUpdateUIElements(); this.dispose(); }), ); this.registerDisposer( - this.viewer.screenshotActionHandler.sendStatisticsRequested.add(() => { + this.viewer.screenshotHandler.sendStatisticsRequested.add(() => { this.populateStatistics(); }), ); @@ -98,14 +98,6 @@ export class ScreenshotDialog extends Overlay { return (this.nameInput = nameInput); } - private updateStatisticsTableDisplayBasedOnMode() { - if (this.screenshotMode === ScreenshotMode.OFF) { - this.statisticsContainer.style.display = "none"; - } else { - this.statisticsContainer.style.display = "block"; - } - } - private createButton( text: string, onClick: () => void, @@ -136,7 +128,7 @@ export class ScreenshotDialog extends Overlay { input.type = "radio"; input.name = "screenshot-scale"; input.value = scale.toString(); - input.checked = scale === this.screenshotHandler.screenshotScale; + input.checked = scale === this.viewer.screenshotManager.screenshotScale; input.classList.add("neuroglancer-screenshot-scale-radio"); label.appendChild(input); @@ -145,7 +137,7 @@ export class ScreenshotDialog extends Overlay { scaleMenu.appendChild(label); input.addEventListener("change", () => { - this.screenshotHandler.screenshotScale = scale; + this.viewer.screenshotManager.screenshotScale = scale; }); }); return scaleMenu; @@ -173,7 +165,7 @@ export class ScreenshotDialog extends Overlay { headerRow.appendChild(valueHeader); // Populate inital table elements with placeholder text - const statsRow = this.screenshotHandler.screenshotStatistics; + const statsRow = this.viewer.screenshotManager.screenshotStatistics; for (const key in statsRow) { if (key === "timeElapsedString") { continue; @@ -182,7 +174,7 @@ export class ScreenshotDialog extends Overlay { const keyCell = row.insertCell(); const valueCell = row.insertCell(); keyCell.textContent = - friendlyNameMap[key as keyof typeof friendlyNameMap]; + statisticsNamesForUI[key as keyof typeof statisticsNamesForUI]; valueCell.textContent = "Loading..."; this.statisticsKeyToCellMap.set(key, valueCell); } @@ -194,18 +186,26 @@ export class ScreenshotDialog extends Overlay { } private forceScreenshot() { - this.screenshotHandler.forceScreenshot(); + this.viewer.screenshotManager.forceScreenshot(); this.debouncedUpdateUIElements(); } private screenshot() { const filename = this.nameInput.value; - this.screenshotHandler.takeScreenshot(filename); + this.viewer.screenshotManager.takeScreenshot(filename); this.debouncedUpdateUIElements(); } + private updateStatisticsTableDisplayBasedOnMode() { + if (this.screenshotMode === ScreenshotMode.OFF) { + this.statisticsContainer.style.display = "none"; + } else { + this.statisticsContainer.style.display = "block"; + } + } + private populateStatistics() { - const statsRow = this.screenshotHandler.screenshotStatistics; + const statsRow = this.viewer.screenshotManager.screenshotStatistics; for (const key in statsRow) { if (key === "timeElapsedString") { @@ -242,8 +242,4 @@ export class ScreenshotDialog extends Overlay { this.scaleSelectContainer.style.display = "none"; } } - - get screenshotHandler() { - return this.viewer.screenshotHandler; - } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index d1f1e00ce..501ce81d0 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -158,7 +158,7 @@ async function extractViewportScreenshot( return croppedBlob; } -export class ScreenshotFromViewer extends RefCounted { +export class ScreenshotManager extends RefCounted { public screenshotId: number = -1; public screenshotScale: number = 1; private filename: string = ""; @@ -179,14 +179,14 @@ export class ScreenshotFromViewer extends RefCounted { super(); this.viewer = viewer; this.registerDisposer( - this.viewer.screenshotActionHandler.sendScreenshotRequested.add( + this.viewer.screenshotHandler.sendScreenshotRequested.add( (actionState) => { this.saveScreenshot(actionState); }, ), ); this.registerDisposer( - this.viewer.screenshotActionHandler.sendStatisticsRequested.add( + this.viewer.screenshotHandler.sendStatisticsRequested.add( (actionState) => { this.persistStatisticsData(actionState); this.checkAndHandleStalledScreenshot(actionState); @@ -243,7 +243,7 @@ export class ScreenshotFromViewer extends RefCounted { // Pass a new screenshot ID to the viewer to trigger a new screenshot. this.screenshotId++; - this.viewer.screenshotActionHandler.requestState.value = + this.viewer.screenshotHandler.requestState.value = this.screenshotId.toString(); // Force handling the canvas size change diff --git a/src/viewer.ts b/src/viewer.ts index 49d421e77..6373b0d85 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -120,7 +120,7 @@ import { EventActionMap, KeyboardEventBinder, } from "#src/util/keyboard_bindings.js"; -import { ScreenshotFromViewer } from "#src/util/screenshot.js"; +import { ScreenshotManager } from "#src/util/screenshot.js"; import { NullarySignal } from "#src/util/signal.js"; import { CompoundTrackable, @@ -491,8 +491,8 @@ export class Viewer extends RefCounted implements ViewerState { resetInitiated = new NullarySignal(); - screenshotActionHandler = this.registerDisposer(new ScreenshotHandler(this)); - screenshotHandler = this.registerDisposer(new ScreenshotFromViewer(this)); + screenshotHandler = this.registerDisposer(new ScreenshotHandler(this)); + screenshotManager = this.registerDisposer(new ScreenshotManager(this)); get chunkManager() { return this.dataContext.chunkManager; @@ -572,6 +572,7 @@ export class Viewer extends RefCounted implements ViewerState { this.display.applyWindowedViewportToElement(element, value); }, this.partialViewport), ); + this.registerDisposer(() => removeFromParent(this.element)); this.dataContext = this.registerDisposer(dataContext); From e0facf0fd4029534369f044f68f2604bcad3c3d0 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 14:35:08 +0200 Subject: [PATCH 34/66] refactor: clarify interaction between screenshot objects --- src/ui/screenshot_menu.ts | 33 +++++++++++++++++---------------- src/util/screenshot.ts | 9 ++++++++- src/viewer.ts | 2 +- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 80712ff21..321f2a570 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -18,8 +18,8 @@ import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; +import type { ScreenshotManager } from "#src/util/screenshot.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; -import type { Viewer } from "#src/viewer.js"; const statisticsNamesForUI = { chunkUsageDescription: "Number of loaded chunks", @@ -36,11 +36,9 @@ export class ScreenshotDialog extends Overlay { private statisticsContainer: HTMLDivElement; private scaleSelectContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; - private screenshotMode: ScreenshotMode; private statisticsKeyToCellMap: Map = new Map(); - constructor(public viewer: Viewer) { + constructor(private screenshotManager: ScreenshotManager) { super(); - this.screenshotMode = this.viewer.display.screenshotMode.value; this.initializeUI(); this.setupEventListeners(); @@ -78,13 +76,12 @@ export class ScreenshotDialog extends Overlay { private setupEventListeners() { this.registerDisposer( - this.viewer.screenshotHandler.sendScreenshotRequested.add(() => { - this.debouncedUpdateUIElements(); + this.screenshotManager.screenshotFinished.add(() => { this.dispose(); }), ); this.registerDisposer( - this.viewer.screenshotHandler.sendStatisticsRequested.add(() => { + this.screenshotManager.statisticsUpdated.add(() => { this.populateStatistics(); }), ); @@ -128,7 +125,7 @@ export class ScreenshotDialog extends Overlay { input.type = "radio"; input.name = "screenshot-scale"; input.value = scale.toString(); - input.checked = scale === this.viewer.screenshotManager.screenshotScale; + input.checked = scale === this.screenshotManager.screenshotScale; input.classList.add("neuroglancer-screenshot-scale-radio"); label.appendChild(input); @@ -137,7 +134,7 @@ export class ScreenshotDialog extends Overlay { scaleMenu.appendChild(label); input.addEventListener("change", () => { - this.viewer.screenshotManager.screenshotScale = scale; + this.screenshotManager.screenshotScale = scale; }); }); return scaleMenu; @@ -165,7 +162,7 @@ export class ScreenshotDialog extends Overlay { headerRow.appendChild(valueHeader); // Populate inital table elements with placeholder text - const statsRow = this.viewer.screenshotManager.screenshotStatistics; + const statsRow = this.screenshotManager.screenshotStatistics; for (const key in statsRow) { if (key === "timeElapsedString") { continue; @@ -186,13 +183,14 @@ export class ScreenshotDialog extends Overlay { } private forceScreenshot() { - this.viewer.screenshotManager.forceScreenshot(); - this.debouncedUpdateUIElements(); + this.screenshotManager.forceScreenshot(); } private screenshot() { const filename = this.nameInput.value; - this.viewer.screenshotManager.takeScreenshot(filename); + this.screenshotManager.takeScreenshot(filename); + // Delay the update because sometimes the screenshot is immediately taken + // And the UI is disposed before the update can happen this.debouncedUpdateUIElements(); } @@ -205,7 +203,7 @@ export class ScreenshotDialog extends Overlay { } private populateStatistics() { - const statsRow = this.viewer.screenshotManager.screenshotStatistics; + const statsRow = this.screenshotManager.screenshotStatistics; for (const key in statsRow) { if (key === "timeElapsedString") { @@ -228,10 +226,9 @@ export class ScreenshotDialog extends Overlay { private debouncedUpdateUIElements = debounce(() => { this.updateSetupUIVisibility(); this.updateStatisticsTableDisplayBasedOnMode(); - }, 200); + }, 100); private updateSetupUIVisibility() { - this.screenshotMode = this.viewer.display.screenshotMode.value; if (this.screenshotMode === ScreenshotMode.OFF) { this.forceScreenshotButton.style.display = "none"; this.filenameAndButtonsContainer.style.display = "block"; @@ -242,4 +239,8 @@ export class ScreenshotDialog extends Overlay { this.scaleSelectContainer.style.display = "none"; } } + + get screenshotMode() { + return this.screenshotManager.screenshotMode; + } } diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index 501ce81d0..d7abb68a5 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -17,6 +17,7 @@ import type { RenderedPanel } from "#src/display_context.js"; import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { RefCounted } from "#src/util/disposable.js"; +import {NullarySignal} from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; @@ -174,6 +175,9 @@ export class ScreenshotManager extends RefCounted { gpuMemoryUsageDescription: "", downloadSpeedDescription: "", }; + screenshotMode: ScreenshotMode = ScreenshotMode.OFF; + statisticsUpdated = new NullarySignal(); + screenshotFinished = new NullarySignal(); constructor(public viewer: Viewer) { super(); @@ -181,6 +185,7 @@ export class ScreenshotManager extends RefCounted { this.registerDisposer( this.viewer.screenshotHandler.sendScreenshotRequested.add( (actionState) => { + this.screenshotFinished.dispatch(); this.saveScreenshot(actionState); }, ), @@ -190,6 +195,7 @@ export class ScreenshotManager extends RefCounted { (actionState) => { this.persistStatisticsData(actionState); this.checkAndHandleStalledScreenshot(actionState); + this.statisticsUpdated.dispatch(); }, ), ); @@ -255,7 +261,8 @@ export class ScreenshotManager extends RefCounted { private handleScreenshotModeChange() { const { display } = this.viewer; - switch (display.screenshotMode.value) { + this.screenshotMode = display.screenshotMode.value; + switch (this.screenshotMode) { case ScreenshotMode.OFF: this.resetCanvasSize(); this.resetStatistics(); diff --git a/src/viewer.ts b/src/viewer.ts index 6373b0d85..2e38fa6d5 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -1151,7 +1151,7 @@ export class Viewer extends RefCounted implements ViewerState { } showScreenshotDialog() { - new ScreenshotDialog(this); + new ScreenshotDialog(this.screenshotManager); } showStatistics(value: boolean | undefined = undefined) { From f928d071b0c62f7c44428cbc05c4647dc4fc6709 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 3 Sep 2024 18:34:45 +0200 Subject: [PATCH 35/66] refactor: make viewer menu responsible for manipulating data from controller to UI also make the "model" responsible for storing interfaces for its own action states --- src/python_integration/screenshots.ts | 41 ++++++- src/ui/screenshot_menu.ts | 81 ++++++++++---- src/util/screenshot.ts | 153 +++++++------------------- 3 files changed, 142 insertions(+), 133 deletions(-) diff --git a/src/python_integration/screenshots.ts b/src/python_integration/screenshots.ts index ac3b0b2d2..836375668 100644 --- a/src/python_integration/screenshots.ts +++ b/src/python_integration/screenshots.ts @@ -31,9 +31,46 @@ import { getCachedJson } from "#src/util/trackable.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; +export interface ScreenshotActionState { + viewerState: any; + selectedValues: any; + screenshot: { + id: string; + image: string; + imageType: string; + depthData: string | undefined; + width: number; + height: number; + }; +} + +export interface ScreenshotChunkStatistics { + downloadLatency: number; + visibleChunksDownloading: number; + visibleChunksFailed: number; + visibleChunksGpuMemory: number; + visibleChunksSystemMemory: number; + visibleChunksTotal: number; + visibleGpuMemory: number; +} + +export interface StatisticsActionState { + viewerState: any; + selectedValues: any; + screenshotStatistics: { + id: string; + chunkSources: any[]; + total: ScreenshotChunkStatistics; + }; +} + export class ScreenshotHandler extends RefCounted { - sendScreenshotRequested = new Signal<(state: any) => void>(); - sendStatisticsRequested = new Signal<(state: any) => void>(); + sendScreenshotRequested = new Signal< + (state: ScreenshotActionState) => void + >(); + sendStatisticsRequested = new Signal< + (state: StatisticsActionState) => void + >(); requestState = new TrackableValue( undefined, verifyOptionalString, diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 321f2a570..49c44570e 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -18,10 +18,21 @@ import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import "#src/ui/screenshot_menu.css"; -import type { ScreenshotManager } from "#src/util/screenshot.js"; +import type { + ScreenshotLoadStatistics, + ScreenshotManager, +} from "#src/util/screenshot.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; +interface UIScreenshotStatistics { + timeElapsedString: string | null; + chunkUsageDescription: string; + gpuMemoryUsageDescription: string; + downloadSpeedDescription: string; +} + const statisticsNamesForUI = { + timeElapsedString: "Screenshot duration", chunkUsageDescription: "Number of loaded chunks", gpuMemoryUsageDescription: "Visible chunk GPU memory usage", downloadSpeedDescription: "Number of downloading chunks", @@ -155,24 +166,26 @@ export class ScreenshotDialog extends Overlay { const headerRow = this.statisticsTable.createTHead().insertRow(); const keyHeader = document.createElement("th"); - keyHeader.textContent = "Screenshot in progress"; + keyHeader.textContent = "Screenshot statistics"; headerRow.appendChild(keyHeader); const valueHeader = document.createElement("th"); valueHeader.textContent = ""; headerRow.appendChild(valueHeader); // Populate inital table elements with placeholder text - const statsRow = this.screenshotManager.screenshotStatistics; - for (const key in statsRow) { - if (key === "timeElapsedString") { - continue; - } + const orderedStatsRow: UIScreenshotStatistics = { + chunkUsageDescription: "Loading...", + gpuMemoryUsageDescription: "Loading...", + downloadSpeedDescription: "Loading...", + timeElapsedString: "Loading...", + }; + for (const key in orderedStatsRow) { const row = this.statisticsTable.insertRow(); const keyCell = row.insertCell(); const valueCell = row.insertCell(); keyCell.textContent = statisticsNamesForUI[key as keyof typeof statisticsNamesForUI]; - valueCell.textContent = "Loading..."; + valueCell.textContent = orderedStatsRow[key as keyof typeof orderedStatsRow]; this.statisticsKeyToCellMap.set(key, valueCell); } @@ -203,26 +216,54 @@ export class ScreenshotDialog extends Overlay { } private populateStatistics() { - const statsRow = this.screenshotManager.screenshotStatistics; + const statsRow = this.parseStatistics( + this.screenshotManager.screenshotLoadStats, + ); for (const key in statsRow) { - if (key === "timeElapsedString") { - const headerRow = this.statisticsTable.rows[0]; - const keyHeader = headerRow.cells[0]; - const time = statsRow[key]; - if (time === null) { - keyHeader.textContent = "Screenshot in progress (statistics loading)"; - } else { - keyHeader.textContent = `Screenshot in progress for ${statsRow[key]}s`; - } - continue; - } this.statisticsKeyToCellMap.get(key)!.textContent = String( statsRow[key as keyof typeof statsRow], ); } } + private parseStatistics( + currentStatistics: ScreenshotLoadStatistics | null, + ): UIScreenshotStatistics { + const nowtime = Date.now(); + if (currentStatistics === null) { + return { + timeElapsedString: "Loading...", + chunkUsageDescription: "Loading...", + gpuMemoryUsageDescription: "Loading...", + downloadSpeedDescription: "Loading...", + }; + } + + const percentLoaded = + currentStatistics.visibleChunksTotal === 0 + ? 0 + : (100 * currentStatistics.visibleChunksGpuMemory) / + currentStatistics.visibleChunksTotal; + const percentGpuUsage = + (100 * currentStatistics.visibleGpuMemory) / + currentStatistics.gpuMemoryCapacity; + const gpuMemoryUsageInMB = currentStatistics.visibleGpuMemory / 1000000; + const totalMemoryInMB = currentStatistics.gpuMemoryCapacity / 1000000; + const latency = isNaN(currentStatistics.downloadLatency) + ? 0 + : currentStatistics.downloadLatency; + const passedTimeInSeconds = + (nowtime - this.screenshotManager.screenshotStartTime) / 1000; + + return { + timeElapsedString: `${passedTimeInSeconds.toFixed(0)} seconds`, + chunkUsageDescription: `${currentStatistics.visibleChunksGpuMemory} out of ${currentStatistics.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, + gpuMemoryUsageDescription: `${gpuMemoryUsageInMB.toFixed(0)}MB / ${totalMemoryInMB.toFixed(0)}MB (${percentGpuUsage.toFixed(2)}% of total)`, + downloadSpeedDescription: `${currentStatistics.visibleChunksDownloading} at ${latency.toFixed(0)}ms latency`, + }; + } + private debouncedUpdateUIElements = debounce(() => { this.updateSetupUIVisibility(); this.updateStatisticsTableDisplayBasedOnMode(); diff --git a/src/util/screenshot.ts b/src/util/screenshot.ts index d7abb68a5..21dd875ca 100644 --- a/src/util/screenshot.ts +++ b/src/util/screenshot.ts @@ -15,55 +15,22 @@ */ import type { RenderedPanel } from "#src/display_context.js"; +import type { + ScreenshotActionState, + StatisticsActionState, + ScreenshotChunkStatistics, +} from "#src/python_integration/screenshots.js"; import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { RefCounted } from "#src/util/disposable.js"; -import {NullarySignal} from "#src/util/signal.js"; +import { NullarySignal, Signal } from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; const SCREENSHOT_TIMEOUT = 5000; -interface ScreenshotLoadStatistics { - numGpuLoadedVisibleChunks: number; +export interface ScreenshotLoadStatistics extends ScreenshotChunkStatistics { timestamp: number; -} - -interface ScreenshotActionState { - viewerState: any; - selectedValues: any; - screenshot: { - id: string; - image: string; - imageType: string; - depthData: string | undefined; - width: number; - height: number; - }; -} - -export interface StatisticsActionState { - viewerState: any; - selectedValues: any; - screenshotStatistics: { - id: string; - chunkSources: any[]; - total: { - downloadLatency: number; - visibleChunksDownloading: number; - visibleChunksFailed: number; - visibleChunksGpuMemory: number; - visibleChunksSystemMemory: number; - visibleChunksTotal: number; - visibleGpuMemory: number; - }; - }; -} - -interface UIScreenshotStatistics { - timeElapsedString: string | null; - chunkUsageDescription: string; - gpuMemoryUsageDescription: string; - downloadSpeedDescription: string; + gpuMemoryCapacity: number; } interface ViewportBounds { @@ -160,23 +127,15 @@ async function extractViewportScreenshot( } export class ScreenshotManager extends RefCounted { - public screenshotId: number = -1; - public screenshotScale: number = 1; private filename: string = ""; - private screenshotLoadStats: ScreenshotLoadStatistics = { - numGpuLoadedVisibleChunks: 0, - timestamp: 0, - }; - private lastUpdateTimestamp = 0; - private screenshotStartTime = 0; - private lastSavedStatistics: UIScreenshotStatistics = { - timeElapsedString: null, - chunkUsageDescription: "", - gpuMemoryUsageDescription: "", - downloadSpeedDescription: "", - }; + private lastUpdateTimestamp: number = 0; + private gpuMemoryChangeTimestamp: number = 0; + screenshotId: number = -1; + screenshotScale: number = 1; + screenshotLoadStats: ScreenshotLoadStatistics | null = null; + screenshotStartTime = 0; screenshotMode: ScreenshotMode = ScreenshotMode.OFF; - statisticsUpdated = new NullarySignal(); + statisticsUpdated = new Signal<(state: ScreenshotLoadStatistics) => void>(); screenshotFinished = new NullarySignal(); constructor(public viewer: Viewer) { @@ -193,9 +152,15 @@ export class ScreenshotManager extends RefCounted { this.registerDisposer( this.viewer.screenshotHandler.sendStatisticsRequested.add( (actionState) => { - this.persistStatisticsData(actionState); this.checkAndHandleStalledScreenshot(actionState); - this.statisticsUpdated.dispatch(); + this.screenshotLoadStats = { + ...actionState.screenshotStatistics.total, + timestamp: Date.now(), + gpuMemoryCapacity: + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit + .value, + }; + this.statisticsUpdated.dispatch(this.screenshotLoadStats); }, ), ); @@ -220,19 +185,15 @@ export class ScreenshotManager extends RefCounted { this.viewer.display.screenshotMode.value = ScreenshotMode.FORCE; } - get screenshotStatistics(): UIScreenshotStatistics { - return this.lastSavedStatistics; - } - private handleScreenshotStarted() { const { viewer } = this; const shouldIncreaseCanvasSize = this.screenshotScale !== 1; - this.screenshotStartTime = this.lastUpdateTimestamp = Date.now(); - this.screenshotLoadStats = { - numGpuLoadedVisibleChunks: 0, - timestamp: 0, - }; + this.screenshotStartTime = + this.lastUpdateTimestamp = + this.gpuMemoryChangeTimestamp = + Date.now(); + this.screenshotLoadStats = null; if (shouldIncreaseCanvasSize) { const oldSize = { @@ -282,31 +243,33 @@ export class ScreenshotManager extends RefCounted { * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. */ private checkAndHandleStalledScreenshot(actionState: StatisticsActionState) { + if (this.screenshotLoadStats === null) { + return; + } const total = actionState.screenshotStatistics.total; - const newStats: ScreenshotLoadStatistics = { - numGpuLoadedVisibleChunks: total.visibleChunksGpuMemory, + const newStats = { + visibleChunksGpuMemory: total.visibleChunksGpuMemory, timestamp: Date.now(), + totalGpuMemory: + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value, }; - if (this.screenshotLoadStats.timestamp === 0) { - this.screenshotLoadStats = newStats; - return; - } const oldStats = this.screenshotLoadStats; if ( - oldStats.numGpuLoadedVisibleChunks === newStats.numGpuLoadedVisibleChunks + oldStats.visibleChunksGpuMemory === newStats.visibleChunksGpuMemory && + oldStats.gpuMemoryCapacity === newStats.totalGpuMemory ) { if ( - newStats.timestamp - oldStats.timestamp > SCREENSHOT_TIMEOUT && + newStats.timestamp - this.gpuMemoryChangeTimestamp > + SCREENSHOT_TIMEOUT && Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT ) { - const totalChunks = total.visibleChunksTotal; console.warn( - `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${newStats.numGpuLoadedVisibleChunks}/${totalChunks}`, + `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${total.visibleChunksGpuMemory}/${total.visibleChunksTotal}`, ); this.forceScreenshot(); } } else { - this.screenshotLoadStats = newStats; + this.gpuMemoryChangeTimestamp = newStats.timestamp; } } @@ -359,39 +322,7 @@ export class ScreenshotManager extends RefCounted { } private resetStatistics() { - this.lastSavedStatistics = { - timeElapsedString: null, - chunkUsageDescription: "", - gpuMemoryUsageDescription: "", - downloadSpeedDescription: "", - }; - } - - private persistStatisticsData(actionState: StatisticsActionState) { - const nowtime = Date.now(); - const total = actionState.screenshotStatistics.total; - const maxGpuMemory = - this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value; - - const percentLoaded = - total.visibleChunksTotal === 0 - ? 0 - : (100 * total.visibleChunksGpuMemory) / total.visibleChunksTotal; - const percentGpuUsage = (100 * total.visibleGpuMemory) / maxGpuMemory; - const gpuMemoryUsageInMB = total.visibleGpuMemory / 1000000; - const totalMemoryInMB = maxGpuMemory / 1000000; - const latency = isNaN(total.downloadLatency) ? 0 : total.downloadLatency; - const passedTimeInSeconds = ( - (nowtime - this.screenshotStartTime) / - 1000 - ).toFixed(0); - - this.lastSavedStatistics = { - timeElapsedString: passedTimeInSeconds, - chunkUsageDescription: `${total.visibleChunksGpuMemory} out of ${total.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, - gpuMemoryUsageDescription: `${gpuMemoryUsageInMB.toFixed(0)}MB / ${totalMemoryInMB.toFixed(0)}MB (${percentGpuUsage.toFixed(2)}% of total)`, - downloadSpeedDescription: `${total.visibleChunksDownloading} at ${latency.toFixed(0)}ms latency`, - }; + this.screenshotLoadStats = null; } private generateFilename(width: number, height: number): string { From cf44402b2b6f4750e61699f2711be922d14e01a9 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 16 Sep 2024 15:48:01 +0200 Subject: [PATCH 36/66] refactor: rename screenshot manager file --- src/ui/screenshot_menu.ts | 2 +- src/util/{screenshot.ts => screenshot_manager.ts} | 0 src/viewer.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/util/{screenshot.ts => screenshot_manager.ts} (100%) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 49c44570e..fe1799c76 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -21,7 +21,7 @@ import "#src/ui/screenshot_menu.css"; import type { ScreenshotLoadStatistics, ScreenshotManager, -} from "#src/util/screenshot.js"; +} from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; interface UIScreenshotStatistics { diff --git a/src/util/screenshot.ts b/src/util/screenshot_manager.ts similarity index 100% rename from src/util/screenshot.ts rename to src/util/screenshot_manager.ts diff --git a/src/viewer.ts b/src/viewer.ts index 2e38fa6d5..bc1edc691 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -120,7 +120,7 @@ import { EventActionMap, KeyboardEventBinder, } from "#src/util/keyboard_bindings.js"; -import { ScreenshotManager } from "#src/util/screenshot.js"; +import { ScreenshotManager } from "#src/util/screenshot_manager.js"; import { NullarySignal } from "#src/util/signal.js"; import { CompoundTrackable, From 4fc91b8b71876562fb29f81b2e07a519234a0fe5 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 16 Sep 2024 16:04:36 +0200 Subject: [PATCH 37/66] feat: lock screenshot menu until cancelled or done --- src/ui/screenshot_menu.ts | 24 ++++++++++++++++++------ src/util/screenshot_manager.ts | 9 +++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index fe1799c76..9939cb9ed 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -41,7 +41,7 @@ const statisticsNamesForUI = { export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private saveButton: HTMLButtonElement; - private closeButton: HTMLButtonElement; + private cancelButton: HTMLButtonElement; private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; private statisticsContainer: HTMLDivElement; @@ -58,9 +58,9 @@ export class ScreenshotDialog extends Overlay { private initializeUI() { this.content.classList.add("neuroglancer-screenshot-dialog"); - this.closeButton = this.createButton( - "Close", - () => this.dispose(), + this.cancelButton = this.createButton( + "Cancel", + () => this.cancelScreenshot(), "neuroglancer-screenshot-close-button", ); this.saveButton = this.createButton("Take screenshot", () => @@ -78,7 +78,7 @@ export class ScreenshotDialog extends Overlay { this.filenameAndButtonsContainer.appendChild(this.createNameInput()); this.filenameAndButtonsContainer.appendChild(this.saveButton); - this.content.appendChild(this.closeButton); + this.content.appendChild(this.cancelButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(this.createStatisticsTable()); @@ -185,7 +185,8 @@ export class ScreenshotDialog extends Overlay { const valueCell = row.insertCell(); keyCell.textContent = statisticsNamesForUI[key as keyof typeof statisticsNamesForUI]; - valueCell.textContent = orderedStatsRow[key as keyof typeof orderedStatsRow]; + valueCell.textContent = + orderedStatsRow[key as keyof typeof orderedStatsRow]; this.statisticsKeyToCellMap.set(key, valueCell); } @@ -197,6 +198,12 @@ export class ScreenshotDialog extends Overlay { private forceScreenshot() { this.screenshotManager.forceScreenshot(); + this.dispose(); + } + + private cancelScreenshot() { + this.screenshotManager.cancelScreenshot(); + this.dispose(); } private screenshot() { @@ -213,6 +220,11 @@ export class ScreenshotDialog extends Overlay { } else { this.statisticsContainer.style.display = "block"; } + if (this.screenshotMode === ScreenshotMode.ON) { + this.cancelButton.textContent = "Cancel"; + } else { + this.cancelButton.textContent = "Close"; + } } private populateStatistics() { diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 21dd875ca..18a03539d 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -185,6 +185,14 @@ export class ScreenshotManager extends RefCounted { this.viewer.display.screenshotMode.value = ScreenshotMode.FORCE; } + cancelScreenshot() { + // Decrement the screenshot ID since the screenshot was cancelled + if (this.screenshotMode === ScreenshotMode.ON) { + this.screenshotId--; + } + this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; + } + private handleScreenshotStarted() { const { viewer } = this; const shouldIncreaseCanvasSize = this.screenshotScale !== 1; @@ -227,6 +235,7 @@ export class ScreenshotManager extends RefCounted { case ScreenshotMode.OFF: this.resetCanvasSize(); this.resetStatistics(); + this.viewer.screenshotHandler.requestState.value = undefined; break; case ScreenshotMode.FORCE: display.scheduleRedraw(); From a2d504b299a14e0711d023c35d333db8670356b4 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 16 Sep 2024 16:21:36 +0200 Subject: [PATCH 38/66] feat: close tool menus before opening JSON state editor or screenshot menus --- src/ui/tool.ts | 4 ++++ src/viewer.ts | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/ui/tool.ts b/src/ui/tool.ts index c422e430f..9dfa7775d 100644 --- a/src/ui/tool.ts +++ b/src/ui/tool.ts @@ -405,6 +405,10 @@ export class GlobalToolBinder extends RefCounted { this.activeTool_ = undefined; activation.dispose(); } + + public deactivate() { + this.debounceDeactivate(); + } } export class LocalToolBinder< diff --git a/src/viewer.ts b/src/viewer.ts index bc1edc691..f0d361c54 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -1146,11 +1146,17 @@ export class Viewer extends RefCounted implements ViewerState { this.globalToolBinder.activate(uppercase); } + deactivateTools() { + this.globalToolBinder.deactivate(); + } + editJsonState() { + this.deactivateTools(); new StateEditorDialog(this); } showScreenshotDialog() { + this.deactivateTools(); new ScreenshotDialog(this.screenshotManager); } From 7f61396eeb6923147eabd9a5d7761a521a340b01 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 16 Sep 2024 16:57:13 +0200 Subject: [PATCH 39/66] feat: show indication of screenshot size, warning if big, don't hide elements --- src/ui/screenshot_menu.css | 5 ++ src/ui/screenshot_menu.ts | 110 +++++++++++++++++++-------------- src/util/screenshot_manager.ts | 15 +++++ 3 files changed, 82 insertions(+), 48 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index 046147eed..79c7b7394 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -38,6 +38,7 @@ .neuroglancer-screenshot-button { cursor: pointer; + margin: 2px; } .neuroglancer-screenshot-close-button { @@ -66,4 +67,8 @@ background-color: #f8f8f8; font-weight: bold; color: #555; +} + +.neuroglancer-screenshot-warning { + color: red; } \ No newline at end of file diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 9939cb9ed..fc0fbd31d 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -14,16 +14,17 @@ * limitations under the License. */ +import "#src/ui/screenshot_menu.css"; import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; -import "#src/ui/screenshot_menu.css"; - import type { ScreenshotLoadStatistics, ScreenshotManager, } from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; +const LARGE_SCREENSHOT_SIZE = 4096 * 4096; + interface UIScreenshotStatistics { timeElapsedString: string | null; chunkUsageDescription: string; @@ -40,13 +41,14 @@ const statisticsNamesForUI = { export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; - private saveButton: HTMLButtonElement; - private cancelButton: HTMLButtonElement; + private takeScreenshotButton: HTMLButtonElement; + private cancelScreenshotButton: HTMLButtonElement; private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; private statisticsContainer: HTMLDivElement; - private scaleSelectContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; + private screenshotSizeText: HTMLDivElement; + private warningElement: HTMLDivElement; private statisticsKeyToCellMap: Map = new Map(); constructor(private screenshotManager: ScreenshotManager) { super(); @@ -58,31 +60,30 @@ export class ScreenshotDialog extends Overlay { private initializeUI() { this.content.classList.add("neuroglancer-screenshot-dialog"); - this.cancelButton = this.createButton( + this.cancelScreenshotButton = this.createButton( "Cancel", () => this.cancelScreenshot(), "neuroglancer-screenshot-close-button", ); - this.saveButton = this.createButton("Take screenshot", () => + this.takeScreenshotButton = this.createButton("Take screenshot", () => this.screenshot(), ); this.forceScreenshotButton = this.createButton("Force screenshot", () => this.forceScreenshot(), ); - this.forceScreenshotButton.title = - "Force a screenshot of the current view without waiting for all data to be loaded and rendered"; this.filenameAndButtonsContainer = document.createElement("div"); this.filenameAndButtonsContainer.classList.add( "neuroglancer-screenshot-filename-and-buttons", ); this.filenameAndButtonsContainer.appendChild(this.createNameInput()); - this.filenameAndButtonsContainer.appendChild(this.saveButton); + this.filenameAndButtonsContainer.appendChild(this.takeScreenshotButton); + this.filenameAndButtonsContainer.appendChild(this.forceScreenshotButton); - this.content.appendChild(this.cancelButton); + this.content.appendChild(this.cancelScreenshotButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(this.createStatisticsTable()); - this.updateSetupUIVisibility(); + this.updateUIBasedOnMode(); } private setupEventListeners() { @@ -120,14 +121,21 @@ export class ScreenshotDialog extends Overlay { } private createScaleRadioButtons() { - const scaleMenu = (this.scaleSelectContainer = - document.createElement("div")); + const scaleMenu = document.createElement("div"); scaleMenu.classList.add("neuroglancer-screenshot-scale-menu"); + this.screenshotSizeText = document.createElement("div"); + this.screenshotSizeText.classList.add("neuroglancer-screenshot-size-text"); + scaleMenu.appendChild(this.screenshotSizeText); + const scaleLabel = document.createElement("label"); scaleLabel.textContent = "Screenshot scale factor:"; scaleMenu.appendChild(scaleLabel); + this.warningElement = document.createElement("div"); + this.warningElement.classList.add("neuroglancer-screenshot-warning"); + this.warningElement.textContent = ""; + const scales = [1, 2, 4]; scales.forEach((scale) => { const label = document.createElement("label"); @@ -146,17 +154,31 @@ export class ScreenshotDialog extends Overlay { input.addEventListener("change", () => { this.screenshotManager.screenshotScale = scale; + this.handleScreenshotResize(); }); }); + scaleMenu.appendChild(this.warningElement); + this.handleScreenshotResize(); return scaleMenu; } + private handleScreenshotResize() { + const screenshotSize = + this.screenshotManager.calculatedScaledAndClippedSize(); + if (screenshotSize.width * screenshotSize.height > LARGE_SCREENSHOT_SIZE) { + this.warningElement.textContent = + "Warning: large screenshots (bigger than 4096x4096) may fail"; + } else { + this.warningElement.textContent = ""; + } + this.screenshotSizeText.textContent = `Screenshot size: ${screenshotSize.width}px, ${screenshotSize.height}px`; + } + private createStatisticsTable() { this.statisticsContainer = document.createElement("div"); this.statisticsContainer.classList.add( "neuroglancer-screenshot-statistics-title", ); - this.statisticsContainer.appendChild(this.forceScreenshotButton); this.statisticsTable = document.createElement("table"); this.statisticsTable.classList.add( @@ -174,10 +196,10 @@ export class ScreenshotDialog extends Overlay { // Populate inital table elements with placeholder text const orderedStatsRow: UIScreenshotStatistics = { - chunkUsageDescription: "Loading...", - gpuMemoryUsageDescription: "Loading...", - downloadSpeedDescription: "Loading...", - timeElapsedString: "Loading...", + chunkUsageDescription: "", + gpuMemoryUsageDescription: "", + downloadSpeedDescription: "", + timeElapsedString: "", }; for (const key in orderedStatsRow) { const row = this.statisticsTable.insertRow(); @@ -191,7 +213,6 @@ export class ScreenshotDialog extends Overlay { } this.populateStatistics(); - this.updateStatisticsTableDisplayBasedOnMode(); this.statisticsContainer.appendChild(this.statisticsTable); return this.statisticsContainer; } @@ -214,23 +235,16 @@ export class ScreenshotDialog extends Overlay { this.debouncedUpdateUIElements(); } - private updateStatisticsTableDisplayBasedOnMode() { + private populateStatistics() { if (this.screenshotMode === ScreenshotMode.OFF) { - this.statisticsContainer.style.display = "none"; - } else { - this.statisticsContainer.style.display = "block"; - } - if (this.screenshotMode === ScreenshotMode.ON) { - this.cancelButton.textContent = "Cancel"; - } else { - this.cancelButton.textContent = "Close"; + return; } - } - - private populateStatistics() { const statsRow = this.parseStatistics( this.screenshotManager.screenshotLoadStats, ); + if (statsRow === null) { + return; + } for (const key in statsRow) { this.statisticsKeyToCellMap.get(key)!.textContent = String( @@ -241,15 +255,10 @@ export class ScreenshotDialog extends Overlay { private parseStatistics( currentStatistics: ScreenshotLoadStatistics | null, - ): UIScreenshotStatistics { + ): UIScreenshotStatistics | null { const nowtime = Date.now(); if (currentStatistics === null) { - return { - timeElapsedString: "Loading...", - chunkUsageDescription: "Loading...", - gpuMemoryUsageDescription: "Loading...", - downloadSpeedDescription: "Loading...", - }; + return null; } const percentLoaded = @@ -277,19 +286,24 @@ export class ScreenshotDialog extends Overlay { } private debouncedUpdateUIElements = debounce(() => { - this.updateSetupUIVisibility(); - this.updateStatisticsTableDisplayBasedOnMode(); + this.updateUIBasedOnMode(); }, 100); - private updateSetupUIVisibility() { + private updateUIBasedOnMode() { if (this.screenshotMode === ScreenshotMode.OFF) { - this.forceScreenshotButton.style.display = "none"; - this.filenameAndButtonsContainer.style.display = "block"; - this.scaleSelectContainer.style.display = "block"; + this.forceScreenshotButton.disabled = true; + this.takeScreenshotButton.disabled = false; + this.forceScreenshotButton.title = ""; + } else { + this.forceScreenshotButton.disabled = false; + this.takeScreenshotButton.disabled = true; + this.forceScreenshotButton.title = + "Force a screenshot of the current view without waiting for all data to be loaded and rendered"; + } + if (this.screenshotMode === ScreenshotMode.ON) { + this.cancelScreenshotButton.textContent = "Cancel"; } else { - this.forceScreenshotButton.style.display = "block"; - this.filenameAndButtonsContainer.style.display = "none"; - this.scaleSelectContainer.style.display = "none"; + this.cancelScreenshotButton.textContent = "Close"; } } diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 18a03539d..90ff88360 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -193,6 +193,21 @@ export class ScreenshotManager extends RefCounted { this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; } + // Scales the screenshot by the given factor, and calculates the cropped area + calculatedScaledAndClippedSize() { + const renderingPanelArea = calculateViewportBounds( + this.viewer.display.panels, + ); + return { + width: + Math.round(renderingPanelArea.right - renderingPanelArea.left) * + this.screenshotScale, + height: + Math.round(renderingPanelArea.bottom - renderingPanelArea.top) * + this.screenshotScale, + }; + } + private handleScreenshotStarted() { const { viewer } = this; const shouldIncreaseCanvasSize = this.screenshotScale !== 1; From 15bf2975cf39c7b130b7aaa89a26ddc8747fd78a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 16 Sep 2024 19:03:58 +0200 Subject: [PATCH 40/66] feat: progress on image (slice and volume) res stats --- src/ui/screenshot_menu.ts | 29 +++++++++++++++++++++ src/volume_rendering/volume_render_layer.ts | 16 +++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index fc0fbd31d..a29019464 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -17,11 +17,14 @@ import "#src/ui/screenshot_menu.css"; import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; +import { RenderLayerRole } from "#src/renderlayer.js"; import type { ScreenshotLoadStatistics, ScreenshotManager, } from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; +import { ImageRenderLayer } from "#src/sliceview/volume/image_renderlayer.js"; +import { VolumeRenderingRenderLayer } from "#src/volume_rendering/volume_render_layer.js"; const LARGE_SCREENSHOT_SIZE = 4096 * 4096; @@ -55,6 +58,7 @@ export class ScreenshotDialog extends Overlay { this.initializeUI(); this.setupEventListeners(); + this.parseLayerStatistics(); } private initializeUI() { @@ -285,6 +289,31 @@ export class ScreenshotDialog extends Overlay { }; } + private parseLayerStatistics() { + const layers = + this.screenshotManager.viewer.layerManager.visibleRenderLayers; + for (const layer of layers) { + if (layer.role === RenderLayerRole.DATA) { + console.log("Layer: ", layer); + if (layer instanceof ImageRenderLayer) { + // Use refCount to see if it is in any panels? + console.log("ImageRenderLayer: ", layer); + const sliceResolution = layer.renderScaleTarget.value; + console.log("Slice Resolution: ", sliceResolution); + } + if (layer instanceof VolumeRenderingRenderLayer) { + console.log("VolumeRenderingRenderLayer: ", layer); + const volumeResolution = layer.depthSamplesTarget.value; + console.log("Volume Resolution: ", volumeResolution); + const physicalSpacing = layer.physicalSpacing; + console.log("Physical Spacing: ", physicalSpacing); + const resolutionIndex = layer.selectedDataResolution; + console.log("Resolution Index: ", resolutionIndex); + } + } + } + } + private debouncedUpdateUIElements = debounce(() => { this.updateUIBasedOnMode(); }, 100); diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index f7b68bfed..a1f55959c 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -223,6 +223,8 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { private modeOverride: TrackableVolumeRenderingModeValue; private vertexIdHelper: VertexIdHelper; private dataHistogramSpecifications: HistogramSpecifications; + private physicalSpacingForDepthSamples: number; + private dataResolutionIndex: number; private shaderGetter: ParameterizedContextDependentShaderGetter< { emitter: ShaderModule; chunkFormat: ChunkFormat; wireFrame: boolean }, @@ -248,6 +250,14 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { return true; } + get physicalSpacing() { + return this.physicalSpacingForDepthSamples; + } + + get selectedDataResolution() { + return this.dataResolutionIndex; + } + getDataHistogramCount() { return this.dataHistogramSpecifications.visibleHistograms; } @@ -747,7 +757,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); if (!renderContext.emitColor) return; const allSources = attachment.state!.sources.value; if (allSources.length === 0) return; - let curPhysicalSpacing = 0; let curOptimalSamples = 0; let curHistogramInformation: HistogramInformation = { spatialScales: new Map(), @@ -827,7 +836,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); }, ); renderScaleHistogram.add( - curPhysicalSpacing, + this.physicalSpacingForDepthSamples, curOptimalSamples, presentCount, notPresentCount, @@ -881,9 +890,10 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); ) => { ignored1; ignored2; - curPhysicalSpacing = physicalSpacing; + this.physicalSpacingForDepthSamples = physicalSpacing; curOptimalSamples = optimalSamples; curHistogramInformation = histogramInformation; + this.dataResolutionIndex = histogramInformation.activeIndex; const chunkLayout = getNormalizedChunkLayout( projectionParameters, transformedSource.chunkLayout, From 3e2afd8875663d07e8b5eed18754b7b5695fa6eb Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 17 Sep 2024 11:22:30 +0200 Subject: [PATCH 41/66] feat: resolution from each layer type --- src/ui/screenshot_menu.ts | 71 +++++++++++++++++------------ src/util/viewer_resolution_stats.ts | 60 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 29 deletions(-) create mode 100644 src/util/viewer_resolution_stats.ts diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index a29019464..0375a74e4 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -17,14 +17,12 @@ import "#src/ui/screenshot_menu.css"; import { debounce } from "lodash-es"; import { Overlay } from "#src/overlay.js"; -import { RenderLayerRole } from "#src/renderlayer.js"; import type { ScreenshotLoadStatistics, ScreenshotManager, } from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; -import { ImageRenderLayer } from "#src/sliceview/volume/image_renderlayer.js"; -import { VolumeRenderingRenderLayer } from "#src/volume_rendering/volume_render_layer.js"; +import { getViewerResolutionState } from "#src/util/viewer_resolution_stats.js"; const LARGE_SCREENSHOT_SIZE = 4096 * 4096; @@ -42,6 +40,13 @@ const statisticsNamesForUI = { downloadSpeedDescription: "Number of downloading chunks", }; +const layerNamesForUI = { + ImageRenderLayer: "Image", + VolumeRenderingRenderLayer: "Volume", + SegmentationRenderLayer: "Segmentation", + MultiscaleMeshLayer: "Mesh", +}; + export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private takeScreenshotButton: HTMLButtonElement; @@ -58,7 +63,6 @@ export class ScreenshotDialog extends Overlay { this.initializeUI(); this.setupEventListeners(); - this.parseLayerStatistics(); } private initializeUI() { @@ -86,6 +90,7 @@ export class ScreenshotDialog extends Overlay { this.content.appendChild(this.cancelScreenshotButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); + this.content.appendChild(this.createResolutionTable()); this.content.appendChild(this.createStatisticsTable()); this.updateUIBasedOnMode(); } @@ -221,6 +226,39 @@ export class ScreenshotDialog extends Overlay { return this.statisticsContainer; } + private createResolutionTable() { + const resolutionTable = document.createElement("table"); + resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); + resolutionTable.title = "Viewer resolution statistics"; + + const headerRow = resolutionTable.createTHead().insertRow(); + const keyHeader = document.createElement("th"); + keyHeader.textContent = "Name"; + headerRow.appendChild(keyHeader); + const typeHeader = document.createElement("th"); + typeHeader.textContent = "Type"; + headerRow.appendChild(typeHeader); + const valueHeader = document.createElement("th"); + valueHeader.textContent = "Resolution"; + headerRow.appendChild(valueHeader); + + const resolutionMap = getViewerResolutionState( + this.screenshotManager.viewer, + ); + for (const [key, value] of resolutionMap) { + const row = resolutionTable.insertRow(); + const keyCell = row.insertCell(); + const typeCell = row.insertCell(); + const valueCell = row.insertCell(); + const name = key[0]; + keyCell.textContent = name; + typeCell.textContent = + layerNamesForUI[key[1] as keyof typeof layerNamesForUI]; + valueCell.textContent = JSON.stringify(value); + } + return resolutionTable; + } + private forceScreenshot() { this.screenshotManager.forceScreenshot(); this.dispose(); @@ -289,31 +327,6 @@ export class ScreenshotDialog extends Overlay { }; } - private parseLayerStatistics() { - const layers = - this.screenshotManager.viewer.layerManager.visibleRenderLayers; - for (const layer of layers) { - if (layer.role === RenderLayerRole.DATA) { - console.log("Layer: ", layer); - if (layer instanceof ImageRenderLayer) { - // Use refCount to see if it is in any panels? - console.log("ImageRenderLayer: ", layer); - const sliceResolution = layer.renderScaleTarget.value; - console.log("Slice Resolution: ", sliceResolution); - } - if (layer instanceof VolumeRenderingRenderLayer) { - console.log("VolumeRenderingRenderLayer: ", layer); - const volumeResolution = layer.depthSamplesTarget.value; - console.log("Volume Resolution: ", volumeResolution); - const physicalSpacing = layer.physicalSpacing; - console.log("Physical Spacing: ", physicalSpacing); - const resolutionIndex = layer.selectedDataResolution; - console.log("Resolution Index: ", resolutionIndex); - } - } - } - } - private debouncedUpdateUIElements = debounce(() => { this.updateUIBasedOnMode(); }, 100); diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts new file mode 100644 index 000000000..1a075cc30 --- /dev/null +++ b/src/util/viewer_resolution_stats.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use viewer file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; +import { MultiscaleMeshLayer } from "#src/mesh/frontend.js"; +import { RenderLayerRole } from "#src/renderlayer.js"; +import { ImageRenderLayer } from "#src/sliceview/volume/image_renderlayer.js"; +import { SegmentationRenderLayer } from "#src/sliceview/volume/segmentation_renderlayer.js"; +import type { Viewer } from "#src/viewer.js"; +import { VolumeRenderingRenderLayer } from "#src/volume_rendering/volume_render_layer.js"; + +export function getViewerResolutionState(viewer: Viewer) { + const layers = viewer.layerManager.visibleRenderLayers; + const map = new Map(); + for (const layer of layers) { + if (layer.role === RenderLayerRole.DATA) { + const layer_name = layer.userLayer!.managedLayer.name; + if (layer instanceof ImageRenderLayer) { + const type = "ImageRenderLayer"; + const sliceResolution = layer.renderScaleTarget.value; + map.set([layer_name, type], { sliceResolution }); + } else if (layer instanceof VolumeRenderingRenderLayer) { + const type = "VolumeRenderingRenderLayer"; + const volumeResolution = layer.depthSamplesTarget.value; + const physicalSpacing = layer.physicalSpacing; + const resolutionIndex = layer.selectedDataResolution; + map.set([layer_name, type], { + volumeResolution, + physicalSpacing, + resolutionIndex, + }); + } else if (layer instanceof SegmentationRenderLayer) { + const type = "SegmentationRenderLayer"; + const segmentationResolution = layer.renderScaleTarget.value; + map.set([layer_name, type], { + sliceResolution: segmentationResolution, + }); + } else if (layer instanceof MultiscaleMeshLayer) { + const type = "MultiscaleMeshLayer"; + const userLayer = layer.userLayer as SegmentationUserLayer; + const meshResolution = userLayer.displayState.renderScaleTarget.value; + map.set([layer_name, type], { meshResolution }); + } + } + } + return map; +} From 5d1a0eee985a589175ced30db38ef81980ea6fc2 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 17 Sep 2024 16:18:45 +0200 Subject: [PATCH 42/66] feat: add per panel resolution indicator --- src/ui/screenshot_menu.ts | 78 ++++++++++++++++-- src/util/viewer_resolution_stats.ts | 122 +++++++++++++++++++++++++--- 2 files changed, 181 insertions(+), 19 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 0375a74e4..f9726dff5 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -22,8 +22,14 @@ import type { ScreenshotManager, } from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; -import { getViewerResolutionState } from "#src/util/viewer_resolution_stats.js"; - +import { + getViewerLayerResolutions, + getViewerPanelResolutions, +} from "#src/util/viewer_resolution_stats.js"; + +// If true, the menu can be closed by clicking the close button +// Usually the user is locked into the screenshot menu until the screenshot is taken or cancelled +const DEBUG_ALLOW_MENU_CLOSE = false; const LARGE_SCREENSHOT_SIZE = 4096 * 4096; interface UIScreenshotStatistics { @@ -90,7 +96,8 @@ export class ScreenshotDialog extends Overlay { this.content.appendChild(this.cancelScreenshotButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); - this.content.appendChild(this.createResolutionTable()); + this.content.appendChild(this.createPanelResolutionTable()); + this.content.appendChild(this.createLayerResolutionTable()); this.content.appendChild(this.createStatisticsTable()); this.updateUIBasedOnMode(); } @@ -226,7 +233,61 @@ export class ScreenshotDialog extends Overlay { return this.statisticsContainer; } - private createResolutionTable() { + private createPanelResolutionTable() { + function formatResolution(resolution: any) { + const first_resolution = resolution[0]; + if (first_resolution.name === "All_") { + return { + type: first_resolution.panelType, + resolution: first_resolution.textContent, + }; + } else { + let text = ""; + for (const res of resolution) { + text += `${res.name}: ${res.textContent}, `; + } + return { + type: first_resolution.panelType, + resolution: text, + }; + } + } + + const resolutionTable = document.createElement("table"); + resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); + resolutionTable.title = "Viewer resolution statistics"; + + const headerRow = resolutionTable.createTHead().insertRow(); + const keyHeader = document.createElement("th"); + keyHeader.textContent = "Panel type"; + headerRow.appendChild(keyHeader); + const valueHeader = document.createElement("th"); + valueHeader.textContent = "Resolution"; + headerRow.appendChild(valueHeader); + + const resolutions = getViewerPanelResolutions( + this.screenshotManager.viewer.display.panels, + ); + for (const resolution of resolutions) { + const resolutionStrings = formatResolution(resolution); + const row = resolutionTable.insertRow(); + const keyCell = row.insertCell(); + const valueCell = row.insertCell(); + keyCell.textContent = resolutionStrings.type; + valueCell.textContent = resolutionStrings.resolution; + } + return resolutionTable; + } + + private createLayerResolutionTable() { + function formatResolution(key: any, value: any) { + const type = key[1]; + const resolution: number = value.resolution; + const unit = type === "VolumeRenderingRenderLayer" ? " Z samples" : "px"; + const roundingLevel = type === "VolumeRenderingRenderLayer" ? 0 : 2; + + return `${resolution.toFixed(roundingLevel)} ${unit}`; + } const resolutionTable = document.createElement("table"); resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); resolutionTable.title = "Viewer resolution statistics"; @@ -242,7 +303,7 @@ export class ScreenshotDialog extends Overlay { valueHeader.textContent = "Resolution"; headerRow.appendChild(valueHeader); - const resolutionMap = getViewerResolutionState( + const resolutionMap = getViewerLayerResolutions( this.screenshotManager.viewer, ); for (const [key, value] of resolutionMap) { @@ -254,7 +315,7 @@ export class ScreenshotDialog extends Overlay { keyCell.textContent = name; typeCell.textContent = layerNamesForUI[key[1] as keyof typeof layerNamesForUI]; - valueCell.textContent = JSON.stringify(value); + valueCell.textContent = formatResolution(key, value); } return resolutionTable; } @@ -265,6 +326,11 @@ export class ScreenshotDialog extends Overlay { } private cancelScreenshot() { + if (DEBUG_ALLOW_MENU_CLOSE) { + this.dispose(); + return; + } + this.screenshotManager.cancelScreenshot(); this.dispose(); } diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 1a075cc30..455d27279 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -14,15 +14,21 @@ * limitations under the License. */ +import type { RenderedPanel } from "#src/display_context.js"; import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; import { MultiscaleMeshLayer } from "#src/mesh/frontend.js"; +import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { RenderLayerRole } from "#src/renderlayer.js"; +import { SliceViewPanel } from "#src/sliceview/panel.js"; import { ImageRenderLayer } from "#src/sliceview/volume/image_renderlayer.js"; import { SegmentationRenderLayer } from "#src/sliceview/volume/segmentation_renderlayer.js"; +import { formatScaleWithUnitAsString } from "#src/util/si_units.js"; import type { Viewer } from "#src/viewer.js"; import { VolumeRenderingRenderLayer } from "#src/volume_rendering/volume_render_layer.js"; -export function getViewerResolutionState(viewer: Viewer) { +export function getViewerLayerResolutions( + viewer: Viewer, +): Map<[string, string], any> { const layers = viewer.layerManager.visibleRenderLayers; const map = new Map(); for (const layer of layers) { @@ -30,31 +36,121 @@ export function getViewerResolutionState(viewer: Viewer) { const layer_name = layer.userLayer!.managedLayer.name; if (layer instanceof ImageRenderLayer) { const type = "ImageRenderLayer"; - const sliceResolution = layer.renderScaleTarget.value; - map.set([layer_name, type], { sliceResolution }); + const resolution = layer.renderScaleTarget.value; + map.set([layer_name, type], { resolution }); } else if (layer instanceof VolumeRenderingRenderLayer) { const type = "VolumeRenderingRenderLayer"; - const volumeResolution = layer.depthSamplesTarget.value; - const physicalSpacing = layer.physicalSpacing; - const resolutionIndex = layer.selectedDataResolution; + const resolution = layer.depthSamplesTarget.value; map.set([layer_name, type], { - volumeResolution, - physicalSpacing, - resolutionIndex, + resolution, }); } else if (layer instanceof SegmentationRenderLayer) { const type = "SegmentationRenderLayer"; - const segmentationResolution = layer.renderScaleTarget.value; + const resolution = layer.renderScaleTarget.value; map.set([layer_name, type], { - sliceResolution: segmentationResolution, + resolution, }); } else if (layer instanceof MultiscaleMeshLayer) { const type = "MultiscaleMeshLayer"; const userLayer = layer.userLayer as SegmentationUserLayer; - const meshResolution = userLayer.displayState.renderScaleTarget.value; - map.set([layer_name, type], { meshResolution }); + const resolution = userLayer.displayState.renderScaleTarget.value; + map.set([layer_name, type], { resolution }); } } } return map; } + +// TODO needs screenshotFactor +export function getViewerPanelResolutions(panels: ReadonlySet) { + function resolutionsEqual(resolution1: any[], resolution2: any[]) { + if (resolution1.length !== resolution2.length) { + return false; + } + for (let i = 0; i < resolution1.length; ++i) { + if (resolution1[i].textContent !== resolution2[i].textContent) { + return false; + } + if (resolution1[i].panelType !== resolution2[i].panelType) { + return false; + } + if (resolution1[i].name !== resolution2[i].name) { + return false; + } + } + return true; + } + + const resolutions: any[] = []; + for (const panel of panels) { + if (!(panel instanceof RenderedDataPanel)) continue; + const panel_resolution = []; + const displayDimensionUnit = panel instanceof SliceViewPanel ? "px" : "vh"; + const panelType = panel instanceof SliceViewPanel ? "Slice" : "3D"; + const { navigationState } = panel; + const { + displayDimensionIndices, + canonicalVoxelFactors, + displayDimensionUnits, + displayDimensionScales, + globalDimensionNames, + } = navigationState.displayDimensionRenderInfo.value; + const { factors } = navigationState.relativeDisplayScales.value; + const zoom = navigationState.zoomFactor.value; + // Check if all units and factors are the same. + const firstDim = displayDimensionIndices[0]; + let singleScale = true; + if (firstDim !== -1) { + const unit = displayDimensionUnits[0]; + const factor = factors[firstDim]; + for (let i = 1; i < 3; ++i) { + const dim = displayDimensionIndices[i]; + if (dim === -1) continue; + if (displayDimensionUnits[i] !== unit || factors[dim] !== factor) { + singleScale = false; + break; + } + } + } + for (let i = 0; i < 3; ++i) { + const dim = displayDimensionIndices[i]; + if (dim !== -1) { + const totalScale = + (displayDimensionScales[i] * zoom) / canonicalVoxelFactors[i]; + let textContent; + const name = globalDimensionNames[dim]; + if (i === 0 || !singleScale) { + const formattedScale = formatScaleWithUnitAsString( + totalScale, + displayDimensionUnits[i], + { precision: 2, elide1: false }, + ); + textContent = `${formattedScale}/${displayDimensionUnit}`; + if (singleScale) { + panel_resolution.push({ panelType, textContent, name: "All_" }); + } else { + panel_resolution.push({ panelType, textContent, name }); + } + } else { + textContent = ""; + } + } + } + resolutions.push(panel_resolution); + } + + const uniqueResolutions: any[] = []; + for (const resolution of resolutions) { + let found = false; + for (const uniqueResolution of uniqueResolutions) { + if (resolutionsEqual(resolution, uniqueResolution)) { + found = true; + break; + } + } + if (!found) { + uniqueResolutions.push(resolution); + } + } + return uniqueResolutions; +} From 7d2930ba85260a3241926f8da73af86bee7197af Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 17 Sep 2024 16:23:14 +0200 Subject: [PATCH 43/66] feat: separate screenshot cancel button --- src/ui/screenshot_menu.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index f9726dff5..092117598 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -56,6 +56,7 @@ const layerNamesForUI = { export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private takeScreenshotButton: HTMLButtonElement; + private closeMenuButton: HTMLButtonElement; private cancelScreenshotButton: HTMLButtonElement; private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; @@ -74,11 +75,14 @@ export class ScreenshotDialog extends Overlay { private initializeUI() { this.content.classList.add("neuroglancer-screenshot-dialog"); - this.cancelScreenshotButton = this.createButton( - "Cancel", - () => this.cancelScreenshot(), + this.closeMenuButton = this.createButton( + "Close", + () => this.dispose(), "neuroglancer-screenshot-close-button", ); + this.cancelScreenshotButton = this.createButton("Cancel screenshot", () => + this.cancelScreenshot(), + ); this.takeScreenshotButton = this.createButton("Take screenshot", () => this.screenshot(), ); @@ -93,6 +97,7 @@ export class ScreenshotDialog extends Overlay { this.filenameAndButtonsContainer.appendChild(this.takeScreenshotButton); this.filenameAndButtonsContainer.appendChild(this.forceScreenshotButton); + this.content.appendChild(this.closeMenuButton); this.content.appendChild(this.cancelScreenshotButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); @@ -400,19 +405,18 @@ export class ScreenshotDialog extends Overlay { private updateUIBasedOnMode() { if (this.screenshotMode === ScreenshotMode.OFF) { this.forceScreenshotButton.disabled = true; + this.cancelScreenshotButton.disabled = true; this.takeScreenshotButton.disabled = false; + this.closeMenuButton.disabled = false; this.forceScreenshotButton.title = ""; } else { this.forceScreenshotButton.disabled = false; + this.cancelScreenshotButton.disabled = false; this.takeScreenshotButton.disabled = true; + this.closeMenuButton.disabled = true; this.forceScreenshotButton.title = "Force a screenshot of the current view without waiting for all data to be loaded and rendered"; } - if (this.screenshotMode === ScreenshotMode.ON) { - this.cancelScreenshotButton.textContent = "Cancel"; - } else { - this.cancelScreenshotButton.textContent = "Close"; - } } get screenshotMode() { From 5f84eb607590156e595acd88051f98fefd112f32 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 17 Sep 2024 17:22:09 +0200 Subject: [PATCH 44/66] feat: separate close button --- src/ui/screenshot_menu.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 092117598..cf7cf88d0 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -299,7 +299,7 @@ export class ScreenshotDialog extends Overlay { const headerRow = resolutionTable.createTHead().insertRow(); const keyHeader = document.createElement("th"); - keyHeader.textContent = "Name"; + keyHeader.textContent = "Layer name"; headerRow.appendChild(keyHeader); const typeHeader = document.createElement("th"); typeHeader.textContent = "Type"; @@ -331,13 +331,8 @@ export class ScreenshotDialog extends Overlay { } private cancelScreenshot() { - if (DEBUG_ALLOW_MENU_CLOSE) { - this.dispose(); - return; - } - this.screenshotManager.cancelScreenshot(); - this.dispose(); + this.updateUIBasedOnMode(); } private screenshot() { @@ -417,6 +412,9 @@ export class ScreenshotDialog extends Overlay { this.forceScreenshotButton.title = "Force a screenshot of the current view without waiting for all data to be loaded and rendered"; } + if (DEBUG_ALLOW_MENU_CLOSE) { + this.closeMenuButton.disabled = false; + } } get screenshotMode() { From beeb35559260c60896ac8c6cb7b752af5a6f210a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 17 Sep 2024 17:58:43 +0200 Subject: [PATCH 45/66] feat(ui): show stats on menu even when screenshot not running --- src/ui/screenshot_menu.ts | 30 +++++++++---------- src/util/screenshot_manager.ts | 53 ++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index cf7cf88d0..de61d2cb8 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -33,14 +33,12 @@ const DEBUG_ALLOW_MENU_CLOSE = false; const LARGE_SCREENSHOT_SIZE = 4096 * 4096; interface UIScreenshotStatistics { - timeElapsedString: string | null; chunkUsageDescription: string; gpuMemoryUsageDescription: string; downloadSpeedDescription: string; } const statisticsNamesForUI = { - timeElapsedString: "Screenshot duration", chunkUsageDescription: "Number of loaded chunks", gpuMemoryUsageDescription: "Visible chunk GPU memory usage", downloadSpeedDescription: "Number of downloading chunks", @@ -70,6 +68,7 @@ export class ScreenshotDialog extends Overlay { this.initializeUI(); this.setupEventListeners(); + this.screenshotManager.throttledSendStatistics(); } private initializeUI() { @@ -114,8 +113,13 @@ export class ScreenshotDialog extends Overlay { }), ); this.registerDisposer( - this.screenshotManager.statisticsUpdated.add(() => { - this.populateStatistics(); + this.screenshotManager.statisticsUpdated.add((screenshotLoadStats) => { + this.populateStatistics(screenshotLoadStats); + }), + ); + this.registerDisposer( + this.screenshotManager.viewer.display.updateFinished.add(() => { + this.screenshotManager.throttledSendStatistics(); }), ); } @@ -220,7 +224,6 @@ export class ScreenshotDialog extends Overlay { chunkUsageDescription: "", gpuMemoryUsageDescription: "", downloadSpeedDescription: "", - timeElapsedString: "", }; for (const key in orderedStatsRow) { const row = this.statisticsTable.insertRow(); @@ -233,7 +236,7 @@ export class ScreenshotDialog extends Overlay { this.statisticsKeyToCellMap.set(key, valueCell); } - this.populateStatistics(); + this.populateStatistics(this.screenshotManager.screenshotLoadStats); this.statisticsContainer.appendChild(this.statisticsTable); return this.statisticsContainer; } @@ -343,13 +346,10 @@ export class ScreenshotDialog extends Overlay { this.debouncedUpdateUIElements(); } - private populateStatistics() { - if (this.screenshotMode === ScreenshotMode.OFF) { - return; - } - const statsRow = this.parseStatistics( - this.screenshotManager.screenshotLoadStats, - ); + private populateStatistics( + screenshotLoadStats: ScreenshotLoadStatistics | null, + ) { + const statsRow = this.parseStatistics(screenshotLoadStats); if (statsRow === null) { return; } @@ -364,7 +364,6 @@ export class ScreenshotDialog extends Overlay { private parseStatistics( currentStatistics: ScreenshotLoadStatistics | null, ): UIScreenshotStatistics | null { - const nowtime = Date.now(); if (currentStatistics === null) { return null; } @@ -382,11 +381,8 @@ export class ScreenshotDialog extends Overlay { const latency = isNaN(currentStatistics.downloadLatency) ? 0 : currentStatistics.downloadLatency; - const passedTimeInSeconds = - (nowtime - this.screenshotManager.screenshotStartTime) / 1000; return { - timeElapsedString: `${passedTimeInSeconds.toFixed(0)} seconds`, chunkUsageDescription: `${currentStatistics.visibleChunksGpuMemory} out of ${currentStatistics.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, gpuMemoryUsageDescription: `${gpuMemoryUsageInMB.toFixed(0)}MB / ${totalMemoryInMB.toFixed(0)}MB (${percentGpuUsage.toFixed(2)}% of total)`, downloadSpeedDescription: `${currentStatistics.visibleChunksDownloading} at ${latency.toFixed(0)}ms latency`, diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 90ff88360..8ac255e8f 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { throttle } from "lodash-es"; +import { numChunkStatistics } from "#src/chunk_manager/base.js"; import type { RenderedPanel } from "#src/display_context.js"; import type { ScreenshotActionState, @@ -21,6 +23,11 @@ import type { ScreenshotChunkStatistics, } from "#src/python_integration/screenshots.js"; import { RenderedDataPanel } from "#src/rendered_data_panel.js"; +import { + columnSpecifications, + getChunkSourceIdentifier, + getFormattedNames, +} from "#src/ui/statistics.js"; import { RefCounted } from "#src/util/disposable.js"; import { NullarySignal, Signal } from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; @@ -127,9 +134,6 @@ async function extractViewportScreenshot( } export class ScreenshotManager extends RefCounted { - private filename: string = ""; - private lastUpdateTimestamp: number = 0; - private gpuMemoryChangeTimestamp: number = 0; screenshotId: number = -1; screenshotScale: number = 1; screenshotLoadStats: ScreenshotLoadStatistics | null = null; @@ -137,6 +141,49 @@ export class ScreenshotManager extends RefCounted { screenshotMode: ScreenshotMode = ScreenshotMode.OFF; statisticsUpdated = new Signal<(state: ScreenshotLoadStatistics) => void>(); screenshotFinished = new NullarySignal(); + private filename: string = ""; + private lastUpdateTimestamp: number = 0; + private gpuMemoryChangeTimestamp: number = 0; + throttledSendStatistics = this.registerCancellable( + throttle( + async () => { + const map = await this.viewer.chunkQueueManager.getStatistics(); + if (this.wasDisposed) return; + const formattedNames = getFormattedNames( + Array.from(map, (x) => getChunkSourceIdentifier(x[0])), + ); + let i = 0; + const rows: any[] = []; + const sumStatistics = new Float64Array(numChunkStatistics); + for (const [source, statistics] of map) { + for (let i = 0; i < numChunkStatistics; ++i) { + sumStatistics[i] += statistics[i]; + } + const row: any = {}; + row.id = getChunkSourceIdentifier(source); + row.distinctId = formattedNames[i]; + for (const column of columnSpecifications) { + row[column.key] = column.getter(statistics); + } + ++i; + rows.push(row); + } + const total: any = {}; + for (const column of columnSpecifications) { + total[column.key] = column.getter(sumStatistics); + } + const screenshotLoadStats = { + ...total, + timestamp: Date.now(), + gpuMemoryCapacity: + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value, + }; + this.statisticsUpdated.dispatch(screenshotLoadStats); + }, + 1000, + { leading: false, trailing: true }, + ), + ); constructor(public viewer: Viewer) { super(); From 949bcdfa5315e415d261d4e2943e0d2bfbd2cf03 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 17 Sep 2024 18:09:16 +0200 Subject: [PATCH 46/66] feat: update resolution rounding for slices > 1px --- src/ui/screenshot_menu.ts | 12 ++++++++++-- src/util/viewer_resolution_stats.ts | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index de61d2cb8..2f07951b7 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -291,8 +291,15 @@ export class ScreenshotDialog extends Overlay { function formatResolution(key: any, value: any) { const type = key[1]; const resolution: number = value.resolution; - const unit = type === "VolumeRenderingRenderLayer" ? " Z samples" : "px"; - const roundingLevel = type === "VolumeRenderingRenderLayer" ? 0 : 2; + const unit = type === "VolumeRenderingRenderLayer" ? "Z samples" : "px"; + + let roundingLevel = 0; + if ( + type === "VolumeRenderingRenderLayer" || + (type === "ImageRenderLayer" && resolution > 1) + ) { + roundingLevel = 0; + } return `${resolution.toFixed(roundingLevel)} ${unit}`; } @@ -311,6 +318,7 @@ export class ScreenshotDialog extends Overlay { valueHeader.textContent = "Resolution"; headerRow.appendChild(valueHeader); + // TODO needs populate with debounce as sometimes the viewer is not ready const resolutionMap = getViewerLayerResolutions( this.screenshotManager.viewer, ); diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 455d27279..525fd26d8 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -61,7 +61,6 @@ export function getViewerLayerResolutions( return map; } -// TODO needs screenshotFactor export function getViewerPanelResolutions(panels: ReadonlySet) { function resolutionsEqual(resolution1: any[], resolution2: any[]) { if (resolution1.length !== resolution2.length) { From 042ee626701412dff4674242a1ab073ae4334bdd Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 19 Sep 2024 16:35:10 +0200 Subject: [PATCH 47/66] feat: populate layer stats in menu as they laod --- src/ui/screenshot_menu.ts | 109 ++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 40 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 2f07951b7..bd8654831 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -15,7 +15,7 @@ */ import "#src/ui/screenshot_menu.css"; -import { debounce } from "lodash-es"; +import { debounce, throttle } from "lodash-es"; import { Overlay } from "#src/overlay.js"; import type { ScreenshotLoadStatistics, @@ -58,11 +58,21 @@ export class ScreenshotDialog extends Overlay { private cancelScreenshotButton: HTMLButtonElement; private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; + private panelResolutionTable: HTMLTableElement; + private layerResolutionTable: HTMLTableElement; private statisticsContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; private screenshotSizeText: HTMLDivElement; private warningElement: HTMLDivElement; private statisticsKeyToCellMap: Map = new Map(); + private layerResolutionKeyToCellMap: Map = + new Map(); + + private throttledUpdateLayerResolutionTable = this.registerCancellable( + throttle(() => { + this.populateLayerResolutionTable(); + }, 1000), + ); constructor(private screenshotManager: ScreenshotManager) { super(); @@ -104,6 +114,8 @@ export class ScreenshotDialog extends Overlay { this.content.appendChild(this.createLayerResolutionTable()); this.content.appendChild(this.createStatisticsTable()); this.updateUIBasedOnMode(); + this.populatePanelResolutionTable(); + this.throttledUpdateLayerResolutionTable(); } private setupEventListeners() { @@ -120,6 +132,7 @@ export class ScreenshotDialog extends Overlay { this.registerDisposer( this.screenshotManager.viewer.display.updateFinished.add(() => { this.screenshotManager.throttledSendStatistics(); + this.throttledUpdateLayerResolutionTable(); }), ); } @@ -242,6 +255,23 @@ export class ScreenshotDialog extends Overlay { } private createPanelResolutionTable() { + const resolutionTable = (this.panelResolutionTable = + document.createElement("table")); + resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); + resolutionTable.title = "Viewer resolution statistics"; + + const headerRow = resolutionTable.createTHead().insertRow(); + const keyHeader = document.createElement("th"); + keyHeader.textContent = "Panel type"; + headerRow.appendChild(keyHeader); + const valueHeader = document.createElement("th"); + valueHeader.textContent = "Resolution"; + headerRow.appendChild(valueHeader); + return resolutionTable; + } + + private populatePanelResolutionTable() { + const resolutionTable = this.panelResolutionTable; function formatResolution(resolution: any) { const first_resolution = resolution[0]; if (first_resolution.name === "All_") { @@ -260,19 +290,6 @@ export class ScreenshotDialog extends Overlay { }; } } - - const resolutionTable = document.createElement("table"); - resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); - resolutionTable.title = "Viewer resolution statistics"; - - const headerRow = resolutionTable.createTHead().insertRow(); - const keyHeader = document.createElement("th"); - keyHeader.textContent = "Panel type"; - headerRow.appendChild(keyHeader); - const valueHeader = document.createElement("th"); - valueHeader.textContent = "Resolution"; - headerRow.appendChild(valueHeader); - const resolutions = getViewerPanelResolutions( this.screenshotManager.viewer.display.panels, ); @@ -288,6 +305,26 @@ export class ScreenshotDialog extends Overlay { } private createLayerResolutionTable() { + const resolutionTable = (this.layerResolutionTable = + document.createElement("table")); + resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); + resolutionTable.title = "Viewer resolution statistics"; + + const headerRow = resolutionTable.createTHead().insertRow(); + const keyHeader = document.createElement("th"); + keyHeader.textContent = "Layer name"; + headerRow.appendChild(keyHeader); + const typeHeader = document.createElement("th"); + typeHeader.textContent = "Type"; + headerRow.appendChild(typeHeader); + const valueHeader = document.createElement("th"); + valueHeader.textContent = "Resolution"; + headerRow.appendChild(valueHeader); + return resolutionTable; + } + + private populateLayerResolutionTable() { + console.log("populateLayerResolutionTable"); function formatResolution(key: any, value: any) { const type = key[1]; const resolution: number = value.resolution; @@ -303,37 +340,29 @@ export class ScreenshotDialog extends Overlay { return `${resolution.toFixed(roundingLevel)} ${unit}`; } - const resolutionTable = document.createElement("table"); - resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); - resolutionTable.title = "Viewer resolution statistics"; - - const headerRow = resolutionTable.createTHead().insertRow(); - const keyHeader = document.createElement("th"); - keyHeader.textContent = "Layer name"; - headerRow.appendChild(keyHeader); - const typeHeader = document.createElement("th"); - typeHeader.textContent = "Type"; - headerRow.appendChild(typeHeader); - const valueHeader = document.createElement("th"); - valueHeader.textContent = "Resolution"; - headerRow.appendChild(valueHeader); - - // TODO needs populate with debounce as sometimes the viewer is not ready + const resolutionTable = this.layerResolutionTable; const resolutionMap = getViewerLayerResolutions( this.screenshotManager.viewer, ); for (const [key, value] of resolutionMap) { - const row = resolutionTable.insertRow(); - const keyCell = row.insertCell(); - const typeCell = row.insertCell(); - const valueCell = row.insertCell(); - const name = key[0]; - keyCell.textContent = name; - typeCell.textContent = - layerNamesForUI[key[1] as keyof typeof layerNamesForUI]; - valueCell.textContent = formatResolution(key, value); + const stringKey = key.join(","); + const resolution = formatResolution(key, value); + let valueCell = this.layerResolutionKeyToCellMap.get(stringKey); + console.log("valueCell", valueCell); + if (valueCell === undefined) { + const row = resolutionTable.insertRow(); + const keyCell = row.insertCell(); + const typeCell = row.insertCell(); + valueCell = row.insertCell(); + const name = key[0]; + keyCell.textContent = name; + typeCell.textContent = + layerNamesForUI[key[1] as keyof typeof layerNamesForUI]; + this.layerResolutionKeyToCellMap.set(stringKey, valueCell); + } + console.log(resolution) + valueCell.textContent = resolution; } - return resolutionTable; } private forceScreenshot() { From b62fe06a1439183793111f3dd77ce7bb41bb3a95 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 19 Sep 2024 16:50:51 +0200 Subject: [PATCH 48/66] feat: hide non-visible stats --- src/ui/screenshot_menu.ts | 3 --- src/util/viewer_resolution_stats.ts | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index bd8654831..77a2077e0 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -324,7 +324,6 @@ export class ScreenshotDialog extends Overlay { } private populateLayerResolutionTable() { - console.log("populateLayerResolutionTable"); function formatResolution(key: any, value: any) { const type = key[1]; const resolution: number = value.resolution; @@ -348,7 +347,6 @@ export class ScreenshotDialog extends Overlay { const stringKey = key.join(","); const resolution = formatResolution(key, value); let valueCell = this.layerResolutionKeyToCellMap.get(stringKey); - console.log("valueCell", valueCell); if (valueCell === undefined) { const row = resolutionTable.insertRow(); const keyCell = row.insertCell(); @@ -360,7 +358,6 @@ export class ScreenshotDialog extends Overlay { layerNamesForUI[key[1] as keyof typeof layerNamesForUI]; this.layerResolutionKeyToCellMap.set(stringKey, valueCell); } - console.log(resolution) valueCell.textContent = resolution; } } diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 525fd26d8..22566abfa 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -30,27 +30,51 @@ export function getViewerLayerResolutions( viewer: Viewer, ): Map<[string, string], any> { const layers = viewer.layerManager.visibleRenderLayers; + const panels = viewer.display.panels; const map = new Map(); + + // Get all the layers in at least one panel. + for (const panel of panels) { + if (!(panel instanceof RenderedDataPanel)) continue; + } + for (const layer of layers) { + //const isLayerInAnyPanel = if (layer.role === RenderLayerRole.DATA) { const layer_name = layer.userLayer!.managedLayer.name; if (layer instanceof ImageRenderLayer) { + const isVisble = layer.visibleSourcesList.length > 0; + if (!isVisble) { + continue; + } const type = "ImageRenderLayer"; const resolution = layer.renderScaleTarget.value; map.set([layer_name, type], { resolution }); } else if (layer instanceof VolumeRenderingRenderLayer) { + const isVisble = layer.visibility.visible; + if (!isVisble) { + continue; + } const type = "VolumeRenderingRenderLayer"; const resolution = layer.depthSamplesTarget.value; map.set([layer_name, type], { resolution, }); } else if (layer instanceof SegmentationRenderLayer) { + const isVisble = layer.visibleSourcesList.length > 0; + if (!isVisble) { + continue; + } const type = "SegmentationRenderLayer"; const resolution = layer.renderScaleTarget.value; map.set([layer_name, type], { resolution, }); } else if (layer instanceof MultiscaleMeshLayer) { + const isVisble = layer.visibility.visible; + if (!isVisble) { + continue; + } const type = "MultiscaleMeshLayer"; const userLayer = layer.userLayer as SegmentationUserLayer; const resolution = userLayer.displayState.renderScaleTarget.value; From 3289c2cdff2324d75cecf436414bbb9d53e3a6fc Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 19 Sep 2024 17:00:24 +0200 Subject: [PATCH 49/66] feat: reduce screenshot hang time --- src/util/screenshot_manager.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 8ac255e8f..2fbfd12f3 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -33,7 +33,7 @@ import { NullarySignal, Signal } from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; -const SCREENSHOT_TIMEOUT = 5000; +const SCREENSHOT_TIMEOUT = 1000; export interface ScreenshotLoadStatistics extends ScreenshotChunkStatistics { timestamp: number; @@ -199,15 +199,15 @@ export class ScreenshotManager extends RefCounted { this.registerDisposer( this.viewer.screenshotHandler.sendStatisticsRequested.add( (actionState) => { - this.checkAndHandleStalledScreenshot(actionState); - this.screenshotLoadStats = { + const newLoadStats = { ...actionState.screenshotStatistics.total, timestamp: Date.now(), gpuMemoryCapacity: this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit .value, }; - this.statisticsUpdated.dispatch(this.screenshotLoadStats); + this.checkAndHandleStalledScreenshot(actionState, newLoadStats); + this.screenshotLoadStats = newLoadStats; }, ), ); @@ -313,7 +313,10 @@ export class ScreenshotManager extends RefCounted { * in the GPU with the previous number of visible chunks. If the number of * visible chunks has not changed after a certain timeout, and the display has not updated, force a screenshot. */ - private checkAndHandleStalledScreenshot(actionState: StatisticsActionState) { + private checkAndHandleStalledScreenshot( + actionState: StatisticsActionState, + fullStats: ScreenshotLoadStatistics, + ) { if (this.screenshotLoadStats === null) { return; } @@ -334,6 +337,7 @@ export class ScreenshotManager extends RefCounted { SCREENSHOT_TIMEOUT && Date.now() - this.lastUpdateTimestamp > SCREENSHOT_TIMEOUT ) { + this.statisticsUpdated.dispatch(fullStats); console.warn( `Forcing screenshot: screenshot is likely stuck, no change in GPU chunks after ${SCREENSHOT_TIMEOUT}ms. Last visible chunks: ${total.visibleChunksGpuMemory}/${total.visibleChunksTotal}`, ); From bd919047dbb03d768535c0e04512ac0acd5ac38b Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 20 Sep 2024 16:16:27 +0200 Subject: [PATCH 50/66] feat: improve stats display and checking for hanging screenshots --- src/ui/screenshot_menu.ts | 8 +++- src/util/screenshot_manager.ts | 71 +++++++++++++++++----------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 77a2077e0..52189541d 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -131,7 +131,6 @@ export class ScreenshotDialog extends Overlay { ); this.registerDisposer( this.screenshotManager.viewer.display.updateFinished.add(() => { - this.screenshotManager.throttledSendStatistics(); this.throttledUpdateLayerResolutionTable(); }), ); @@ -416,10 +415,15 @@ export class ScreenshotDialog extends Overlay { ? 0 : currentStatistics.downloadLatency; + const downloadString = + currentStatistics.visibleChunksDownloading == 0 + ? "0" + : `${currentStatistics.visibleChunksDownloading} at ${latency.toFixed(0)}ms latency`; + return { chunkUsageDescription: `${currentStatistics.visibleChunksGpuMemory} out of ${currentStatistics.visibleChunksTotal} (${percentLoaded.toFixed(2)}%)`, gpuMemoryUsageDescription: `${gpuMemoryUsageInMB.toFixed(0)}MB / ${totalMemoryInMB.toFixed(0)}MB (${percentGpuUsage.toFixed(2)}% of total)`, - downloadSpeedDescription: `${currentStatistics.visibleChunksDownloading} at ${latency.toFixed(0)}ms latency`, + downloadSpeedDescription: downloadString, }; } diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 2fbfd12f3..a54497cea 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -145,44 +145,40 @@ export class ScreenshotManager extends RefCounted { private lastUpdateTimestamp: number = 0; private gpuMemoryChangeTimestamp: number = 0; throttledSendStatistics = this.registerCancellable( - throttle( - async () => { - const map = await this.viewer.chunkQueueManager.getStatistics(); - if (this.wasDisposed) return; - const formattedNames = getFormattedNames( - Array.from(map, (x) => getChunkSourceIdentifier(x[0])), - ); - let i = 0; - const rows: any[] = []; - const sumStatistics = new Float64Array(numChunkStatistics); - for (const [source, statistics] of map) { - for (let i = 0; i < numChunkStatistics; ++i) { - sumStatistics[i] += statistics[i]; - } - const row: any = {}; - row.id = getChunkSourceIdentifier(source); - row.distinctId = formattedNames[i]; - for (const column of columnSpecifications) { - row[column.key] = column.getter(statistics); - } - ++i; - rows.push(row); + throttle(async () => { + const map = await this.viewer.chunkQueueManager.getStatistics(); + if (this.wasDisposed) return; + const formattedNames = getFormattedNames( + Array.from(map, (x) => getChunkSourceIdentifier(x[0])), + ); + let i = 0; + const rows: any[] = []; + const sumStatistics = new Float64Array(numChunkStatistics); + for (const [source, statistics] of map) { + for (let i = 0; i < numChunkStatistics; ++i) { + sumStatistics[i] += statistics[i]; } - const total: any = {}; + const row: any = {}; + row.id = getChunkSourceIdentifier(source); + row.distinctId = formattedNames[i]; for (const column of columnSpecifications) { - total[column.key] = column.getter(sumStatistics); + row[column.key] = column.getter(statistics); } - const screenshotLoadStats = { - ...total, - timestamp: Date.now(), - gpuMemoryCapacity: - this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value, - }; - this.statisticsUpdated.dispatch(screenshotLoadStats); - }, - 1000, - { leading: false, trailing: true }, - ), + ++i; + rows.push(row); + } + const total: any = {}; + for (const column of columnSpecifications) { + total[column.key] = column.getter(sumStatistics); + } + const screenshotLoadStats = { + ...total, + timestamp: Date.now(), + gpuMemoryCapacity: + this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value, + }; + this.statisticsUpdated.dispatch(screenshotLoadStats); + }, 1000), ); constructor(public viewer: Viewer) { @@ -214,6 +210,7 @@ export class ScreenshotManager extends RefCounted { this.registerDisposer( this.viewer.display.updateFinished.add(() => { this.lastUpdateTimestamp = Date.now(); + this.throttledSendStatistics(); }), ); this.registerDisposer( @@ -326,11 +323,13 @@ export class ScreenshotManager extends RefCounted { timestamp: Date.now(), totalGpuMemory: this.viewer.chunkQueueManager.capacities.gpuMemory.sizeLimit.value, + numDownloadingChunks: total.visibleChunksDownloading, }; const oldStats = this.screenshotLoadStats; if ( oldStats.visibleChunksGpuMemory === newStats.visibleChunksGpuMemory && - oldStats.gpuMemoryCapacity === newStats.totalGpuMemory + (oldStats.gpuMemoryCapacity === newStats.totalGpuMemory || + newStats.numDownloadingChunks == 0) ) { if ( newStats.timestamp - this.gpuMemoryChangeTimestamp > From bb07ae8b7f8b16e017f0c4d7922367e0c0db6301 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 23 Sep 2024 11:55:48 +0200 Subject: [PATCH 51/66] feat: detect ortographic view stats --- src/util/viewer_resolution_stats.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 22566abfa..ee11c9bda 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -17,6 +17,7 @@ import type { RenderedPanel } from "#src/display_context.js"; import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; import { MultiscaleMeshLayer } from "#src/mesh/frontend.js"; +import { PerspectivePanel } from "#src/perspective_view/panel.js"; import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { RenderLayerRole } from "#src/renderlayer.js"; import { SliceViewPanel } from "#src/sliceview/panel.js"; @@ -108,8 +109,22 @@ export function getViewerPanelResolutions(panels: ReadonlySet) { for (const panel of panels) { if (!(panel instanceof RenderedDataPanel)) continue; const panel_resolution = []; - const displayDimensionUnit = panel instanceof SliceViewPanel ? "px" : "vh"; - const panelType = panel instanceof SliceViewPanel ? "Slice" : "3D"; + const isOrtographicProjection = + panel instanceof PerspectivePanel && + panel.viewer.orthographicProjection.value; + + const displayDimensionUnit = + panel instanceof SliceViewPanel || isOrtographicProjection ? "px" : "vh"; + let panelType: string; + if (panel instanceof SliceViewPanel) { + panelType = "Slice view"; + } else if (isOrtographicProjection) { + panelType = "Orthographic view"; + } else if (panel instanceof PerspectivePanel) { + panelType = "Perspective view"; + } else { + panelType = "Unknown"; + } const { navigationState } = panel; const { displayDimensionIndices, From 54a20d267f485aea830cca6f5c9c9078848ae57a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 23 Sep 2024 12:02:54 +0200 Subject: [PATCH 52/66] fix(ui): don't round resolution < 1 in display --- src/ui/screenshot_menu.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 52189541d..18f716092 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -328,7 +328,7 @@ export class ScreenshotDialog extends Overlay { const resolution: number = value.resolution; const unit = type === "VolumeRenderingRenderLayer" ? "Z samples" : "px"; - let roundingLevel = 0; + let roundingLevel = 2; if ( type === "VolumeRenderingRenderLayer" || (type === "ImageRenderLayer" && resolution > 1) From 4230a4ca56570bfd4c61d6cffcdddf63723a859f Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 23 Sep 2024 12:19:05 +0200 Subject: [PATCH 53/66] docs: note about why use debug close menu --- src/ui/screenshot_menu.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 18f716092..e786e2648 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -29,6 +29,9 @@ import { // If true, the menu can be closed by clicking the close button // Usually the user is locked into the screenshot menu until the screenshot is taken or cancelled +// Setting this to true, and setting the SCREENSHOT_MENU_CLOSE_TIMEOUT in screenshot_manager.ts +// to a high value can be useful for debugging canvas handling of the resize + const DEBUG_ALLOW_MENU_CLOSE = false; const LARGE_SCREENSHOT_SIZE = 4096 * 4096; From 7f0d199400c6c41f46af6ae6bad8e2fa123541dc Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 26 Sep 2024 11:08:39 +0200 Subject: [PATCH 54/66] revert: don't grab JSON state with screenshot --- src/util/screenshot_manager.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index a54497cea..098b91288 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -371,22 +371,10 @@ export class ScreenshotManager extends RefCounted { } catch (error) { console.error("Failed to save screenshot:", error); } finally { - this.saveScreenshotLog(actionState); this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; } } - private saveScreenshotLog(actionState: ScreenshotActionState) { - const { viewerState } = actionState; - const stateString = JSON.stringify(viewerState); - this.downloadState(stateString); - } - - private downloadState(state: string) { - const blob = new Blob([state], { type: "text/json" }); - saveBlobToFile(blob, setExtension(this.filename, "_state.json")); - } - private resetCanvasSize() { // Reset the canvas size to the original size // No need to manually pass the correct sizes, the viewer will handle it From beab5f4ede0b19f517163349aec041ccb5145615 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 26 Sep 2024 13:43:13 +0200 Subject: [PATCH 55/66] feat: include voxel resolution in screenshot --- src/sliceview/volume/renderlayer.ts | 14 ++ src/ui/screenshot_menu.ts | 80 ++++++----- src/util/viewer_resolution_stats.ts | 144 ++++++++++++-------- src/volume_rendering/volume_render_layer.ts | 13 +- 4 files changed, 144 insertions(+), 107 deletions(-) diff --git a/src/sliceview/volume/renderlayer.ts b/src/sliceview/volume/renderlayer.ts index a01325eb1..d8224efba 100644 --- a/src/sliceview/volume/renderlayer.ts +++ b/src/sliceview/volume/renderlayer.ts @@ -342,6 +342,7 @@ export abstract class SliceViewVolumeRenderLayer< private tempChunkPosition: Float32Array; shaderParameters: WatchableValueInterface; private vertexIdHelper: VertexIdHelper; + public highestResolutionLoadedVoxelSize: Float32Array | undefined; constructor( multiscaleSource: MultiscaleVolumeChunkSource, @@ -570,6 +571,7 @@ void main() { this.chunkManager.chunkQueueManager.frameNumberCounter.frameNumber, ); } + this.highestResolutionLoadedVoxelSize = undefined; let shaderResult: ParameterizedShaderGetterResult< ShaderParameters, @@ -692,6 +694,18 @@ void main() { effectiveVoxelSize[1], effectiveVoxelSize[2], ); + if (presentCount > 0) { + const medianStoredVoxelSize = this.highestResolutionLoadedVoxelSize + ? medianOf3( + this.highestResolutionLoadedVoxelSize[0], + this.highestResolutionLoadedVoxelSize[1], + this.highestResolutionLoadedVoxelSize[2], + ) + : Infinity; + if (medianVoxelSize <= medianStoredVoxelSize) { + this.highestResolutionLoadedVoxelSize = effectiveVoxelSize; + } + } renderScaleHistogram.add( medianVoxelSize, medianVoxelSize / projectionParameters.pixelSize, diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index e786e2648..610f3dd1d 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -22,6 +22,7 @@ import type { ScreenshotManager, } from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; +import type { DimensionResolutionStats } from "#src/util/viewer_resolution_stats.js"; import { getViewerLayerResolutions, getViewerPanelResolutions, @@ -48,12 +49,38 @@ const statisticsNamesForUI = { }; const layerNamesForUI = { - ImageRenderLayer: "Image", - VolumeRenderingRenderLayer: "Volume", - SegmentationRenderLayer: "Segmentation", - MultiscaleMeshLayer: "Mesh", + ImageRenderLayer: "Image slice (2D)", + VolumeRenderingRenderLayer: "Volume rendering (3D)", + SegmentationRenderLayer: "Segmentation slice (2D)", }; +function formatResolution(resolution: DimensionResolutionStats[]) { + if (resolution.length === 0) { + return { + type: "Loading...", + resolution: "Loading...", + }; + } + const first_resolution = resolution[0]; + // If the resolution is the same for all dimensions, display it as a single line + if (first_resolution.dimensionName === "All_") { + return { + type: first_resolution.parentType, + resolution: ` ${first_resolution.resolutionWithUnit}`, + }; + } else { + let text = ""; + for (const res of resolution) { + text += `${res.dimensionName}: ${res.resolutionWithUnit}, `; + } + text = text.slice(0, -2); + return { + type: first_resolution.parentType, + resolution: text, + }; + } +} + export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private takeScreenshotButton: HTMLButtonElement; @@ -274,24 +301,6 @@ export class ScreenshotDialog extends Overlay { private populatePanelResolutionTable() { const resolutionTable = this.panelResolutionTable; - function formatResolution(resolution: any) { - const first_resolution = resolution[0]; - if (first_resolution.name === "All_") { - return { - type: first_resolution.panelType, - resolution: first_resolution.textContent, - }; - } else { - let text = ""; - for (const res of resolution) { - text += `${res.name}: ${res.textContent}, `; - } - return { - type: first_resolution.panelType, - resolution: text, - }; - } - } const resolutions = getViewerPanelResolutions( this.screenshotManager.viewer.display.panels, ); @@ -326,41 +335,28 @@ export class ScreenshotDialog extends Overlay { } private populateLayerResolutionTable() { - function formatResolution(key: any, value: any) { - const type = key[1]; - const resolution: number = value.resolution; - const unit = type === "VolumeRenderingRenderLayer" ? "Z samples" : "px"; - - let roundingLevel = 2; - if ( - type === "VolumeRenderingRenderLayer" || - (type === "ImageRenderLayer" && resolution > 1) - ) { - roundingLevel = 0; - } - - return `${resolution.toFixed(roundingLevel)} ${unit}`; - } const resolutionTable = this.layerResolutionTable; const resolutionMap = getViewerLayerResolutions( this.screenshotManager.viewer, ); for (const [key, value] of resolutionMap) { - const stringKey = key.join(","); - const resolution = formatResolution(key, value); + const { name, type } = key; + if (type === "MultiscaleMeshLayer") { + continue; + } + const stringKey = `{${name}--${type}}`; let valueCell = this.layerResolutionKeyToCellMap.get(stringKey); if (valueCell === undefined) { const row = resolutionTable.insertRow(); const keyCell = row.insertCell(); const typeCell = row.insertCell(); valueCell = row.insertCell(); - const name = key[0]; keyCell.textContent = name; typeCell.textContent = - layerNamesForUI[key[1] as keyof typeof layerNamesForUI]; + layerNamesForUI[type as keyof typeof layerNamesForUI]; this.layerResolutionKeyToCellMap.set(stringKey, valueCell); } - valueCell.textContent = resolution; + valueCell.textContent = formatResolution(value).resolution; } } diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index ee11c9bda..70bd64dc0 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -15,8 +15,6 @@ */ import type { RenderedPanel } from "#src/display_context.js"; -import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; -import { MultiscaleMeshLayer } from "#src/mesh/frontend.js"; import { PerspectivePanel } from "#src/perspective_view/panel.js"; import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { RenderLayerRole } from "#src/renderlayer.js"; @@ -27,60 +25,94 @@ import { formatScaleWithUnitAsString } from "#src/util/si_units.js"; import type { Viewer } from "#src/viewer.js"; import { VolumeRenderingRenderLayer } from "#src/volume_rendering/volume_render_layer.js"; +export interface DimensionResolutionStats { + parentType: string; + dimensionName: string; + resolutionWithUnit: string; +} + +interface LayerIdentifier { + name: string; + type: string; +} + export function getViewerLayerResolutions( viewer: Viewer, -): Map<[string, string], any> { - const layers = viewer.layerManager.visibleRenderLayers; - const panels = viewer.display.panels; - const map = new Map(); +): Map { + function formatResolution( + resolution: Float32Array | undefined, + parentType: string, + ): DimensionResolutionStats[] { + if (resolution === undefined) return []; - // Get all the layers in at least one panel. - for (const panel of panels) { - if (!(panel instanceof RenderedDataPanel)) continue; + const resolution_stats: DimensionResolutionStats[] = []; + const { + globalDimensionNames, + displayDimensionUnits, + displayDimensionIndices, + } = viewer.navigationState.displayDimensionRenderInfo.value; + + // Check if all units and factors are the same. + const firstDim = displayDimensionIndices[0]; + let singleScale = true; + if (firstDim !== -1) { + const unit = displayDimensionUnits[0]; + const factor = resolution[0]; + for (let i = 1; i < 3; ++i) { + const dim = displayDimensionIndices[i]; + if (dim === -1) continue; + if (displayDimensionUnits[i] !== unit || factor !== resolution[i]) { + singleScale = false; + break; + } + } + } + + for (let i = 0; i < 3; ++i) { + const dim = displayDimensionIndices[i]; + if (dim !== -1) { + const dimensionName = globalDimensionNames[dim]; + if (i === 0 || !singleScale) { + const formattedScale = formatScaleWithUnitAsString( + resolution[i], + displayDimensionUnits[i], + { precision: 2, elide1: false }, + ); + resolution_stats.push({ + parentType: parentType, + resolutionWithUnit: `${formattedScale}`, + dimensionName: singleScale ? "All_" : dimensionName, + }); + } + } + } + return resolution_stats; } + const layers = viewer.layerManager.visibleRenderLayers; + const map = new Map(); + for (const layer of layers) { - //const isLayerInAnyPanel = if (layer.role === RenderLayerRole.DATA) { - const layer_name = layer.userLayer!.managedLayer.name; + let isVisble = false; + const name = layer.userLayer!.managedLayer.name; + let type: string = ""; + let resolution: Float32Array | undefined; if (layer instanceof ImageRenderLayer) { - const isVisble = layer.visibleSourcesList.length > 0; - if (!isVisble) { - continue; - } - const type = "ImageRenderLayer"; - const resolution = layer.renderScaleTarget.value; - map.set([layer_name, type], { resolution }); + type = "ImageRenderLayer"; + isVisble = layer.visibleSourcesList.length > 0; + resolution = layer.highestResolutionLoadedVoxelSize; } else if (layer instanceof VolumeRenderingRenderLayer) { - const isVisble = layer.visibility.visible; - if (!isVisble) { - continue; - } - const type = "VolumeRenderingRenderLayer"; - const resolution = layer.depthSamplesTarget.value; - map.set([layer_name, type], { - resolution, - }); + type = "VolumeRenderingRenderLayer"; + isVisble = layer.visibility.visible; + resolution = layer.highestResolutionLoadedVoxelSize; } else if (layer instanceof SegmentationRenderLayer) { - const isVisble = layer.visibleSourcesList.length > 0; - if (!isVisble) { - continue; - } - const type = "SegmentationRenderLayer"; - const resolution = layer.renderScaleTarget.value; - map.set([layer_name, type], { - resolution, - }); - } else if (layer instanceof MultiscaleMeshLayer) { - const isVisble = layer.visibility.visible; - if (!isVisble) { - continue; - } - const type = "MultiscaleMeshLayer"; - const userLayer = layer.userLayer as SegmentationUserLayer; - const resolution = userLayer.displayState.renderScaleTarget.value; - map.set([layer_name, type], { resolution }); + type = "SegmentationRenderLayer"; + isVisble = layer.visibleSourcesList.length > 0; + resolution = layer.highestResolutionLoadedVoxelSize; } + if (!isVisble) continue; + map.set({ name, type }, formatResolution(resolution, type)); } } return map; @@ -105,7 +137,7 @@ export function getViewerPanelResolutions(panels: ReadonlySet) { return true; } - const resolutions: any[] = []; + const resolutions: DimensionResolutionStats[][] = []; for (const panel of panels) { if (!(panel instanceof RenderedDataPanel)) continue; const panel_resolution = []; @@ -113,7 +145,7 @@ export function getViewerPanelResolutions(panels: ReadonlySet) { panel instanceof PerspectivePanel && panel.viewer.orthographicProjection.value; - const displayDimensionUnit = + const panelDimensionUnit = panel instanceof SliceViewPanel || isOrtographicProjection ? "px" : "vh"; let panelType: string; if (panel instanceof SliceViewPanel) { @@ -155,29 +187,25 @@ export function getViewerPanelResolutions(panels: ReadonlySet) { if (dim !== -1) { const totalScale = (displayDimensionScales[i] * zoom) / canonicalVoxelFactors[i]; - let textContent; - const name = globalDimensionNames[dim]; + const dimensionName = globalDimensionNames[dim]; if (i === 0 || !singleScale) { const formattedScale = formatScaleWithUnitAsString( totalScale, displayDimensionUnits[i], { precision: 2, elide1: false }, ); - textContent = `${formattedScale}/${displayDimensionUnit}`; - if (singleScale) { - panel_resolution.push({ panelType, textContent, name: "All_" }); - } else { - panel_resolution.push({ panelType, textContent, name }); - } - } else { - textContent = ""; + panel_resolution.push({ + parentType: panelType, + resolutionWithUnit: `${formattedScale}/${panelDimensionUnit}`, + dimensionName: singleScale ? "All_" : dimensionName, + }); } } } resolutions.push(panel_resolution); } - const uniqueResolutions: any[] = []; + const uniqueResolutions: DimensionResolutionStats[][] = []; for (const resolution of resolutions) { let found = false; for (const uniqueResolution of uniqueResolutions) { diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index a1f55959c..a3b005060 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -223,8 +223,8 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { private modeOverride: TrackableVolumeRenderingModeValue; private vertexIdHelper: VertexIdHelper; private dataHistogramSpecifications: HistogramSpecifications; - private physicalSpacingForDepthSamples: number; private dataResolutionIndex: number; + public highestResolutionLoadedVoxelSize: Float32Array | undefined; private shaderGetter: ParameterizedContextDependentShaderGetter< { emitter: ShaderModule; chunkFormat: ChunkFormat; wireFrame: boolean }, @@ -250,10 +250,6 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { return true; } - get physicalSpacing() { - return this.physicalSpacingForDepthSamples; - } - get selectedDataResolution() { return this.dataResolutionIndex; } @@ -768,6 +764,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); ShaderControlsBuilderState, VolumeRenderingShaderParameters >; + let physicalSpacingForOptimalSamples = 0; // Size of chunk (in voxels) in the "display" subspace of the chunk coordinate space. const chunkDataDisplaySize = vec3.create(); @@ -836,7 +833,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); }, ); renderScaleHistogram.add( - this.physicalSpacingForDepthSamples, + physicalSpacingForOptimalSamples, curOptimalSamples, presentCount, notPresentCount, @@ -890,7 +887,9 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); ) => { ignored1; ignored2; - this.physicalSpacingForDepthSamples = physicalSpacing; + this.highestResolutionLoadedVoxelSize = + transformedSource.effectiveVoxelSize; + physicalSpacingForOptimalSamples = physicalSpacing; curOptimalSamples = optimalSamples; curHistogramInformation = histogramInformation; this.dataResolutionIndex = histogramInformation.activeIndex; From 681d923efde2db41cd263ca6766aae7c2fd83a35 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 26 Sep 2024 16:12:00 +0200 Subject: [PATCH 56/66] feat: allow fixed 2D panel FOV in screenshots with checkbox --- src/ui/screenshot_menu.ts | 36 ++++++++++++++++++++++++++- src/util/screenshot_manager.ts | 45 +++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 610f3dd1d..73b4d9279 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -101,7 +101,7 @@ export class ScreenshotDialog extends Overlay { private throttledUpdateLayerResolutionTable = this.registerCancellable( throttle(() => { this.populateLayerResolutionTable(); - }, 1000), + }, 500), ); constructor(private screenshotManager: ScreenshotManager) { super(); @@ -111,6 +111,13 @@ export class ScreenshotDialog extends Overlay { this.screenshotManager.throttledSendStatistics(); } + dispose(): void { + super.dispose(); + if (!DEBUG_ALLOW_MENU_CLOSE) { + this.screenshotManager.screenshotScale = 1; + } + } + private initializeUI() { this.content.classList.add("neuroglancer-screenshot-dialog"); @@ -164,6 +171,11 @@ export class ScreenshotDialog extends Overlay { this.throttledUpdateLayerResolutionTable(); }), ); + this.registerDisposer( + this.screenshotManager.zoomMaybeChanged.add(() => { + this.populatePanelResolutionTable(); + }), + ); } private createNameInput(): HTMLInputElement { @@ -225,6 +237,24 @@ export class ScreenshotDialog extends Overlay { }); }); scaleMenu.appendChild(this.warningElement); + + const keepSliceFOVFixedDiv = document.createElement("div"); + keepSliceFOVFixedDiv.textContent = "Keep slice FOV fixed with scale change"; + + const keepSliceFOVFixedCheckbox = document.createElement("input"); + keepSliceFOVFixedCheckbox.classList.add( + "neuroglancer-screenshot-keep-slice-fov-checkbox", + ); + keepSliceFOVFixedCheckbox.type = "checkbox"; + keepSliceFOVFixedCheckbox.checked = + this.screenshotManager.shouldKeepSliceViewFOVFixed; + keepSliceFOVFixedCheckbox.addEventListener("change", () => { + this.screenshotManager.shouldKeepSliceViewFOVFixed = + keepSliceFOVFixedCheckbox.checked; + }); + keepSliceFOVFixedDiv.appendChild(keepSliceFOVFixedCheckbox); + scaleMenu.appendChild(keepSliceFOVFixedDiv); + this.handleScreenshotResize(); return scaleMenu; } @@ -300,6 +330,10 @@ export class ScreenshotDialog extends Overlay { } private populatePanelResolutionTable() { + // Clear the table before populating it + while (this.panelResolutionTable.rows.length > 1) { + this.panelResolutionTable.deleteRow(1); + } const resolutionTable = this.panelResolutionTable; const resolutions = getViewerPanelResolutions( this.screenshotManager.viewer.display.panels, diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 098b91288..dd595da65 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -32,6 +32,7 @@ import { RefCounted } from "#src/util/disposable.js"; import { NullarySignal, Signal } from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; +import { SliceViewPanel } from "#src/sliceview/panel.js"; const SCREENSHOT_TIMEOUT = 1000; @@ -135,12 +136,14 @@ async function extractViewportScreenshot( export class ScreenshotManager extends RefCounted { screenshotId: number = -1; - screenshotScale: number = 1; screenshotLoadStats: ScreenshotLoadStatistics | null = null; screenshotStartTime = 0; screenshotMode: ScreenshotMode = ScreenshotMode.OFF; statisticsUpdated = new Signal<(state: ScreenshotLoadStatistics) => void>(); + zoomMaybeChanged = new NullarySignal(); screenshotFinished = new NullarySignal(); + private _shouldKeepSliceViewFOVFixed: boolean = true; + private _screenshotScale: number = 1; private filename: string = ""; private lastUpdateTimestamp: number = 0; private gpuMemoryChangeTimestamp: number = 0; @@ -220,6 +223,29 @@ export class ScreenshotManager extends RefCounted { ); } + public get screenshotScale() { + return this._screenshotScale; + } + + public set screenshotScale(scale: number) { + this.handleScreenshotZoom(scale); + this._screenshotScale = scale; + } + + public get shouldKeepSliceViewFOVFixed() { + return this._shouldKeepSliceViewFOVFixed; + } + + public set shouldKeepSliceViewFOVFixed(enableFixedFOV: boolean) { + const wasInFixedFOVMode = this.shouldKeepSliceViewFOVFixed; + this._shouldKeepSliceViewFOVFixed = enableFixedFOV; + if (!enableFixedFOV && wasInFixedFOVMode) { + this.handleScreenshotZoom(1 / this.screenshotScale, true /* resetZoom */); + } else if (enableFixedFOV && !wasInFixedFOVMode) { + this.handleScreenshotZoom(this.screenshotScale, true /* resetZoom */); + } + } + takeScreenshot(filename: string = "") { this.filename = filename; this.viewer.display.screenshotMode.value = ScreenshotMode.ON; @@ -305,6 +331,23 @@ export class ScreenshotManager extends RefCounted { } } + private handleScreenshotZoom(scale: number, resetZoom: boolean = false) { + const oldScale = this.screenshotScale; + const scaleFactor = resetZoom ? scale : oldScale / scale; + + if (this.shouldKeepSliceViewFOVFixed || resetZoom) { + const { navigationState } = this.viewer; + for (const panel of this.viewer.display.panels) { + if (panel instanceof SliceViewPanel) { + const zoom = navigationState.zoomFactor.value; + navigationState.zoomFactor.value = zoom * scaleFactor; + break; + } + } + this.zoomMaybeChanged.dispatch(); + } + } + /** * Check if the screenshot is stuck by comparing the number of visible chunks * in the GPU with the previous number of visible chunks. If the number of From 7e5dda2f511ff1e45c79b94c75507f58dfb1e829 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 26 Sep 2024 16:25:00 +0200 Subject: [PATCH 57/66] fix: formatting --- src/ui/screenshot_menu.css | 49 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index 79c7b7394..f2861bfb3 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -14,31 +14,30 @@ * limitations under the License. */ - -.neuroglancer-screenshot-dialog{ - width: 60%; +.neuroglancer-screenshot-dialog { + width: 60%; } .neuroglancer-screenshot-scale-radio { - display: inline-block; - width: 20px; - margin-right: -2px; - cursor: pointer; + display: inline-block; + width: 20px; + margin-right: -2px; + cursor: pointer; } .neuroglancer-screenshot-filename-and-buttons { - margin-bottom: 5px; + margin-bottom: 5px; } .neuroglancer-screenshot-name-input { - width: 50%; - margin-right: 10px; - border: 1px solid #ccc; + width: 50%; + margin-right: 10px; + border: 1px solid #ccc; } .neuroglancer-screenshot-button { - cursor: pointer; - margin: 2px; + cursor: pointer; + margin: 2px; } .neuroglancer-screenshot-close-button { @@ -47,28 +46,28 @@ } .neuroglancer-screenshot-statistics-title { - margin-top: 5px; + margin-top: 5px; } .neuroglancer-screenshot-statistics-table { - width: 100%; - border-collapse: collapse; - margin-top: 5px; + width: 100%; + border-collapse: collapse; + margin-top: 5px; } .neuroglancer-screenshot-statistics-table th, .neuroglancer-screenshot-statistics-table td { - padding: 12px 15px; - text-align: left; - border-bottom: 1px solid #ddd; + padding: 12px 15px; + text-align: left; + border-bottom: 1px solid #ddd; } .neuroglancer-screenshot-statistics-table th { - background-color: #f8f8f8; - font-weight: bold; - color: #555; + background-color: #f8f8f8; + font-weight: bold; + color: #555; } .neuroglancer-screenshot-warning { - color: red; -} \ No newline at end of file + color: red; +} From 5862bf91ebe9386617400dcf334173704260f85d Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 26 Sep 2024 16:25:17 +0200 Subject: [PATCH 58/66] fix: possible race condition between screenshot taking and resetting zoom back to original --- src/util/screenshot_manager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index dd595da65..ddbc6749f 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -23,6 +23,7 @@ import type { ScreenshotChunkStatistics, } from "#src/python_integration/screenshots.js"; import { RenderedDataPanel } from "#src/rendered_data_panel.js"; +import { SliceViewPanel } from "#src/sliceview/panel.js"; import { columnSpecifications, getChunkSourceIdentifier, @@ -32,7 +33,6 @@ import { RefCounted } from "#src/util/disposable.js"; import { NullarySignal, Signal } from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; import type { Viewer } from "#src/viewer.js"; -import { SliceViewPanel } from "#src/sliceview/panel.js"; const SCREENSHOT_TIMEOUT = 1000; @@ -190,7 +190,6 @@ export class ScreenshotManager extends RefCounted { this.registerDisposer( this.viewer.screenshotHandler.sendScreenshotRequested.add( (actionState) => { - this.screenshotFinished.dispatch(); this.saveScreenshot(actionState); }, ), @@ -415,6 +414,7 @@ export class ScreenshotManager extends RefCounted { console.error("Failed to save screenshot:", error); } finally { this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; + this.screenshotFinished.dispatch(); } } From 7ebf5584ab6fd50cab617ad4c22253323af88f0b Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 26 Sep 2024 18:28:21 +0200 Subject: [PATCH 59/66] feat: add panel pixel sizes --- src/ui/screenshot_menu.ts | 38 +++++++++++++++++++++++++ src/util/screenshot_manager.ts | 43 ++++++++++++++++++++++++++--- src/util/viewer_resolution_stats.ts | 13 ++++++--- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 73b4d9279..88c048ff7 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -89,6 +89,7 @@ export class ScreenshotDialog extends Overlay { private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; private panelResolutionTable: HTMLTableElement; + private panelPixelSizeTable: HTMLTableElement; private layerResolutionTable: HTMLTableElement; private statisticsContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; @@ -101,6 +102,8 @@ export class ScreenshotDialog extends Overlay { private throttledUpdateLayerResolutionTable = this.registerCancellable( throttle(() => { this.populateLayerResolutionTable(); + this.populatePanelPixelSizeTable(); + this.handleScreenshotResize(); }, 500), ); constructor(private screenshotManager: ScreenshotManager) { @@ -148,10 +151,12 @@ export class ScreenshotDialog extends Overlay { this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(this.createPanelResolutionTable()); + this.content.appendChild(this.createPanelPixelSizeTable()); this.content.appendChild(this.createLayerResolutionTable()); this.content.appendChild(this.createStatisticsTable()); this.updateUIBasedOnMode(); this.populatePanelResolutionTable(); + this.populatePanelPixelSizeTable(); this.throttledUpdateLayerResolutionTable(); } @@ -349,6 +354,39 @@ export class ScreenshotDialog extends Overlay { return resolutionTable; } + private createPanelPixelSizeTable() { + const resolutionTable = (this.panelPixelSizeTable = + document.createElement("table")); + resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); + resolutionTable.title = "Viewer resolution statistics"; + + const headerRow = resolutionTable.createTHead().insertRow(); + const keyHeader = document.createElement("th"); + keyHeader.textContent = "Panel type"; + headerRow.appendChild(keyHeader); + const valueHeader = document.createElement("th"); + valueHeader.textContent = "Resolution"; + headerRow.appendChild(valueHeader); + return resolutionTable; + } + + private populatePanelPixelSizeTable() { + // Clear the table before populating it + while (this.panelPixelSizeTable.rows.length > 1) { + this.panelPixelSizeTable.deleteRow(1); + } + const pixelSizeTable = this.panelPixelSizeTable; + const panelPixelSizes = + this.screenshotManager.calculateUniqueScaledPanelViewportSizes(); + for (const value of panelPixelSizes) { + const row = pixelSizeTable.insertRow(); + const keyCell = row.insertCell(); + const valueCell = row.insertCell(); + keyCell.textContent = value.type; + valueCell.textContent = `${value.width}x${value.height} px`; + } + } + private createLayerResolutionTable() { const resolutionTable = (this.layerResolutionTable = document.createElement("table")); diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index ddbc6749f..3df86d3a8 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -46,6 +46,12 @@ interface ViewportBounds { right: number; top: number; bottom: number; + panelType: string; +} + +interface CanvasSizeStatistics { + totalRenderPanelViewport: ViewportBounds; + individualRenderPanelViewports: ViewportBounds[]; } function saveBlobToFile(blob: Blob, filename: string) { @@ -74,13 +80,15 @@ function setExtension(filename: string, extension: string = ".png"): string { function calculateViewportBounds( panels: ReadonlySet, -): ViewportBounds { +): CanvasSizeStatistics { const viewportBounds = { left: Number.POSITIVE_INFINITY, right: Number.NEGATIVE_INFINITY, top: Number.POSITIVE_INFINITY, bottom: Number.NEGATIVE_INFINITY, + panelType: "All", }; + const allPanelViewports: ViewportBounds[] = []; for (const panel of panels) { if (!(panel instanceof RenderedDataPanel)) continue; const viewport = panel.renderViewport; @@ -93,8 +101,18 @@ function calculateViewportBounds( viewportBounds.right = Math.max(viewportBounds.right, panelRight); viewportBounds.top = Math.min(viewportBounds.top, panelTop); viewportBounds.bottom = Math.max(viewportBounds.bottom, panelBottom); + allPanelViewports.push({ + left: panelLeft, + right: panelRight, + top: panelTop, + bottom: panelBottom, + panelType: panel instanceof SliceViewPanel ? "SliceView" : "Rendered", + }); } - return viewportBounds; + return { + totalRenderPanelViewport: viewportBounds, + individualRenderPanelViewports: allPanelViewports, + }; } function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { @@ -266,7 +284,7 @@ export class ScreenshotManager extends RefCounted { calculatedScaledAndClippedSize() { const renderingPanelArea = calculateViewportBounds( this.viewer.display.panels, - ); + ).totalRenderPanelViewport; return { width: Math.round(renderingPanelArea.right - renderingPanelArea.left) * @@ -277,6 +295,23 @@ export class ScreenshotManager extends RefCounted { }; } + calculateUniqueScaledPanelViewportSizes() { + const panelAreas = calculateViewportBounds( + this.viewer.display.panels, + ).individualRenderPanelViewports; + const scaledPanelAreas = panelAreas.map((panelArea) => ({ + width: + Math.round(panelArea.right - panelArea.left) * this.screenshotScale, + height: + Math.round(panelArea.bottom - panelArea.top) * this.screenshotScale, + type: panelArea.panelType, + })); + const uniquePanelAreas = Array.from( + new Set(scaledPanelAreas.map((area) => JSON.stringify(area))), + ).map((area) => JSON.parse(area)); + return uniquePanelAreas; + } + private handleScreenshotStarted() { const { viewer } = this; const shouldIncreaseCanvasSize = this.screenshotScale !== 1; @@ -399,7 +434,7 @@ export class ScreenshotManager extends RefCounted { } const renderingPanelArea = calculateViewportBounds( this.viewer.display.panels, - ); + ).totalRenderPanelViewport; try { const croppedImage = await extractViewportScreenshot( this.viewer, diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 70bd64dc0..6be937500 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -119,18 +119,23 @@ export function getViewerLayerResolutions( } export function getViewerPanelResolutions(panels: ReadonlySet) { - function resolutionsEqual(resolution1: any[], resolution2: any[]) { + function resolutionsEqual( + resolution1: DimensionResolutionStats[], + resolution2: DimensionResolutionStats[], + ) { if (resolution1.length !== resolution2.length) { return false; } for (let i = 0; i < resolution1.length; ++i) { - if (resolution1[i].textContent !== resolution2[i].textContent) { + if ( + resolution1[i].resolutionWithUnit !== resolution2[i].resolutionWithUnit + ) { return false; } - if (resolution1[i].panelType !== resolution2[i].panelType) { + if (resolution1[i].parentType !== resolution2[i].parentType) { return false; } - if (resolution1[i].name !== resolution2[i].name) { + if (resolution1[i].dimensionName !== resolution2[i].dimensionName) { return false; } } From 7e9404f3bcb184c1908ff4c30210dd31babc3dc2 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 10:27:13 +0200 Subject: [PATCH 60/66] refactor: small changes for clarity --- src/sliceview/volume/renderlayer.ts | 2 +- src/ui/screenshot_menu.ts | 30 ++++++++++----------- src/util/viewer_resolution_stats.ts | 4 ++- src/volume_rendering/volume_render_layer.ts | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/sliceview/volume/renderlayer.ts b/src/sliceview/volume/renderlayer.ts index d8224efba..84d114bbb 100644 --- a/src/sliceview/volume/renderlayer.ts +++ b/src/sliceview/volume/renderlayer.ts @@ -341,8 +341,8 @@ export abstract class SliceViewVolumeRenderLayer< >; private tempChunkPosition: Float32Array; shaderParameters: WatchableValueInterface; + highestResolutionLoadedVoxelSize: Float32Array | undefined; private vertexIdHelper: VertexIdHelper; - public highestResolutionLoadedVoxelSize: Float32Array | undefined; constructor( multiscaleSource: MultiscaleVolumeChunkSource, diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 88c048ff7..5299caf5d 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -99,7 +99,7 @@ export class ScreenshotDialog extends Overlay { private layerResolutionKeyToCellMap: Map = new Map(); - private throttledUpdateLayerResolutionTable = this.registerCancellable( + private throttledUpdateTableStatistics = this.registerCancellable( throttle(() => { this.populateLayerResolutionTable(); this.populatePanelPixelSizeTable(); @@ -157,7 +157,7 @@ export class ScreenshotDialog extends Overlay { this.updateUIBasedOnMode(); this.populatePanelResolutionTable(); this.populatePanelPixelSizeTable(); - this.throttledUpdateLayerResolutionTable(); + this.throttledUpdateTableStatistics(); } private setupEventListeners() { @@ -173,7 +173,7 @@ export class ScreenshotDialog extends Overlay { ); this.registerDisposer( this.screenshotManager.viewer.display.updateFinished.add(() => { - this.throttledUpdateLayerResolutionTable(); + this.throttledUpdateTableStatistics(); }), ); this.registerDisposer( @@ -264,18 +264,6 @@ export class ScreenshotDialog extends Overlay { return scaleMenu; } - private handleScreenshotResize() { - const screenshotSize = - this.screenshotManager.calculatedScaledAndClippedSize(); - if (screenshotSize.width * screenshotSize.height > LARGE_SCREENSHOT_SIZE) { - this.warningElement.textContent = - "Warning: large screenshots (bigger than 4096x4096) may fail"; - } else { - this.warningElement.textContent = ""; - } - this.screenshotSizeText.textContent = `Screenshot size: ${screenshotSize.width}px, ${screenshotSize.height}px`; - } - private createStatisticsTable() { this.statisticsContainer = document.createElement("div"); this.statisticsContainer.classList.add( @@ -465,6 +453,18 @@ export class ScreenshotDialog extends Overlay { } } + private handleScreenshotResize() { + const screenshotSize = + this.screenshotManager.calculatedScaledAndClippedSize(); + if (screenshotSize.width * screenshotSize.height > LARGE_SCREENSHOT_SIZE) { + this.warningElement.textContent = + "Warning: large screenshots (bigger than 4096x4096) may fail"; + } else { + this.warningElement.textContent = ""; + } + this.screenshotSizeText.textContent = `Screenshot size: ${screenshotSize.width}px, ${screenshotSize.height}px`; + } + private parseStatistics( currentStatistics: ScreenshotLoadStatistics | null, ): UIScreenshotStatistics | null { diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 6be937500..ca704fc49 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -118,7 +118,9 @@ export function getViewerLayerResolutions( return map; } -export function getViewerPanelResolutions(panels: ReadonlySet) { +export function getViewerPanelResolutions( + panels: ReadonlySet, +): DimensionResolutionStats[][] { function resolutionsEqual( resolution1: DimensionResolutionStats[], resolution2: DimensionResolutionStats[], diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index a3b005060..1d55cd435 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -220,11 +220,11 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { chunkResolutionHistogram: RenderScaleHistogram; mode: TrackableVolumeRenderingModeValue; backend: ChunkRenderLayerFrontend; + highestResolutionLoadedVoxelSize: Float32Array | undefined; private modeOverride: TrackableVolumeRenderingModeValue; private vertexIdHelper: VertexIdHelper; private dataHistogramSpecifications: HistogramSpecifications; private dataResolutionIndex: number; - public highestResolutionLoadedVoxelSize: Float32Array | undefined; private shaderGetter: ParameterizedContextDependentShaderGetter< { emitter: ShaderModule; chunkFormat: ChunkFormat; wireFrame: boolean }, From 461a79eec0f3ecb6bb13bc33c63d08adaca47eca Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 10:49:37 +0200 Subject: [PATCH 61/66] docs: add docstring for resolution functions --- src/util/viewer_resolution_stats.ts | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index ca704fc49..7dc9f3e7c 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -36,6 +36,23 @@ interface LayerIdentifier { type: string; } +/** + * For each visible data layer, returns the resolution of the voxels + * in physical units for the most detailed resolution of the data for + * which any data is actually loaded. + * + * The resolution is for loaded data, so may be lower than the resolution requested + * for the layer, such as when there are memory constraints. + * + * The key for the returned map is the layer name and type. + * A single layer name can have multiple types, such as ImageRenderLayer and + * VolumeRenderingRenderLayer from the same named layer. + * + * As the dimensions of the voxels can be the same in each dimension, the + * function will return a single resolution if all dimensions in the layer are the + * same, with the name "All_". Otherwise, it will return the resolution for + * each dimension, with the name of the dimension as per the global viewer dim names. + */ export function getViewerLayerResolutions( viewer: Viewer, ): Map { @@ -118,8 +135,25 @@ export function getViewerLayerResolutions( return map; } +/** + * For each viewer panel, returns the scale in each dimension for that panel. + * + * It is quite common for all dimensions to have the same scale, so the function + * will return a single resolution for a panel if all dimensions in the panel are + * the same, with the name "All_". Otherwise, it will return the resolution for + * each dimension, with the name of the dimension as per the global dimension names. + * + * For orthographic projections or slice views, the scale is in pixels, otherwise it is in vh. + * + * @param panels The set of panels to get the resolutions for. E.g. viewer.display.panels + * @param onlyUniqueResolutions If true, only return panels with unique resolutions. + * It is quite common for all slice view panels to have the same resolution. + * + * @returns An array of resolutions for each panel. + */ export function getViewerPanelResolutions( panels: ReadonlySet, + onlyUniqueResolutions = true, ): DimensionResolutionStats[][] { function resolutionsEqual( resolution1: DimensionResolutionStats[], @@ -212,6 +246,9 @@ export function getViewerPanelResolutions( resolutions.push(panel_resolution); } + if (!onlyUniqueResolutions) { + return resolutions; + } const uniqueResolutions: DimensionResolutionStats[][] = []; for (const resolution of resolutions) { let found = false; From 970a3f7e48d9b51c2cf8134cf8a85ec82c965c76 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 11:17:28 +0200 Subject: [PATCH 62/66] docs, refactor: improve screenshot clarity --- src/ui/screenshot_menu.ts | 34 +++++++-- src/util/screenshot_manager.ts | 66 +++--------------- src/util/viewer_resolution_stats.ts | 104 +++++++++++++++++++++++----- 3 files changed, 127 insertions(+), 77 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 5299caf5d..5727423d1 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -12,6 +12,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * @file UI menu for taking screenshots from the viewer. */ import "#src/ui/screenshot_menu.css"; @@ -28,7 +30,7 @@ import { getViewerPanelResolutions, } from "#src/util/viewer_resolution_stats.js"; -// If true, the menu can be closed by clicking the close button +// If DEBUG_ALLOW_MENU_CLOSE is true, the menu can be closed by clicking the close button // Usually the user is locked into the screenshot menu until the screenshot is taken or cancelled // Setting this to true, and setting the SCREENSHOT_MENU_CLOSE_TIMEOUT in screenshot_manager.ts // to a high value can be useful for debugging canvas handling of the resize @@ -54,6 +56,9 @@ const layerNamesForUI = { SegmentationRenderLayer: "Segmentation slice (2D)", }; +/** + * Combine the resolution of all dimensions into a single string for UI display + */ function formatResolution(resolution: DimensionResolutionStats[]) { if (resolution.length === 0) { return { @@ -81,6 +86,27 @@ function formatResolution(resolution: DimensionResolutionStats[]) { } } +/** + * This menu allows the user to take a screenshot of the current view, with options to + * set the filename, scale, and force the screenshot to be taken immediately. + * Once a screenshot is initiated, the user is locked into the menu until the + * screenshot is taken or cancelled, to prevent + * the user from interacting with the viewer while the screenshot is being taken. + * + * The menu displays statistics about the current view, such as the number of loaded + * chunks, GPU memory usage, and download speed. These are to inform the user about the + * progress of the screenshot, as it may take some time to load all the data. + * + * The menu also displays the resolution of each panel in the viewer, as well as the resolution + * of the voxels loaded for each Image, Volume, and Segmentation layer. + * This is to inform the user about the the physical units of the data and panels, + * and to help them decide on the scale of the screenshot. + * + * The screenshot menu supports keeping the slice view FOV fixed when changing the scale of the screenshot. + * This will cause the viewer to zoom in or out to keep the same FOV in the slice view. + * For example, an x2 scale will cause the viewer in slice views to zoom in by a factor of 2 + * such that when the number of pixels in the slice view is doubled, the FOV remains the same. + */ export class ScreenshotDialog extends Overlay { private nameInput: HTMLInputElement; private takeScreenshotButton: HTMLButtonElement; @@ -317,7 +343,7 @@ export class ScreenshotDialog extends Overlay { keyHeader.textContent = "Panel type"; headerRow.appendChild(keyHeader); const valueHeader = document.createElement("th"); - valueHeader.textContent = "Resolution"; + valueHeader.textContent = "Physical resolution"; headerRow.appendChild(valueHeader); return resolutionTable; } @@ -353,7 +379,7 @@ export class ScreenshotDialog extends Overlay { keyHeader.textContent = "Panel type"; headerRow.appendChild(keyHeader); const valueHeader = document.createElement("th"); - valueHeader.textContent = "Resolution"; + valueHeader.textContent = "Pixel resolution"; headerRow.appendChild(valueHeader); return resolutionTable; } @@ -389,7 +415,7 @@ export class ScreenshotDialog extends Overlay { typeHeader.textContent = "Type"; headerRow.appendChild(typeHeader); const valueHeader = document.createElement("th"); - valueHeader.textContent = "Resolution"; + valueHeader.textContent = "Physical voxel resolution"; headerRow.appendChild(valueHeader); return resolutionTable; } diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 3df86d3a8..29c1e6937 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -12,17 +12,17 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * @file Builds upon the Python screenshot tool to allow viewer screenshots to be taken and saved. */ import { throttle } from "lodash-es"; import { numChunkStatistics } from "#src/chunk_manager/base.js"; -import type { RenderedPanel } from "#src/display_context.js"; import type { ScreenshotActionState, StatisticsActionState, ScreenshotChunkStatistics, } from "#src/python_integration/screenshots.js"; -import { RenderedDataPanel } from "#src/rendered_data_panel.js"; import { SliceViewPanel } from "#src/sliceview/panel.js"; import { columnSpecifications, @@ -32,6 +32,10 @@ import { import { RefCounted } from "#src/util/disposable.js"; import { NullarySignal, Signal } from "#src/util/signal.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; +import { + calculatePanelViewportBounds, + type PanelViewport, +} from "#src/util/viewer_resolution_stats.js"; import type { Viewer } from "#src/viewer.js"; const SCREENSHOT_TIMEOUT = 1000; @@ -41,19 +45,6 @@ export interface ScreenshotLoadStatistics extends ScreenshotChunkStatistics { gpuMemoryCapacity: number; } -interface ViewportBounds { - left: number; - right: number; - top: number; - bottom: number; - panelType: string; -} - -interface CanvasSizeStatistics { - totalRenderPanelViewport: ViewportBounds; - individualRenderPanelViewports: ViewportBounds[]; -} - function saveBlobToFile(blob: Blob, filename: string) { const a = document.createElement("a"); const url = URL.createObjectURL(blob); @@ -78,43 +69,6 @@ function setExtension(filename: string, extension: string = ".png"): string { return filename.endsWith(extension) ? filename : replaceExtension(filename); } -function calculateViewportBounds( - panels: ReadonlySet, -): CanvasSizeStatistics { - const viewportBounds = { - left: Number.POSITIVE_INFINITY, - right: Number.NEGATIVE_INFINITY, - top: Number.POSITIVE_INFINITY, - bottom: Number.NEGATIVE_INFINITY, - panelType: "All", - }; - const allPanelViewports: ViewportBounds[] = []; - for (const panel of panels) { - if (!(panel instanceof RenderedDataPanel)) continue; - const viewport = panel.renderViewport; - const { width, height } = viewport; - const panelLeft = panel.canvasRelativeClippedLeft; - const panelTop = panel.canvasRelativeClippedTop; - const panelRight = panelLeft + width; - const panelBottom = panelTop + height; - viewportBounds.left = Math.min(viewportBounds.left, panelLeft); - viewportBounds.right = Math.max(viewportBounds.right, panelRight); - viewportBounds.top = Math.min(viewportBounds.top, panelTop); - viewportBounds.bottom = Math.max(viewportBounds.bottom, panelBottom); - allPanelViewports.push({ - left: panelLeft, - right: panelRight, - top: panelTop, - bottom: panelBottom, - panelType: panel instanceof SliceViewPanel ? "SliceView" : "Rendered", - }); - } - return { - totalRenderPanelViewport: viewportBounds, - individualRenderPanelViewports: allPanelViewports, - }; -} - function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { return new Promise((resolve, reject) => { canvas.toBlob((blob) => { @@ -129,7 +83,7 @@ function canvasToBlob(canvas: HTMLCanvasElement, type: string): Promise { async function extractViewportScreenshot( viewer: Viewer, - viewportBounds: ViewportBounds, + viewportBounds: PanelViewport, ): Promise { const cropWidth = viewportBounds.right - viewportBounds.left; const cropHeight = viewportBounds.bottom - viewportBounds.top; @@ -282,7 +236,7 @@ export class ScreenshotManager extends RefCounted { // Scales the screenshot by the given factor, and calculates the cropped area calculatedScaledAndClippedSize() { - const renderingPanelArea = calculateViewportBounds( + const renderingPanelArea = calculatePanelViewportBounds( this.viewer.display.panels, ).totalRenderPanelViewport; return { @@ -296,7 +250,7 @@ export class ScreenshotManager extends RefCounted { } calculateUniqueScaledPanelViewportSizes() { - const panelAreas = calculateViewportBounds( + const panelAreas = calculatePanelViewportBounds( this.viewer.display.panels, ).individualRenderPanelViewports; const scaledPanelAreas = panelAreas.map((panelArea) => ({ @@ -432,7 +386,7 @@ export class ScreenshotManager extends RefCounted { this.viewer.display.screenshotMode.value = ScreenshotMode.OFF; return; } - const renderingPanelArea = calculateViewportBounds( + const renderingPanelArea = calculatePanelViewportBounds( this.viewer.display.panels, ).totalRenderPanelViewport; try { diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index 7dc9f3e7c..e7a5754c6 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -12,6 +12,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * @file Helper functions to get the resolution of the viewer layers and panels. */ import type { RenderedPanel } from "#src/display_context.js"; @@ -36,6 +38,19 @@ interface LayerIdentifier { type: string; } +export interface PanelViewport { + left: number; + right: number; + top: number; + bottom: number; + panelType: string; +} + +interface CanvasSizeStatistics { + totalRenderPanelViewport: PanelViewport; + individualRenderPanelViewports: PanelViewport[]; +} + /** * For each visible data layer, returns the resolution of the voxels * in physical units for the most detailed resolution of the data for @@ -148,7 +163,7 @@ export function getViewerLayerResolutions( * @param panels The set of panels to get the resolutions for. E.g. viewer.display.panels * @param onlyUniqueResolutions If true, only return panels with unique resolutions. * It is quite common for all slice view panels to have the same resolution. - * + * * @returns An array of resolutions for each panel. */ export function getViewerPanelResolutions( @@ -182,22 +197,11 @@ export function getViewerPanelResolutions( for (const panel of panels) { if (!(panel instanceof RenderedDataPanel)) continue; const panel_resolution = []; - const isOrtographicProjection = - panel instanceof PerspectivePanel && - panel.viewer.orthographicProjection.value; - - const panelDimensionUnit = - panel instanceof SliceViewPanel || isOrtographicProjection ? "px" : "vh"; - let panelType: string; - if (panel instanceof SliceViewPanel) { - panelType = "Slice view"; - } else if (isOrtographicProjection) { - panelType = "Orthographic view"; - } else if (panel instanceof PerspectivePanel) { - panelType = "Perspective view"; - } else { - panelType = "Unknown"; - } + const { + panelType, + panelDimensionUnit, + }: { panelType: string; panelDimensionUnit: string } = + determinePanelTypeAndUnit(panel); const { navigationState } = panel; const { displayDimensionIndices, @@ -264,3 +268,69 @@ export function getViewerPanelResolutions( } return uniqueResolutions; } + +function determinePanelTypeAndUnit(panel: RenderedDataPanel) { + const isOrtographicProjection = + panel instanceof PerspectivePanel && + panel.viewer.orthographicProjection.value; + + const panelDimensionUnit = + panel instanceof SliceViewPanel || isOrtographicProjection ? "px" : "vh"; + let panelType: string; + if (panel instanceof SliceViewPanel) { + panelType = "Slice view (2D)"; + } else if (isOrtographicProjection) { + panelType = "Orthographic projection (3D)"; + } else if (panel instanceof PerspectivePanel) { + panelType = "Perspective projection (3D)"; + } else { + panelType = "Unknown"; + } + return { panelType, panelDimensionUnit }; +} + +/** + * Calculates the viewport bounds of the viewer render data panels individually. + * And also calculates the total viewport bounds of all the render data panels combined. + * + * The total bounds can contain some non-panel areas, such as the layer bar if + * the panels have been duplicated so that the layer bar sits in the middle + * of the visible rendered panels. + */ +export function calculatePanelViewportBounds( + panels: ReadonlySet, +): CanvasSizeStatistics { + const viewportBounds = { + left: Number.POSITIVE_INFINITY, + right: Number.NEGATIVE_INFINITY, + top: Number.POSITIVE_INFINITY, + bottom: Number.NEGATIVE_INFINITY, + panelType: "All", + }; + const allPanelViewports: PanelViewport[] = []; + for (const panel of panels) { + if (!(panel instanceof RenderedDataPanel)) continue; + const viewport = panel.renderViewport; + const { width, height } = viewport; + const panelLeft = panel.canvasRelativeClippedLeft; + const panelTop = panel.canvasRelativeClippedTop; + const panelRight = panelLeft + width; + const panelBottom = panelTop + height; + viewportBounds.left = Math.min(viewportBounds.left, panelLeft); + viewportBounds.right = Math.max(viewportBounds.right, panelRight); + viewportBounds.top = Math.min(viewportBounds.top, panelTop); + viewportBounds.bottom = Math.max(viewportBounds.bottom, panelBottom); + + allPanelViewports.push({ + left: panelLeft, + right: panelRight, + top: panelTop, + bottom: panelBottom, + panelType: determinePanelTypeAndUnit(panel).panelType, + }); + } + return { + totalRenderPanelViewport: viewportBounds, + individualRenderPanelViewports: allPanelViewports, + }; +} From 0cb7ef2746ba8253d637a2fc462c18f1e5634c48 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 11:29:37 +0200 Subject: [PATCH 63/66] docs: screenshot manager notes --- src/util/screenshot_manager.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 29c1e6937..6e5f72406 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -106,6 +106,13 @@ async function extractViewportScreenshot( return croppedBlob; } +/** + * Manages the screenshot functionality from the viewer viewer. + * + * Responsible for linking up the Python screenshot tool with the viewer, and handling the screenshot process. + * The screenshot manager provides information about updates in the screenshot process, and allows for the screenshot to be taken and saved. + * The screenshot UI menu listens to the signals emitted by the screenshot manager to update the UI. + */ export class ScreenshotManager extends RefCounted { screenshotId: number = -1; screenshotLoadStats: ScreenshotLoadStatistics | null = null; From 6c171dfd26bb842eff9cde88661a6fa7bdcf4e66 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 11:43:16 +0200 Subject: [PATCH 64/66] feat: combine panel physical and pixel resolution in UI --- src/ui/screenshot_menu.ts | 80 ++++++++++++----------------- src/util/screenshot_manager.ts | 19 +------ src/util/viewer_resolution_stats.ts | 65 ++++++++++++++++++----- 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 5727423d1..5c8bd8429 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -24,7 +24,10 @@ import type { ScreenshotManager, } from "#src/util/screenshot_manager.js"; import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js"; -import type { DimensionResolutionStats } from "#src/util/viewer_resolution_stats.js"; +import type { + DimensionResolutionStats, + PanelViewport, +} from "#src/util/viewer_resolution_stats.js"; import { getViewerLayerResolutions, getViewerPanelResolutions, @@ -59,7 +62,7 @@ const layerNamesForUI = { /** * Combine the resolution of all dimensions into a single string for UI display */ -function formatResolution(resolution: DimensionResolutionStats[]) { +function formatPhysicalResolution(resolution: DimensionResolutionStats[]) { if (resolution.length === 0) { return { type: "Loading...", @@ -101,7 +104,7 @@ function formatResolution(resolution: DimensionResolutionStats[]) { * of the voxels loaded for each Image, Volume, and Segmentation layer. * This is to inform the user about the the physical units of the data and panels, * and to help them decide on the scale of the screenshot. - * + * * The screenshot menu supports keeping the slice view FOV fixed when changing the scale of the screenshot. * This will cause the viewer to zoom in or out to keep the same FOV in the slice view. * For example, an x2 scale will cause the viewer in slice views to zoom in by a factor of 2 @@ -115,7 +118,6 @@ export class ScreenshotDialog extends Overlay { private forceScreenshotButton: HTMLButtonElement; private statisticsTable: HTMLTableElement; private panelResolutionTable: HTMLTableElement; - private panelPixelSizeTable: HTMLTableElement; private layerResolutionTable: HTMLTableElement; private statisticsContainer: HTMLDivElement; private filenameAndButtonsContainer: HTMLDivElement; @@ -128,7 +130,6 @@ export class ScreenshotDialog extends Overlay { private throttledUpdateTableStatistics = this.registerCancellable( throttle(() => { this.populateLayerResolutionTable(); - this.populatePanelPixelSizeTable(); this.handleScreenshotResize(); }, 500), ); @@ -177,12 +178,10 @@ export class ScreenshotDialog extends Overlay { this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); this.content.appendChild(this.createPanelResolutionTable()); - this.content.appendChild(this.createPanelPixelSizeTable()); this.content.appendChild(this.createLayerResolutionTable()); this.content.appendChild(this.createStatisticsTable()); this.updateUIBasedOnMode(); this.populatePanelResolutionTable(); - this.populatePanelPixelSizeTable(); this.throttledUpdateTableStatistics(); } @@ -342,13 +341,23 @@ export class ScreenshotDialog extends Overlay { const keyHeader = document.createElement("th"); keyHeader.textContent = "Panel type"; headerRow.appendChild(keyHeader); - const valueHeader = document.createElement("th"); - valueHeader.textContent = "Physical resolution"; - headerRow.appendChild(valueHeader); + const physicalValueHeader = document.createElement("th"); + physicalValueHeader.textContent = "Physical resolution"; + headerRow.appendChild(physicalValueHeader); + const pixelValueHeader = document.createElement("th"); + pixelValueHeader.textContent = "Pixel resolution"; + headerRow.appendChild(pixelValueHeader); return resolutionTable; } private populatePanelResolutionTable() { + function formatPixelResolution(panelArea: PanelViewport, scale: number) { + const width = Math.round(panelArea.right - panelArea.left) * scale; + const height = Math.round(panelArea.bottom - panelArea.top) * scale; + const type = panelArea.panelType; + return { width, height, type }; + } + // Clear the table before populating it while (this.panelResolutionTable.rows.length > 1) { this.panelResolutionTable.deleteRow(1); @@ -358,49 +367,24 @@ export class ScreenshotDialog extends Overlay { this.screenshotManager.viewer.display.panels, ); for (const resolution of resolutions) { - const resolutionStrings = formatResolution(resolution); + const physicalResolution = formatPhysicalResolution( + resolution.physicalResolution, + ); + const pixelResolution = formatPixelResolution( + resolution.pixelResolution, + this.screenshotManager.screenshotScale, + ); const row = resolutionTable.insertRow(); const keyCell = row.insertCell(); - const valueCell = row.insertCell(); - keyCell.textContent = resolutionStrings.type; - valueCell.textContent = resolutionStrings.resolution; + const physicalValueCell = row.insertCell(); + keyCell.textContent = physicalResolution.type; + physicalValueCell.textContent = physicalResolution.resolution; + const pixelValueCell = row.insertCell(); + pixelValueCell.textContent = `${pixelResolution.width}x${pixelResolution.height} px`; } return resolutionTable; } - private createPanelPixelSizeTable() { - const resolutionTable = (this.panelPixelSizeTable = - document.createElement("table")); - resolutionTable.classList.add("neuroglancer-screenshot-resolution-table"); - resolutionTable.title = "Viewer resolution statistics"; - - const headerRow = resolutionTable.createTHead().insertRow(); - const keyHeader = document.createElement("th"); - keyHeader.textContent = "Panel type"; - headerRow.appendChild(keyHeader); - const valueHeader = document.createElement("th"); - valueHeader.textContent = "Pixel resolution"; - headerRow.appendChild(valueHeader); - return resolutionTable; - } - - private populatePanelPixelSizeTable() { - // Clear the table before populating it - while (this.panelPixelSizeTable.rows.length > 1) { - this.panelPixelSizeTable.deleteRow(1); - } - const pixelSizeTable = this.panelPixelSizeTable; - const panelPixelSizes = - this.screenshotManager.calculateUniqueScaledPanelViewportSizes(); - for (const value of panelPixelSizes) { - const row = pixelSizeTable.insertRow(); - const keyCell = row.insertCell(); - const valueCell = row.insertCell(); - keyCell.textContent = value.type; - valueCell.textContent = `${value.width}x${value.height} px`; - } - } - private createLayerResolutionTable() { const resolutionTable = (this.layerResolutionTable = document.createElement("table")); @@ -442,7 +426,7 @@ export class ScreenshotDialog extends Overlay { layerNamesForUI[type as keyof typeof layerNamesForUI]; this.layerResolutionKeyToCellMap.set(stringKey, valueCell); } - valueCell.textContent = formatResolution(value).resolution; + valueCell.textContent = formatPhysicalResolution(value).resolution; } } diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 6e5f72406..8c6da8d25 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -108,7 +108,7 @@ async function extractViewportScreenshot( /** * Manages the screenshot functionality from the viewer viewer. - * + * * Responsible for linking up the Python screenshot tool with the viewer, and handling the screenshot process. * The screenshot manager provides information about updates in the screenshot process, and allows for the screenshot to be taken and saved. * The screenshot UI menu listens to the signals emitted by the screenshot manager to update the UI. @@ -256,23 +256,6 @@ export class ScreenshotManager extends RefCounted { }; } - calculateUniqueScaledPanelViewportSizes() { - const panelAreas = calculatePanelViewportBounds( - this.viewer.display.panels, - ).individualRenderPanelViewports; - const scaledPanelAreas = panelAreas.map((panelArea) => ({ - width: - Math.round(panelArea.right - panelArea.left) * this.screenshotScale, - height: - Math.round(panelArea.bottom - panelArea.top) * this.screenshotScale, - type: panelArea.panelType, - })); - const uniquePanelAreas = Array.from( - new Set(scaledPanelAreas.map((area) => JSON.stringify(area))), - ).map((area) => JSON.parse(area)); - return uniquePanelAreas; - } - private handleScreenshotStarted() { const { viewer } = this; const shouldIncreaseCanvasSize = this.screenshotScale !== 1; diff --git a/src/util/viewer_resolution_stats.ts b/src/util/viewer_resolution_stats.ts index e7a5754c6..44fcf204b 100644 --- a/src/util/viewer_resolution_stats.ts +++ b/src/util/viewer_resolution_stats.ts @@ -46,6 +46,11 @@ export interface PanelViewport { panelType: string; } +export interface PanelResolutionStats { + pixelResolution: PanelViewport; + physicalResolution: DimensionResolutionStats[]; +} + interface CanvasSizeStatistics { totalRenderPanelViewport: PanelViewport; individualRenderPanelViewports: PanelViewport[]; @@ -164,44 +169,78 @@ export function getViewerLayerResolutions( * @param onlyUniqueResolutions If true, only return panels with unique resolutions. * It is quite common for all slice view panels to have the same resolution. * - * @returns An array of resolutions for each panel. + * @returns An array of resolutions for each panel, both in physical units and pixel units. */ export function getViewerPanelResolutions( panels: ReadonlySet, onlyUniqueResolutions = true, -): DimensionResolutionStats[][] { +): PanelResolutionStats[] { function resolutionsEqual( - resolution1: DimensionResolutionStats[], - resolution2: DimensionResolutionStats[], + panelResolution1: PanelResolutionStats, + panelResolution2: PanelResolutionStats, ) { - if (resolution1.length !== resolution2.length) { + const physicalResolution1 = panelResolution1.physicalResolution; + const physicalResolution2 = panelResolution2.physicalResolution; + if (physicalResolution1.length !== physicalResolution2.length) { return false; } - for (let i = 0; i < resolution1.length; ++i) { + for (let i = 0; i < physicalResolution1.length; ++i) { if ( - resolution1[i].resolutionWithUnit !== resolution2[i].resolutionWithUnit + physicalResolution1[i].resolutionWithUnit !== + physicalResolution2[i].resolutionWithUnit ) { return false; } - if (resolution1[i].parentType !== resolution2[i].parentType) { + if ( + physicalResolution1[i].parentType !== physicalResolution2[i].parentType + ) { return false; } - if (resolution1[i].dimensionName !== resolution2[i].dimensionName) { + if ( + physicalResolution1[i].dimensionName !== + physicalResolution2[i].dimensionName + ) { return false; } } + const pixelResolution1 = panelResolution1.pixelResolution; + const pixelResolution2 = panelResolution2.pixelResolution; + const width1 = pixelResolution1.right - pixelResolution1.left; + const width2 = pixelResolution2.right - pixelResolution2.left; + const height1 = pixelResolution1.bottom - pixelResolution1.top; + const height2 = pixelResolution2.bottom - pixelResolution2.top; + if (width1 !== width2 || height1 !== height2) { + return false; + } + return true; } - const resolutions: DimensionResolutionStats[][] = []; + const resolutions: PanelResolutionStats[] = []; for (const panel of panels) { if (!(panel instanceof RenderedDataPanel)) continue; - const panel_resolution = []; + const viewport = panel.renderViewport; + const { width, height } = viewport; + const panelLeft = panel.canvasRelativeClippedLeft; + const panelTop = panel.canvasRelativeClippedTop; + const panelRight = panelLeft + width; + const panelBottom = panelTop + height; const { panelType, panelDimensionUnit, }: { panelType: string; panelDimensionUnit: string } = determinePanelTypeAndUnit(panel); + const panel_resolution: PanelResolutionStats = { + pixelResolution: { + left: panelLeft, + right: panelRight, + top: panelTop, + bottom: panelBottom, + panelType, + }, + physicalResolution: [], + }; + const { physicalResolution } = panel_resolution; const { navigationState } = panel; const { displayDimensionIndices, @@ -239,7 +278,7 @@ export function getViewerPanelResolutions( displayDimensionUnits[i], { precision: 2, elide1: false }, ); - panel_resolution.push({ + physicalResolution.push({ parentType: panelType, resolutionWithUnit: `${formattedScale}/${panelDimensionUnit}`, dimensionName: singleScale ? "All_" : dimensionName, @@ -253,7 +292,7 @@ export function getViewerPanelResolutions( if (!onlyUniqueResolutions) { return resolutions; } - const uniqueResolutions: DimensionResolutionStats[][] = []; + const uniqueResolutions: PanelResolutionStats[] = []; for (const resolution of resolutions) { let found = false; for (const uniqueResolution of uniqueResolutions) { From 9b4c306ff4275c0569c887e4cbfeb7b2dcd31a5e Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 11:58:38 +0200 Subject: [PATCH 65/66] fix: handle correclty the pixel indicator on zoom in UI --- src/ui/screenshot_menu.ts | 1 + src/util/screenshot_manager.ts | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index 5c8bd8429..b27bb7fcf 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -131,6 +131,7 @@ export class ScreenshotDialog extends Overlay { throttle(() => { this.populateLayerResolutionTable(); this.handleScreenshotResize(); + this.populatePanelResolutionTable(); }, 500), ); constructor(private screenshotManager: ScreenshotManager) { diff --git a/src/util/screenshot_manager.ts b/src/util/screenshot_manager.ts index 8c6da8d25..abcbda08e 100644 --- a/src/util/screenshot_manager.ts +++ b/src/util/screenshot_manager.ts @@ -119,8 +119,8 @@ export class ScreenshotManager extends RefCounted { screenshotStartTime = 0; screenshotMode: ScreenshotMode = ScreenshotMode.OFF; statisticsUpdated = new Signal<(state: ScreenshotLoadStatistics) => void>(); - zoomMaybeChanged = new NullarySignal(); screenshotFinished = new NullarySignal(); + zoomMaybeChanged = new NullarySignal(); private _shouldKeepSliceViewFOVFixed: boolean = true; private _screenshotScale: number = 1; private filename: string = ""; @@ -208,6 +208,7 @@ export class ScreenshotManager extends RefCounted { public set screenshotScale(scale: number) { this.handleScreenshotZoom(scale); this._screenshotScale = scale; + this.zoomMaybeChanged.dispatch(); } public get shouldKeepSliceViewFOVFixed() { @@ -219,8 +220,10 @@ export class ScreenshotManager extends RefCounted { this._shouldKeepSliceViewFOVFixed = enableFixedFOV; if (!enableFixedFOV && wasInFixedFOVMode) { this.handleScreenshotZoom(1 / this.screenshotScale, true /* resetZoom */); + this.zoomMaybeChanged.dispatch(); } else if (enableFixedFOV && !wasInFixedFOVMode) { this.handleScreenshotZoom(this.screenshotScale, true /* resetZoom */); + this.zoomMaybeChanged.dispatch(); } } @@ -322,7 +325,6 @@ export class ScreenshotManager extends RefCounted { break; } } - this.zoomMaybeChanged.dispatch(); } } From c9eb62a0c9e184ad63e361a7a713686e87b7fc52 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Fri, 27 Sep 2024 15:10:17 +0200 Subject: [PATCH 66/66] feat(ui): initial version of tooltips --- src/ui/screenshot_menu.css | 10 ++++++++++ src/ui/screenshot_menu.ts | 28 +++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/ui/screenshot_menu.css b/src/ui/screenshot_menu.css index f2861bfb3..589f80c94 100644 --- a/src/ui/screenshot_menu.css +++ b/src/ui/screenshot_menu.css @@ -71,3 +71,13 @@ .neuroglancer-screenshot-warning { color: red; } + +.neuroglancer-screenshot-tooltip { + user-select: none; + text-align: center; + padding: 0.1em; + border: 0.1em solid black; + width: 1.1em; + height: 1.1em; + border-radius: 100%; +} \ No newline at end of file diff --git a/src/ui/screenshot_menu.ts b/src/ui/screenshot_menu.ts index b27bb7fcf..2924296c0 100644 --- a/src/ui/screenshot_menu.ts +++ b/src/ui/screenshot_menu.ts @@ -149,6 +149,22 @@ export class ScreenshotDialog extends Overlay { } } + private setupHelpTooltips() { + const generalSettingsTooltip = document.createElement("div"); + generalSettingsTooltip.classList.add("neuroglancer-screenshot-tooltip"); + generalSettingsTooltip.title = + "In the main viewer, see the settings (cog icon, top right) for options to turn off the axis line indicators, the scale bar, and the default annotations (yellow bounding box)"; + generalSettingsTooltip.textContent = "?"; + + const orthographicSettingsTooltip = document.createElement("div"); + orthographicSettingsTooltip.classList.add("neuroglancer-screenshot-tooltip"); + orthographicSettingsTooltip.title = + "In the main viewer, press 'o' to toggle between perspective and orthographic views"; + orthographicSettingsTooltip.textContent = "?"; + + return { generalSettingsTooltip, orthographicSettingsTooltip }; + } + private initializeUI() { this.content.classList.add("neuroglancer-screenshot-dialog"); @@ -174,10 +190,20 @@ export class ScreenshotDialog extends Overlay { this.filenameAndButtonsContainer.appendChild(this.takeScreenshotButton); this.filenameAndButtonsContainer.appendChild(this.forceScreenshotButton); - this.content.appendChild(this.closeMenuButton); + const tooltip = this.setupHelpTooltips(); + + const closeAndHelpContainer = document.createElement("div"); + closeAndHelpContainer.classList.add( + "neuroglancer-screenshot-close-and-help", + ); + closeAndHelpContainer.appendChild(tooltip.generalSettingsTooltip); + closeAndHelpContainer.appendChild(this.closeMenuButton); + + this.content.appendChild(closeAndHelpContainer); this.content.appendChild(this.cancelScreenshotButton); this.content.appendChild(this.filenameAndButtonsContainer); this.content.appendChild(this.createScaleRadioButtons()); + this.content.appendChild(tooltip.orthographicSettingsTooltip); this.content.appendChild(this.createPanelResolutionTable()); this.content.appendChild(this.createLayerResolutionTable()); this.content.appendChild(this.createStatisticsTable());