diff --git a/build.gradle.kts b/build.gradle.kts index 533340dc6..6da63a564 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { } implementation("net.java.dev.jna:jna-platform:5.14.0") +// implementation("net.clearvolume:cleargl") implementation("org.janelia.saalfeldlab:n5") implementation("org.janelia.saalfeldlab:n5-imglib2") implementation("org.apache.logging.log4j:log4j-api:2.20.0") @@ -148,6 +149,19 @@ dependencies { val isRelease: Boolean get() = System.getProperty("release") == "true" +//kotlin { +// jvmToolchain(21) +//// compilerOptions { +//// jvmTarget = JvmTarget.JVM_21 +//// freeCompilerArgs = listOf("-Xinline-classes", "-opt-in=kotlin.RequiresOptIn") +//// } +//} +// +//java { +// targetCompatibility = JavaVersion.VERSION_21 +// sourceCompatibility = JavaVersion.VERSION_21 +//} + tasks { withType().all { val version = System.getProperty("java.version").substringBefore('.').toInt() @@ -414,30 +428,32 @@ tasks { register(name = "run") { classpath = sourceSets.main.get().runtimeClasspath - if (project.hasProperty("target")) { - project.property("target")?.let { target -> - classpath = sourceSets.test.get().runtimeClasspath - - println("Target is $target") - // if(target.endsWith(".kt")) { - // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") - // } else { - // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") - // } - - mainClass.set("$target") - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } + var target: Any? = null + if (project.hasProperty("target")) + target = project.property("target") + target = "StartEyeTrackingDirectlyKt" + if (target != null) { + classpath = sourceSets.test.get().runtimeClasspath + + println("Target is $target") + // if(target.endsWith(".kt")) { + // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") + // } else { + // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") + // } + + mainClass.set("$target") + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - println("Will run target $target with classpath $classpath, main=${mainClass.get()}") - println("JVM arguments passed to target: $allJvmArgs") + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } } + + println("Will run target $target with classpath $classpath, main=${mainClass.get()}") + println("JVM arguments passed to target: $allJvmArgs") } } diff --git a/gradle.properties b/gradle.properties index 91aaef276..a380f1c73 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=2g org.gradle.caching=true jvmTarget=21 -#useLocalScenery=true +useLocalScenery=true kotlinVersion=1.9.23 dokkaVersion=1.9.10 scijavaParentPOMVersion=37.0.0 diff --git a/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java b/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java index 91c12ad8a..24119d0ed 100644 --- a/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java +++ b/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java @@ -22,6 +22,8 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; + + public class DefaultLogger extends Logger { @SuppressWarnings("UnusedParameters") public DefaultLogger(String category) { } diff --git a/src/main/java/sc/iview/commands/file/OpenDirofTif.java b/src/main/java/sc/iview/commands/file/OpenDirofTif.java new file mode 100644 index 000000000..e565332c2 --- /dev/null +++ b/src/main/java/sc/iview/commands/file/OpenDirofTif.java @@ -0,0 +1,85 @@ +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview.commands.file; + +import org.scijava.command.Command; +import org.scijava.io.IOService; +import org.scijava.log.LogService; +import org.scijava.plugin.Menu; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import sc.iview.SciView; + +import java.io.File; +import java.io.IOException; + +import static sc.iview.commands.MenuWeights.FILE; +import static sc.iview.commands.MenuWeights.FILE_OPEN; + +/** + * Command to open a file in SciView + * + * @author Kyle Harrington + * + */ +@Plugin(type = Command.class, menuRoot = "SciView", // + menu = { @Menu(label = "File", weight = FILE), // + @Menu(label = "Open Directory of tif...", weight = FILE_OPEN) }) +public class OpenDirofTif implements Command { + + @Parameter + private IOService io; + + @Parameter + private LogService log; + + @Parameter + private SciView sciView; + + // TODO: Find a more extensible way than hard-coding the extensions. + @Parameter(style = "directory") + private File file; + + @Parameter + private int onlyFirst = 0; + + @Override + public void run() { + try { + if(onlyFirst > 0) { + sciView.openDirTiff(file.toPath(), onlyFirst); + } else { + sciView.openDirTiff(file.toPath(), null); + } + } + catch (final IOException | IllegalArgumentException exc) { + log.error( exc ); + } + } +} diff --git a/src/main/java/sc/iview/commands/file/OpenTrackFile.java b/src/main/java/sc/iview/commands/file/OpenTrackFile.java new file mode 100644 index 000000000..b2c3e7a7f --- /dev/null +++ b/src/main/java/sc/iview/commands/file/OpenTrackFile.java @@ -0,0 +1,79 @@ +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview.commands.file; + +import org.scijava.command.Command; +import org.scijava.io.IOService; +import org.scijava.log.LogService; +import org.scijava.plugin.Menu; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import sc.iview.SciView; + +import java.io.File; +import java.io.IOException; + +import static sc.iview.commands.MenuWeights.FILE; +import static sc.iview.commands.MenuWeights.FILE_OPEN; + +/** + * Command to open a file in SciView + * + * @author Kyle Harrington + * + */ +@Plugin(type = Command.class, menuRoot = "SciView", // + menu = { @Menu(label = "File", weight = FILE), // + @Menu(label = "Open Track File", weight = FILE_OPEN) }) +public class OpenTrackFile implements Command { + + @Parameter + private IOService io; + + @Parameter + private LogService log; + + @Parameter + private SciView sciView; + + // TODO: Find a more extensible way than hard-coding the extensions. + @Parameter(style = "open,extensions:csv") + private File file; + + @Override + public void run() { + try { + sciView.openTrackFile(file); + + } + catch (final IOException | IllegalArgumentException exc) { + log.error( exc ); + } + } +} diff --git a/src/main/java/sc/iview/event/NodeTaggedEvent.java b/src/main/java/sc/iview/event/NodeTaggedEvent.java new file mode 100644 index 000000000..c0dc6feb4 --- /dev/null +++ b/src/main/java/sc/iview/event/NodeTaggedEvent.java @@ -0,0 +1,10 @@ +package sc.iview.event; + +import graphics.scenery.Node; + + +public class NodeTaggedEvent extends NodeEvent { + public NodeTaggedEvent(final Node node ) { + super( node ); + } +} diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 5e7d92578..8230a1221 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -58,6 +58,7 @@ import graphics.scenery.utils.Statistics import graphics.scenery.utils.extensions.times import graphics.scenery.volumes.Colormap import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume import graphics.scenery.volumes.Volume.Companion.fromXML import graphics.scenery.volumes.Volume.Companion.setupId @@ -85,6 +86,7 @@ import net.imglib2.type.numeric.integer.UnsignedByteType import net.imglib2.view.Views import org.joml.Quaternionf import org.joml.Vector3f +import org.joml.Vector4f import org.scijava.Context import org.scijava.`object`.ObjectService import org.scijava.display.Display @@ -100,6 +102,7 @@ import org.scijava.thread.ThreadService import org.scijava.util.ColorRGB import org.scijava.util.Colors import org.scijava.util.VersionUtils +import sc.iview.commands.demo.animation.ParticleDemo import sc.iview.commands.edit.InspectorInteractiveCommand import sc.iview.event.NodeActivatedEvent import sc.iview.event.NodeAddedEvent @@ -118,6 +121,7 @@ import java.net.JarURLConnection import java.net.URL import java.nio.ByteBuffer import java.nio.FloatBuffer +import java.nio.file.Path import java.time.LocalDate import java.time.format.DateTimeFormatter import java.util.* @@ -134,6 +138,7 @@ import kotlin.concurrent.thread import javax.swing.JOptionPane import kotlin.math.cos import kotlin.math.sin +import kotlin.system.measureTimeMillis /** * Main SciView class. @@ -784,6 +789,132 @@ class SciView : SceneryBase, CalibratedRealInterval { } } + @Throws(IOException::class) + fun openDirTiff(source: Path, onlyFirst: Int? = null) + { + val v = Volume.fromPath(source, hub, onlyFirst) + v.name = "volume" + v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) + v.colormap = Colormap.get("jet") + v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) + v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + v.metadata["animating"] = true + v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + v.visible = true + + v.spatial().wantsComposeModel = true + v.spatial().updateWorld(true) +// System.out.println("v.model: " + v.model) + addChild(v) +// System.out.println("v.getDimensions: "+ v.getDimensions()) +// +// System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) +// System.out.println("v.world.matrix: " + v.spatial().world) + } + + data class PointInTrack( + val t: Int, + val loc: Vector3f, + val cellId: Long, + val parentId: Long, + val nodeScore: Float, + val edgeScore: Float + ) + + data class Track( + val track: List, + val trackId: Int + ) + + @Throws(IOException::class) + fun openTrackFile(file: File) + { + val lines = file.readLines() + var track = ArrayList() + val tracks = ArrayList() + val separator = "," + + var lastTrackId = -1 + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val t = tokens[0].toInt() + val z = tokens[1].toFloat() -2000f + val y = tokens[2].toFloat() -800f + val x = tokens[3].toFloat() -1300f + val cellId = tokens[4].toLong() + val parentId = tokens[5].toLong() + val trackId = tokens[6].toInt() + val nodeScore = tokens[7].toFloat() + val edgeScore = tokens[8].toFloat()/45.0f + + val currentPointInTrack = PointInTrack( + t, + Vector3f(x,y,z), + cellId, + parentId, + nodeScore, + edgeScore + ) + if(lastTrackId != trackId) + { + lastTrackId = trackId + val sortedTrack = track.sortedBy { it.t } + tracks.add(Track(sortedTrack, trackId)) + + track.clear() + } + track.add(currentPointInTrack) + } + val timeCost = measureTimeMillis { + addTracks(tracks) + } + println("time: $timeCost") + } + + fun addTracks(tracks: ArrayList) + { + val rng = Random(17) + for(track in tracks) + { + if(track.trackId > 10) + { + continue + } + System.out.println("add track: "+ track.trackId.toString() ) + val master = Cylinder(0.1f, 1.0f, 10) +// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Vector3f(0.05f, 0f, 0f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + mInstanced.name = "TrackID-${track.trackId}" + mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } + addNode(mInstanced) + + var cnt = 0 + val a = rng.nextFloat() + val b = rng.nextFloat() + track.track.windowed(2,1).forEach { pair -> + cnt = cnt + 1 + val element = mInstanced.addInstance() + element.name ="EdgeID-$cnt" + element.instancedProperties["Color"] = { Vector4f( a,b,pair[0].edgeScore, 1.0f) } + element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) + //mInstanced.instances.add(element) + + } + } + + } + + + + /** * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud * @param source string of a data source @@ -911,7 +1042,7 @@ class SciView : SceneryBase, CalibratedRealInterval { } /** - * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true. + * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true * @param n node to add to scene * @param activePublish flag to specify whether the node becomes active *and* is published in the inspector/services * @param block an optional code that will be executed as a part of adding the node @@ -1715,6 +1846,7 @@ class SciView : SceneryBase, CalibratedRealInterval { } if (vrActive && ti != null) { cam.tracker = ti + logger.info("tracker set") } else { cam.tracker = null } diff --git a/src/main/kotlin/sc/iview/SplashLabel.kt b/src/main/kotlin/sc/iview/SplashLabel.kt index a5eadcb67..c2526a7f6 100644 --- a/src/main/kotlin/sc/iview/SplashLabel.kt +++ b/src/main/kotlin/sc/iview/SplashLabel.kt @@ -87,7 +87,7 @@ class SplashLabel : JPanel(), ItemListener { splashImage = try { ImageIO.read(this.javaClass.getResourceAsStream("sciview-logo.png")) - } catch (e: IOException) { + } catch (e: IOException ) { logger.warn("Could not read splash image 'sciview-logo.png'") BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB) } diff --git a/src/main/kotlin/sc/iview/commands/MenuWeights.kt b/src/main/kotlin/sc/iview/commands/MenuWeights.kt index 610deeadb..9cefbddd8 100644 --- a/src/main/kotlin/sc/iview/commands/MenuWeights.kt +++ b/src/main/kotlin/sc/iview/commands/MenuWeights.kt @@ -121,6 +121,7 @@ object MenuWeights { const val DEMO_ADVANCED_BDVSLICING = 2.0 const val DEMO_ADVANCED_MESHTEXTURE = 3.0 const val DEMO_ADVANCED_INSTANCE_BENCHMARK = 4.0 + const val DEMO_ADVANCED_EYETRACKING =5.0 // Help const val HELP_HELP = 0.0 const val HELP_ABOUT = 200.0 diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt new file mode 100644 index 000000000..58aef285c --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt @@ -0,0 +1,34 @@ +package sc.iview.commands.demo.advanced + +import org.scijava.ui.behaviour.ClickBehaviour +import kotlin.concurrent.thread + + +/** + * [ClickBehaviour] that waits [timeout] for confirmation by re-executing the behaviour. + * Executes [armedAction] on first invocation, and [confirmAction] on second invocation, if + * it happens within [timeout]. + * + * @author Ulrik Guenther + */ +class ConfirmableClickBehaviour(val armedAction: (Long) -> Any, val confirmAction: (Long) -> Any, var timeout: Long = 3000): ClickBehaviour { + /** Whether the action is armed at the moment. Action becomes disarmed after [timeout]. */ + private var armed: Boolean = false + + /** + * Action fired at position [x]/[y]. Parameters not used in VR actions. + */ + override fun click(x : Int, y : Int) { + if(!armed) { + armed = true + armedAction.invoke(timeout) + + thread { + Thread.sleep(timeout) + armed = false + } + } else { + confirmAction.invoke(timeout) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt new file mode 100644 index 000000000..267c85074 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -0,0 +1,695 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Volume +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.volumes.RAIVolume + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class EyeTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var logger: LogService + + @Parameter + private lateinit var mastodonCallbackLinkCreate: (HedgehogAnalysis.SpineGraphVertex) -> Unit + + @Parameter + private lateinit var mastodonUpdateGraph: () -> Unit + + val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + logger.info("VR mode has been toggled") + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera?.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.camera?.addChild(calibrationTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + laser.name = "Laser" + sciview.addNode(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addNode(shell) + + val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } + + val v = (volnodes.firstOrNull() as? Volume) + if(v == null) { + logger.warn("No volume found, bailing") + return + } else { + logger.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") + volume = v + } + volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addNode(hedgehogs) + + val eyeFrames = Mesh("eyeFrames") + val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) + val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) + left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) + left.spatial().rotation = left.spatial().rotation.rotationZ(PI.toFloat()) + right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) + eyeFrames.addChild(left) + eyeFrames.addChild(right) + + sciview.addNode(eyeFrames) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + pupilTracker.subscribeFrames { eye, texture -> + if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { + return@subscribeFrames + } + + val node = if(eye == 1) { + left + } else { + right + } + + val stream = ByteArrayInputStream(texture) + val image = ImageIO.read(stream) + val data = (image.raster.dataBuffer as DataBufferByte).data + + node.ifMaterial { + textures["diffuse"] = Texture( + Vector3i(image.width, image.height, 1), + 3, + UnsignedByteType(), + BufferUtils.allocateByteAndPut(data) + ) } + + lastFrame = System.nanoTime() + } + + // TODO: Replace with cam.showMessage() + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addNode(it) } + + thread { + logger.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + logger.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + logger.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + logger.info("started thread for inputSetup") + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode + hedgehog.instances.forEach { + if (it.metadata.isNotEmpty()) { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } else { + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + logger.info("added hedgehog") + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false + hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) + val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.entries.indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.entries.get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden",distance = 2f, size = 0.2f, centered = true) + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint",distance = 2f, size = 0.2f, centered = true) + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible",distance = 2f, size = 0.2f, centered = true) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f, centered = true) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.spatial().scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 2f, size = 0.2f, centered = true) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.spatial().scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 2f, size = 0.2f, centered = true) + } else { + cam.showMessage("Paused",distance = 2f, size = 0.2f, centered = true) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt(), + centered = true) + + }, + confirmAction = { + hedgehogs.children.removeLast() + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000, + centered = true + ) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 2f, centered = true) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 2f, duration = 1000) + dumpHedgehog() + addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + logger.info("calibration should start now") + setupCalibration() + + } + + private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { + val startCalibration = ClickBehaviour { _, _ -> + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated) { + logger.info("pupil is currently uncalibrated") + pupilTracker.onCalibrationInProgress = { + cam.showMessage("Crunching equations ...",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000, centered = true) + } + + pupilTracker.onCalibrationFailed = { + cam.showMessage("Calibration failed.",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), centered = true) + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage("Calibration succeeded!", distance = 2f, size = 0.2f, messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), centered = true) +// cam.children.find { it.name == "debugBoard" }?.visible = true + + for (i in 0 until 20) { + referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} + Thread.sleep(100) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} + Thread.sleep(30) + } + + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + logger.info("deactivating tracking...") + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) + dumpHedgehog() + } else { + logger.info("activating tracking...") + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) + } + tracking = !tracking + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + logger.info("Starting eye tracker calibration") + cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) + pupilTracker.calibrate(cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + //NEW + PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.spatial().position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" + + val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) + } + } + } + +// else -> {gaze-> } + } + + logger.info("Calibration routine done.") + } + + // bind calibration start to menu key on controller + + } + } + hmd.addBehaviour("start_calibration", startCalibration) + hmd.addKeyBinding("start_calibration", keybindingCalibration) + } + + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) + spine.visible = true + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + // TODO: Allow for sampling a given time point of a volume +// val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) + // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now + val localDirection = Vector3f(0f) + val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) + val volumeScale = (volume as RAIVolume).getVoxelScale() +// if (samples != null && localDirection != null) { +// val metadata = SpineMetadata( +// timepoint, +// center, +// direction, +// intersection.distance, +// localEntry, +// localExit, +// localDirection, +// cam.headPosition, +// cam.headOrientation, +// cam.spatial().position, +// confidence, +// samples.map { it ?: 0.0f } +// ) + if (samples != null && samplePos != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.spatial().position, + confidence, + samples.map { it ?: 0.0f }, + samplePos.map { it?.mul(volumeScale) ?: Vector3f(0f) } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + // TODO: Show confidence as color for the spine + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + logger.info("dumping hedgehog...") + val lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + +// val cylinder = Cylinder(0.1f, 1.0f, 6, smoothSides = true) +// cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { +// diffuse = Vector3f(1f) +// ambient = Vector3f(1f) +// roughness = 1f +// } + +// cylinder.name = "Track-$hedgehogId" +// val mainTrack = InstancedNode(cylinder) +// mainTrack.instancedProperties["Color"] = { Vector4f(1f) } + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + mastodonCallbackLinkCreate(pair[0].second) +// val element = mainTrack.addInstance() +// element.addAttribute(Material::class.java, cylinder.material()) +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) +// element.parent = volume +// mainTrack.instances.add(element) + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + mastodonUpdateGraph() + +// mainTrack.let { sciview.addNode(it, parent = volume) } + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt new file mode 100644 index 000000000..bf183131c --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -0,0 +1,426 @@ +package sc.iview.commands.demo.advanced + +import org.joml.Vector3f +import org.joml.Matrix4f +import org.joml.Quaternionf +import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.lazyLogger +import org.slf4j.LoggerFactory +import java.io.File +import kotlin.math.sqrt + +/** + * + * + * @author Ulrik Günther + */ +class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { + + private val logger by lazyLogger() + + val timepoints = LinkedHashMap>() + + var avgConfidence = 0.0f + private set + var totalSampleCount = 0 + private set + + data class Track( + val points: List>, + val confidence: Float + ) + + init { + logger.info("Starting analysis with ${spines.size} spines") + + spines.forEach { spine -> + val timepoint = spine.timepoint + val current = timepoints[timepoint] + + if(current == null) { + timepoints[timepoint] = arrayListOf(spine) + } else { + current.add(spine) + } + + avgConfidence += spine.confidence + totalSampleCount++ + } + + avgConfidence /= totalSampleCount + } + + /** + * From a [list] of Floats, return both the index of local maxima, and their value, + * packaged nicely as a Pair + */ + private fun localMaxima(list: List): List> { + return list.windowed(3, 1).mapIndexed { index, l -> + val left = l[0] + val center = l[1] + val right = l[2] + + // we have a match at center + if (left < center && center > right) { + index * 1 + 1 to center + } else { + null + } + }.filterNotNull() + } + + data class SpineGraphVertex(val timepoint: Int, + val position: Vector3f, + val worldPosition: Vector3f, + val index: Int, + val value: Float, + val metadata : SpineMetadata, + var previous: SpineGraphVertex? = null, + var next: SpineGraphVertex? = null) { + + fun distance(): Float { + val n = next + return if(n != null) { + val t = (n.worldPosition - this.worldPosition) + sqrt(t.x*t.x + t.y*t.y + t.z*t.z) + } else { + 0.0f + } + } + + fun drop() { + previous?.next = next + next?.previous = previous + } + + override fun toString() : String { + return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" + } + } + + fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) + + fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { + val cross = forward.cross(this) + val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) + + val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) + + return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) + } + + data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) + + fun run(): Track? { + + val startingThreshold = 0.002f + val localMaxThreshold = 0.001f + val zscoreThreshold = 2.0f + val removeTooFarThreshold = 5.0f + + if(timepoints.isEmpty()) { + return null + } + + + //step1: find the startingPoint by using startingthreshold + val startingPoint = timepoints.entries.firstOrNull { entry -> + entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } + } ?: return null + + logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") + + // filter timepoints, remove all before the starting point + timepoints.filter { it.key > startingPoint.key } + .forEach { timepoints.remove(it.key) } + + logger.info("${timepoints.size} timepoints left") + + fun gaussSmoothing(samples: List, iterations: Int): List { + var smoothed = samples.toList() + val kernel = listOf(0.25f, 0.5f, 0.25f) + for (i in 0 until iterations) { + val newSmoothed = ArrayList(smoothed.size) + // Handle the first element + newSmoothed.add(smoothed[0] * 0.75f + smoothed[1] * 0.25f) + // Apply smoothing to the middle elements + for (j in 1 until smoothed.size - 1) { + val value = kernel[0] * smoothed[j-1] + kernel[1] * smoothed[j] + kernel[2] * smoothed[j+1] + newSmoothed.add(value) + } + // Handle the last element + newSmoothed.add(smoothed[smoothed.size - 2] * 0.25f + smoothed[smoothed.size - 1] * 0.75f) + + smoothed = newSmoothed + } + return smoothed + } + + //step2: find the maxIndices along the spine + // this will be a list of lists, where each entry in the first-level list + // corresponds to a time point, which then contains a list of vertices within that timepoint. + val candidates: List> = timepoints.map { tp -> + val vs = tp.value.mapIndexedNotNull { i, spine -> + // determine local maxima (and their indices) along the spine, aka, actual things the user might have + // seen when looking into the direction of the spine + val maxIndices = localMaxima(spine.samples.filterNotNull()) + logger.debug("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + + // if there actually are local maxima, generate a graph vertex for them with all the necessary metadata + if(maxIndices.isNotEmpty()) { + //maxIndices. +// filter the maxIndices which are too far away, which can be removed + //filter { it.first <1200}. + maxIndices.map { index -> + logger.debug("Generating vertex at index $index") + // get the position of the current index along the spine + val position = spine.samplePosList[index.first] + val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() + SpineGraphVertex(tp.key, + position, + worldPosition, + index.first, + index.second, + spine) + + } + } else { + null + } + } + vs + }.flatten() + + logger.info("SpineGraphVertices extracted") + + // step3: connect localMaximal points between 2 candidate spines according to the shortest path principle + // get the initial vertex, this one is assumed to always be in front, and have a local maximum - aka, what + // the user looks at first is assumed to be the actual cell they want to track + val initial = candidates.first().first { it.value > startingThreshold } + var current = initial + var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> + // calculate world-space distances between current point, and all candidate + // vertices, sorting them by distance + val vertices = vs + .filter { it.value > localMaxThreshold } + .map { vertex -> + val t = current.worldPosition - vertex.worldPosition + val distance = t.length() + VertexWithDistance(vertex, distance) + } + .sortedBy { it.distance } + + val closest = vertices.firstOrNull() + if(closest != null && closest.distance > 0) { + // create a linked list between current and closest vertices + current.next = closest.vertex + closest.vertex.previous = current + current = closest.vertex + current + } else { + null + } + }.toMutableList() + + // calculate average path lengths over all + val beforeCount = shortestPath.size + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) + + //step4: if some path is longer than multiple average length, it should be removed + while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { + shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() + shortestPath.windowed(3, 1, partialWindows = true).forEach { + // this reconnects the neighbors after the offending vertex has been removed + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + + } + + // recalculate statistics after offending vertex removal + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + + //step5: remove some vertices according to zscoreThreshold + var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + while(remaining > 0) { + val outliers = shortestPath + .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + .map { + val idx = shortestPath.indexOf(it) + listOf(idx-1,idx,idx+1) + }.flatten() + + shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() + remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + } + + val afterCount = shortestPath.size + logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") + val singlePoints = shortestPath + .groupBy { it.timepoint } + .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } + .filter { + it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f + } + + + logger.info("Returning ${singlePoints.size} points") + + + return Track(singlePoints.map { it.position to it}, avgConfidence) + } + + companion object { + private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) + + fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { + logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val confidence = tokens[1].toFloat() + val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + Vector3f(0.0f), + Vector3f(0.0f), + 0.0f, + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Quaternionf(), + Vector3f(0.0f), + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + + private fun String.toVector3f(): Vector3f { + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + + if (array[0] == "+Inf" || array[0] == "-Inf") + return Vector3f(0.0f,0.0f,0.0f) + + return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) + } + + private fun String.toQuaternionf(): Quaternionf { + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) + } + fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f,separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV with Matrix at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + logger.info("lines number: " + lines.size) + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, matrix4f) + } + + fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + } +} + +fun main(args: Array) { + val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") + + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") + val analysis = HedgehogAnalysis.fromCSV(file) + val results = analysis.run() + logger.info("Results: \n$results") +} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt new file mode 100644 index 000000000..cbbaf4a45 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt @@ -0,0 +1,24 @@ +package sc.iview.commands.demo.advanced + +import org.joml.Quaternionf +import org.joml.Vector3f + +/** + * Data class to store metadata for spines of the hedgehog. + */ +data class SpineMetadata( + val timepoint: Int, + val origin: Vector3f, + val direction: Vector3f, + val distance: Float, + val localEntry: Vector3f, + val localExit: Vector3f, + val localDirection: Vector3f, + val headPosition: Vector3f, + val headOrientation: Quaternionf, +// val headOrientation: Quaternion, + val position: Vector3f, + val confidence: Float, + val samples: List, + val samplePosList: List = ArrayList() +) \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt new file mode 100644 index 000000000..c57150ba1 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -0,0 +1,502 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.numerics.Random +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Volume +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.text.DecimalFormat +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class Test: Command{ + @Parameter + private lateinit var sciview: SciView + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + //val calibrationTarget = Icosphere(0.02f, 2) + val TestTarget = Icosphere(0.1f, 2) + + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var point1:Icosphere + lateinit var point2:Icosphere + + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Backward + @Parameter(label = "Volumes per second") + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.addChild(TestTarget) + TestTarget.visible = false + + +// sciview.toggleVRRendering() +// hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + shell.name = "shell" + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + + point1 = Icosphere(0.1f, 2) + point1.spatial().position = Vector3f(1.858f,2f,8.432f) + point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + sciview.addChild(point1) + + point2 = Icosphere(0.1f, 2) + point2.spatial().position = Vector3f(1.858f, 2f, -10.39f) + point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} + sciview.addChild(point2) + + + val connector = Cylinder.betweenPoints(point1.position, point2.position) + connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(connector) + + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + //println("timepoint: "+ newTimepoint); + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + //dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + tracking = true + //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position + + if(true) + { +// val p = Vector3f(0f,0f,-1f) +// referenceTarget.position = p +// referenceTarget.visible = true + val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() +// + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + //System.out.println("tracking!!!!!!!!!!") +// println("direction:"+ direction.toString()) + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + showTrack() + } + Thread.sleep(200) + } + //referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + + + } // bind calibration start to menu key on controller + + } + + private fun showTrack() + { +// val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-05-25 16.04.52\\Hedgehog_1_2022-05-25 16.06.03.csv") + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-10-19 13.48.51\\Hedgehog_1_2022-10-19 13.49.41.csv") + + var volumeDimensions = volume.getDimensions() + var selfdefineworlfmatrix = volume.spatial().world + // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) +// selfdefineworlfmatrix = Matrix4f( +// 0.015f, 0f, 0f, 0f, +// 0f, -0.015f, 0f, 0f, +// 0f, 0f, 0.045f, 0f, +// -5f, 8f, -2f, 1f +// ) + val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix) + print("volume.getDimensions(): "+ volume.getDimensions()) + print("volume.spatial().world: "+ volume.spatial().world) + print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) + + val track = analysis.run() + + print("flag1") + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + print("flag2") + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + sciview.addNode(mInstanced) + + + print("flag3") + if(track == null) + { + return + } + print("flag4") + track.points.windowed(2, 1).forEach { pair -> + + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first)//direct product + val p1 = Vector3f(pair[1].first) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + + val tp = pair[0].second.timepoint + val pp = Icosphere(0.1f, 1) + pp.name = "trackpoint_${tp}_${pair[0].first.x}_${pair[0].first.y}_${pair[0].first.z}" +// println("volumeDimensions: " + volumeDimensions) +// println("volume.spatial().world: " + volume.spatial().world) + println("the local position of the point is:" + pair[0].first) + println("the world position of the point is: "+ p0w) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addNode(pp) + } + } + + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + + + val p2 = Vector3f(center).add(temp) + + +// print("center position: " + p1.toString()) +// print("p2 position" + p2.toString()) + +// TestTarget.visible = true +// TestTarget.ifSpatial { position = p2} + + +// val spine = (hedgehogs.children.last() as InstancedNode).addInstance() +// spine.spatial().orientBetweenPoints(p1, p2, true, true) +// spine.visible = true + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val nf = DecimalFormat("0.0000") + println("Ray intersects volume at world=${intersection.entry.toString(nf)}/${intersection.exit.toString(nf)} local=${localEntry.toString(nf)}/${localExit.toString(nf)} ") + +// System.out.println("localEntry:"+ localEntry.toString()) +// System.out.println("localExit:" + localExit.toString()) +// val worldpositionEntry = volume.spatial().world.transform((Vector3f(localEntry)).xyzw()).xyz() +// val worldpositionExit = volume.spatial().world.transform((Vector3f(localExit)).xyzw()).xyz() +// System.out.println("worldEntry:"+ worldpositionEntry.toString()) +// System.out.println("worldExit:" + worldpositionExit.toString()) + + + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.002f } + + println("count of samples: "+ count.toString()) +println(samples) + +// spine.metadata["spine"] = metadata +// spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ +// fun dumpHedgehog() { +// var lastHedgehog = hedgehogs.children.last() as InstancedNode +// val hedgehogId = hedgehogIds.incrementAndGet() +// +// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() +// val hedgehogFileWriter = hedgehogFile.bufferedWriter() +// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") +// +// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() +// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) +// if(!trackFile.exists()) { +// trackFile.createNewFile() +// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") +// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") +// } +// +// +// val spines = lastHedgehog.instances.mapNotNull { spine -> +// spine.metadata["spine"] as? SpineMetadata +// } +// +// spines.forEach { metadata -> +// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") +// } +// hedgehogFileWriter.close() +// +// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track +// val track = if(existingAnalysis is HedgehogAnalysis.Track) { +// existingAnalysis +// } else { +// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) +// h.run() +// } +// +// if(track == null) { +//// logger.warn("No track returned") +// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) +// return +// } +// +// lastHedgehog.metadata["HedgehogAnalysis"] = track +// lastHedgehog.metadata["Spines"] = spines +// +//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") +// +// val master = if(lastHedgehog == null) { +// val m = Cylinder(3f, 1.0f, 10) +// m.ifMaterial { +// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") +// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) +// roughness = 1.0f +// metallic = 0.0f +// cullingMode = Material.CullingMode.None +// } +// m.name = "Track-$hedgehogId" +// val mInstanced = InstancedNode(m) +// mInstanced +// } else { +// null +// } +// +// val parentId = 0 +// val volumeDimensions = volume.getDimensions() +// +// trackFileWriter.newLine() +// trackFileWriter.newLine() +// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") +// track.points.windowed(2, 1).forEach { pair -> +// if(master != null) { +// val element = master.addInstance() +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) +// element.parent = volume +// master.instances.add(element) +// } +// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product +// val tp = pair[0].second.timepoint +// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") +// } +// +// master?.let { volume.addChild(it) } +// +// trackFileWriter.close() +// } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt new file mode 100644 index 000000000..ce7b42b99 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt @@ -0,0 +1,17 @@ +package sc.iview.commands.demo.advanced + +import org.scijava.ui.behaviour.DragBehaviour + +class TrackingDragBehaviour():DragBehaviour{ + override fun init(x: Int, y: Int) { + TODO("Not yet implemented") + } + + override fun drag(x: Int, y: Int) { + TODO("Not yet implemented") + } + + override fun end(x: Int, y: Int) { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt new file mode 100644 index 000000000..c3099ff6f --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -0,0 +1,604 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.numerics.Random +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Volume +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.BufferedWriter +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRControllerTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.04f, 2) + val testTarget1 = Icosphere(0.01f, 2) + val testTarget2 = Icosphere(0.04f, 2) + val laser = Cylinder(0.0025f, 1f, 20) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var rightController: TrackedDevice + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + laser.material().diffuse = Vector3f(5.0f, 0.0f, 0.02f) + laser.material().metallic = 0.0f + laser.material().roughness = 1.0f + laser.visible = false + sciview.addNode(laser) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(referenceTarget) + + testTarget1.visible = false + testTarget1.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget1) + + + testTarget2.visible = false + testTarget2.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget2) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume +// volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") +// if(device.role == TrackerRole.RightHand) { +// rightController = device +// log.info("rightController is found, its location is in ${rightController.position}") +// } +// rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) + //dumpHedgehog() + //addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + dumpHedgehog() + println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + + + while(true) + { + /** + * the following code is added to detect right controller + */ + if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) + { + continue + } + else + { + rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! + + if (rightController.model?.spatialOrNull() == null) { + //println("spatial null") + } + else + { + val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() + val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + + println(headCenter.toString()) + println(pointWorld.toString()) + testTarget1.visible = true + testTarget1.ifSpatial { position = headCenter} + + testTarget2.visible = true + testTarget2.ifSpatial { position = pointWorld} + + laser.visible = true + laser.spatial().orientBetweenPoints(headCenter, pointWorld,true,true) + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = pointWorld} + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + + } + + } + + } + + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.02f } + //println("cnt: " + count.toString()) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + master.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(master) + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + sciview.addNode(mInstanced) + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + val pp = Icosphere(0.01f, 1) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addChild(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt new file mode 100644 index 000000000..eb4a63427 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -0,0 +1,622 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.numerics.Random +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.volumes.Volume +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.BufferedWriter +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.behaviours.* +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.utils.extensions.* +import org.scijava.event.EventService +import sc.iview.commands.file.OpenDirofTif +import sc.iview.event.NodeAddedEvent +import sc.iview.event.NodeChangedEvent +import sc.iview.event.NodeTaggedEvent + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRHeadSetTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + @Parameter + private lateinit var eventService: EventService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.02f, 2) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + private var selectionStorage: Node? = null + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = true + var direction = PlaybackDirection.Backward + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + +// val testtarget = Icosphere(2f, 2) +// volume.addChild(testtarget) +// testtarget.addAttribute(Grabable::class.java,Grabable()) +// testtarget.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = testtarget})) + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + thread { + dumpHedgehog() + } + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + //set up move action for moving forward, back, left and right + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Up), + "move_back" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Down), + "move_left" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Left), + "move_right" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Right) + ).forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + //hedgehog has three modes of visibility, 1. hide hedgehog, 2, show hedgehog for per timepoint, 3. show full hedgehog + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + //adjust the direction of playing volume + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + //Speeding up the playing of volume or enlarge the scale of volume depending on whether the volume is in playing + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.1f, 1.2f) + volume.spatial().scale *= Vector3f(volumeScaleFactor) + } + } + + //slower the playing of volume or reduce the scale of volume depending on whether the volume is in play + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.1f, 0.9f) + volume.spatial().scale *= Vector3f(volumeScaleFactor) + } + } + + //click the button to play or pause the volume + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + //delete the last hedgehog + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + }, + confirmAction = { + if(hedgehogsList.size != 0) + { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + } + }) + + val playbackDirection = ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + } + + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("playback_direction",playbackDirection) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + + + + hmd.addKeyBinding("skip_to_next", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Right)) // RightController. right + hmd.addKeyBinding("skip_to_prev", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Left)) // RightController. left + hmd.addKeyBinding("faster_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Up)) // RightController. up + hmd.addKeyBinding("slower_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Down)) //RightController. down + hmd.addKeyBinding("play_pause", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Menu)) // LeftController.Menu + hmd.addKeyBinding("playback_direction", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Menu)) //RightController.Menu + hmd.addKeyBinding("delete_hedgehog", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.A)) //RightController.Side + hmd.addKeyBinding("toggle_hedgehog", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.A)) //LeftController.Side + + + + //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) + //left trigger button can validate or delete a track, the function should be arranged to two different button in the future + VRSelect.createAndSet(sciview.currentScene, + hmd, + listOf(OpenVRHMD.OpenVRButton.Trigger), + listOf(TrackerRole.LeftHand), + { n -> + println("the spot ${n.name} is selected") + + /** + * delete the selected node from volume + **/ +// volume.runRecursive{it.removeChild(n)} +// eventService.publish(NodeRemovedEvent(n)) + + + /* + validate the selected node from volume, the tag event is designed specially for tag of Elephant + */ + eventService.publish(NodeTaggedEvent(n)) + + }, + true) + + + VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand,TrackerRole.RightHand),true) + + VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand,TrackerRole.LeftHand)) + setupControllerforTracking() + } + + + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + thread { + dumpHedgehog() + println("before dumphedgehog: " + hedgehogsList.last().instances.size.toString()) + } + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + //RightController.trigger + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + +// playing = false + + while(true) + { + + val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() + val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-1f) } + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + + Thread.sleep(2) + } + } + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + //println("try to find intersection"); + + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val dim = volume.getDimensions() + + val entryUV = Vector3f(intersection.relativeEntry).div(Vector3f(dim)) + val exitUV = Vector3f(intersection.relativeExit).div(Vector3f(dim)) + val (samples, localDirection) = volume.sampleRay(entryUV, exitUV) ?: (null to null) + + + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + entryUV, + exitUV, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + println("do hedgehog Analysis") + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + //check whether track is null, if it is null, then let the camera show "No track returned", otherwise do analysis + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val parent = RichNode() + parent.name = "Track-$hedgehogId" + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = Cylinder(3.0f, 1.0f, 5)//edgeMaster.addInstance() + val p0 = Vector3f(pair[0].first) * Vector3f(volumeDimensions) + val p1 = Vector3f(pair[1].first) * Vector3f(volumeDimensions) + + val tp = pair[0].second.timepoint + + element.spatial().orientBetweenPoints(p0, p1, rescale = true, reposition = true) + element.name = "edge" + element.metadata["Type"] = "edge" + parent.addChild(element) + + val pp = Icosphere(5.0f, 1)//nodeMaster.addInstance() + log.info("Local position: $p0 / $p1") + pp.name = "node-$tp" + pp.metadata["NodeTimepoint"] = tp + pp.metadata["NodePosition"] = p0 + pp.metadata["Type"] = "node" + pp.spatial().position = p0 + + //give attributes to these nodes to make them grable, touchable and selectable, for more detailed usage check VRControllerExample in scenery + pp.addAttribute(Grabable::class.java, Grabable()) + pp.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = pp})) + pp.addAttribute(Touchable::class.java, Touchable(onTouch = { device -> + if (device.role == TrackerRole.LeftHand) { + pp.ifSpatial { + position = (device.velocity ?: Vector3f(0.0f)) * 5f + position + eventService.publish(NodeChangedEvent(pp)) + } + } + + })) + parent.addChild(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + + volume.addChild(parent) + eventService.publish(NodeAddedEvent(parent)) + + trackFileWriter.close() + } + + companion object { + //run function from here, it will automatically choose the volume for rendering, please give the correct location of volume + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + + command.run(OpenDirofTif::class.java, true, + hashMapOf( + "file" to File("E:\\dataset\\Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone"), + "onlyFirst" to 10 + )) + .get() + + val argmap = HashMap() + command.run(VRHeadSetTrackingDemo::class.java, true, argmap) + .get() + } + } +} \ No newline at end of file diff --git a/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert b/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert index 4a3d2b12f..1f612d645 100644 --- a/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert +++ b/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert @@ -58,7 +58,7 @@ mat4 mv; mv = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] * iModelMatrix + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye] * iModelMatrix); projectionMatrix = (vrParameters.stereoEnabled ^ 1) * ProjectionMatrix + vrParameters.stereoEnabled * vrParameters.projectionMatrices[currentEye.eye]; - if(ubo.isBillboard > 0) { + if(ubo.isBillboard == 1) { mv[0][0] = 1.0f; mv[0][1] = .0f; mv[0][2] = .0f; diff --git a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt new file mode 100644 index 000000000..88c812e58 --- /dev/null +++ b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt @@ -0,0 +1,41 @@ +import graphics.scenery.utils.extensions.times +import graphics.scenery.utils.lazyLogger +import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction +import org.joml.Vector3f +import org.scijava.command.CommandService +import org.scijava.ui.UIService +import sc.iview.SciView +import sc.iview.commands.demo.advanced.EyeTrackingDemo + +//object StartEye { + +// val logger by lazyLogger() + +// @JvmStatic +fun main() { + val sv = SciView.create() + val context = sv.scijavaContext + val uiService = context?.service(UIService::class.java) + uiService?.showUI() + + sv.open("C:/Software/datasets/MastodonTutorialDataset1/datasethdf5.xml") + val volumes = sv.findNodes { it.javaClass == RAIVolume::class.java } + volumes.first().let { + it as RAIVolume + it.minDisplayRange = 400f + it.maxDisplayRange = 1500f + val tf = TransferFunction() + tf.addControlPoint(0f, 0f) + tf.addControlPoint(1f, 1f) + it.transferFunction = tf + it.spatial().scale *= 50f + it.spatial().scale.z *= -1f + } + + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + +} +//} \ No newline at end of file