New Controller's setting to disable hand tracking

There maybe be cases where the user wants to prevent the hand tracking
to take the control when the controllers got untracked.

This change defines a new switch in the Controller's settings dialog to
to disable the hand trakckig capabilities. This setting has been defined
as non-persistent, so it will be reset just restarting Wolvic.

app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java

@@ -243,6 +243,9 @@ public void run() {
     private ScheduledFuture<?> mNativeWidgetUpdatesTask = null;
     private Media mPrevActiveMedia = null;
     private boolean mIsPassthroughEnabled = false;
+    private boolean mIsHandTrackingEnabled = true;
+    private boolean mIsHandTrackingSupported = false;
+    private boolean mAreControllersAvailable = false;
     private long mLastBatteryUpdate = System.nanoTime();
     private int mLastBatteryLevel = -1;
     private PlatformActivityPlugin mPlatformPlugin;
@@ -1599,6 +1602,19 @@ private void onAppFocusChanged(final boolean aIsFocused) {
     @SuppressWarnings("unused")
     private void setEyeTrackingSupported(final boolean isSupported) { mIsEyeTrackingSupported = isSupported; }
 
+    @Keep
+    @SuppressWarnings("unused")
+    private void setHandTrackingSupported(final boolean isSupported) {
+        mIsHandTrackingSupported = isSupported;
+    }
+
+    @Keep
+    @SuppressWarnings("unused")
+    private void onControllersAvailable() {
+        Log.e(LOGTAG, "Physical controllers are available");
+        mAreControllersAvailable = true;
+    }
+
     private SurfaceTexture createSurfaceTexture() {
         int[] ids = new int[1];
         GLES20.glGenTextures(1, ids, 0);
@@ -2018,6 +2034,10 @@ public boolean isPassthroughSupported() {
         return DeviceType.isOculusBuild() || DeviceType.isLynx() || DeviceType.isSnapdragonSpaces() ||
                (DeviceType.isPicoXR() && Build.ID.compareTo(kPicoVersionPassthroughUpdate) >= 0);
     }
+    @Override
+    public boolean areControllersAvailable() {
+        return mAreControllersAvailable;
+    }
 
     @Override
     public boolean isPageZoomEnabled() {
@@ -2152,6 +2172,22 @@ public void setPointerMode(@PointerMode int mode) {
         queueRunnable(() -> setPointerModeNative(mode));
     }
 
+    @Override
+    public void setHandTrackingEnabled(boolean value) {
+        mIsHandTrackingEnabled = value;
+        queueRunnable(() -> setHandTrackingEnabledNative(value));
+    }
+
+    @Override
+    public boolean isHandTrackingEnabled() {
+        return mIsHandTrackingEnabled;
+    }
+
+    @Override
+    public boolean isHandTrackingSupported() {
+        return mIsHandTrackingEnabled;
+    }
+
     @Override
     @NonNull
     public AppServicesProvider getServicesProvider() {
@@ -2230,4 +2266,6 @@ public void reject() {
     private native void setWebXRIntersitialStateNative(@WebXRInterstitialState int aState);
     private native void setIsServo(boolean aIsServo);
     private native void setPointerModeNative(@PointerMode int aMode);
+    private native void setHandTrackingEnabledNative(boolean value);
+
 }

app/src/common/shared/com/igalia/wolvic/ui/widgets/WidgetManagerDelegate.java

@@ -137,6 +137,11 @@ enum OriginatorType {WEBSITE, APPLICATION}
     AppServicesProvider getServicesProvider();
     KeyboardWidget getKeyboard();
     void setPointerMode(@PointerMode int mode);
+    void setHandTrackingEnabled(boolean value);
+    boolean isHandTrackingEnabled();
     void checkEyeTrackingPermissions(@NonNull EyeTrackingCallback callback);
     boolean isEyeTrackingSupported();
+    boolean isHandTrackingSupported();
+    boolean areControllersAvailable();
+
 }

app/src/common/shared/com/igalia/wolvic/ui/widgets/settings/ControllerOptionsView.java

@@ -64,6 +64,12 @@ protected void updateUI() {
         } else {
             mBinding.pointerModeRadio.setVisibility(GONE);
         }
+
+        if (mWidgetManager.isHandTrackingSupported() && mWidgetManager.areControllersAvailable()) {
+            setHandTrackingEnabled(mWidgetManager.isHandTrackingEnabled(), false);
+        } else {
+            mBinding.handtrackingSwitch.setVisibility(GONE);
+        }
     }
 
     private void resetOptions() {
@@ -75,6 +81,7 @@ private void resetOptions() {
         }
         setHapticFeedbackEnabled(SettingsStore.HAPTIC_FEEDBACK_ENABLED, true);
         setPointerMode(SettingsStore.POINTER_MODE_DEFAULT, true);
+        setHandTrackingEnabled(true, true);
     }
 
     private void setPointerColor(int checkedId, boolean doApply) {
@@ -119,6 +126,16 @@ private void setPointerMode(@WidgetManagerDelegate.PointerMode int value, boolea
         }
     }
 
+    private void setHandTrackingEnabled(boolean value, boolean doApply) {
+        mBinding.handtrackingSwitch.setOnCheckedChangeListener(null);
+        mBinding.handtrackingSwitch.setValue(value, false);
+        mBinding.handtrackingSwitch.setOnCheckedChangeListener(mHandtrackingListener);
+
+        if (doApply) {
+            mWidgetManager.setHandTrackingEnabled(value);
+        }
+    }
+
     private RadioGroupSetting.OnCheckedChangeListener mPointerColorListener = (radioGroup, checkedId, doApply) -> {
         setPointerColor(checkedId, doApply);
     };
@@ -146,6 +163,9 @@ private void setPointerMode(@WidgetManagerDelegate.PointerMode int value, boolea
         });
     };
 
+    private SwitchSetting.OnCheckedChangeListener mHandtrackingListener = (compoundButton, enabled, apply) ->
+            setHandTrackingEnabled(enabled, true);
+
     @Override
     protected SettingViewType getType() {
         return SettingViewType.CONTROLLER;

app/src/main/cpp/BrowserWorld.cpp

@@ -1703,6 +1703,11 @@ BrowserWorld::SetPointerMode(crow::DeviceDelegate::PointerMode pointerMode) {
   m.device->SetPointerMode(pointerMode);
 }
 
+void
+BrowserWorld::SetHandTrackingEnabled(bool value) {
+    m.device->SetHandTrackingEnabled(value);
+}
+
 JNIEnv*
 BrowserWorld::GetJNIEnv() const {
   ASSERT_ON_RENDER_THREAD(nullptr);
@@ -2219,4 +2224,9 @@ JNI_METHOD(void, setPointerModeNative)
     crow::BrowserWorld::Instance().SetPointerMode(pointerMode);
 }
 
+JNI_METHOD(void, setHandTrackingEnabledNative)
+(JNIEnv*, jobject, jboolean value) {
+    crow::BrowserWorld::Instance().SetHandTrackingEnabled(value);
+}
+
 } // extern "C"

app/src/main/cpp/BrowserWorld.h

@@ -78,6 +78,7 @@ class BrowserWorld : public DeviceDelegate::ReorientClient {
   void SetIsServo(const bool aIsServo);
   void SetCPULevel(const device::CPULevel aLevel);
   void SetPointerMode(crow::DeviceDelegate::PointerMode);
+  void SetHandTrackingEnabled(bool value);
   JNIEnv* GetJNIEnv() const;
   void OnReorient() override;
 #if HVR

app/src/main/cpp/DeviceDelegate.h

@@ -123,6 +123,7 @@ class DeviceDelegate {
     TRACKED_EYE
   };
   virtual void SetPointerMode(const PointerMode) {};
+  virtual void SetHandTrackingEnabled(bool value) {};
 
 protected:
   DeviceDelegate() {}

app/src/main/cpp/VRBrowser.cpp

@@ -74,6 +74,11 @@ const char* const kOnAppFocusChangedName = "onAppFocusChanged";
 const char* const kOnAppFocusChangedSignature = "(Z)V";
 const char* const kSetEyeTrackingSupported = "setEyeTrackingSupported";
 const char* const kSetEyeTrackingSupportedSignature = "(Z)V";
+const char* const kSetHandTrackingSupported = "setHandTrackingSupported";
+const char* const kSetHandTrackingSupportedSignature = "(Z)V";
+const char* const kOnControllersAvailable = "onControllersAvailable";
+const char* const kOnControllersAvailableSignature = "()V";
+
 
 JNIEnv* sEnv = nullptr;
 jclass sBrowserClass = nullptr;
@@ -110,6 +115,8 @@ jmethodID sAppendAppNotesToCrashReport = nullptr;
 jmethodID sUpdateControllerBatteryLevels = nullptr;
 jmethodID sOnAppFocusChanged = nullptr;
 jmethodID sSetEyeTrackingSupported = nullptr;
+jmethodID sSetHandTrackingSupported = nullptr;
+jmethodID sOnControllersAvailable = nullptr;
 
 } // namespace
 
@@ -162,6 +169,8 @@ VRBrowser::InitializeJava(JNIEnv* aEnv, jobject aActivity) {
   sUpdateControllerBatteryLevels = FindJNIMethodID(sEnv, sBrowserClass, kUpdateControllerBatteryLevelsName, kUpdateControllerBatteryLevelsSignature);
   sOnAppFocusChanged = FindJNIMethodID(sEnv, sBrowserClass, kOnAppFocusChangedName, kOnAppFocusChangedSignature);
   sSetEyeTrackingSupported = FindJNIMethodID(sEnv, sBrowserClass, kSetEyeTrackingSupported, kSetEyeTrackingSupportedSignature);
+  sSetHandTrackingSupported = FindJNIMethodID(sEnv, sBrowserClass, kSetHandTrackingSupported, kSetHandTrackingSupportedSignature);
+  sOnControllersAvailable = FindJNIMethodID(sEnv, sBrowserClass, kOnControllersAvailable, kOnControllersAvailableSignature);
 }
 
 JNIEnv * VRBrowser::Env()
@@ -482,4 +491,19 @@ VRBrowser::SetEyeTrackingSupported(bool aIsSupported) {
     CheckJNIException(sEnv, __FUNCTION__);
 }
 
+void
+VRBrowser::SetHandTrackingSupported(bool aIsSupported) {
+    if (!ValidateMethodID(sEnv, sActivity, sSetHandTrackingSupported, __FUNCTION__)) { return; }
+    sEnv->CallVoidMethod(sActivity, sSetHandTrackingSupported, (jboolean) aIsSupported);
+    CheckJNIException(sEnv, __FUNCTION__);
+}
+
+void
+VRBrowser::OnControllersAvailable() {
+    if (!ValidateMethodID(sEnv, sActivity, sOnControllersAvailable, __FUNCTION__)) { return; }
+    sEnv->CallVoidMethod(sActivity, sOnControllersAvailable);
+    CheckJNIException(sEnv, __FUNCTION__);
+
+}
+
 } // namespace crow

app/src/main/cpp/VRBrowser.h

@@ -51,6 +51,8 @@ void AppendAppNotesToCrashLog(const std::string& aNotes);
 void UpdateControllerBatteryLevels(const jint aLeftBatteryLevel, const jint aRightBatteryLevel);
 void OnAppFocusChanged(const bool aIsFocused);
 void SetEyeTrackingSupported(bool aIsSupported);
+void SetHandTrackingSupported(bool aIsSupported);
+void OnControllersAvailable();
 } // namespace VRBrowser;
 
 } // namespace crow

app/src/main/res/layout/options_controller.xml

@@ -54,6 +54,12 @@
                     android:layout_height="wrap_content"
                     app:description="@string/controller_options_haptic_feedback" />
 
+                <com.igalia.wolvic.ui.views.settings.SwitchSetting
+                    android:id="@+id/handtracking_switch"
+                    android:layout_width="match_parent"
+                    android:layout_height="wrap_content"
+                    app:description="@string/controller_options_handtracking" />
+
                 <com.igalia.wolvic.ui.views.settings.RadioGroupVSetting
                     android:id="@+id/pointer_color_radio"
                     android:layout_width="match_parent"

app/src/main/res/values/strings.xml

@@ -685,6 +685,10 @@
          haptic feedback of the remote controller. -->
     <string name="controller_options_haptic_feedback">Haptic Feedback</string>
 
+    <!-- This string is used to label the switch that allow the user to enable/disable
+         the hand-tracking capabilities. -->
+    <string name="controller_options_handtracking">Hand Tracking</string>
+
     <!-- This string describes what the 'Reset' button in the developer options does which is
          restore all the developer settings to their default value. -->
     <string name="developer_options_reset">Reset Developer Settings</string>

app/src/openxr/cpp/DeviceDelegateOpenXR.cpp

@@ -122,6 +122,7 @@ struct DeviceDelegateOpenXR::State {
   float furthestHitDistance { near };
   OpenXRActionSetPtr eyeActionSet;
   PointerMode pointerMode { PointerMode::TRACKED_POINTER };
+  bool handTrackingEnabled { true };
 
   bool IsPositionTrackingSupported() {
       CHECK(system != XR_NULL_SYSTEM_ID);
@@ -304,6 +305,7 @@ struct DeviceDelegateOpenXR::State {
         VRB_ERROR("OpenXR runtime reports 0 layers. There must be at least 1");
 
     mHandTrackingSupported = handTrackingProperties.supportsHandTracking;
+    VRBrowser::SetHandTrackingSupported(mHandTrackingSupported);
     VRB_LOG("OpenXR runtime %s hand tracking", mHandTrackingSupported ? "does support" : "doesn't support");
     VRB_LOG("OpenXR runtime %s FB passthrough extension", passthroughProperties.supportsPassthrough ? "does support" : "doesn't support");
 
@@ -801,8 +803,17 @@ struct DeviceDelegateOpenXR::State {
     // Invoke the callback. We only clear it if the load has started. Otherwise we will retry later.
     // This happens for example if a profile with no 3D models like hand interaction is loaded on
     // start. If we clear the callback it won't ever be called in case the user grabs a controller.
-    if (controllersReadyCallback())
+    if (controllersReadyCallback()) {
+        MaybeNotifyControllersAvailable();
         controllersReadyCallback = nullptr;
+    }
+  }
+
+  void MaybeNotifyControllersAvailable() {
+      if (input->HasPhysicalControllersAvailable()) {
+          VRB_LOG("DeviceDelegateOpenXR -- Physical controllers available");
+          VRBrowser::OnControllersAvailable();
+      }
   }
 
   void UpdateInteractionProfile() {
@@ -1173,7 +1184,7 @@ DeviceDelegateOpenXR::StartFrame(const FramePrediction aPrediction) {
     offsets.y() = -0.05;
     offsets.z() = 0.05;
 #endif
-    m.input->Update(frameState, m.localSpace, head, offsets, m.renderMode, m.pointerMode, *m.controller);
+    m.input->Update(frameState, m.localSpace, head, offsets, m.renderMode, m.pointerMode, m.handTrackingEnabled, *m.controller);
   }
 
   if (m.reorientRequested && m.renderMode == device::RenderMode::StandAlone) {
@@ -1694,4 +1705,8 @@ void DeviceDelegateOpenXR::SetPointerMode(const DeviceDelegate::PointerMode mode
   m.pointerMode = mode;
 }
 
+void DeviceDelegateOpenXR::SetHandTrackingEnabled(bool value) {
+    m.handTrackingEnabled = value;
+}
+
 } // namespace crow

app/src/openxr/cpp/DeviceDelegateOpenXR.h

@@ -67,6 +67,7 @@ class DeviceDelegateOpenXR : public DeviceDelegate {
   void DrawHandMesh(const uint32_t aControllerIndex, const vrb::Camera&) override;
   void SetHitDistance(const float) override;
   void SetPointerMode(const PointerMode mode) override;
+  void SetHandTrackingEnabled(bool value) override;
   // Custom methods for NativeActivity render loop based devices.
   void BeginXRSession();
   void EnterVR(const crow::BrowserEGLContext& aEGLContext);

app/src/openxr/cpp/OpenXRInput.cpp

@@ -75,7 +75,7 @@ XrResult OpenXRInput::Initialize(ControllerDelegate& delegate)
   return XR_SUCCESS;
 }
 
-XrResult OpenXRInput::Update(const XrFrameState& frameState, XrSpace baseSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, ControllerDelegate& delegate)
+XrResult OpenXRInput::Update(const XrFrameState& frameState, XrSpace baseSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool handTrackingEnabled, ControllerDelegate& delegate)
 {
   XrActiveActionSet activeActionSet {
     mActionSet->ActionSet(), XR_NULL_PATH
@@ -89,7 +89,7 @@ XrResult OpenXRInput::Update(const XrFrameState& frameState, XrSpace baseSpace,
   bool usingEyeTracking = pointerMode == DeviceDelegate::PointerMode::TRACKED_EYE && updateEyeGaze(frameState, head, delegate);
 
   for (auto& input : mInputSources) {
-    input->Update(frameState, baseSpace, head, offsets, renderMode, pointerMode, usingEyeTracking, delegate);
+    input->Update(frameState, baseSpace, head, offsets, renderMode, pointerMode, usingEyeTracking, handTrackingEnabled, delegate);
   }
 
   // Update tracked keyboard
@@ -146,6 +146,14 @@ OpenXRInputMapping* OpenXRInput::GetActiveInputMapping() const
   return nullptr;
 }
 
+bool OpenXRInput::HasPhysicalControllersAvailable() const {
+    if (!AreControllersReady())
+        return false;
+    if (mInputSources[0]->IsUsingHandInteracionProfile())
+        return mInputSources[0]->NumberOfMappings() > 1;
+    return true;
+}
+
 void OpenXRInput::SetHandMeshBufferSizes(const uint32_t indexCount, const uint32_t vertexCount) {
   for (auto& input : mInputSources) {
     input->SetHandMeshBufferSizes(indexCount, vertexCount);

app/src/openxr/cpp/OpenXRInput.h

@@ -59,11 +59,12 @@ class OpenXRInput {
 public:
   static OpenXRInputPtr Create(XrInstance, XrSession, XrSystemProperties, XrSpace localSpace, ControllerDelegate& delegate);
   XrResult Initialize(ControllerDelegate& delegate);
-  XrResult Update(const XrFrameState&, XrSpace baseSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode, ControllerDelegate &);
+  XrResult Update(const XrFrameState&, XrSpace baseSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode, bool handTrackingEnabled, ControllerDelegate &);
   int32_t GetControllerModelCount() const;
   std::string GetControllerModelName(const int32_t aModelIndex) const;
   void UpdateInteractionProfile(ControllerDelegate&);
   bool AreControllersReady() const;
+  bool HasPhysicalControllersAvailable() const;
   void SetHandMeshBufferSizes(const uint32_t indexCount, const uint32_t vertexCount);
   HandMeshBufferPtr GetNextHandMeshBuffer(const int32_t aControllerIndex);
   void SetKeyboardTrackingEnabled(bool enabled);

app/src/openxr/cpp/OpenXRInputSource.cpp

@@ -752,7 +752,7 @@ void OpenXRInputSource::EmulateControllerFromHand(device::RenderMode renderMode,
     delegate.SetCapabilityFlags(mIndex, flags);
 }
 
-void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpace, const vrb::Matrix &head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, ControllerDelegate& delegate)
+void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpace, const vrb::Matrix &head, const vrb::Vector& offsets, device::RenderMode renderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, ControllerDelegate& delegate)
 {
     if (mActiveMapping &&
         ((mHandeness == OpenXRHandFlags::Left && !mActiveMapping->leftControllerModel) ||
@@ -806,6 +806,10 @@ void OpenXRInputSource::Update(const XrFrameState& frameState, XrSpace localSpac
     auto gotHandTrackingInfo = false;
     auto handFacesHead = false;
     if (isControllerUnavailable || mUsingHandInteractionProfile) {
+        if (!handTrackingEnabled) {
+            delegate.SetEnabled(mIndex, false);
+            return;
+        }
         gotHandTrackingInfo = GetHandTrackingInfo(frameState.predictedDisplayTime, localSpace, head);
         if (gotHandTrackingInfo) {
             std::vector<vrb::Matrix> jointTransforms;

app/src/openxr/cpp/OpenXRInputSource.h

@@ -113,10 +113,12 @@ class OpenXRInputSource {
     ~OpenXRInputSource();
 
     XrResult SuggestBindings(SuggestedBindings&) const;
-    void Update(const XrFrameState&, XrSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, ControllerDelegate& delegate);
+    void Update(const XrFrameState&, XrSpace, const vrb::Matrix& head, const vrb::Vector& offsets, device::RenderMode, DeviceDelegate::PointerMode pointerMode, bool usingEyeTracking, bool handTrackingEnabled, ControllerDelegate& delegate);
     XrResult UpdateInteractionProfile(ControllerDelegate&);
     std::string ControllerModelName() const;
     OpenXRInputMapping* GetActiveMapping() const { return mActiveMapping; }
+    int NumberOfMappings() { return mMappings.size(); }
+    bool IsUsingHandInteracionProfile() { return mUsingHandInteractionProfile; }
     void SetHandMeshBufferSizes(const uint32_t indexCount, const uint32_t vertexCount);
     HandMeshBufferPtr GetNextHandMeshBuffer();
 };