Skip to content

Commit

Permalink
Open immersive experiences directly
Browse files Browse the repository at this point in the history
This PR implements support for opening immersive experiences directly
as soon as the application is launched.

We do this by adding a built-in extension which will activate a
particular element in the page.

We need to set the preference dom.vr.require-gesture to false so this
script can launch immersive WebXR experiences.

The information required to identify the element to activate is passed
as parameters to the Intent:

- open_in_immersive
- open_in_immersive_parent_xpath
- open_in_immersive_element_xpath
- a target URL to open
  • Loading branch information
felipeerias committed Jul 25, 2024
1 parent 58c7851 commit 701c442
Show file tree
Hide file tree
Showing 5 changed files with 229 additions and 2 deletions.
27 changes: 26 additions & 1 deletion app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,12 @@ public class VRBrowserActivity extends PlatformActivity implements WidgetManager
public static final String EXTRA_KIOSK = "kiosk";
private static final long BATTERY_UPDATE_INTERVAL = 60 * 1_000_000_000L; // 60 seconds

boolean mOpenInImmersive = false;
public static final String EXTRA_OPEN_IN_IMMERSIVE = "open_in_immersive";
// Element where a click would be simulated to launch the WebXR experience.
public static final String EXTRA_OPEN_IN_IMMERSIVE_PARENT_XPATH = "open_in_immersive_parent_xpath";
public static final String EXTRA_OPEN_IN_IMMERSIVE_ELEMENT_XPATH = "open_in_immersive_element_xpath";

private BroadcastReceiver mCrashReceiver = new BroadcastReceiver() {
@Override
public void onReceive(Context context, Intent intent) {
Expand Down Expand Up @@ -247,6 +253,8 @@ public void run() {
private int mLastBatteryLevel = -1;
private PlatformActivityPlugin mPlatformPlugin;
private int mLastMotionEventWidgetHandle;
private String mImmersiveParentElementXPath;
private String mImmersiveTargetElementXPath;

private boolean callOnAudioManager(Consumer<AudioManager> fn) {
if (mAudioManager == null) {
Expand Down Expand Up @@ -880,6 +888,14 @@ void loadFromIntent(final Intent intent) {
}

openInKioskMode = extras.getBoolean(EXTRA_KIOSK, false);

if (extras.getBoolean(EXTRA_OPEN_IN_IMMERSIVE)) {
mImmersiveParentElementXPath = extras.getString(EXTRA_OPEN_IN_IMMERSIVE_PARENT_XPATH);
mImmersiveTargetElementXPath = extras.getString(EXTRA_OPEN_IN_IMMERSIVE_ELEMENT_XPATH);

// Open in immersive requires specific information to be present
mOpenInImmersive = targetUri != null && mImmersiveTargetElementXPath != null;
}
}

// If there is a target URI we open it
Expand All @@ -891,6 +907,8 @@ void loadFromIntent(final Intent intent) {
if (openInKioskMode) {
// FIXME this might not work as expected if the app was already running
mWindows.openInKioskMode(targetUri.toString());
} if (mOpenInImmersive) {
mWindows.openInImmersiveMode(targetUri, mImmersiveParentElementXPath, mImmersiveTargetElementXPath);
} else {
if (openInWindow) {
location = Windows.OPEN_IN_NEW_WINDOW;
Expand Down Expand Up @@ -1332,6 +1350,13 @@ void onExitWebXR(long aCallback) {
return;
}
mIsPresentingImmersive = false;
TelemetryService.stopImmersive();

if (mOpenInImmersive) {
Log.d(LOGTAG, "Started in immersive mode: exiting WebXR will finish the app");
finish();
}

runOnUiThread(() -> {
mWindows.exitImmersiveMode();
for (WebXRListener listener: mWebXRListeners) {
Expand All @@ -1342,7 +1367,6 @@ void onExitWebXR(long aCallback) {
// Show the window in front of you when you exit immersive mode.
recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL);

TelemetryService.stopImmersive();
Handler handler = new Handler(Looper.getMainLooper());
handler.postDelayed(() -> {
if (!mWindows.isPaused()) {
Expand All @@ -1355,6 +1379,7 @@ void onExitWebXR(long aCallback) {
}
}, 20);
}

@Keep
@SuppressWarnings("unused")
void onDismissWebXRInterstitial() {
Expand Down
35 changes: 35 additions & 0 deletions app/src/common/shared/com/igalia/wolvic/ui/widgets/Windows.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import android.content.Context;
import android.content.SharedPreferences;
import android.net.Uri;
import android.util.Log;

import androidx.annotation.IntDef;
Expand Down Expand Up @@ -68,6 +69,8 @@ public class Windows implements TrayListener, TopBarWidget.Delegate, TitleBarWid

public static final int WHITE = 0xFFFFFFFF;
public static final int GRAY = 0x555555FF;
public static final String PARENT_ELEMENT_XPATH_PARAMETER = "wolvic-autowebxr-parentElementXPath";
public static final String TARGET_ELEMENT_XPATH_PARAMETER = "wolvic-autowebxr-targetElementXPath";

@IntDef(value = { OPEN_IN_FOREGROUND, OPEN_IN_BACKGROUND, OPEN_IN_NEW_WINDOW})
public @interface NewTabLocation {}
Expand Down Expand Up @@ -1470,6 +1473,38 @@ public void openInKioskMode(@NonNull String aUri) {
mFocusedWindow.setKioskMode(true);
}

public void openInImmersiveMode(Uri targetUri, String immersiveParentElementXPath, String immersiveTargetElementXPath) {
String extensionId = "wolvic-autowebxr@igalia.com";
String extensionUrl = "resource://android/assets/extensions/wolvic_autowebxr/";

Uri.Builder uriBuilder = targetUri.buildUpon();
if (!StringUtils.isEmpty(immersiveParentElementXPath)) {
uriBuilder.appendQueryParameter(PARENT_ELEMENT_XPATH_PARAMETER, immersiveParentElementXPath);
}
if (!StringUtils.isEmpty(immersiveTargetElementXPath)) {
uriBuilder.appendQueryParameter(TARGET_ELEMENT_XPATH_PARAMETER, immersiveTargetElementXPath);
}
Uri extendedUri = uriBuilder.build();

Session session = SessionStore.get().createSuspendedSession(extendedUri.toString(), true);

mFocusedWindow.setKioskMode(true);

SessionStore.get().getWebExtensionRuntime().installWebExtension(
extensionId,
extensionUrl,
webExtension -> {
setFirstPaint(mFocusedWindow, session);
mFocusedWindow.setSession(session, WindowWidget.DEACTIVATE_CURRENT_SESSION);
return null;
},
(s, throwable) -> {
Log.e(LOGTAG, "Error installing the " + extensionId + " Web Extension: " + throwable.getLocalizedMessage());
return null;
}
);
}

public void addTab(@NonNull WindowWidget targetWindow, @Nullable String aUri) {
Session session = SessionStore.get().createSuspendedSession(aUri, targetWindow.getSession().isPrivateMode());
session.setParentSession(targetWindow.getSession());
Expand Down
142 changes: 142 additions & 0 deletions app/src/main/assets/extensions/wolvic_autowebxr/content.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
const LOGTAG = '[wolvic:autowebxr]';
const ENABLE_LOGS = true;
const logDebug = (...args) => ENABLE_LOGS && console.log(LOGTAG, ...args);

const PARENT_ELEMENT_XPATH_PARAMETER = 'wolvic-autowebxr-parentElementXPath';
const TARGET_ELEMENT_XPATH_PARAMETER = 'wolvic-autowebxr-targetElementXPath';

const IFRAME_READY_MSG = 'wolvic-autowebxr-iframeReady';
const TARGET_ELEMENT_MSG = 'wolvic-autowebxr-targetElement';

var parentElementXPath;
var targetElementXPath;

function getElementByXPath(document, xpath) {
let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
return result.singleNodeValue;
}

function clickImmersiveElement() {
// Check if the current URL has extra query parameters
parentElementXPath = undefined;
targetElementXPath = undefined;

let url = document.URL;
let params = new URLSearchParams(new URL(url).search);
for (let [key, value] of params) {
if (key === PARENT_ELEMENT_XPATH_PARAMETER)
parentElementXPath = value;
else if (key === TARGET_ELEMENT_XPATH_PARAMETER)
targetElementXPath = value;
}

// We need at least the target element to click
if (!targetElementXPath)
return;

logDebug('Preparing to open immersive WebXR; parentElementXPath: ' + parentElementXPath + ' ; targetElementXPath: ' + targetElementXPath);

// The parent element is typically an iframe and, if it comes from a different origin,
// we might not be able to access its contents directly.
// If parentElementXPath is null, we will use the root to find the target element.

var parent, parentDocument;
if (parentElementXPath) {
parent = getElementByXPath(document, parentElementXPath);
if (!parent) {
logDebug('Parent element not found, retrying');
setTimeout(clickImmersiveElement, 1000);
return;
}

try {
parentDocument = parent.contentDocument || parent.contentWindow.document;
} catch (e) {
logDebug('Parent iframe is from a different origin');
const iframeWindow = parent.contentWindow;

const targetElementMsg = {
action: TARGET_ELEMENT_MSG,
targetElementXPath: targetElementXPath
};

iframeWindow.postMessage(targetElementMsg, '*');

// The iframe might not be ready yet, so we set up a listener for the "iframe ready" message.
const handleIframeReady = (event) => {
if (event.source === iframeWindow && event.data === IFRAME_READY_MSG) {
window.removeEventListener('message', handleIframeReady);
iframeWindow.postMessage(targetElementMsg, '*');
}
};
window.addEventListener('message', handleIframeReady);

return;
}
} else {
parent = window;
parentDocument = document;
}

if (parentDocument.readyState !== 'complete') {
logDebug('Parent is still loading');
parent.addEventListener('load', function() {
logDebug('Parent has finished loading');
clickImmersiveElement();
});
return;
} else {
logDebug('Parent is loaded');
}

let targetElement = getElementByXPath(parentDocument, targetElementXPath);

if (targetElement) {
logDebug('Target element found, calling click()');
targetElement.click();
targetElement.click();
} else {
logDebug('Target element not found, we will try again');
setTimeout(clickImmersiveElement, 1000);
}
}

function launchImmersiveFromIframe() {
window.addEventListener('message', function(event) {
if (event.data.action === TARGET_ELEMENT_MSG) {
let targetElement = getElementByXPath(document, event.data.targetElementXPath);

if (targetElement) {
logDebug('Target element found in iframe, calling click()');
targetElement.click();
targetElement.click();
} else {
logDebug('Target element not found in iframe, retrying');
setTimeout(function() {
window.postMessage(event.data, '*');
}, 1000);
}
}
});

window.parent.postMessage(IFRAME_READY_MSG, '*');
}

// Main script execution
if (window.top === window.self) {
if (document.readyState === 'complete') {
logDebug('Root document is completely ready');
clickImmersiveElement();
} else {
logDebug('Root document is not ready yet');
window.addEventListener('load', clickImmersiveElement);
}
} else {
if (document.readyState === 'complete') {
logDebug('Iframe is completely ready');
launchImmersiveFromIframe();
} else {
logDebug('Iframe is not ready yet');
window.addEventListener('load', launchImmersiveFromIframe);
}
}
23 changes: 23 additions & 0 deletions app/src/main/assets/extensions/wolvic_autowebxr/manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"manifest_version": 2,
"name": "Wolvic WebXR Automator",
"version": "1.0",
"description": "Enter WebXR Experiences Automatically",
"browser_specific_settings": {
"gecko": {
"id": "wolvic-autowebxr@igalia.com"
}
},
"content_scripts": [
{
"matches": [
"*://*/*"
],
"js": [
"content.js"
],
"run_at": "document_idle",
"all_frames": true
}
]
}
4 changes: 3 additions & 1 deletion app/src/main/res/raw/fxr_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,6 @@ prefs:
browser.gesture.pinch.in: ''
browser.gesture.pinch.out.shift: ''
browser.gesture.pinch.in.shift: ''
apz.one_touch_pinch.enabled: false
apz.one_touch_pinch.enabled: false,
# allows scripts to open WebXR immersive sessions
dom.vr.require-gesture: false

0 comments on commit 701c442

Please sign in to comment.