Skip to content

Commit

Permalink
Open immersive experiences directly
Browse files Browse the repository at this point in the history
This PR implements support for opening immersive experiences directly
as soon as the application is launched.

We do this by adding a built-in extension which will activate a
particular element in the page.

We need to set the preference dom.vr.require-gesture to false so this
script can launch immersive WebXR experiences.

Media autoplay is also allowed in this mode.

The information required to identify the element to activate is passed
as parameters to the Intent:

- open_in_immersive
- open_in_immersive_parent_xpath
- open_in_immersive_element_xpath
- a target URL to open
  • Loading branch information
felipeerias committed Sep 3, 2024
1 parent cc5a654 commit a2b14df
Show file tree
Hide file tree
Showing 7 changed files with 248 additions and 4 deletions.
31 changes: 30 additions & 1 deletion app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,12 @@ public class VRBrowserActivity extends PlatformActivity implements WidgetManager
public static final String EXTRA_KIOSK = "kiosk";
private static final long BATTERY_UPDATE_INTERVAL = 60 * 1_000_000_000L; // 60 seconds

private boolean mOpenInImmersive = false;
public static final String EXTRA_OPEN_IN_IMMERSIVE = "open_in_immersive";
// Element where a click would be simulated to launch the WebXR experience.
public static final String EXTRA_OPEN_IN_IMMERSIVE_PARENT_XPATH = "open_in_immersive_parent_xpath";
public static final String EXTRA_OPEN_IN_IMMERSIVE_ELEMENT_XPATH = "open_in_immersive_element_xpath";

private BroadcastReceiver mCrashReceiver = new BroadcastReceiver() {
@Override
public void onReceive(Context context, Intent intent) {
Expand Down Expand Up @@ -248,6 +254,8 @@ public void run() {
private PlatformActivityPlugin mPlatformPlugin;
private int mLastMotionEventWidgetHandle;
private boolean mIsEyeTrackingSupported;
private String mImmersiveParentElementXPath;
private String mImmersiveTargetElementXPath;

private boolean callOnAudioManager(Consumer<AudioManager> fn) {
if (mAudioManager == null) {
Expand Down Expand Up @@ -886,6 +894,14 @@ void loadFromIntent(final Intent intent) {
}

openInKioskMode = extras.getBoolean(EXTRA_KIOSK, false);

if (extras.getBoolean(EXTRA_OPEN_IN_IMMERSIVE)) {
mImmersiveParentElementXPath = extras.getString(EXTRA_OPEN_IN_IMMERSIVE_PARENT_XPATH);
mImmersiveTargetElementXPath = extras.getString(EXTRA_OPEN_IN_IMMERSIVE_ELEMENT_XPATH);

// Open in immersive requires specific information to be present
mOpenInImmersive = targetUri != null && mImmersiveTargetElementXPath != null;
}
}

// If there is a target URI we open it
Expand All @@ -897,6 +913,8 @@ void loadFromIntent(final Intent intent) {
if (openInKioskMode) {
// FIXME this might not work as expected if the app was already running
mWindows.openInKioskMode(targetUri.toString());
} if (mOpenInImmersive) {
mWindows.openInImmersiveMode(targetUri, mImmersiveParentElementXPath, mImmersiveTargetElementXPath);
} else {
if (openInWindow) {
location = Windows.OPEN_IN_NEW_WINDOW;
Expand Down Expand Up @@ -1338,6 +1356,13 @@ void onExitWebXR(long aCallback) {
return;
}
mIsPresentingImmersive = false;
TelemetryService.stopImmersive();

if (mOpenInImmersive) {
Log.d(LOGTAG, "Started in immersive mode: exiting WebXR will finish the app");
finish();
}

runOnUiThread(() -> {
mWindows.exitImmersiveMode();
for (WebXRListener listener: mWebXRListeners) {
Expand All @@ -1348,7 +1373,6 @@ void onExitWebXR(long aCallback) {
// Show the window in front of you when you exit immersive mode.
recenterUIYaw(WidgetManagerDelegate.YAW_TARGET_ALL);

TelemetryService.stopImmersive();
Handler handler = new Handler(Looper.getMainLooper());
handler.postDelayed(() -> {
if (!mWindows.isPaused()) {
Expand All @@ -1361,6 +1385,7 @@ void onExitWebXR(long aCallback) {
}
}, 20);
}

@Keep
@SuppressWarnings("unused")
void onDismissWebXRInterstitial() {
Expand Down Expand Up @@ -1875,6 +1900,10 @@ public boolean isWebXRPresenting() {
return mIsPresentingImmersive;
}

@Override public boolean isOpenInImmersive() {
return mOpenInImmersive;
}

@Override
public void pushBackHandler(@NonNull Runnable aRunnable) {
mBackHandlers.addLast(aRunnable);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,8 @@ public WResult<Integer> onContentPermissionRequest(WSession aSession, ContentPer
// https://hacks.mozilla.org/2019/02/firefox-66-to-block-automatically-playing-audible-video-and-audio/
return WResult.fromValue(ContentPermission.VALUE_ALLOW);
} else if(perm.permission == PERMISSION_AUTOPLAY_AUDIBLE) {
if (SettingsStore.getInstance(mContext).isAutoplayEnabled()) {
// allow autoplay when we start Wolvic in immersive mode automatically
if (SettingsStore.getInstance(mContext).isAutoplayEnabled() || mWidgetManager.isOpenInImmersive()) {
return WResult.fromValue(ContentPermission.VALUE_ALLOW);
} else {
return WResult.fromValue(ContentPermission.VALUE_DENY);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ enum OriginatorType {WEBSITE, APPLICATION}
boolean isWebXRIntersitialHidden();
boolean isWebXRPresenting();
boolean isPermissionGranted(@NonNull String permission);
boolean isOpenInImmersive();
void requestPermission(String originator, @NonNull String permission, OriginatorType originatorType, WSession.PermissionDelegate.Callback aCallback);
boolean canOpenNewWindow();
void openNewWindow(String uri);
Expand Down
37 changes: 36 additions & 1 deletion app/src/common/shared/com/igalia/wolvic/ui/widgets/Windows.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import android.content.Context;
import android.content.SharedPreferences;
import android.net.Uri;
import android.util.Log;

import androidx.annotation.IntDef;
Expand Down Expand Up @@ -75,14 +76,19 @@ public class Windows implements TrayListener, TopBarWidget.Delegate, TitleBarWid
public static final int OPEN_IN_BACKGROUND = 1;
public static final int OPEN_IN_NEW_WINDOW = 2;


private static final String WINDOWS_SAVE_FILENAME = "windows_state.json";

private static final int TAB_ADDED_NOTIFICATION_ID = 0;
private static final int TAB_SENT_NOTIFICATION_ID = 1;
private static final int BOOKMARK_ADDED_NOTIFICATION_ID = 2;
private static final int WEB_APP_ADDED_NOTIFICATION_ID = 3;

// start Wolvic in immersive mode automatically
private static final String PARENT_ELEMENT_XPATH_PARAMETER = "wolvic-autowebxr-parentElementXPath";
private static final String TARGET_ELEMENT_XPATH_PARAMETER = "wolvic-autowebxr-targetElementXPath";
private static final String IMMERSIVE_EXTENSION_ID = "wolvic-autowebxr@igalia.com";
private static final String IMMERSIVE_EXTENSION_URL = "resource://android/assets/extensions/wolvic_autowebxr/";

class WindowState {
WindowPlacement placement;
int textureWidth;
Expand Down Expand Up @@ -1475,6 +1481,35 @@ public void openInKioskMode(@NonNull String aUri) {
mFocusedWindow.setKioskMode(true);
}

public void openInImmersiveMode(Uri targetUri, String immersiveParentElementXPath, String immersiveTargetElementXPath) {
Uri.Builder uriBuilder = targetUri.buildUpon();
if (!StringUtils.isEmpty(immersiveParentElementXPath)) {
uriBuilder.appendQueryParameter(PARENT_ELEMENT_XPATH_PARAMETER, immersiveParentElementXPath);
}
if (!StringUtils.isEmpty(immersiveTargetElementXPath)) {
uriBuilder.appendQueryParameter(TARGET_ELEMENT_XPATH_PARAMETER, immersiveTargetElementXPath);
}
Uri extendedUri = uriBuilder.build();

Session session = SessionStore.get().createSuspendedSession(extendedUri.toString(), true);

mFocusedWindow.setKioskMode(true);

SessionStore.get().getWebExtensionRuntime().installBuiltInWebExtension(
IMMERSIVE_EXTENSION_ID,
IMMERSIVE_EXTENSION_URL,
webExtension -> {
setFirstPaint(mFocusedWindow, session);
mFocusedWindow.setSession(session, WindowWidget.DEACTIVATE_CURRENT_SESSION);
return null;
},
(throwable) -> {
Log.e(LOGTAG, "Error installing the " + IMMERSIVE_EXTENSION_ID + " from " + IMMERSIVE_EXTENSION_URL + " Web Extension: " + throwable.getLocalizedMessage());
return null;
}
);
}

public void addTab(@NonNull WindowWidget targetWindow, @Nullable String aUri) {
Session session = SessionStore.get().createSuspendedSession(aUri, targetWindow.getSession().isPrivateMode());
session.setParentSession(targetWindow.getSession());
Expand Down
153 changes: 153 additions & 0 deletions app/src/main/assets/extensions/wolvic_autowebxr/content.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
const LOGTAG = '[wolvic:autowebxr]';
const ENABLE_LOGS = true;
const logDebug = (...args) => ENABLE_LOGS && console.log(LOGTAG, ...args);

const PARENT_ELEMENT_XPATH_PARAMETER = 'wolvic-autowebxr-parentElementXPath';
const TARGET_ELEMENT_XPATH_PARAMETER = 'wolvic-autowebxr-targetElementXPath';

const IFRAME_READY_MSG = 'wolvic-autowebxr-iframeReady';
const TARGET_ELEMENT_MSG = 'wolvic-autowebxr-targetElement';

var parentElementXPath;
var targetElementXPath;

function getElementByXPath(document, xpath) {
let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
return result.singleNodeValue;
}

// Limit the number of times that we can try to launch the experience, to avoid an infinite loop.
var retryCounter = 0;
const RETRY_LIMIT = 20;
function retryAfterTimeout(code, delay) {
if (retryCounter < RETRY_LIMIT) {
retryCounter++;
setTimeout(code, delay);
} else {
logDebug('Retry limit reached, will not try again');
}
}

function clickImmersiveElement() {
// Check if the current URL has extra query parameters
parentElementXPath = undefined;
targetElementXPath = undefined;

let url = document.URL;
let params = new URLSearchParams(new URL(url).search);
for (let [key, value] of params) {
if (key === PARENT_ELEMENT_XPATH_PARAMETER)
parentElementXPath = value;
else if (key === TARGET_ELEMENT_XPATH_PARAMETER)
targetElementXPath = value;
}

// We need at least the target element to click
if (!targetElementXPath)
return;

logDebug('Preparing to open immersive WebXR; parentElementXPath: ' + parentElementXPath + ' ; targetElementXPath: ' + targetElementXPath);

// The parent element is typically an iframe and, if it comes from a different origin,
// we might not be able to access its contents directly.
// If parentElementXPath is null, we will use the root to find the target element.

var parent, parentDocument;
if (parentElementXPath) {
parent = getElementByXPath(document, parentElementXPath);
if (!parent) {
logDebug('Parent element not found, retrying');
retryAfterTimeout(clickImmersiveElement, 1000);
return;
}

try {
parentDocument = parent.contentDocument || parent.contentWindow.document;
} catch (e) {
logDebug('Parent iframe is from a different origin');
const iframeWindow = parent.contentWindow;

const targetElementMsg = {
action: TARGET_ELEMENT_MSG,
targetElementXPath: targetElementXPath
};

iframeWindow.postMessage(targetElementMsg, '*');

// The iframe might not be ready yet, so we set up a listener for the "iframe ready" message.
const handleIframeReady = (event) => {
if (event.source === iframeWindow && event.data === IFRAME_READY_MSG) {
window.removeEventListener('message', handleIframeReady);
iframeWindow.postMessage(targetElementMsg, '*');
}
};
window.addEventListener('message', handleIframeReady);

return;
}
} else {
parent = window;
parentDocument = document;
}

if (parentDocument.readyState !== 'complete') {
logDebug('Parent is still loading');
parent.addEventListener('load', function() {
logDebug('Parent has finished loading');
clickImmersiveElement();
});
return;
} else {
logDebug('Parent is loaded');
}

let targetElement = getElementByXPath(parentDocument, targetElementXPath);

if (targetElement) {
logDebug('Target element found, calling click()');
targetElement.click();
targetElement.click();
} else {
logDebug('Target element not found, we will try again');
retryAfterTimeout(clickImmersiveElement, 1000);
}
}

function launchImmersiveFromIframe() {
window.addEventListener('message', function(event) {
if (event.data.action === TARGET_ELEMENT_MSG) {
let targetElement = getElementByXPath(document, event.data.targetElementXPath);

if (targetElement) {
logDebug('Target element found in iframe, calling click()');
targetElement.click();
} else {
logDebug('Target element not found in iframe, retrying');
retryAfterTimeout(function() {
window.postMessage(event.data, '*');
}, 1000);
}
}
});

window.parent.postMessage(IFRAME_READY_MSG, '*');
}

// Main script execution
if (window.top === window.self) {
if (document.readyState === 'complete') {
logDebug('Root document is completely ready');
clickImmersiveElement();
} else {
logDebug('Root document is not ready yet');
window.addEventListener('load', clickImmersiveElement);
}
} else {
if (document.readyState === 'complete') {
logDebug('Iframe is completely ready');
launchImmersiveFromIframe();
} else {
logDebug('Iframe is not ready yet');
window.addEventListener('load', launchImmersiveFromIframe);
}
}
23 changes: 23 additions & 0 deletions app/src/main/assets/extensions/wolvic_autowebxr/manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"manifest_version": 2,
"name": "Wolvic WebXR Automator",
"version": "1.0",
"description": "Enter WebXR Experiences Automatically",
"browser_specific_settings": {
"gecko": {
"id": "wolvic-autowebxr@igalia.com"
}
},
"content_scripts": [
{
"matches": [
"*://*/*"
],
"js": [
"content.js"
],
"run_at": "document_idle",
"all_frames": true
}
]
}
4 changes: 3 additions & 1 deletion app/src/main/res/raw/fxr_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,6 @@ prefs:
browser.gesture.pinch.in: ''
browser.gesture.pinch.out.shift: ''
browser.gesture.pinch.in.shift: ''
apz.one_touch_pinch.enabled: false
apz.one_touch_pinch.enabled: false,
# allows scripts to open WebXR immersive sessions
dom.vr.require-gesture: false

0 comments on commit a2b14df

Please sign in to comment.