Load github resources from web editor (#68)

* wasm url loader

* testing

* style fix and format fix

* forgot to save

* update botw-map

* update botw-map

* update default branch and add preset placeholders

* run fixer

* small fixes

compiler-core/Cargo.toml

@@ -32,7 +32,7 @@ log = "0.4.20"
 default = [
     "tokio", 
     "tokio-stream", 
-    "cached/async_tokio_rt_multi_thread"
+    "cached/async_tokio_rt_multi_thread",
 ]
 wasm = [
     "cached/wasm", 

compiler-core/src/pack/mod.rs

@@ -47,6 +47,9 @@ pub enum PackerError {
     #[error("Invalid path: {0}")]
     InvalidPath(String),
 
+    #[error("Invalid url: {0}")]
+    InvalidUrl(String),
+
     #[error("Max depth of {0} levels of `use` is reached. Please make sure there are no circular dependencies.")]
     MaxUseDepthExceeded(usize),
 

compiler-core/src/pack/resource/loader_cache.rs

@@ -20,22 +20,45 @@ impl GlobalCacheLoader {
 #[cfg_attr(feature = "wasm", async_trait::async_trait(?Send))]
 impl ResourceLoader for GlobalCacheLoader {
     async fn load_raw(&self, r: &str) -> PackerResult<Vec<u8>> {
-        self.delegate.load_raw(r).await
+        load_raw_internal(&self.delegate, r).await
     }
 
     async fn load_image_url(&self, path: &str) -> PackerResult<String> {
-        self.delegate.load_image_url(path).await
+        load_image_url_internal(&self.delegate, path).await
     }
 
     async fn load_structured(&self, path: &str) -> PackerResult<Value> {
-        // associative function not supported by cached crate
-        // so we need to use a helper
         load_structured_internal(&self.delegate, path).await
     }
 }
 
+// associative function not supported by cached crate
+// so we need to use helpers
+
+#[cached(
+    size=256,
+    key="String",
+    convert = r#"{ path.to_string() }"#,
+    // TTL of 5 minutes
+    time=300,
+)]
+async fn load_raw_internal(loader: &ArcLoader, path: &str) -> PackerResult<Vec<u8>> {
+    loader.load_raw(path).await
+}
+
+#[cached(
+    size=256,
+    key="String",
+    convert = r#"{ path.to_string() }"#,
+    // TTL of 5 minutes
+    time=300,
+)]
+async fn load_image_url_internal(loader: &ArcLoader, path: &str) -> PackerResult<String> {
+    loader.load_image_url(path).await
+}
+
 #[cached(
-    size=512,
+    size=256,
     key="String",
     convert = r#"{ path.to_string() }"#,
     // TTL of 5 minutes

compiler-core/src/pack/resource/mod.rs

@@ -4,8 +4,6 @@ mod loader_cache;
 pub use loader_cache::*;
 mod loader_empty;
 pub use loader_empty::*;
-mod loader_url;
-pub use loader_url::*;
 mod resource_github;
 pub use resource_github::*;
 mod resource_impl;

compiler-core/src/pack/resource/resource_github.rs

@@ -2,7 +2,6 @@
 use std::sync::Arc;
 
 use crate::util::Path;
-use cached::proc_macro::cached;
 
 use super::{ArcLoader, EmptyLoader, Resource, ResourcePath, ResourceResolver};
 use crate::pack::{PackerError, PackerResult, ValidUse};
@@ -128,28 +127,7 @@ async fn get_github_url(
     reference: Option<&str>,
 ) -> PackerResult<String> {
     let path = path.as_ref();
-    let url = match reference {
-        Some(reference) => {
-            format!("https://raw.githubusercontent.com/{owner}/{repo}/{reference}/{path}")
-        }
-        None => {
-            let default_branch = get_default_branch(owner, repo).await?;
-            format!("https://raw.githubusercontent.com/{owner}/{repo}/{default_branch}/{path}")
-        }
-    };
+    let branch = reference.unwrap_or("main");
+    let url = format!("https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}");
     Ok(url)
 }
-
-/// Get the default branch of a repository.
-#[cached(
-    key="String", 
-    convert = r#"{ format!("{}/{}", _owner, _repo) }"#,
-    // 1 hour TTL
-    time=3600,
-)]
-async fn get_default_branch(_owner: &str, _repo: &str) -> PackerResult<String> {
-    // TODO #31
-    Err(PackerError::NotImpl(
-        "getting default branch not implemented".to_string(),
-    ))
-}

compiler-core/src/util/async_macro_wasm.rs

@@ -13,6 +13,11 @@ thread_local! {
     static CANCELLED: UnsafeCell<bool> = UnsafeCell::new(false);
 }
 
+const BUDGET_MAX: u8 = 64;
+thread_local! {
+    static BUDGET: UnsafeCell<u8> = UnsafeCell::new(BUDGET_MAX);
+}
+
 /// A signal for cancellation
 #[derive(Debug, Clone, PartialEq, thiserror::Error)]
 pub enum WasmError {
@@ -30,6 +35,19 @@ pub fn set_cancelled(value: bool) {
 ///
 /// Shared code for server and WASM should use the [`yield_now`] macro instead of calling this directly.
 pub async fn set_timeout_yield() -> Result<(), WasmError> {
+    let has_budget = BUDGET.with(|budget| unsafe {
+        let b_ref = budget.get();
+        if *b_ref == 0 {
+            *b_ref = BUDGET_MAX;
+            false
+        } else {
+            *b_ref -= 1;
+            true
+        }
+    });
+    if has_budget {
+        return Ok(());
+    }
     let promise = WINDOW.with(|window| {
         Promise::new(&mut |resolve, _| {
             let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, 0);

compiler-wasm/src/api.rs

@@ -2,18 +2,17 @@ use std::cell::RefCell;
 use std::sync::Arc;
 
 use celerc::api::{CompilerMetadata, CompilerOutput, Setting};
-use celerc::pack::{
-    ArcLoader, GlobalCacheLoader, LocalResourceResolver, Resource, ResourcePath, UrlLoader,
-};
+use celerc::pack::{LocalResourceResolver, Resource, ResourcePath};
 use celerc::util::Path;
 use celerctypes::ExecDoc;
 use js_sys::Function;
 use log::{info, LevelFilter};
 use wasm_bindgen::JsValue;
 use web_sys::console;
 
+use crate::loader_file::FileLoader;
+use crate::loader_url::UrlLoader;
 use crate::logger::{self, Logger};
-use crate::resource::FileLoader;
 
 const LOGGER: Logger = Logger;
 
@@ -24,15 +23,15 @@ thread_local! {
 
 thread_local! {
     #[allow(clippy::arc_with_non_send_sync)]
-    static URL_LOADER: ArcLoader = Arc::new(GlobalCacheLoader::new(Arc::new(UrlLoader)));
+    static URL_LOADER: Arc<UrlLoader> = Arc::new(UrlLoader::new());
 }
 
 thread_local! {
     static COMPILER_META: RefCell<Option<CompilerMetadata>> = RefCell::new(None);
 }
 
 /// Initialize
-pub fn init(logger: JsValue, load_file: Function) {
+pub fn init(logger: JsValue, load_file: Function, fetch: Function) {
     if let Err(e) = logger::bind_logger(logger) {
         console::error_1(&e);
     }
@@ -51,6 +50,9 @@ pub fn init(logger: JsValue, load_file: Function) {
     FILE_LOADER.with(|loader| {
         loader.init(load_file);
     });
+    URL_LOADER.with(|loader| {
+        loader.init(fetch);
+    });
 
     info!("compiler initialized");
 }

compiler-wasm/src/build.rs

@@ -62,6 +62,7 @@ fn build_wasm_pack_command() -> Command {
 fn override_typescript_definitions() -> io::Result<()> {
     println!("generating typescript definitions");
     let mut d_ts = celercwasm::generate_d_ts_imports();
+    d_ts.push_str(include_str!("./wasm.ts"));
     d_ts.push_str(&celercwasm::generate_d_ts());
     fs::write(Path::new(OUTPUT_DIR).join("celercwasm.d.ts"), d_ts)?;
     Ok(())

compiler-wasm/src/lib.rs

@@ -7,8 +7,9 @@ mod api;
 mod wasm;
 use wasm::*;
 
+mod loader_file;
+mod loader_url;
 mod logger;
-mod resource;
 
 // WASM output types
 import! {
@@ -21,12 +22,14 @@ into! {ExecDoc}
 
 ffi!(
     /// Initialize
-    pub async fn initCompiler(logger: JsValue, load_file: Function) -> void {
-        api::init(logger, load_file);
+    pub async fn initCompiler(logger: JsValue, load_file: Function, fetch: Function) -> void {
+        api::init(logger, load_file, fetch);
         JsValue::UNDEFINED
     }
 
     /// Compile a document from web editor
+    ///
+    /// If use_cache is true, the compiler will use cached results loaded from URLs
     pub async fn compileDocument() -> Option<ExecDoc> {
         api::compile_document().await
     }

compiler-wasm/src/loader_url.rs

@@ -0,0 +1,53 @@
+use std::cell::RefCell;
+
+use celerc::pack::{PackerError, PackerResult, ResourceLoader};
+use celerc::yield_now;
+use js_sys::{Function, Uint8Array};
+use wasm_bindgen::{JsCast, JsValue};
+
+use crate::wasm::{into_future, stub_function};
+
+/// Loader for loading a URL using a provided JS function
+pub struct UrlLoader {
+    /// Callback function to ask JS to load a file
+    ///
+    /// Takes in a string (url) as argument.
+    /// Returns a promise that resolves to a Uint8Array that could throw
+    fetch: RefCell<Function>,
+}
+
+impl UrlLoader {
+    pub fn new() -> Self {
+        Self {
+            fetch: RefCell::new(stub_function()),
+        }
+    }
+    pub fn init(&self, fetch: Function) {
+        self.fetch.replace(fetch);
+    }
+}
+
+#[async_trait::async_trait(?Send)]
+impl ResourceLoader for UrlLoader {
+    async fn load_raw(&self, url: &str) -> PackerResult<Vec<u8>> {
+        yield_now!()?;
+        let result: Result<Uint8Array, JsValue> = async {
+            let promise = self
+                .fetch
+                .borrow()
+                .call1(&JsValue::UNDEFINED, &JsValue::from(url))?;
+            let vec: Uint8Array = into_future(promise).await?.dyn_into()?;
+            Ok(vec)
+        }
+        .await;
+        // see if JS call is successful
+        let uint8arr =
+            result.map_err(|_| PackerError::LoadUrl(format!("loading URL failed: {url}")))?;
+        Ok(uint8arr.to_vec())
+    }
+
+    async fn load_image_url(&self, url: &str) -> PackerResult<String> {
+        // image is already a URL, so just return it
+        Ok(url.to_string())
+    }
+}

compiler-wasm/src/wasm.rs

@@ -1,6 +1,6 @@
 //! Utils for gluing WASM and JS
 
-use js_sys::{Function, Promise};
+use js_sys::{Boolean, Function, Promise};
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::JsFuture;
 
@@ -148,6 +148,13 @@ macro_rules! into {
 }
 pub(crate) use into;
 
+impl WasmFrom for bool {
+    fn from_wasm(value: JsValue) -> Result<Self, JsValue> {
+        let b: Boolean = value.dyn_into()?;
+        Ok(b.into())
+    }
+}
+
 /// Take a promise and return a future
 pub async fn into_future(promise: JsValue) -> Result<JsValue, JsValue> {
     let promise: Promise = promise.dyn_into()?;

compiler-wasm/src/wasm.ts

@@ -0,0 +1,2 @@
+type bool = boolean;
+type JsValue = any;

docs/src/route/file-structure.md

@@ -43,7 +43,8 @@ If the path does not start with `.`, `..`, or `/`, it will be considered a GitHu
 ```
 {owner}/{repo}/{path/to/file}:{ref}
 ```
-The `:{ref}` can be a branch, tag, or commit hash. It is optional and when omitted, it will resolve to the default branch of that repo.
+The `:{ref}` can be a branch, tag, or commit hash. 
+It is optional and when omitted, it will resolve to the `main` branch.
 
 Examples:
 ```yaml

presets/botw/README.md

@@ -0,0 +1,2 @@
+# Presets for Breath of the Wild
+README under construction. Tracked by https://github.com/Pistonite/celer/issues/27

presets/botw/map.yaml

@@ -0,0 +1,20 @@
+map:
+  layers:
+  - name: Overworld
+    template-url: https://objmap.zeldamods.org/game_files/maptex/{z}/{x}/{y}.png
+    size: [24000, 20000]
+    max-native-zoom: 7
+    zoom-bounds: [2, 9]
+    transform:
+      translate: [12000, 10000]
+      scale: [2, 2]
+    start-z: -5000
+    attribution:
+      link: https://objmap.zeldamods.org
+  coord-map:
+    "2d": [x, y]
+    "3d": [x, z, y]
+  initial-coord: [-1099.10, 242.00, 1876.31]
+  initial-zoom: 3
+  initial-color: "#38F"
+

web-client/src/core/kernel/editor/CompMgr.ts

@@ -25,8 +25,14 @@ export class CompMgr {
         this.compiling = false;
     }
 
-    public async init(loadFile: RequestFileFunction) {
-        initCompiler(CompilerLog, loadFile);
+    public async init(
+        loadFile: RequestFileFunction,
+        loadUrl: RequestFileFunction,
+    ) {
+        initCompiler(CompilerLog, loadFile, (url: string) => {
+            CompilerLog.info(`loading ${url}`);
+            return loadUrl(url);
+        });
     }
 
     /// Trigger compilation of the document
@@ -80,7 +86,5 @@ export class CompMgr {
         window.clearTimeout(handle);
         this.dispatcher.dispatch(viewActions.setCompileInProgress(false));
         this.compiling = false;
-        //wasm api should be something like:
-        //compile(requestfunction) -> Promise<result>
     }
 }