From 4aeca5ab0fe4b041d502486a18cc2b66e18ba4ce Mon Sep 17 00:00:00 2001
From: Johnathan Sharratt <johnathan.sharratt@gmail.com>
Date: Fri, 9 Jun 2023 12:11:44 +1000
Subject: [PATCH] Upgraded to the latest resolver
---
lib/api/src/js/externals/memory.rs | 2 +-
lib/wasi-web/Cargo.lock | 129 +++++++++++++++++++---
lib/wasi-web/src/glue.rs | 9 +-
lib/wasi-web/src/runtime.rs | 48 +++++++-
lib/wasi/src/runtime/resolver/registry.rs | 2 +-
lib/wasi/src/wapm/mod.rs | 48 ++++----
6 files changed, 192 insertions(+), 46 deletions(-)
diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs
index a33645c3c60..456c98b57cf 100644
--- a/lib/api/src/js/externals/memory.rs
+++ b/lib/api/src/js/externals/memory.rs
@@ -151,7 +151,7 @@ impl Memory {
new_store: &mut impl AsStoreMut,
) -> Option<Self> {
self.try_clone(&store)
- .and_then(|mut memory| memory.duplicate().ok())
+ .and_then(|memory| memory.try_clone())
.map(|new_memory| Self::new_from_existing(new_store, new_memory.into()))
}
diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock
index f86357bc294..303964767bd 100644
--- a/lib/wasi-web/Cargo.lock
+++ b/lib/wasi-web/Cargo.lock
@@ -507,6 +507,27 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "errno"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
[[package]]
name = "fastrand"
version = "1.9.0"
@@ -701,6 +722,12 @@ dependencies = [
"unicode-segmentation",
]
+[[package]]
+name = "hermit-abi"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+
[[package]]
name = "hex"
version = "0.4.3"
@@ -787,6 +814,17 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "io-lifetimes"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "itoa"
version = "1.0.6"
@@ -825,9 +863,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.141"
+version = "0.2.146"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
+checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b"
[[package]]
name = "link-cplusplus"
@@ -853,6 +891,12 @@ dependencies = [
"linked-hash-map",
]
+[[package]]
+name = "linux-raw-sys"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
+
[[package]]
name = "lock_api"
version = "0.4.9"
@@ -1010,7 +1054,7 @@ dependencies = [
"cfg-if",
"instant",
"libc",
- "redox_syscall",
+ "redox_syscall 0.2.16",
"smallvec",
"winapi",
]
@@ -1023,7 +1067,7 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall",
+ "redox_syscall 0.2.16",
"smallvec",
"windows-sys 0.45.0",
]
@@ -1206,6 +1250,15 @@ dependencies = [
"bitflags",
]
+[[package]]
+name = "redox_syscall"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+dependencies = [
+ "bitflags",
+]
+
[[package]]
name = "redox_users"
version = "0.4.3"
@@ -1213,7 +1266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
- "redox_syscall",
+ "redox_syscall 0.2.16",
"thiserror",
]
@@ -1314,6 +1367,20 @@ dependencies = [
"semver 1.0.17",
]
+[[package]]
+name = "rustix"
+version = "0.37.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "ryu"
version = "1.0.13"
@@ -1617,6 +1684,20 @@ version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5"
+[[package]]
+name = "tempfile"
+version = "3.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "fastrand",
+ "redox_syscall 0.3.5",
+ "rustix",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "term_size"
version = "0.3.2"
@@ -2057,7 +2138,7 @@ dependencies = [
[[package]]
name = "wai-bindgen-wasmer"
-version = "0.3.1"
+version = "0.4.0"
dependencies = [
"anyhow",
"bitflags",
@@ -2241,7 +2322,7 @@ dependencies = [
[[package]]
name = "wasmer"
-version = "3.2.1"
+version = "3.3.0"
dependencies = [
"bytes",
"cfg-if",
@@ -2260,14 +2341,15 @@ dependencies = [
"wasmer-derive",
"wasmer-types",
"wasmer-vm",
- "wasmparser",
+ "wasmparser 0.83.0",
+ "wasmparser 0.95.0",
"wat",
"winapi",
]
[[package]]
name = "wasmer-compiler"
-version = "3.2.1"
+version = "3.3.0"
dependencies = [
"backtrace",
"cfg-if",
@@ -2287,7 +2369,7 @@ dependencies = [
[[package]]
name = "wasmer-derive"
-version = "3.2.1"
+version = "3.3.0"
dependencies = [
"proc-macro-error",
"proc-macro2",
@@ -2297,7 +2379,7 @@ dependencies = [
[[package]]
name = "wasmer-types"
-version = "3.2.1"
+version = "3.3.0"
dependencies = [
"bytecheck",
"enum-iterator",
@@ -2311,7 +2393,7 @@ dependencies = [
[[package]]
name = "wasmer-vm"
-version = "3.2.1"
+version = "3.3.0"
dependencies = [
"backtrace",
"cc",
@@ -2336,7 +2418,7 @@ dependencies = [
[[package]]
name = "wasmer-wasix"
-version = "0.3.1"
+version = "0.4.0"
dependencies = [
"anyhow",
"async-trait",
@@ -2345,6 +2427,7 @@ dependencies = [
"cfg-if",
"chrono",
"cooked-waker",
+ "dashmap",
"derivative",
"futures",
"getrandom",
@@ -2357,17 +2440,20 @@ dependencies = [
"once_cell",
"pin-project",
"rand",
+ "semver 1.0.17",
"serde",
"serde_derive",
"serde_json",
"serde_yaml",
"sha2 0.10.6",
"shellexpand",
+ "tempfile",
"term_size",
"termios",
"thiserror",
"tokio",
"tracing",
+ "url",
"urlencoding",
"virtual-fs",
"virtual-net",
@@ -2384,7 +2470,7 @@ dependencies = [
[[package]]
name = "wasmer-wasix-types"
-version = "0.3.1"
+version = "0.4.0"
dependencies = [
"anyhow",
"bitflags",
@@ -2441,6 +2527,12 @@ dependencies = [
"xterm-js-rs",
]
+[[package]]
+name = "wasmparser"
+version = "0.83.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "718ed7c55c2add6548cca3ddd6383d738cd73b892df400e96b9aa876f0141d7a"
+
[[package]]
name = "wasmparser"
version = "0.95.0"
@@ -2575,6 +2667,15 @@ dependencies = [
"windows-targets 0.42.2",
]
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
+]
+
[[package]]
name = "windows-targets"
version = "0.42.2"
diff --git a/lib/wasi-web/src/glue.rs b/lib/wasi-web/src/glue.rs
index 52ce3b30a63..5a23fb84ad7 100644
--- a/lib/wasi-web/src/glue.rs
+++ b/lib/wasi-web/src/glue.rs
@@ -9,7 +9,6 @@ use tokio::sync::mpsc;
use tracing::{debug, error, info, trace, warn};
use wasm_bindgen::{prelude::*, JsCast};
use wasmer_wasix::{
- bin_factory::ModuleCache,
capabilities::Capabilities,
os::{Console, InputEvent, Tty, TtyOptions},
Pipe,
@@ -40,7 +39,7 @@ pub fn main() {
pub const DEFAULT_BOOT_WEBC: &'static str = "sharrattj/bash";
//pub const DEFAULT_BOOT_WEBC: &str = "sharrattj/dash";
-pub const DEFAULT_BOOT_USES: [&'static str; 2] = ["sharrattj/coreutils", "sharrattj/catsay"];
+pub const DEFAULT_BOOT_USES: [&'static str; 1] = ["sharrattj/coreutils"];
#[wasm_bindgen]
pub fn start() -> Result<(), JsValue> {
@@ -134,8 +133,6 @@ pub fn start() -> Result<(), JsValue> {
tty_options,
);
- let compiled_modules = Arc::new(ModuleCache::new(None, None, false));
-
let location = url::Url::parse(location.as_str()).unwrap();
let mut console = if let Some(init) = location
.query_pairs()
@@ -143,11 +140,11 @@ pub fn start() -> Result<(), JsValue> {
.next()
.map(|(_, val)| val.to_string())
{
- let mut console = Console::new(init.as_str(), runtime.clone(), compiled_modules);
+ let mut console = Console::new(init.as_str(), runtime.clone());
console = console.with_no_welcome(true);
console
} else {
- let mut console = Console::new(DEFAULT_BOOT_WEBC, runtime.clone(), compiled_modules);
+ let mut console = Console::new(DEFAULT_BOOT_WEBC, runtime.clone());
console = console.with_uses(DEFAULT_BOOT_USES.iter().map(|a| a.to_string()).collect());
console
};
diff --git a/lib/wasi-web/src/runtime.rs b/lib/wasi-web/src/runtime.rs
index 332782d08cd..3ecfc1eb1ea 100644
--- a/lib/wasi-web/src/runtime.rs
+++ b/lib/wasi-web/src/runtime.rs
@@ -17,9 +17,14 @@ use tracing::{debug, error, info, trace, warn};
use wasm_bindgen::prelude::*;
use wasm_bindgen_futures::JsFuture;
use wasmer_wasix::{
- http::{DynHttpClient, HttpRequest, HttpResponse},
+ bin_factory::BinaryPackage,
+ http::{DynHttpClient, HttpClient, HttpRequest, HttpResponse},
os::{TtyBridge, TtyOptions},
- runtime::task_manager::TaskWasm,
+ runtime::{
+ module_cache::{ModuleCache, ThreadLocalCache},
+ resolver::{PackageResolver, ResolverError, WebcIdentifier},
+ task_manager::TaskWasm,
+ },
VirtualFile, VirtualNetworking, VirtualTaskManager, WasiRuntime, WasiThreadError, WasiTtyState,
};
use web_sys::WebGl2RenderingContext;
@@ -47,11 +52,36 @@ pub(crate) struct WebRuntime {
tty: TtyOptions,
http_client: DynHttpClient,
+ package_resolver: Arc<dyn PackageResolver + Send + Sync>,
+ module_cache: Arc<dyn ModuleCache + Send + Sync>,
net: wasmer_wasix::virtual_net::DynVirtualNetworking,
tasks: Arc<dyn VirtualTaskManager>,
}
+#[derive(Debug, Default)]
+struct BasicResolver {}
+
+pub const WAPM_PROD_ENDPOINT: &str = "https://registry.wapm.io/graphql";
+
+#[async_trait::async_trait]
+impl PackageResolver for BasicResolver {
+ async fn resolve_package(
+ &self,
+ pkg: &WebcIdentifier,
+ client: &(dyn HttpClient + Send + Sync),
+ ) -> Result<BinaryPackage, ResolverError> {
+ wasmer_wasix::wapm::fetch_webc(
+ None,
+ &pkg.full_name,
+ client,
+ &url::Url::parse(WAPM_PROD_ENDPOINT).unwrap(),
+ )
+ .await
+ .map_err(|e| ResolverError::Other(e.into()))
+ }
+}
+
impl WebRuntime {
#[allow(unused_variables)]
pub(crate) fn new(
@@ -68,6 +98,10 @@ impl WebRuntime {
runtime,
});
+ let http_client = Arc::new(WebHttpClient { pool: pool.clone() });
+
+ let package_resolver = BasicResolver::default();
+ let module_cache = ThreadLocalCache::default();
WebRuntime {
pool: pool.clone(),
tasks: runtime,
@@ -75,7 +109,9 @@ impl WebRuntime {
#[cfg(feature = "webgl")]
webgl_tx,
http_client: Arc::new(WebHttpClient { pool }),
+ package_resolver: Arc::new(package_resolver),
net: Arc::new(WebVirtualNetworking),
+ module_cache: Arc::new(module_cache),
}
}
}
@@ -429,6 +465,14 @@ impl WasiRuntime for WebRuntime {
Some(self)
}
+ fn module_cache(&self) -> Arc<dyn ModuleCache + Send + Sync> {
+ Arc::clone(&self.module_cache)
+ }
+
+ fn package_resolver(&self) -> Arc<dyn PackageResolver + Send + Sync> {
+ Arc::clone(&self.package_resolver) as Arc<dyn PackageResolver + Send + Sync>
+ }
+
fn http_client(&self) -> Option<&DynHttpClient> {
Some(&self.http_client)
}
diff --git a/lib/wasi/src/runtime/resolver/registry.rs b/lib/wasi/src/runtime/resolver/registry.rs
index f9761792b5e..95b8ea13e97 100644
--- a/lib/wasi/src/runtime/resolver/registry.rs
+++ b/lib/wasi/src/runtime/resolver/registry.rs
@@ -89,7 +89,7 @@ impl PackageResolver for RegistryResolver {
}
crate::wapm::fetch_webc(
- &self.cache_dir,
+ Some(&self.cache_dir),
&pkg.full_name,
client,
&self.registry_endpoint,
diff --git a/lib/wasi/src/wapm/mod.rs b/lib/wasi/src/wapm/mod.rs
index 22f2d8cb900..577054363cb 100644
--- a/lib/wasi/src/wapm/mod.rs
+++ b/lib/wasi/src/wapm/mod.rs
@@ -27,8 +27,8 @@ mod pirita;
use crate::http::{HttpRequest, HttpRequestOptions};
use pirita::*;
-pub(crate) async fn fetch_webc(
- cache_dir: &Path,
+pub async fn fetch_webc(
+ cache_dir: Option<&Path>,
webc: &str,
client: &(dyn HttpClient + Send + Sync),
registry_endpoint: &Url,
@@ -109,7 +109,7 @@ pub fn parse_static_webc(data: Vec<u8>) -> Result<BinaryPackage, anyhow::Error>
}
async fn download_webc(
- cache_dir: &Path,
+ cache_dir: Option<&Path>,
name: &str,
pirita_download_url: String,
client: &(dyn HttpClient + Send + Sync),
@@ -132,29 +132,33 @@ async fn download_webc(
std::path::Path::new(cache_dir).join(&name)
};
- // fast path
- let path = compute_path(cache_dir, name);
-
#[cfg(feature = "sys")]
- if path.exists() {
- tracing::debug!(path=%path.display(), "Parsing cached WEBC file");
+ {
+ // fast path
+ let path = compute_path(cache_dir, name);
- match Container::from_disk(&path) {
- Ok(webc) => {
- return parse_webc_v2(&webc)
- .with_context(|| format!("Could not parse webc at path '{}'", path.display()));
- }
- Err(err) => {
- tracing::warn!(
- error = &err as &dyn std::error::Error,
- "failed to parse WEBC",
- );
+ #[cfg(feature = "sys")]
+ if path.exists() {
+ tracing::debug!(path=%path.display(), "Parsing cached WEBC file");
+
+ match Container::from_disk(&path) {
+ Ok(webc) => {
+ return parse_webc_v2(&webc).with_context(|| {
+ format!("Could not parse webc at path '{}'", path.display())
+ });
+ }
+ Err(err) => {
+ tracing::warn!(
+ error = &err as &dyn std::error::Error,
+ "failed to parse WEBC",
+ );
+ }
}
}
- }
- if let Ok(data) = std::fs::read(&path) {
- if let Ok(webc) = parse_static_webc(data) {
- return Ok(webc);
+ if let Ok(data) = std::fs::read(&path) {
+ if let Ok(webc) = parse_static_webc(data) {
+ return Ok(webc);
+ }
}
}
--
GitLab