Merge branch 'master' into jupyter_base_url

.github/workflows/ci.yml

@@ -14,13 +14,27 @@ jobs:
     env:
       JULIA_PKG_SERVER: ""
       JULIA_NUM_THREADS: "4"
+      DISPLAY: ":99"
+      LIBGL_ALWAYS_SOFTWARE: "1"
+      MESA_GL_VERSION_OVERRIDE: "4.5"
+      MESA_GLSL_VERSION_OVERRIDE: "450"
     strategy:
       fail-fast: false
       matrix:
         version:
           - '1.6'
-          - '1' # Leave this line unchanged. '1' will automatically expand to the latest stable 1.x release of Julia.
+          - '1'
     steps:
+      - name: Install Dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y xvfb mesa-utils mesa-vulkan-drivers dbus
+      - name: Start Virtual Display
+        run: |
+          Xvfb :99 -screen 0 1920x1080x24 &
+          eval $(dbus-launch)
+          export DBUS_SESSION_BUS_ADDRESS
+          export DBUS_SESSION_BUS_PID
       - name: Checkout
         uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1

js_dependencies/Bonito.js

@@ -59,6 +59,7 @@ function update_dom_node(dom, html) {
 function fetch_binary(url) {
     return fetch(url).then((response) => {
         if (!response.ok) {
+
             throw new Error("HTTP error, status = " + response.status);
         }
         return response.arrayBuffer();

js_dependencies/Websocket.bundled.js

@@ -18,7 +18,7 @@ class Websocket {
         this.#onopen_callbacks.push(f);
     }
     tryconnect() {
-        console.log(`tries; ${this.#tries}`);
+        console.log(`tries: ${this.#tries}`);
         if (this.#websocket) {
             this.#websocket.close();
             this.#websocket = undefined;

js_dependencies/Websocket.js

@@ -28,7 +28,7 @@ class Websocket {
     }
 
     tryconnect() {
-        console.log(`tries; ${this.#tries}`);
+        console.log(`tries: ${this.#tries}`);
         if (this.#websocket) {
             this.#websocket.close();
             this.#websocket = undefined;

src/Bonito.jl

@@ -43,7 +43,7 @@ function HTTPServer.route!(server::HTTPServer.Server, routes::Routes)
 end
 
 
-import .HTTPServer: browser_display
+import .HTTPServer: browser_display, EWindow
 using .HTTPServer: Server, html, online_url, route!, file_mimetype, delete_websocket_route!, delete_route!, use_electron_display
 
 include("js_source.jl")

src/HTTPServer/browser-display.jl

@@ -130,9 +130,20 @@ struct ElectronDisplay{EWindow} <: Base.Multimedia.AbstractDisplay
     browserdisplay::BrowserDisplay
 end
 
+function EWindow(args...)
+    app = Electron().Application(;
+        additional_electron_args=[
+            "--disable-logging",
+            "--no-sandbox",
+            "--user-data-dir=$(mktempdir())",
+            "--disable-features=AccessibilityObjectModel",
+        ],
+    )
+    return Electron().Window(app, args...)
+end
+
 function ElectronDisplay(; devtools = false)
-    app = Electron().Application(; additional_electron_args=["--disable-logging"])
-    w = Electron().Window(app)
+    w = EWindow()
     devtools && Electron().toggle_devtools(w)
     return ElectronDisplay(w, BrowserDisplay(; open_browser=false))
 end

src/asset-serving/asset-serving.jl

@@ -4,14 +4,14 @@ include("asset.jl")
 include("no-server.jl")
 include("http.jl")
 
-const JS_DEPENDENCIES = @path joinpath(@__DIR__, "..", "..", "js_dependencies")
+const JS_DEPENDENCIES = joinpath(@__DIR__, "..", "..", "js_dependencies")
 
 """
     dependency_path(paths...)
 
 Path to serve downloaded dependencies
 """
-dependency_path(paths...) = @path joinpath(JS_DEPENDENCIES, paths...)
+dependency_path(paths...) = joinpath(JS_DEPENDENCIES, paths...)
 
 const BonitoLib = ES6Module(dependency_path("Bonito.js"))
 const Websocket = ES6Module(dependency_path("Websocket.js"))

src/asset-serving/asset.jl

@@ -5,7 +5,11 @@ function get_path(asset::Asset)
     isempty(asset.online_path) ? asset.local_path : asset.online_path
 end
 
-unique_file_key(path::String) = bytes2hex(sha1(abspath(path))) * "-" * basename(path)
+hash_content(x) = bytes2hex(sha1(x))
+
+function unique_file_key(path::String)
+    return hash_content(abspath(path)) * "-" * Bonito.URIs.escapeuri(basename(path))
+end
 unique_file_key(path) = unique_file_key(string(path))
 function unique_key(asset::Asset)
     if isempty(asset.online_path)
@@ -116,6 +120,37 @@ function Base.show(io::IO, asset::Asset)
     print(io, get_path(asset))
 end
 
+function bundle_folder(bundle_dir, local_path, name, media_type)
+    bundle_dir = if !isnothing(bundle_dir) && !isempty(bundle_dir)
+        bundle_dir
+    elseif isempty(local_path)
+        get_deps_path(name)
+    else
+        dirname(local_path)
+    end
+    isdir(bundle_dir) || mkpath(bundle_dir)
+    return joinpath(bundle_dir, string(name, ".bundled.", media_type))
+end
+
+
+function generate_bundle_file(file, bundle_file)
+    if isfile(file) || is_online(file)
+        if needs_bundling(file, bundle_file)
+            bundled, err = deno_bundle(file, bundle_file)
+            if !bundled
+                if isfile(bundle_file)
+                    @warn "Failed to bundle $file: $err"
+                else
+                    error("Failed to bundle $file: $err")
+                end
+            end
+        end
+        return bundle_file
+    else
+        return bundle_file
+    end
+end
+
 function Asset(path_or_url::Union{String,Path}; name=nothing, es6module=false, check_isfile=false, bundle_dir::Union{Nothing,String,Path}=nothing, mediatype=Symbol(getextension(path_or_url)))
     local_path = ""; real_online_path = ""
     if is_online(path_or_url)
@@ -124,10 +159,26 @@ function Asset(path_or_url::Union{String,Path}; name=nothing, es6module=false, c
     else
         local_path = normalize_path(path_or_url; check_isfile=check_isfile)
     end
-    _bundle_dir = isnothing(bundle_dir) ? dirname(local_path) : bundle_dir
-    return Asset(name, es6module, mediatype, real_online_path, local_path, @path _bundle_dir)
+     if es6module
+        path = bundle_folder(bundle_dir, local_path, name, mediatype)
+        # We may need to bundle immediately, since otherwise the dependencies for bunddling may be gone!
+        source = is_online(path_or_url) ? real_online_path : local_path
+        bundle_file = generate_bundle_file(source, path)
+        if !isfile(bundle_file)
+            error("Failed to bundle $source: $path. bundle_dir: $(bundle_dir)")
+        end
+        bundle_data = read(bundle_file) # read the into memory to make it relocatable
+        content_hash = RefValue{String}(hash_content(bundle_data))
+    else
+
+        bundle_file = ""
+        bundle_data = UInt8[]
+        content_hash = RefValue{String}("")
+    end
+    return Asset(name, es6module, mediatype, real_online_path, local_path, bundle_file, bundle_data, content_hash)
 end
 
+
 function ES6Module(path)
     name = String(splitext(basename(path))[1])
     asset = Asset(path; name=name, es6module=true)
@@ -192,43 +243,49 @@ function get_deps_path(name)
 end
 
 function bundle_path(asset::Asset)
-    @assert !isempty(asset.name) "Asset has no name, which may happen if Asset constructor was called wrongly"
-    bundle_dir = if !isempty(asset.bundle_dir)
-        asset.bundle_dir
-    elseif isempty(asset.local_path)
-        get_deps_path(asset.name)
-    else
-        dirname(asset.local_path)
-    end
-    isdir(bundle_dir) || mkpath(bundle_dir)
-    return joinpath(bundle_dir, string(asset.name, ".bundled.", asset.media_type))
+    return asset.bundle_file
 end
 
 last_modified(path::Path) = last_modified(Bonito.getroot(path))
 function last_modified(path::String)
     Dates.unix2datetime(Base.Filesystem.mtime(path))
 end
 
+function needs_bundling(path, bundled)
+    is_online(path) && return !isfile(bundled)
+    !isfile(bundled) && return true
+    # If bundled happen after last modification of asset
+    return last_modified(path) > last_modified(bundled)
+end
+
 function needs_bundling(asset::Asset)
     asset.es6module || return false
     path = get_path(asset)
     bundled = bundle_path(asset)
-    !isfile(bundled) && return true
-    # If bundled happen after last modification of asset
-    return last_modified(path) > last_modified(bundled)
+    return needs_bundling(path, bundled)
 end
 
+
+
 bundle!(asset::BinaryAsset) = nothing
 
 function bundle!(asset::Asset)
     needs_bundling(asset) || return
-    has_been_bundled = deno_bundle(get_path(asset), bundle_path(asset))
-    if !has_been_bundled && isfile(bundle_path(asset))
+    bundle_file = String(bundle_path(asset))
+    source = String(get_path(asset))
+    has_been_bundled, err = deno_bundle(source, bundle_file)
+    if isfile(bundle_file)
+        data = read(bundle_file)
+        resize!(asset.bundle_data, length(data))
+        copyto!(asset.bundle_data, data)
+        asset.content_hash[] = hash_content(data)
         # when shipping, we don't have the correct time stamps, so we can't accurately say if we need bundling :(
         # So we need to rely on the package authors to bundle before creating a new release!
         return
     end
-    if !has_been_bundled
+    # if bundle_data is stored in the asset, we dont necessarily need to bundle
+    # But it's likely outdated - which is fine for e.g. relocatable packages
+    if !has_been_bundled && isempty(asset.bundle_data)
         # Not bundling if bundling is needed is an error...
         # In theory it could be a warning, but this way we make CI fail, so that
         # PRs that forget to bundle JS dependencies will fail!
@@ -237,7 +294,11 @@ function bundle!(asset::Asset)
             which is an optional dependency needed for Developing Bonito Assets.
             After that, assets should be bundled on precompile and whenever they're used after editing the asset.
             If you're just using a package, please open an issue with the Package maintainers,
-            they must have forgotten bundling.")
+            they must have forgotten bundling.
+            Error: $err")
+    end
+    if !isempty(asset.bundle_data) && !has_been_bundled && isfile(source)
+        @warn "Asset $(asset) being served from memory, but failed to bundle with error: $(err)."
     end
     return
 end

src/asset-serving/http.jl

@@ -3,7 +3,7 @@ using .HTTPServer: has_route, get_route, route!
 mutable struct HTTPAssetServer <: AbstractAssetServer
     # Reference count the files/binary assets, so we can clean them up for child sessions
     registered_files::Dict{
-        String,Tuple{Set{UInt},Union{Path, String, BinaryAsset}}
+        String,Tuple{Set{UInt}, AbstractAsset}
     }
     server::Server
     lock::ReentrantLock
@@ -21,7 +21,8 @@ end
 const MATCH_HEX = r"[\da-f]"
 const ASSET_ROUTE_REGEX = r"assets-(?:\d+){20}"
 const UNIQUE_FILE_KEY_REGEX = MATCH_HEX^40 * r"-.*"
-const ASSET_URL_REGEX = "/" * ASSET_ROUTE_REGEX * "/" * UNIQUE_FILE_KEY_REGEX
+const ASSET_URL_REGEX =
+    "/" * ASSET_ROUTE_REGEX * "/" * UNIQUE_FILE_KEY_REGEX * r"(/?(?:\d+){20})?"
 const HTTP_ASSET_ROUTE_KEY = "/assets/" * UNIQUE_FILE_KEY_REGEX
 
 HTTPAssetServer() = HTTPAssetServer(get_server())
@@ -65,15 +66,13 @@ function Base.close(server::ChildAssetServer)
     end
 end
 
-serving_target(path::Path) = path
-serving_target(path::AbstractString) = normpath(abspath(expanduser(path)))
-serving_target(asset::Asset) = serving_target(local_path(asset))
-serving_target(asset::AbstractAsset) = asset
-
 function refs_and_url(server, asset::AbstractAsset)
     key = "/assets/" * unique_file_key(asset)
-    refs, target = get!(server.registered_files, key) do
-        return (Set{UInt}(), serving_target(asset))
+    refs, _ = get!(server.registered_files, key) do
+        return (Set{UInt}(), asset)
+    end
+    if asset isa Asset && asset.es6module
+        key = key * "?" * asset.content_hash[]
     end
     return refs, HTTPServer.online_url(server.server, key)
 end
@@ -99,9 +98,9 @@ function js_to_local_url(server::HTTPAssetServer, url::AbstractString)
     if isnothing(m) || isempty(m)
         return url
     else
-        key = m[1]
-        refs, path = server.registered_files[string(key)]
-        return path * ":" * m[2]
+        key = URIs.URI(m[1]).path
+        _, asset = server.registered_files[string(key)]
+        return local_path(asset) * ":" * m[2]
     end
 end
 
@@ -113,22 +112,32 @@ function js_to_local_stacktrace(server::HTTPAssetServer, line::AbstractString)
 end
 
 function (server::HTTPAssetServer)(context)
-    path = context.request.target
+    path = URIs.URI(context.request.target).path
     rf = server.registered_files
     if haskey(rf, path)
-        refs, filepath = rf[path]
-        if filepath isa BinaryAsset
+        _, asset = rf[path]
+        if asset isa BinaryAsset
             header = ["Access-Control-Allow-Origin" => "*",
                 "Content-Type" => "application/octet-stream"]
-            return HTTP.Response(200, header, body=filepath.data)
+            return HTTP.Response(200, header; body=asset.data)
         else
-            if isfile(filepath)
+            data = nothing
+            if isempty(asset.bundle_data)
+                data = asset.bundle_data
+            else
+                if isfile(local_path(asset))
+                    data = read(local_path(asset))
+                end
+            end
+            if !isnothing(data)
                 header = ["Access-Control-Allow-Origin" => "*",
-                    "Content-Type" => file_mimetype(filepath)]
-                return HTTP.Response(200, header, body = read(filepath))
+                    "Content-Type" => file_mimetype(local_path(asset)),
+                ]
+                return HTTP.Response(200, header, body = data)
             end
         end
     end
+
     return HTTP.Response(404)
 end
 

src/deno.jl

@@ -20,20 +20,20 @@ function deno_bundle(path_to_js::AbstractString, output_file::String)
     iswriteable = filemode(output_file) & Base.S_IWUSR != 0
     # bundles shipped as part of a package end up as read only
     # So we can't overwrite them
-    isfile(output_file) && !iswriteable && return false
+    isfile(output_file) && !iswriteable && return false, "Output file is not writeable"
     Deno_jll = Deno()
     # We treat Deno as a development dependency,
     # so if deno isn't loaded, don't bundle!
-    isnothing(Deno_jll) && return false
+    isnothing(Deno_jll) && return false, "Deno not loaded"
     exe = Deno_jll.deno()
     stdout = IOBuffer()
     err = IOBuffer()
     try
         run(pipeline(`$exe bundle $(path_to_js)`; stdout=stdout, stderr=err))
     catch e
-        write(stderr, seekstart(err))
-        return false
+        err_str = String(take!(err))
+        return false, err_str
     end
     write(output_file, seekstart(stdout))
-    return true
+    return true, ""
 end

src/types.jl

@@ -38,7 +38,11 @@ struct Asset <: AbstractAsset
     # to also be able to host it locally
     online_path::String
     local_path::Union{String, Path}
-    bundle_dir::Union{String, Path}
+
+    # only used if es6module
+    bundle_file::Union{String, Path}
+    bundle_data::Vector{UInt8}
+    content_hash::RefValue{String}
 end