WIP: add LocalRemoteCard and LocalRemoteSection

docs/quickstart/usage_example/basics/configure_sec_and_page.md

@@ -46,7 +46,7 @@ There are three possible options for `theme`:
 You can check the "Theme Gallery" to see how different themes look like.
 
 
-## [Override default values with properties](@ref page_sec_properties)
+## [Override default values with properties](@id page_sec_properties)
 
 Another item that could be useful is `properties`, it is written as a dictionary-like format, and
 provides a way to override some default values. Properties can be configured at any level and it

docs/quickstart/usage_example/julia_demos/1.julia_demo.jl

@@ -69,7 +69,7 @@ img = testimage("lena")
 
 # Another extra keyword is `notebook`, it allows you to control whether you needs to generate
 # jupyter notebook `.ipynb` for the demo. The valid values are `true` and `false`. See also
-# [Override default values with properties](@id page_sec_properties) on how to disable all notebook
+# [Override default values with properties](@ref page_sec_properties) on how to disable all notebook
 # generation via the properties entry.
 
 # !!! warning

src/generate.jl

@@ -143,13 +143,24 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing;
 
     # we can directly pass it to Documenter.makedocs
     if isnothing(templates)
-        out_path = walkpage(page; flatten=false) do item
-            splitext(joinpath(relative_root, relpath(item.path, page_root)))[1] * ".md"
+        out_path = walkpage(page; flatten=false) do dir, item
+            joinpath(
+                basename(source),
+                relpath(dir, page_root),
+                splitext(basename(item))[1] * ".md"
+            )
         end
     else
+        # For themes that requires an index page
+        # This will not generate a multi-level structure in the sidebar
         out_path = joinpath(relative_root, "index.md")
     end
 
+    # Ensure every path exists before we actually do the work
+    walkpage(page) do dir, item
+        @assert isfile(item.path) || isdir(item.path)
+    end
+
     @info "SetupDemoCardsDirectory: setting up \"$(source)\" directory."
     if isdir(absolute_root)
         # a typical and probably safe case -- that we're still in docs/ folder
@@ -173,6 +184,7 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing;
     end
 
     mkpath(absolute_root)
+    local clean_up_temp_remote_files
     try
         # hard coded "covers" should be consistant to card template
         isnothing(templates) || mkpath(joinpath(absolute_root, "covers"))
@@ -181,6 +193,10 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing;
         source_files = [x.path for x in walkpage(page)[2]]
 
         # pipeline
+        # prepare remote files into current folder structure, and provide a callback
+        # to clean it up after the generation
+        page, clean_up_temp_remote_files = prepare_remote_files(page)
+
         # for themeless version, we don't need to generate covers and index page
         copy_assets(absolute_root, page)
         # WARNING: julia cards are reconfigured here
@@ -198,6 +214,9 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing;
             @info "Redirect page URL: redirect docs-edit-link for demos in \"$(source)\" directory."
             isnothing(templates) || push!(source_files, joinpath(page_root, "index.md"))
             foreach(source_files) do source_file
+                # LocalRemoteCard is a virtual placeholder and does not exist here
+                isfile(source_file) && return
+                # only redirect to "real" files
                 redirect_link(source_file, source, root, src, build, edit_branch)
             end
 
@@ -229,6 +248,8 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing;
         rm(absolute_root; force=true, recursive=true)
         @error "Errors when building demo dir" pwd=pwd() source root src
         rethrow(err)
+    finally
+        clean_up_temp_remote_files()
     end
 end
 
@@ -249,18 +270,18 @@ end
 function generate(cards::AbstractVector{<:AbstractDemoCard}, template; properties=Dict{String, Any}())
     # for those hidden cards, only generate the necessary assets and files, but don't add them into
     # the index.md page
-    foreach(filter(x->x.hidden, cards)) do x
+    foreach(filter(ishidden, cards)) do x
         generate(x, template; properties=properties)
     end
 
-    mapreduce(*, filter(x->!x.hidden, cards); init="") do x
+    mapreduce(*, filter(x->!ishidden(x), cards); init="") do x
         generate(x, template; properties=properties)
     end
 end
 
 function generate(secs::AbstractVector{DemoSection}, templates; level=1, properties=Dict{String, Any}())
     mapreduce(*, secs; init="") do x
-        properties = merge(properties, x.properties) # sec.properties has higher priority
+        properties = merge(properties, x.properties)
         generate(x, templates; level=level, properties=properties)
     end
 end
@@ -323,7 +344,7 @@ function save_cover(path::String, sec::DemoSection)
 end
 
 """
-    save_cover(path::String, card::AbstractDemoCard)
+    save_cover(path::String, card)
 
 process the cover image and save it.
 """
@@ -358,7 +379,7 @@ function save_cover(path::String, card::AbstractDemoCard)
     end
 end
 
-function get_covername(card::AbstractDemoCard)
+function get_covername(card)
     isnothing(card.cover) && return nothing
     is_remote_url(card.cover) && return card.cover
 
@@ -410,6 +431,50 @@ function _copy_assets(dest_root::String, src_root::String)
     end
 end
 
+### prepare_remote_files
+
+function prepare_remote_files(page)
+    # 1. copy all remote files into its corresponding folders
+    # 2. record all temporarily remote files
+    # 3. rebuild the demo page in no-remote mode
+    temp_entry_list = []
+    for (root, dirs, files) in walkdir(page.root)
+        config_filename in files || continue
+
+        config_path = joinpath(root, config_filename)
+        config = JSON.parsefile(config_path)
+
+        if haskey(config, "remote")
+            remotes = config["remote"]
+            for (dst_entry_name, src_entry_path) in remotes
+                dst_entry_path = joinpath(root, dst_entry_name)
+                src_entry_path = normpath(root, src_entry_path)
+                if !ispath(src_entry_path)
+                    @warn "File/folder doesn't exist, skip it." path=src_entry_path
+                    continue
+                end
+                if ispath(dst_entry_path)
+                    @warn "A file/folder already exists for remote path $dst_entry_name, skip it." path=dst_entry_path
+                    continue
+                end
+
+                cp(src_entry_path, dst_entry_path)
+                push!(temp_entry_list, dst_entry_path)
+            end
+        end
+    end
+
+    page = isempty(temp_entry_list) ? page : DemoPage(page.root; ignore_remote=true)
+    function clean_up_temp_remote_files()
+        Base.Sys.iswindows() && GC.gc()
+        foreach(temp_entry_list) do x
+            rm(x; force=true, recursive=true)
+        end
+    end
+    return page, clean_up_temp_remote_files
+end
+
+
 ### postprocess
 
 """

src/preview.jl

@@ -54,7 +54,7 @@ function preview_demos(demo_path::String;
     end
 
     page_dir = generate_or_copy_pagedir(demo_path, build_dir)
-    copy_assets_and_configs(page_dir, build_dir)
+    copy_assets(page_dir, build_dir)
 
     cd(build_dir) do
         page = @suppress_err DemoPage(page_dir)
@@ -158,9 +158,12 @@ function generate_or_copy_pagedir(src_path, build_dir)
         end
         mkpath(dirname(dst_sec_dir))
         cp(sec_dir, dst_sec_dir; force=true)
+        config_rewrite(sec_dir, dst_sec_dir)
     elseif is_demopage(src_path)
         page_dir = src_path
-        cp(page_dir, abspath(build_dir, basename(page_dir)); force=true)
+        dst_page_dir = abspath(build_dir, basename(page_dir))
+        cp(page_dir, dst_page_dir; force=true)
+        config_rewrite(page_dir, dst_page_dir)
     else
         throw(ArgumentError("failed to parse demo page structure from path: $src_path. There might be some invalid demo files."))
     end
@@ -169,14 +172,12 @@ function generate_or_copy_pagedir(src_path, build_dir)
 end
 
 """
-    copy_assets_and_configs(src_page_dir, dst_build_dir=pwd())
+    copy_assets(src_page_dir, dst_build_dir=pwd())
 
-copy only assets, configs and templates from `src_page_dir` to `dst_build_dir`. The folder structure
+copy only assets and templates from `src_page_dir` to `dst_build_dir`. The folder structure
 is preserved under `dst_build_dir/\$(basename(src_page_dir))`
-
-`order` in config files are modified accordingly.
 """
-function copy_assets_and_configs(src_page_dir, dst_build_dir=pwd())
+function copy_assets(src_page_dir, dst_build_dir=pwd())
     (isabspath(src_page_dir) && isdir(src_page_dir)) || throw(ArgumentError("src_page_dir is expected to be absolute folder path: $src_page_dir"))
 
     for (root, dirs, files) in walkdir(src_page_dir)
@@ -207,32 +208,40 @@ function copy_assets_and_configs(src_page_dir, dst_build_dir=pwd())
             ispath(dst_template_path) || cp(src_template_path, dst_template_path)
         end
     end
+end
 
-    # modify and copy config.json after the folder structure is already set up
-    for (root, dirs, files) in walkdir(src_page_dir)
-        if config_filename in files
-            src_config_path = abspath(root, config_filename)
-            dst_config_path = abspath(dst_build_dir, relpath(src_config_path, dirname(src_page_dir)))
-            mkpath(dirname(dst_config_path))
-
-            config = JSON.parsefile(src_config_path)
-            config_dir = dirname(dst_config_path)
-            if haskey(config, "order")
-                order =  [x for x in config["order"] if x in readdir(config_dir)]
-                if isempty(order)
-                    delete!(config, "order")
-                else
-                    config["order"] = order
-                end
-            end
+function config_rewrite(src_dir, dst_dir)
+    for (dst_root, dirs, files) in walkdir(dst_dir)
+        src_root = joinpath(dirname(src_dir), relpath(dst_root, dirname(dst_dir)))
+
+        config_filename in files || continue
 
-            if !isempty(config)
-                # Ref: https://discourse.julialang.org/t/find-what-has-locked-held-a-file/23278/2
-                Base.Sys.iswindows() && GC.gc()
-                open(dst_config_path, "w") do io
-                    JSON.print(io, config)
+        dst_config_path = joinpath(dst_root, config_filename)
+        config = JSON.parsefile(dst_config_path)
+
+        if haskey(config, "remote")
+            remotes = config["remote"]
+            for (dst_entry_name, src_entry_path) in remotes
+                src_entry_path = isabspath(src_entry_path) ? src_entry_path : normpath(src_root, src_entry_path)
+                if !ispath(src_entry_path)
+                    @warn "File/folder doesn't exist, skip it." path=src_entry_path
+                    continue
+                end
+
+                dst_entry_path = joinpath(dst_root, dst_entry_name)
+                if ispath(dst_entry_path)
+                    @warn "A file/folder already exists for remote path $dst_entry_name, skip it." path=dst_entry_path
+                    continue
                 end
+                cp(src_entry_path, dst_entry_path)
             end
+            delete!(config, "remote")
+        end
+
+        # Ref: https://discourse.julialang.org/t/find-what-has-locked-held-a-file/23278/2
+        Base.Sys.iswindows() && GC.gc()
+        open(dst_config_path, "w") do io
+            JSON.print(io, config)
         end
     end
 end

src/show.jl

@@ -3,26 +3,27 @@ import Base: show
 const indent_spaces = "  "
 
 function show(io::IO, card::AbstractDemoCard)
-    println(io, basename(card))
+    print(io, basename(card))
 end
 
-function show(io::IO, sec::DemoSection; level=1)
+function show(io::IO, sec::AbstractDemoSection; level=1)
     print(io, repeat(indent_spaces, level-1))
-    println(io, repeat("#", level), " ", sec.title)
+    println(io, repeat("#", level), " ", compact_title(sec))
 
     # a section either holds cards or subsections
     # here it's a mere coincident to show cards first
-    foreach(sec.cards) do x
+    foreach(cards(sec)) do x
         print(io, repeat(indent_spaces, level))
         show(io, x)
+        println(io)
     end
 
-    foreach(x->show(io, x; level=level+1), sec.subsections)
+    foreach(x->show(io, x; level=level+1), subsections(sec))
 end
 
 function show(io::IO, page::DemoPage)
     page_root = replace(page.root, "\\" => "/")
     println(io, "DemoPage(\"", page_root, "\"):\n")
     println(io, "# ", page.title)
-    foreach(x->show(io, x; level=2), page.sections)
+    foreach(x->show(io, x; level=2), subsections(page))
 end

src/types/card.jl

@@ -1,5 +1,12 @@
 abstract type AbstractDemoCard end
 
+"""
+    UnmatchedCard(path)
+
+A dummy placeholder card for files that DemoCards doesn't support
+yet. No operation will be applied on this type of file, except for
+warnings.
+"""
 struct UnmatchedCard <: AbstractDemoCard
     path::String
 end
@@ -18,7 +25,6 @@ to your demofile. Currently supported types are:
 
 """
 function democard(path::String)::AbstractDemoCard
-    validate_file(path)
     _, ext = splitext(path)
     if ext in markdown_exts
         return MarkdownDemoCard(path)
@@ -28,6 +34,9 @@ function democard(path::String)::AbstractDemoCard
         return UnmatchedCard(path)
     end
 end
+function democard((name, path)::Pair{String, String})
+    LocalRemoteCard(name, path, democard(path))
+end
 
 basename(x::AbstractDemoCard) = basename(x.path)
 
@@ -46,8 +55,8 @@ end
 
 function is_democard(file)
     try
-        @suppress_err democard(file)
-        return true
+        card = @suppress_err democard(file)
+        return !isa(card, UnmatchedCard)
     catch err
         @debug err
         return false
@@ -122,3 +131,4 @@ end
 
 include("markdown.jl")
 include("julia.jl")
+include("remote_card.jl")

src/types/julia.jl

@@ -76,6 +76,21 @@ mutable struct JuliaDemoCard <: AbstractDemoCard
     julia::VersionNumber
     hidden::Bool
     notebook::Union{Nothing, Bool}
+    function JuliaDemoCard(
+        path::String,
+        cover::Union{String, Nothing},
+        id::String,
+        title::String,
+        description::String,
+        author::String,
+        date::DateTime,
+        julia::VersionNumber,
+        hidden::Bool,
+        notebook::Union{Nothing, Bool}
+    )
+        isfile(path) || throw(ArgumentError("$(path) is not a valid file"))
+        new(path, cover, id, title, description, author, date, julia, hidden, notebook)
+    end
 end
 
 function JuliaDemoCard(path::String)::JuliaDemoCard
@@ -274,3 +289,5 @@ function make_badges(card::JuliaDemoCard; src, card_dir, nbviewer_root_url, proj
 
     join(badges, " ")
 end
+
+ishidden(x::JuliaDemoCard) = x.hidden