Skip to content

Commit

Permalink
Add geojson example
Browse files Browse the repository at this point in the history
  • Loading branch information
patricoferris committed Jul 6, 2024
1 parent d78e1fc commit 0cb9757
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 21 deletions.
1 change: 1 addition & 0 deletions specs/shark.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ using that environment.

```shark-run:gdal-env
$ gdalinfo --version > /data/gdal.version
$ curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson
```

Shark keeps track of inputs and outputs. In the next code block, Shark knows to wire
Expand Down
13 changes: 7 additions & 6 deletions specs/shark.out.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ built and can be referenced as the context for future `shark-run` blocks.

## Shark Build

```shark-build:gdal-env:ec610a45b8d858c2eba37fd40dd1764890828557c1c43fa84ec88c7fcdc087c1
```shark-build:gdal-env
((from osgeo/gdal:ubuntu-small-3.6.3)
(run (shell "mkdir -p /data && echo 'Something for the log!'")))
```
Expand All @@ -18,15 +18,16 @@ using that environment.

## Shark Run

```shark-run:gdal-env:1dd3d7fdb8f1f485dd5aa0d5f383209a60aca98e67552d03a54c99be8b610eca
$ gdalinfo --version > /data/gdal.version
```shark-run:gdal-env:bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b
gdalinfo --version > /data/gdal.version
curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson
```

Shark keeps track of inputs and outputs. In the next code block, Shark knows to wire
up `/data/gdal.version` into the container.

```shark-run:gdal-env:e02469d800253ccf95e53b583e4a91465375a4e41479a67408331ecdeedb713e
$ cat /data/gdal.version
```shark-run:gdal-env:a4438aaea4711bfb05d666f91bb6fa8ba69d0bcd3e4398027538b3346855273b
cat /shark/bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b/gdal.version
GDAL 3.6.3, released 2023/03/07
```
Expand All @@ -38,5 +39,5 @@ this will publish to a `_shark` directory in the current working directory. Use
conventions to export data blobs.

```shark-publish
/data/gdal.version
/obuilder-zfs/result/bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b/.zfs/snapshot/snap/rootfs/data/gdal.version
```
7 changes: 1 addition & 6 deletions src/lib/server/build.ml
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,7 @@ let render (Obuilder.Store_spec.Store ((module Store), store)) id =
find_paths (fun p -> Filename.extension p = ".json") src_manifest
in
let geojsons =
List.map Uri.pct_encode geojsons
|> List.map (fun v ->
"." / id
/ (Uri.with_query (Uri.of_string "serve")
[ ("file", [ v ]) ]
|> Uri.to_string))
List.map (fun g -> Fmt.str "/file/%s/%s" id g) geojsons
in
let jsons =
List.map
Expand Down
2 changes: 1 addition & 1 deletion src/lib/server/dune
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@
(library
(name shark_server)
(public_name shark.server)
(libraries shark cohttp-eio htmlit routes csv))
(libraries shark cohttp-eio htmlit routes csv magic-mime))
4 changes: 2 additions & 2 deletions src/lib/server/pages.ml
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ let map geojsons =
if (i == 0) { map.setView(k.getBounds().getCenter(), 6) };
var obj = {};
// Extracting filepath from query params of URL
var file = new URLSearchParams(url.split("?")[1]).get("file").split("%2F")[1]
var file = url.split("/")[url.split("/").length - 1]
obj[file] = k;
return obj;
})|}
Expand Down Expand Up @@ -262,7 +262,7 @@ let build ?inputs ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = [])
feasible on relatively small datasets.";
];
El.pre
~at:[ At.style "height:300px; overflow-y:scroll" ]
~at:[ At.style "max-height:300px; overflow-y:scroll" ]
[ El.code [ El.txt manifest ] ];
(match geojsons with
| [] -> El.void
Expand Down
26 changes: 20 additions & 6 deletions src/lib/server/shark_server.ml
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,12 @@ let with_file f fn =
Lwt_unix.openfile f [ Unix.O_RDWR; Unix.O_CREAT ] 0o644 >>= fun fd ->
Lwt.finalize (fun () -> fn (f, fd)) (fun () -> Lwt_unix.close fd)

let respond_file ~fs path =
let mime = Magic_mime.lookup path in
let headers = Http.Header.add_opt_unless_exists None "content-type" mime in
let body = Cohttp_eio.Body.of_string Eio.Path.(load (fs / path)) in
Cohttp_eio.Server.respond ~status:`OK ~headers ~body ()

let download ~fs (Obuilder.Store_spec.Store ((module Store), store)) id =
match Lwt_eio.run_lwt @@ fun () -> Store.result store id with
| None -> Cohttp_eio.Server.respond_string ~status:`Not_found ~body:"" ()
Expand All @@ -471,14 +477,17 @@ let download ~fs (Obuilder.Store_spec.Store ((module Store), store)) id =
~src_manifest:[ src_manifest ] ~dst_dir:"" ~to_untar:fd
~user:(`Unix Obuilder_spec.{ uid = 1000; gid = 1000 })
in
let headers =
Http.Header.add_opt_unless_exists None "content-type"
"application/zip"
in
(* Some respond_file API would be nice here *)
let () = Lwt_eio.run_lwt tar in
let body = Cohttp_eio.Body.of_string Eio.Path.(load (fs / fname)) in
Cohttp_eio.Server.respond ~status:`OK ~headers ~body ())
respond_file ~fs fname)

let serve_file ~fs (Obuilder.Store_spec.Store ((module Store), store)) id path =
match Lwt_eio.run_lwt @@ fun () -> Store.result store id with
| None -> Cohttp_eio.Server.respond_string ~status:`Not_found ~body:"" ()
| Some src_dir ->
let src_dir = Filename.concat src_dir "rootfs" in
let path = Filename.concat src_dir path in
respond_file ~fs path

let edit_routes ~proc md_file (_conn : Cohttp_eio.Server.conn) request body =
let open Routes in
Expand All @@ -496,6 +505,11 @@ let router ~proc ~fs ~store md_file (conn : Cohttp_eio.Server.conn) request body
route nil (serve md_file);
route (s "logs" / str /? nil) (serve_logs fs store);
route (s "data" / str /? nil) (serve_data store);
route
(s "file" / str /? wildcard)
(fun id path ->
let dir = Parts.wildcard_match path in
serve_file ~fs store id dir);
route (s "download" / str /? nil) (download ~fs store);
route
(s "files" / str /? nil)
Expand Down

0 comments on commit 0cb9757

Please sign in to comment.