diff --git a/.eslintrc.json b/.eslintrc.json
index 742ad18da..6229eba8c 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -24,8 +24,8 @@
"@typescript-eslint/no-unused-vars": ["error", {"ignoreRestSiblings": true}],
"import/first": "warn",
"import/newline-after-import": "warn",
+ "import/no-duplicates": "off",
"import/no-import-module-exports": "warn",
- "import/no-namespace": "error",
"import/no-relative-packages": "warn",
"import/no-unresolved": "off",
"import/no-unused-modules": "error",
diff --git a/docs/.observablehq/config.ts b/docs/.observablehq/config.ts
index cc4379370..1b56a900b 100644
--- a/docs/.observablehq/config.ts
+++ b/docs/.observablehq/config.ts
@@ -2,9 +2,10 @@ export default {
title: "Observable CLI",
pages: [
{name: "Getting started", path: "/getting-started"},
- {name: "Markdown", path: "/markdown"},
{name: "JavaScript", path: "/javascript"},
{name: "Data loaders", path: "/loaders"},
+ {name: "Markdown", path: "/markdown"},
+ {name: "HTML", path: "/html"},
{
name: "JavaScript",
open: false,
@@ -17,17 +18,26 @@ export default {
{name: "Mutables", path: "/javascript/mutables"},
{name: "Imports", path: "/javascript/imports"},
{name: "Inputs", path: "/javascript/inputs"},
- {name: "DuckDB", path: "/javascript/duckdb"}
]
},
{
- name: "Features",
+ name: "Libraries",
open: false,
pages: [
- {name: "HTML", path: "/html"},
- {name: "DOT", path: "/dot"},
- {name: "Mermaid", path: "/mermaid"},
- {name: "TeX", path: "/tex"}
+ {name: "Arquero", path: "/lib/arquero"},
+ {name: "Apache Arrow", path: "/lib/arrow"},
+ {name: "D3", path: "/lib/d3"},
+ {name: "DOT (Graphviz)", path: "/lib/dot"},
+ {name: "DuckDB", path: "/lib/duckdb"},
+ {name: "Hypertext Literal", path: "/lib/htl"},
+ {name: "Leaflet", path: "/lib/leaflet"},
+ {name: "Lodash", path: "/lib/lodash"},
+ {name: "Mermaid", path: "/lib/mermaid"},
+ {name: "Observable Inputs", path: "/lib/inputs"},
+ {name: "Observable Plot", path: "/lib/plot"},
+ {name: "SQLite", path: "/lib/sqlite"},
+ {name: "TeX", path: "/lib/tex"},
+ {name: "TopoJSON", path: "/lib/topojson"}
]
},
{name: "Contributing", path: "/contributing"}
diff --git a/docs/javascript/inputs.md b/docs/javascript/inputs.md
index 70f0bf49e..1eaeb7411 100644
--- a/docs/javascript/inputs.md
+++ b/docs/javascript/inputs.md
@@ -17,223 +17,3 @@ The current gain is ${gain.toFixed(1)}!
## view(*input*)
As described above, this function [displays](./display) the given input element and then returns its corresponding [generator](./generators) via `Generators.input`. Use this to display an input element while also exposing the input’s value as a [reactive top-level variable](./reactivity).
-
-## Built-in inputs
-
-These basic inputs will get you started.
-
-* [Button](#button) - do something when a button is clicked
-* [Toggle](#toggle) - toggle between two values (on or off)
-* [Checkbox](#checkbox) - choose any from a set
-* [Radio](#radio) - choose one from a set
-* [Range](#range) or [Number](https://observablehq.com/@observablehq/input-range) - choose a number in a range (slider)
-* [Select](#select) - choose one or any from a set (drop-down menu)
-* [Text](#text) - enter freeform single-line text
-* [Textarea](#textarea) - enter freeform multi-line text
-* [Date](#date) or [Datetime](https://observablehq.com/@observablehq/input-date) - choose a date
-* [Color](#color) - choose a color
-* [File](#file) - choose a local file
-
-These fancy inputs are designed to work with tabular data such as CSV or TSV [file attachments](./files).
-
-* [Search](#search) - query a tabular dataset
-* [Table](#table) - browse a tabular dataset
-
----
-
-### Button
-
-Do something when a button is clicked. [Examples ›](https://observablehq.com/@observablehq/input-button) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#button)
-
-```js echo
-const clicks = view(Inputs.button("Click me"));
-```
-
-```js
-clicks
-```
-
----
-
-### Toggle
-
-Toggle between two values (on or off). [Examples ›](https://observablehq.com/@observablehq/input-toggle) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#toggle)
-
-```js echo
-const mute = view(Inputs.toggle({label: "Mute"}));
-```
-
-```js
-mute
-```
-
----
-
-### Checkbox
-
-Choose any from a set. [Examples ›](https://observablehq.com/@observablehq/input-checkbox) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#checkbox)
-
-```js echo
-const flavors = view(Inputs.checkbox(["salty", "sweet", "bitter", "sour", "umami"], {label: "Flavors"}));
-```
-
-```js
-flavors
-```
-
----
-
-### Radio
-
-Choose one from a set. [Examples ›](https://observablehq.com/@observablehq/input-radio) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#radio)
-
-```js echo
-const flavor = view(Inputs.radio(["salty", "sweet", "bitter", "sour", "umami"], {label: "Flavor"}));
-```
-
-```js
-flavor
-```
-
----
-
-### Range
-
-Pick a number. [Examples ›](https://observablehq.com/@observablehq/input-range) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#range)
-
-```js echo
-const n = view(Inputs.range([0, 255], {step: 1, label: "Favorite number"}));
-```
-
-```js
-n
-```
-
----
-
-### Select
-
-Choose one, or any, from a menu. [Examples ›](https://observablehq.com/@observablehq/input-select) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#select)
-
-```js
-const capitals = FileAttachment("us-state-capitals.tsv").tsv({typed: true});
-const stateNames = capitals.then((capitals) => capitals.map(d => d.State));
-```
-
-```js echo
-const homeState = view(Inputs.select([null].concat(stateNames), {label: "Home state"}));
-```
-
-```js
-homeState
-```
-
-```js echo
-const visitedStates = view(Inputs.select(stateNames, {label: "Visited states", multiple: true}));
-```
-
-```js
-visitedStates
-```
-
----
-
-### Text
-
-Enter freeform single-line text. [Examples ›](https://observablehq.com/@observablehq/input-text) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#text)
-
-```js echo
-const name = view(Inputs.text({label: "Name", placeholder: "What’s your name?"}));
-```
-
-```js
-name
-```
-
----
-
-### Textarea
-
-Enter freeform multi-line text. [Examples ›](https://observablehq.com/@observablehq/input-textarea) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#textarea)
-
-```js echo
-const bio = view(Inputs.textarea({label: "Biography", placeholder: "What’s your story?"}));
-```
-
-```js
-bio
-```
-
----
-
-### Date
-
-Choose a date, or a date and time. [Examples ›](https://observablehq.com/@observablehq/input-date) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#date)
-
-```js echo
-const birthday = view(Inputs.date({label: "Birthday"}));
-```
-
-```js
-birthday
-```
-
----
-
-### Color
-
-Choose a color. [Examples ›](https://observablehq.com/@observablehq/input-color) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#color)
-
-```js echo
-const color = view(Inputs.color({label: "Favorite color", value: "#4682b4"}));
-```
-
-```js
-color
-```
-
----
-
-### File
-
-Choose a local file. [Examples ›](https://observablehq.com/@observablehq/input-file) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#file)
-
-```js echo
-const file = view(Inputs.file({label: "CSV file", accept: ".csv", required: true}));
-```
-
-```js
-data = file.csv({typed: true})
-```
-
----
-
-### Search
-
-Query a tabular dataset. [Examples ›](https://observablehq.com/@observablehq/input-search) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#search)
-
-```js echo
-const search = view(Inputs.search(capitals, {placeholder: "Search U.S. capitals"}));
-```
-
-```js
-search // see table below!
-```
-
----
-
-### Table
-
-Browse a tabular dataset. [Examples ›](https://observablehq.com/@observablehq/input-table) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#table)
-
-```js echo
-const rows = view(Inputs.table(search));
-```
-
-```js
-rows // click a checkbox in the leftmost column
-```
-
----
-
-* [Form](https://observablehq.com/@observablehq/input-form?collection=@observablehq/inputs) - combine multiple inputs for a compact display
diff --git a/docs/lib/arquero.md b/docs/lib/arquero.md
new file mode 100644
index 000000000..94b92a5c0
--- /dev/null
+++ b/docs/lib/arquero.md
@@ -0,0 +1,5 @@
+# Arquero
+
+```js echo
+aq
+```
diff --git a/docs/lib/arrow.md b/docs/lib/arrow.md
new file mode 100644
index 000000000..6629d2850
--- /dev/null
+++ b/docs/lib/arrow.md
@@ -0,0 +1,5 @@
+# Apache Arrow
+
+```js echo
+Arrow
+```
diff --git a/docs/lib/chinook.db b/docs/lib/chinook.db
new file mode 100644
index 000000000..38a98b391
Binary files /dev/null and b/docs/lib/chinook.db differ
diff --git a/docs/lib/d3.md b/docs/lib/d3.md
new file mode 100644
index 000000000..0d914313a
--- /dev/null
+++ b/docs/lib/d3.md
@@ -0,0 +1,5 @@
+# D3
+
+```js echo
+d3
+```
diff --git a/docs/dot.md b/docs/lib/dot.md
similarity index 100%
rename from docs/dot.md
rename to docs/lib/dot.md
diff --git a/docs/javascript/duckdb.md b/docs/lib/duckdb.md
similarity index 87%
rename from docs/javascript/duckdb.md
rename to docs/lib/duckdb.md
index aa9c47071..b1495081e 100644
--- a/docs/javascript/duckdb.md
+++ b/docs/lib/duckdb.md
@@ -1,6 +1,10 @@
# DuckDB
-Observable Markdown has built-in support for DuckDB via [duckdb-wasm](https://github.com/duckdb/duckdb-wasm). It’s easiest to use in conjunction with [`FileAttachment`](./files). Declare a database with `DuckDBClient`, passing in a set of named tables:
+Observable Markdown has built-in support for DuckDB via [duckdb-wasm](https://github.com/duckdb/duckdb-wasm). It’s easiest to use in conjunction with [`FileAttachment`](../javascript/files). Declare a database with `DuckDBClient`, passing in a set of named tables:
+
+```js echo
+duckdb
+```
```js echo
const db = DuckDBClient.of({gaia: FileAttachment("gaia-sample.parquet")});
diff --git a/docs/javascript/gaia-sample.parquet b/docs/lib/gaia-sample.parquet
similarity index 100%
rename from docs/javascript/gaia-sample.parquet
rename to docs/lib/gaia-sample.parquet
diff --git a/docs/lib/htl.md b/docs/lib/htl.md
new file mode 100644
index 000000000..8d173d023
--- /dev/null
+++ b/docs/lib/htl.md
@@ -0,0 +1,13 @@
+# Hypertext Literal
+
+```js echo
+htl
+```
+
+```js echo
+html
+```
+
+```js echo
+svg
+```
diff --git a/docs/lib/inputs.md b/docs/lib/inputs.md
new file mode 100644
index 000000000..abc01638c
--- /dev/null
+++ b/docs/lib/inputs.md
@@ -0,0 +1,219 @@
+# Observable Inputs
+
+These basic inputs will get you started.
+
+* [Button](#button) - do something when a button is clicked
+* [Toggle](#toggle) - toggle between two values (on or off)
+* [Checkbox](#checkbox) - choose any from a set
+* [Radio](#radio) - choose one from a set
+* [Range](#range) or [Number](https://observablehq.com/@observablehq/input-range) - choose a number in a range (slider)
+* [Select](#select) - choose one or any from a set (drop-down menu)
+* [Text](#text) - enter freeform single-line text
+* [Textarea](#textarea) - enter freeform multi-line text
+* [Date](#date) or [Datetime](https://observablehq.com/@observablehq/input-date) - choose a date
+* [Color](#color) - choose a color
+* [File](#file) - choose a local file
+
+These fancy inputs are designed to work with tabular data such as CSV or TSV [file attachments](./files).
+
+* [Search](#search) - query a tabular dataset
+* [Table](#table) - browse a tabular dataset
+
+---
+
+### Button
+
+Do something when a button is clicked. [Examples ›](https://observablehq.com/@observablehq/input-button) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#button)
+
+```js echo
+const clicks = view(Inputs.button("Click me"));
+```
+
+```js
+clicks
+```
+
+---
+
+### Toggle
+
+Toggle between two values (on or off). [Examples ›](https://observablehq.com/@observablehq/input-toggle) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#toggle)
+
+```js echo
+const mute = view(Inputs.toggle({label: "Mute"}));
+```
+
+```js
+mute
+```
+
+---
+
+### Checkbox
+
+Choose any from a set. [Examples ›](https://observablehq.com/@observablehq/input-checkbox) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#checkbox)
+
+```js echo
+const flavors = view(Inputs.checkbox(["salty", "sweet", "bitter", "sour", "umami"], {label: "Flavors"}));
+```
+
+```js
+flavors
+```
+
+---
+
+### Radio
+
+Choose one from a set. [Examples ›](https://observablehq.com/@observablehq/input-radio) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#radio)
+
+```js echo
+const flavor = view(Inputs.radio(["salty", "sweet", "bitter", "sour", "umami"], {label: "Flavor"}));
+```
+
+```js
+flavor
+```
+
+---
+
+### Range
+
+Pick a number. [Examples ›](https://observablehq.com/@observablehq/input-range) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#range)
+
+```js echo
+const n = view(Inputs.range([0, 255], {step: 1, label: "Favorite number"}));
+```
+
+```js
+n
+```
+
+---
+
+### Select
+
+Choose one, or any, from a menu. [Examples ›](https://observablehq.com/@observablehq/input-select) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#select)
+
+```js
+const capitals = FileAttachment("us-state-capitals.tsv").tsv({typed: true});
+const stateNames = capitals.then((capitals) => capitals.map(d => d.State));
+```
+
+```js echo
+const homeState = view(Inputs.select([null].concat(stateNames), {label: "Home state"}));
+```
+
+```js
+homeState
+```
+
+```js echo
+const visitedStates = view(Inputs.select(stateNames, {label: "Visited states", multiple: true}));
+```
+
+```js
+visitedStates
+```
+
+---
+
+### Text
+
+Enter freeform single-line text. [Examples ›](https://observablehq.com/@observablehq/input-text) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#text)
+
+```js echo
+const name = view(Inputs.text({label: "Name", placeholder: "What’s your name?"}));
+```
+
+```js
+name
+```
+
+---
+
+### Textarea
+
+Enter freeform multi-line text. [Examples ›](https://observablehq.com/@observablehq/input-textarea) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#textarea)
+
+```js echo
+const bio = view(Inputs.textarea({label: "Biography", placeholder: "What’s your story?"}));
+```
+
+```js
+bio
+```
+
+---
+
+### Date
+
+Choose a date, or a date and time. [Examples ›](https://observablehq.com/@observablehq/input-date) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#date)
+
+```js echo
+const birthday = view(Inputs.date({label: "Birthday"}));
+```
+
+```js
+birthday
+```
+
+---
+
+### Color
+
+Choose a color. [Examples ›](https://observablehq.com/@observablehq/input-color) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#color)
+
+```js echo
+const color = view(Inputs.color({label: "Favorite color", value: "#4682b4"}));
+```
+
+```js
+color
+```
+
+---
+
+### File
+
+Choose a local file. [Examples ›](https://observablehq.com/@observablehq/input-file) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#file)
+
+```js echo
+const file = view(Inputs.file({label: "CSV file", accept: ".csv", required: true}));
+```
+
+```js
+data = file.csv({typed: true})
+```
+
+---
+
+### Search
+
+Query a tabular dataset. [Examples ›](https://observablehq.com/@observablehq/input-search) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#search)
+
+```js echo
+const search = view(Inputs.search(capitals, {placeholder: "Search U.S. capitals"}));
+```
+
+```js
+search // see table below!
+```
+
+---
+
+### Table
+
+Browse a tabular dataset. [Examples ›](https://observablehq.com/@observablehq/input-table) [API Reference ›](https://github.com/observablehq/inputs/blob/main/README.md#table)
+
+```js echo
+const rows = view(Inputs.table(search));
+```
+
+```js
+rows // click a checkbox in the leftmost column
+```
+
+---
+
+* [Form](https://observablehq.com/@observablehq/input-form?collection=@observablehq/inputs) - combine multiple inputs for a compact display
diff --git a/docs/lib/laser-report.xlsx b/docs/lib/laser-report.xlsx
new file mode 100644
index 000000000..98f8ca9e3
Binary files /dev/null and b/docs/lib/laser-report.xlsx differ
diff --git a/docs/lib/leaflet.md b/docs/lib/leaflet.md
new file mode 100644
index 000000000..157ffff97
--- /dev/null
+++ b/docs/lib/leaflet.md
@@ -0,0 +1,8 @@
+# Leaflet
+
+```js echo
+const div = Object.assign(display(document.createElement("div")), {style: "height: 400px;"});
+const map = L.map(div).setView([51.505, -0.09], 13);
+L.tileLayer("https://tile.openstreetmap.org/{z}/{x}/{y}.png").addTo(map);
+L.marker([51.5, -0.09]).addTo(map).bindPopup("A pretty CSS popup.
Easily customizable.").openPopup();
+```
diff --git a/docs/lib/lodash.md b/docs/lib/lodash.md
new file mode 100644
index 000000000..500849afe
--- /dev/null
+++ b/docs/lib/lodash.md
@@ -0,0 +1,5 @@
+# Lodash
+
+```js echo
+_
+```
diff --git a/docs/mermaid.md b/docs/lib/mermaid.md
similarity index 100%
rename from docs/mermaid.md
rename to docs/lib/mermaid.md
diff --git a/docs/lib/plot.md b/docs/lib/plot.md
new file mode 100644
index 000000000..a5b3f414f
--- /dev/null
+++ b/docs/lib/plot.md
@@ -0,0 +1,5 @@
+# Observable Plot
+
+```js echo
+Plot
+```
diff --git a/docs/lib/sqlite.md b/docs/lib/sqlite.md
new file mode 100644
index 000000000..a367dde59
--- /dev/null
+++ b/docs/lib/sqlite.md
@@ -0,0 +1,77 @@
+# SQLite
+
+[SQLite](https://sqlite.org/) is “a small, fast, self-contained, high-reliability, full-featured, SQL database engine” and “the most used database engine in the world.” We provide built-in support for [sql.js](https://sql.js.org), a WASM-based distribution of SQLite.
+
+```js echo
+import SQLite from "npm:@observablehq/sqlite";
+```
+
+We also provide a `sql` template literal implementation using `SQLiteDatabaseClient`.
+
+```js echo
+import {SQLiteDatabaseClient} from "npm:@observablehq/sqlite";
+```
+
+(Both of these are available by default in [Observable Markdown](../markdown) but are not loaded unless you reference them.)
+
+The easiest way to construct a `SQLiteDatabaseClient` is to call `file.sqlite()` to load a SQLite database file. This returns a promise. (Here we rely on [implicit await](../javascript/promises).)
+
+```js echo
+const db = FileAttachment("chinook.db").sqlite();
+```
+
+Alternatively you can use `SQLiteDatabaseClient` and pass in a string (URL), `Blob`, `ArrayBuffer`, `Uint8Array`, `FileAttachment`, or promise to the same:
+
+```js run=false
+const db = SQLiteDatabaseClient.open(FileAttachment("chinook.db"));
+```
+
+We recommend using `FileAttachment` for local files so that the files are automatically copied to `dist` during build, and so you can generate SQLite files using [data loaders](../loaders). But if you want to “hot” load a live file from an external server (not generally recommended since it’s usually slower and less reliable than a data loader), pass a string to `SQLiteDatabaseClient.open`.
+
+```js run=false
+const db = SQLiteDatabaseClient.open("https://static.observableusercontent.com/files/b3711cfd9bdf50cbe4e74751164d28e907ce366cd4bf56a39a980a48fdc5f998c42a019716a8033e2b54defdd97e4a55ebe4f6464b4f0678ea0311532605a115");
+```
+
+Once you’ve loaded your `db` you can write SQL queries.
+
+```js echo
+const customers = db.sql`SELECT * FROM customers`;
+```
+
+```js
+customers
+```
+
+This returns a promise to an array of objects; each object represents a row returned from the query.
+
+For better readability, you can display query result sets using [Inputs.table](./inputs#table).
+
+```js echo
+Inputs.table(customers)
+```
+
+For interactive or dynamic queries, you can interpolate reactive variables into SQL queries. For example, you can declare a [text input](./inputs#text) to prompt the query to enter a search term, and then interpolate the input into the query parameter.
+
+```js echo
+const name = view(Inputs.text({label: "Name", placeholder: "Search track names"}));
+```
+
+```js echo
+const tracks = db.sql`SELECT * FROM tracks WHERE Name LIKE ${`%${name}%`}`;
+```
+
+```js
+Inputs.table(tracks)
+```
+
+As an alternative to `db.sql`, you can call `db.query`.
+
+```js run=false
+await db.query(`SELECT * FROM tracks WHERE Name LIKE $1`, [`%${name}%`])
+```
+
+There’s also `db.queryRow` for just getting a single row.
+
+```js echo
+await db.queryRow(`SELECT sqlite_version()`)
+```
diff --git a/docs/tex.md b/docs/lib/tex.md
similarity index 100%
rename from docs/tex.md
rename to docs/lib/tex.md
diff --git a/docs/lib/topojson.md b/docs/lib/topojson.md
new file mode 100644
index 000000000..740839cc8
--- /dev/null
+++ b/docs/lib/topojson.md
@@ -0,0 +1,5 @@
+# TopoJSON
+
+```js echo
+topojson
+```
diff --git a/docs/javascript/us-state-capitals.tsv b/docs/lib/us-state-capitals.tsv
similarity index 100%
rename from docs/javascript/us-state-capitals.tsv
rename to docs/lib/us-state-capitals.tsv
diff --git a/docs/lib/xslx.md b/docs/lib/xslx.md
new file mode 100644
index 000000000..dfb2d99b6
--- /dev/null
+++ b/docs/lib/xslx.md
@@ -0,0 +1,28 @@
+# XLSX
+
+Let’s read a Microsoft Excel spreadsheet using [ExcelJS](https://github.com/exceljs/exceljs).
+
+```js echo
+const workbook = await FileAttachment("laser-report.xlsx").xlsx();
+const reports = workbook.sheet("Laser Report 2020", {range: "A:J", headers: true});
+```
+
+```js echo
+Inputs.table(reports)
+```
+
+```js echo
+Plot.plot({
+ y: {
+ label: "Altitude (feet, thousands)",
+ domain: [0, 100],
+ transform: (y) => y / 1000,
+ grid: true,
+ clamp: true
+ },
+ marks: [
+ Plot.ruleY([0]),
+ Plot.dot(reports, {x: "Incident Date", y: "Altitude", r: 1, stroke: "Incident Time", tip: true})
+ ]
+})
+```
diff --git a/src/build.ts b/src/build.ts
index 32bfa9dad..8021205ac 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -10,13 +10,24 @@ import {prepareOutput, visitFiles, visitMarkdownFiles} from "./files.js";
import {createImportResolver, rewriteModule} from "./javascript/imports.js";
import type {Logger, Writer} from "./logger.js";
import {renderServerless} from "./render.js";
-import {makeCLIResolver} from "./resolver.js";
import {getClientPath, rollupClient} from "./rollup.js";
import {faint} from "./tty.js";
import {resolvePath} from "./url.js";
const EXTRA_FILES = new Map([["node_modules/@observablehq/runtime/dist/runtime.js", "_observablehq/runtime.js"]]);
+// TODO Remove library helpers (e.g., duckdb) when they are published to npm.
+const CLIENT_BUNDLES: [entry: string, name: string][] = [
+ ["./src/client/index.js", "client.js"],
+ ["./src/client/stdlib.js", "stdlib.js"],
+ ["./src/client/stdlib/dot.js", "stdlib/dot.js"],
+ ["./src/client/stdlib/duckdb.js", "stdlib/duckdb.js"],
+ ["./src/client/stdlib/mermaid.js", "stdlib/mermaid.js"],
+ ["./src/client/stdlib/sqlite.js", "stdlib/sqlite.js"],
+ ["./src/client/stdlib/tex.js", "stdlib/tex.js"],
+ ["./src/client/stdlib/xslx.js", "stdlib/xslx.js"]
+];
+
export interface BuildOptions {
sourceRoot: string;
outputRoot?: string;
@@ -53,13 +64,12 @@ export async function build(
const config = await readConfig(root);
const files: string[] = [];
const imports: string[] = [];
- const resolver = await makeCLIResolver();
for await (const sourceFile of visitMarkdownFiles(root)) {
const sourcePath = join(root, sourceFile);
const outputPath = join(dirname(sourceFile), basename(sourceFile, ".md") + ".html");
effects.output.write(`${faint("render")} ${sourcePath} ${faint("→")} `);
const path = join("/", dirname(sourceFile), basename(sourceFile, ".md"));
- const render = await renderServerless(await readFile(sourcePath, "utf-8"), {root, path, resolver, ...config});
+ const render = await renderServerless(await readFile(sourcePath, "utf-8"), {root, path, ...config});
const resolveFile = ({name}) => resolvePath(sourceFile, name);
files.push(...render.files.map(resolveFile));
imports.push(...render.imports.filter((i) => i.type === "local").map(resolveFile));
@@ -67,12 +77,14 @@ export async function build(
}
if (addPublic) {
- // Generate the client bundle.
- const clientPath = getClientPath();
- const outputPath = join("_observablehq", "client.js");
- effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `);
- const code = await rollupClient(clientPath, {minify: true});
- await effects.writeFile(outputPath, code);
+ // Generate the client bundles.
+ for (const [entry, name] of CLIENT_BUNDLES) {
+ const clientPath = getClientPath(entry);
+ const outputPath = join("_observablehq", name);
+ effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `);
+ const code = await rollupClient(clientPath, {minify: true});
+ await effects.writeFile(outputPath, code);
+ }
// Copy over the public directory.
const publicRoot = relative(cwd(), join(dirname(fileURLToPath(import.meta.url)), "..", "public"));
for await (const publicFile of visitFiles(publicRoot)) {
diff --git a/src/client/dot.js b/src/client/dot.js
deleted file mode 100644
index e2efebe38..000000000
--- a/src/client/dot.js
+++ /dev/null
@@ -1,39 +0,0 @@
-export async function dot() {
- const {instance} = await import("npm:@viz-js/viz");
- const viz = await instance();
- return function dot(strings) {
- let string = strings[0] + "";
- let i = 0;
- let n = arguments.length;
- while (++i < n) string += arguments[i] + "" + strings[i];
- const svg = viz.renderSVGElement(string, {
- graphAttributes: {
- bgcolor: "none",
- color: "#00000101",
- fontcolor: "#00000101",
- fontname: "var(--sans-serif)",
- fontsize: "12"
- },
- nodeAttributes: {
- color: "#00000101",
- fontcolor: "#00000101",
- fontname: "var(--sans-serif)",
- fontsize: "12"
- },
- edgeAttributes: {
- color: "#00000101"
- }
- });
- for (const e of svg.querySelectorAll("[stroke='#000001'][stroke-opacity='0.003922']")) {
- e.setAttribute("stroke", "currentColor");
- e.removeAttribute("stroke-opacity");
- }
- for (const e of svg.querySelectorAll("[fill='#000001'][fill-opacity='0.003922']")) {
- e.setAttribute("fill", "currentColor");
- e.removeAttribute("fill-opacity");
- }
- svg.remove();
- svg.style = "max-width: 100%; height: auto;";
- return svg;
- };
-}
diff --git a/src/client/inspect.js b/src/client/inspect.js
index 81547cca2..e23080b62 100644
--- a/src/client/inspect.js
+++ b/src/client/inspect.js
@@ -1,4 +1,4 @@
-import {Inspector} from "./runtime.js";
+import {Inspector} from "observablehq:runtime";
export function inspect(value) {
const inspector = new Inspector(document.createElement("div"));
diff --git a/src/client/library.js b/src/client/library.js
deleted file mode 100644
index df683f494..000000000
--- a/src/client/library.js
+++ /dev/null
@@ -1,36 +0,0 @@
-import {dot} from "./dot.js";
-import {mermaid} from "./mermaid.js";
-import {Mutable} from "./mutable.js";
-import {Library} from "./runtime.js";
-import {tex} from "./tex.js";
-import {width} from "./width.js";
-
-export function makeLibrary() {
- const library = new Library();
- return Object.assign(library, {
- width: () => width(library),
- Mutable: () => Mutable(library),
- // Override the common recommended libraries so that if a user imports them,
- // they get the same version that the standard library provides (rather than
- // loading the library twice). Also, it’s nice to avoid require!
- d3: () => import("npm:d3"),
- htl: () => import("npm:htl"),
- html: () => import("npm:htl").then((htl) => htl.html),
- svg: () => import("npm:htl").then((htl) => htl.svg),
- Plot: () => import("npm:@observablehq/plot"),
- Inputs: () => {
- // TODO Observable Inputs needs to include the CSS in the dist folder
- // published to npm, and we should replace the __ns__ namespace with
- // oi-{hash} in the ES module distribution, somehow.
- const inputs = import("npm:@observablehq/inputs");
- const link = document.createElement("link");
- link.rel = "stylesheet";
- link.href = "https://cdn.jsdelivr.net/gh/observablehq/inputs/src/style.css";
- document.head.append(link);
- return inputs;
- },
- tex,
- dot,
- mermaid
- });
-}
diff --git a/src/client/main.js b/src/client/main.js
index c33257215..06ca98d3f 100644
--- a/src/client/main.js
+++ b/src/client/main.js
@@ -1,26 +1,24 @@
-import {makeDatabaseClient} from "./database.js";
+import {Runtime} from "observablehq:runtime";
+import {registerDatabase, registerFile} from "observablehq:stdlib";
+import {DatabaseClient, FileAttachment, Generators, Mutable, now, width} from "observablehq:stdlib";
import {inspect, inspectError} from "./inspect.js";
-import {makeLibrary} from "./library.js";
-import {Runtime} from "./runtime.js";
+import * as recommendedLibraries from "./stdlib/recommendedLibraries.js";
+import * as sampleDatasets from "./stdlib/sampleDatasets.js";
-const library = makeLibrary();
-const {Generators} = library;
-library.DatabaseClient = () => makeDatabaseClient(resolveDatabaseToken);
+const library = {
+ now,
+ width,
+ DatabaseClient: () => DatabaseClient,
+ FileAttachment: () => FileAttachment,
+ Generators: () => Generators,
+ Mutable: () => Mutable,
+ ...recommendedLibraries,
+ ...sampleDatasets
+};
const runtime = new Runtime(library);
export const main = runtime.module();
-const attachedFiles = new Map();
-const resolveFile = (name) => attachedFiles.get(name);
-main.builtin("FileAttachment", runtime.fileAttachments(resolveFile));
-
-const databaseTokens = new Map();
-async function resolveDatabaseToken(name) {
- const token = databaseTokens.get(name);
- if (!token) throw new Error(`Database configuration for ${name} not found`);
- return token;
-}
-
export const cellsById = new Map(); // TODO hide
export function define(cell) {
@@ -59,8 +57,8 @@ export function define(cell) {
v.define(outputs.length ? `cell ${id}` : null, inputs, body);
variables.push(v);
for (const o of outputs) variables.push(main.variable(true).define(o, [`cell ${id}`], (exports) => exports[o]));
- for (const f of files) attachedFiles.set(f.name, {url: f.path, mimeType: f.mimeType});
- for (const d of databases) databaseTokens.set(d.name, d);
+ for (const f of files) registerFile(f.name, {url: f.path, mimeType: f.mimeType});
+ for (const d of databases) registerDatabase(d.name, d);
}
// Note: Element.prototype is instanceof Node, but cannot be inserted!
diff --git a/src/client/mermaid.js b/src/client/mermaid.js
deleted file mode 100644
index 6f83c0271..000000000
--- a/src/client/mermaid.js
+++ /dev/null
@@ -1,11 +0,0 @@
-export async function mermaid() {
- let nextId = 0;
- const {default: mer} = await import("npm:mermaid");
- const theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "neutral";
- mer.initialize({startOnLoad: false, securityLevel: "loose", theme});
- return async function mermaid() {
- const root = document.createElement("div");
- root.innerHTML = (await mer.render(`mermaid-${++nextId}`, String.raw.apply(String, arguments))).svg;
- return root.removeChild(root.firstChild);
- };
-}
diff --git a/src/client/mutable.js b/src/client/mutable.js
deleted file mode 100644
index fb1dfa401..000000000
--- a/src/client/mutable.js
+++ /dev/null
@@ -1,20 +0,0 @@
-// Mutable returns a generator with a value getter/setting that allows the
-// generated value to be mutated. Therefore, direct mutation is only allowed
-// within the defining cell, but the cell can also export functions that allows
-// other cells to mutate the value as desired.
-export function Mutable({Generators}) {
- return function Mutable(value) {
- let change;
- return Object.defineProperty(
- Generators.observe((_) => {
- change = _;
- if (value !== undefined) change(value);
- }),
- "value",
- {
- get: () => value,
- set: (x) => void change((value = x)) // eslint-disable-line no-setter-return
- }
- );
- };
-}
diff --git a/src/client/preview.js b/src/client/preview.js
index 91cce9772..cf169555b 100644
--- a/src/client/preview.js
+++ b/src/client/preview.js
@@ -19,12 +19,13 @@ export function open({hash, eval: compile} = {}) {
location.reload();
break;
}
- case "refresh":
+ case "refresh": {
message.cellIds.forEach((id) => {
const cell = cellsById.get(id);
if (cell) define(cell.cell);
});
break;
+ }
case "update": {
const root = document.querySelector("main");
if (message.previousHash !== hash) {
@@ -92,6 +93,19 @@ export function open({hash, eval: compile} = {}) {
enableCopyButtons();
break;
}
+ case "add-stylesheet": {
+ const link = document.createElement("link");
+ link.rel = "stylesheet";
+ link.type = "text/css";
+ link.crossOrigin = "";
+ link.href = message.href;
+ document.head.appendChild(link);
+ break;
+ }
+ case "remove-stylesheet": {
+ document.head.querySelector(`link[href="${message.href}"]`)?.remove();
+ break;
+ }
}
};
diff --git a/src/client/sidebar.js b/src/client/sidebar.js
index 648a014f8..1e921579b 100644
--- a/src/client/sidebar.js
+++ b/src/client/sidebar.js
@@ -23,7 +23,7 @@ function preventDoubleClick(event) {
}
function persistOpen() {
- sessionStorage.setItem(`observablehq-sidebar:${this.firstElementChild.textContent}`, String(this.open));
+ sessionStorage.setItem(`observablehq-sidebar:${this.firstElementChild.textContent}`, this.open);
}
for (const summary of document.querySelectorAll("#observablehq-sidebar summary")) {
diff --git a/src/client/stdlib.js b/src/client/stdlib.js
new file mode 100644
index 000000000..74db28e28
--- /dev/null
+++ b/src/client/stdlib.js
@@ -0,0 +1,6 @@
+export {DatabaseClient, registerDatabase} from "./stdlib/databaseClient.js";
+export {FileAttachment, registerFile} from "./stdlib/fileAttachment.js";
+export * as Generators from "./stdlib/generators/index.js";
+export {Mutable} from "./stdlib/mutable.js";
+export {now} from "./stdlib/now.js";
+export {width} from "./stdlib/width.js";
diff --git a/src/client/database.js b/src/client/stdlib/databaseClient.js
similarity index 90%
rename from src/client/database.js
rename to src/client/stdlib/databaseClient.js
index d7ceefde4..be86eac10 100644
--- a/src/client/database.js
+++ b/src/client/stdlib/databaseClient.js
@@ -1,11 +1,18 @@
-export function makeDatabaseClient(resolveToken) {
- return function DatabaseClient(name) {
- if (new.target !== undefined) throw new TypeError("DatabaseClient is not a constructor");
- return resolveToken((name += "")).then((token) => new DatabaseClientImpl(name, token));
- };
+const databases = new Map();
+
+export function registerDatabase(name, token) {
+ if (token == null) databases.delete(name);
+ else databases.set(name, token);
+}
+
+export function DatabaseClient(name) {
+ if (new.target !== undefined) throw new TypeError("DatabaseClient is not a constructor");
+ const token = databases.get((name = `${name}`));
+ if (!token) throw new Error(`Database not found: ${name}`);
+ return new DatabaseClientImpl(name, token);
}
-class DatabaseClientImpl {
+const DatabaseClientImpl = class DatabaseClient {
#token;
constructor(name, token) {
@@ -64,7 +71,9 @@ class DatabaseClientImpl {
async sql() {
return this.query(...this.queryTag.apply(this, arguments));
}
-}
+};
+
+DatabaseClient.prototype = DatabaseClientImpl.prototype; // instanceof
function coerceBuffer(d) {
return Uint8Array.from(d.data).buffer;
diff --git a/src/client/stdlib/dot.js b/src/client/stdlib/dot.js
new file mode 100644
index 000000000..a46d26faa
--- /dev/null
+++ b/src/client/stdlib/dot.js
@@ -0,0 +1,39 @@
+import {instance} from "npm:@viz-js/viz";
+
+const viz = await instance();
+
+export default function dot(strings) {
+ let string = strings[0] + "";
+ let i = 0;
+ let n = arguments.length;
+ while (++i < n) string += arguments[i] + "" + strings[i];
+ const svg = viz.renderSVGElement(string, {
+ graphAttributes: {
+ bgcolor: "none",
+ color: "#00000101",
+ fontcolor: "#00000101",
+ fontname: "var(--sans-serif)",
+ fontsize: "12"
+ },
+ nodeAttributes: {
+ color: "#00000101",
+ fontcolor: "#00000101",
+ fontname: "var(--sans-serif)",
+ fontsize: "12"
+ },
+ edgeAttributes: {
+ color: "#00000101"
+ }
+ });
+ for (const e of svg.querySelectorAll("[stroke='#000001'][stroke-opacity='0.003922']")) {
+ e.setAttribute("stroke", "currentColor");
+ e.removeAttribute("stroke-opacity");
+ }
+ for (const e of svg.querySelectorAll("[fill='#000001'][fill-opacity='0.003922']")) {
+ e.setAttribute("fill", "currentColor");
+ e.removeAttribute("fill-opacity");
+ }
+ svg.remove();
+ svg.style = "max-width: 100%; height: auto;";
+ return svg;
+}
diff --git a/src/client/stdlib/duckdb.js b/src/client/stdlib/duckdb.js
new file mode 100644
index 000000000..ab0b531d0
--- /dev/null
+++ b/src/client/stdlib/duckdb.js
@@ -0,0 +1,382 @@
+import * as duckdb from "npm:@duckdb/duckdb-wasm";
+
+// Adapted from https://observablehq.com/@cmudig/duckdb-client
+// Copyright 2021 CMU Data Interaction Group
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const bundle = await duckdb.selectBundle({
+ mvp: {
+ mainModule: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm",
+ mainWorker: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js"
+ },
+ eh: {
+ mainModule: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-eh.wasm",
+ mainWorker: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js"
+ }
+});
+
+const logger = new duckdb.ConsoleLogger();
+
+export default class DuckDBClient {
+ constructor(db) {
+ Object.defineProperties(this, {
+ _db: {value: db}
+ });
+ }
+
+ async queryStream(query, params) {
+ const connection = await this._db.connect();
+ let reader, batch;
+ try {
+ if (params?.length > 0) {
+ const statement = await connection.prepare(query);
+ reader = await statement.send(...params);
+ } else {
+ reader = await connection.send(query);
+ }
+ batch = await reader.next();
+ if (batch.done) throw new Error("missing first batch");
+ } catch (error) {
+ await connection.close();
+ throw error;
+ }
+ return {
+ schema: getArrowTableSchema(batch.value),
+ async *readRows() {
+ try {
+ while (!batch.done) {
+ yield batch.value.toArray();
+ batch = await reader.next();
+ }
+ } finally {
+ await connection.close();
+ }
+ }
+ };
+ }
+
+ async query(query, params) {
+ const result = await this.queryStream(query, params);
+ const results = [];
+ for await (const rows of result.readRows()) {
+ for (const row of rows) {
+ results.push(row);
+ }
+ }
+ results.schema = result.schema;
+ return results;
+ }
+
+ async queryRow(query, params) {
+ const result = await this.queryStream(query, params);
+ const reader = result.readRows();
+ try {
+ const {done, value} = await reader.next();
+ return done || !value.length ? null : value[0];
+ } finally {
+ await reader.return();
+ }
+ }
+
+ async sql(strings, ...args) {
+ return await this.query(strings.join("?"), args);
+ }
+
+ queryTag(strings, ...params) {
+ return [strings.join("?"), params];
+ }
+
+ escape(name) {
+ return `"${name}"`;
+ }
+
+ async describeTables() {
+ const tables = await this.query("SHOW TABLES");
+ return tables.map(({name}) => ({name}));
+ }
+
+ async describeColumns({table} = {}) {
+ const columns = await this.query(`DESCRIBE ${this.escape(table)}`);
+ return columns.map(({column_name, column_type, null: nullable}) => ({
+ name: column_name,
+ type: getDuckDBType(column_type),
+ nullable: nullable !== "NO",
+ databaseType: column_type
+ }));
+ }
+
+ static async of(sources = {}, config = {}) {
+ const db = await createDuckDB();
+ if (config.query?.castTimestampToDate === undefined) {
+ config = {...config, query: {...config.query, castTimestampToDate: true}};
+ }
+ if (config.query?.castBigIntToDouble === undefined) {
+ config = {...config, query: {...config.query, castBigIntToDouble: true}};
+ }
+ await db.open(config);
+ await Promise.all(
+ Object.entries(sources).map(async ([name, source]) => {
+ if (isFileAttachment(source)) {
+ // bare file
+ await insertFile(db, name, source);
+ } else if (isArrowTable(source)) {
+ // bare arrow table
+ await insertArrowTable(db, name, source);
+ } else if (Array.isArray(source)) {
+ // bare array of objects
+ await insertArray(db, name, source);
+ } else if (isArqueroTable(source)) {
+ await insertArqueroTable(db, name, source);
+ } else if ("data" in source) {
+ // data + options
+ const {data, ...options} = source;
+ if (isArrowTable(data)) {
+ await insertArrowTable(db, name, data, options);
+ } else {
+ await insertArray(db, name, data, options);
+ }
+ } else if ("file" in source) {
+ // file + options
+ const {file, ...options} = source;
+ await insertFile(db, name, file, options);
+ } else {
+ throw new Error(`invalid source: ${source}`);
+ }
+ })
+ );
+ return new DuckDBClient(db);
+ }
+}
+
+Object.defineProperty(DuckDBClient.prototype, "dialect", {
+ value: "duckdb"
+});
+
+async function insertFile(database, name, file, options) {
+ const url = await file.url();
+ if (url.startsWith("blob:")) {
+ const buffer = await file.arrayBuffer();
+ await database.registerFileBuffer(file.name, new Uint8Array(buffer));
+ } else {
+ await database.registerFileURL(file.name, new URL(url, location).href, 4); // duckdb.DuckDBDataProtocol.HTTP
+ }
+ const connection = await database.connect();
+ try {
+ switch (file.mimeType) {
+ case "text/csv":
+ case "text/tab-separated-values": {
+ return await connection
+ .insertCSVFromPath(file.name, {
+ name,
+ schema: "main",
+ ...options
+ })
+ .catch(async (error) => {
+ // If initial attempt to insert CSV resulted in a conversion
+ // error, try again, this time treating all columns as strings.
+ if (error.toString().includes("Could not convert")) {
+ return await insertUntypedCSV(connection, file, name);
+ }
+ throw error;
+ });
+ }
+ case "application/json":
+ return await connection.insertJSONFromPath(file.name, {
+ name,
+ schema: "main",
+ ...options
+ });
+ default:
+ if (/\.arrow$/i.test(file.name)) {
+ const buffer = new Uint8Array(await file.arrayBuffer());
+ return await connection.insertArrowFromIPCStream(buffer, {
+ name,
+ schema: "main",
+ ...options
+ });
+ }
+ if (/\.parquet$/i.test(file.name)) {
+ return await connection.query(`CREATE VIEW '${name}' AS SELECT * FROM parquet_scan('${file.name}')`);
+ }
+ throw new Error(`unknown file type: ${file.mimeType}`);
+ }
+ } finally {
+ await connection.close();
+ }
+}
+
+async function insertUntypedCSV(connection, file, name) {
+ const statement = await connection.prepare(
+ `CREATE TABLE '${name}' AS SELECT * FROM read_csv_auto(?, ALL_VARCHAR=TRUE)`
+ );
+ return await statement.send(file.name);
+}
+
+async function insertArrowTable(database, name, table, options) {
+ const connection = await database.connect();
+ try {
+ await connection.insertArrowTable(table, {
+ name,
+ schema: "main",
+ ...options
+ });
+ } finally {
+ await connection.close();
+ }
+}
+
+async function insertArqueroTable(database, name, source) {
+ // TODO When we have stdlib versioning and can upgrade Arquero to version 5,
+ // we can then call source.toArrow() directly, with insertArrowTable()
+ const arrow = await import("npm:apache-arrow");
+ const table = arrow.tableFromIPC(source.toArrowBuffer());
+ return await insertArrowTable(database, name, table);
+}
+
+async function insertArray(database, name, array, options) {
+ const arrow = await import("npm:apache-arrow");
+ const table = arrow.tableFromJSON(array);
+ return await insertArrowTable(database, name, table, options);
+}
+
+async function createDuckDB() {
+ const worker = await duckdb.createWorker(bundle.mainWorker);
+ const db = new duckdb.AsyncDuckDB(logger, worker);
+ await db.instantiate(bundle.mainModule);
+ return db;
+}
+
+// https://duckdb.org/docs/sql/data_types/overview
+function getDuckDBType(type) {
+ switch (type) {
+ case "BIGINT":
+ case "HUGEINT":
+ case "UBIGINT":
+ return "bigint";
+ case "DOUBLE":
+ case "REAL":
+ case "FLOAT":
+ return "number";
+ case "INTEGER":
+ case "SMALLINT":
+ case "TINYINT":
+ case "USMALLINT":
+ case "UINTEGER":
+ case "UTINYINT":
+ return "integer";
+ case "BOOLEAN":
+ return "boolean";
+ case "DATE":
+ case "TIMESTAMP":
+ case "TIMESTAMP WITH TIME ZONE":
+ return "date";
+ case "VARCHAR":
+ case "UUID":
+ return "string";
+ // case "BLOB":
+ // case "INTERVAL":
+ // case "TIME":
+ default:
+ if (/^DECIMAL\(/.test(type)) return "integer";
+ return "other";
+ }
+}
+
+// Returns true if the given value is an Observable FileAttachment.
+function isFileAttachment(value) {
+ return (
+ value &&
+ typeof value.name === "string" &&
+ typeof value.url === "function" &&
+ typeof value.arrayBuffer === "function"
+ );
+}
+
+// Arquero tables have a `toArrowBuffer` function
+function isArqueroTable(value) {
+ return value && typeof value.toArrowBuffer === "function";
+}
+
+// Returns true if the value is an Apache Arrow table. This uses a “duck” test
+// (instead of strict instanceof) because we want it to work with a range of
+// Apache Arrow versions at least 7.0.0 or above.
+// https://arrow.apache.org/docs/7.0/js/classes/Arrow_dom.Table.html
+function isArrowTable(value) {
+ return (
+ value &&
+ typeof value.getChild === "function" &&
+ typeof value.toArray === "function" &&
+ value.schema &&
+ Array.isArray(value.schema.fields)
+ );
+}
+
+function getArrowTableSchema(table) {
+ return table.schema.fields.map(getArrowFieldSchema);
+}
+
+function getArrowFieldSchema(field) {
+ return {
+ name: field.name,
+ type: getArrowType(field.type),
+ nullable: field.nullable,
+ databaseType: `${field.type}`
+ };
+}
+
+// https://github.com/apache/arrow/blob/89f9a0948961f6e94f1ef5e4f310b707d22a3c11/js/src/enum.ts#L140-L141
+function getArrowType(type) {
+ switch (type.typeId) {
+ case 2: // Int
+ return "integer";
+ case 3: // Float
+ case 7: // Decimal
+ return "number";
+ case 4: // Binary
+ case 15: // FixedSizeBinary
+ return "buffer";
+ case 5: // Utf8
+ return "string";
+ case 6: // Bool
+ return "boolean";
+ case 8: // Date
+ case 9: // Time
+ case 10: // Timestamp
+ return "date";
+ case 12: // List
+ case 16: // FixedSizeList
+ return "array";
+ case 13: // Struct
+ case 14: // Union
+ return "object";
+ case 11: // Interval
+ case 17: // Map
+ default:
+ return "other";
+ }
+}
diff --git a/src/client/stdlib/fileAttachment.js b/src/client/stdlib/fileAttachment.js
new file mode 100644
index 000000000..147c21bfc
--- /dev/null
+++ b/src/client/stdlib/fileAttachment.js
@@ -0,0 +1,133 @@
+const files = new Map();
+
+export function registerFile(name, file) {
+ if (file == null) files.delete(name);
+ else files.set(name, file);
+}
+
+export function FileAttachment(name) {
+ if (new.target !== undefined) throw new TypeError("FileAttachment is not a constructor");
+ const file = files.get((name = `${name}`));
+ if (!file) throw new Error(`File not found: ${name}`);
+ const {url, mimeType} = file;
+ return new FileAttachmentImpl(url, name, mimeType);
+}
+
+async function remote_fetch(file) {
+ const response = await fetch(await file.url());
+ if (!response.ok) throw new Error(`Unable to load file: ${file.name}`);
+ return response;
+}
+
+async function dsv(file, delimiter, {array = false, typed = false} = {}) {
+ const [text, d3] = await Promise.all([file.text(), import("npm:d3-dsv")]);
+ const parse = delimiter === "\t" ? (array ? d3.tsvParseRows : d3.tsvParse) : array ? d3.csvParseRows : d3.csvParse;
+ return parse(text, typed && d3.autoType);
+}
+
+class AbstractFile {
+ constructor(name, mimeType) {
+ Object.defineProperty(this, "name", {value: name, enumerable: true});
+ if (mimeType !== undefined) Object.defineProperty(this, "mimeType", {value: mimeType + "", enumerable: true});
+ }
+ async blob() {
+ return (await remote_fetch(this)).blob();
+ }
+ async arrayBuffer() {
+ return (await remote_fetch(this)).arrayBuffer();
+ }
+ async text() {
+ return (await remote_fetch(this)).text();
+ }
+ async json() {
+ return (await remote_fetch(this)).json();
+ }
+ async stream() {
+ return (await remote_fetch(this)).body;
+ }
+ async csv(options) {
+ return dsv(this, ",", options);
+ }
+ async tsv(options) {
+ return dsv(this, "\t", options);
+ }
+ async image(props) {
+ const url = await this.url();
+ return new Promise((resolve, reject) => {
+ const i = new Image();
+ if (new URL(url, document.baseURI).origin !== new URL(location).origin) i.crossOrigin = "anonymous";
+ Object.assign(i, props);
+ i.onload = () => resolve(i);
+ i.onerror = () => reject(new Error(`Unable to load file: ${this.name}`));
+ i.src = url;
+ });
+ }
+ async arrow() {
+ const [Arrow, response] = await Promise.all([import("npm:apache-arrow"), remote_fetch(this)]);
+ return Arrow.tableFromIPC(response);
+ }
+ async sqlite() {
+ return import("observablehq:stdlib/sqlite").then((sqlite) => sqlite.SQLiteDatabaseClient.open(remote_fetch(this)));
+ }
+ async zip() {
+ const [{default: JSZip}, buffer] = await Promise.all([import("npm:jszip"), this.arrayBuffer()]);
+ return new ZipArchive(await JSZip.loadAsync(buffer));
+ }
+ async xml(mimeType = "application/xml") {
+ return new DOMParser().parseFromString(await this.text(), mimeType);
+ }
+ async html() {
+ return this.xml("text/html");
+ }
+ async xlsx() {
+ const [{Workbook}, buffer] = await Promise.all([import("observablehq:stdlib/xslx"), this.arrayBuffer()]);
+ return Workbook.load(buffer);
+ }
+}
+
+const FileAttachmentImpl = class FileAttachment extends AbstractFile {
+ constructor(url, name, mimeType) {
+ super(name, mimeType);
+ Object.defineProperty(this, "_url", {value: url});
+ }
+ async url() {
+ return (await this._url) + "";
+ }
+};
+
+FileAttachment.prototype = FileAttachmentImpl.prototype; // instanceof
+
+class ZipArchive {
+ constructor(archive) {
+ Object.defineProperty(this, "_", {value: archive});
+ this.filenames = Object.keys(archive.files).filter((name) => !archive.files[name].dir);
+ }
+ file(path) {
+ const object = this._.file((path = `${path}`));
+ if (!object || object.dir) throw new Error(`file not found: ${path}`);
+ return new ZipArchiveEntry(object);
+ }
+}
+
+class ZipArchiveEntry extends AbstractFile {
+ constructor(object) {
+ super(object.name);
+ Object.defineProperty(this, "_", {value: object});
+ Object.defineProperty(this, "_url", {writable: true});
+ }
+ async url() {
+ return this._url || (this._url = this.blob().then(URL.createObjectURL));
+ }
+ async blob() {
+ return this._.async("blob");
+ }
+ async arrayBuffer() {
+ return this._.async("arraybuffer");
+ }
+ async text() {
+ return this._.async("text");
+ }
+ async json() {
+ return JSON.parse(await this.text());
+ }
+}
diff --git a/src/client/stdlib/generators/index.js b/src/client/stdlib/generators/index.js
new file mode 100644
index 000000000..24387edc0
--- /dev/null
+++ b/src/client/stdlib/generators/index.js
@@ -0,0 +1,3 @@
+export {input} from "./input.js";
+export {observe} from "./observe.js";
+export {queue} from "./queue.js";
diff --git a/src/client/stdlib/generators/input.js b/src/client/stdlib/generators/input.js
new file mode 100644
index 000000000..9d7d8ba37
--- /dev/null
+++ b/src/client/stdlib/generators/input.js
@@ -0,0 +1,43 @@
+import {observe} from "./observe.js";
+
+export function input(element) {
+ return observe((change) => {
+ const event = eventof(element);
+ let value = valueof(element);
+ const inputted = () => change(valueof(element));
+ element.addEventListener(event, inputted);
+ if (value !== undefined) change(value);
+ return () => element.removeEventListener(event, inputted);
+ });
+}
+
+function valueof(element) {
+ switch (element.type) {
+ case "range":
+ case "number":
+ return element.valueAsNumber;
+ case "date":
+ return element.valueAsDate;
+ case "checkbox":
+ return element.checked;
+ case "file":
+ return element.multiple ? element.files : element.files[0];
+ case "select-multiple":
+ return Array.from(element.selectedOptions, (o) => o.value);
+ default:
+ return element.value;
+ }
+}
+
+function eventof(element) {
+ switch (element.type) {
+ case "button":
+ case "submit":
+ case "checkbox":
+ return "click";
+ case "file":
+ return "change";
+ default:
+ return "input";
+ }
+}
diff --git a/src/client/stdlib/generators/observe.js b/src/client/stdlib/generators/observe.js
new file mode 100644
index 000000000..5c98428cb
--- /dev/null
+++ b/src/client/stdlib/generators/observe.js
@@ -0,0 +1,30 @@
+export async function* observe(initialize) {
+ let resolve;
+ let value;
+ let stale = false;
+
+ const dispose = initialize((x) => {
+ value = x;
+ if (resolve) resolve(x), (resolve = null);
+ else stale = true;
+ return x;
+ });
+
+ if (dispose != null && typeof dispose !== "function") {
+ throw new Error(
+ typeof dispose.then === "function"
+ ? "async initializers are not supported"
+ : "initializer returned something, but not a dispose function"
+ );
+ }
+
+ try {
+ while (true) {
+ yield stale ? ((stale = false), value) : new Promise((_) => (resolve = _));
+ }
+ } finally {
+ if (dispose != null) {
+ dispose();
+ }
+ }
+}
diff --git a/src/client/stdlib/generators/queue.js b/src/client/stdlib/generators/queue.js
new file mode 100644
index 000000000..7fd98fd54
--- /dev/null
+++ b/src/client/stdlib/generators/queue.js
@@ -0,0 +1,28 @@
+export async function* queue(initialize) {
+ let resolve;
+ const values = [];
+
+ const dispose = initialize((x) => {
+ values.push(x);
+ if (resolve) resolve(values.shift()), (resolve = null);
+ return x;
+ });
+
+ if (dispose != null && typeof dispose !== "function") {
+ throw new Error(
+ typeof dispose.then === "function"
+ ? "async initializers are not supported"
+ : "initializer returned something, but not a dispose function"
+ );
+ }
+
+ try {
+ while (true) {
+ yield values.length ? values.shift() : new Promise((_) => (resolve = _));
+ }
+ } finally {
+ if (dispose != null) {
+ dispose();
+ }
+ }
+}
diff --git a/src/client/stdlib/mermaid.js b/src/client/stdlib/mermaid.js
new file mode 100644
index 000000000..4edca066b
--- /dev/null
+++ b/src/client/stdlib/mermaid.js
@@ -0,0 +1,11 @@
+import mer from "npm:mermaid";
+
+let nextId = 0;
+const theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "neutral";
+mer.initialize({startOnLoad: false, securityLevel: "loose", theme});
+
+export default async function mermaid() {
+ const root = document.createElement("div");
+ root.innerHTML = (await mer.render(`mermaid-${++nextId}`, String.raw.apply(String, arguments))).svg;
+ return root.removeChild(root.firstChild);
+}
diff --git a/src/client/stdlib/mutable.js b/src/client/stdlib/mutable.js
new file mode 100644
index 000000000..eb571fa79
--- /dev/null
+++ b/src/client/stdlib/mutable.js
@@ -0,0 +1,20 @@
+import {observe} from "./generators/observe.js";
+
+// Mutable returns a generator with a value getter/setting that allows the
+// generated value to be mutated. Therefore, direct mutation is only allowed
+// within the defining cell, but the cell can also export functions that allows
+// other cells to mutate the value as desired.
+export function Mutable(value) {
+ let change;
+ return Object.defineProperty(
+ observe((_) => {
+ change = _;
+ if (value !== undefined) change(value);
+ }),
+ "value",
+ {
+ get: () => value,
+ set: (x) => void change((value = x)) // eslint-disable-line no-setter-return
+ }
+ );
+}
diff --git a/src/client/stdlib/now.js b/src/client/stdlib/now.js
new file mode 100644
index 000000000..3424a4966
--- /dev/null
+++ b/src/client/stdlib/now.js
@@ -0,0 +1,5 @@
+export function* now() {
+ while (true) {
+ yield Date.now();
+ }
+}
diff --git a/src/client/stdlib/recommendedLibraries.js b/src/client/stdlib/recommendedLibraries.js
new file mode 100644
index 000000000..2ebda4f95
--- /dev/null
+++ b/src/client/stdlib/recommendedLibraries.js
@@ -0,0 +1,18 @@
+export const _ = () => import("npm:lodash").then((lodash) => lodash.default);
+export const aq = () => import("npm:arquero");
+export const Arrow = () => import("npm:apache-arrow");
+export const d3 = () => import("npm:d3");
+export const dot = () => import("observablehq:stdlib/dot").then((dot) => dot.default);
+export const duckdb = () => import("npm:@duckdb/duckdb-wasm");
+export const DuckDBClient = () => import("observablehq:stdlib/duckdb").then((duckdb) => duckdb.default);
+export const htl = () => import("npm:htl");
+export const html = () => import("npm:htl").then((htl) => htl.html);
+export const svg = () => import("npm:htl").then((htl) => htl.svg);
+export const Inputs = () => import("npm:@observablehq/inputs");
+export const L = () => import("npm:leaflet");
+export const mermaid = () => import("observablehq:stdlib/mermaid").then((mermaid) => mermaid.default);
+export const Plot = () => import("npm:@observablehq/plot");
+export const SQLite = () => import("observablehq:stdlib/sqlite").then((sqlite) => sqlite.default);
+export const SQLiteDatabaseClient = () => import("observablehq:stdlib/sqlite").then((sqlite) => sqlite.SQLiteDatabaseClient); // prettier-ignore
+export const tex = () => import("observablehq:stdlib/tex").then((tex) => tex.default);
+export const topojson = () => import("npm:topojson-client");
diff --git a/src/client/stdlib/sampleDatasets.js b/src/client/stdlib/sampleDatasets.js
new file mode 100644
index 000000000..86d5ea322
--- /dev/null
+++ b/src/client/stdlib/sampleDatasets.js
@@ -0,0 +1,29 @@
+export const aapl = () => csv("https://static.observableusercontent.com/files/3ccff97fd2d93da734e76829b2b066eafdaac6a1fafdec0faf6ebc443271cfc109d29e80dd217468fcb2aff1e6bffdc73f356cc48feb657f35378e6abbbb63b9", true); // prettier-ignore
+export const alphabet = () => csv("https://static.observableusercontent.com/files/75d52e6c3130b1cae83cda89305e17b50f33e7420ef205587a135e8562bcfd22e483cf4fa2fb5df6dff66f9c5d19740be1cfaf47406286e2eb6574b49ffc685d", true); // prettier-ignore
+export const cars = () => csv("https://static.observableusercontent.com/files/048ec3dfd528110c0665dfa363dd28bc516ffb7247231f3ab25005036717f5c4c232a5efc7bb74bc03037155cb72b1abe85a33d86eb9f1a336196030443be4f6", true); // prettier-ignore
+export const citywages = () => csv("https://static.observableusercontent.com/files/39837ec5121fcc163131dbc2fe8c1a2e0b3423a5d1e96b5ce371e2ac2e20a290d78b71a4fb08b9fa6a0107776e17fb78af313b8ea70f4cc6648fad68ddf06f7a", true); // prettier-ignore
+export const diamonds = () => csv("https://static.observableusercontent.com/files/87942b1f5d061a21fa4bb8f2162db44e3ef0f7391301f867ab5ba718b225a63091af20675f0bfe7f922db097b217b377135203a7eab34651e21a8d09f4e37252", true); // prettier-ignore
+export const flare = () => csv("https://static.observableusercontent.com/files/a6b0d94a7f5828fd133765a934f4c9746d2010e2f342d335923991f31b14120de96b5cb4f160d509d8dc627f0107d7f5b5070d2516f01e4c862b5b4867533000", true); // prettier-ignore
+export const industries = () => csv("https://static.observableusercontent.com/files/76f13741128340cc88798c0a0b7fa5a2df8370f57554000774ab8ee9ae785ffa2903010cad670d4939af3e9c17e5e18e7e05ed2b38b848ac2fc1a0066aa0005f", true); // prettier-ignore
+export const miserables = () => json("https://static.observableusercontent.com/files/31d904f6e21d42d4963ece9c8cc4fbd75efcbdc404bf511bc79906f0a1be68b5a01e935f65123670ed04e35ca8cae3c2b943f82bf8db49c5a67c85cbb58db052"); // prettier-ignore
+export const olympians = () => csv("https://static.observableusercontent.com/files/31ca24545a0603dce099d10ee89ee5ae72d29fa55e8fc7c9ffb5ded87ac83060d80f1d9e21f4ae8eb04c1e8940b7287d179fe8060d887fb1f055f430e210007c", true); // prettier-ignore
+export const penguins = () => csv("https://static.observableusercontent.com/files/715db1223e067f00500780077febc6cebbdd90c151d3d78317c802732252052ab0e367039872ab9c77d6ef99e5f55a0724b35ddc898a1c99cb14c31a379af80a", true); // prettier-ignore
+export const pizza = () => csv("https://static.observableusercontent.com/files/c653108ab176088cacbb338eaf2344c4f5781681702bd6afb55697a3f91b511c6686ff469f3e3a27c75400001a2334dbd39a4499fe46b50a8b3c278b7d2f7fb5", true); // prettier-ignore
+export const weather = () => csv("https://static.observableusercontent.com/files/693a46b22b33db0f042728700e0c73e836fa13d55446df89120682d55339c6db7cc9e574d3d73f24ecc9bc7eb9ac9a1e7e104a1ee52c00aab1e77eb102913c1f", true); // prettier-ignore
+
+async function json(url) {
+ const response = await fetch(url);
+ if (!response.ok) throw new Error(`unable to fetch ${url}: status ${response.status}`);
+ return response.json();
+}
+
+async function text(url) {
+ const response = await fetch(url);
+ if (!response.ok) throw new Error(`unable to fetch ${url}: status ${response.status}`);
+ return response.text();
+}
+
+async function csv(url, typed) {
+ const [contents, d3] = await Promise.all([text(url), import("npm:d3-dsv")]);
+ return d3.csvParse(contents, typed && d3.autoType);
+}
diff --git a/src/client/stdlib/sqlite.js b/src/client/stdlib/sqlite.js
new file mode 100644
index 000000000..a0265a7c5
--- /dev/null
+++ b/src/client/stdlib/sqlite.js
@@ -0,0 +1,155 @@
+// https://github.com/sql-js/sql.js/issues/284
+const SQLite = await (async () => {
+ const exports = {};
+ const response = await fetch("https://cdn.jsdelivr.net/npm/sql.js/dist/sql-wasm.js");
+ new Function("exports", await response.text())(exports);
+ return exports.Module({locateFile: (name) => `https://cdn.jsdelivr.net/npm/sql.js/dist/${name}`});
+})();
+
+export default SQLite;
+
+export class SQLiteDatabaseClient {
+ constructor(db) {
+ Object.defineProperties(this, {
+ _db: {value: db}
+ });
+ }
+ static async open(source) {
+ return new SQLiteDatabaseClient(new SQLite.Database(await load(await source)));
+ }
+ async query(query, params) {
+ return await exec(this._db, query, params);
+ }
+ async queryRow(query, params) {
+ return (await this.query(query, params))[0] || null;
+ }
+ async explain(query, params) {
+ const rows = await this.query(`EXPLAIN QUERY PLAN ${query}`, params);
+ return element("pre", {className: "observablehq--inspect"}, [text(rows.map((row) => row.detail).join("\n"))]);
+ }
+ async describeTables({schema} = {}) {
+ return this.query(
+ `SELECT NULLIF(schema, 'main') AS schema, name FROM pragma_table_list() WHERE type = 'table'${
+ schema == null ? "" : " AND schema = ?"
+ } AND name NOT LIKE 'sqlite_%' ORDER BY schema, name`,
+ schema == null ? [] : [schema]
+ );
+ }
+ async describeColumns({schema, table} = {}) {
+ if (table == null) throw new Error("missing table");
+ const rows = await this.query(
+ `SELECT name, type, "notnull" FROM pragma_table_info(?${schema == null ? "" : ", ?"}) ORDER BY cid`,
+ schema == null ? [table] : [table, schema]
+ );
+ if (!rows.length) throw new Error(`table not found: ${table}`);
+ return rows.map(({name, type, notnull}) => ({
+ name,
+ type: sqliteType(type),
+ databaseType: type,
+ nullable: !notnull
+ }));
+ }
+ async describe(object) {
+ const rows = await (object === undefined
+ ? this.query("SELECT name FROM sqlite_master WHERE type = 'table'")
+ : this.query("SELECT * FROM pragma_table_info(?)", [object]));
+ if (!rows.length) throw new Error("Not found");
+ const {columns} = rows;
+ return element("table", {value: rows}, [
+ element("thead", [
+ element(
+ "tr",
+ columns.map((c) => element("th", [text(c)]))
+ )
+ ]),
+ element(
+ "tbody",
+ rows.map((r) =>
+ element(
+ "tr",
+ columns.map((c) => element("td", [text(r[c])]))
+ )
+ )
+ )
+ ]);
+ }
+ async sql() {
+ return this.query(...this.queryTag.apply(this, arguments));
+ }
+ queryTag(strings, ...params) {
+ return [strings.join("?"), params];
+ }
+}
+
+Object.defineProperty(SQLiteDatabaseClient.prototype, "dialect", {
+ value: "sqlite"
+});
+
+// https://www.sqlite.org/datatype3.html
+function sqliteType(type) {
+ switch (type) {
+ case "NULL":
+ return "null";
+ case "INT":
+ case "INTEGER":
+ case "TINYINT":
+ case "SMALLINT":
+ case "MEDIUMINT":
+ case "BIGINT":
+ case "UNSIGNED BIG INT":
+ case "INT2":
+ case "INT8":
+ return "integer";
+ case "TEXT":
+ case "CLOB":
+ return "string";
+ case "REAL":
+ case "DOUBLE":
+ case "DOUBLE PRECISION":
+ case "FLOAT":
+ case "NUMERIC":
+ return "number";
+ case "BLOB":
+ return "buffer";
+ case "DATE":
+ case "DATETIME":
+ return "string"; // TODO convert strings to Date instances in sql.js
+ default:
+ return /^(?:(?:(?:VARYING|NATIVE) )?CHARACTER|(?:N|VAR|NVAR)CHAR)\(/.test(type)
+ ? "string"
+ : /^(?:DECIMAL|NUMERIC)\(/.test(type)
+ ? "number"
+ : "other";
+ }
+}
+
+function load(source) {
+ return typeof source === "string"
+ ? fetch(source).then(load)
+ : source && typeof source.arrayBuffer === "function" // Response, Blob, FileAttachment
+ ? source.arrayBuffer().then(load)
+ : source instanceof ArrayBuffer
+ ? new Uint8Array(source)
+ : source;
+}
+
+async function exec(db, query, params) {
+ const [result] = await db.exec(query, params);
+ if (!result) return [];
+ const {columns, values} = result;
+ const rows = values.map((row) => Object.fromEntries(row.map((value, i) => [columns[i], value])));
+ rows.columns = columns;
+ return rows;
+}
+
+function element(name, props, children) {
+ if (arguments.length === 2) (children = props), (props = undefined);
+ const element = document.createElement(name);
+ if (props !== undefined) for (const p in props) element[p] = props[p];
+ if (children !== undefined) for (const c of children) element.appendChild(c);
+ return element;
+}
+
+function text(value) {
+ return document.createTextNode(value);
+}
diff --git a/src/client/stdlib/tex.js b/src/client/stdlib/tex.js
new file mode 100644
index 000000000..28a6f8892
--- /dev/null
+++ b/src/client/stdlib/tex.js
@@ -0,0 +1,16 @@
+import katex from "npm:katex";
+
+const tex = renderer();
+
+function renderer(options) {
+ return function () {
+ const root = document.createElement("div");
+ katex.render(String.raw.apply(String, arguments), root, {...options, output: "html"});
+ return root.removeChild(root.firstChild);
+ };
+}
+
+tex.options = renderer;
+tex.block = renderer({displayMode: true});
+
+export default tex;
diff --git a/src/client/width.ts b/src/client/stdlib/width.ts
similarity index 62%
rename from src/client/width.ts
rename to src/client/stdlib/width.ts
index 91d679fc0..1505d7484 100644
--- a/src/client/width.ts
+++ b/src/client/stdlib/width.ts
@@ -1,13 +1,15 @@
+import {observe} from "./generators/observe.js";
+
// Override the width definition to use main instead of body (and also use a
// ResizeObserver instead of listening for window resize events).
-export function width({Generators}) {
- return Generators.observe((notify: (width: number) => void) => {
+export function width(target = document.querySelector("main")!) {
+ return observe((notify: (width: number) => void) => {
let width: number;
const observer = new ResizeObserver(([entry]) => {
const w = entry.contentRect.width;
if (w !== width) notify((width = w));
});
- observer.observe(document.querySelector("main")!);
+ observer.observe(target);
return () => observer.disconnect();
});
}
diff --git a/src/client/stdlib/xslx.js b/src/client/stdlib/xslx.js
new file mode 100644
index 000000000..8a7c9aaf9
--- /dev/null
+++ b/src/client/stdlib/xslx.js
@@ -0,0 +1,106 @@
+import Excel from "npm:exceljs";
+
+export class Workbook {
+ constructor(workbook) {
+ Object.defineProperties(this, {
+ _: {value: workbook},
+ sheetNames: {
+ value: workbook.worksheets.map((s) => s.name),
+ enumerable: true
+ }
+ });
+ }
+ static async load(buffer) {
+ const workbook = new Excel.Workbook();
+ await workbook.xlsx.load(buffer);
+ return new Workbook(workbook);
+ }
+ sheet(name, options) {
+ const sname =
+ typeof name === "number" ? this.sheetNames[name] : this.sheetNames.includes((name = `${name}`)) ? name : null;
+ if (sname == null) throw new Error(`Sheet not found: ${name}`);
+ const sheet = this._.getWorksheet(sname);
+ return extract(sheet, options);
+ }
+}
+
+function extract(sheet, {range, headers} = {}) {
+ let [[c0, r0], [c1, r1]] = parseRange(range, sheet);
+ const headerRow = headers ? sheet._rows[r0++] : null;
+ let names = new Set(["#"]);
+ for (let n = c0; n <= c1; n++) {
+ const value = headerRow ? valueOf(headerRow.findCell(n + 1)) : null;
+ let name = (value && value + "") || toColumn(n);
+ while (names.has(name)) name += "_";
+ names.add(name);
+ }
+ names = new Array(c0).concat(Array.from(names));
+
+ const output = new Array(r1 - r0 + 1);
+ for (let r = r0; r <= r1; r++) {
+ const row = (output[r - r0] = Object.create(null, {"#": {value: r + 1}}));
+ const _row = sheet.getRow(r + 1);
+ if (_row.hasValues)
+ for (let c = c0; c <= c1; c++) {
+ const value = valueOf(_row.findCell(c + 1));
+ if (value != null) row[names[c + 1]] = value;
+ }
+ }
+
+ output.columns = names.filter(() => true); // Filter sparse columns
+ return output;
+}
+
+function valueOf(cell) {
+ if (!cell) return;
+ const {value} = cell;
+ if (value && typeof value === "object" && !(value instanceof Date)) {
+ if (value.formula || value.sharedFormula) {
+ return value.result && value.result.error ? NaN : value.result;
+ }
+ if (value.richText) {
+ return richText(value);
+ }
+ if (value.text) {
+ let {text} = value;
+ if (text.richText) text = richText(text);
+ return value.hyperlink && value.hyperlink !== text ? `${value.hyperlink} ${text}` : text;
+ }
+ return value;
+ }
+ return value;
+}
+
+function richText(value) {
+ return value.richText.map((d) => d.text).join("");
+}
+
+function parseRange(specifier = ":", {columnCount, rowCount}) {
+ specifier = `${specifier}`;
+ if (!specifier.match(/^[A-Z]*\d*:[A-Z]*\d*$/)) throw new Error("Malformed range specifier");
+ const [[c0 = 0, r0 = 0], [c1 = columnCount - 1, r1 = rowCount - 1]] = specifier.split(":").map(fromCellReference);
+ return [
+ [c0, r0],
+ [c1, r1]
+ ];
+}
+
+// Returns the default column name for a zero-based column index.
+// For example: 0 -> "A", 1 -> "B", 25 -> "Z", 26 -> "AA", 27 -> "AB".
+function toColumn(c) {
+ let sc = "";
+ c++;
+ do {
+ sc = String.fromCharCode(64 + (c % 26 || 26)) + sc;
+ } while ((c = Math.floor((c - 1) / 26)));
+ return sc;
+}
+
+// Returns the zero-based indexes from a cell reference.
+// For example: "A1" -> [0, 0], "B2" -> [1, 1], "AA10" -> [26, 9].
+function fromCellReference(s) {
+ const [, sc, sr] = s.match(/^([A-Z]*)(\d*)$/);
+ let c = 0;
+ if (sc) for (let i = 0; i < sc.length; i++) c += Math.pow(26, sc.length - i - 1) * (sc.charCodeAt(i) - 64);
+ return [c ? c - 1 : undefined, sr ? +sr - 1 : undefined];
+}
diff --git a/src/client/tex.js b/src/client/tex.js
deleted file mode 100644
index 1b9a515e7..000000000
--- a/src/client/tex.js
+++ /dev/null
@@ -1,22 +0,0 @@
-export async function tex() {
- const link = document.createElement("link");
- link.rel = "stylesheet";
- link.href = "https://cdn.jsdelivr.net/npm/katex/dist/katex.min.css";
- link.crossOrigin = "anonymous";
- document.head.appendChild(link);
-
- const {default: katex} = await import("npm:katex");
- const tex = renderer();
-
- function renderer(options) {
- return function () {
- const root = document.createElement("div");
- katex.render(String.raw.apply(String, arguments), root, {...options, output: "html"});
- return root.removeChild(root.firstChild);
- };
- }
-
- tex.options = renderer;
- tex.block = renderer({displayMode: true});
- return tex;
-}
diff --git a/src/javascript.ts b/src/javascript.ts
index 733f8ce5c..29b163267 100644
--- a/src/javascript.ts
+++ b/src/javascript.ts
@@ -3,6 +3,7 @@ import type {Expression, Identifier, Node, Options, Program} from "acorn";
import {fileReference} from "./files.js";
import {findAssignments} from "./javascript/assignments.js";
import {findAwaits} from "./javascript/awaits.js";
+import {resolveDatabases} from "./javascript/databases.js";
import {findDeclarations} from "./javascript/declarations.js";
import {findFeatures} from "./javascript/features.js";
import {rewriteFetches} from "./javascript/fetches.js";
@@ -80,7 +81,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans
...(inputs.length ? {inputs} : null),
...(options.inline ? {inline: true} : null),
...(node.declarations?.length ? {outputs: node.declarations.map(({name}) => name)} : null),
- ...(databases.length ? {databases} : null),
+ ...(databases.length ? {databases: resolveDatabases(databases)} : null),
...(files.length ? {files} : null),
body: `${node.async ? "async " : ""}(${inputs}) => {
${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""}
@@ -140,7 +141,6 @@ function parseJavaScript(input: string, options: ParseOptions): JavaScriptNode {
const declarations = expression ? null : findDeclarations(body as Program, globals, input);
const {imports, fetches} = findImports(body, root, sourcePath);
const features = findFeatures(body, root, sourcePath, references, input);
-
return {
body,
declarations,
diff --git a/src/javascript/databases.ts b/src/javascript/databases.ts
new file mode 100644
index 000000000..211f8f6c3
--- /dev/null
+++ b/src/javascript/databases.ts
@@ -0,0 +1,63 @@
+import {createHmac} from "node:crypto";
+import type {DatabaseReference} from "../javascript.js";
+
+// TODO config
+const DATABASE_TOKEN_DURATION = 60 * 60 * 1000 * 36; // 36 hours in ms
+
+export interface ResolvedDatabaseReference extends DatabaseReference {
+ token: string;
+ type: string;
+ url: string;
+}
+
+interface DatabaseConfig {
+ host: string;
+ name: string;
+ origin?: string;
+ port: number;
+ secret: string;
+ ssl: "disabled" | "enabled";
+ type: string;
+}
+
+function getDatabaseProxyConfig(env: typeof process.env, name: string): DatabaseConfig | undefined {
+ const property = `OBSERVABLEHQ_DB_SECRET_${name}`;
+ const secret = env[property];
+ if (!secret) return;
+ const config = JSON.parse(Buffer.from(secret, "base64").toString("utf-8")) as DatabaseConfig;
+ if (!config.host || !config.port || !config.secret) throw new Error(`Invalid database config: ${property}`);
+ return config;
+}
+
+function encodeToken(payload: {name: string; exp: number}, secret: string): string {
+ const data = JSON.stringify(payload);
+ const hmac = createHmac("sha256", Buffer.from(secret, "hex")).update(data).digest();
+ return `${Buffer.from(data).toString("base64")}.${Buffer.from(hmac).toString("base64")}`;
+}
+
+export function resolveDatabases(
+ databases: DatabaseReference[],
+ {exp = Date.now() + DATABASE_TOKEN_DURATION, env = process.env} = {}
+): ResolvedDatabaseReference[] {
+ const options = {exp, env};
+ return databases.map((d) => resolveDatabase(d, options)).filter((d): d is ResolvedDatabaseReference => !!d);
+}
+
+export function resolveDatabase(
+ database: DatabaseReference,
+ {exp = Date.now() + DATABASE_TOKEN_DURATION, env = process.env} = {}
+): ResolvedDatabaseReference | undefined {
+ const {name} = database;
+ const config = getDatabaseProxyConfig(env, name);
+ if (!config) return void console.warn(`Unable to resolve database: ${name}`);
+ const url = new URL("http://localhost");
+ url.protocol = config.ssl !== "disabled" ? "https:" : "http:";
+ url.host = config.host;
+ url.port = String(config.port);
+ return {
+ name,
+ token: encodeToken({name, exp}, config.secret),
+ type: config.type,
+ url: url.toString()
+ };
+}
diff --git a/src/javascript/imports.ts b/src/javascript/imports.ts
index 8a46f0be4..6c69cd5d3 100644
--- a/src/javascript/imports.ts
+++ b/src/javascript/imports.ts
@@ -212,18 +212,36 @@ export function rewriteImports(
export type ImportResolver = (path: string, specifier: string) => string;
-export function createImportResolver(root: string, base = "."): ImportResolver {
+export function createImportResolver(root: string, base: "." | "_import" = "."): ImportResolver {
return (path, specifier) => {
return isLocalImport(specifier, path)
? relativeUrl(path, resolvePath(base, path, resolveImportHash(root, path, specifier)))
: specifier === "npm:@observablehq/runtime"
- ? relativeUrl(path, "_observablehq/runtime.js")
+ ? resolveBuiltin(base, path, "runtime.js")
+ : specifier === "npm:@observablehq/stdlib"
+ ? resolveBuiltin(base, path, "stdlib.js")
+ : specifier === "npm:@observablehq/dot"
+ ? resolveBuiltin(base, path, "stdlib/dot.js") // TODO publish to npm
+ : specifier === "npm:@observablehq/duckdb"
+ ? resolveBuiltin(base, path, "stdlib/duckdb.js") // TODO publish to npm
+ : specifier === "npm:@observablehq/mermaid"
+ ? resolveBuiltin(base, path, "stdlib/mermaid.js") // TODO publish to npm
+ : specifier === "npm:@observablehq/tex"
+ ? resolveBuiltin(base, path, "stdlib/tex.js") // TODO publish to npm
+ : specifier === "npm:@observablehq/sqlite"
+ ? resolveBuiltin(base, path, "stdlib/sqlite.js") // TODO publish to npm
+ : specifier === "npm:@observablehq/xslx"
+ ? resolveBuiltin(base, path, "stdlib/xslx.js") // TODO publish to npm
: specifier.startsWith("npm:")
? `https://cdn.jsdelivr.net/npm/${specifier.slice("npm:".length)}/+esm`
: specifier;
};
}
+function resolveBuiltin(base: "." | "_import", path: string, specifier: string): string {
+ return relativeUrl(join(base === "." ? "_import" : ".", path), join("_observablehq", specifier));
+}
+
/**
* Given the specified local import, applies the ?sha query string based on the
* content hash of the imported module and its transitively imported modules.
diff --git a/src/libraries.ts b/src/libraries.ts
new file mode 100644
index 000000000..3c2e4d4f7
--- /dev/null
+++ b/src/libraries.ts
@@ -0,0 +1,33 @@
+export function getImplicitSpecifiers(inputs: Set): Set {
+ return addImplicitSpecifiers(new Set(), inputs);
+}
+
+export function addImplicitSpecifiers(specifiers: Set, inputs: Set): typeof specifiers {
+ if (inputs.has("d3")) specifiers.add("npm:d3");
+ if (inputs.has("Plot")) specifiers.add("npm:d3").add("npm:@observablehq/plot");
+ if (inputs.has("htl") || inputs.has("html") || inputs.has("svg")) specifiers.add("npm:htl");
+ if (inputs.has("Inputs")) specifiers.add("npm:htl").add("npm:@observablehq/inputs");
+ if (inputs.has("dot")) specifiers.add("npm:@observablehq/dot").add("npm:@viz-js/viz");
+ if (inputs.has("duckdb")) specifiers.add("npm:@duckdb/duckdb-wasm");
+ if (inputs.has("DuckDBClient")) specifiers.add("npm:@observablehq/duckdb").add("npm:@duckdb/duckdb-wasm");
+ if (inputs.has("_")) specifiers.add("npm:lodash");
+ if (inputs.has("aq")) specifiers.add("npm:arquero");
+ if (inputs.has("Arrow")) specifiers.add("npm:apache-arrow");
+ if (inputs.has("L")) specifiers.add("npm:leaflet");
+ if (inputs.has("mermaid")) specifiers.add("npm:@observablehq/mermaid").add("npm:mermaid").add("npm:d3");
+ if (inputs.has("SQLite") || inputs.has("SQLiteDatabaseClient")) specifiers.add("npm:@observablehq/sqlite");
+ if (inputs.has("tex")) specifiers.add("npm:@observablehq/tex").add("npm:katex");
+ if (inputs.has("topojson")) specifiers.add("npm:topojson-client");
+ return specifiers;
+}
+
+export function getImplicitStylesheets(specifiers: Set): Set {
+ return addImplicitStylesheets(new Set(), specifiers);
+}
+
+export function addImplicitStylesheets(stylesheets: Set, specifiers: Set): typeof stylesheets {
+ if (specifiers.has("npm:katex")) stylesheets.add("https://cdn.jsdelivr.net/npm/katex/dist/katex.min.css");
+ if (specifiers.has("npm:@observablehq/inputs")) stylesheets.add("https://cdn.jsdelivr.net/gh/observablehq/inputs/src/style.css"); // prettier-ignore
+ if (specifiers.has("npm:leaflet")) stylesheets.add("https://cdn.jsdelivr.net/npm/leaflet/dist/leaflet.css");
+ return stylesheets;
+}
diff --git a/src/markdown.ts b/src/markdown.ts
index 34024ed94..d4fc0480c 100644
--- a/src/markdown.ts
+++ b/src/markdown.ts
@@ -332,6 +332,7 @@ const SUPPORTED_PROPERTIES: readonly {query: string; src: "href" | "src" | "srcs
{query: "video[src]", src: "src"},
{query: "video source[src]", src: "src"}
]);
+
export function normalizePieceHtml(html: string, sourcePath: string, context: ParseContext): string {
const {document} = parseHTML(html);
@@ -419,11 +420,10 @@ function toParseCells(pieces: RenderPiece[]): CellPiece[] {
return cellPieces;
}
+// TODO We need to know what line in the source the markdown starts on and pass
+// that as startLine in the parse context below.
export async function parseMarkdown(source: string, root: string, sourcePath: string): Promise {
const parts = matter(source, {});
-
- // TODO: We need to know what line in the source the markdown starts on and pass that
- // as startLine in the parse context below.
const md = MarkdownIt({html: true});
md.use(MarkdownItAnchor, {permalink: MarkdownItAnchor.permalink.headerLink({class: "observablehq-header-anchor"})});
md.inline.ruler.push("placeholder", transformPlaceholderInline);
@@ -497,12 +497,13 @@ function diffReducer(patch: PatchItem) {
if (patch.type === "remove") {
return {
...patch,
+ type: "remove",
items: patch.items.map((item) => ({
type: item.type,
id: item.id,
...("cellIds" in item ? {cellIds: item.cellIds} : null)
}))
- };
+ } as const;
}
return patch;
}
diff --git a/src/preview.ts b/src/preview.ts
index 237eba44f..bd0c5f74e 100644
--- a/src/preview.ts
+++ b/src/preview.ts
@@ -6,6 +6,7 @@ import {createServer} from "node:http";
import type {IncomingMessage, RequestListener, Server, ServerResponse} from "node:http";
import {basename, dirname, extname, join, normalize} from "node:path";
import {fileURLToPath} from "node:url";
+import {difference} from "d3-array";
import send from "send";
import {type WebSocket, WebSocketServer} from "ws";
import {version} from "../package.json";
@@ -14,10 +15,10 @@ import {Loader} from "./dataloader.js";
import {HttpError, isEnoent, isHttpError, isSystemError} from "./error.js";
import {FileWatchers} from "./fileWatchers.js";
import {createImportResolver, rewriteModule} from "./javascript/imports.js";
+import {getImplicitSpecifiers, getImplicitStylesheets} from "./libraries.js";
import {diffMarkdown, readMarkdown} from "./markdown.js";
import type {ParseResult, ReadMarkdownResult} from "./markdown.js";
import {renderPreview} from "./render.js";
-import {type CellResolver, makeCLIResolver} from "./resolver.js";
import {getClientPath, rollupClient} from "./rollup.js";
import {bold, faint, green, underline} from "./tty.js";
@@ -37,21 +38,16 @@ export async function preview(options: PreviewOptions): Promise {
export class PreviewServer {
private readonly _server: ReturnType;
private readonly _socketServer: WebSocketServer;
- private readonly _resolver: CellResolver;
private readonly _verbose: boolean;
readonly root: string;
- private constructor(
- {server, root, verbose}: {server: Server; root: string; verbose: boolean},
- resolver: CellResolver
- ) {
+ private constructor({server, root, verbose}: {server: Server; root: string; verbose: boolean}) {
this.root = root;
this._verbose = verbose;
this._server = server;
this._server.on("request", this._handleRequest);
this._socketServer = new WebSocketServer({server: this._server});
this._socketServer.on("connection", this._handleConnection);
- this._resolver = resolver;
}
static async start({verbose = true, hostname, port, ...options}: PreviewOptions) {
@@ -76,7 +72,7 @@ export class PreviewServer {
console.log(`${faint("↳")} ${underline(`http://${hostname}:${port}/`)}`);
console.log("");
}
- return new PreviewServer({server, verbose, ...options}, await makeCLIResolver());
+ return new PreviewServer({server, verbose, ...options});
}
_handleRequest: RequestListener = async (req, res) => {
@@ -86,6 +82,10 @@ export class PreviewServer {
let {pathname} = url;
if (pathname === "/_observablehq/runtime.js") {
send(req, "/@observablehq/runtime/dist/runtime.js", {root: "./node_modules"}).pipe(res);
+ } else if (pathname.startsWith("/_observablehq/stdlib.js")) {
+ end(req, res, await rollupClient(getClientPath("./src/client/stdlib.js")), "text/javascript");
+ } else if (pathname.startsWith("/_observablehq/stdlib/")) {
+ end(req, res, await rollupClient(getClientPath("./src/client/" + pathname.slice("/_observablehq/".length))), "text/javascript"); // prettier-ignore
} else if (pathname === "/_observablehq/client.js") {
end(req, res, await rollupClient(getClientPath("./src/client/preview.js")), "text/javascript");
} else if (pathname.startsWith("/_observablehq/")) {
@@ -175,7 +175,6 @@ export class PreviewServer {
const {html} = await renderPreview(await readFile(path + ".md", "utf-8"), {
root: this.root,
path: pathname,
- resolver: this._resolver,
...config
});
end(req, res, html, "text/html");
@@ -197,7 +196,6 @@ export class PreviewServer {
const {html} = await renderPreview(await readFile(join(this.root, "404.md"), "utf-8"), {
root: this.root,
path: "/404",
- resolver: this._resolver,
...config
});
end(req, res, html, "text/html");
@@ -213,7 +211,7 @@ export class PreviewServer {
_handleConnection = (socket: WebSocket, req: IncomingMessage) => {
if (req.url === "/_observablehq") {
- handleWatch(socket, req, {root: this.root, resolver: this._resolver});
+ handleWatch(socket, req, {root: this.root});
} else {
socket.close();
}
@@ -242,14 +240,6 @@ function end(req: IncomingMessage, res: ServerResponse, content: string, type: s
}
}
-function resolveDiffs(diff: ReturnType, resolver: CellResolver): ReturnType {
- return diff.map((item) =>
- item.type === "add"
- ? {...item, items: item.items.map((addItem) => (addItem.type === "cell" ? resolver(addItem) : addItem))}
- : item
- );
-}
-
function getWatchPaths(parseResult: ParseResult): string[] {
const paths: string[] = [];
const {files, imports} = parseResult;
@@ -258,10 +248,17 @@ function getWatchPaths(parseResult: ParseResult): string[] {
return paths;
}
-function handleWatch(socket: WebSocket, req: IncomingMessage, options: {root: string; resolver: CellResolver}) {
- const {root, resolver} = options;
+function getStylesheets({cells}: ParseResult): Set {
+ const inputs = new Set();
+ for (const cell of cells) cell.inputs?.forEach(inputs.add, inputs);
+ return getImplicitStylesheets(getImplicitSpecifiers(inputs));
+}
+
+function handleWatch(socket: WebSocket, req: IncomingMessage, options: {root: string}) {
+ const {root} = options;
let path: string | null = null;
let current: ReadMarkdownResult | null = null;
+ let stylesheets: Set | null = null;
let markdownWatcher: FSWatcher | null = null;
let attachmentWatcher: FileWatchers | null = null;
console.log(faint("socket open"), req.url);
@@ -297,7 +294,11 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, options: {root: st
case "change": {
const updated = await readMarkdown(path, root);
if (current.parse.hash === updated.parse.hash) break;
- const diff = resolveDiffs(diffMarkdown(current, updated), resolver);
+ const updatedStylesheets = getStylesheets(updated.parse);
+ for (const href of difference(stylesheets, updatedStylesheets)) send({type: "remove-stylesheet", href});
+ for (const href of difference(updatedStylesheets, stylesheets)) send({type: "add-stylesheet", href});
+ stylesheets = updatedStylesheets;
+ const diff = diffMarkdown(current, updated);
send({type: "update", diff, previousHash: current.parse.hash, updatedHash: updated.parse.hash});
current = updated;
attachmentWatcher?.close();
@@ -315,6 +316,7 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, options: {root: st
path += ".md";
current = await readMarkdown(path, root);
if (current.parse.hash !== initialHash) return void send({type: "reload"});
+ stylesheets = getStylesheets(current.parse);
attachmentWatcher = await FileWatchers.of(root, path, getWatchPaths(current.parse), refreshAttachment);
markdownWatcher = watch(join(root, path), watcher);
}
diff --git a/src/render.ts b/src/render.ts
index e488b3cb3..413201783 100644
--- a/src/render.ts
+++ b/src/render.ts
@@ -3,7 +3,8 @@ import {type Config, type Page, type Section, mergeToc} from "./config.js";
import {type Html, html} from "./html.js";
import {type ImportResolver, createImportResolver} from "./javascript/imports.js";
import {type FileReference, type ImportReference} from "./javascript.js";
-import {type CellPiece, type ParseResult, parseMarkdown} from "./markdown.js";
+import {addImplicitSpecifiers, addImplicitStylesheets} from "./libraries.js";
+import {type ParseResult, parseMarkdown} from "./markdown.js";
import {type PageLink, findLink} from "./pager.js";
import {relativeUrl} from "./url.js";
@@ -16,7 +17,6 @@ export interface Render {
export interface RenderOptions extends Config {
root: string;
path: string;
- resolver: (cell: CellPiece) => CellPiece;
}
export async function renderPreview(source: string, options: RenderOptions): Promise {
@@ -50,7 +50,7 @@ type RenderInternalOptions =
| {preview: true}; // preview
function render(parseResult: ParseResult, options: RenderOptions & RenderInternalOptions): string {
- const {root, path, pages, title, preview, resolver} = options;
+ const {root, path, pages, title, preview} = options;
const toc = mergeToc(parseResult.data?.toc, options.toc);
const headers = toc.show ? findHeaders(parseResult) : [];
return String(html`
@@ -62,13 +62,7 @@ ${
.filter((title): title is string => !!title)
.join(" | ")}\n`
: ""
-}
-
-${renderImportPreloads(
- parseResult,
- path,
- createImportResolver(root, "_import")
- )}${
+}${renderLinks(parseResult, path, createImportResolver(root, "_import"))}${
path === "/404"
? html.unsafe(`\n${pages.length > 0 ? html`\n${renderSidebar(title, pages, path)}` : ""}${
headers.length > 0 ? html`\n${renderToc(headers, toc.label)}` : ""
}
@@ -181,25 +172,29 @@ function prettyPath(path: string): string {
return path.replace(/\/index$/, "/") || "/";
}
-function renderImportPreloads(parseResult: ParseResult, path: string, resolver: ImportResolver): Html {
- const specifiers = new Set(["npm:@observablehq/runtime"]);
+function renderLinks(parseResult: ParseResult, path: string, resolver: ImportResolver): Html {
+ const stylesheets = new Set([relativeUrl(path, "/_observablehq/style.css"), "https://fonts.googleapis.com/css2?family=Source+Serif+Pro:ital,wght@0,400;0,600;0,700;1,400;1,600;1,700&display=swap"]); // prettier-ignore
+ const specifiers = new Set(["npm:@observablehq/runtime", "npm:@observablehq/stdlib"]);
for (const {name} of parseResult.imports) specifiers.add(name);
const inputs = new Set(parseResult.cells.flatMap((cell) => cell.inputs ?? []));
- if (inputs.has("d3") || inputs.has("Plot")) specifiers.add("npm:d3");
- if (inputs.has("Plot")) specifiers.add("npm:@observablehq/plot");
- if (inputs.has("htl") || inputs.has("html") || inputs.has("svg") || inputs.has("Inputs")) specifiers.add("npm:htl");
- if (inputs.has("Inputs")) specifiers.add("npm:@observablehq/inputs");
- if (inputs.has("dot")) specifiers.add("npm:@viz-js/viz");
- if (inputs.has("mermaid")) specifiers.add("npm:mermaid").add("npm:d3");
- if (inputs.has("tex")) specifiers.add("npm:katex");
+ addImplicitSpecifiers(specifiers, inputs);
+ addImplicitStylesheets(stylesheets, specifiers);
const preloads = new Set();
- for (const specifier of specifiers) {
- preloads.add(resolver(path, specifier));
- }
- if (parseResult.cells.some((cell) => cell.databases?.length)) {
- preloads.add(relativeUrl(path, "/_observablehq/database.js"));
- }
- return html`${Array.from(preloads, (href) => html`\n`)}`;
+ for (const specifier of specifiers) preloads.add(resolver(path, specifier));
+ if (parseResult.cells.some((cell) => cell.databases?.length)) preloads.add(relativeUrl(path, "/_observablehq/database.js")); // prettier-ignore
+ return html`${
+ Array.from(stylesheets).sort().map(renderStylesheet) //
+ }${
+ Array.from(preloads).sort().map(renderModulePreload) //
+ }`;
+}
+
+function renderStylesheet(href: string): Html {
+ return html`\n`;
+}
+
+function renderModulePreload(href: string): Html {
+ return html`\n`;
}
function renderFooter(path: string, options: Pick): Html {
diff --git a/src/resolver.ts b/src/resolver.ts
deleted file mode 100644
index e893050cb..000000000
--- a/src/resolver.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-import {createHmac} from "node:crypto";
-import type {CellPiece} from "./markdown.js";
-
-const DEFAULT_DATABASE_TOKEN_DURATION = 60 * 60 * 1000 * 36; // 36 hours in ms
-
-export type CellResolver = (cell: CellPiece) => CellPiece;
-
-interface DatabaseConfig {
- host: string;
- name: string;
- origin?: string;
- port: number;
- secret: string;
- ssl: "disabled" | "enabled";
- type: string;
-}
-
-function getDatabaseProxyConfig(env: typeof process.env, name: string): DatabaseConfig | null {
- const property = `OBSERVABLEHQ_DB_SECRET_${name}`;
- if (env[property]) {
- const config = JSON.parse(Buffer.from(env[property]!, "base64").toString("utf8")) as DatabaseConfig;
- if (!config.host || !config.port || !config.secret) {
- throw new Error(`Invalid database config: ${property}`);
- }
- return config;
- }
- return null;
-}
-
-function encodeToken(payload: {name: string; exp: number}, secret: string): string {
- const data = JSON.stringify(payload);
- const hmac = createHmac("sha256", Buffer.from(secret, "hex")).update(data).digest();
- return `${Buffer.from(data).toString("base64")}.${Buffer.from(hmac).toString("base64")}`;
-}
-
-interface ResolverOptions {
- databaseTokenDuration?: number;
- env?: typeof process.env;
-}
-
-export async function makeCLIResolver({
- databaseTokenDuration = DEFAULT_DATABASE_TOKEN_DURATION,
- env = process.env
-}: ResolverOptions = {}): Promise {
- return (cell: CellPiece): CellPiece => {
- if (cell.databases !== undefined) {
- cell = {
- ...cell,
- databases: cell.databases.map((ref) => {
- const db = getDatabaseProxyConfig(env, ref.name);
- if (db) {
- const url = new URL("http://localhost");
- url.protocol = db.ssl !== "disabled" ? "https:" : "http:";
- url.host = db.host;
- url.port = String(db.port);
- return {
- ...ref,
- token: encodeToken({...ref, exp: Date.now() + databaseTokenDuration}, db.secret),
- type: db.type,
- url: url.toString()
- };
- }
- throw new Error(`Unable to resolve database "${ref.name}"`);
- })
- };
- }
- return cell;
- };
-}
diff --git a/src/rollup.ts b/src/rollup.ts
index a0599257e..15de1bde8 100644
--- a/src/rollup.ts
+++ b/src/rollup.ts
@@ -1,23 +1,22 @@
import {dirname, join, relative} from "node:path";
import {cwd} from "node:process";
import {fileURLToPath} from "node:url";
-import {type OutputChunk, rollup} from "rollup";
+import type {AstNode, OutputChunk, ResolveIdResult} from "rollup";
+import {rollup} from "rollup";
import esbuild from "rollup-plugin-esbuild";
+import {relativeUrl} from "./url.js";
-export async function rollupClient(clientPath = getClientPath(), {minify = false} = {}): Promise {
+export async function rollupClient(clientPath: string, {minify = false} = {}): Promise {
const bundle = await rollup({
input: clientPath,
- external: ["./runtime.js", /^https:/],
+ external: [/^https:/],
plugins: [
{
- name: "resolve-npm-import",
- resolveDynamicImport(specifier) {
- return typeof specifier === "string" && specifier.startsWith("npm:")
- ? {id: `https://cdn.jsdelivr.net/npm/${specifier.slice("npm:".length)}/+esm`}
- : null;
- }
+ name: "resolve-import",
+ resolveId: (specifier) => resolveImport(clientPath, specifier),
+ resolveDynamicImport: (specifier) => resolveImport(clientPath, specifier)
},
- esbuild({minify})
+ esbuild({target: "es2022", minify})
]
});
try {
@@ -28,6 +27,16 @@ export async function rollupClient(clientPath = getClientPath(), {minify = false
}
}
-export function getClientPath(entry = "./src/client/index.js"): string {
+function resolveImport(source: string, specifier: string | AstNode): ResolveIdResult {
+ return typeof specifier !== "string"
+ ? null
+ : specifier.startsWith("observablehq:")
+ ? {id: relativeUrl(source, `./src/client/${specifier.slice("observablehq:".length)}.js`), external: true}
+ : specifier.startsWith("npm:")
+ ? {id: `https://cdn.jsdelivr.net/npm/${specifier.slice("npm:".length)}/+esm`}
+ : null;
+}
+
+export function getClientPath(entry: string): string {
return relative(cwd(), join(dirname(fileURLToPath(import.meta.url)), "..", entry));
}
diff --git a/test/deploy-test.ts b/test/deploy-test.ts
index 8175302a7..d8a6d0a5c 100644
--- a/test/deploy-test.ts
+++ b/test/deploy-test.ts
@@ -7,13 +7,24 @@ import type {Logger} from "../src/logger.js";
import {commandRequiresAuthenticationMessage} from "../src/observableApiAuth.js";
import type {DeployConfig} from "../src/observableApiConfig.js";
import {MockLogger} from "./mocks/logger.js";
-import {
- ObservableApiMock,
- invalidApiKey,
- userWithTwoWorkspaces,
- userWithZeroWorkspaces,
- validApiKey
-} from "./mocks/observableApi.js";
+import {ObservableApiMock} from "./mocks/observableApi.js";
+import {invalidApiKey, userWithTwoWorkspaces, userWithZeroWorkspaces, validApiKey} from "./mocks/observableApi.js";
+
+// These files are implicitly generated by the CLI. This may change over time,
+// so they’re enumerated here for clarity. TODO We should enforce that these
+// files are specifically uploaded, rather than just the number of files.
+const EXTRA_FILES: string[] = [
+ "_observablehq/client.js",
+ "_observablehq/runtime.js",
+ "_observablehq/stdlib.js",
+ "_observablehq/stdlib/dot.js",
+ "_observablehq/stdlib/duckdb.js",
+ "_observablehq/stdlib/mermaid.js",
+ "_observablehq/stdlib/sqlite.js",
+ "_observablehq/stdlib/tex.js",
+ "_observablehq/stdlib/xslx.js",
+ "_observablehq/style.css"
+];
class MockDeployEffects implements DeployEffects {
public logger = new MockLogger();
@@ -67,7 +78,9 @@ class MockDeployEffects implements DeployEffects {
}
}
-const TEST_SOURCE_ROOT = "test/example-dist";
+// This test should have exactly one index.md in it, and nothing else; that one
+// page is why we +1 to the number of extra files.
+const TEST_SOURCE_ROOT = "test/input/build/simple-public";
describe("deploy", () => {
it("makes expected API calls for a new project", async () => {
@@ -77,7 +90,7 @@ describe("deploy", () => {
.handleGetUser()
.handlePostProject({projectId})
.handlePostDeploy({projectId, deployId})
- .handlePostDeployFile({deployId, repeat: 3})
+ .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
.handlePostDeployUploaded({deployId})
.start();
@@ -96,7 +109,7 @@ describe("deploy", () => {
const deployId = "deploy456";
const apiMock = new ObservableApiMock()
.handlePostDeploy({projectId, deployId})
- .handlePostDeployFile({deployId, repeat: 3})
+ .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
.handlePostDeployUploaded({deployId})
.start();
@@ -129,7 +142,7 @@ describe("deploy", () => {
.handleGetUser({user: userWithTwoWorkspaces})
.handlePostProject({projectId})
.handlePostDeploy({projectId, deployId})
- .handlePostDeployFile({deployId, repeat: 3})
+ .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
.handlePostDeployUploaded({deployId})
.start();
const effects = new MockDeployEffects();
@@ -232,7 +245,7 @@ describe("deploy", () => {
.handleGetUser()
.handlePostProject({projectId})
.handlePostDeploy({projectId, deployId})
- .handlePostDeployFile({deployId, repeat: 3})
+ .handlePostDeployFile({deployId, repeat: EXTRA_FILES.length + 1})
.handlePostDeployUploaded({deployId, status: 500})
.start();
const effects = new MockDeployEffects();
diff --git a/test/example-dist/index.html b/test/example-dist/index.html
deleted file mode 100644
index 701b40345..000000000
--- a/test/example-dist/index.html
+++ /dev/null
@@ -1,9 +0,0 @@
-
-
-
- Hello, world
-
-
- Hello, world!
-
-
\ No newline at end of file
diff --git a/test/javascript/databases-test.ts b/test/javascript/databases-test.ts
new file mode 100644
index 000000000..ac5bd22c4
--- /dev/null
+++ b/test/javascript/databases-test.ts
@@ -0,0 +1,57 @@
+import assert from "node:assert";
+import {createHmac} from "node:crypto";
+import {type ResolvedDatabaseReference, resolveDatabase, resolveDatabases} from "../../src/javascript/databases.js";
+import type {DatabaseReference} from "../../src/javascript.js";
+
+const exp = Date.UTC(2020, 0, 1);
+
+const snow2 = {
+ name: "snow2",
+ type: "snowflake",
+ host: "localhost",
+ port: "2899",
+ ssl: "disabled",
+ origin: "http://127.0.0.1:3000",
+ secret: "0249f7ba6fff3856cb9868d115dbcd6b2ef51f1e9a55fdc6f7484013b7c191c5"
+};
+
+const env = {
+ OBSERVABLEHQ_DB_SECRET_snow2: Buffer.from(JSON.stringify(snow2), "utf-8").toString("base64")
+};
+
+describe("resolveDatabase", () => {
+ it("returns a resolved database reference", async () => {
+ const input: DatabaseReference = {name: "snow2"};
+ const output = resolveDatabase(input, {env, exp});
+ assert.deepStrictEqual(output, {
+ name: "snow2",
+ token: "eyJuYW1lIjoic25vdzIiLCJleHAiOjE1Nzc4MzY4MDAwMDB9.P4Zhh8OSkq7TGARf2Rid6OyL1m0WL3nWe+w01DWtTng=",
+ type: "snowflake",
+ url: "http://localhost:2899/"
+ });
+ const [encodedData, hmac] = output.token.split(".");
+ const decodedData = Buffer.from(encodedData, "base64").toString("utf-8");
+ assert.deepStrictEqual(JSON.parse(decodedData), {name: "snow2", exp});
+ const expectedHmac = createHmac("sha256", Buffer.from(snow2.secret, "hex")).update(decodedData).digest("base64");
+ assert.strictEqual(hmac, expectedHmac);
+ });
+ it("does not mutate the passed-in database reference", async () => {
+ const input = {name: "snow2"};
+ resolveDatabase(input, {env});
+ assert.deepStrictEqual(input, {name: "snow2"});
+ });
+ it("returns undefined when it can’t be resolved", async () => {
+ const input = {name: "notthere"};
+ assert.strictEqual(resolveDatabase(input), undefined);
+ assert.deepStrictEqual(input, {name: "notthere"});
+ });
+});
+
+describe("resolveDatabases", () => {
+ it("returns only resolved database references", async () => {
+ const notthere: DatabaseReference = {name: "notthere"};
+ const snow2: DatabaseReference = {name: "snow2"};
+ const output = resolveDatabases([notthere, snow2], {env, exp});
+ assert.deepStrictEqual(output, [resolveDatabase(snow2, {env, exp})!]);
+ });
+});
diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts
index d672b9e5d..a9c7e2374 100644
--- a/test/mocks/observableApi.ts
+++ b/test/mocks/observableApi.ts
@@ -3,7 +3,9 @@ import {getObservableApiHost} from "../../src/observableApiClient.js";
export const validApiKey = "MOCK-VALID-KEY";
export const invalidApiKey = "MOCK-INVALID-KEY";
+
const emptyErrorBody = JSON.stringify({errors: []});
+
export class ObservableApiMock {
private _agent: MockAgent | null = null;
private _handlers: ((pool: Interceptable) => void)[] = [];
@@ -76,11 +78,10 @@ export class ObservableApiMock {
const response = status == 204 ? "" : emptyErrorBody;
const headers = authorizationHeader(status != 401);
this._handlers.push((pool) => {
- for (let i = 0; i < repeat; i++) {
- pool
- .intercept({path: `/cli/deploy/${deployId}/file`, method: "POST", headers: headersMatcher(headers)})
- .reply(status, response);
- }
+ pool
+ .intercept({path: `/cli/deploy/${deployId}/file`, method: "POST", headers: headersMatcher(headers)})
+ .reply(status, response)
+ .times(repeat);
});
return this;
}
diff --git a/test/output/build/404/404.html b/test/output/build/404/404.html
index 133ee0439..c83e18824 100644
--- a/test/output/build/404/404.html
+++ b/test/output/build/404/404.html
@@ -4,9 +4,10 @@
Page not found
-
+
+