From 5d132d654ee7136cfdfde519c9811ee689ed83bd Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 9 Jan 2025 16:55:45 +0000 Subject: [PATCH] feat: optimize order of entries --- lib/utils/templates.js | 4 +- lib/writer.js | 78 +++++++++++++++++- .../valid-styles/all-types.input.json | 25 ++++++ test/write-read.js | 81 ++++++++++++++++++- 4 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/valid-styles/all-types.input.json diff --git a/lib/utils/templates.js b/lib/utils/templates.js index 1d3f8b1..691fb1b 100644 --- a/lib/utils/templates.js +++ b/lib/utils/templates.js @@ -3,9 +3,9 @@ export const URI_BASE = URI_SCHEME + '://maps.v1/' // These constants determine the file format structure export const STYLE_FILE = 'style.json' -const SOURCES_FOLDER = 's' +export const SOURCES_FOLDER = 's' const SPRITES_FOLDER = 'sprites' -const FONTS_FOLDER = 'fonts' +export const FONTS_FOLDER = 'fonts' // This must include placeholders `{z}`, `{x}`, `{y}`, since these are used to // define the tile URL, and this is a TileJSON standard. diff --git a/lib/writer.js b/lib/writer.js index 0d2d944..54a3542 100644 --- a/lib/writer.js +++ b/lib/writer.js @@ -14,12 +14,14 @@ import { clone } from './utils/misc.js' import { writeStreamFromAsync } from './utils/streams.js' import { replaceFontStacks } from './utils/style.js' import { + FONTS_FOLDER, getGlyphFilename, getSpriteFilename, getSpriteUri, getTileFilename, getTileUri, GLYPH_URI, + SOURCES_FOLDER, STYLE_FILE, } from './utils/templates.js' @@ -338,11 +340,12 @@ export default class Writer extends EventEmitter { * This method must be called to complete the archive. * You must wait for your destination write stream to 'finish' before using the output. */ - finish() { + async finish() { this.#prepareStyle() const style = JSON.stringify(this.#style) - this.#append(style, { name: STYLE_FILE }) - this.#archive.finalize() + await this.#append(style, { name: STYLE_FILE }) + sortEntries(this.#archive) + await this.#archive.finalize() } /** @@ -476,3 +479,72 @@ function get2DBBox(bbox) { if (bbox.length === 4) return bbox return [bbox[0], bbox[1], bbox[3], bbox[4]] } + +/** + * @typedef {object} ZipEntry + * @property {string} name + */ + +/** + * Dive into the internals of Archiver to sort the central directory entries of + * the zip, so that the style.json, ASCII glyphs, and initial tiles are listed + * first, which improves read speed (the map can be displayed before the entire + * central directory is indexed) + * @param {import('archiver').Archiver} archive + */ +function sortEntries(archive) { + // @ts-expect-error + const entries = /** @type {unknown} */ (archive._module?.engine?._entries) + if (!Array.isArray(entries)) { + throw new Error( + 'Cannot find zip entries: check implementation changes in Archiver', + ) + } + const sortedEntries = entries.sort( + /** + * @param {unknown} a + * @param {unknown} b + */ + function (a, b) { + assertValidEntry(a) + assertValidEntry(b) + if (a.name === 'style.json') return -1 + if (b.name === 'style.json') return 1 + const foldersA = a.name.split('/') + const foldersB = b.name.split('/') + if (foldersA[0] === FONTS_FOLDER && foldersA[2] === '0-255.pbf.gz') + return -1 + if (foldersB[0] === FONTS_FOLDER && foldersB[2] === '0-255.pbf.gz') + return 1 + if (foldersA[0] === SOURCES_FOLDER && foldersB[0] !== SOURCES_FOLDER) + return -1 + if (foldersB[0] === SOURCES_FOLDER && foldersA[0] !== SOURCES_FOLDER) + return 1 + if (foldersA[0] === SOURCES_FOLDER && foldersB[0] === SOURCES_FOLDER) { + const zoomA = +foldersA[2] + const zoomB = +foldersB[2] + return zoomA - zoomB + } + return 0 + }, + ) + // @ts-expect-error + archive._module.engine._entries = sortedEntries +} + +/** + * @param {unknown} maybeEntry + * @returns {asserts maybeEntry is ZipEntry} + */ +function assertValidEntry(maybeEntry) { + if ( + !maybeEntry || + typeof maybeEntry !== 'object' || + !('name' in maybeEntry) || + typeof maybeEntry.name !== 'string' + ) { + throw new Error( + 'Unexpected zip entry type: check implementation changes in Archiver', + ) + } +} diff --git a/test/fixtures/valid-styles/all-types.input.json b/test/fixtures/valid-styles/all-types.input.json new file mode 100644 index 0000000..58604f8 --- /dev/null +++ b/test/fixtures/valid-styles/all-types.input.json @@ -0,0 +1,25 @@ +{ + "version": 8, + "name": "Example (fake)", + "sources": { + "source1": { + "type": "vector", + "url": "https://example.com/source1.json" + }, + "source2": { + "type": "vector", + "url": "https://example.com/source2.json" + } + }, + "glyph": "https://example.com/font/{fontstack}/{range}.pbf", + "sprite": "https://example.com/sprite", + "layers": [ + { + "id": "background", + "type": "background", + "paint": { + "background-color": "#f8f4f0" + } + } + ] +} diff --git a/test/write-read.js b/test/write-read.js index d6b0bd9..6ce6731 100644 --- a/test/write-read.js +++ b/test/write-read.js @@ -1,7 +1,7 @@ import SphericalMercator from '@mapbox/sphericalmercator' import { bbox as turfBbox } from '@turf/bbox' import randomStream from 'random-bytes-readable-stream' -import { fromBuffer as zipFromBuffer } from 'yauzl-promise' +import { fromBuffer, fromBuffer as zipFromBuffer } from 'yauzl-promise' import assert from 'node:assert/strict' import fs from 'node:fs/promises' @@ -573,6 +573,85 @@ test('Raster tiles write and read', async () => { assert.equal(jpgTileHashOut, jpgTileHash, 'JPG tile is the same') }) +test.only('Optimized central directory order', async () => { + const styleInUrl = new URL( + './fixtures/valid-styles/all-types.input.json', + import.meta.url, + ) + + /** @type {import('@maplibre/maplibre-gl-style-spec').StyleSpecification} */ + const styleIn = await readJson(styleInUrl) + const writer = new Writer(styleIn) + + const bounds = /** @type {BBox} */ ([-40.6, -50.6, 151.6, 76.0]) + + for (const { x, y, z } of tileIterator({ maxzoom: 5, bounds })) { + for (const sourceId of ['source1', 'source2']) { + const stream = randomStream({ size: random(2048, 4096) }).pipe( + new DigestStream('md5'), + ) + await writer.addTile(stream, { x, y, z, sourceId, format: 'mvt' }) + } + } + + for (const range of glyphRanges()) { + for (const font of ['font1', 'font2']) { + const stream = randomStream({ size: random(256, 1024) }).pipe( + new DigestStream('md5'), + ) + await writer.addGlyphs(stream, { range, font }) + } + } + + const spriteImageStream = randomStream({ size: random(1024, 2048) }).pipe( + new DigestStream('md5'), + ) + const spriteLayoutIn = { + airfield_11: { + height: 17, + pixelRatio: 1, + width: 17, + x: 21, + y: 0, + }, + } + await writer.addSprite({ + png: spriteImageStream, + json: JSON.stringify(spriteLayoutIn), + }) + + writer.finish() + + const smp = await streamToBuffer(writer.outputStream) + const zip = await fromBuffer(smp) + const entries = await zip.readEntries() + const entriesFilenames = entries.map((e) => e.filename) + + // 1. style.json + // 2. glyphs for 0-255 UTF codes + // 3. sources ordered by zoom level + const expectedFirstEntriesFilenames = [ + 'style.json', + 'fonts/font2/0-255.pbf.gz', + 'fonts/font1/0-255.pbf.gz', + 's/0/0/0/0.mvt.gz', + 's/1/0/0/0.mvt.gz', + 's/0/1/0/0.mvt.gz', + 's/1/1/0/0.mvt.gz', + 's/0/1/0/1.mvt.gz', + 's/1/1/0/1.mvt.gz', + 's/0/1/1/0.mvt.gz', + 's/1/1/1/0.mvt.gz', + 's/0/1/1/1.mvt.gz', + 's/1/1/1/1.mvt.gz', + ] + + assert.deepStrictEqual( + entriesFilenames.slice(0, expectedFirstEntriesFilenames.length), + expectedFirstEntriesFilenames, + ) +}) + /** * * @param {number} min