From f6959a6c06fb086430c0468b035620c76462582d Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Fri, 5 Jan 2024 14:04:44 -0800 Subject: [PATCH 1/4] Stop using PouchDB (#9810) * migrate pouchDB without a dependency on pouchDB * obliterate PouchDb * copy extra info to current db * fix copyToLegacyEditor * don't delete projects from pouchdb --- pxtlib/browserutils.ts | 28 +++ webapp/src/app.tsx | 3 + webapp/src/browserworkspace.ts | 11 +- webapp/src/db.ts | 65 +------ webapp/src/idbworkspace.ts | 312 +++++++++++++++++++++++++++++---- webapp/src/package.ts | 37 ++-- webapp/src/workspace.ts | 45 +++-- 7 files changed, 367 insertions(+), 134 deletions(-) diff --git a/pxtlib/browserutils.ts b/pxtlib/browserutils.ts index 376ccc7b0e72..ec67ac1b5b24 100644 --- a/pxtlib/browserutils.ts +++ b/pxtlib/browserutils.ts @@ -945,6 +945,34 @@ namespace pxt.BrowserUtils { request.onerror = () => this.errorHandler(request.error, "deleteAll", reject); }); } + + public getObjectStoreWrapper(storeName: string): IDBObjectStoreWrapper { + return new IDBObjectStoreWrapper(this, storeName); + } + } + + export class IDBObjectStoreWrapper { + constructor(protected db: IDBWrapper, protected storeName: string) {} + + public getAsync(id: string): Promise { + return this.db.getAsync(this.storeName, id); + } + + public getAllAsync(): Promise { + return this.db.getAllAsync(this.storeName); + } + + public setAsync(data: T): Promise { + return this.db.setAsync(this.storeName, data); + } + + public async deleteAsync(id: string): Promise { + await this.db.deleteAsync(this.storeName, id); + } + + public async deleteAllAsync(): Promise { + await this.db.deleteAllAsync(this.storeName); + } } class IndexedDbTranslationDb implements ITranslationDb { diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index f69ebfe3dc2b..3ad74d4ddf78 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -76,6 +76,7 @@ import { CodeCardView } from "./codecard"; import { mergeProjectCode, appendTemporaryAssets } from "./mergeProjects"; import { Tour } from "./components/onboarding/Tour"; import { parseTourStepsAsync } from "./onboarding"; +import { initGitHubDb } from "./idbworkspace"; pxsim.util.injectPolyphils(); @@ -5735,6 +5736,8 @@ document.addEventListener("DOMContentLoaded", async () => { pxt.analytics.consoleTicks = pxt.analytics.ConsoleTickOptions.Short; } + initGitHubDb(); + pxt.perf.measureStart("setAppTarget"); pkg.setupAppTarget((window as any).pxtTargetBundle); diff --git a/webapp/src/browserworkspace.ts b/webapp/src/browserworkspace.ts index 5267a6c4de8f..78cae1e9ebca 100644 --- a/webapp/src/browserworkspace.ts +++ b/webapp/src/browserworkspace.ts @@ -121,20 +121,13 @@ function setCoreAsync(headers: db.Table, texts: db.Table, h: Header, prevVer: an return headerRes } -export function copyProjectToLegacyEditor(h: Header, majorVersion: number): Promise
{ +export async function copyProjectToLegacyEditor(header: Header, script: pxt.workspace.ScriptText, majorVersion: number): Promise { const prefix = pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[majorVersion]; const oldHeaders = new db.Table(prefix ? `${prefix}-header` : `header`); const oldTexts = new db.Table(prefix ? `${prefix}-text` : `text`); - const header = pxt.Util.clone(h); - delete (header as any)._id; - delete header._rev; - header.id = pxt.Util.guidGen(); - - return getAsync(h) - .then(resp => setCoreAsync(oldHeaders, oldTexts, header, undefined, resp.text)) - .then(rev => header); + await setCoreAsync(oldHeaders, oldTexts, header, undefined, script); } function deleteAsync(h: Header, prevVer: any) { diff --git a/webapp/src/db.ts b/webapp/src/db.ts index 4260ff18d985..a0082147e9fa 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -88,67 +88,4 @@ export class Table { obj._id = this.name + "--" + obj.id return getDbAsync().then(db => db.put(obj)).then((resp: any) => resp.rev) } -} - -class GithubDb implements pxt.github.IGithubDb { - // in memory cache - private mem = new pxt.github.MemoryGithubDb(); - private table = new Table("github"); - - latestVersionAsync(repopath: string, config: pxt.PackagesConfig): Promise { - return this.mem.latestVersionAsync(repopath, config) - } - - loadConfigAsync(repopath: string, tag: string): Promise { - // don't cache master - if (tag == "master") - return this.mem.loadConfigAsync(repopath, tag); - - const id = `config-${repopath}-${tag}`; - return this.table.getAsync(id).then( - entry => { - pxt.debug(`github offline cache hit ${id}`); - return entry.config as pxt.PackageConfig; - }, - e => { - pxt.debug(`github offline cache miss ${id}`); - return this.mem.loadConfigAsync(repopath, tag) - .then(config => { - return this.table.forceSetAsync({ - id, - config - }).then(() => config, e => config); - }) - } // not found - ); - } - loadPackageAsync(repopath: string, tag: string): Promise { - if (!tag) { - pxt.debug(`dep: default to master`) - tag = "master" - } - // don't cache master - if (tag == "master") - return this.mem.loadPackageAsync(repopath, tag); - - const id = `pkg-${repopath}-${tag}`; - return this.table.getAsync(id).then( - entry => { - pxt.debug(`github offline cache hit ${id}`); - return entry.package as pxt.github.CachedPackage; - }, - e => { - pxt.debug(`github offline cache miss ${id}`); - return this.mem.loadPackageAsync(repopath, tag) - .then(p => { - return this.table.forceSetAsync({ - id, - package: p - }).then(() => p, e => p); - }) - } // not found - ); - } -} - -pxt.github.db = new GithubDb(); \ No newline at end of file +} \ No newline at end of file diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index 1890063fc156..7d445379fb61 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -1,9 +1,3 @@ -/** - * A workspace implementation that uses IndexedDB directly (bypassing PouchDB), to support WKWebview where PouchDB - * doesn't work. - */ -import * as browserworkspace from "./browserworkspace" - type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; @@ -17,47 +11,168 @@ interface StoredText { const TEXTS_TABLE = "texts"; const HEADERS_TABLE = "headers"; const KEYPATH = "id"; +export const SCRIPT_TABLE = "script"; +export const GITHUB_TABLE = "github"; +export const HOSTCACHE_TABLE = "hostcache"; + +let _migrationPromise: Promise; +async function performMigrationsAsync() { + if (!_migrationPromise) { + _migrationPromise = (async () => { + try { + await migratePouchAsync(); + } + catch (e) { + pxt.reportException(e); + pxt.log("Unable to migrate pouchDB") + } + + try { + await migrateOldIndexedDbAsync(); + } + catch (e) { + pxt.reportException(e); + pxt.log("Unable to migrate old indexed db format") + } -// This function migrates existing projectes in pouchDb to indexDb -// From browserworkspace to idbworkspace -async function migrateBrowserWorkspaceAsync(): Promise { - const db = await getDbAsync(); - const allDbHeaders = await db.getAllAsync(HEADERS_TABLE); - if (allDbHeaders.length) { - // There are already scripts using the idbworkspace, so a migration has already happened - return; + try { + await migratePrefixesAsync(); + } + catch (e) { + pxt.reportException(e); + pxt.log("Unable to migrate projects from other prefixes"); + } + })(); } - const copyProject = async (h: pxt.workspace.Header): Promise => { - const resp = await browserworkspace.provider.getAsync(h); + return _migrationPromise; +} - // Ignore metadata of the previous script so they get re-generated for the new copy - delete (resp as any)._id; - delete (resp as any)._rev; +async function migratePouchAsync() { + const POUCH_OBJECT_STORE = "by-sequence"; + const oldDb = new pxt.BrowserUtils.IDBWrapper("_pouch_pxt-" + pxt.storage.storageId(), 99999, () => {}); + await oldDb.openAsync(); + const entries = await oldDb.getAllAsync(POUCH_OBJECT_STORE); - await setAsync(h, undefined, resp.text); - }; + for (const entry of entries) { + if (entry._migrated) continue; + // format is (prefix-)?tableName--id::rev + const docId: string = entry._doc_id_rev; + + const revSeparatorIndex = docId.lastIndexOf("::"); + const rev = docId.substring(revSeparatorIndex + 2); + + const tableSeparatorIndex = docId.indexOf("--"); + let table = docId.substring(0, tableSeparatorIndex); + + const id = docId.substring(tableSeparatorIndex + 2, revSeparatorIndex); + + let prefix: string; + let prefixSeparatorIndex = table.indexOf("-") + if (prefixSeparatorIndex !== -1) { + prefix = table.substring(0, prefixSeparatorIndex); + table = table.substring(prefixSeparatorIndex + 1); + } + + pxtc.assert(id === entry.id, "ID mismatch!"); - const previousHeaders = await browserworkspace.provider.listAsync(); + switch (table) { + case "header": + table = HEADERS_TABLE; + break; + case "text": + table = TEXTS_TABLE; + break; + case "script": + table = SCRIPT_TABLE; + prefix = prefix || getCurrentDBPrefix(); + break; + case "github": + table = GITHUB_TABLE; + prefix = prefix || getCurrentDBPrefix(); + break; + case "hostcache": + table = HOSTCACHE_TABLE; + prefix = prefix || getCurrentDBPrefix(); + break; + default: + console.warn("Unknown database table " + table); + continue; + } - await Promise.all(previousHeaders.map(h => copyProject(h))); + const db = await getDbAsync(prefix) + const existing = await db.getAsync(table, id); + + if (!existing) { + await db.setAsync(table, entry); + } + + entry._migrated = true; + await oldDb.setAsync(POUCH_OBJECT_STORE, entry); + } } -let _dbPromise: Promise; -async function getDbAsync(): Promise { - if (_dbPromise) { - return await _dbPromise; +async function migrateOldIndexedDbAsync() { + const legacyDb = new pxt.BrowserUtils.IDBWrapper(`__pxt_idb_workspace`, 1, (ev, r) => { + const db = r.result as IDBDatabase; + db.createObjectStore(TEXTS_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(HEADERS_TABLE, { keyPath: KEYPATH }); + }, async () => { + await pxt.BrowserUtils.clearTranslationDbAsync(); + await pxt.BrowserUtils.clearTutorialInfoDbAsync(); + }); + + try { + await legacyDb.openAsync(); + const currentDb = await getCurrentDbAsync(); + + await copyTableEntriesAsync(legacyDb, currentDb, HEADERS_TABLE, true); + await copyTableEntriesAsync(legacyDb, currentDb, TEXTS_TABLE, true); + } catch (e) { + pxt.reportException(e); } +} + +async function migratePrefixesAsync() { + if (!getCurrentDBPrefix()) return; + + const currentVersion = pxt.semver.parse(pxt.appTarget.versions.target); + const currentMajor = currentVersion.major; + const previousMajor = currentMajor - 1; + const previousDbPrefix = previousMajor < 0 ? "" : pxt.appTarget.appTheme.browserDbPrefixes[previousMajor]; + + if (!previousDbPrefix) return; + const currentDb = await getCurrentDbAsync(); - _dbPromise = createDbAsync(); + // If headers are already in the new db, migration must have already happened + if ((await currentDb.getAllAsync(HEADERS_TABLE)).length) return; - return _dbPromise; + const prevDb = await getDbAsync(previousDbPrefix); + + await copyTableEntriesAsync(prevDb, currentDb, HEADERS_TABLE, false); + await copyTableEntriesAsync(prevDb, currentDb, TEXTS_TABLE, false); + await copyTableEntriesAsync(prevDb, currentDb, SCRIPT_TABLE, false); + await copyTableEntriesAsync(prevDb, currentDb, HOSTCACHE_TABLE, false); + await copyTableEntriesAsync(prevDb, currentDb, GITHUB_TABLE, false); +} + +let _dbPromises: pxt.Map> = {}; + +async function getDbAsync(prefix = "__default") { + if (_dbPromises[prefix]) return _dbPromises[prefix]; + + _dbPromises[prefix] = createDbAsync(); + + return _dbPromises[prefix]; async function createDbAsync(): Promise { - const idbDb = new pxt.BrowserUtils.IDBWrapper("__pxt_idb_workspace", 1, (ev, r) => { + const idbDb = new pxt.BrowserUtils.IDBWrapper(`__pxt_idb_workspace_${pxt.storage.storageId()}_${prefix}`, 1, (ev, r) => { const db = r.result as IDBDatabase; db.createObjectStore(TEXTS_TABLE, { keyPath: KEYPATH }); db.createObjectStore(HEADERS_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(SCRIPT_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(HOSTCACHE_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(GITHUB_TABLE, { keyPath: KEYPATH }); }, async () => { await pxt.BrowserUtils.clearTranslationDbAsync(); await pxt.BrowserUtils.clearTutorialInfoDbAsync(); @@ -74,14 +189,36 @@ async function getDbAsync(): Promise { } } +async function copyTableEntriesAsync(fromDb: pxt.BrowserUtils.IDBWrapper, toDb: pxt.BrowserUtils.IDBWrapper, storeName: string, dontOverwrite: boolean) { + for (const entry of await fromDb.getAllAsync(storeName)) { + const existing = dontOverwrite && !!(await toDb.getAsync(storeName, entry.id)); + + if (!existing) { + await toDb.setAsync(storeName, entry); + } + } +} + +async function getCurrentDbAsync(): Promise { + return getDbAsync(getCurrentDBPrefix()); +} + +function getCurrentDBPrefix() { + if (!pxt.appTarget.appTheme.browserDbPrefixes) return undefined; + + const currentVersion = pxt.semver.parse(pxt.appTarget.versions.target); + const currentMajor = currentVersion.major; + return pxt.appTarget.appTheme.browserDbPrefixes[currentMajor]; +} + async function listAsync(): Promise { - await migrateBrowserWorkspaceAsync(); - const db = await getDbAsync(); + await performMigrationsAsync(); + const db = await getCurrentDbAsync(); return db.getAllAsync(HEADERS_TABLE); } async function getAsync(h: Header): Promise { - const db = await getDbAsync(); + const db = await getCurrentDbAsync(); const res = await db.getAsync(TEXTS_TABLE, h.id); return { header: h, @@ -91,7 +228,7 @@ async function getAsync(h: Header): Promise { } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { - const db = await getDbAsync(); + const db = await getCurrentDbAsync(); try { await setCoreAsync(db, h, prevVer, text); @@ -137,17 +274,120 @@ async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { - const db = await getDbAsync(); + const db = await getCurrentDbAsync(); await db.deleteAsync(TEXTS_TABLE, h.id); await db.deleteAsync(HEADERS_TABLE, h.id); } async function resetAsync(): Promise { - const db = await getDbAsync(); + const db = await getCurrentDbAsync(); await db.deleteAllAsync(TEXTS_TABLE); await db.deleteAllAsync(HEADERS_TABLE); } +export async function getObjectStoreAsync(storeName: string) { + const db = await getCurrentDbAsync(); + return db.getObjectStoreWrapper(storeName); +} + +export async function copyProjectToLegacyEditorAsync(header: Header, script: pxt.workspace.ScriptText, majorVersion: number): Promise { + const prefix = pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[majorVersion]; + + const oldDB = await getDbAsync(prefix); + + await oldDB.setAsync(HEADERS_TABLE, header); + await oldDB.setAsync(TEXTS_TABLE, { + id: header.id, + files: script, + _rev: null + } as StoredText); +} + +export function initGitHubDb() { + class GithubDb implements pxt.github.IGithubDb { + // in memory cache + private mem = new pxt.github.MemoryGithubDb(); + + latestVersionAsync(repopath: string, config: pxt.PackagesConfig): Promise { + return this.mem.latestVersionAsync(repopath, config) + } + + async loadConfigAsync(repopath: string, tag: string): Promise { + // don't cache master + if (tag == "master") + return this.mem.loadConfigAsync(repopath, tag); + + const id = `config-${repopath}-${tag}`; + + const cache = await getGitHubCacheAsync(); + + try { + const entry = await cache.getAsync(id); + return entry.config; + } + catch (e) { + pxt.debug(`github offline cache miss ${id}`); + const config = await this.mem.loadConfigAsync(repopath, tag); + + try { + await cache.setAsync({ + id, + config + }) + } + catch (e) { + } + + return config; + } + } + + async loadPackageAsync(repopath: string, tag: string): Promise { + if (!tag) { + pxt.debug(`dep: default to master`) + tag = "master" + } + // don't cache master + if (tag == "master") + return this.mem.loadPackageAsync(repopath, tag); + + const id = `pkg-${repopath}-${tag}`; + const cache = await getGitHubCacheAsync(); + + try { + const entry = await cache.getAsync(id); + pxt.debug(`github offline cache hit ${id}`); + return entry.package; + } + catch (e) { + pxt.debug(`github offline cache miss ${id}`); + const p = await this.mem.loadPackageAsync(repopath, tag); + + try { + await cache.setAsync({ + id, + package: p + }) + } + catch (e) { + } + + return p; + } + } + } + + function getGitHubCacheAsync() { + return getObjectStoreAsync<{ + id: string; + package?: pxt.github.CachedPackage; + config?: pxt.PackageConfig; + }>(GITHUB_TABLE) + } + + pxt.github.db = new GithubDb(); +} + export const provider: WorkspaceProvider = { getAsync, setAsync, diff --git a/webapp/src/package.ts b/webapp/src/package.ts index 3a9464f93cff..a1925d6701c0 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -1,15 +1,10 @@ import * as workspace from "./workspace"; import * as data from "./data"; import * as core from "./core"; -import * as db from "./db"; import * as compiler from "./compiler"; -import * as auth from "./auth"; -import * as toolbox from "./toolbox" import Util = pxt.Util; -import { getBlocksEditor } from "./app"; - -let hostCache = new db.Table("hostcache") +import { HOSTCACHE_TABLE, getObjectStoreAsync } from "./idbworkspace"; let extWeight: pxt.Map = { "ts": 10, @@ -686,19 +681,27 @@ class Host return pxt.hexloader.getHexInfoAsync(this, extInfo).catch(core.handleNetworkError); } - cacheStoreAsync(id: string, val: string): Promise { - return hostCache.forceSetAsync({ - id: id, - val: val - }).then(() => { }, e => { + async cacheStoreAsync(id: string, val: string): Promise { + const hostCache = await getHostCacheAsync(); + + try { + await hostCache.setAsync({ id, val }); + } + catch (e) { pxt.tickEvent('cache.store.failed', { error: e.name }); pxt.log(`cache store failed for ${id}: ${e.name}`) - }) + } } - cacheGetAsync(id: string): Promise { - return hostCache.getAsync(id) - .then(v => v.val, e => null) + async cacheGetAsync(id: string): Promise { + const hostCache = await getHostCacheAsync(); + + try { + return (await hostCache.getAsync(id)).val; + } + catch (e) { + return null; + } } downloadPackageAsync(pkg: pxt.Package): Promise { @@ -993,3 +996,7 @@ export function getExtensionOfFileName(filename: string) { if (m) return m[1] return "" } + +async function getHostCacheAsync(): Promise> { + return getObjectStoreAsync(HOSTCACHE_TABLE) +} \ No newline at end of file diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 6824a0abe594..d14f6d081330 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -53,11 +53,31 @@ export function gitsha(data: string, encoding: "utf-8" | "base64" = "utf-8") { return (sha1("blob " + U.toUTF8(data).length + "\u0000" + data) + "") } -export function copyProjectToLegacyEditor(header: Header, majorVersion: number): Promise
{ +export async function copyProjectToLegacyEditor(header: Header, majorVersion: number): Promise
{ if (!isBrowserWorkspace()) { return Promise.reject("Copy operation only works in browser workspace"); } - return browserworkspace.copyProjectToLegacyEditor(header, majorVersion); + + const script = await getTextAsync(header.id); + + const newHeader = pxt.Util.clone(header); + delete (newHeader as any)._id; + delete newHeader._rev; + newHeader.id = pxt.Util.guidGen(); + + // We don't know if the legacy editor uses the indexedDB or PouchDB workspace, so we're going + // to copy the project to both places + try { + await browserworkspace.copyProjectToLegacyEditor(newHeader, script, majorVersion); + } + catch (e) { + pxt.reportException(e); + pxt.log("Unable to port project to PouchDB") + } + + await indexedDBWorkspace.copyProjectToLegacyEditorAsync(newHeader, script, majorVersion); + + return newHeader; } export function setupWorkspace(id: string) { @@ -78,12 +98,13 @@ export function setupWorkspace(id: string) { // Iframe workspace, the editor relays sync messages back and forth when hosted in an iframe impl = iframeworkspace.provider; break; - case "idb": - impl = indexedDBWorkspace.provider; + case "pouch": + impl = browserworkspace.provider break; + case "idb": case "browser": default: - impl = browserworkspace.provider + impl = indexedDBWorkspace.provider; break; } } @@ -269,7 +290,7 @@ export function getLastCloudSync(): number { export function initAsync() { if (!impl) { - impl = browserworkspace.provider; + impl = indexedDBWorkspace.provider; implType = "browser"; } @@ -809,7 +830,6 @@ export function fixupFileNames(txt: ScriptText) { const scriptDlQ = new U.PromiseQueue(); -const scripts = new db.Table("script"); // cache for published scripts export async function getPublishedScriptAsync(id: string) { if (pxt.github.isGithubId(id)) id = pxt.github.normalizeRepoId(id) @@ -817,8 +837,9 @@ export async function getPublishedScriptAsync(id: string) { const eid = encodeURIComponent(pxt.github.upgradedPackageId(config, id)) return await scriptDlQ.enqueue(eid, async () => { let files: ScriptText + const scriptCache = await getScriptCacheAsync(); try { - files = (await scripts.getAsync(eid)).files + files = (await scriptCache.getAsync(eid)).files } catch { if (pxt.github.isGithubId(id)) { files = (await pxt.github.downloadPackageAsync(id, config)).files @@ -827,7 +848,7 @@ export async function getPublishedScriptAsync(id: string) { .catch(core.handleNetworkError)) } try { - await scripts.setAsync({ id: eid, files: files }) + await scriptCache.setAsync({ id: eid, files: files }) } catch (e) { // Don't fail if the indexeddb fails, but log it @@ -1720,7 +1741,7 @@ export function listAssetsAsync(id: string): Promise { } export function isBrowserWorkspace() { - return impl === browserworkspace.provider; + return impl === indexedDBWorkspace.provider; } export function fireEvent(ev: pxt.editor.events.Event) { @@ -1746,6 +1767,10 @@ export function dbgHdrToString(h: Header): string { return `${h.name} ${h.id.substr(0, 4)}..v${dbgShorten(h.cloudVersion)}@${h.modificationTime % 100}-${U.timeSince(h.modificationTime)}`; } +async function getScriptCacheAsync(): Promise> { + return indexedDBWorkspace.getObjectStoreAsync(indexedDBWorkspace.SCRIPT_TABLE) +} + /* header: - one header header:* - all headers From a59a6aa75bddb563dc5ffccdc4f98d3dea199984 Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Tue, 9 Jan 2024 13:35:45 -0800 Subject: [PATCH 2/4] Don't give a ridiculously high pouchdb version (#9819) --- webapp/src/idbworkspace.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index 7d445379fb61..a914f6548f34 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -50,7 +50,7 @@ async function performMigrationsAsync() { async function migratePouchAsync() { const POUCH_OBJECT_STORE = "by-sequence"; - const oldDb = new pxt.BrowserUtils.IDBWrapper("_pouch_pxt-" + pxt.storage.storageId(), 99999, () => {}); + const oldDb = new pxt.BrowserUtils.IDBWrapper("_pouch_pxt-" + pxt.storage.storageId(), 5, () => {}); await oldDb.openAsync(); const entries = await oldDb.getAllAsync(POUCH_OBJECT_STORE); @@ -394,4 +394,4 @@ export const provider: WorkspaceProvider = { deleteAsync, listAsync, resetAsync, -} \ No newline at end of file +} From b5fea9bce93a464f21b131d7f83d477b35ce482c Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Wed, 10 Jan 2024 10:48:08 -0800 Subject: [PATCH 3/4] fix pouch db migration edge cases (#9820) --- pxtlib/browserutils.ts | 7 +++- webapp/src/idbworkspace.ts | 66 ++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/pxtlib/browserutils.ts b/pxtlib/browserutils.ts index ec67ac1b5b24..1e8bb9c02d6c 100644 --- a/pxtlib/browserutils.ts +++ b/pxtlib/browserutils.ts @@ -835,7 +835,8 @@ namespace pxt.BrowserUtils { private name: string, private version: number, private upgradeHandler?: IDBUpgradeHandler, - private quotaExceededHandler?: () => void) { + private quotaExceededHandler?: () => void, + private skipErrorLog = false) { } private throwIfNotOpened(): void { @@ -845,6 +846,10 @@ namespace pxt.BrowserUtils { } private errorHandler(err: Error, op: string, reject: (err: Error) => void): void { + if (this.skipErrorLog) { + reject(err); + return; + } console.error(new Error(`${this.name} IDBWrapper error for ${op}: ${err.message}`)); reject(err); // special case for quota exceeded diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index a914f6548f34..2df4c163060e 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -48,14 +48,42 @@ async function performMigrationsAsync() { return _migrationPromise; } +const POUCH_OBJECT_STORE = "by-sequence"; +const POUCH_DB_VERSION = 5; + +async function checkIfPouchDbExistsAsync() { + // Unfortunately, there is no simple cross-browser way to check + // if an indexedDb already exists. This works by requesting the + // db with a version lower than the current version. If it + // throws an exception, then the db must already exist with the + // higher version. If it tries to upgrade, then the db doesn't + // exist. We abort the transaction to avoid poisoning pouchdb + // if anyone ever visits an old version of the editor. + let result = true; + try { + const db = new pxt.BrowserUtils.IDBWrapper("_pouch_pxt-" + pxt.storage.storageId(), 1, (e, r) => { + result = false; + r.transaction.abort(); + }, null, true); + await db.openAsync(); + } + catch (e) { + // This will always throw an exception + } + + return result; +} + async function migratePouchAsync() { - const POUCH_OBJECT_STORE = "by-sequence"; - const oldDb = new pxt.BrowserUtils.IDBWrapper("_pouch_pxt-" + pxt.storage.storageId(), 5, () => {}); + if (!await checkIfPouchDbExistsAsync()) return; + + const oldDb = new pxt.BrowserUtils.IDBWrapper("_pouch_pxt-" + pxt.storage.storageId(), POUCH_DB_VERSION, () => {}); await oldDb.openAsync(); + const entries = await oldDb.getAllAsync(POUCH_OBJECT_STORE); + const alreadyMigratedList = await getMigrationDbAsync(); for (const entry of entries) { - if (entry._migrated) continue; // format is (prefix-)?tableName--id::rev const docId: string = entry._doc_id_rev; @@ -100,15 +128,18 @@ async function migratePouchAsync() { continue; } + if (await alreadyMigratedList.getAsync(table, id)) { + continue; + } + + alreadyMigratedList.setAsync(table, { id }); + const db = await getDbAsync(prefix) const existing = await db.getAsync(table, id); if (!existing) { await db.setAsync(table, entry); } - - entry._migrated = true; - await oldDb.setAsync(POUCH_OBJECT_STORE, entry); } } @@ -303,6 +334,29 @@ export async function copyProjectToLegacyEditorAsync(header: Header, script: pxt } as StoredText); } +async function getMigrationDbAsync() { + const idbDb = new pxt.BrowserUtils.IDBWrapper(`__pxt_idb_migration_${pxt.storage.storageId()}`, 1, (ev, r) => { + const db = r.result as IDBDatabase; + db.createObjectStore(TEXTS_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(HEADERS_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(SCRIPT_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(HOSTCACHE_TABLE, { keyPath: KEYPATH }); + db.createObjectStore(GITHUB_TABLE, { keyPath: KEYPATH }); + }, async () => { + await pxt.BrowserUtils.clearTranslationDbAsync(); + await pxt.BrowserUtils.clearTutorialInfoDbAsync(); + }); + + try { + await idbDb.openAsync(); + } catch (e) { + pxt.reportException(e); + return Promise.reject(e); + } + + return idbDb; +} + export function initGitHubDb() { class GithubDb implements pxt.github.IGithubDb { // in memory cache From 84ccf1f98da35413c3bfe09454283de5b50e5bee Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Thu, 11 Jan 2024 16:25:33 -0800 Subject: [PATCH 4/4] delete the _doc_id_rev when migrating entries (#9822) --- webapp/src/idbworkspace.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index 2df4c163060e..048630d0ce5f 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -138,6 +138,7 @@ async function migratePouchAsync() { const existing = await db.getAsync(table, id); if (!existing) { + delete entry._doc_id_rev await db.setAsync(table, entry); } }