From bfb542894e2083d4dd5b4c37a5a657a97207d895 Mon Sep 17 00:00:00 2001 From: Julian Bilcke Date: Mon, 22 Jul 2024 02:03:43 +0200 Subject: [PATCH] work on video import --- package-lock.json | 365 ++++++++++--------- package.json | 4 +- src/lib/core/constants.ts | 2 +- src/lib/utils/base64DataUriToFile.ts | 8 +- src/services/io/extractCaptionsFromFrames.ts | 13 +- src/services/io/parseFileIntoSegments.ts | 89 +++-- src/services/io/useIO.ts | 43 ++- 7 files changed, 291 insertions(+), 233 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6337c889..b76f0f9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,9 +11,9 @@ "dependencies": { "@aitube/broadway": "0.0.22", "@aitube/clap": "0.0.30", - "@aitube/clapper-services": "0.0.28", + "@aitube/clapper-services": "0.0.29", "@aitube/engine": "0.0.26", - "@aitube/timeline": "0.0.42", + "@aitube/timeline": "0.0.43", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", @@ -161,12 +161,12 @@ } }, "node_modules/@aitube/clapper-services": { - "version": "0.0.28", - "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.28.tgz", - "integrity": "sha512-YmiPGAGtZcgqqnXmgtWvFyd9TL5eCpF343+ndvDa+aorQ43PkIKo7f3df550njOICM5KZhSBRFEIh1c5Dsi7Yg==", + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.29.tgz", + "integrity": "sha512-61UH/TQwPcvXArEkPnGNm+IQulaW3zNh73pzihdU2kkqufGzUYCNSd/jHJh9dLqQm3lZtm6QMN2RReFrzGuLNQ==", "peerDependencies": { "@aitube/clap": "0.0.30", - "@aitube/timeline": "0.0.42", + "@aitube/timeline": "0.0.43", "@monaco-editor/react": "4.6.0", "monaco-editor": "0.50.0", "react": "*", @@ -192,9 +192,9 @@ } }, "node_modules/@aitube/timeline": { - "version": "0.0.42", - "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.42.tgz", - "integrity": "sha512-H9vvHrrOBsTpU7AeM1+PJ7TbGr0Jb63aRGBXXL8CUVawX2L92BIkoEBXhGSVePP6yeS6oUv/u86Z2rHPLW9ChQ==", + "version": "0.0.43", + "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.43.tgz", + "integrity": "sha512-TnzKrB955YeDKOMWsnniGbQ+qulCmGptMfhNjDLEqA6jRcsnPVUFCR2dQBqWGNn6KfFPXmDvSi0Sihy7Oj98Aw==", "dependencies": { "date-fns": "^3.6.0", "react-virtualized-auto-sizer": "^1.0.24" @@ -2551,6 +2551,18 @@ "node": ">=12" } }, + "node_modules/@electron/rebuild/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/@electron/rebuild/node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -2613,6 +2625,18 @@ "node": ">=10" } }, + "node_modules/@electron/rebuild/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@electron/rebuild/node_modules/minipass-collect": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", @@ -2732,6 +2756,12 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/@electron/rebuild/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/@electron/universal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@electron/universal/-/universal-2.0.1.tgz", @@ -3884,24 +3914,24 @@ } }, "node_modules/@inquirer/confirm": { - "version": "3.1.16", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.16.tgz", - "integrity": "sha512-DXgLZim+YVTk05zRywvFRfJt2Jje7sZ4DO6Ss9RpGtgXEd/T0IiTqubHWst0IazCwdPI9g/06Rtm/nm4IBFJBA==", + "version": "3.1.17", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.17.tgz", + "integrity": "sha512-qCpt/AABzPynz8tr69VDvhcjwmzAryipWXtW8Vi6m651da4H/d0Bdn55LkxXD7Rp2gfgxvxzTdb66AhIA8gzBA==", "dependencies": { - "@inquirer/core": "^9.0.4", - "@inquirer/type": "^1.5.0" + "@inquirer/core": "^9.0.5", + "@inquirer/type": "^1.5.1" }, "engines": { "node": ">=18" } }, "node_modules/@inquirer/core": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.0.4.tgz", - "integrity": "sha512-46LaWACIctSfVKTu71ziFlqO8SVLhWGSxvaHpf0frfDTphSSpIfeNo5ZH/kJPHYJw4VgPGf/9c3zJN/FnCdaIQ==", + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.0.5.tgz", + "integrity": "sha512-QWG41I7vn62O9stYKg/juKXt1PEbr/4ZZCPb4KgXDQGwgA9M5NBTQ7FnOvT1ridbxkm/wTxLCNraUs7y47pIRQ==", "dependencies": { - "@inquirer/figures": "^1.0.4", - "@inquirer/type": "^1.5.0", + "@inquirer/figures": "^1.0.5", + "@inquirer/type": "^1.5.1", "@types/mute-stream": "^0.0.4", "@types/node": "^20.14.11", "@types/wrap-ansi": "^3.0.0", @@ -4019,17 +4049,17 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.4.tgz", - "integrity": "sha512-R7Gsg6elpuqdn55fBH2y9oYzrU/yKrSmIsDX4ROT51vohrECFzTf2zw9BfUbOW8xjfmM2QbVoVYdTwhrtEKWSQ==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.5.tgz", + "integrity": "sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA==", "engines": { "node": ">=18" } }, "node_modules/@inquirer/type": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.0.tgz", - "integrity": "sha512-L/UdayX9Z1lLN+itoTKqJ/X4DX5DaWu2Sruwt4XgZzMNv32x4qllbzMX4MbJlz0yxAQtU19UvABGOjmdq1u3qA==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.1.tgz", + "integrity": "sha512-m3YgGQlKNS0BM+8AFiJkCsTqHEFCWn6s/Rqye3mYwvqY6LdfUv12eSwbsgNzrYyrLXiy7IrrjDLPysaSBwEfhw==", "dependencies": { "mute-stream": "^1.0.0" }, @@ -4090,15 +4120,6 @@ "node": ">=18.0.0" } }, - "node_modules/@isaacs/fs-minipass/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "optional": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -4487,15 +4508,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@next/eslint-plugin-next/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/@next/swc-darwin-arm64": { "version": "14.2.5", "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.5.tgz", @@ -6062,25 +6074,25 @@ "integrity": "sha512-/RVXdLvJxLg4QKvMoM5WlwNR9ViO9z8B/qPcc+C0Sa/teJY7QG7kJ441DwzOjMYEY7GmU4dj5EcGHIkKZiQZCA==" }, "node_modules/@react-spring/animated": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.3.tgz", - "integrity": "sha512-5CWeNJt9pNgyvuSzQH+uy2pvTg8Y4/OisoscZIR8/ZNLIOI+CatFBhGZpDGTF/OzdNFsAoGk3wiUYTwoJ0YIvw==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.4.tgz", + "integrity": "sha512-7As+8Pty2QlemJ9O5ecsuPKjmO0NKvmVkRR1n6mEotFgWar8FKuQt2xgxz3RTgxcccghpx1YdS1FCdElQNexmQ==", "dependencies": { - "@react-spring/shared": "~9.7.3", - "@react-spring/types": "~9.7.3" + "@react-spring/shared": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, "node_modules/@react-spring/core": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.3.tgz", - "integrity": "sha512-IqFdPVf3ZOC1Cx7+M0cXf4odNLxDC+n7IN3MDcVCTIOSBfqEcBebSv+vlY5AhM0zw05PDbjKrNmBpzv/AqpjnQ==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.4.tgz", + "integrity": "sha512-GzjA44niEJBFUe9jN3zubRDDDP2E4tBlhNlSIkTChiNf9p4ZQlgXBg50qbXfSXHQPHak/ExYxwhipKVsQ/sUTw==", "dependencies": { - "@react-spring/animated": "~9.7.3", - "@react-spring/shared": "~9.7.3", - "@react-spring/types": "~9.7.3" + "@react-spring/animated": "~9.7.4", + "@react-spring/shared": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "funding": { "type": "opencollective", @@ -6091,30 +6103,31 @@ } }, "node_modules/@react-spring/rafz": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.6.1.tgz", - "integrity": "sha512-v6qbgNRpztJFFfSE3e2W1Uz+g8KnIBs6SmzCzcVVF61GdGfGOuBrbjIcp+nUz301awVmREKi4eMQb2Ab2gGgyQ==" + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.7.4.tgz", + "integrity": "sha512-mqDI6rW0Ca8IdryOMiXRhMtVGiEGLIO89vIOyFQXRIwwIMX30HLya24g9z4olDvFyeDW3+kibiKwtZnA4xhldA==" }, "node_modules/@react-spring/shared": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.3.tgz", - "integrity": "sha512-NEopD+9S5xYyQ0pGtioacLhL2luflh6HACSSDUZOwLHoxA5eku1UPuqcJqjwSD6luKjjLfiLOspxo43FUHKKSA==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.4.tgz", + "integrity": "sha512-bEPI7cQp94dOtCFSEYpxvLxj0+xQfB5r9Ru1h8OMycsIq7zFZon1G0sHrBLaLQIWeMCllc4tVDYRTLIRv70C8w==", "dependencies": { - "@react-spring/types": "~9.7.3" + "@react-spring/rafz": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, "node_modules/@react-spring/three": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.3.tgz", - "integrity": "sha512-Q1p512CqUlmMK8UMBF/Rj79qndhOWq4XUTayxMP9S892jiXzWQuj+xC3Xvm59DP/D4JXusXpxxqfgoH+hmOktA==", + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.4.tgz", + "integrity": "sha512-HKUhrrvWW7F/MAroObOloqcYyFqsUHp1ANIDvPVxk9cSh7veW7gQbJm2Sc7Ka+L4gVJEwSkS+MRfr8kk+sRZBw==", "dependencies": { - "@react-spring/animated": "~9.7.3", - "@react-spring/core": "~9.7.3", - "@react-spring/shared": "~9.7.3", - "@react-spring/types": "~9.7.3" + "@react-spring/animated": "~9.7.4", + "@react-spring/core": "~9.7.4", + "@react-spring/shared": "~9.7.4", + "@react-spring/types": "~9.7.4" }, "peerDependencies": { "@react-three/fiber": ">=6.0", @@ -6123,9 +6136,9 @@ } }, "node_modules/@react-spring/types": { - "version": "9.7.3", - "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.3.tgz", - "integrity": "sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw==" + "version": "9.7.4", + "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.4.tgz", + "integrity": "sha512-iQVztO09ZVfsletMiY+DpT/JRiBntdsdJ4uqk3UJFhrhS8mIC9ZOZbmfGSRs/kdbNPQkVyzucceDicQ/3Mlj9g==" }, "node_modules/@react-three/drei": { "version": "9.108.4", @@ -6198,6 +6211,11 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@react-three/drei/node_modules/@react-spring/rafz": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.6.1.tgz", + "integrity": "sha512-v6qbgNRpztJFFfSE3e2W1Uz+g8KnIBs6SmzCzcVVF61GdGfGOuBrbjIcp+nUz301awVmREKi4eMQb2Ab2gGgyQ==" + }, "node_modules/@react-three/drei/node_modules/@react-spring/shared": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.6.1.tgz", @@ -8840,18 +8858,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/cacache/node_modules/fs-minipass": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", - "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", - "dev": true, - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, "node_modules/cacache/node_modules/glob": { "version": "10.4.5", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", @@ -8908,15 +8914,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/cacache/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/cacheable-lookup": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", @@ -9022,9 +9019,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001642", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz", - "integrity": "sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA==", + "version": "1.0.30001643", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001643.tgz", + "integrity": "sha512-ERgWGNleEilSrHM6iUz/zJNSQTP8Mr21wDWpdgvRwcTXGAq6jMtOUPP4dqFPTdKqZ2wKTdtB+uucZ3MRpAUSmg==", "funding": [ { "type": "opencollective", @@ -12377,15 +12374,15 @@ } }, "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", + "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", "dev": true, "dependencies": { - "minipass": "^3.0.0" + "minipass": "^7.0.3" }, "engines": { - "node": ">= 8" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/fs-temp": { @@ -14638,15 +14635,6 @@ "node": "^16.14.0 || >=18.0.0" } }, - "node_modules/make-fetch-happen/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/map-age-cleaner": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", @@ -14807,15 +14795,11 @@ } }, "node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" } }, "node_modules/minipass-collect": { @@ -14830,15 +14814,6 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/minipass-collect/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/minipass-fetch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.5.tgz", @@ -14856,15 +14831,6 @@ "encoding": "^0.1.13" } }, - "node_modules/minipass-fetch/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/minipass-flush": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", @@ -14877,6 +14843,24 @@ "node": ">= 8" } }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/minipass-pipeline": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", @@ -14889,6 +14873,24 @@ "node": ">=8" } }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/minipass-sized": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", @@ -14901,7 +14903,19 @@ "node": ">=8" } }, - "node_modules/minipass/node_modules/yallist": { + "node_modules/minipass-sized/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", @@ -14920,6 +14934,18 @@ "node": ">= 8" } }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/minizlib/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -15401,15 +15427,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/node-gyp/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/node-gyp/node_modules/which": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", @@ -15813,15 +15830,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/onnxruntime-node/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "optional": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/onnxruntime-node/node_modules/minizlib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", @@ -16258,14 +16266,6 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, - "node_modules/path-scurry/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/path-to-regexp": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", @@ -18267,15 +18267,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/ssri/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -18790,14 +18781,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/sucrase/node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/sudo-prompt": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-9.2.1.tgz", @@ -18943,6 +18926,30 @@ "node": ">=10" } }, + "node_modules/tar/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/tar/node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", diff --git a/package.json b/package.json index bb62ae79..3db6581f 100644 --- a/package.json +++ b/package.json @@ -37,9 +37,9 @@ "dependencies": { "@aitube/broadway": "0.0.22", "@aitube/clap": "0.0.30", - "@aitube/clapper-services": "0.0.28", + "@aitube/clapper-services": "0.0.29", "@aitube/engine": "0.0.26", - "@aitube/timeline": "0.0.42", + "@aitube/timeline": "0.0.43", "@fal-ai/serverless-client": "^0.13.0", "@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/util": "^0.12.1", diff --git a/src/lib/core/constants.ts b/src/lib/core/constants.ts index 08169274..8942edfc 100644 --- a/src/lib/core/constants.ts +++ b/src/lib/core/constants.ts @@ -3,7 +3,7 @@ export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32 export const APP_NAME = 'Clapper.app' -export const APP_REVISION = 'r20240721-0835' +export const APP_REVISION = 'r20240722-0205' export const APP_DOMAIN = 'Clapper.app' export const APP_LINK = 'https://clapper.app' diff --git a/src/lib/utils/base64DataUriToFile.ts b/src/lib/utils/base64DataUriToFile.ts index 404a737b..184b9aca 100644 --- a/src/lib/utils/base64DataUriToFile.ts +++ b/src/lib/utils/base64DataUriToFile.ts @@ -5,8 +5,8 @@ export function base64DataUriToFile(dataUrl: string, fileName: string) { const bstr = atob(arr[arr.length - 1]) let n = bstr.length const u8arr = new Uint8Array(n) - while(n--){ - u8arr[n] = bstr.charCodeAt(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n) } - return new File([u8arr], fileName, {type:mime}); -} \ No newline at end of file + return new File([u8arr], fileName, { type: mime }) +} diff --git a/src/services/io/extractCaptionsFromFrames.ts b/src/services/io/extractCaptionsFromFrames.ts index 575ba547..2017b1ac 100644 --- a/src/services/io/extractCaptionsFromFrames.ts +++ b/src/services/io/extractCaptionsFromFrames.ts @@ -7,7 +7,11 @@ import { export async function extractCaptionsFromFrames( images: string[] = [], - onProgress: (progress: number, storyboardIndex: number, nbStoryboards: number) => void + onProgress: ( + progress: number, + storyboardIndex: number, + nbStoryboards: number + ) => void ): Promise { if (!(navigator as any).gpu) { throw new Error(`Please enable WebGPU to analyze video frames: @@ -40,15 +44,15 @@ Linux experimental support also requires launching the browser with --enable-fea } ) - onProgress(progress = 5, 0, images.length) + onProgress((progress = 5), 0, images.length) const processor = await AutoProcessor.from_pretrained(model_id) - onProgress(progress = 10, 0, images.length) + onProgress((progress = 10), 0, images.length) const tokenizer = await AutoTokenizer.from_pretrained(model_id) - onProgress(progress = 15, 0, images.length) + onProgress((progress = 15), 0, images.length) // not all prompts will work properly, see the official examples: // https://huggingface.co/microsoft/Florence-2-base-ft/blob/e7a5acc73559546de6e12ec0319cd7cc1fa2437c/processing_florence2.py#L115-L117 @@ -60,7 +64,6 @@ Linux experimental support also requires launching the browser with --enable-fea let i = 1 const captions: string[] = [] for (const imageInBase64DataUri of images) { - console.log('analyzing image:', imageInBase64DataUri.slice(0, 64)) // Prepare vision inputs const image = await RawImage.fromURL(imageInBase64DataUri) diff --git a/src/services/io/parseFileIntoSegments.ts b/src/services/io/parseFileIntoSegments.ts index 9c14daa3..15519349 100644 --- a/src/services/io/parseFileIntoSegments.ts +++ b/src/services/io/parseFileIntoSegments.ts @@ -5,6 +5,7 @@ import { ClapOutputType, ClapSegmentCategory, ClapSegmentStatus, + isValidNumber, newSegment, UUID, } from '@aitube/clap' @@ -14,7 +15,8 @@ import { SegmentVisibility, TimelineSegment, useTimeline, - TimelineStore + TimelineStore, + DEFAULT_DURATION_IN_MS_PER_STEP, } from '@aitube/timeline' import { blobToBase64DataUri } from '@/lib/utils/blobToBase64DataUri' @@ -24,13 +26,21 @@ import { ResourceCategory, ResourceType } from '@aitube/clapper-services' export async function parseFileIntoSegments({ file, + track, + startTimeInMs: maybeStartTimeInMs, + endTimeInMs: maybeEndTimeInMs, }: { /** * The file to import */ file: File + + track?: number + startTimeInMs?: number + endTimeInMs?: number }): Promise { const timeline: TimelineStore = useTimeline.getState() + const { cursorTimestampAtInMs } = timeline // console.log(`parseFileIntoSegments(): filename = ${file.name}`) // console.log(`parseFileIntoSegments(): file size = ${file.size} bytes`) // console.log(`parseFileIntoSegments(): file type = ${file.type}`) @@ -41,9 +51,6 @@ export async function parseFileIntoSegments({ 'TODO: open a popup to ask if this is a voice character sample, dialogue, music etc' ) - let type: ResourceType = 'misc' - let resourceCategory: ResourceCategory = 'misc' - const newSegments: TimelineSegment[] = [] switch (file.type) { @@ -51,13 +58,9 @@ export async function parseFileIntoSegments({ case 'image/png': case 'image/avif': case 'image/heic': - case 'image/webp': - type = 'image' - resourceCategory = 'control_image' - const startTimeInMs = cursorInSteps * DEFAULT_DURATION_IN_MS_PER_STEP - const durationInSteps = 4 - const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP - const endTimeInMs = startTimeInMs + durationInMs + case 'image/webp': { + const type: ResourceType = 'image' + const resourceCategory: ResourceCategory = 'control_image' // ok let's stop for a minute there: // if someone drops a .mp3, and assuming we don't yet have the UI to select the category, @@ -65,17 +68,26 @@ export async function parseFileIntoSegments({ // I expect people will use AI service providers for sound and voice, // maybe in some case music too, but there are also many people // who will want to use their own track eg. to create a music video - const category = ClapSegmentCategory.MUSIC + const category = ClapSegmentCategory.STORYBOARD const assetUrl = await blobToBase64DataUri(file) + const startTimeInMs = isValidNumber(maybeStartTimeInMs) + ? maybeStartTimeInMs! + : cursorTimestampAtInMs + const durationInSteps = 4 + const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP + const endTimeInMs = isValidNumber(maybeEndTimeInMs) + ? maybeEndTimeInMs! + : startTimeInMs + durationInMs + const newSegmentData: Partial = { - prompt: 'audio track', + prompt: 'Storyboard', // note: this can be set later with an automatic captioning worker startTimeInMs, // start time of the segment endTimeInMs, // end time of the segment (startTimeInMs + durationInMs) status: ClapSegmentStatus.COMPLETED, // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index - label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined) + label: `${file.name}`, // a short label to name the segment (optional, can be human or LLM-defined) category, assetUrl, assetDurationInMs: endTimeInMs, @@ -86,9 +98,12 @@ export async function parseFileIntoSegments({ const timelineSegment = await clapSegmentToTimelineSegment( newSegment(newSegmentData) ) - timelineSegment.outputType = ClapOutputType.AUDIO - timelineSegment.outputGain = 1.0 - timelineSegment.audioBuffer = audioBuffer + + if (isValidNumber(track)) { + timelineSegment.track = track + } + + timelineSegment.outputType = ClapOutputType.IMAGE // we assume we want it to be immediately visible timelineSegment.visibility = SegmentVisibility.VISIBLE @@ -98,9 +113,7 @@ export async function parseFileIntoSegments({ // poof! type disappears.. it's magic newSegments.push(timelineSegment) break - - break - + } case 'audio/mpeg': // this is the "official" one case 'audio/mp3': // this is just an alias @@ -109,10 +122,10 @@ export async function parseFileIntoSegments({ case 'audio/x-mp4': // should be rare, normally is is audio/mp4 case 'audio/m4a': // shouldn't exist case 'audio/x-m4a': // should be rare, normally is is audio/mp4 - case 'audio/webm': + case 'audio/webm': { // for background track, or as an inspiration track, or a voice etc - type = 'audio' - resourceCategory = 'background_music' + const type: ResourceType = 'audio' + const resourceCategory: ResourceCategory = 'background_music' // TODO: add caption analysis const { durationInMs, durationInSteps, bpm, audioBuffer } = @@ -124,11 +137,12 @@ export async function parseFileIntoSegments({ }) // TODO: use the correct drop time - const startTimeInMs = 0 - const startTimeInSteps = 1 - - const endTimeInSteps = durationInSteps - const endTimeInMs = startTimeInMs + durationInMs + const startTimeInMs = isValidNumber(maybeStartTimeInMs) + ? maybeStartTimeInMs! + : 0 + const endTimeInMs = isValidNumber(maybeEndTimeInMs) + ? maybeEndTimeInMs! + : startTimeInMs + durationInMs // ok let's stop for a minute there: // if someone drops a .mp3, and assuming we don't yet have the UI to select the category, @@ -145,6 +159,7 @@ export async function parseFileIntoSegments({ startTimeInMs, // start time of the segment endTimeInMs, // end time of the segment (startTimeInMs + durationInMs) status: ClapSegmentStatus.COMPLETED, + track, // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined) category, @@ -157,6 +172,11 @@ export async function parseFileIntoSegments({ const timelineSegment = await clapSegmentToTimelineSegment( newSegment(newSegmentData) ) + + if (isValidNumber(track)) { + timelineSegment.track = track + } + timelineSegment.outputType = ClapOutputType.AUDIO timelineSegment.outputGain = 1.0 timelineSegment.audioBuffer = audioBuffer @@ -169,28 +189,31 @@ export async function parseFileIntoSegments({ // poof! type disappears.. it's magic newSegments.push(timelineSegment) break + } - case 'text/plain': + case 'text/plain': { // for dialogue, prompts.. - type = 'text' - resourceCategory = 'text_prompt' + const type: ResourceType = 'text' + const resourceCategory: ResourceCategory = 'text_prompt' break + } - default: + default: { console.log(`unrecognized file type "${file.type}"`) break + } } // note: we always upload the files, because even if it is an unhandled format (eg. a PDF) // this can still be part of the project as a resource for humans (inspiration, guidelines etc) + /* const id = UUID() const fileName = `${id}.${extension}` const storage = `resources` const filePath = `${type}/${fileName}` - /* const { data, error } = await supabase .storage .from('avatars') diff --git a/src/services/io/useIO.ts b/src/services/io/useIO.ts index e1d05f0b..8e056198 100644 --- a/src/services/io/useIO.ts +++ b/src/services/io/useIO.ts @@ -18,6 +18,7 @@ import { TimelineSegment, removeFinalVideosAndConvertToTimelineSegments, getFinalVideo, + DEFAULT_DURATION_IN_MS_PER_STEP, } from '@aitube/timeline' import { ParseScriptProgressUpdate, parseScriptToClap } from '@aitube/broadway' import { IOStore, TaskCategory, TaskVisibility } from '@aitube/clapper-services' @@ -129,14 +130,32 @@ export const useIO = create((set, get) => ({ }, }) + // optional: reset the project + await timeline.setClap(newClap()) + + const track = 1 let i = 0 + let startTimeInMs = 0 + const durationInSteps = 4 + const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP + let endTimeInMs = startTimeInMs + durationInMs + for (const frame of frames) { const frameFile = base64DataUriToFile(frame, `storyboard_${i++}.png`) - const newSegments = await parseFileIntoSegments({ file: frameFile }) + const newSegments = await parseFileIntoSegments({ + file: frameFile, + startTimeInMs, + endTimeInMs, + track, + }) + + startTimeInMs += durationInMs + endTimeInMs += durationInMs console.log('calling timeline.addSegments with:', newSegments) await timeline.addSegments({ segments: newSegments, + track, }) } @@ -157,18 +176,24 @@ export const useIO = create((set, get) => ({ }) console.log('calling extractCaptionsFromFrames() with:', frames) - const captions = await extractCaptionsFromFrames(frames, (progress: number, storyboardIndex: number, nbStoryboards: number) => { - captioningTask.setProgress({ - message: `Analyzing storyboards (${progress}%)`, - value: progress, - }) - }) + const captions = await extractCaptionsFromFrames( + frames, + ( + progress: number, + storyboardIndex: number, + nbStoryboards: number + ) => { + captioningTask.setProgress({ + message: `Analyzing storyboards (${progress}%)`, + value: progress, + }) + } + ) console.log('captions:', captions) - // TODO: add + // TODO: add captioningTask.success() } - } } },