From c9a8710e248ff781e8956e392322e52c62b95b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Birm=C3=A9?= Date: Tue, 2 Jul 2024 15:09:18 +0200 Subject: [PATCH] feat: support download from OSC encore and upload to S3 (#1) * feat: support download from OSC encore and upload to S3 * chore: fix unit tests * chore: fix prettier * chore: add AWS keys in documentation --- README.md | 3 +++ package-lock.json | 34 +++++++++++++++++++++++----------- package.json | 3 ++- src/config.ts | 8 ++++++-- src/encorePackager.test.ts | 4 +++- src/encorePackager.ts | 37 ++++++++++++++++++++++++++++++++++--- 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 84a3f51..c9df1d8 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ package the output of the transcoding job referenced by the message. | `PACKAGE_CONCURRENCY` | Number of concurrent packaging jobs | `1` | | `PACKAGE_LISTENER_PLUGIN` | Optional path to a javascript file containing a custom listener for packaging event, see below | | | `ENCORE_PASSWORD` | Optional password for the encore instance `user` user | | +| `OSC_ACCESS_TOKEN` | Optional OSC access token for accessing Encore instance in OSC | | +| `AWS_ACCESS_KEY_ID` | Optional AWS access key id when `PACKAGE_OUTPUT_FOLDER` is an AWS S3 bucket | | +| `AWS_SECRET_ACCESS_KEY` | Optional AWS secret access key when `PACKAGE_OUTPUT_FOLDER` is an AWS S3 bucket | | ```bash npm run start diff --git a/package-lock.json b/package-lock.json index c8a3d94..58420f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,10 @@ "version": "0.6.0", "license": "MIT", "dependencies": { - "@eyevinn/shaka-packager-s3": "^0.2.2", + "@eyevinn/shaka-packager-s3": "^0.3.0", "@fastify/cors": "^8.2.0", "@fastify/type-provider-typebox": "^2.4.0", + "@osaas/client-core": "^0.8.0", "@sinclair/typebox": "^0.25.24", "commander": "^12.1.0", "fastify": "^4.12.0", @@ -989,9 +990,9 @@ } }, "node_modules/@eyevinn/shaka-packager-s3": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@eyevinn/shaka-packager-s3/-/shaka-packager-s3-0.2.2.tgz", - "integrity": "sha512-M9CHKPsLOP+mrY0FTqIipWidGOJ6cUpkJSGNLE3NjurX7ce08m2K3QHMCKStQf5NF07nNyRnbQ2OGLJaBwHKRQ==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@eyevinn/shaka-packager-s3/-/shaka-packager-s3-0.3.0.tgz", + "integrity": "sha512-QS3GvytfX9F7sBdHHA/zyS9+iHmWY04W/+2K8GOUSBn4lv+np8gz3fqMF/6GxoryJJfyNegkUMtE8AAEXJmYuw==", "dependencies": { "commander": "^12.1.0", "mv": "^2.1.1" @@ -1569,6 +1570,23 @@ "node": ">= 8" } }, + "node_modules/@osaas/client-core": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@osaas/client-core/-/client-core-0.8.0.tgz", + "integrity": "sha512-3mQrZCEztCR3T3lmAOK3cmO3lOo41BEAc8FqBQAL+UAVdgXTg/21QwCN5LAJQBO2IU6sswEthmCc9481n1sKbA==", + "dependencies": { + "@osaas/client-core": "^0.4.0", + "chalk": "4.1.2" + } + }, + "node_modules/@osaas/client-core/node_modules/@osaas/client-core": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@osaas/client-core/-/client-core-0.4.0.tgz", + "integrity": "sha512-gp+wGhEuBSJwpl/yYs9p31Z5NjWWeT/dT5SPO2NbTs88qG70cbdvzKheSjck9bxl+Ol/yED3fnL67jCk0EUxbA==", + "dependencies": { + "chalk": "4.1.2" + } + }, "node_modules/@redis/bloom": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", @@ -2119,7 +2137,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -2506,7 +2523,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -2624,7 +2640,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -2635,8 +2650,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/commander": { "version": "12.1.0", @@ -3772,7 +3786,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "engines": { "node": ">=8" } @@ -6523,7 +6536,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, diff --git a/package.json b/package.json index e6bbebd..1640330 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,10 @@ "node": ">=18.15.0" }, "dependencies": { - "@eyevinn/shaka-packager-s3": "^0.2.2", + "@eyevinn/shaka-packager-s3": "^0.3.0", "@fastify/cors": "^8.2.0", "@fastify/type-provider-typebox": "^2.4.0", + "@osaas/client-core": "^0.8.0", "@sinclair/typebox": "^0.25.24", "commander": "^12.1.0", "fastify": "^4.12.0", diff --git a/src/config.ts b/src/config.ts index e160b7a..56315b1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -23,6 +23,7 @@ export interface PackagingConfig { shakaExecutable?: string; packageListenerPlugin?: string; encorePassword?: string; + oscAccessToken?: string; } function readRedisConfig(): RedisConfig { @@ -34,11 +35,14 @@ function readRedisConfig(): RedisConfig { function readPackagingConfig(): PackagingConfig { return { - outputFolder: resolve(process.env.PACKAGE_OUTPUT_FOLDER || 'packaged'), + outputFolder: process.env.PACKAGE_OUTPUT_FOLDER?.match(/^s3:/) + ? new URL(process.env.PACKAGE_OUTPUT_FOLDER).toString() + : resolve(process.env.PACKAGE_OUTPUT_FOLDER || 'packaged'), shakaExecutable: process.env.SHAKA_PACKAGER_EXECUTABLE, concurrency: parseInt(process.env.PACKAGE_CONCURRENCY || '1'), packageListenerPlugin: process.env.PACKAGE_LISTENER_PLUGIN, - encorePassword: process.env.ENCORE_PASSWORD + encorePassword: process.env.ENCORE_PASSWORD, + oscAccessToken: process.env.OSC_ACCESS_TOKEN }; } diff --git a/src/encorePackager.test.ts b/src/encorePackager.test.ts index b01f2ea..8494a12 100644 --- a/src/encorePackager.test.ts +++ b/src/encorePackager.test.ts @@ -8,6 +8,7 @@ describe('Test parseInputsFromEncoreJob', () => { output: [ { file: '/data/out/e5e76304-744c-41d6-85f7-69007b3b1a65/test3_x264_3100.mp4', + format: 'mp4', fileSize: 3757912, overallBitrate: 2982469, videoStreams: [ @@ -19,7 +20,8 @@ describe('Test parseInputsFromEncoreJob', () => { audioStreams: [], type: 'VideoFile' } - ] + ], + inputs: [] }; const inputs = parseInputsFromEncoreJob(job); console.log(inputs); diff --git a/src/encorePackager.ts b/src/encorePackager.ts index dc8d745..4cec1f0 100644 --- a/src/encorePackager.ts +++ b/src/encorePackager.ts @@ -2,6 +2,7 @@ import { Input, doPackage, PackageOptions } from '@eyevinn/shaka-packager-s3'; import { resolve } from 'node:path'; import { PackagingConfig } from './config'; import { basename, extname } from 'node:path'; +import { Context } from '@osaas/client-core'; export interface EncoreJob { id: string; @@ -12,6 +13,7 @@ export interface EncoreJob { export interface Output { type: string; + format: string; file: string; fileSize: number; overallBitrate: number; @@ -31,10 +33,19 @@ export class EncorePackager { async package(jobUrl: string) { const job = await this.getEncoreJob(jobUrl); const inputs = parseInputsFromEncoreJob(job); + let serviceAccessToken = undefined; + if (this.config.oscAccessToken) { + const ctx = new Context({ + personalAccessToken: this.config.oscAccessToken + }); + serviceAccessToken = await ctx.getServiceAccessToken('encore'); + } const dest = this.getPackageDestination(job); await doPackage({ dest, inputs, + source: this.config.oscAccessToken ? new URL(jobUrl).origin : undefined, + serviceAccessToken, noImplicitAudio: true, shakaExecutable: this.config.shakaExecutable } as PackageOptions); @@ -44,7 +55,13 @@ export class EncorePackager { getPackageDestination(job: EncoreJob) { const inputUri = job.inputs[0].uri; const inputBasename = basename(inputUri, extname(inputUri)); - return resolve(this.config.outputFolder, inputBasename, job.id); + if (this.config.outputFolder.match(/^s3:/)) { + return new URL( + this.config.outputFolder + inputBasename + '/' + job.id + ).toString(); + } else { + return resolve(this.config.outputFolder, inputBasename, job.id); + } } async getEncoreJob(url: string): Promise { @@ -58,8 +75,20 @@ export class EncorePackager { ).toString('base64') } : {}; + let sat; + if (this.config.oscAccessToken) { + const ctx = new Context({ + personalAccessToken: this.config.oscAccessToken + }); + sat = await ctx.getServiceAccessToken('encore'); + } + const jwtHeader: { 'x-jwt': string } | Record = sat + ? { + 'x-jwt': `Bearer ${sat}` + } + : {}; const response = await fetch(url, { - headers: { ...authHeader } + headers: { ...authHeader, ...jwtHeader } }); if (!response.ok) { throw new Error( @@ -80,7 +109,9 @@ export function parseInputsFromEncoreJob(job: EncoreJob) { throw new Error('Encore job has no output'); } const video = job.output - .filter((output) => output.type === 'VideoFile') + .filter( + (output) => output.type === 'VideoFile' && output.format.includes('mp4') + ) .map((output) => ({ output, videoStream: output.videoStreams?.[0] })); const audio = job.output .filter((output) => output.type === 'AudioFile')