From 8f031d2a703244d34bb981a420b4922efeea182c Mon Sep 17 00:00:00 2001 From: Vittorio Palmisano Date: Tue, 16 Apr 2024 21:42:16 +0200 Subject: [PATCH] added detailed stats writer --- src/app.ts | 4 ---- src/config.ts | 20 +++++++++---------- src/server.ts | 23 +++++++++++++++++++++- src/stats.ts | 54 +++++++++++++++++++++++++++++++++++++++++++-------- src/vmaf.ts | 11 ++++------- 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/app.ts b/src/app.ts index 4c448d7..2da7ca0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -183,10 +183,6 @@ async function main(): Promise { await stopApplication() - if (config.postProcessVideoRecordings && config.serverData) { - await fixIvfFiles(config.serverData, config.vmafKeepSourceFiles) - } - process.exit(0) } registerExitHandler(() => stop()) diff --git a/src/config.ts b/src/config.ts index 7396352..b706c71 100644 --- a/src/config.ts +++ b/src/config.ts @@ -542,14 +542,21 @@ is provided.`, arg: 'show-stats', }, statsPath: { - doc: `The log file directory path; if set, the log data will be written in \ -a .csv file inside this directory; if the directory path does not exist, it \ -will be created.`, + doc: `The log file path; if set, the stats will be written in \ +a .csv file inside that file.`, format: String, default: '', env: 'STATS_PATH', arg: 'stats-path', }, + detailedStatsPath: { + doc: `The log file path; if set, the detailed stats will be written in \ +a .csv file inside that file.`, + format: String, + default: '', + env: 'DETAILED_STATS_PATH', + arg: 'detailed-stats-path', + }, statsInterval: { doc: `The stats collect interval in seconds. It should be lower than the \ Prometheus scraping interval.`, @@ -680,13 +687,6 @@ alert will be successful only when at least 95% of the checks pass.`, env: 'SERVER_DATA', arg: 'server-data', }, - postProcessVideoRecordings: { - doc: `When true, it post-processes the video recordings stored into the \`serverData\` directory.`, - format: 'Boolean', - default: false, - env: 'POST_PROCESS_VIDEO_RECORDINGS', - arg: 'post-process-video-recordings', - }, vmafPath: { doc: `When set, it runs the VMAF calculator for the videos saved under the provided directory path.`, format: String, diff --git a/src/server.ts b/src/server.ts index 3705df7..04fbeed 100644 --- a/src/server.ts +++ b/src/server.ts @@ -100,6 +100,10 @@ export class Server { this.app.get('/view/docker.log', this.getDockerLog.bind(this)) this.app.get('/download/alert-rules', this.getAlertRules.bind(this)) this.app.get('/download/stats', this.getStatsFile.bind(this)) + this.app.get( + '/download/detailed-stats', + this.getDetailedStatsFile.bind(this), + ) this.app.get('/empty-page', this.getEmptyPage.bind(this)) if (this.serverData) { @@ -179,7 +183,24 @@ export class Server { if (!this.stats.statsWriter) { return next(new Error('statsPath not set')) } - res.download(this.stats.statsWriter.fname) + res.download(this.stats.statsPath) + } + + /** + * GET /download/detailed-stats endpoint. + * + * Returns the {@link Stats.detailedStatsWriter} file content. + */ + private getDetailedStatsFile( + req: express.Request, + res: express.Response, + next: express.NextFunction, + ): void { + log.debug(`/download/detailed-stats`, req.query) + if (!this.stats.detailedStatsWriter) { + return next(new Error('detailedStatsPath not set')) + } + res.download(this.stats.detailedStatsPath) } /** diff --git a/src/stats.ts b/src/stats.ts index 131a34f..047c009 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -324,6 +324,7 @@ const calculateFailAmount = (checkValue: number, ruleValue: number): number => { */ export class Stats extends events.EventEmitter { readonly statsPath: string + readonly detailedStatsPath: string readonly prometheusPushgateway: string readonly prometheusPushgatewayJobName: string readonly prometheusPushgatewayAuth?: string @@ -340,6 +341,7 @@ export class Stats extends events.EventEmitter { readonly sessions = new Map() nextSessionId: number statsWriter: StatsWriter | null + detailedStatsWriter: StatsWriter | null private scheduler?: Scheduler // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -414,6 +416,7 @@ export class Stats extends events.EventEmitter { */ constructor({ statsPath, + detailedStatsPath, prometheusPushgateway, prometheusPushgatewayJobName, prometheusPushgatewayAuth, @@ -434,6 +437,7 @@ export class Stats extends events.EventEmitter { enableDetailedStats, }: { statsPath: string + detailedStatsPath: string prometheusPushgateway: string prometheusPushgatewayJobName: string prometheusPushgatewayAuth: string @@ -455,6 +459,7 @@ export class Stats extends events.EventEmitter { }) { super() this.statsPath = statsPath + this.detailedStatsPath = detailedStatsPath this.prometheusPushgateway = prometheusPushgateway this.prometheusPushgatewayJobName = prometheusPushgatewayJobName || 'default' @@ -483,6 +488,7 @@ export class Stats extends events.EventEmitter { this.enableDetailedStats = enableDetailedStats this.statsWriter = null + this.detailedStatsWriter = null if (alertRules.trim()) { this.alertRules = json5.parse(alertRules) log.debug( @@ -595,16 +601,20 @@ export class Stats extends events.EventEmitter { this.running = true if (this.statsPath) { - const logPath = path.join( - this.statsPath, - `${moment().format('YYYY-MM-DD_HH.mm.ss')}.csv`, - ) - log.info(`Logging into ${logPath}`) - const headers: string[] = this.statsNames.reduce( + log.info(`Logging stats into ${this.statsPath}`) + const headers = this.statsNames.reduce( (v: string[], name) => v.concat(formatStatsColumns(name)), [], ) - this.statsWriter = new StatsWriter(logPath, headers) + this.statsWriter = new StatsWriter(this.statsPath, headers) + } + + if (this.detailedStatsPath) { + log.info(`Logging stats into ${this.statsPath}`) + this.detailedStatsWriter = new StatsWriter(this.detailedStatsPath, [ + 'participantName', + ...this.statsNames, + ]) } if (this.prometheusPushgateway) { @@ -935,7 +945,7 @@ export class Stats extends events.EventEmitter { this.consoleShowStats() // Write stats to file. if (this.statsWriter) { - const values: string[] = this.statsNames.reduce( + const values = this.statsNames.reduce( (v: string[], name) => v.concat( formatStats(this.collectedStats[name].all, true) as string[], @@ -944,6 +954,34 @@ export class Stats extends events.EventEmitter { ) await this.statsWriter.push(values) } + if (this.detailedStatsWriter) { + const participants = new Map>() + Object.entries(this.collectedStats).forEach(([name, stats]) => { + Object.entries(stats.byParticipantAndTrack).forEach( + ([label, value]) => { + const [participantName] = label.split(':', 2) + let participant = participants.get(participantName) + if (!participant) { + participant = {} + participants.set(participantName, participant) + } + if (!participant[name]) { + participant[name] = new FastStats({ store_data: false }) + } + participant[name].push(value) + }, + ) + }) + for (const [participantName, participant] of participants.entries()) { + const values = [participantName] + for (const name of this.statsNames) { + values.push( + participant[name] ? toPrecision(participant[name].amean()) : '', + ) + } + await this.detailedStatsWriter.push(values) + } + } // Send to pushgateway. await this.sendToPushGateway() // Write alert rules diff --git a/src/vmaf.ts b/src/vmaf.ts index b08a7fe..adf4a11 100644 --- a/src/vmaf.ts +++ b/src/vmaf.ts @@ -316,11 +316,8 @@ export async function fixIvfFrames(fpath: string, outDir: string) { return { participantDisplayName, outFilePath, startPts } } -export async function fixIvfFiles( - vmafPath: string, - vmafKeepSourceFiles = true, -) { - const files = await await getFiles(vmafPath, '.ivf.raw') +export async function fixIvfFiles(directory: string, keepSourceFiles = true) { + const files = await await getFiles(directory, '.ivf.raw') log.info(`fixIvfFiles files=${files}`) const reference = new Map() @@ -329,7 +326,7 @@ export async function fixIvfFiles( try { const { participantDisplayName, outFilePath } = await fixIvfFrames( filePath, - vmafPath, + directory, ) if (outFilePath.includes('_recv-by_')) { if (!degraded.has(participantDisplayName)) { @@ -339,7 +336,7 @@ export async function fixIvfFiles( } else { reference.set(participantDisplayName, outFilePath) } - if (!vmafKeepSourceFiles) { + if (!keepSourceFiles) { await fs.promises.unlink(filePath) } } catch (err) {