added detailed stats writer

src/app.ts

@@ -183,10 +183,6 @@ async function main(): Promise<void> {
 
     await stopApplication()
 
-    if (config.postProcessVideoRecordings && config.serverData) {
-      await fixIvfFiles(config.serverData, config.vmafKeepSourceFiles)
-    }
-
     process.exit(0)
   }
   registerExitHandler(() => stop())

src/config.ts

@@ -542,14 +542,21 @@ is provided.`,
     arg: 'show-stats',
   },
   statsPath: {
-    doc: `The log file directory path; if set, the log data will be written in \
-a .csv file inside this directory; if the directory path does not exist, it \
-will be created.`,
+    doc: `The log file path; if set, the stats will be written in \
+a .csv file inside that file.`,
     format: String,
     default: '',
     env: 'STATS_PATH',
     arg: 'stats-path',
   },
+  detailedStatsPath: {
+    doc: `The log file path; if set, the detailed stats will be written in \
+a .csv file inside that file.`,
+    format: String,
+    default: '',
+    env: 'DETAILED_STATS_PATH',
+    arg: 'detailed-stats-path',
+  },
   statsInterval: {
     doc: `The stats collect interval in seconds. It should be lower than the \
 Prometheus scraping interval.`,
@@ -680,13 +687,6 @@ alert will be successful only when at least 95% of the checks pass.`,
     env: 'SERVER_DATA',
     arg: 'server-data',
   },
-  postProcessVideoRecordings: {
-    doc: `When true, it post-processes the video recordings stored into the \`serverData\` directory.`,
-    format: 'Boolean',
-    default: false,
-    env: 'POST_PROCESS_VIDEO_RECORDINGS',
-    arg: 'post-process-video-recordings',
-  },
   vmafPath: {
     doc: `When set, it runs the VMAF calculator for the videos saved under the provided directory path.`,
     format: String,

src/server.ts

@@ -100,6 +100,10 @@ export class Server {
     this.app.get('/view/docker.log', this.getDockerLog.bind(this))
     this.app.get('/download/alert-rules', this.getAlertRules.bind(this))
     this.app.get('/download/stats', this.getStatsFile.bind(this))
+    this.app.get(
+      '/download/detailed-stats',
+      this.getDetailedStatsFile.bind(this),
+    )
     this.app.get('/empty-page', this.getEmptyPage.bind(this))
 
     if (this.serverData) {
@@ -179,7 +183,24 @@ export class Server {
     if (!this.stats.statsWriter) {
       return next(new Error('statsPath not set'))
     }
-    res.download(this.stats.statsWriter.fname)
+    res.download(this.stats.statsPath)
+  }
+
+  /**
+   * GET /download/detailed-stats endpoint.
+   *
+   * Returns the {@link Stats.detailedStatsWriter} file content.
+   */
+  private getDetailedStatsFile(
+    req: express.Request,
+    res: express.Response,
+    next: express.NextFunction,
+  ): void {
+    log.debug(`/download/detailed-stats`, req.query)
+    if (!this.stats.detailedStatsWriter) {
+      return next(new Error('detailedStatsPath not set'))
+    }
+    res.download(this.stats.detailedStatsPath)
   }
 
   /**

src/stats.ts

@@ -324,6 +324,7 @@ const calculateFailAmount = (checkValue: number, ruleValue: number): number => {
  */
 export class Stats extends events.EventEmitter {
   readonly statsPath: string
+  readonly detailedStatsPath: string
   readonly prometheusPushgateway: string
   readonly prometheusPushgatewayJobName: string
   readonly prometheusPushgatewayAuth?: string
@@ -340,6 +341,7 @@ export class Stats extends events.EventEmitter {
   readonly sessions = new Map<number, Session>()
   nextSessionId: number
   statsWriter: StatsWriter | null
+  detailedStatsWriter: StatsWriter | null
   private scheduler?: Scheduler
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -414,6 +416,7 @@ export class Stats extends events.EventEmitter {
    */
   constructor({
     statsPath,
+    detailedStatsPath,
     prometheusPushgateway,
     prometheusPushgatewayJobName,
     prometheusPushgatewayAuth,
@@ -434,6 +437,7 @@ export class Stats extends events.EventEmitter {
     enableDetailedStats,
   }: {
     statsPath: string
+    detailedStatsPath: string
     prometheusPushgateway: string
     prometheusPushgatewayJobName: string
     prometheusPushgatewayAuth: string
@@ -455,6 +459,7 @@ export class Stats extends events.EventEmitter {
   }) {
     super()
     this.statsPath = statsPath
+    this.detailedStatsPath = detailedStatsPath
     this.prometheusPushgateway = prometheusPushgateway
     this.prometheusPushgatewayJobName =
       prometheusPushgatewayJobName || 'default'
@@ -483,6 +488,7 @@ export class Stats extends events.EventEmitter {
     this.enableDetailedStats = enableDetailedStats
 
     this.statsWriter = null
+    this.detailedStatsWriter = null
     if (alertRules.trim()) {
       this.alertRules = json5.parse(alertRules)
       log.debug(
@@ -595,16 +601,20 @@ export class Stats extends events.EventEmitter {
     this.running = true
 
     if (this.statsPath) {
-      const logPath = path.join(
-        this.statsPath,
-        `${moment().format('YYYY-MM-DD_HH.mm.ss')}.csv`,
-      )
-      log.info(`Logging into ${logPath}`)
-      const headers: string[] = this.statsNames.reduce(
+      log.info(`Logging stats into ${this.statsPath}`)
+      const headers = this.statsNames.reduce(
         (v: string[], name) => v.concat(formatStatsColumns(name)),
         [],
       )
-      this.statsWriter = new StatsWriter(logPath, headers)
+      this.statsWriter = new StatsWriter(this.statsPath, headers)
+    }
+
+    if (this.detailedStatsPath) {
+      log.info(`Logging stats into ${this.statsPath}`)
+      this.detailedStatsWriter = new StatsWriter(this.detailedStatsPath, [
+        'participantName',
+        ...this.statsNames,
+      ])
     }
 
     if (this.prometheusPushgateway) {
@@ -935,7 +945,7 @@ export class Stats extends events.EventEmitter {
     this.consoleShowStats()
     // Write stats to file.
     if (this.statsWriter) {
-      const values: string[] = this.statsNames.reduce(
+      const values = this.statsNames.reduce(
         (v: string[], name) =>
           v.concat(
             formatStats(this.collectedStats[name].all, true) as string[],
@@ -944,6 +954,34 @@ export class Stats extends events.EventEmitter {
       )
       await this.statsWriter.push(values)
     }
+    if (this.detailedStatsWriter) {
+      const participants = new Map<string, Record<string, FastStats>>()
+      Object.entries(this.collectedStats).forEach(([name, stats]) => {
+        Object.entries(stats.byParticipantAndTrack).forEach(
+          ([label, value]) => {
+            const [participantName] = label.split(':', 2)
+            let participant = participants.get(participantName)
+            if (!participant) {
+              participant = {}
+              participants.set(participantName, participant)
+            }
+            if (!participant[name]) {
+              participant[name] = new FastStats({ store_data: false })
+            }
+            participant[name].push(value)
+          },
+        )
+      })
+      for (const [participantName, participant] of participants.entries()) {
+        const values = [participantName]
+        for (const name of this.statsNames) {
+          values.push(
+            participant[name] ? toPrecision(participant[name].amean()) : '',
+          )
+        }
+        await this.detailedStatsWriter.push(values)
+      }
+    }
     // Send to pushgateway.
     await this.sendToPushGateway()
     // Write alert rules

src/vmaf.ts

@@ -316,11 +316,8 @@ export async function fixIvfFrames(fpath: string, outDir: string) {
   return { participantDisplayName, outFilePath, startPts }
 }
 
-export async function fixIvfFiles(
-  vmafPath: string,
-  vmafKeepSourceFiles = true,
-) {
-  const files = await await getFiles(vmafPath, '.ivf.raw')
+export async function fixIvfFiles(directory: string, keepSourceFiles = true) {
+  const files = await await getFiles(directory, '.ivf.raw')
   log.info(`fixIvfFiles files=${files}`)
 
   const reference = new Map<string, string>()
@@ -329,7 +326,7 @@ export async function fixIvfFiles(
     try {
       const { participantDisplayName, outFilePath } = await fixIvfFrames(
         filePath,
-        vmafPath,
+        directory,
       )
       if (outFilePath.includes('_recv-by_')) {
         if (!degraded.has(participantDisplayName)) {
@@ -339,7 +336,7 @@ export async function fixIvfFiles(
       } else {
         reference.set(participantDisplayName, outFilePath)
       }
-      if (!vmafKeepSourceFiles) {
+      if (!keepSourceFiles) {
         await fs.promises.unlink(filePath)
       }
     } catch (err) {