tde17: scoreDir & leaderboard (WIP)

zerospeech/benchmarks/tde17.py

@@ -3,7 +3,6 @@
 from pydantic import Field
 
 from zerospeech.datasets import ZRC2017Dataset
-from zerospeech.generics import FileItem
 from zerospeech.submissions import Submission
 from zerospeech.submissions.tde17 import TDE17Submission
 from zerospeech.tasks import tde
@@ -13,24 +12,19 @@
 class TDE17Task(tde.TDETask):
     tasks: Tuple = ('english', 'french', 'mandarin', 'german', 'wolof')
 
-    def from_submission(self, submission: "TDE17Submission", lang: str) -> FileItem:
-        """ Extract current input class file from submission """
-        current_input_classes_file = submission.items.get(lang)
-        if current_input_classes_file is None:
-            raise ValueError(f'Language {lang} was not found in current submission : {submission.location}')
-
-        return current_input_classes_file
-
-    def load_gold(self, dataset: ZRC2017Dataset, lang: str):
-        """ Load gold object for current language set """
+    def gather_items(self, lang: str, submission: "TDE17Submission", dataset: ZRC2017Dataset):
         current_data = dataset.index.subsets.get(lang)
         if current_data is None:
             raise ValueError(f'Language {lang} was not found in {dataset.name}')
 
-        # load gold files
-        return tde.Gold(
-            wrd_path=str(current_data.items.alignment_words.file),
-            phn_path=str(current_data.items.alignment_phones.file)
+        current_input_classes_file = submission.items.get(lang)
+        if current_input_classes_file is None:
+            raise ValueError(f'Language {lang} was not found in current submission : {submission.location}')
+
+        return tde.TDEItems(
+            wrd_path=current_data.items.alignment_words.file,
+            phn_path=current_data.items.alignment_phones.file,
+            input_classes=current_input_classes_file.file
         )
 
 

zerospeech/submissions/tde17.py

@@ -2,17 +2,21 @@
 import os
 import shutil
 import sys
+from datetime import datetime
 from pathlib import Path
 from typing import Tuple, Dict, Any, Optional, Type, List
 
 import yaml
 from pydantic import Field
 
+from zerospeech.datasets.zrc_2017 import ZRC2017Dataset
 from zerospeech.generics import FileItem, Item, Namespace
+from zerospeech.leaderboards import EntryDetails, LeaderboardBenchmarkName, LeaderboardEntry
+from zerospeech.leaderboards.tde17 import TDE17Scores, TDE17Entry
 from zerospeech.misc import load_obj
 from zerospeech.tasks import BenchmarkParameters, tde
-from zerospeech.datasets.zrc_2017 import ZRC2017Dataset
 from zerospeech.validators import BASE_VALIDATOR_FN_TYPE
+from . import ScoreDir
 from ._model import (
     MetaFile, Submission, validation_fn, SubmissionValidation,
     ValidationResponse, ValidationError, ValidationOK
@@ -22,6 +26,7 @@
 class TDE17BenchmarkParams(BenchmarkParameters):
     """ Parameters for the TDE-17 benchmark """
     # location to output the results
+    njobs: int = 1  # CPU cores to use for eval
     out: Optional[str] = None
     result_filename: str = "scores.json"
 
@@ -102,6 +107,77 @@ def validating_english(self, class_file: FileItem):
         return tde_class_file_check(class_file.file)
 
 
+class TDE17ScoreDir(ScoreDir):
+    params: Optional[TDE17BenchmarkParams] = TDE17BenchmarkParams()
+
+    @property
+    def scores(self):
+        pass
+
+    def get_details(self) -> EntryDetails:
+        """ Build entry details """
+        train_set = ""
+        gpu_budget = ""
+
+        if self.meta_file is not None:
+            train_set = self.meta_file.model_info.train_set
+            gpu_budget = self.meta_file.model_info.gpu_budget
+
+        return EntryDetails(
+            train_set=train_set,
+            benchmarks=[LeaderboardBenchmarkName.TDE_17],
+            gpu_budget=gpu_budget,
+            parameters=self.params.to_meta()
+        )
+
+    def build_scores(self) -> TDE17Scores:
+        df = self.scores
+        # todo: compile scores into format
+        return ...
+
+    def build_meta_data(self):
+        """ Build leaderboard metadata """
+        return dict(
+            model_id=self.meta_file.model_info.model_id,
+            submission_id="",
+            index=None,
+            submission_date=datetime.now(),
+            submitted_by=self.meta_file.username,
+            description=self.meta_file.model_info.system_description,
+            publication=dict(
+                author_short=self.meta_file.publication.author_label,
+                authors=self.meta_file.publication.authors,
+                paper_title=self.meta_file.publication.paper_title,
+                paper_ref=self.meta_file.publication.paper_url,
+                bib_ref=self.meta_file.publication.bib_reference,
+                paper_url=self.meta_file.publication.paper_url,
+                pub_year=self.meta_file.publication.publication_year,
+                team_name=self.meta_file.publication.team,
+                institution=self.meta_file.publication.institution,
+                code=self.meta_file.code_url,
+                DOI=self.meta_file.publication.DOI,
+                open_science=self.meta_file.open_source,
+            ),
+            details=dict(
+                train_set=self.meta_file.model_info.train_set,
+                benchmarks=[],
+                gpu_budget=self.meta_file.model_info.gpu_budget,
+                parameters=self.params.to_meta(),
+            )
+        )
+
+    def build_leaderboard(self) -> LeaderboardEntry:
+        """ Build leaderboard entry for the current submission """
+        self.load_meta()
+
+        return TDE17Entry.parse_obj(
+            dict(
+                **self.build_meta_data(),
+                scores=self.build_scores()
+            )
+        )
+
+
 class TDE17Submission(Submission):
     """ Submission for TDE-17 """
     sets: Tuple = ('1s', '10s', '120s')

zerospeech/tasks/tde.py

@@ -1,10 +1,10 @@
 import abc
 import json
 import os
-import signal
 import sys
 import warnings
-from typing import Tuple, Set, TYPE_CHECKING
+from pathlib import Path
+from typing import Tuple, Set, TYPE_CHECKING, NamedTuple
 
 import joblib
 
@@ -30,22 +30,28 @@
     from zerospeech.submissions import Submission
 
 
+class TDEItems(NamedTuple):
+    wrd_path: Path
+    phn_path: Path
+    input_classes: Path
+
+
 class TDETask(Task, abc.ABC):
     """ TDE Task """
     _name = "tde-task"
     tasks: Tuple
     metrics: Set = {'grouping', 'matching', 'boundary', 'token_type', 'nlp'}
-    n_jobs: int = 1
+    njobs: int = 1
     result_filename: str = "scores.json"
     grouping_max_time: int = 7200
 
     @staticmethod
-    def read_discovered(item: FileItem, gold: Gold):
+    def read_discovered(item: Path, gold: Gold):
         """ Load discovered Intervals """
         # Disc class prints a bunch of nonsense, so we force it to be quiet
         sys.stdout = open(os.devnull, 'w')
         try:
-            return Disc(str(item.file), gold)
+            return Disc(str(item), gold)
         finally:
             sys.stdout = sys.__stdout__
 
@@ -127,38 +133,44 @@ def score_or_none(data):
 
         return {m: score_or_none(score) for m, score in scores.items()}
 
-    @abc.abstractmethod
-    def from_submission(self, submission: "Submission", lang: str) -> FileItem:
-        """ Extract current input class file from submission """
-        pass
-
-    @abc.abstractmethod
-    def load_gold(self, dataset: "Dataset", lang: str) -> Gold:
+    @staticmethod
+    def load_gold(wrd: Path, phn: Path) -> Gold:
         """ Load gold object for current language set """
-        pass
+        # load gold files
+        return Gold(
+            wrd_path=str(wrd),
+            phn_path=str(phn)
+        )
 
-    def _eval_lang(self, submission: "Submission", dataset: "Dataset", lang: str):
+    def _eval_lang(self, lang: str, items: TDEItems):
         """ Evaluate tde for specific language """
-        current_input_classes_file = self.from_submission(submission, lang)
         self.console.print(f"Loading gold for {lang}...")
-        gold = self.load_gold(dataset, lang)
+        gold = self.load_gold(wrd=items.wrd_path, phn=items.phn_path)
 
         # load discovered intervals
         self.console.print(f"Loading class discovery for {lang}...")
         discovered = self.read_discovered(
-            current_input_classes_file, gold
+            items.input_classes, gold
         )
 
-        # return results
         self.console.print(f"Gathering metrics for {lang} ...")
         return lang, self.gather_metrics(gold, discovered, lang)
 
+    @abc.abstractmethod
+    def gather_items(self, lang: str, submission: "Submission", dataset: "Dataset") -> TDEItems:
+        pass
+
     def eval(self, submission: "Submission", dataset: "Dataset"):
         """ Evaluate the submission """
-
+        print(f"Running with {self.njobs} cores!!")
         # Run evaluation with multiprocess if specified
-        res = joblib.Parallel(n_jobs=self.n_jobs)(
-            joblib.delayed(self._eval_lang)(submission, dataset, lang) for lang in self.tasks
+        eval_items = {
+            lang: self.gather_items(lang=lang, submission=submission, dataset=dataset)
+            for lang in self.tasks
+        }
+
+        res = joblib.Parallel(n_jobs=self.njobs)(
+            joblib.delayed(self._eval_lang)(lang, items) for lang, items in eval_items.items()
         )
         scores = dict(res)
         self.console.print(f":pencil: writing scores {self.result_filename}", style="underline yellow4")