diff --git a/zerospeech/benchmarks/tde17.py b/zerospeech/benchmarks/tde17.py index 73bafbd..a6e5516 100644 --- a/zerospeech/benchmarks/tde17.py +++ b/zerospeech/benchmarks/tde17.py @@ -3,7 +3,6 @@ from pydantic import Field from zerospeech.datasets import ZRC2017Dataset -from zerospeech.generics import FileItem from zerospeech.submissions import Submission from zerospeech.submissions.tde17 import TDE17Submission from zerospeech.tasks import tde @@ -13,24 +12,19 @@ class TDE17Task(tde.TDETask): tasks: Tuple = ('english', 'french', 'mandarin', 'german', 'wolof') - def from_submission(self, submission: "TDE17Submission", lang: str) -> FileItem: - """ Extract current input class file from submission """ - current_input_classes_file = submission.items.get(lang) - if current_input_classes_file is None: - raise ValueError(f'Language {lang} was not found in current submission : {submission.location}') - - return current_input_classes_file - - def load_gold(self, dataset: ZRC2017Dataset, lang: str): - """ Load gold object for current language set """ + def gather_items(self, lang: str, submission: "TDE17Submission", dataset: ZRC2017Dataset): current_data = dataset.index.subsets.get(lang) if current_data is None: raise ValueError(f'Language {lang} was not found in {dataset.name}') - # load gold files - return tde.Gold( - wrd_path=str(current_data.items.alignment_words.file), - phn_path=str(current_data.items.alignment_phones.file) + current_input_classes_file = submission.items.get(lang) + if current_input_classes_file is None: + raise ValueError(f'Language {lang} was not found in current submission : {submission.location}') + + return tde.TDEItems( + wrd_path=current_data.items.alignment_words.file, + phn_path=current_data.items.alignment_phones.file, + input_classes=current_input_classes_file.file ) diff --git a/zerospeech/submissions/tde17.py b/zerospeech/submissions/tde17.py index 876bd76..428964b 100644 --- a/zerospeech/submissions/tde17.py +++ b/zerospeech/submissions/tde17.py @@ -2,17 +2,21 @@ import os import shutil import sys +from datetime import datetime from pathlib import Path from typing import Tuple, Dict, Any, Optional, Type, List import yaml from pydantic import Field +from zerospeech.datasets.zrc_2017 import ZRC2017Dataset from zerospeech.generics import FileItem, Item, Namespace +from zerospeech.leaderboards import EntryDetails, LeaderboardBenchmarkName, LeaderboardEntry +from zerospeech.leaderboards.tde17 import TDE17Scores, TDE17Entry from zerospeech.misc import load_obj from zerospeech.tasks import BenchmarkParameters, tde -from zerospeech.datasets.zrc_2017 import ZRC2017Dataset from zerospeech.validators import BASE_VALIDATOR_FN_TYPE +from . import ScoreDir from ._model import ( MetaFile, Submission, validation_fn, SubmissionValidation, ValidationResponse, ValidationError, ValidationOK @@ -22,6 +26,7 @@ class TDE17BenchmarkParams(BenchmarkParameters): """ Parameters for the TDE-17 benchmark """ # location to output the results + njobs: int = 1 # CPU cores to use for eval out: Optional[str] = None result_filename: str = "scores.json" @@ -102,6 +107,77 @@ def validating_english(self, class_file: FileItem): return tde_class_file_check(class_file.file) +class TDE17ScoreDir(ScoreDir): + params: Optional[TDE17BenchmarkParams] = TDE17BenchmarkParams() + + @property + def scores(self): + pass + + def get_details(self) -> EntryDetails: + """ Build entry details """ + train_set = "" + gpu_budget = "" + + if self.meta_file is not None: + train_set = self.meta_file.model_info.train_set + gpu_budget = self.meta_file.model_info.gpu_budget + + return EntryDetails( + train_set=train_set, + benchmarks=[LeaderboardBenchmarkName.TDE_17], + gpu_budget=gpu_budget, + parameters=self.params.to_meta() + ) + + def build_scores(self) -> TDE17Scores: + df = self.scores + # todo: compile scores into format + return ... + + def build_meta_data(self): + """ Build leaderboard metadata """ + return dict( + model_id=self.meta_file.model_info.model_id, + submission_id="", + index=None, + submission_date=datetime.now(), + submitted_by=self.meta_file.username, + description=self.meta_file.model_info.system_description, + publication=dict( + author_short=self.meta_file.publication.author_label, + authors=self.meta_file.publication.authors, + paper_title=self.meta_file.publication.paper_title, + paper_ref=self.meta_file.publication.paper_url, + bib_ref=self.meta_file.publication.bib_reference, + paper_url=self.meta_file.publication.paper_url, + pub_year=self.meta_file.publication.publication_year, + team_name=self.meta_file.publication.team, + institution=self.meta_file.publication.institution, + code=self.meta_file.code_url, + DOI=self.meta_file.publication.DOI, + open_science=self.meta_file.open_source, + ), + details=dict( + train_set=self.meta_file.model_info.train_set, + benchmarks=[], + gpu_budget=self.meta_file.model_info.gpu_budget, + parameters=self.params.to_meta(), + ) + ) + + def build_leaderboard(self) -> LeaderboardEntry: + """ Build leaderboard entry for the current submission """ + self.load_meta() + + return TDE17Entry.parse_obj( + dict( + **self.build_meta_data(), + scores=self.build_scores() + ) + ) + + class TDE17Submission(Submission): """ Submission for TDE-17 """ sets: Tuple = ('1s', '10s', '120s') diff --git a/zerospeech/tasks/tde.py b/zerospeech/tasks/tde.py index 3bb37cc..a39b656 100644 --- a/zerospeech/tasks/tde.py +++ b/zerospeech/tasks/tde.py @@ -1,10 +1,10 @@ import abc import json import os -import signal import sys import warnings -from typing import Tuple, Set, TYPE_CHECKING +from pathlib import Path +from typing import Tuple, Set, TYPE_CHECKING, NamedTuple import joblib @@ -30,22 +30,28 @@ from zerospeech.submissions import Submission +class TDEItems(NamedTuple): + wrd_path: Path + phn_path: Path + input_classes: Path + + class TDETask(Task, abc.ABC): """ TDE Task """ _name = "tde-task" tasks: Tuple metrics: Set = {'grouping', 'matching', 'boundary', 'token_type', 'nlp'} - n_jobs: int = 1 + njobs: int = 1 result_filename: str = "scores.json" grouping_max_time: int = 7200 @staticmethod - def read_discovered(item: FileItem, gold: Gold): + def read_discovered(item: Path, gold: Gold): """ Load discovered Intervals """ # Disc class prints a bunch of nonsense, so we force it to be quiet sys.stdout = open(os.devnull, 'w') try: - return Disc(str(item.file), gold) + return Disc(str(item), gold) finally: sys.stdout = sys.__stdout__ @@ -127,38 +133,44 @@ def score_or_none(data): return {m: score_or_none(score) for m, score in scores.items()} - @abc.abstractmethod - def from_submission(self, submission: "Submission", lang: str) -> FileItem: - """ Extract current input class file from submission """ - pass - - @abc.abstractmethod - def load_gold(self, dataset: "Dataset", lang: str) -> Gold: + @staticmethod + def load_gold(wrd: Path, phn: Path) -> Gold: """ Load gold object for current language set """ - pass + # load gold files + return Gold( + wrd_path=str(wrd), + phn_path=str(phn) + ) - def _eval_lang(self, submission: "Submission", dataset: "Dataset", lang: str): + def _eval_lang(self, lang: str, items: TDEItems): """ Evaluate tde for specific language """ - current_input_classes_file = self.from_submission(submission, lang) self.console.print(f"Loading gold for {lang}...") - gold = self.load_gold(dataset, lang) + gold = self.load_gold(wrd=items.wrd_path, phn=items.phn_path) # load discovered intervals self.console.print(f"Loading class discovery for {lang}...") discovered = self.read_discovered( - current_input_classes_file, gold + items.input_classes, gold ) - # return results self.console.print(f"Gathering metrics for {lang} ...") return lang, self.gather_metrics(gold, discovered, lang) + @abc.abstractmethod + def gather_items(self, lang: str, submission: "Submission", dataset: "Dataset") -> TDEItems: + pass + def eval(self, submission: "Submission", dataset: "Dataset"): """ Evaluate the submission """ - + print(f"Running with {self.njobs} cores!!") # Run evaluation with multiprocess if specified - res = joblib.Parallel(n_jobs=self.n_jobs)( - joblib.delayed(self._eval_lang)(submission, dataset, lang) for lang in self.tasks + eval_items = { + lang: self.gather_items(lang=lang, submission=submission, dataset=dataset) + for lang in self.tasks + } + + res = joblib.Parallel(n_jobs=self.njobs)( + joblib.delayed(self._eval_lang)(lang, items) for lang, items in eval_items.items() ) scores = dict(res) self.console.print(f":pencil: writing scores {self.result_filename}", style="underline yellow4")