Skip to content

Commit

Permalink
Added the transformation of the extended contingency matrix into JSON…
Browse files Browse the repository at this point in the history
… LD that is given to the user. Removed the German and french DBpedias from the entity checking properities.
  • Loading branch information
MichaelRoeder committed Sep 2, 2024
1 parent 331498f commit fd83ddd
Show file tree
Hide file tree
Showing 8 changed files with 615 additions and 476 deletions.
939 changes: 487 additions & 452 deletions src/main/java/org/aksw/gerbil/database/ExperimentDAOImpl.java

Large diffs are not rendered by default.

59 changes: 59 additions & 0 deletions src/main/java/org/aksw/gerbil/database/IntByteArrayPair.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package org.aksw.gerbil.database;

import java.util.Arrays;

public class IntByteArrayPair {

public int first;
public byte[] second;

public IntByteArrayPair() {
}

public IntByteArrayPair(int first, byte[] second) {
this.first = first;
this.second = second;
}

public int getFirst() {
return first;
}

public void setFirst(int first) {
this.first = first;
}

public byte[] getSecond() {
return second;
}

public void setSecond(byte[] second) {
this.second = second;
}

@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + first;
result = prime * result + Arrays.hashCode(second);
return result;
}

@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
IntByteArrayPair other = (IntByteArrayPair) obj;
if (first != other.first)
return false;
if (!Arrays.equals(second, other.second))
return false;
return true;
}

}
52 changes: 43 additions & 9 deletions src/main/java/org/aksw/gerbil/dataid/DataIDGenerator.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@

import org.aksw.gerbil.database.ResultNameToIdMapping;
import org.aksw.gerbil.datatypes.ExperimentTaskResult;
import org.aksw.gerbil.evaluate.AggregatedContingencyMetricsReport;
import org.aksw.gerbil.evaluate.ExtendedContingencyMetrics;
import org.aksw.gerbil.semantic.vocabs.CUBE;
import org.aksw.gerbil.semantic.vocabs.GERBIL;
import org.aksw.gerbil.web.ExperimentTaskStateHelper;
Expand All @@ -47,7 +49,7 @@ public class DataIDGenerator {
private static final String DATASET_DATAID = "dataId/corpora/";
private static final String ANNOTATOR_DATAID = "dataId/annotators/";
private static final String DATAID_EXTENSION = "";
private static final String LANGUAGE_DATAID = "dataId/languages/";
private static final String LANGUAGE_DATAID = "dataId/languages/";

private String gerbilURL;

Expand Down Expand Up @@ -159,15 +161,8 @@ public void createExperimentTask(Model model, ExperimentTaskResult result, Resou
}
experimentTask.addProperty(RDF.type, CUBE.Observation);

// add annotator and dataset
experimentTask.addProperty(GERBIL.annotator,
gerbilURL + ANNOTATOR_DATAID + DataIDUtils.treatsNames(result.dataset) + DATAID_EXTENSION);
experimentTask.addProperty(GERBIL.dataset,
gerbilURL + DATASET_DATAID + DataIDUtils.treatsNames(result.annotator) + DATAID_EXTENSION);
experimentTask.addProperty(GERBIL.language,
gerbilURL + LANGUAGE_DATAID + DataIDUtils.treatsNames(result.language) + DATAID_EXTENSION);
addExperimentTaskDetails(experimentTask, result);


// set the status of this task
model.add(experimentTask, GERBIL.statusCode, model.createTypedLiteral(result.state));

Expand Down Expand Up @@ -207,6 +202,9 @@ public void createExperimentTask(Model model, ExperimentTaskResult result, Resou
}
}
}
if (result.getContingencyMetricsReport() != null) {
addContinguencyMatrix(result.getContingencyMetricsReport(), result, model, experimentTask);
}
if (result.hasSubTasks()) {
for (ExperimentTaskResult subResult : result.getSubTasks()) {
createExperimentTask(model, subResult, experimentTask, experimentTasks);
Expand All @@ -219,6 +217,42 @@ public void createExperimentTask(Model model, ExperimentTaskResult result, Resou
model.add(experimentTask, GERBIL.timestamp, model.createTypedLiteral(cal));
}

protected void addExperimentTaskDetails(Resource observation, ExperimentTaskResult result) {
// add annotator and dataset
observation.addProperty(GERBIL.annotator,
gerbilURL + ANNOTATOR_DATAID + DataIDUtils.treatsNames(result.dataset) + DATAID_EXTENSION);
observation.addProperty(GERBIL.dataset,
gerbilURL + DATASET_DATAID + DataIDUtils.treatsNames(result.annotator) + DATAID_EXTENSION);
observation.addProperty(GERBIL.language,
gerbilURL + LANGUAGE_DATAID + DataIDUtils.treatsNames(result.language) + DATAID_EXTENSION);
}

protected void addContinguencyMatrix(AggregatedContingencyMetricsReport contingencyMetricsReport,
ExperimentTaskResult result, Model model, Resource experimentTask) {
int internalId = 0;
for (ExtendedContingencyMetrics metrics : contingencyMetricsReport.getValue()) {
addContinguencyMatrixPoint(metrics, internalId, result, model, experimentTask);
++internalId;
}
}

protected void addContinguencyMatrixPoint(ExtendedContingencyMetrics metrics, int internalId,
ExperimentTaskResult result, Model model, Resource experimentTask) {
// create Resource
Resource observation = model.createResource(generateExperimentTaskUri(result.idInDb) + "_" + internalId);
observation.addProperty(RDF.type, CUBE.Observation);
addExperimentTaskDetails(observation, result);
// connect it with the ID of the document / element of the dataset
observation.addProperty(GERBIL.datasetElement, model.createResource(metrics.getId()));

observation.addLiteral(GERBIL.precision, metrics.getPrecision());
observation.addLiteral(GERBIL.recall, metrics.getRecall());
observation.addLiteral(GERBIL.f1, metrics.getF1Score());
observation.addLiteral(GERBIL.truePositiveCount, metrics.getCount().truePositives);
observation.addLiteral(GERBIL.falsePositiveCount, metrics.getCount().falsePositives);
observation.addLiteral(GERBIL.falseNegativeCount, metrics.getCount().falseNegatives);
}

public String generateExperimentTaskUri(int taskId) {
return gerbilURL + EXPERIMENT_TASK_PREFIX + taskId;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,21 @@

import org.aksw.gerbil.evaluate.impl.FMeasureCalculator;

public enum ExerimentTaskBlobResultType {
public enum ExperimentTaskBlobResultType {

CONTINGENCY_MATRIX(FMeasureCalculator.CONTINGENCY_MATRIX_NAME, 1);

private final String resultType;

private final int resultId;
ExerimentTaskBlobResultType(String resultType, int resultId) {

ExperimentTaskBlobResultType(String resultType, int resultId) {
this.resultType = resultType;
this.resultId = resultId;
}

public static int getResultId(String resultType) {
for (ExerimentTaskBlobResultType result : ExerimentTaskBlobResultType.values()) {
for (ExperimentTaskBlobResultType result : ExperimentTaskBlobResultType.values()) {
if (result.resultType.equals(resultType)) {
return result.resultId;
}
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/aksw/gerbil/semantic/vocabs/GERBIL.java
Original file line number Diff line number Diff line change
Expand Up @@ -74,19 +74,27 @@ protected static final Property property(String local) {

public static final Property annotator = property("annotator");
public static final Property dataset = property("dataset");
public static final Property datasetElement = property("datasetElement");
public static final Property language = property("language");
public static final Property experimentType = property("experimentType");
public static final Property errorCount = property("errorCount");
public static final Property f1 = property("f1");
public static final Property falseNegativeCount = property("falseNegativeCount");
public static final Property falsePositiveCount = property("falsePositiveCount");
public static final Property macroF1 = property("macroF1");
public static final Property macroPrecision = property("macroPrecision");
public static final Property macroRecall = property("macroRecall");
public static final Property matching = property("matching");
public static final Property microF1 = property("microF1");
public static final Property microPrecision = property("microPrecision");
public static final Property microRecall = property("microRecall");
public static final Property precision = property("precision");
public static final Property recall = property("recall");
public static final Property statusCode = property("statusCode");
public static final Property subExperimentOf = property("subExperimentOf");
public static final Property timestamp = property("timestamp");
public static final Property trueNegativeCount = property("trueNegativeCount");
public static final Property truePositiveCount = property("truePositiveCount");
public static final Property topic = property("topic");

public static Resource getMatchingResource(Matching matching) {
Expand Down
6 changes: 3 additions & 3 deletions src/main/properties/entity_checking.properties
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://dbpedia.or
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://bg.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://ca.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://cs.dbpedia.org
org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://de.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://de.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://es.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://eu.dbpedia.org
org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://fr.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://fr.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://hu.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://id.dbpedia.org
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://it.dbpedia.org
Expand All @@ -31,5 +31,5 @@ org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://fr.dbpedia
### Wikipedia
org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://wikipedia.org/wiki
org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://en.wikipedia.org/wiki
org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://de.wikipedia.org/wiki
#org.aksw.gerbil.dataset.check.HttpBasedEntityChecker.namespace=http://de.wikipedia.org/wiki

2 changes: 1 addition & 1 deletion src/main/properties/log4j2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ Configuration:
- ref: STDOUT
- ref: File
Root:
level: warn
level: info
AppenderRef:
- ref: STDOUT
- ref: File
17 changes: 9 additions & 8 deletions src/test/java/org/aksw/gerbil/SingleRunTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,17 @@ public class SingleRunTest implements TaskObserver {

private static final Logger LOGGER = LoggerFactory.getLogger(SingleRunTest.class);


private static final String ANNOTATOR_NAME = "NIFWS_Tremblay(http://qald7rest.azurewebsites.net/api/question)";
private static final String DATASET_NAME = "QALD7 Train Multilingual";
//private static final String ANNOTATOR_NAME = "NIFWS_Tremblay(http://qald7rest.azurewebsites.net/api/question)";
//private static final String ANNOTATOR_NAME = "AF_Test(../datasets/qald10/qald_9_plus_train_wikidata.json)(undefined)(QALD10 Train Multilingual)";
private static final String ANNOTATOR_NAME = "AF_Test(../../../../Downloads/e2eResultsfreebase.json)(undefined)(QALD10 Train Multilingual)";
private static final String DATASET_NAME = "QALD10 Train Multilingual";
private static final ExperimentType EXPERIMENT_TYPE = ExperimentType.QA;
private static final String QUESTION_LANGUAGE = "fr";
private static final String QUESTION_LANGUAGE = "en";

private static final Matching MATCHING = Matching.STRONG_ENTITY_MATCH;

private static final boolean USE_SAME_AS_RETRIEVAL = false;
private static final boolean USE_ENTITY_CHECKING = false;
private static final boolean USE_SAME_AS_RETRIEVAL = true;
private static final boolean USE_ENTITY_CHECKING = true;

private static final SameAsRetriever SAME_AS_RETRIEVER = USE_SAME_AS_RETRIEVAL
? SameAsRetrieverSingleton4Tests.getInstance() : null;
Expand All @@ -87,9 +88,9 @@ public void run() throws Exception {
adapterManager.setAnnotators(AnnotatorsConfig.annotators());
adapterManager.setDatasets(DatasetsConfig.datasets(ENTITY_CHECKER_MANAGER, SAME_AS_RETRIEVER));

AnnotatorConfiguration annotatorConfig = adapterManager.getAnnotatorConfig(ANNOTATOR_NAME, EXPERIMENT_TYPE, "fr");
AnnotatorConfiguration annotatorConfig = adapterManager.getAnnotatorConfig(ANNOTATOR_NAME, EXPERIMENT_TYPE, QUESTION_LANGUAGE);
Assert.assertNotNull(annotatorConfig);
DatasetConfiguration datasetConfig = adapterManager.getDatasetConfig(DATASET_NAME, EXPERIMENT_TYPE, "fr");
DatasetConfiguration datasetConfig = adapterManager.getDatasetConfig(DATASET_NAME, EXPERIMENT_TYPE, QUESTION_LANGUAGE);
Assert.assertNotNull(datasetConfig);
// DatasetConfiguration datasetConfig = new
// DatasetConfigurationImpl("Test", false,
Expand Down

0 comments on commit fd83ddd

Please sign in to comment.