Skip to content

Commit

Permalink
fix(rdf): make entity iris use roottablename instead of subtablename …
Browse files Browse the repository at this point in the history 
…so references to subclasses can be resolved (#3249)
  • Loading branch information
@mswertz
mswertz authored Feb 13, 2024
1 parent ce53d7c commit 3869338
Show file tree
Hide file tree
Showing 12 changed files with 220 additions and 133 deletions.
57 changes: 27 additions & 30 deletions backend/molgenis-emx2-rdf/src/main/java/org/molgenis/emx2/rdf/PrimaryKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,74 +4,71 @@
import static org.molgenis.emx2.FilterBean.f;
import static org.molgenis.emx2.Operator.EQUALS;

import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.*;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.molgenis.emx2.Filter;
import org.molgenis.emx2.MolgenisException;

class PrimaryKey {

public static final String NAME_VALUE_SEPARATOR = "&";
public static final String KEY_PARTS_SEPARATOR = ";";
private final List<NameValuePair> keys;
public static final String NAME_VALUE_SEPARATOR = "=";
public static final String KEY_PARTS_SEPARATOR = "&";
private final Map<String, String> keys;
// use map instead of list<NameValuePair> to prevent duplicate entries
// some foreign key have overlapping relationships which resulted in a bug

static PrimaryKey makePrimaryKeyFromEncodedKey(String encodedValue) {
Base64.Decoder decoder = Base64.getDecoder();
String[] encodedPairs = encodedValue.split(KEY_PARTS_SEPARATOR);
if (encodedPairs.length == 0) {
throw new IllegalArgumentException("There must be at least one key.");
} else {
List<NameValuePair> pairs = new ArrayList<>();
Map<String, String> pairs = new LinkedHashMap();
for (var pair : encodedPairs) {
var parts = pair.split(NAME_VALUE_SEPARATOR);
if (parts.length != 2) {
throw new IllegalArgumentException(
"Can't decode the key, name value pair is incomplete.");
}
var name = new String(decoder.decode(parts[0]), StandardCharsets.UTF_8);
var value = new String(decoder.decode(parts[1]), StandardCharsets.UTF_8);
pairs.add(new BasicNameValuePair(name, value));
var name = URLDecoder.decode(parts[0], StandardCharsets.UTF_8);
var value = URLDecoder.decode(parts[1], StandardCharsets.UTF_8);
pairs.put(name, value);
}
return new PrimaryKey(pairs);
}
}

PrimaryKey(List<NameValuePair> keys) {
PrimaryKey(Map<String, String> keys) {
if (keys.isEmpty()) {
throw new IllegalArgumentException("There must be at least one key.");
}
this.keys = keys;
}

String getEncodedValue() {
Base64.Encoder encoder = Base64.getEncoder();
List<String> encodedPairs = new ArrayList<>();
// Sort the list to have a stable order
var sortedList = keys.stream().sorted(new NameValuePairComparator()).toList();
for (var pair : sortedList) {
var name = encoder.encodeToString(pair.getName().getBytes(StandardCharsets.UTF_8));
var value = encoder.encodeToString(pair.getValue().getBytes(StandardCharsets.UTF_8));
encodedPairs.add(name + NAME_VALUE_SEPARATOR + value);
try {
List<String> encodedPairs = new ArrayList<>();
// Sort the list to have a stable order
var sortedMap = new TreeMap<>(this.keys);
for (var pair : sortedMap.entrySet()) {
var name = URLEncoder.encode(pair.getKey(), StandardCharsets.UTF_8.toString());
var value = URLEncoder.encode(pair.getValue(), StandardCharsets.UTF_8.toString());
encodedPairs.add(name + NAME_VALUE_SEPARATOR + value);
}
return String.join(KEY_PARTS_SEPARATOR, encodedPairs);
} catch (Exception e) {
throw new MolgenisException("Error encoding" + e);
}
return String.join(KEY_PARTS_SEPARATOR, encodedPairs);
}

Filter getFilter() {
final List<Filter> filters =
keys.stream().map(param -> f(param.getName(), EQUALS, param.getValue())).toList();
keys.entrySet().stream().map(param -> f(param.getKey(), EQUALS, param.getValue())).toList();
return and(filters);
}

List<NameValuePair> getKeys() {
Map<String, String> getKeys() {
return keys;
}

static class NameValuePairComparator implements Comparator<NameValuePair> {

@Override
public int compare(NameValuePair left, NameValuePair right) {
return left.getName().compareTo(right.getName());
}
}
}
124 changes: 70 additions & 54 deletions backend/molgenis-emx2-rdf/src/main/java/org/molgenis/emx2/rdf/RDFService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import static org.eclipse.rdf4j.model.util.Values.iri;
import static org.eclipse.rdf4j.model.util.Values.literal;
import static org.molgenis.emx2.Constants.MG_TABLECLASS;
import static org.molgenis.emx2.FilterBean.f;
import static org.molgenis.emx2.Operator.EQUALS;
import static org.molgenis.emx2.rdf.RDFUtils.*;
Expand All @@ -10,8 +11,6 @@
import java.io.OutputStream;
import java.time.format.DateTimeFormatter;
import java.util.*;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Namespace;
import org.eclipse.rdf4j.model.Value;
Expand Down Expand Up @@ -89,8 +88,6 @@ public class RDFService {
* schemas.
*/
private final String baseURL;
/** The rdfAPIPath is the relative path for the RDF api within a schema. */
private final String rdfAPIPath;

/**
* Construct an RDF Service.
Expand All @@ -107,16 +104,6 @@ public RDFService(final String baseURL, final String rdfAPIPath, final RDFFormat
} else {
this.baseURL = baseURL.trim() + "/";
}
// Ensure that the stored rdfAPIPath has a leading and trailing "/" so we
// can use it easily to construct URL paths.
String temp = rdfAPIPath.trim();
if (!temp.startsWith("/")) {
temp = "/" + temp;
}
if (!temp.endsWith("/")) {
temp = temp + "/";
}
this.rdfAPIPath = temp;
this.rdfFormat = format == null ? RDFFormat.TURTLE : format;

this.config = new WriterConfig();
Expand Down Expand Up @@ -169,14 +156,20 @@ public void describeAsRDF(
builder.setNamespace(ns);
}

describeRoot(builder);
if (table == null) {
describeRoot(builder);
}

for (final Schema schema : schemas) {
describeSchema(builder, schema);
if (table == null) {
describeSchema(builder, schema);
}
final List<Table> tables = table != null ? Arrays.asList(table) : schema.getTablesSorted();
for (final Table tableToDescribe : tables) {
describeTable(builder, tableToDescribe);
describeColumns(builder, tableToDescribe, columnName);
if (rowId == null) {
describeTable(builder, tableToDescribe);
describeColumns(builder, tableToDescribe, columnName);
}
// if a column name is provided then only provide column metadata, no row values
if (columnName == null) {
rowsToRdf(builder, tableToDescribe, rowId);
Expand Down Expand Up @@ -226,13 +219,17 @@ protected void describeRoot(final ModelBuilder builder) {
* @param schema the schema
* @return A namespace that defines a local unique prefix for this schema.
*/
private Namespace getSchemaNamespace(final Schema schema) {
private Namespace getSchemaNamespace(final SchemaMetadata schema) {
final String schemaName = UrlEscapers.urlPathSegmentEscaper().escape(schema.getName());
final String url = baseURL + schemaName + rdfAPIPath;
final String url = baseURL + schemaName + "/api/rdf/";
final String prefix = TypeUtils.convertToPascalCase(schema.getName());
return Values.namespace(prefix, url);
}

private Namespace getSchemaNamespace(final Schema schema) {
return getSchemaNamespace(schema.getMetadata());
}

/**
* Get an IRI for the table. Taking the schema in which the table resides into consideration.
*
Expand Down Expand Up @@ -293,7 +290,7 @@ private void describeTable(final ModelBuilder builder, final Table table) {
private void describeColumns(
final ModelBuilder builder, final Table table, final String columnName) {
if (table.getMetadata().getTableType() == TableType.DATA) {
for (final Column column : table.getMetadata().getColumns()) {
for (final Column column : table.getMetadata().getNonInheritedColumns()) {
// Exclude the system columns like mg_insertedBy
if (column.isSystemColumn()) {
continue;
Expand All @@ -310,13 +307,6 @@ private void describeColumns(
private IRI getColumnIRI(final Column column) {
TableMetadata table = column.getTable();
Schema schema = table.getTable().getSchema();
final Database db = schema.getDatabase();
while (table.getLocalColumn(column.getName()) == null) {
var inherited = table.getInheritedTable();
// Don't use the copy from the inherited table metadata, because that might not be complete.
schema = db.getSchema(inherited.getSchemaName());
table = schema.getTable(inherited.getTableName()).getMetadata();
}
final String tableName = UrlEscapers.urlPathSegmentEscaper().escape(table.getIdentifier());
final String columnName = UrlEscapers.urlPathSegmentEscaper().escape(column.getIdentifier());
final Namespace ns = getSchemaNamespace(schema);
Expand Down Expand Up @@ -395,7 +385,7 @@ private CoreDatatype.XSD columnTypeToXSD(final ColumnType columnType) {
* @param table the table for which to fetch the rows
* @param rowId optional rowId
*/
public void rowsToRdf(final ModelBuilder builder, final Table table, final String rowId) {
public void rowsToRdf(final ModelBuilder builder, Table table, final String rowId) {
final IRI tableIRI = getTableIRI(table);
for (final Row row : getRows(table, rowId)) {
IRI subject = getIriForRow(row, table.getMetadata());
Expand Down Expand Up @@ -435,7 +425,12 @@ public void rowsToRdf(final ModelBuilder builder, final Table table, final Strin
builder.add(subject, RDF.TYPE, IRI_OBSERVATION);
builder.add(subject, IRI_DATASET_PREDICATE, tableIRI);
builder.add(subject, RDFS.LABEL, Values.literal(getLabelForRow(row, table.getMetadata())));

// via rowId might be subclass
if (rowId != null) {
// because row IRI point to root tables we need to find actual subclass table to ensure we
// get all columns
table = getSubclassTableForRowBasedOnMgTableclass(table, row);
}
for (final Column column : table.getMetadata().getColumns()) {
// Exclude the system columns like mg_insertedBy
if (column.isSystemColumn()) {
Expand All @@ -455,6 +450,18 @@ public void rowsToRdf(final ModelBuilder builder, final Table table, final Strin
}
}

private static Table getSubclassTableForRowBasedOnMgTableclass(Table table, Row row) {
if (row.getString(MG_TABLECLASS) != null) {
table =
table
.getSchema()
.getDatabase()
.getSchema(row.getSchemaName())
.getTable(row.getTableName());
}
return table;
}

private String getLabelForRow(final Row row, final TableMetadata metadata) {
List<String> primaryKeyValues = new ArrayList<>();
for (Column column : metadata.getPrimaryKeyColumns()) {
Expand All @@ -470,73 +477,82 @@ private String getLabelForRow(final Row row, final TableMetadata metadata) {
return String.join(" ", primaryKeyValues);
}

private List<Row> getRows(final Table table, final String rowId) {
private List<Row> getRows(Table table, final String rowId) {
Query query = table.query();
if (rowId != null) {
// first find from root table
PrimaryKey key = PrimaryKey.makePrimaryKeyFromEncodedKey(rowId);
query.where(key.getFilter());
}
// If a table is extended then we get only those rows that are for the base table.
if (table.getMetadata().getColumnNames().contains("mg_tableclass")) {
var tableName = table.getSchema().getName() + "." + table.getName();
query.where(f("mg_tableclass", EQUALS, tableName));
List<Row> oneRow = query.where(key.getFilter()).retrieveRows();
// if subclass
if (oneRow.size() == 1 && oneRow.get(0).getString(MG_TABLECLASS) != null) {
Row row = oneRow.get(0);
table = getSubclassTableForRowBasedOnMgTableclass(table, row);
return table.query().where(key.getFilter()).retrieveRows();
}
return oneRow;
} else {
if (table.getMetadata().getColumnNames().contains(MG_TABLECLASS)) {
var tableName = table.getSchema().getName() + "." + table.getName();
query.where(f("mg_tableclass", EQUALS, tableName));
}
return query.retrieveRows();
}
return query.retrieveRows();
}

private IRI getIriForRow(final Row row, final TableMetadata metadata) {
final String tableName =
UrlEscapers.urlPathSegmentEscaper().escape(metadata.getTable().getIdentifier());
final List<NameValuePair> keyParts = new ArrayList<>();
final String rootTableName =
UrlEscapers.urlPathSegmentEscaper().escape(metadata.getRootTable().getIdentifier());
final Map<String, String> keyParts = new LinkedHashMap<>();
for (final Column column : metadata.getPrimaryKeyColumns()) {
if (column.isReference()) {
for (final Reference reference : column.getReferences()) {
final String[] values = row.getStringArray(reference.getName());
for (final String value : values) {
keyParts.add(new BasicNameValuePair(reference.getName(), value));
keyParts.put(reference.getName(), value);
}
}
} else {
keyParts.add(new BasicNameValuePair(column.getIdentifier(), row.get(column).toString()));
keyParts.put(column.getIdentifier(), row.get(column).toString());
}
}
final Namespace ns = getSchemaNamespace(metadata.getTable().getSchema());
final Namespace ns = getSchemaNamespace(metadata.getRootTable().getSchema());
PrimaryKey key = new PrimaryKey(keyParts);
return Values.iri(ns, tableName + "/" + key.getEncodedValue());
return Values.iri(ns, rootTableName + "?" + key.getEncodedValue());
}

private List<IRI> getIriValue(final Row row, final Column column) {
final TableMetadata target = column.getRefTable();
final String tableName = UrlEscapers.urlPathSegmentEscaper().escape(target.getIdentifier());
final Namespace ns = getSchemaNamespace(target.getTable().getSchema());
final String rootTableName =
UrlEscapers.urlPathSegmentEscaper().escape(target.getRootTable().getIdentifier());
final Namespace ns = getSchemaNamespace(target.getRootTable().getSchema());

final Set<IRI> iris = new HashSet<>();
final Map<Integer, List<NameValuePair>> items = new HashMap<>();
final Map<Integer, Map<String, String>> items = new HashMap<>();
for (final Reference reference : column.getReferences()) {
final String localColumn = reference.getName();
final String targetColumn = reference.getPath().get(0);
final String targetColumn = reference.getRefTo();
if (column.isArray()) {
final String[] values = row.getStringArray(localColumn);
if (values != null) {
for (int i = 0; i < values.length; i++) {
var keyValuePairs = items.getOrDefault(i, new ArrayList<>());
keyValuePairs.add(new BasicNameValuePair(targetColumn, values[i]));
var keyValuePairs = items.getOrDefault(i, new LinkedHashMap<>());
keyValuePairs.put(targetColumn, values[i]);
items.put(i, keyValuePairs);
}
}
} else {
final String value = row.getString(localColumn);
if (value != null) {
var keyValuePairs = items.getOrDefault(0, new ArrayList<>());
keyValuePairs.add(new BasicNameValuePair(targetColumn, value));
var keyValuePairs = items.getOrDefault(0, new LinkedHashMap<>());
keyValuePairs.put(targetColumn, value);
items.put(0, keyValuePairs);
}
}
}

for (final var item : items.values()) {
PrimaryKey key = new PrimaryKey(item);
iris.add(Values.iri(ns, tableName + "/" + key.getEncodedValue()));
iris.add(Values.iri(ns, rootTableName + "?" + key.getEncodedValue()));
}
return List.copyOf(iris);
}
Expand Down
Loading

0 comments on commit 3869338

Please sign in to comment.