diff --git a/.changeset/forty-hotels-unite.md b/.changeset/forty-hotels-unite.md new file mode 100644 index 00000000..91086144 --- /dev/null +++ b/.changeset/forty-hotels-unite.md @@ -0,0 +1,5 @@ +--- +"barnard59-cube": patch +--- + +Fix batch size in check-observations diff --git a/packages/cube/lib/merge.js b/packages/cube/lib/merge.js new file mode 100644 index 00000000..0ff8371b --- /dev/null +++ b/packages/cube/lib/merge.js @@ -0,0 +1,7 @@ +export default function (batch) { + const result = this.env.dataset() + for (const quads of batch) { + result.addAll(quads) + } + return result +} diff --git a/packages/cube/pipeline/check-observations.ttl b/packages/cube/pipeline/check-observations.ttl index 48eec477..70654c21 100644 --- a/packages/cube/pipeline/check-observations.ttl +++ b/packages/cube/pipeline/check-observations.ttl @@ -46,7 +46,7 @@ _:sortChunkSize a p:Variable ; _:sortBySubject [ splitDataset:bySubject () ] [ base:batch ("batchSize"^^p:VariableName) ] - [ base:flatten () ] + _:merge [ shacl:report [code:name "shape" ; code:value _:getConstraint ] , [code:name "maxErrors" ; code:value "maxViolations"^^p:VariableName ] @@ -80,6 +80,13 @@ _:readConstraint a p:Step ; code:arguments ("constraint"^^p:VariableName) . +_:merge base:map ( + [ + a code:EcmaScriptModule ; + code:link + ] +) . + _:addTarget base:map ( [ a code:EcmaScriptModule ; diff --git a/packages/cube/test/support/constraint.ttl b/packages/cube/test/support/constraint.ttl new file mode 100644 index 00000000..0c91d824 --- /dev/null +++ b/packages/cube/test/support/constraint.ttl @@ -0,0 +1,67 @@ +@base . + +@prefix dc: . +@prefix cube: . +@prefix dh: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xml: . +@prefix xsd: . +@prefix ex: . +@prefix skos: . +@prefix schema: . +@prefix qudt: . + + a sh:NodeShape, cube:Constraint ; + sh:closed true ; + sh:property [ + sh:path rdf:type ; + sh:nodeKind sh:IRI ; + sh:in ( cube:Observation ) + ], [ + sh:path cube:observedBy ; + sh:nodeKind sh:IRI ; + sh:in ( ) + ], [ + schema:name "Date and Time"@en, "Datum und Zeit"@de, "Date et heure"@fr; # optional, label of the vocab can be used + sh:path dc:date ; + sh:datatype xsd:dateTime ; + sh:minCount 1 ; + sh:maxCount 1 ; + qudt:scaleType qudt:IntervalScale ; + ], [ + schema:name "Humidity"@en, "Feuchtigkeit"@de, "Humidité"@fr ; # optional, label of the vocab can be used + sh:path dh:humidity ; + sh:datatype xsd:decimal ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:minInclusive 0 ; + sh:maxInclusive 100 ; + qudt:scaleType qudt:IntervalScale ; + ], [ + schema:name "Low Battery Power"@en, "Niedrige Batterieleistung"@de, "Piles faibles"@fr ; # optional, label of the vocab can be used + sh:path dh:lowBatteryPower ; + sh:datatype xsd:boolean ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "true"^^xsd:boolean "false"^^xsd:boolean ) ; + qudt:scaleType qudt:NominalScale ; + ], [ + schema:name "Temperature"@en, "Temperatur"@de, "Température"@fr ; # optional, label of the vocab can be used + sh:path dh:temperature ; + sh:datatype xsd:decimal ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:minInclusive -50 ; + sh:maxInclusive 50 ; + qudt:scaleType qudt:IntervalScale ; + ], [ + schema:name "Raum"@de , "Pièce"@fr ,"Room"@en ; + sh:path dh:room ; + sh:maxCount 1 ; + sh:class schema:Place ; + sh:nodeKind sh:IRI ; + sh:in ( ) ; + qudt:scaleType qudt:NominalScale ; + ] . diff --git a/packages/cube/test/support/cube.ttl b/packages/cube/test/support/cube.ttl new file mode 100644 index 00000000..b5035a2b --- /dev/null +++ b/packages/cube/test/support/cube.ttl @@ -0,0 +1,248 @@ +@base . + +@prefix dc: . +@prefix cube: . +@prefix dh: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix xml: . +@prefix xsd: . +@prefix ex: . +@prefix skos: . +@prefix schema: . + + a dh:Device, dh:TemperatureSensor ; + schema:name "Thermometer"@en ; + schema:name "Thermometer"@de ; + schema:name "Thermomètre"@fr . + + a cube:Cube, schema:Dataset ; + schema:name "Thermometer Historical Data"@en ; + schema:name "Historische Daten vom Thermometer"@de ; + schema:name "Données historiques du thermomètre"@fr ; + schema:description "Die Daten werden von Temperatursensoren in den verschiedenen Räumen automatisch erfasst."@de ; + schema:description "Les données sont automatiquement collectées par des capteurs de température dans les différentes pièces."@fr ; + schema:description "The data is automatically collected by temperature sensors in the various rooms."@en ; + schema:publisher ; + schema:creator ; + schema:contactPoint ; + schema:contributor ; + cube:observationSet ; + cube:observationConstraint ; + schema:dateCreated "2020-05-04"^^xsd:date ; # The date on which the CreativeWork was created or the item was added to a DataFeed. + schema:datePublished "2020-05-04"^^xsd:date ; # Date of first broadcast/publication. + schema:dateModified "2020-05-04"^^xsd:date ; # The date on which the CreativeWork was most recently modified or when the item's entry was modified within a DataFeed. + schema:workExample . # What apps use/visualize this cube data + + a cube:ObservationSet ; + cube:observation , + , + , + , + , + , + , + , + , + , + , + , + . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 79.0 ; + dh:lowBatteryPower false ; + dh:temperature 2.8999999999999999112 ; + dc:date "2019-01-02T12:00:00.058000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 75.0 ; + dh:lowBatteryPower false ; + dh:temperature 0.0 ; + dc:date "2019-01-03T12:00:00.055000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 83.0 ; + dh:lowBatteryPower false ; + dh:temperature 0.0 ; + dc:date "2019-01-04T12:00:00.013000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 87.0 ; + dh:lowBatteryPower false ; + dh:temperature 1.3000000000000000444 ; + dc:date "2019-01-05T12:00:00.041000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 88.0 ; + dh:lowBatteryPower false ; + dh:temperature 2.3999999999999999112 ; + dc:date "2019-01-06T12:00:00.057000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 89.0 ; + dh:lowBatteryPower false ; + dh:temperature 2.7999999999999998224 ; + dc:date "2019-01-07T12:00:00.059000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 88.0 ; + dh:lowBatteryPower false ; + dh:temperature 2.7000000000000001776 ; + dc:date "2019-01-08T12:00:00.017000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 86.0 ; + dh:lowBatteryPower false ; + dh:temperature 1.3000000000000000444 ; + dc:date "2019-01-09T12:00:00.055000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 86.0 ; + dh:lowBatteryPower false ; + dh:temperature 1.1999999999999999556 ; + dc:date "2019-01-10T12:00:00.059000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 80.0 ; + dh:lowBatteryPower false ; + dh:temperature -0.80000000000000004441 ; + dc:date "2019-01-11T12:00:00.055000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 87.0 ; + dh:lowBatteryPower false ; + dh:temperature 1.5 ; + dc:date "2019-01-12T12:00:00.055000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 89.0 ; + dh:lowBatteryPower false ; + dh:temperature 2.5 ; + dc:date "2019-01-13T12:00:00.055000+00:00"^^xsd:dateTime . + + a cube:Observation ; + cube:observedBy ; + dh:room ; + dh:humidity 87.0 ; + dh:lowBatteryPower false ; + dh:temperature 3.5 ; + dc:date "2019-01-14T12:00:00.015000+00:00"^^xsd:dateTime . + +## TODO we should have a unit example IMO + +## Example of a nested structure. + + a schema:Place ; + schema:name "Room 1101"@en, "Raum 1101"@de, "Pièce 1101"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 1102"@en, "Raum 1102"@de, "Pièce 1102"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 1201"@en, "Raum 1201"@de, "Pièce 1201"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 1202"@en, "Raum 1202"@de, "Pièce 1202"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 2101"@en, "Raum 2101"@de, "Pièce 2101"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 2102"@en, "Raum 2102"@de, "Pièce 2102"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 2201"@en, "Raum 2201"@de, "Pièce 2201"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Room 2202"@en, "Raum 2202"@de, "Pièce 2202"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Building 1, Level 1"@en, "Gebäude 1, Etage 1"@de, "Bâtiment 1, étage 1"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Building 1, Level 2"@en, "Gebäude 1, Etage 2"@de, "Bâtiment 1, étage 2"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Building 2, Level 1"@en, "Gebäude 2, Etage 1"@de, "Bâtiment 2, étage 1"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Building 2, Level 2"@en, "Gebäude 2, Etage 2"@de, "Bâtiment 2, étage 2"@fr ; + schema:inDefinedTermSet ; + schema:containedInPlace . + + a schema:Place ; + schema:name "Building 1"@en, "Gebäude 1"@de, "Bâtiment 1"@fr ; + schema:latitude 47.16277 ; + schema:longitude 7.28952 ; + schema:inDefinedTermSet . + + a schema:Place ; + schema:name "Building 2"@en, "Gebäude 2"@de, "Bâtiment 2"@fr ; + schema:latitude 47.14396 ; + schema:longitude 7.26027 ; + schema:inDefinedTermSet . + + a schema:DefinedTermSet ; + schema:name "Räume"@de, "Rooms"@en, "Pièces"@fr . + + a schema:DefinedTermSet ; + schema:name "Gebäude"@de, "Buildings"@en, "Immeubles"@fr . + + a schema:DefinedTermSet ; + schema:name "Etage"@de, "Level"@en, "Etage"@fr . + +# Orgs + + a schema:Corporation ; + schema:name "Zazuko GmbH" ; + rdfs:seeAlso ; + schema:url + . \ No newline at end of file diff --git a/packages/cube/test/validation.pipeline.test.js b/packages/cube/test/validation.pipeline.test.js index 213c9806..898d8c0d 100644 --- a/packages/cube/test/validation.pipeline.test.js +++ b/packages/cube/test/validation.pipeline.test.js @@ -51,4 +51,27 @@ describe('cube validation pipeline', function () { strictEqual(result.stdout, '') strictEqual(result.code, 0) }) + + it('should run check-cube-observations pipeline with error for cube.link examples', () => { + const constraintFile = `${support}/constraint.ttl` + // sh:class constraint fails because of batching + const command = `cat ${support}/cube.ttl | barnard59 cube check-observations --constraint ${constraintFile} --batchSize 10` + + const result = shell.exec(command, { silent: true, cwd }) + + strictEqual(result.code, 1) + ok(result.stdout.includes('_:report "false"^^')) + }) + + it('should run check-cube-observations pipeline with cube.link examples', () => { + const constraintFile = `${support}/constraint.ttl` + // disabling batching makes all data available when checking the sh:class constraint + const command = `cat ${support}/cube.ttl | barnard59 cube check-observations --constraint ${constraintFile} --batchSize 0` + + const result = shell.exec(command, { silent: true, cwd }) + + strictEqual(result.stderr, '') + strictEqual(result.stdout, '') + strictEqual(result.code, 0) + }) })