diff --git a/CHANGELOG.md b/CHANGELOG.md index b03d3b0f..2d86b650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,43 @@ # Changelog +## Unreleased + +### General + +- Add `migrations-publication-triplestore` as a migration service for the publication-triplestore [DL-6349] +- Re-export 12 submissions and their related subjects, because they are missing from Worship Decisions Database, see migrations and deploy instructions [DL-6349] + +### Deploy instructions + +**For re-exporting submissions** + +- Pull and start the new service + * `drc up -d migrations-publication-triplestore` +- Make sure migrations have run `drc logs --tail 1000 -f migrations-publication-triplestore` +- Run the following commands to send submissions to be marked for publication to Worship Decisions Database: + +``` +cd scripts/send-submissions-to-prepare-for-export/ + +echo 'http://data.lblod.info/form-data/95c393b0-07f1-11ef-8911-1be0e451139c +http://data.lblod.info/form-data/d86d0350-f8eb-11ee-8911-1be0e451139c +http://data.lblod.info/form-data/54e5ea70-f89a-11ee-8911-1be0e451139c +http://data.lblod.info/form-data/2fe55480-f710-11ee-8911-1be0e451139c +http://data.lblod.info/form-data/dc3c4ab0-e048-11ee-8911-1be0e451139c' > form-data-uris.txt + +./run.sh + +rm form-data-uris.txt + +# Make sure that the prepare service has processed everything +# This can take about 2 minutes +drc logs --tail 1000 -f prepare-submissions-for-export +``` + +- Start healing on the `delta-producer-publication-graph-maintainer` via the `delta-producer-background-jobs-initiator` + * `drc exec delta-producer-background-jobs-initiator curl -X POST http://localhost/worship-submissions/healing-jobs` + * This can also take a while, up to an hour. + ## 1.108.0 (2025-01-24) ### General diff --git a/config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..7c23a2a7 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..ba79b357 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..353113bc --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..735c8d9c --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..ad81bd44 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..84de622c --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..6c865ae9 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7458859f..06b705c7 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -33,6 +33,8 @@ services: restart: "no" migrations: restart: "no" + migrations-publication-triplestore: + restart: "no" cache: restart: "no" resource: diff --git a/docker-compose.yml b/docker-compose.yml index 811f5abd..6ca4efe5 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,6 +96,18 @@ services: - "logging=true" restart: always logging: *default-logging + migrations-publication-triplestore: + image: semtech/mu-migrations-service:0.9.0 + links: + - publication-triplestore:database + environment: + MU_SPARQL_TIMEOUT: "300" + volumes: + - ./config/migrations-publication-triplestore:/data/migrations + labels: + - "logging=true" + restart: always + logging: *default-logging cache: image: semtech/mu-cache:2.0.1 links: