From a35c327abda0d1f27039bfbd77a254063bf979ae Mon Sep 17 00:00:00 2001 From: benjay10 Date: Fri, 31 Jan 2025 13:17:36 +0100 Subject: [PATCH 1/4] Introduce migrations on publication triplestore --- docker-compose.dev.yml | 2 ++ docker-compose.yml | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7458859f..06b705c7 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -33,6 +33,8 @@ services: restart: "no" migrations: restart: "no" + migrations-publication-triplestore: + restart: "no" cache: restart: "no" resource: diff --git a/docker-compose.yml b/docker-compose.yml index 811f5abd..6ca4efe5 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,6 +96,18 @@ services: - "logging=true" restart: always logging: *default-logging + migrations-publication-triplestore: + image: semtech/mu-migrations-service:0.9.0 + links: + - publication-triplestore:database + environment: + MU_SPARQL_TIMEOUT: "300" + volumes: + - ./config/migrations-publication-triplestore:/data/migrations + labels: + - "logging=true" + restart: always + logging: *default-logging cache: image: semtech/mu-cache:2.0.1 links: From 6c84b07ca6f7763ea359c837883f0eabe6ebb486 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Fri, 31 Jan 2025 13:18:00 +0100 Subject: [PATCH 2/4] Migrations: remove submissions for re-export Remove these submissions and related data from the publication triplestore. --- ...0000-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ ...0001-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ ...0002-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ ...0003-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ ...0004-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ ...0005-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ ...0006-remove-submission-to-re-export.sparql | 43 +++++++++++++++++++ 7 files changed, 301 insertions(+) create mode 100644 config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql create mode 100644 config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql create mode 100644 config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql create mode 100644 config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql create mode 100644 config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql create mode 100644 config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql create mode 100644 config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql diff --git a/config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..7c23a2a7 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150000-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..ba79b357 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150001-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..353113bc --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150002-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..735c8d9c --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150003-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..ad81bd44 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150004-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..84de622c --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150005-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} diff --git a/config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql b/config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql new file mode 100644 index 00000000..6c865ae9 --- /dev/null +++ b/config/migrations-publication-triplestore/2025/20250129150006-remove-submission-to-re-export.sparql @@ -0,0 +1,43 @@ +PREFIX nie: +PREFIX prov: +PREFIX dct: +PREFIX rdf: +PREFIX rdfs: + +DELETE { + GRAPH ?g { + ?submission ?sp ?so . + ?submissionDocument ?sdp ?sdo . + ?pFile ?pp ?po . + ?formData ?fp ?fo . + ?lFile ?lp ?lo . + ?pFile2 ?p2p ?p2o . + } +} +WHERE { + BIND ( as ?submission) + GRAPH ?g { + { + ?submission ?sp ?so . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument ?sdp ?sdo . + } UNION { + ?submission dct:subject ?submissionDocument . + ?submissionDocument dct:source ?pFile . + ?pFile ?pp ?po . + } UNION { + ?submission prov:generated ?formData . + ?formData ?fp ?fo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?lFile ?lp ?lo . + } UNION { + ?submission prov:generated ?formData . + ?formData dct:hasPart ?lFile . + ?pFile2 nie:dataSource ?lFile . + ?pFile2 ?p2p ?p2o . + } + } +} From 7d1f0c219880971d3adba81a06eccf8e87b79e41 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Fri, 31 Jan 2025 13:53:45 +0100 Subject: [PATCH 3/4] CHANGELOG: add deploy instruction for missing submissions --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b03d3b0f..2d86b650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,43 @@ # Changelog +## Unreleased + +### General + +- Add `migrations-publication-triplestore` as a migration service for the publication-triplestore [DL-6349] +- Re-export 12 submissions and their related subjects, because they are missing from Worship Decisions Database, see migrations and deploy instructions [DL-6349] + +### Deploy instructions + +**For re-exporting submissions** + +- Pull and start the new service + * `drc up -d migrations-publication-triplestore` +- Make sure migrations have run `drc logs --tail 1000 -f migrations-publication-triplestore` +- Run the following commands to send submissions to be marked for publication to Worship Decisions Database: + +``` +cd scripts/send-submissions-to-prepare-for-export/ + +echo 'http://data.lblod.info/form-data/95c393b0-07f1-11ef-8911-1be0e451139c +http://data.lblod.info/form-data/d86d0350-f8eb-11ee-8911-1be0e451139c +http://data.lblod.info/form-data/54e5ea70-f89a-11ee-8911-1be0e451139c +http://data.lblod.info/form-data/2fe55480-f710-11ee-8911-1be0e451139c +http://data.lblod.info/form-data/dc3c4ab0-e048-11ee-8911-1be0e451139c' > form-data-uris.txt + +./run.sh + +rm form-data-uris.txt + +# Make sure that the prepare service has processed everything +# This can take about 2 minutes +drc logs --tail 1000 -f prepare-submissions-for-export +``` + +- Start healing on the `delta-producer-publication-graph-maintainer` via the `delta-producer-background-jobs-initiator` + * `drc exec delta-producer-background-jobs-initiator curl -X POST http://localhost/worship-submissions/healing-jobs` + * This can also take a while, up to an hour. + ## 1.108.0 (2025-01-24) ### General From 6d4378656af02a13a6c080772b922f61ecd5668d Mon Sep 17 00:00:00 2001 From: benjay10 Date: Tue, 4 Feb 2025 15:40:58 +0100 Subject: [PATCH 4/4] Add warning about migrations-publication-triplestore --- .../migrations-publication-triplestore/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 config/migrations-publication-triplestore/README.md diff --git a/config/migrations-publication-triplestore/README.md b/config/migrations-publication-triplestore/README.md new file mode 100644 index 00000000..f73eae18 --- /dev/null +++ b/config/migrations-publication-triplestore/README.md @@ -0,0 +1,16 @@ +# Migrations on the publication-triplestore + +If you want to fix something in the frontend or other service in this stack, DO +NOT MAKE ANY MIGRATIONS HERE. + +The publication triplestore is only meant to be used as a intermediate for the +producers in the stack. It is used for calculating differences between what's +in the actual triplestore that is used for all the services, and what has +already been sent to the consuming apps in terms of delta files. + +It can be used as a trick to re-export certain data. For example, if some +Sumbission is missing from consuming apps, you can remove it and related +subjects from the publication-triplestore and trigger healing on the producer +in question. This will generate the delta files for those subjects, ready to be +consumed. Look at some existing migrations for examples. +