delta generation service

lblod · Sep 12, 2023 · 7c554c3 · 7c554c3
1 parent dd1dd8a
commit 7c554c3
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 6 deletions.
diff --git a/config/delta/rules.dev.js b/config/delta/rules.dev.js
@@ -59,7 +59,28 @@ export default [
       optOutMuScopeIds: ['http://redpencil.data.gift/id/concept/muScope/deltas/initialSync'],
     },
   },
-
+  {
+    match: {
+      predicate: {
+        type: 'uri',
+        value: 'http://www.w3.org/ns/adms#status',
+      },
+      object: {
+        type: 'uri',
+        value: 'http://redpencil.data.gift/id/concept/JobStatus/scheduled',
+      },
+    },
+    callback: {
+      method: 'POST',
+      url: 'http://harvest_gen_delta/delta',
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+      ignoreFromSelf: true,
+      optOutMuScopeIds: ['http://redpencil.data.gift/id/concept/muScope/deltas/initialSync'],
+    },
+  },
 
   {
     match: {

diff --git a/config/delta/rules.js b/config/delta/rules.js
@@ -124,6 +124,28 @@ export default [
       optOutMuScopeIds: ['http://redpencil.data.gift/id/concept/muScope/deltas/initialSync'],
     },
   },
+  {
+    match: {
+      predicate: {
+        type: 'uri',
+        value: 'http://www.w3.org/ns/adms#status',
+      },
+      object: {
+        type: 'uri',
+        value: 'http://redpencil.data.gift/id/concept/JobStatus/scheduled',
+      },
+    },
+    callback: {
+      method: 'POST',
+      url: 'http://harvest_gen_delta/delta',
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+      ignoreFromSelf: true,
+      optOutMuScopeIds: ['http://redpencil.data.gift/id/concept/muScope/deltas/initialSync'],
+    },
+  },
   {
     match: {
       predicate: {

diff --git a/config/job-controller/config.json b/config/job-controller/config.json
@@ -106,8 +106,13 @@
       },
       {
         "currentOperation": "http://lblod.data.gift/id/jobs/concept/TaskOperation/compressFiles",
-        "nextOperation": "http://lblod.data.gift/id/jobs/concept/TaskOperation/checking-urls",
+        "nextOperation": "http://lblod.data.gift/id/jobs/concept/TaskOperation/generatingDelta",
         "nextIndex": "9"
+      },
+      {
+        "currentOperation": "http://lblod.data.gift/id/jobs/concept/TaskOperation/generatingDelta",
+        "nextOperation": "http://lblod.data.gift/id/jobs/concept/TaskOperation/checking-urls",
+        "nextIndex": "10"
       }
     ]
   },

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -52,6 +52,10 @@ services:
     restart: "no"
     environment:
       LOGGING_LEVEL: "DEBUG" # optionals
+  harvest_compression:
+    restart: "no"
+  harvest_gen_delta:
+    restart: "no"
 
   harvest_diff:
     restart: "no"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -122,7 +122,7 @@ services:
     logging: *default-logging
 
   harvest_scraper:
-    image: nvdk/lblod-scraper:1.0.0-beta.2
+    image: lblod/scraper:1.0.0-beta.4
     volumes:
       - ./data/files:/share
     environment:
@@ -132,7 +132,7 @@ services:
     restart: always
     logging: *default-logging
   harvest_compression:
-    image: lblod/job-compression-service:0.1.1
+    image: lblod/job-compression-service:0.1.4
     volumes:
       - ./data/files:/share
     environment:
@@ -142,7 +142,7 @@ services:
     restart: always
     logging: *default-logging
   harvest_check-url:
-    image: lblod/harvest-check-url-collection-service:1.2.1
+    image: lblod/harvest-check-url-collection-service:1.3.1
     volumes:
       - ./data/files:/share
     labels:
@@ -197,6 +197,20 @@ services:
       - "logging=true"
     restart: always
     logging: *default-logging
+  harvest_gen_delta:
+   image: lblod/harvesting-generation-delta-service:0.0.8
+   environment:
+     TARGET_DIRECTORY_DELTA_PATH: "/share/deltas/besluiten"
+     BUFFER_SIZE: "100"
+     PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager-besluiten"
+     TARGET_PUBLISHER_GRAPH: "http://redpencil.data.gift/id/deltas/producer/lblod-harvester-besluiten-producer"
+
+   volumes:
+     - ./data/files:/share
+   labels:
+      - "logging=true"
+   restart: always
+   logging: *default-logging
 
   job-controller:
     image: lblod/job-controller-service:1.0.0
@@ -208,7 +222,7 @@ services:
     logging: *default-logging
 
   scheduled-job-controller:
-    image: lblod/scheduled-job-controller-service:1.0.1
+    image: lblod/scheduled-job-controller-service:1.0.2
     environment:
       # NOTE: if the delta-events prove to inaccurate,
       # you can go back to previous behavior by commenting out env.var. below