From 118a8c5a1a87f59713e7a79c4f818eea6ebb7976 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Tue, 20 Aug 2024 09:12:02 +0200 Subject: [PATCH] #469 Update README with new features. --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90c1a94de..fc8508d0f 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,15 @@ pramen.sources = [ format = "parquet" minimum.records = 0 + + # If true, fails the pipeline is there is no data any time when it is expected fail.if.no.data = false + + # If true, fails the pipeline is there is no data for jobs trying to catch late data + fail.if.no.late.data = false + + # If true, fails the pipeline is there is no data for jobs checking new data as expected + fail.if.no.new.data = false } ] ``` @@ -534,10 +542,16 @@ is determined by the pipeline configuration. # SQL queries that do not start with "SELECT". use.jdbc.native = false - # Consider the pipeline as failed if at least one table has no data at the scheduled time. + # Consider the pipeline as failed if at least one table has no data at the scheduled time (new or late). # Useful for auto-retrying ingestion pipelines. fail.if.no.data = false + # If true, fails the pipeline is there is no data for jobs trying to catch late data + fail.if.no.late.data = false + + # If true, fails the pipeline is there is no data for jobs checking new data as expected + fail.if.no.new.data = false + # One of: auto (default), always, never # - When 'auto', an identifier will be quoted if it contains invalid characters. This includes any characters # outside the scope of A-Z, a-z, 0-9, and underscore (_). @@ -1952,7 +1966,7 @@ Here is an example configuration for a JDBC source: # You can override any of source settings here source { minimum.records = 1000 - fail.if.no.data = true + fail.if.no.new.data = true has.information.date.column = true use.jdbc.native = true information.date.column = "info_date"