FluentBit log collection setup support for old-style playbooks (#58)

* added 'check_database_access_permission: true' to configuration samples

* custom fact_path defined

* revert adding fact_path as it becomes deprecated

* better rolling restart for db nodes

* error fixes

* Print the parameters on database creation

* error fixes

* fluentbit role support

* extra variables added to all.example

examples/fluentbit/ddl.sql

@@ -0,0 +1,24 @@
+
+CREATE USER fluentbit PASSWORD "....";
+GRANT CONNECT ON `/cluster1/admin` TO fluentbit;
+
+CREATE TABLE `ydblogs` (
+  `ts` Timestamp NOT NULL,
+  `datahash` Uint64 NOT NULL,
+  `dbname` Text NOT NULL,
+  `hostname` Text NOT NULL,
+  `input` Text NOT NULL,
+  `ts_orig` Text,   
+  `level` Text,
+  `unit` Text,
+  `service` Text,
+  `msg` Text,
+  PRIMARY KEY (
+    `ts`, `datahash`, `dbname`, `hostname`, `input`
+  )
+) PARTITION BY HASH(`ts`, `dbname`, `hostname`, `input`)
+WITH (
+  STORE = COLUMN
+);
+
+GRANT USE ON `/cluster1/admin/ydblogs` TO fluentbit;

examples/fluentbit/dynamic-config.txt

@@ -0,0 +1,5 @@
+ydb auth get-token -f >ydbd-token
+./ydbd -s grpcs://bigpig1:2135 --ca-file ../tls/ca.crt -f ydbd-token admin console configs dump-yaml >step0.yaml
+cp step0.yaml step1.yaml
+vi step1.yaml
+ydb admin config replace -f step1.yaml 

examples/fluentbit/dynamic-config.yaml

@@ -0,0 +1,115 @@
+metadata:
+  kind: MainConfig
+  cluster: ""
+  version: 2
+config:
+  yaml_config_enabled: true
+  table_profiles_config:
+    table_profiles:
+    - name: default
+      compaction_policy: default
+      execution_policy: default
+      partitioning_policy: default
+      storage_policy: default
+      replication_policy: default
+      caching_policy: default
+    compaction_policies:
+    - name: default
+    execution_policies:
+    - name: default
+    partitioning_policies:
+    - name: default
+      auto_split: true
+      auto_merge: true
+      size_to_split: 2147483648
+    storage_policies:
+    - name: default
+      column_families:
+      - storage_config:
+          sys_log:
+            preferred_pool_kind: ssd
+          log:
+            preferred_pool_kind: ssd
+          data:
+            preferred_pool_kind: ssd
+    replication_policies:
+    - name: default
+    caching_policies:
+    - name: default
+allowed_labels:
+  node_id:
+    type: string
+  host:
+    type: string
+  tenant:
+    type: string
+selector_config:
+- description: cookie=testdb-01
+  selector:
+    tenant: /cluster1/testdb
+  config:
+    shared_cache_config:
+      memory_limit: 51539607552
+    feature_flags: !inherit
+      enable_views: true
+      enable_data_shard_volatile_transactions: true
+    table_service_config:
+      sql_version: 1
+      index_auto_choose_mode: MAX_USED_PREFIX
+      enable_implicit_query_parameter_types: true
+      enable_kqp_data_query_stream_lookup: true
+      enable_kqp_data_query_stream_idx_lookup_join: true
+      resource_manager:
+        kqp_pattern_cache_compiled_capacity_bytes: 524288000
+        kqp_pattern_cache_capacity_bytes: 524288000
+      query_limits:
+        result_rows_limit: 20000
+    resource_broker_config:
+      queues:
+      - name: queue_restore
+        limit:
+          cpu: 8
+      - name: queue_backup
+        limit:
+          cpu: 8
+    actor_system_config:
+      executor:
+      - {name: System, spin_threshold: '10', threads: 2, max_threads: 11, type: BASIC}
+      - {name: User, spin_threshold: '1', threads: 11, max_threads: 15, type: BASIC}
+      - {name: Batch, spin_threshold: '1', threads: 2, max_threads: 6, type: BASIC}
+      - {name: IO, threads: 1, type: IO}
+      - {name: IC, spin_threshold: '10', threads: 7, max_threads: 11, time_per_mailbox_micro_secs: 100, max_avg_ping_deviation: 500, type: BASIC}
+      scheduler: {progress_threshold: '10000', resolution: '64', spin_threshold: '0'}
+      sys_executor: 0
+      user_executor: 1
+      batch_executor: 2
+      io_executor: 3
+      service_executor:
+      - {executor_id: 4, service_name: Interconnect}
+- description: cookie=admin-01
+  selector:
+    tenant: /cluster1/admin
+  config:
+    shared_cache_config:
+      memory_limit: 17179869184
+    feature_flags: !inherit
+      enable_views: false
+      enable_data_shard_volatile_transactions: false
+    table_service_config:
+      sql_version: 1
+    column_shard_config:
+      disabled_on_scheme_shard: false
+    actor_system_config:
+      executor:
+      - {name: System, spin_threshold: '10', threads: 1, max_threads: 4, type: BASIC}
+      - {name: User, spin_threshold: '1', threads: 3, max_threads: 7, type: BASIC}
+      - {name: Batch, spin_threshold: '1', threads: 1, max_threads: 4, type: BASIC}
+      - {name: IO, threads: 1, type: IO}
+      - {name: IC, spin_threshold: '10', threads: 2, max_threads: 4, time_per_mailbox_micro_secs: 100, max_avg_ping_deviation: 500, type: BASIC}
+      scheduler: {progress_threshold: '10000', resolution: '64', spin_threshold: '0'}
+      sys_executor: 0
+      user_executor: 1
+      batch_executor: 2
+      io_executor: 3
+      service_executor:
+      - {executor_id: 4, service_name: Interconnect}

examples/fluentbit/fluent-bit.conf

@@ -0,0 +1,66 @@
+[SERVICE]
+    flush        2
+    log_level    info
+    parsers_file parsers.conf
+    plugins_file plugins.conf
+    storage.path /opt/ydb/fluentbit/data/storage
+
+[INPUT]
+    name systemd
+    Tag ydbd-storage
+    storage.type filesystem
+    DB /opt/ydb/fluentbit/data/ydbd-storage.db
+    Systemd_Filter  _SYSTEMD_UNIT=ydbd-storage.service
+
+[INPUT]
+    name systemd
+    Tag ydbd-admin
+    storage.type filesystem
+    DB /opt/ydb/fluentbit/data/ydbd-admin.db
+    Systemd_Filter  _SYSTEMD_UNIT=ydbd-admin-a.service
+
+[FILTER]
+    Name parser
+    Match ydbd-*
+    Key_Name MESSAGE
+    Parser ydb
+    Reserve_Data On
+    Preserve_Key On
+
+[FILTER]
+    Name modify
+    Match ydbd-*
+    Condition Key_does_not_exist P_LEVEL
+    Set P_LEVEL unknown
+    Hard_copy MESSAGE P_MESSAGE
+
+[FILTER]
+    Name modify
+    Match ydbd-storage
+    Add P_DB /cluster1
+
+[FILTER]
+    Name modify
+    Match ydbd-admin
+    Add P_DB /cluster1/admin
+
+[FILTER]
+    Name record_modifier
+    Match ydbd-*
+    Allowlist_key P_DB
+    Allowlist_key P_SERVICE
+    Allowlist_key P_DTTM
+    Allowlist_key P_LEVEL
+    Allowlist_key P_MESSAGE
+    Allowlist_key _HOSTNAME
+    Allowlist_key _SYSTEMD_UNIT
+
+[OUTPUT]
+    Name ydb
+    LogLevel error
+    ConnectionURL grpcs://ydbd-1.front.private:2135/cluster1/admin
+    CredentialsStatic fluentbit:password
+    Certificates /opt/ydb/cfg/ca.crt
+    Match ydbd-*
+    TablePath ydblogs
+    Columns {".timestamp": "ts",".input":"input",".hash":"datahash","P_DB":"dbname","P_SERVICE":"service","P_DTTM":"ts_orig","P_LEVEL":"level","P_MESSAGE":"msg","_HOSTNAME":"hostname","_SYSTEMD_UNIT":"unit"}

examples/fluentbit/fluent-bit.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Fluent Bit
+Documentation=https://docs.fluentbit.io/manual/
+Requires=network.target
+After=network.target
+
+[Service]
+Type=simple
+EnvironmentFile=-/etc/sysconfig/fluent-bit
+EnvironmentFile=-/etc/default/fluent-bit
+ExecStart=/usr/local/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

examples/fluentbit/parsers.conf

@@ -0,0 +1,5 @@
+[PARSER]
+    Name    ydb
+    Format  regex
+    Regex   ^(?<P_DTTM>[^ ]*) :(?<P_SERVICE>[^ ]*) (?<P_LEVEL>[^ ]*): (?<P_MESSAGE>.*).*$
+

examples/fluentbit/plugins.conf

@@ -0,0 +1,2 @@
+[PLUGINS]
+    out_ydb.so /opt/ydb/fluentbit/lib/out_ydb.so

examples/full-dualhomed/cluster-config.yaml

@@ -236,6 +236,7 @@ table_service_config:
     kqp_pattern_cache_capacity_bytes: 524288000
 
 feature_flags:
+  check_database_access_permission: true
   enable_views: true
   enable_data_shard_volatile_transactions: true
   enable_grpc_audit: false

files/config-1node.yaml

@@ -107,8 +107,7 @@ table_service_config:
   enable_kqp_data_query_stream_lookup: true
 
 feature_flags:
-  enable_grpc_audit: false
-  enable_get_node_labels: true
+  check_database_access_permission: true
   enable_views: true
 
 log_config:

files/config-3nodes-extra.yaml

@@ -184,8 +184,7 @@ bootstrap_config:
   shared_cache_config: {memory_limit: '4294967296'}
 
 feature_flags:
-  enable_grpc_audit: false
-  enable_get_node_labels: true
+  check_database_access_permission: true
   enable_views: true
 
 log_config:

files/config-3nodes.yaml

@@ -132,6 +132,7 @@ blob_storage_config:         # configuration of static blobstorage group.
           - node_id: ycydb-s3
             pdisk_category: SSD
             path: /dev/disk/by-partlabel/ydb_disk_3
+
 channel_profile_config:
   profile:
   - channel:
@@ -145,26 +146,35 @@ channel_profile_config:
       pdisk_category: 1
       storage_pool_kind: ssd
     profile_id: 0
+
 interconnect_config:
   start_tcp: true
   encryption_mode: OPTIONAL
   path_to_certificate_file: "/opt/ydb/certs/node.crt"
   path_to_private_key_file: "/opt/ydb/certs/node.key"
   path_to_ca_file: "/opt/ydb/certs/ca.crt"
+
 grpc_config:
     cert: "/opt/ydb/certs/node.crt"
     key: "/opt/ydb/certs/node.key"
     ca: "/opt/ydb/certs/ca.crt"
     services_enabled:
     - legacy
+
+feature_flags:
+  check_database_access_permission: true
+
 table_service_config:
   sql_version: 1
+
 bootstrap_config:
   shared_cache_config: {memory_limit: '4294967296'}
+
 audit_config:
   file_backend:
     format: JSON
     file_path: "ydb-audit.log"
+
 # Actor system config is appended to the configuration file by Ansible task.
 # Provided here in the commented form for reference
 #actor_system_config:

files/config-9nodes.yaml

@@ -5,6 +5,7 @@ host_configs:
   - path: /dev/disk/by-partlabel/ydb_disk_1
     type: SSD
   host_config_id: 1
+
 hosts:
 - host: ycydb-s1
   node_id: 1
@@ -69,6 +70,7 @@ hosts:
     body: 1
     data_center: 'zone-c'
     rack: '3'
+
 domains_config:
   domain:
   - name: Domain0
@@ -190,6 +192,9 @@ grpc_config:
 bootstrap_config:
   shared_cache_config: {memory_limit: '4294967296'}
 
+feature_flags:
+  check_database_access_permission: true
+
 table_service_config:
   sql_version: 1
 

fluentbit-all.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  any_errors_fatal: true
+  roles:
+  - role: ydbd_fluentbit

group_vars/all.example

@@ -75,6 +75,19 @@ ydb_congestion_setting: htcp
 # Optional module to load the congestion control algorithm
 ydb_congestion_module: tcp_htcp
 
+# Fluentbit software with the included YDB plugin
+ydb_fluentbit_archive: "../fluent-bit-ydb.tar.xz"
+
+# Destination database for logs delivery
+ydb_fluentbit_destination: "grpcs://ycydb-s1:2135/Domain0/admin"
+
+# Logs delivery destination database trusted CA
+ydb_fluentbit_ca: "/opt/ydb/cfg/ca.crt"
+
+# Logs delivery destination database credentials.
+# See examples/fluentbit for DDL and permissions.
+ydb_fluentbit_credentials: "fluentbit:password"
+
 # Should only be defined when YDB cluster extension is performed.
-# Must not be defined during the initial cluster setup.
+# Must NOT be defined during the initial cluster setup.
 # ydb_cluster_extension: true