chore: support parameters api for pulsar (#1365)

(cherry picked from commit ab80d7e)

addons-cluster/pulsar/templates/cluster.yaml

@@ -10,7 +10,7 @@ metadata:
   {{- end }}
 spec:
   terminationPolicy: {{ $.Values.terminationPolicy }}
-  {{- $topologyEnable :=  eq .Values.mode "" | default false }}
+  {{- $topologyEnable :=  ne .Values.mode "" | default false }}
   {{- if $topologyEnable}}
   clusterDef: pulsar
   topology: {{ .Values.mode }}

addons/pulsar/config/2.11.2/bookies-config.tpl

@@ -75,4 +75,6 @@ zkLedgersRootPath=/ledgers
     {{- $autoRecoveryDaemonEnabled = "false" }}
   {{- end }}
 {{- end }}
-autoRecoveryDaemonEnabled={{ $autoRecoveryDaemonEnabled }}
+autoRecoveryDaemonEnabled={{ $autoRecoveryDaemonEnabled }}
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/bookies-recovery-config.tpl

@@ -0,0 +1,9 @@
+httpServerEnabled=true
+httpServerPort=8000
+prometheusStatsHttpPort=8000
+useHostNameAsBookieID=true
+# how long to wait, in seconds, before starting autorecovery of a lost bookie.
+# TODO: set to 0 after opsRequest for rollingUpdate supports hooks
+lostBookieRecoveryDelay=300
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/broker-config.tpl

@@ -57,3 +57,6 @@ brokerDeleteInactiveTopicsEnabled=false
 # configurations in KoP because of KIP-679.
 kafkaTransactionCoordinatorEnabled=true
 brokerDeduplicationEnabled=true
+
+zookeeperServers={{ .ZOOKEEPER_SERVERS }}
+configurationStoreServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/proxy-config.tpl

@@ -1,2 +1,4 @@
 statusFilePath=/pulsar/status
-maxMessageSize=5242880
+maxMessageSize=5242880
+
+metadataStoreUrl={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/zookeeper-config.tpl

@@ -0,0 +1,2 @@
+dataDir=/pulsar/data/zookeeper
+serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory

addons/pulsar/config/3.0.2/bookies-config.tpl

@@ -78,3 +78,5 @@ zkLedgersRootPath=/ledgers
   {{- end }}
 {{- end }}
 autoRecoveryDaemonEnabled={{ $autoRecoveryDaemonEnabled }}
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/bookies-recovery-config.tpl

@@ -0,0 +1,9 @@
+httpServerEnabled=true
+httpServerPort=8000
+prometheusStatsHttpPort=8000
+useHostNameAsBookieID=true
+# how long to wait, in seconds, before starting autorecovery of a lost bookie.
+# TODO: set to 0 after opsRequest for rollingUpdate supports hooks
+lostBookieRecoveryDelay=300
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/broker-config.tpl

@@ -57,3 +57,6 @@ brokerDeleteInactiveTopicsEnabled=false
 # configurations in KoP because of KIP-679.
 kafkaTransactionCoordinatorEnabled=true
 brokerDeduplicationEnabled=true
+
+zookeeperServers={{ .ZOOKEEPER_SERVERS }}
+configurationStoreServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/proxy-config.tpl

@@ -1,2 +1,4 @@
 statusFilePath=/pulsar/status
-maxMessageSize=5242880
+maxMessageSize=5242880
+
+metadataStoreUrl={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/zookeeper-config.tpl

@@ -0,0 +1,2 @@
+dataDir=/pulsar/data/zookeeper
+serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory

addons/pulsar/config/pulsar-env-optimize.tpl

@@ -0,0 +1,9 @@
+## for proxy or broker
+export PULSAR_EXTRA_OPTS="-Dpulsar.allocator.exit_on_oom=true -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024 -Dnetworkaddress.cache.ttl=60 -XX:ActiveProcessorCount=1 -XshowSettings:vm -Ddepth=64"
+export PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB"
+{{- $maxDirectMemorySize := "" }}
+{{- $phyMemory := getContainerMemory ( index $.podSpec.containers 0 ) }}
+{{- if gt $phyMemory 0 }}
+  {{- $maxDirectMemorySize = printf "-XX:MaxDirectMemorySize=%dm" (mul (div $phyMemory ( mul 1024 1024 10)) 6) }}
+{{- end }}
+export PULSAR_MEM="-XX:MinRAMPercentage=30 -XX:MaxRAMPercentage=30 {{ $maxDirectMemorySize }}"

addons/pulsar/config/pulsar-env.tpl

@@ -0,0 +1,8 @@
+export PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XshowSettings:vm -Ddepth=64"
+{{- $maxDirectMemorySize := "" }}
+{{- $phyMemory := getContainerMemory ( index $.podSpec.containers 0 ) }}
+{{- if gt $phyMemory 0 }}
+  {{- $phyMemoryMB := div $phyMemory ( mul 1024 1024 ) }}
+  {{- $maxDirectMemorySize = printf "-XX:MaxDirectMemorySize=%dm" (div ( mul $phyMemoryMB 3 ) 4 ) }}
+{{- end }}
+export PULSAR_MEM="-XX:MinRAMPercentage=25 -XX:MaxRAMPercentage=50 {{ $maxDirectMemorySize }}"

addons/pulsar/config/zookeeper-env.tpl

@@ -1,5 +1,2 @@
-PULSAR_GC: -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:ActiveProcessorCount=1
-PULSAR_MEM: -XX:MinRAMPercentage=40 -XX:MaxRAMPercentage=60
-PULSAR_PREFIX_serverCnxnFactory: org.apache.zookeeper.server.NIOServerCnxnFactory
-dataDir: /pulsar/data/zookeeper
-serverCnxnFactory: org.apache.zookeeper.server.NIOServerCnxnFactory
+export PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:ActiveProcessorCount=1"
+export PULSAR_MEM="-XX:MinRAMPercentage=40 -XX:MaxRAMPercentage=60"

addons/pulsar/scripts-ut-spec/init_bookies_spec.sh

@@ -89,7 +89,7 @@ Describe "Pulsar Init Bookies Bash Script Tests"
 
   Describe "init_bookies()"
     It "initializes bookies"
-      export zkServers="zookeeper.example.com:2181"
+      export ZOOKEEPER_SERVERS="zookeeper.example.com:2181"
 
       wait_for_zookeeper() {
         echo "wait_for_zookeeper called"
@@ -114,11 +114,11 @@ Describe "Pulsar Init Bookies Bash Script Tests"
       The output should include "init_bookkeeper_cluster called"
     End
 
-    It "exits with status 1 when zkServers environment variable is not set"
-      unset zkServers
+    It "exits with status 1 when ZOOKEEPER_SERVERS environment variable is not set"
+      unset ZOOKEEPER_SERVERS
 
       When run init_bookies
-      The output should include "Error: zkServers environment variable is not set, Please set the zkServers environment variable and try again."
+      The output should include "Error: ZOOKEEPER_SERVERS environment variable is not set, Please set the ZOOKEEPER_SERVERS environment variable and try again."
       The status should be failure
     End
   End

addons/pulsar/scripts-ut-spec/init_broker_spec.sh

@@ -18,7 +18,7 @@ Describe "Pulsar Init Broker Bash Script Tests"
 
   Describe "check_env_variables()"
     It "checks if required environment variables are set"
-      export zookeeperServers="localhost:2181"
+      export ZOOKEEPER_SERVERS="localhost:2181"
       export POD_NAME="broker-0"
       export clusterName="my-cluster"
       export webServiceUrl="http://localhost:8080"
@@ -32,7 +32,7 @@ Describe "Pulsar Init Broker Bash Script Tests"
       unset zookeeperServers
 
       When run check_env_variables
-      The output should include "Error: zookeeperServers environment variable is not set, Please set the zookeeperServers environment variable and try again."
+      The output should include "Error: ZOOKEEPER_SERVERS environment variable is not set, Please set the ZOOKEEPER_SERVERS environment variable and try again."
       The status should be failure
     End
   End

addons/pulsar/scripts-ut-spec/init_proxy_spec.sh

@@ -32,7 +32,7 @@ Describe "Pulsar Init Proxy Bash Script Tests"
 
   Describe "main()"
     It "waits for Zookeeper when metadataStoreUrl is provided"
-      export metadataStoreUrl="zookeeper.example.com:2181"
+      export ZOOKEEPER_SERVERS="zookeeper.example.com:2181"
 
       wait_for_zookeeper() {
         echo "wait_for_zookeeper called with $1"
@@ -43,7 +43,7 @@ Describe "Pulsar Init Proxy Bash Script Tests"
     End
 
     It "skips Zookeeper readiness check when metadataStoreUrl is not provided"
-      unset metadataStoreUrl
+      unset ZOOKEEPER_SERVERS
 
       wait_for_zookeeper() {
         echo "wait_for_zookeeper called"

addons/pulsar/scripts-ut-spec/start_bookies_spec.sh

@@ -90,15 +90,15 @@ Describe "Pulsar Start Bookies Bash Script Tests"
       zkURL="zookeeper.example.com"
 
       When run handle_empty_directories
-      The stderr should include "Error: BOOKKEEPER_POD_FQDN_LIST or CURRENT_POD_NAME or zkServers is empty. Exiting."
+      The stderr should include "Error: BOOKKEEPER_POD_FQDN_LIST or CURRENT_POD_NAME or ZOOKEEPER_SERVERS is empty. Exiting."
       The stdout should include "journalRes and ledgerRes directory is empty, check whether the remote cookies is empty either"
       The status should be failure
     End
 
     It "removes redundant bookieID if necessary"
       BOOKKEEPER_POD_FQDN_LIST="pod1,pod2"
       CURRENT_POD_NAME="pod1"
-      zkServers="pod1.svc.cluster.local"
+      ZOOKEEPER_SERVERS="pod1.svc.cluster.local"
 
       get_target_pod_fqdn_from_pod_fqdn_vars() {
         echo "pod1.example.com"

addons/pulsar/scripts/check-bookies.sh

@@ -23,6 +23,24 @@ wait_for_bookkeeper() {
   echo "Bookkeeper started successfully"
 }
 
+merge_bookkeeper_config() {
+  local src_config="conf/bookkeeper.conf"
+  local dest_config="/opt/pulsar/conf/bookkeeper.conf"
+
+  echo "Merging Pulsar configuration files:"
+  echo "  - Source: $src_config"
+  echo "  - Destination: $dest_config"
+  python3 /kb-scripts/merge_pulsar_config.py "$src_config" "$dest_config"
+}
+
+load_env_file() {
+  local pulsar_env_config="/opt/pulsar/conf/pulsar.env"
+
+  if [ -f "${pulsar_env_config}" ];then
+     source ${pulsar_env_config}
+  fi
+}
+
 set_tcp_keepalive() {
   local keepalive_time=1
   local keepalive_intvl=11
@@ -46,6 +64,8 @@ set_tcp_keepalive() {
 ${__SOURCED__:+false} : || return 0
 
 # main
+load_env_file
+merge_bookkeeper_config
 apply_config_from_env
 wait_for_bookkeeper
 set_tcp_keepalive