Skip to content

Commit

Permalink
Merge branch 'OpenAtomFoundation:unstable' into unstable
Browse files Browse the repository at this point in the history
  • Loading branch information
@chejinge
chejinge authored Nov 20, 2024
2 parents b3d763e + 77177c7 commit 29bfafa
Show file tree
Hide file tree
Showing 4 changed files with 188 additions and 42 deletions.
37 changes: 9 additions & 28 deletions .github/workflows/pika.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,51 +84,32 @@ jobs:
chmod +x integrate_test.sh
sh integrate_test.sh
build_on_rocky:
build_on_centos:
runs-on: ubuntu-latest
container:
image: rockylinux:9
image: cheniujh/pika-centos7-ci:v5

steps:
- name: Install deps
run: |
dnf update -y
dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.19

- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v1
with:
fetch-depth: 0

- name: Configure CMake
run: |
source /opt/rh/gcc-toolset-13/enable
cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address .
- uses: actions/cache@v3
with:
path: ${{ github.workspace }}/deps
key: ${{ runner.os }}-rocky-deps-${{ hashFiles('**/CMakeLists.txt') }}

- uses: actions/cache@v3
with:
path: ${{ github.workspace }}/buildtrees
key: ${{ runner.os }}-rocky-buildtrees-${{ hashFiles('**/CMakeLists.txt') }}
source /opt/rh/devtoolset-10/enable
cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
- name: Build
run: |
source /opt/rh/gcc-toolset-13/enable
source /opt/rh/devtoolset-10/enable
cmake --build build --config ${{ env.BUILD_TYPE }}
- name: Cleanup
run: |
rm -rf ./deps
rm -rf ./buildtrees
rm -rf ./buildtrees
- name: Test
working-directory: ${{ github.workspace }}/build
Expand All @@ -145,7 +126,6 @@ jobs:
../tests/integration/start_master_and_slave.sh
chmod +x ../tests/integration/start_codis.sh
../tests/integration/start_codis.sh
- name: Run Go E2E Tests
working-directory: ${{ github.workspace }}/build
run: |
Expand All @@ -155,6 +135,7 @@ jobs:
chmod +x integrate_test.sh
sh integrate_test.sh
build_on_macos:

runs-on: macos-13
Expand Down
2 changes: 1 addition & 1 deletion src/pika_repl_bgworker.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ void PikaReplBgWorker::WriteDBInSyncWay(const std::shared_ptr<Cmd>& c_ptr) {
if (duration > g_pika_conf->slowlog_slower_than()) {
g_pika_server->SlowlogPushEntry(argv, start_time, duration);
if (g_pika_conf->slowlog_write_errorlog()) {
LOG(ERROR) << "command: " << argv[0] << ", start_time(s): " << start_time << ", duration(us): " << duration;
LOG(INFO) << "command: " << argv[0] << ", start_time(s): " << start_time << ", duration(us): " << duration;
}
}
}
Expand Down
153 changes: 143 additions & 10 deletions tools/pika_exporter/exporter/metrics/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,141 @@ const (
)

type ParseOption struct {
Version *semver.Version
Extracts map[string]string
ExtractsProxy map[string][]int64
Info string
Version *semver.Version
Extracts map[string]string
ExtractsProxy map[string][]int64
Info string
CurrentVersion VersionChecker
}
type VersionChecker interface {
CheckContainsEmptyValueName(key string) bool
CheckContainsEmptyRegexName(key string) bool
InitVersionChecker()
}
type VersionChecker336 struct {
EmptyValueName []string
EmptyRegexName []string
}

func (v *VersionChecker336) InitVersionChecker() {
if v.EmptyValueName == nil {
v.EmptyValueName = []string{
"instantaneous_output_repl_kbps",
"total_net_output_bytes",
"cache_db_num",
"hits_per_sec",
"cache_status",
"total_net_input_bytes",
"instantaneous_output_kbps",
"instantaneous_input_kbps",
"total_net_repl_input_bytes",
"instantaneous_input_repl_kbps",
"slow_logs_count",
"total_net_repl_output_bytes",
"cache_memory",
}
}
if v.EmptyRegexName == nil {
v.EmptyRegexName = []string{
"hitratio_per_sec",
}
}
}
func (v *VersionChecker336) CheckContainsEmptyValueName(key string) bool {
for _, str := range v.EmptyValueName {
if str == key {
return true
}
}
return false
}
func (v *VersionChecker336) CheckContainsEmptyRegexName(key string) bool {
for _, str := range v.EmptyRegexName {
if str == key {
return true
}
}
return false
}

type VersionChecker350 struct {
EmptyValueName []string
EmptyRegexName []string
}

func (v *VersionChecker350) InitVersionChecker() {
if v.EmptyValueName == nil {
v.EmptyValueName = []string{
"cache_db_num",
"cache_status",
"cache_memory",
"hits_per_sec",
"slow_logs_count",
}
}
if v.EmptyRegexName == nil {
v.EmptyRegexName = []string{
"hitratio_per_sec",
}
}
}
func (v *VersionChecker350) CheckContainsEmptyValueName(key string) bool {
for _, str := range v.EmptyValueName {
if str == key {
return true
}
}
return false
}
func (v *VersionChecker350) CheckContainsEmptyRegexName(key string) bool {
for _, str := range v.EmptyRegexName {
if str == key {
return true
}
}
return false
}

type VersionChecker355 struct {
EmptyValueName []string
EmptyRegexName []string
}

func (v *VersionChecker355) InitVersionChecker() {
if v.EmptyValueName == nil {
v.EmptyValueName = []string{
"cache_db_num",
"cache_status",
"cache_memory",
"hits_per_sec",
}
}
if v.EmptyRegexName == nil {
v.EmptyRegexName = []string{
"hitratio_per_sec",
"keyspace_info_>=3.1.0",
"keyspace_info_all_>=3.3.3",
"binlog_>=3.2.0",
"keyspace_last_start_time",
"is_scaning_keyspace",
}
}
}
func (v *VersionChecker355) CheckContainsEmptyValueName(key string) bool {
for _, str := range v.EmptyValueName {
if str == key {
return true
}
}
return false
}
func (v *VersionChecker355) CheckContainsEmptyRegexName(key string) bool {
for _, str := range v.EmptyRegexName {
if str == key {
return true
}
}
return false
}

type Parser interface {
Expand Down Expand Up @@ -112,9 +243,10 @@ func (p *regexParser) Parse(m MetricMeta, c Collector, opt ParseOption) {

matchMaps := p.regMatchesToMap(s)
if len(matchMaps) == 0 {
log.Warnf("regexParser::Parse reg find sub match nil. name:%s", p.name)
if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyRegexName(p.name) {
log.Warnf("regexParser::Parse reg find sub match nil. name:%s", p.name)
}
}

extracts := make(map[string]string)
for k, v := range opt.Extracts {
extracts[k] = v
Expand All @@ -136,7 +268,6 @@ func (p *regexParser) regMatchesToMap(s string) []map[string]string {

multiMatches := p.reg.FindAllStringSubmatch(s, -1)
if len(multiMatches) == 0 {
log.Errorf("regexParser::regMatchesToMap reg find sub match nil. name:%s", p.name)
return nil
}

Expand Down Expand Up @@ -172,8 +303,11 @@ func (p *normalParser) Parse(m MetricMeta, c Collector, opt ParseOption) {

if m.ValueName != "" {
if v, ok := findInMap(m.ValueName, opt.Extracts); !ok {
log.Warnf("normalParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName)
if opt.CurrentVersion == nil || !opt.CurrentVersion.CheckContainsEmptyValueName(m.ValueName) {
log.Warnf("normalParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName)
}
return

} else {
metric.Value = convertToFloat64(v)
}
Expand Down Expand Up @@ -208,7 +342,6 @@ func (p *timeParser) Parse(m MetricMeta, c Collector, opt ParseOption) {

if m.ValueName != "" {
if v, ok := findInMap(m.ValueName, opt.Extracts); !ok {
log.Warnf("timeParser::Parse not found value. metricName:%s valueName:%s", m.Name, m.ValueName)
return
} else {
t, err := convertTimeToUnix(v)
Expand All @@ -227,14 +360,14 @@ func (p *timeParser) Parse(m MetricMeta, c Collector, opt ParseOption) {
}

func findInMap(key string, ms ...map[string]string) (string, bool) {

for _, m := range ms {
if v, ok := m[key]; ok {
return v, true
}
}
return "", false
}

func trimSpace(s string) string {
return strings.TrimRight(strings.TrimLeft(s, " "), " ")
}
Expand Down
38 changes: 35 additions & 3 deletions tools/pika_exporter/exporter/pika.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,9 +252,10 @@ func (e *exporter) collectInfo(c *client, ch chan<- prometheus.Metric) error {
return nil
})
parseOpt := metrics.ParseOption{
Version: version,
Extracts: extracts,
Info: info,
Version: version,
Extracts: extracts,
Info: info,
CurrentVersion: selectversion(version.Original()),
}
for _, m := range metrics.MetricConfigs {
m.Parse(m, collector, parseOpt)
Expand All @@ -263,6 +264,37 @@ func (e *exporter) collectInfo(c *client, ch chan<- prometheus.Metric) error {
return nil
}

const (
VERSION_336 = "3.3.6"
VERSION_350 = "3.5.0"
VERSION_355 = "3.5.5"
)

func selectversion(version string) metrics.VersionChecker {
if !isValidVersion(version) {
log.Warnf("Invalid version format: %s", version)
return nil
}
var v metrics.VersionChecker
switch version {
case VERSION_336:
v = &metrics.VersionChecker336{}
case VERSION_355:
v = &metrics.VersionChecker355{}
case VERSION_350:
v = &metrics.VersionChecker350{}
default:
return nil
}
v.InitVersionChecker()
return v
}

// isValidVersion validates the version string format (e.g., x.y.z)
func isValidVersion(version string) bool {
matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+$`, version)
return matched
}
func (e *exporter) collectKeys(c *client) error {
allKeys := append([]dbKeyPair{}, e.keys...)
keys, err := getKeysFromPatterns(c, e.keyPatterns, e.scanCount)
Expand Down

0 comments on commit 29bfafa

Please sign in to comment.