Skip to content

Commit

Permalink
use slp instead of pt-query-digest
Browse files Browse the repository at this point in the history
  • Loading branch information
kaz committed Aug 21, 2022
1 parent 29898d2 commit be31fba
Show file tree
Hide file tree
Showing 11 changed files with 215 additions and 478 deletions.
14 changes: 4 additions & 10 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,13 @@ RUN make build

# --------------------------------------------------

FROM alpine AS alp
FROM alpine AS tools

RUN apk add go

ENV GOPATH /go
RUN go install github.com/tkuchiki/alp/cli/alp@latest

# --------------------------------------------------

FROM alpine AS percona-toolkit

RUN wget https://downloads.percona.com/downloads/percona-toolkit/3.4.0/binary/tarball/percona-toolkit-3.4.0_x86_64.tar.gz
RUN tar zxvf percona-toolkit-3.4.0_x86_64.tar.gz
RUN go install github.com/tkuchiki/slp/cmd/slp@latest

# --------------------------------------------------

Expand All @@ -36,8 +30,8 @@ RUN apk add --no-cache bash perl perl-dbd-mysql perl-time-hires graphviz

COPY --from=pprotein /go/src/app/pprotein /usr/local/bin/
COPY --from=pprotein /go/src/app/pprotein-agent /usr/local/bin/
COPY --from=alp /go/bin/alp /usr/local/bin/
COPY --from=percona-toolkit /percona-toolkit-3.4.0/bin/* /usr/local/bin/
COPY --from=tools /go/bin/alp /usr/local/bin/
COPY --from=tools /go/bin/slp /usr/local/bin/

RUN mkdir -p /opt/pprotein
WORKDIR /opt/pprotein
Expand Down
24 changes: 14 additions & 10 deletions cli/pprotein/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/kaz/pprotein/internal/collect/group"
"github.com/kaz/pprotein/internal/event"
"github.com/kaz/pprotein/internal/extproc/alp"
"github.com/kaz/pprotein/internal/extproc/querydigest"
"github.com/kaz/pprotein/internal/extproc/slp"
"github.com/kaz/pprotein/internal/memo"
"github.com/kaz/pprotein/internal/pprof"
"github.com/kaz/pprotein/internal/storage"
Expand Down Expand Up @@ -47,47 +47,51 @@ func start() error {
hub := event.NewHub()
hub.RegisterHandlers(api.Group("/event"))

pOpts := &collect.Options{
pprofOpts := &collect.Options{
Type: "pprof",
Ext: "-pprof.pb.gz",
Store: store,
EventHub: hub,
}
if err := pprof.NewHandler(pOpts).Register(api.Group("/pprof")); err != nil {
if err := pprof.NewHandler(pprofOpts).Register(api.Group("/pprof")); err != nil {
return err
}

aOpts := &collect.Options{
alpOpts := &collect.Options{
Type: "httplog",
Ext: "-httplog.log",
Store: store,
EventHub: hub,
}
aHandler, err := alp.NewHandler(aOpts, store)
alpHandler, err := alp.NewHandler(alpOpts, store)
if err != nil {
return err
}
if err := aHandler.Register(api.Group("/httplog")); err != nil {
if err := alpHandler.Register(api.Group("/httplog")); err != nil {
return err
}

qOpts := &collect.Options{
slpOpts := &collect.Options{
Type: "slowlog",
Ext: "-slowlog.log",
Store: store,
EventHub: hub,
}
if err := querydigest.NewHandler(qOpts).Register(api.Group("/slowlog")); err != nil {
slpHandler, err := slp.NewHandler(slpOpts, store)
if err != nil {
return err
}
if err := slpHandler.Register(api.Group("/slowlog")); err != nil {
return err
}

mOpts := &collect.Options{
memoOpts := &collect.Options{
Type: "memo",
Ext: "-memo.log",
Store: store,
EventHub: hub,
}
if err := memo.NewHandler(mOpts).Register(api.Group("/memo")); err != nil {
if err := memo.NewHandler(memoOpts).Register(api.Group("/memo")); err != nil {
return err
}

Expand Down
26 changes: 0 additions & 26 deletions internal/extproc/querydigest/handler.go

This file was deleted.

59 changes: 59 additions & 0 deletions internal/extproc/slp/handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package slp

import (
_ "embed"
"fmt"

"github.com/kaz/pprotein/internal/collect"
"github.com/kaz/pprotein/internal/extproc"
"github.com/kaz/pprotein/internal/persistent"
"github.com/kaz/pprotein/internal/storage"
"github.com/labstack/echo/v4"
"gopkg.in/yaml.v3"
)

type (
handler struct {
opts *collect.Options
config *persistent.Handler
}
)

//go:embed slp.yml
var defaultConfig []byte

func NewHandler(opts *collect.Options, store storage.Storage) (*handler, error) {
h := &handler{
opts: opts,
}

config, err := persistent.New(store, "slp.yml", defaultConfig, h.sanitize)
if err != nil {
return nil, fmt.Errorf("failed to create targets: %w", err)
}
h.config = config

return h, nil
}

func (h *handler) Register(g *echo.Group) error {
h.config.RegisterHandlers(g.Group("/config"))

if err := extproc.NewHandler(&processor{confPath: h.config.GetPath()}, h.opts).Register(g); err != nil {
return fmt.Errorf("failed to register extproc handlers: %w", err)
}
return nil
}

func (h *handler) sanitize(raw []byte) ([]byte, error) {
var config interface{}
if err := yaml.Unmarshal(raw, &config); err != nil {
return nil, fmt.Errorf("failed to unmarshal: %w", err)
}

res, err := yaml.Marshal(config)
if err != nil {
return nil, fmt.Errorf("failed to marshal: %w", err)
}
return res, nil
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package querydigest
package slp

import (
"bytes"
Expand All @@ -10,7 +10,9 @@ import (
)

type (
processor struct{}
processor struct {
confPath string
}
)

func (p *processor) Cacheable() bool {
Expand All @@ -23,13 +25,12 @@ func (p *processor) Process(snapshot *collect.Snapshot) (io.ReadCloser, error) {
return nil, fmt.Errorf("failed to find snapshot body: %w", err)
}

cmd := exec.Command("pt-query-digest", "--limit", "100%", "--output", "json", bodyPath)
cmd := exec.Command("slp", "--config", p.confPath, "--output", "standard", "--format", "tsv", "--file", bodyPath)

res, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("external process aborted: %w", err)
}

starts := bytes.IndexByte(res, '{')
return io.NopCloser(bytes.NewBuffer(res[starts:])), nil
return io.NopCloser(bytes.NewBuffer(res)), nil
}
4 changes: 4 additions & 0 deletions internal/extproc/slp/slp.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
bundle_values: true
bundle_where_in: true
filters: Query matches "^(SELECT|INSERT|UPDATE|REPLACE) "
limit: 65536
133 changes: 7 additions & 126 deletions view/src/components/HttpLogEntry.vue
Original file line number Diff line number Diff line change
@@ -1,142 +1,23 @@
<template>
<section>
<pre v-if="!logData">Loading ...</pre>
<div v-else class="container">
<table border="1">
<thead>
<tr>
<th v-for="h in header" :key="h" @click="sortBy(h)">
<span>
{{ h }}
<span v-if="sort == h" class="sortOrder">
{{ order == "asc" ? "▲" : "▼" }}
</span>
</span>
</th>
</tr>
</thead>
<tbody>
<tr v-for="d in sortedLogData" :key="d.toString()">
<td v-for="(v, idx) in d" :key="header[idx]">{{ v }}</td>
</tr>
</tbody>
</table>
<div class="tips">Click header to sort</div>
</div>
</section>
<TsvTable :tsv="tsv" />
</template>

<script lang="ts">
import { defineComponent } from "vue";
import Pase, { ParseResult } from "papaparse";
const isNumeric = (v: string) => {
return !isNaN(+v);
};
import TsvTable from "./TsvTable.vue";
export default defineComponent({
components: {
TsvTable,
},
data() {
return {
logData: {} as ParseResult<string[]>,
sort: "count",
order: "desc",
tsv: "",
};
},
computed: {
header: function () {
if (!this.logData.data) {
return [];
}
let data: string[][];
data = this.logData.data;
return data[0];
},
sortedLogData: function () {
if (!this.logData.data) {
return [];
}
let data: string[][];
data = this.logData.data.slice(1);
let index = this.header.findIndex((h: string) => h === this.sort);
if (index == -1) {
index = 0;
}
data.sort((a: string[], b: string[]): number => {
if (isNumeric(a[index])) {
return +a[index] - +b[index];
} else {
if (a[index] < b[index]) {
return -1;
} else if (a[index] > b[index]) {
return 1;
} else {
return 0;
}
}
});
if (this.order == "desc") {
return data.reverse();
} else {
return data;
}
},
},
async beforeCreate() {
const resp = await fetch(`/api/httplog/${this.$route.params.id}`);
const text = await resp.text();
const logData = Pase.parse<string[]>(text, { skipEmptyLines: true });
this.logData = logData;
},
methods: {
sortBy(key: string) {
if (this.sort === key) {
this.order = this.order == "desc" ? "asc" : "desc";
} else {
this.sort = key;
this.order = "desc";
}
},
this.tsv = await resp.text();
},
});
</script>

<style scoped lang="scss">
section {
margin: 2em;
}
pre {
overflow: auto;
flex: 1 0 auto;
}
.container {
overflow: scroll;
}
table {
border-collapse: collapse;
}
td,
th {
padding: 0.5em 1em;
border: 1px solid #999;
}
th > span {
cursor: pointer;
.sortOrder {
font-size: 0.4em;
color: blue;
}
}
.tips {
margin-top: 3em;
text-align: right;
}
</style>
Loading

0 comments on commit be31fba

Please sign in to comment.