Skip to content

Commit

Permalink
cache test
Browse files Browse the repository at this point in the history
  • Loading branch information
pirosiki197 committed Jan 29, 2025
1 parent ae155e7 commit 6129918
Show file tree
Hide file tree
Showing 17 changed files with 1,085 additions and 33 deletions.
4 changes: 0 additions & 4 deletions extractor/dynamic/driver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,6 @@ import (
"github.com/traP-jp/isuc/testutil"
)

func TestMain(m *testing.M) {
testutil.InitialSetupDB(m)
}

func TestExtract(t *testing.T) {
db := testutil.SetupMysqlDB(t, "mysql+analyzer")
defer db.Close()
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rH
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
Expand All @@ -63,10 +65,12 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
Expand Down
21 changes: 21 additions & 0 deletions template/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,27 @@ func ExportMetrics() string {
return res
}

type CacheStats struct {
Query string
HitRatio float64
Hits int
Misses int
}

func ExportCacheStats() map[string]CacheStats {
res := make(map[string]CacheStats)
for query, cache := range caches {
stats := cache.cache.Stats()
res[query] = CacheStats{
Query: query,
HitRatio: stats.HitRatio(),
Hits: int(stats.Hits),
Misses: int(stats.Misses),
}
}
return res
}

func PurgeAllCaches() {
for _, cache := range caches {
cache.cache.Purge()
Expand Down
21 changes: 21 additions & 0 deletions template/cache.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,27 @@ func ExportMetrics() string {
return res
}

type CacheStats struct {
Query string
HitRatio float64
Hits int
Misses int
}

func ExportCacheStats() map[string]CacheStats {
res := make(map[string]CacheStats)
for query, cache := range caches {
stats := cache.cache.Stats()
res[query] = CacheStats{
Query: query,
HitRatio: stats.HitRatio(),
Hits: int(stats.Hits),
Misses: int(stats.Misses),
}
}
return res
}

func PurgeAllCaches() {
for _, cache := range caches {
cache.cache.Purge()
Expand Down
4 changes: 0 additions & 4 deletions template/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ import (
"github.com/traP-jp/isuc/testutil"
)

func TestMain(m *testing.M) {
testutil.InitialSetupDB(m)
}

func TestCacheRows(t *testing.T) {
db := testutil.SetupMysqlDB(t, "mysql")
defer db.Close()
Expand Down
8 changes: 5 additions & 3 deletions template/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,17 @@ func init() {
}

for _, query := range plan.Queries {
queryMap[query.Query] = *query
normalized := normalizer.NormalizeQuery(query.Query)
query.Query = normalized // make sure to use normalized query
queryMap[normalized] = *query
if query.Type != domains.CachePlanQueryType_SELECT || !query.Select.Cache {
continue
}

conditions := query.Select.Conditions
if isSingleUniqueCondition(conditions, query.Select.Table) {
caches[query.Query] = cacheWithInfo{
query: query.Query,
caches[normalized] = cacheWithInfo{
query: normalized,
info: *query.Select,
cache: sc.NewMust(replaceFn, 10*time.Minute, 10*time.Minute),
uniqueOnly: true,
Expand Down
8 changes: 5 additions & 3 deletions template/driver.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,17 @@ func init() {
}

for _, query := range plan.Queries {
queryMap[query.Query] = *query
normalized := normalizer.NormalizeQuery(query.Query)
query.Query = normalized // make sure to use normalized query
queryMap[normalized] = *query
if query.Type != domains.CachePlanQueryType_SELECT || !query.Select.Cache {
continue
}

conditions := query.Select.Conditions
if isSingleUniqueCondition(conditions, query.Select.Table) {
caches[query.Query] = cacheWithInfo{
query: query.Query,
caches[normalized] = cacheWithInfo{
query: normalized,
info: *query.Select,
cache: sc.NewMust(replaceFn, 10*time.Minute, 10*time.Minute),
uniqueOnly: true,
Expand Down
111 changes: 111 additions & 0 deletions test/cache/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package cache

import (
"context"
"database/sql/driver"
"fmt"
"strings"
)

type (
queryKey struct{}
stmtKey struct{}
argsKey struct{}
queryerCtxKey struct{}
namedValueArgsKey struct{}
)

func ExportMetrics() string {
res := ""
for query, cache := range caches {
stats := cache.cache.Stats()
progress := "["
for i := 0; i < 20; i++ {
if i < int(stats.HitRatio()*20) {
progress += "#"
} else {
progress += "-"
}
}
statsStr := fmt.Sprintf("%s (%.2f%% - %d/%d) (%d replace) (size %d)", progress, stats.HitRatio()*100, stats.Hits, stats.Misses+stats.Hits, stats.Replacements, stats.Size)
res += fmt.Sprintf("query: \"%s\"\n%s\n\n", query, statsStr)
}
return res
}

type CacheStats struct {
Query string
HitRatio float64
Hits int
Misses int
}

func ExportCacheStats() map[string]CacheStats {
res := make(map[string]CacheStats)
for query, cache := range caches {
stats := cache.cache.Stats()
res[query] = CacheStats{
Query: query,
HitRatio: stats.HitRatio(),
Hits: int(stats.Hits),
Misses: int(stats.Misses),
}
}
return res
}

func PurgeAllCaches() {
for _, cache := range caches {
cache.cache.Purge()
}
}

func cacheName(query string) string {
return query
}

func cacheKey(args []driver.Value) string {
var b strings.Builder
for _, arg := range args {
switch v := arg.(type) {
case string:
b.WriteString(v)
case []byte:
b.Write(v)
default:
fmt.Fprintf(&b, "%v", v)
}
// delimiter
b.WriteByte(0)
}
return b.String()
}

func replaceFn(ctx context.Context, key string) (*cacheRows, error) {
queryerCtx, ok := ctx.Value(queryerCtxKey{}).(driver.QueryerContext)
if ok {
query := ctx.Value(queryKey{}).(string)
nvargs := ctx.Value(namedValueArgsKey{}).([]driver.NamedValue)
rows, err := queryerCtx.QueryContext(ctx, query, nvargs)
if err != nil {
return nil, err
}
cacheRows, err := newCacheRows(rows)
if err != nil {
return nil, err
}
return cacheRows.clone(), nil
}

stmt := ctx.Value(stmtKey{}).(*customCacheStatement)
args := ctx.Value(argsKey{}).([]driver.Value)
rows, err := stmt.inner.Query(args)
if err != nil {
return nil, err
}
cacheRows, err := newCacheRows(rows)
if err != nil {
return nil, err
}
return cacheRows.clone(), nil
}
Loading

0 comments on commit 6129918

Please sign in to comment.