From 9b2e9aef2973f7c4ec2c088eabb18d6afdeec652 Mon Sep 17 00:00:00 2001 From: AndreMouche Date: Mon, 16 Dec 2024 10:33:23 -0800 Subject: [PATCH 1/3] init failpointSendReqResult Signed-off-by: AndreMouche --- internal/locate/region_request.go | 98 +++++++++++++++++++------------ 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go index d5450b435f..4b227c58d4 100644 --- a/internal/locate/region_request.go +++ b/internal/locate/region_request.go @@ -730,43 +730,8 @@ func (s *RegionRequestSender) SendReqCtx( bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) } - if val, err := util.EvalFailpoint("tikvStoreSendReqResult"); err == nil { - if s, ok := val.(string); ok { - switch s { - case "timeout": - return nil, nil, 0, errors.New("timeout") - case "GCNotLeader": - if req.Type == tikvrpc.CmdGC { - return &tikvrpc.Response{ - Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, - }, nil, 0, nil - } - case "PessimisticLockNotLeader": - if req.Type == tikvrpc.CmdPessimisticLock { - return &tikvrpc.Response{ - Resp: &kvrpcpb.PessimisticLockResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, - }, nil, 0, nil - } - case "GCServerIsBusy": - if req.Type == tikvrpc.CmdGC { - return &tikvrpc.Response{ - Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, - }, nil, 0, nil - } - case "busy": - return &tikvrpc.Response{ - Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, - }, nil, 0, nil - case "requestTiDBStoreError": - if et == tikvrpc.TiDB { - return nil, nil, 0, errors.WithStack(tikverr.ErrTiKVServerTimeout) - } - case "requestTiFlashError": - if et == tikvrpc.TiFlash { - return nil, nil, 0, errors.WithStack(tikverr.ErrTiFlashServerTimeout) - } - } - } + if resp, err = failpointSendReqResult(req, et); err != nil || resp != nil { + return } if err = s.validateReadTS(bo.GetCtx(), req); err != nil { @@ -1929,3 +1894,62 @@ func (s *baseReplicaSelector) backoffOnNoCandidate(bo *retry.Backoffer) error { } return bo.Backoff(args.cfg, args.err) } + +// failpointSendReqResult is used to process the failpoint For tikvStoreSendReqResult. +func failpointSendReqResult(req *tikvrpc.Request, et tikvrpc.EndpointType) ( + resp *tikvrpc.Response, + err error, +) { + val, e := util.EvalFailpoint("tikvStoreSendReqResult") + if e != nil { + return + } + errMsg, ok := val.(string) + if !ok { + return + } + switch errMsg { + case "timeout": + { + err = errors.New("timeout") + return + } + case "GCNotLeader": + if req.Type == tikvrpc.CmdGC { + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + } + return + } + case "PessimisticLockNotLeader": + if req.Type == tikvrpc.CmdPessimisticLock { + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.PessimisticLockResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + } + return + } + case "GCServerIsBusy": + if req.Type == tikvrpc.CmdGC { + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, + } + return + } + case "busy": + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, + } + return + case "requestTiDBStoreError": + if et == tikvrpc.TiDB { + err = errors.WithStack(tikverr.ErrTiKVServerTimeout) + return + } + case "requestTiFlashError": + if et == tikvrpc.TiFlash { + err = errors.WithStack(tikverr.ErrTiFlashServerTimeout) + return + } + } + return +} From 60f0b147a132cc1177a4cd26fbcd0ae5fefe58f3 Mon Sep 17 00:00:00 2001 From: AndreMouche Date: Thu, 2 Jan 2025 17:29:21 +0800 Subject: [PATCH 2/3] address comments Signed-off-by: AndreMouche --- internal/locate/region_request.go | 80 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go index 4b227c58d4..947c786be9 100644 --- a/internal/locate/region_request.go +++ b/internal/locate/region_request.go @@ -1900,55 +1900,53 @@ func failpointSendReqResult(req *tikvrpc.Request, et tikvrpc.EndpointType) ( resp *tikvrpc.Response, err error, ) { - val, e := util.EvalFailpoint("tikvStoreSendReqResult") - if e != nil { - return - } - errMsg, ok := val.(string) - if !ok { - return - } - switch errMsg { - case "timeout": - { - err = errors.New("timeout") + if val, e := util.EvalFailpoint("tikvStoreSendReqResult"); e == nil { + errMsg, ok := val.(string) + if !ok { return } - case "GCNotLeader": - if req.Type == tikvrpc.CmdGC { - resp = &tikvrpc.Response{ - Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + switch errMsg { + case "timeout": + { + err = errors.New("timeout") + return } - return - } - case "PessimisticLockNotLeader": - if req.Type == tikvrpc.CmdPessimisticLock { - resp = &tikvrpc.Response{ - Resp: &kvrpcpb.PessimisticLockResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + case "GCNotLeader": + if req.Type == tikvrpc.CmdGC { + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + } + return } - return - } - case "GCServerIsBusy": - if req.Type == tikvrpc.CmdGC { + case "PessimisticLockNotLeader": + if req.Type == tikvrpc.CmdPessimisticLock { + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.PessimisticLockResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + } + return + } + case "GCServerIsBusy": + if req.Type == tikvrpc.CmdGC { + resp = &tikvrpc.Response{ + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, + } + return + } + case "busy": resp = &tikvrpc.Response{ Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, } return - } - case "busy": - resp = &tikvrpc.Response{ - Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, - } - return - case "requestTiDBStoreError": - if et == tikvrpc.TiDB { - err = errors.WithStack(tikverr.ErrTiKVServerTimeout) - return - } - case "requestTiFlashError": - if et == tikvrpc.TiFlash { - err = errors.WithStack(tikverr.ErrTiFlashServerTimeout) - return + case "requestTiDBStoreError": + if et == tikvrpc.TiDB { + err = errors.WithStack(tikverr.ErrTiKVServerTimeout) + return + } + case "requestTiFlashError": + if et == tikvrpc.TiFlash { + err = errors.WithStack(tikverr.ErrTiFlashServerTimeout) + return + } } } return From 2904fce4e0c3b55a332f83a7a4938d80b5ec97e4 Mon Sep 17 00:00:00 2001 From: AndreMouche Date: Thu, 2 Jan 2025 17:43:59 +0800 Subject: [PATCH 3/3] address comments Signed-off-by: AndreMouche --- internal/locate/region_request.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go index 947c786be9..d055029e1b 100644 --- a/internal/locate/region_request.go +++ b/internal/locate/region_request.go @@ -1901,11 +1901,11 @@ func failpointSendReqResult(req *tikvrpc.Request, et tikvrpc.EndpointType) ( err error, ) { if val, e := util.EvalFailpoint("tikvStoreSendReqResult"); e == nil { - errMsg, ok := val.(string) + failpointCfg, ok := val.(string) if !ok { return } - switch errMsg { + switch failpointCfg { case "timeout": { err = errors.New("timeout")