Skip to content

Commit

Permalink
feat: change stalled disk propagation from an error to a field in ful…
Browse files Browse the repository at this point in the history
…l status output

Signed-off-by: Manan Gupta <[email protected]>
  • Loading branch information
GuptaManan100 committed Jan 24, 2025
1 parent a6c2896 commit 00d359b
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 17 deletions.
30 changes: 20 additions & 10 deletions go/vt/proto/replicationdata/replicationdata.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 36 additions & 0 deletions go/vt/proto/replicationdata/replicationdata_vtproto.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions go/vt/vtorc/inst/instance_dao.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,10 @@ func ReadTopologyInstanceBufferable(tabletAlias string, latency *stopwatch.Named

fs, err = fullStatus(tabletAlias)
if err != nil {
if config.GetStalledDiskPrimaryRecovery() && strings.Contains(err.Error(), "stalled disk") {
stalledDisk = true
}
goto Cleanup
}
if config.GetStalledDiskPrimaryRecovery() && fs.DiskStalled {
stalledDisk = true
goto Cleanup
}
partialSuccess = true // We at least managed to read something from the server.
Expand Down
10 changes: 6 additions & 4 deletions go/vt/vttablet/tabletmanager/rpc_replication.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package tabletmanager

import (
"context"
"errors"
"fmt"
"runtime"
"strings"
Expand Down Expand Up @@ -62,10 +61,13 @@ func (tm *TabletManager) FullStatus(ctx context.Context) (*replicationdatapb.Ful
return nil, err
}

// Return error if the disk is stalled or rejecting writes.
// Noop by default, must be enabled with the flag "disk-write-dir".
// Return if the disk is stalled or rejecting writes.
// If the disk is stalled, we can't be sure if reads will go through
// or not, so we should not run any reads either.
if tm.dhMonitor.IsDiskStalled() {
return nil, errors.New("stalled disk")
return &replicationdatapb.FullStatus{
DiskStalled: true,
}, nil
}

// Server ID - "select @@global.server_id"
Expand Down
1 change: 1 addition & 0 deletions proto/replicationdata.proto
Original file line number Diff line number Diff line change
Expand Up @@ -105,4 +105,5 @@ message FullStatus {
uint32 semi_sync_wait_for_replica_count = 20;
bool super_read_only = 21;
replicationdata.Configuration replication_configuration = 22;
bool disk_stalled = 23;
}
6 changes: 6 additions & 0 deletions web/vtadmin/src/proto/vtadmin.d.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 23 additions & 0 deletions web/vtadmin/src/proto/vtadmin.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 00d359b

Please sign in to comment.