Skip to content

Commit

Permalink
sidecar: for some images don't consider error in logs as a reason of …
Browse files Browse the repository at this point in the history
…sync failure
  • Loading branch information
absorbb committed Oct 24, 2024
1 parent 15476ee commit 8faea2f
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions sync-sidecar/read.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func (s *ReadSideCar) Run() {
s.registerErr(fmt.Errorf("%v", r))
s.closeActiveStreams(false)
} else {
s.closeActiveStreams(!cancelled && !s.isErr())
s.closeActiveStreams(!cancelled && !s.isCriticalError())
}
if len(s.processedStreams) > 0 {
statusMap := types2.NewOrderedMap[string, any]()
Expand Down Expand Up @@ -282,7 +282,7 @@ func (s *ReadSideCar) saveState(stream string, data any) {
return
}
} else {
if s.isErr() {
if s.isCriticalError() {
s.errprint("STATE: not saving '%s' state because of previous errors", stream)
return
}
Expand Down Expand Up @@ -638,6 +638,12 @@ func (s *ReadSideCar) loadDestinationConfig() error {
return nil
}

// isCriticalError returns true if first error occurred during sync run
// and we consider it as valid reason to tread all streams that haven't received COMPLETE status as FAILED
func (s *ReadSideCar) isCriticalError() bool {
return s.isErr() && s.packageName != "airbyte/source-netsuite"
}

type StreamStat struct {
EventsCount int `json:"events"`
BytesProcessed int `json:"bytes"`
Expand Down

0 comments on commit 8faea2f

Please sign in to comment.