fix: open the WAL file as O_SYNC and get rid of file.Sync() calls con…

…suming 1-5ms
internetarchive · Jul 27, 2024 · 98ec9fa · 98ec9fa
1 parent 67c3384
commit 98ec9fa
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/internal/pkg/queue/index/manager.go b/internal/pkg/queue/index/manager.go
@@ -45,7 +45,7 @@ type IndexManager struct {
 
 // NewIndexManager creates a new IndexManager instance and loads the index from the index file.
 func NewIndexManager(walPath, indexPath, queueDirPath string) (*IndexManager, error) {
-	walFile, err := os.OpenFile(walPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644)
+	walFile, err := os.OpenFile(walPath, os.O_APPEND|os.O_SYNC|os.O_CREATE|os.O_RDWR, 0644)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open WAL file: %w", err)
 	}

diff --git a/internal/pkg/queue/index/wal.go b/internal/pkg/queue/index/wal.go
@@ -17,12 +17,10 @@ func (im *IndexManager) unsafeWriteToWAL(Op Operation, host, id string, position
 		Position: position,
 		Size:     size,
 	}
+
 	if err := im.walEncoder.Encode(entry); err != nil {
 		return fmt.Errorf("failed to write to WAL: %w", err)
 	}
-	if err := im.walFile.Sync(); err != nil {
-		return fmt.Errorf("failed to sync WAL: %w", err)
-	}
 
 	return nil
 }
@@ -89,7 +87,7 @@ func (im *IndexManager) unsafeTruncateWAL() error {
 	}
 
 	// Reopen the file with O_TRUNC flag to truncate it
-	walFile, err := os.OpenFile(walPath, os.O_TRUNC|os.O_APPEND|os.O_RDWR, 0644)
+	walFile, err := os.OpenFile(walPath, os.O_TRUNC|os.O_SYNC|os.O_APPEND|os.O_RDWR, 0644)
 	if err != nil {
 		return fmt.Errorf("failed to truncate WAL file: %w", err)
 	}