diff --git a/pmtiles/makesync.go b/pmtiles/makesync.go index b0753df..8236cbe 100644 --- a/pmtiles/makesync.go +++ b/pmtiles/makesync.go @@ -31,6 +31,12 @@ type Result struct { Hash uint64 } +type Syncline struct { + Offset uint64 + Length uint64 + Hash uint64 +} + func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb int, checksum string) error { ctx := context.Background() start := time.Now() @@ -122,20 +128,6 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb var current Block - GetHash := func(offset uint64, length uint64) uint64 { - hasher := fnv.New64a() - r, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+offset), int64(length)) - if err != nil { - log.Fatal(err) - } - - if _, err := io.Copy(hasher, r); err != nil { - log.Fatal(err) - } - r.Close() - return hasher.Sum64() - } - tasks := make(chan Block, 10000) intermediate := make(chan Result, 10000) @@ -143,8 +135,20 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb // workers for i := 0; i < runtime.GOMAXPROCS(0); i++ { errs.Go(func() error { + hasher := fnv.New64a() for block := range tasks { - intermediate <- Result{block, GetHash(block.Offset, block.Length)} + r, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+block.Offset), int64(block.Length)) + if err != nil { + log.Fatal(err) + } + + if _, err := io.Copy(hasher, r); err != nil { + log.Fatal(err) + } + r.Close() + + intermediate <- Result{block, hasher.Sum64()} + hasher.Reset() } return nil }) @@ -152,6 +156,8 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb done := make(chan struct{}) + synclines := make([][]uint64, 0) + go func() { buffer := make(map[uint64]Result) nextIndex := uint64(0) @@ -162,7 +168,9 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb for { if next, ok := buffer[nextIndex]; ok { - output.Write([]byte(fmt.Sprintf("%d\t%d\t%d\t%x\n", next.Block.Start, next.Block.Offset, next.Block.Length, next.Hash))) + // output.Write([]byte(fmt.Sprintf("%d\t%d\t%d\t%x\n", next.Block.Start, next.Block.Offset, next.Block.Length, next.Hash))) + + synclines = append(synclines, []uint64{next.Block.Start, next.Block.Offset, next.Block.Length, next.Hash}) delete(buffer, nextIndex) nextIndex++ @@ -215,17 +223,16 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb close(tasks) <-done + + for _, s := range synclines { + output.Write([]byte(fmt.Sprintf("%d\t%d\t%d\t%x\n", s[0], s[1], s[2], s[3]))) + } + fmt.Printf("Created syncfile with %d blocks.\n", blocks) fmt.Printf("Completed makesync in %v.\n", time.Since(start)) return nil } -type Syncline struct { - Offset uint64 - Length uint64 - Hash uint64 -} - func Sync(logger *log.Logger, file string, syncfile string) error { start := time.Now() total_remote_bytes := uint64(0)