From dfe87f03a8f0013f022a78b4058e12518e8bc835 Mon Sep 17 00:00:00 2001
From: Brandon Liu <bdon@bdon.org>
Date: Tue, 26 Dec 2023 13:37:55 +0800
Subject: [PATCH] remove md5 from syncfile as it's too slow

---
 main.go             | 10 +++++-----
 pmtiles/makesync.go | 20 +-------------------
 2 files changed, 6 insertions(+), 24 deletions(-)

diff --git a/main.go b/main.go
index 1b35900..d23aab7 100644
--- a/main.go
+++ b/main.go
@@ -65,12 +65,12 @@ var cli struct {
 		Input              string `arg:"" type:"existingfile"`
 		BlockSizeMegabytes int    `default:1 help:"The block size, in megabytes. Must be greater than the max tile size in the archive."`
 		HashFunction       string `default:fnv1a help:"The hash function."`
-	} `cmd:"" help:"Create an **experimental** sync control file (.pmtiles.sync) for a local archive. Do not use this for anything in production."`
+	} `cmd:"" hidden:""`
 
 	Sync struct {
 		Existing string `arg:"" type:"existingfile"`
 		Syncfile string `arg:"" type:"existingfile"`
-	} `cmd:"" help:"This command isn't supported yet."`
+	} `cmd:"" hidden:""`
 
 	Serve struct {
 		Path           string `arg:"" help:"Local path or bucket prefix"`
@@ -83,9 +83,9 @@ var cli struct {
 	} `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."`
 
 	Download struct {
-		OldFile 			 string `type:"existingfile" help:"The old archive on disk. Providing this will check the new archive for a .sync file"`
-		NewFile string `arg:"The remote file."`
-		Bucket         string `required:"" help:"Bucket of file to download."`
+		OldFile         string  `type:"existingfile" help:"The old archive on disk. Providing this will check the new archive for a .sync file"`
+		NewFile         string  `arg:"The remote file."`
+		Bucket          string  `required:"" help:"Bucket of file to download."`
 		DownloadThreads int     `default:4 help:"Number of download threads."`
 		DryRun          bool    `help:"Calculate new parts to download, but don't download them."`
 		Overfetch       float32 `default:0.05 help:"What ratio of extra data to download to minimize # requests; 0.2 is 20%"`
diff --git a/pmtiles/makesync.go b/pmtiles/makesync.go
index d7fe264..80729f9 100644
--- a/pmtiles/makesync.go
+++ b/pmtiles/makesync.go
@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"crypto/md5"
 	"fmt"
 	"github.com/dustin/go-humanize"
 	"github.com/schollz/progressbar/v3"
@@ -33,6 +32,7 @@ type Result struct {
 
 func Makesync(logger *log.Logger, file string, block_size_megabytes int) error {
 	ctx := context.Background()
+	start := time.Now()
 
 	bucketURL, key, err := NormalizeBucketKey("", "", file)
 	max_block_bytes := uint64(1024 * 1024 * block_size_megabytes)
@@ -94,23 +94,6 @@ func Makesync(logger *log.Logger, file string, block_size_megabytes int) error {
 	}
 	defer output.Close()
 
-	// while we're developing this let's store the md5 in the file as well
-	start := time.Now()
-	localfile, err := os.Open(file)
-	if err != nil {
-		panic(err)
-	}
-	defer localfile.Close()
-	reader := bufio.NewReader(localfile)
-	md5hasher := md5.New()
-	if _, err := io.Copy(md5hasher, reader); err != nil {
-		panic(err)
-	}
-	md5checksum := md5hasher.Sum(nil)
-	fmt.Printf("Completed md5 in %v.\n", time.Since(start))
-
-	start = time.Now()
-	output.Write([]byte(fmt.Sprintf("md5=%x\n", md5checksum)))
 	output.Write([]byte("hash=fnv1a\n"))
 	output.Write([]byte(fmt.Sprintf("maxblocksize=%d\n", max_block_bytes)))
 
@@ -289,7 +272,6 @@ func Sync(logger *log.Logger, file string, syncfile string) error {
 		return fmt.Errorf("Error: archive must be clustered for makesync.")
 	}
 
-
 	GetHash := func(offset uint64, length uint64) uint64 {
 		hasher := fnv.New64a()
 		r, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+offset), int64(length))