fix(crawler): Visit function should write error in errCh once (#36)

pkg/crawler/crawler.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/hashicorp/go-retryablehttp"
@@ -32,6 +33,7 @@ type Crawler struct {
 	rootUrl         string
 	wg              sync.WaitGroup
 	urlCh           chan string
+	errOnce         sync.Once
 	limit           *semaphore.Weighted
 	wrongSHA1Values []string
 }
@@ -46,6 +48,7 @@ func NewCrawler(opt Option) Crawler {
 	client := retryablehttp.NewClient()
 	client.RetryMax = 10
 	client.Logger = nil
+	client.RetryWaitMin = 10 * time.Second
 
 	if opt.RootUrl == "" {
 		opt.RootUrl = mavenRepoURL
@@ -61,6 +64,7 @@ func NewCrawler(opt Option) Crawler {
 		rootUrl: opt.RootUrl,
 		urlCh:   make(chan string, opt.Limit*10),
 		limit:   semaphore.NewWeighted(opt.Limit),
+		errOnce: sync.Once{},
 	}
 }
 
@@ -101,7 +105,13 @@ func (c *Crawler) Crawl(ctx context.Context) error {
 				defer c.limit.Release(1)
 				defer c.wg.Done()
 				if err := c.Visit(ctx, url); err != nil {
-					errCh <- xerrors.Errorf("visit error: %w", err)
+					// There might be a case where we get 2 errors at the same time.
+					// In this case we close `errCh` after reading the first error
+					// and get panic for the second error
+					// That's why we need to return the error once.
+					c.errOnce.Do(func() {
+						errCh <- xerrors.Errorf("visit error: %w", err)
+					})
 				}
 			}(url)
 		}