From c77e62344f0c47af4a0c0238b30384fec2803211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Wed, 17 Jan 2024 15:27:44 +0100 Subject: [PATCH] fix: do not handle instances of eTLD domains as errors --- internal/keyvalue/report.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/keyvalue/report.go b/internal/keyvalue/report.go index 91d23e1..a147647 100644 --- a/internal/keyvalue/report.go +++ b/internal/keyvalue/report.go @@ -148,14 +148,13 @@ func (r ReportGenerator) Generate(ctx context.Context, req index.Request) (*sche default: } resp = result.(CdxResponse) - key = make([]byte, len(resp.Key)) - copy(key, resp.Key) - log.Debug().Str("key", string(key)).Msg("") + key = resp.Key cdx = resp.Value + report.Progress = string(key) + if tock || updateCount%r.UpdateThreshold == 0 { report.Duration = durationpb.New(time.Since(report.StartTime.AsTime())) - report.Progress = string(key) updateCount = 0 err = r.SaveReport(ctx, report) @@ -181,11 +180,13 @@ func (r ReportGenerator) Generate(ctx context.Context, req index.Request) (*sche reportData.NrOfDomains++ // Update target + domain := deSurtDomain(surtDomain) prevTarget = target - target, err = publicsuffix.EffectiveTLDPlusOne(deSurtDomain(surtDomain)) + target, err = publicsuffix.EffectiveTLDPlusOne(domain) if err != nil { - err = fmt.Errorf("failed to get effective tld plus one for '%s': %w", surtDomain, err) - return + log.Warn().Err(err).Str("domain", domain).Msg("failed to get effective tld plus one") + err = nil + target = domain } if prevTarget != target { // Increment number of targets