feat: limit number of results for prefix searches

The maximum number of records that is returned when performing a prefix search
is now configurable (warcserver API).

If an exact match is not found when searching the index, pywb retries queries
using prefix matching without limit to the number of results. This often leads
to timeouts as the number of results can be large if the index is large.

cmd/serve/serve.go

@@ -60,6 +60,9 @@ func NewCommand() *cobra.Command {
 	cmd.Flags().String("path-prefix", "", "path prefix for all server endpoints")
 	cmd.Flags().Bool("log-requests", false, "log incoming http requests")
 
+	// warcserver API options
+	cmd.Flags().Int("warcserver-prefix-max-records", 1000, "limit number of responses for prefix searches (warcserver)")
+
 	// index options
 	cmd.Flags().StringP("index-source", "s", "file", `index source: "file" or "kafka"`)
 	cmd.Flags().StringP("index-format", "o", "badger", `index format: "badger", "tikv"`)
@@ -240,6 +243,9 @@ func serveCmd(_ *cobra.Command, _ []string) error {
 		FileAPI:    fileApi,
 		IdAPI:      idApi,
 		WarcLoader: l,
+		Config: &warcserver.Config{
+			PrefixSearchLimit: viper.GetInt("warcserver-prefix-max-records"),
+		},
 	}, handler, mw, pathPrefix+"/warcserver")
 
 	// register core API

server/api/api.go

@@ -135,6 +135,10 @@ func (c *SearchRequest) Fields() []string {
 	return c.fields
 }
 
+func (c *SearchRequest) SetLimit(limit int) {
+	c.limit = limit
+}
+
 var schemeRegExp = regexp.MustCompile(`^[a-z][a-z0-9+\-.]+(:.*)`)
 
 func Parse(values url.Values) (req *SearchRequest, err error) {

server/warcserver/handler.go

@@ -20,11 +20,16 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+type Config struct {
+	PrefixSearchLimit int
+}
+
 type Handler struct {
 	CdxAPI     index.CdxAPI
 	FileAPI    index.FileAPI
 	IdAPI      index.IdAPI
 	WarcLoader loader.WarcLoader
+	Config     *Config
 }
 
 func (h Handler) index(w http.ResponseWriter, r *http.Request) {
@@ -34,6 +39,12 @@ func (h Handler) index(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// limit the number of results when prefix searching
+	if coreAPI.MatchType() == index.MatchTypePrefix &&
+		coreAPI.Limit() == 0 {
+		coreAPI.SetLimit(h.Config.PrefixSearchLimit)
+	}
+
 	start := time.Now()
 	count := 0
 	defer func() {