feat: max buffer size (#282)

CHANGELOG.md

@@ -18,6 +18,7 @@ Types of changes
 
 - `Added` new features `findInCSV` mask to get one or multiple csv lines which matched with Json entry value from CSV files.
 - `Added` new flag `--serve` to start a HTTP server that apply masking to JSON body of requests.
+- `Added` new flag `--buffer-size` to increase the buffer used to load files from URI
 - `Fixed` some bugs with selectors.
 
 ## [1.20.0]

cmd/pimo/main.go

@@ -18,6 +18,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"net/http"
@@ -33,6 +34,7 @@ import (
 	"github.com/cgi-fr/pimo/pkg/flow"
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/cgi-fr/pimo/pkg/statistics"
+	"github.com/cgi-fr/pimo/pkg/uri"
 	"github.com/labstack/echo/v4"
 	"github.com/mattn/go-isatty"
 	"github.com/rs/zerolog"
@@ -71,6 +73,7 @@ var (
 	statsTemplateEnv      = os.Getenv("PIMO_STATS_TEMPLATE")
 	xmlSubscriberName     map[string]string
 	serve                 string
+	maxBufferCapacity     int
 )
 
 func main() {
@@ -113,6 +116,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa
 	rootCmd.PersistentFlags().StringVar(&statisticsDestination, "stats", statsDestinationEnv, "generate execution statistics in the specified dump file")
 	rootCmd.PersistentFlags().StringVar(&statsTemplate, "statsTemplate", statsTemplateEnv, "template string to format stats (to include them you have to specify them as `{{ .Stats }}` like `{\"software\":\"PIMO\",\"stats\":{{ .Stats }}}`)")
 	rootCmd.Flags().StringVar(&serve, "serve", "", "listen/respond to HTTP interface and port instead of stdin/stdout, <ip>:<port> or :<port> to listen to all local networks")
+	rootCmd.Flags().IntVar(&maxBufferCapacity, "buffer-size", bufio.MaxScanTokenSize, "buffer size in kB to load data from uri for each line")
 
 	rootCmd.AddCommand(&cobra.Command{
 		Use: "jsonschema",
@@ -247,6 +251,10 @@ func run(cmd *cobra.Command) {
 		skipLogFile = catchErrors
 	}
 
+	if maxBufferCapacity > 0 {
+		uri.MaxCapacityForEachLine = maxBufferCapacity * 1024
+	}
+
 	config := pimo.Config{
 		EmptyInput:       emptyInput,
 		RepeatUntil:      repeatUntil,

pkg/uri/uri.go

@@ -20,6 +20,7 @@ package uri
 import (
 	"bufio"
 	"encoding/csv"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -30,6 +31,8 @@ import (
 	"github.com/cgi-fr/pimo/pkg/model"
 )
 
+var MaxCapacityForEachLine = bufio.MaxScanTokenSize
+
 func ReadCsv(uri string, sep rune, comment rune, fieldsPerRecord int, trimLeadingSpaces bool) ([][]string, error) {
 	u, err := url.Parse(uri)
 	if err != nil {
@@ -92,6 +95,7 @@ func Read(uri string) ([]model.Entry, error) {
 			return nil, err
 		}
 		scanner := bufio.NewScanner(file)
+		scanner.Buffer(make([]byte, MaxCapacityForEachLine), MaxCapacityForEachLine)
 		for scanner.Scan() {
 			value := scanner.Text()
 			intValue, err := strconv.Atoi(value)
@@ -101,7 +105,10 @@ func Read(uri string) ([]model.Entry, error) {
 				result = append(result, scanner.Text())
 			}
 		}
-		return result, nil
+		if errors.Is(scanner.Err(), bufio.ErrTooLong) {
+			return result, errors.New("error reading " + uri + ": line is too long, use --buffer-size parameter to increase buffer size")
+		}
+		return result, scanner.Err()
 	}
 	if u.Scheme == "http" || u.Scheme == "https" {
 		/* #nosec */
@@ -111,6 +118,7 @@ func Read(uri string) ([]model.Entry, error) {
 		}
 		defer rep.Body.Close()
 		scanner := bufio.NewScanner(rep.Body)
+		scanner.Buffer(make([]byte, MaxCapacityForEachLine), MaxCapacityForEachLine)
 		for scanner.Scan() {
 			value := scanner.Text()
 			intValue, err := strconv.Atoi(value)
@@ -120,7 +128,10 @@ func Read(uri string) ([]model.Entry, error) {
 				result = append(result, scanner.Text())
 			}
 		}
-		return result, nil
+		if errors.Is(scanner.Err(), bufio.ErrTooLong) {
+			return result, errors.New("error reading " + uri + ": line is too long, use --buffer-size parameter to increase buffer size")
+		}
+		return result, scanner.Err()
 	}
 	if u.Scheme == "pimo" {
 		list, ok := maskingdata.MapData[u.Host]