From 32d2e261271bb5f1e0f80a5d74100f7496612907 Mon Sep 17 00:00:00 2001 From: Aksel Hjerpbakk Date: Mon, 1 Mar 2021 12:40:00 +0100 Subject: [PATCH 1/5] Simplify bug report template --- .github/ISSUE_TEMPLATE/bug_report.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a4d3e5a..29cb8e8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,16 +23,5 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - **Additional context** Add any other context about the problem here. From f3425e0495d5d64a83a5baff0e0dd58ae3eb455e Mon Sep 17 00:00:00 2001 From: Aksel Hjerpbakk Date: Sun, 7 Mar 2021 15:19:14 +0100 Subject: [PATCH 2/5] #12: tests for cmd functionality Got some help by @maeb to resolve test data issues :) --- cmd/warcserver/cmd/index/index.go | 34 +------ cmd/warcserver/cmd/index/io.go | 57 ++++++++++++ cmd/warcserver/cmd/index/io_test.go | 139 ++++++++++++++++++++++++++++ testdata/example-trunc.warc | Bin 3370 -> 3882 bytes testdata/example.warc | Bin 5120 -> 5632 bytes 5 files changed, 197 insertions(+), 33 deletions(-) create mode 100644 cmd/warcserver/cmd/index/io.go create mode 100644 cmd/warcserver/cmd/index/io_test.go diff --git a/cmd/warcserver/cmd/index/index.go b/cmd/warcserver/cmd/index/index.go index 9064dd6..2a244cc 100644 --- a/cmd/warcserver/cmd/index/index.go +++ b/cmd/warcserver/cmd/index/index.go @@ -18,12 +18,7 @@ package index import ( "errors" "fmt" - "io" - "os" - "strconv" - "github.com/nlnwa/gowarc/warcoptions" - "github.com/nlnwa/gowarc/warcreader" "github.com/nlnwa/gowarcserver/pkg/index" "github.com/spf13/cobra" ) @@ -84,33 +79,6 @@ func runE(c *conf) error { } defer c.writer.Close() - readFile(c) + ReadFile(c) return nil } - -// TODO: return error -func readFile(c *conf) { - opts := &warcoptions.WarcOptions{Strict: false} - wf, err := warcreader.NewWarcFilename(c.fileName, 0, opts) - if err != nil { - return - } - defer wf.Close() - - count := 0 - - for { - wr, currentOffset, err := wf.Next() - if err == io.EOF { - break - } - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error: %v, rec num: %v, Offset %v\n", err.Error(), strconv.Itoa(count), currentOffset) - break - } - count++ - - c.writer.Write(wr, c.fileName, currentOffset) - } - fmt.Fprintln(os.Stderr, "Count: ", count) -} diff --git a/cmd/warcserver/cmd/index/io.go b/cmd/warcserver/cmd/index/io.go new file mode 100644 index 0000000..e5cdc23 --- /dev/null +++ b/cmd/warcserver/cmd/index/io.go @@ -0,0 +1,57 @@ +package index + +import ( + "fmt" + "io" + "strconv" + + "github.com/nlnwa/gowarc/warcoptions" + "github.com/nlnwa/gowarc/warcreader" + "github.com/nlnwa/gowarcserver/pkg/index" + logrus "github.com/sirupsen/logrus" +) + +func ParseFormat(format string) (index.CdxWriter, error) { + switch format { + case "cdx": + return &index.CdxLegacy{}, nil + case "cdxj": + return &index.CdxJ{}, nil + case "cdxpb": + return &index.CdxPb{}, nil + case "db": + return &index.CdxDb{}, nil + } + return nil, fmt.Errorf("unknwon format %v, valid formats are: 'cdx', 'cdxj', 'cdxpb', 'db'", format) +} + +func ReadFile(c *conf) error { + opts := &warcoptions.WarcOptions{Strict: false} + wf, err := warcreader.NewWarcFilename(c.fileName, 0, opts) + if err != nil { + return err + } + defer wf.Close() + + count := 0 + + // avoid defer copy value by using a anonymous function + // At the end, print count even if an error occurs + defer func() { + logrus.Printf("Count: %d", count) + }() + + for { + wr, currentOffset, err := wf.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("Error: %v, rec num: %v, Offset %v\n", err.Error(), strconv.Itoa(count), currentOffset) + } + count++ + + c.writer.Write(wr, c.fileName, currentOffset) + } + return nil +} diff --git a/cmd/warcserver/cmd/index/io_test.go b/cmd/warcserver/cmd/index/io_test.go new file mode 100644 index 0000000..280f9a0 --- /dev/null +++ b/cmd/warcserver/cmd/index/io_test.go @@ -0,0 +1,139 @@ +package index + +import ( + "fmt" + "os" + "path" + "reflect" + "testing" + + "github.com/nlnwa/gowarcserver/pkg/index" + log "github.com/sirupsen/logrus" +) + +func TestParseFormat(t *testing.T) { + tests := []struct { + name string + format string + expected reflect.Type + errorState bool + }{ + { + "'cdx' results in CdxLegacy writer", + "cdx", + reflect.TypeOf((*index.CdxLegacy)(nil)), + false, + }, + { + "'cdxj' results in CdxJ writer", + "cdxj", + reflect.TypeOf((*index.CdxJ)(nil)), + false, + }, + { + "'db' results in CdxDb writer", + "db", + reflect.TypeOf((*index.CdxDb)(nil)), + false, + }, + { + "'cdxpb' results in CdxPd writer", + "cdxpb", + reflect.TypeOf((*index.CdxPb)(nil)), + false, + }, + { + "'cd' results in error", + "cd", + nil, + true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseFormat(tt.format) + if err != nil && !tt.errorState { + t.Errorf("Unexpected failure: %v", err) + } else if err == nil && tt.errorState { + t.Errorf("Expected error parsing '%v', got type %T", tt.format, got) + } + + if reflect.TypeOf(got) != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, got) + } + }) + } +} + +// TODO: this was hard to write tests for and therefore ReadFile +// should probably be refactored +func TestReadFile(t *testing.T) { + log.SetLevel(log.WarnLevel) + // same as testdata/example.warc except removed gzip content because of illegal go str characters + testFileContent := []byte(`WARC/1.0 +WARC-Date: 2017-03-06T04:03:53Z +WARC-Record-ID: +WARC-Type: warcinfo +Content-Length: 0`) + + filepath := path.Join(t.TempDir(), "test.warc") + file, err := os.Create(filepath) + if err != nil { + t.Fatalf("Failed to create testfile at '%s'", filepath) + } + // This is not strictly needed because of tmp, but to be platform agnostic it might be a good idea + defer file.Close() + + _, err = file.Write(testFileContent) + if err != nil { + t.Fatalf("Failed to write to testfile at '%s'", filepath) + } + + err = file.Sync() + if err != nil { + t.Fatalf("Failed to sync testfile at '%s'", filepath) + } + + tests := []struct { + writerFormat string + writer index.CdxWriter + }{ + { + "cdx", + &index.CdxLegacy{}, + }, + { + "cdxj", + &index.CdxJ{}, + }, + { + + "cdxpd", + &index.CdxPb{}, + }, + { + "db", + &index.CdxDb{}, + }, + } + + for _, tt := range tests { + testName := fmt.Sprintf("Readfile: %T successfully indexes", tt.writer) + t.Run(testName, func(t *testing.T) { + c := &conf{ + filepath, + tt.writerFormat, + tt.writer, + } + c.writer.Init() + defer c.writer.Close() + + err := ReadFile(c) + if err != nil { + t.Errorf("Unexpected failure: %v", err) + } + + }) + } +} diff --git a/testdata/example-trunc.warc b/testdata/example-trunc.warc index 6a1e7351529d61b3c626edc6a63e6ca0667d1845..43ca9c2175ef0cff54c410b5e290523d8336e645 100644 GIT binary patch delta 1135 zcmYjQOHUI~6sDB6w4F**V3cgoG>_Kg<~qgvUjuQ>^| zC2Fj&(Q^>su*5zF%Uk9&&Kx6fwIc$pZRJwWP4;oN0PHA-(8aIxEJh^6-bU}tfl<&P z0hYjcDVgC6+JD0h-N!lo^9+Yj%9IOEXjINN!u()5lJ>aZpI*`i7IxAShd{<1ww8HO zX-Cvhtr+wJ3s-|4*C|9RpC5gH66X?>B6~N4cBOzo;6CJ1uIuf_)B!|t(U*kfhp(0H zk%J6fI*5Sw#t6eCqXeBYYsTLe@+06-CMSJd?KQjGO@AjcUHY|N_9sMLw}m|}nhnWtYynBHp=Y_ao9(>##g&U9L~xfAMJaw;t3_P+ zw<+$ZY;^oU3Z!}U(~74-^SN=~;s7#+5u^Fj$muKx!7tDHxK+nCu)n8IEzn$&Eszne zdR(2(0*76&84iWg9ohjZdubN(p`Xv6irxc`lsP!dj`V$vsLJ1JJ;Ux>j!+ykVJZ zmZeY?=bvfj6nY$-6&d|UfKEdenU9tkGe1>Y&^T!2wu3E5tKI%2=u8GdnsTEbLeIGc z9;;TS%{Tmb{Lmk|jvC+vX)9Ut`E;27qY23_QE<>?fJ9V>PRNtD4NJY~B6#)N8M6KX D9V*_x delta 619 zcmV-x0+ju#9;zC!r3ZhD2mm{Q1yTS5jZ{Ie(=ZIa?^l>|0clBn_T9cF{?KYwJ-*MZ$uI24OE47XtfBtO8NuEiQ`*V#@dR1kMW((ja@#HJa zKA(iDD}+x*3DQY4Xcxm4gX6F`(rjHUN=h`)auYSZAO(H^xAyRZS| z!zv9to_y4W-;PkXkhD;k1})*SHr7x4k0gDWN^B{|uwN{JG;gA z3@(9l=YqkuGx7|7da+-M{IIvC(K5_UWo!)3i#5Nmw&NBh?4EQC>#M33Qj*@!bOkdF z3!$hw#}It4QNw=^pBsPPy{=DBdETP}(Rg!%?)}W+R_?_l{?1s5Hnc)g<6^i&wVhBBaYg#DsarxWLFZyH;MWqnsxFbiH3PxBwcDeo=oy`kjuL#qsgklR%bs; zav*y>u!PQwT}7_sUU&mLgvPO07zqGQgcd7f_t6l~MVDwn_SIPA@QkPW7^@Y7iL3`qqP|i|bg5f3TyK8?2$iaf_Ot&B FD+By7H8KDI diff --git a/testdata/example.warc b/testdata/example.warc index 4bc3089a5de83170fa6711bb7ebfacd5545d6dd1..41edf11705c2c7212a3c92064cc29dfa37c93fba 100644 GIT binary patch delta 1135 zcmYjQOHUI~6lN%G>9m!m#(@YJqD7$v6`MQ;L@AG}t& zM-DP{*FgldH%1WVGD^@1vu6BlAwL2RWOA;LE8S*yyQ#Dy(_Ozd)|I%Z>(-!KrU?Nj zK+h6n5X7FgG?3v;cGBCd)U{m=`b8sei_RhGHuNml_OhK9zqoQygb40(qN>Gj&#Dm@ z{%wjoNj5ruAO+IAdTGUzp!wXGcfJo9!-&%SY2L~lW_aBq z*QA9&1?QjH%_;OaI4v^zj{u!UNMv4GX4L#tX+fi)h1+&EKdpBAlb|!{2WiUn3WT0> z3p`dSPnvJ|@%W(W=4WPo@?hmM8DZyQpr=puOa+abw+ E03@Q`WB>pF delta 619 zcmV-x0+juLEPyDmr3ZhD2mm{Q1yTS5jZ{Ie(=ZIa?^l>|0clBn_T9cF{?KYwJ-*MZ$uI24OE47XtfBtO8NuEiQ`*V#@dR1kMW((ja@#HJa zKA(iDD}+x*3DQY4Xcxm4gX6F`(rjHUN=h`)auYSZAO(H^xAyRZS| z!zv9to_y4W-;PkXkhD;k1})*SHr7x4k0gDWN^B{|uwN{JG;gA z3@(9l=YqkuGx7|7da+-M{IIvC(K5_UWo!)3i#5Nmw&NBh?4EQC>#M33Qj*@!bOkdF z3!$hw#}It4QNw=^pBsPPy{=DBdETP}(Rg!%?)}W+R_?_l{?1s5Hnc)g<6^i&wVhBBaYg#DsarxWLFZyH;MWqnsxFbiH3PxBwcDeo=oy`kjuL#qsgklR%bs; zav*y>u!PQwT}7_sUU&mLgvPO07zqGQgcd7f_t6l~MVDwn_SIPA@QkPW7^@Y7iL3`qqP|i|bg5f3TyK8?2$iaf_Ot&B F0Sa`OG}izC From 3e5d7c24b1ed32cdf6ac6f04b6578dda3f8aa9f4 Mon Sep 17 00:00:00 2001 From: Aksel Hjerpbakk Date: Mon, 22 Mar 2021 12:11:32 +0100 Subject: [PATCH 3/5] Removed sideffects from ReadFile tests --- cmd/warcserver/cmd/index/index.go | 5 +++-- cmd/warcserver/cmd/index/io_test.go | 2 +- pkg/index/indexwriter.go | 12 +++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/warcserver/cmd/index/index.go b/cmd/warcserver/cmd/index/index.go index 2a244cc..7063ec7 100644 --- a/cmd/warcserver/cmd/index/index.go +++ b/cmd/warcserver/cmd/index/index.go @@ -21,6 +21,7 @@ import ( "github.com/nlnwa/gowarcserver/pkg/index" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func parseFormat(format string) (index.CdxWriter, error) { @@ -72,8 +73,8 @@ func NewCommand() *cobra.Command { func runE(c *conf) error { fmt.Printf("Format: %v\n", c.writerFormat) - - err := c.writer.Init() + dbDir := viper.GetString("indexdir") + err := c.writer.Init(dbDir) if err != nil { return err } diff --git a/cmd/warcserver/cmd/index/io_test.go b/cmd/warcserver/cmd/index/io_test.go index 280f9a0..bacf618 100644 --- a/cmd/warcserver/cmd/index/io_test.go +++ b/cmd/warcserver/cmd/index/io_test.go @@ -126,7 +126,7 @@ Content-Length: 0`) tt.writerFormat, tt.writer, } - c.writer.Init() + c.writer.Init(t.TempDir()) defer c.writer.Close() err := ReadFile(c) diff --git a/pkg/index/indexwriter.go b/pkg/index/indexwriter.go index 6021016..8c2158f 100644 --- a/pkg/index/indexwriter.go +++ b/pkg/index/indexwriter.go @@ -22,11 +22,10 @@ import ( "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "github.com/nlnwa/gowarc/warcrecord" - "github.com/spf13/viper" ) type CdxWriter interface { - Init() error + Init(dbDir string) error Close() Write(wr warcrecord.WarcRecord, fileName string, offset int64) error } @@ -43,8 +42,7 @@ type CdxDb struct { db *Db } -func (c *CdxDb) Init() (err error) { - dbDir := viper.GetString("indexdir") +func (c *CdxDb) Init(dbDir string) (err error) { c.db, err = NewIndexDb(dbDir) if err != nil { return err @@ -61,7 +59,7 @@ func (c *CdxDb) Write(wr warcrecord.WarcRecord, fileName string, offset int64) e return c.db.Add(wr, fileName, offset) } -func (c *CdxLegacy) Init() (err error) { +func (c *CdxLegacy) Init(dbDir string) (err error) { return nil } @@ -72,7 +70,7 @@ func (c *CdxLegacy) Write(wr warcrecord.WarcRecord, fileName string, offset int6 return nil } -func (c *CdxJ) Init() (err error) { +func (c *CdxJ) Init(dbDir string) (err error) { c.jsonMarshaler = &jsonpb.Marshaler{} return nil } @@ -92,7 +90,7 @@ func (c *CdxJ) Write(wr warcrecord.WarcRecord, fileName string, offset int64) er return nil } -func (c *CdxPb) Init() (err error) { +func (c *CdxPb) Init(dbDir string) (err error) { return nil } From af8024c55ae451931469cb19fcb84aa5f5786a4c Mon Sep 17 00:00:00 2001 From: Aksel Hjerpbakk Date: Wed, 24 Mar 2021 19:36:43 +0100 Subject: [PATCH 4/5] upgrade go to 1.15 --- Dockerfile | 2 +- README.md | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 83208eb..4a9d5fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13 as build +FROM golang:1.15 as build WORKDIR /build diff --git a/README.md b/README.md index 13c1d45..a6516e5 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The gowarc server module. This tool can be used to index and serve warc files # Requirements -go 1.13 or newer +go 1.15 or newer # Build diff --git a/go.mod b/go.mod index e364661..f6517c1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/nlnwa/gowarcserver -go 1.13 +go 1.15 require ( github.com/dgraph-io/badger/v2 v2.2007.2 From 512c8f3b34c9311db82b9e0fc2779ed388cb392c Mon Sep 17 00:00:00 2001 From: Aksel Hjerpbakk Date: Tue, 30 Mar 2021 13:19:26 +0200 Subject: [PATCH 5/5] .#14: properly bind viper and cobra * Update readme documentation * BREAKING: use one naming convention across cobra and viper (camelCase) * Move cmd specific config variables from root to their specific cmd --- README.md | 10 +++---- cmd/warcserver/cmd/root.go | 41 ++++++++++++------------- cmd/warcserver/cmd/serve/serve.go | 50 +++++++++++++++++-------------- exampleconfig.yaml | 1 + pkg/server/server.go | 5 ++-- 5 files changed, 55 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 13c1d45..c692db2 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ You can configure certain aspect of gowarcserver with a config file. Here are al | Name | Type | Description | Default | | ------------- | ------------- | ----------- | ------- | -| warcdir | List of paths | The path to directories where warcs that should be auto indexed | ["."] | -| indexdir | path | The root directory for index files | "." | -| autoindex | bool | Whether gowarc should index from the warcdir(s) when serving automatically or not | true | -| warcport | int | The port that the serve command will use if not overridden as argument to serve | 9999 | -| loglevel | string | Change the application log level manually | "info" | +| warcDir | List of paths | The path to directories where warcs that should be auto indexed | ["."] | +| indexDir | path | The root directory for index files | "." | +| autoIndex | bool | Whether gowarc should index from the warcdir(s) when serving automatically or not | true | +| warcPort | int | The port that the serve command will use if not overridden as argument to serve | 9999 | +| logLevel | string | Change the application log level manually | "info" | diff --git a/cmd/warcserver/cmd/root.go b/cmd/warcserver/cmd/root.go index a6e8d87..05509af 100644 --- a/cmd/warcserver/cmd/root.go +++ b/cmd/warcserver/cmd/root.go @@ -27,14 +27,10 @@ import ( "github.com/spf13/viper" ) -type conf struct { - cfgFile string - logLevel string -} - // NewCommand returns a new cobra.Command implementing the root command for warc func NewCommand() *cobra.Command { - c := &conf{} + cobra.OnInitialize(func() { initConfig() }) + cmd := &cobra.Command{ Use: "warcserver", Short: "Server capable of indexing and serving warc files", @@ -45,26 +41,30 @@ func NewCommand() *cobra.Command { // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use runtime.GOMAXPROCS(128) - if c.logLevel == "" { - c.logLevel = viper.GetString("loglevel") - } - - level, err := log.ParseLevel(c.logLevel) + logLevel := viper.GetString("logLevel") + level, err := log.ParseLevel(logLevel) if err != nil { - return fmt.Errorf("'%s' is not part of the valid levels: 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace'", c.logLevel) + return fmt.Errorf("'%s' is not part of the valid levels: 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug', 'trace'", logLevel) } - log.SetLevel(level) + return nil }, } - cobra.OnInitialize(func() { c.initConfig() }) + // Stub to store cobra variables + c := &struct { + cfgFile string + logLevel string + }{} // Flags - cmd.PersistentFlags().StringVarP(&c.logLevel, "log-level", "l", "", "set the log level of gowarc, it will take precedence over config 'loglevel'") + cmd.PersistentFlags().StringVarP(&c.logLevel, "logLevel", "l", "info", "set the log level of gowarc, it will take precedence over config 'loglevel'") cmd.PersistentFlags().StringVar(&c.cfgFile, "config", "", "config file. If not set, /etc/warc/, $HOME/.warc/ and current working dir will be searched for file config.yaml") - viper.BindPFlag("config", cmd.PersistentFlags().Lookup("config")) + + if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil { + log.Fatalf("Failed to bind root flags, err: %v", err) + } // Subcommands cmd.AddCommand(serve.NewCommand()) @@ -74,13 +74,10 @@ func NewCommand() *cobra.Command { } // initConfig reads in config file and ENV variables if set. -func (c *conf) initConfig() { +func initConfig() { viper.SetTypeByDefaultValue(true) - viper.SetDefault("warcdir", []string{"."}) - viper.SetDefault("indexdir", ".") - viper.SetDefault("autoindex", true) - viper.SetDefault("warcport", 9999) - viper.SetDefault("loglevel", "info") + viper.SetDefault("warcDir", []string{"."}) + viper.SetDefault("indexDir", ".") viper.AutomaticEnv() // read in environment variables that match diff --git a/cmd/warcserver/cmd/serve/serve.go b/cmd/warcserver/cmd/serve/serve.go index f4b7568..ea2db1a 100644 --- a/cmd/warcserver/cmd/serve/serve.go +++ b/cmd/warcserver/cmd/serve/serve.go @@ -23,53 +23,59 @@ import ( "github.com/spf13/viper" ) -type conf struct { - port int - warcDirs []string - watchDepth int -} - func NewCommand() *cobra.Command { - c := &conf{} var cmd = &cobra.Command{ Use: "serve", Short: "Start the warc server to serve warc records", Long: ``, RunE: func(cmd *cobra.Command, args []string) error { + var warcDirs []string if len(args) > 0 { - c.warcDirs = args + warcDirs = args } else { - c.warcDirs = viper.GetStringSlice("warcdir") + warcDirs = viper.GetStringSlice("warcDir") } - return runE(c) + return runE(warcDirs) }, } - cmd.Flags().IntVarP(&c.port, "port", "p", -1, "the port that should be used to serve, will use config value otherwise") - cmd.Flags().IntVarP(&c.watchDepth, "watch-depth", "w", 4, "The maximum depth when indexing warc") + // Stub to hold flags + c := &struct { + warcPort int + watchDepth int + autoIndex bool + }{} + cmd.Flags().IntVarP(&c.warcPort, "warcPort", "p", 9999, "Port that should be used to serve, will use config value otherwise") + cmd.Flags().IntVarP(&c.watchDepth, "watchDepth", "w", 4, "Maximum depth when indexing warc") + cmd.Flags().BoolVarP(&c.autoIndex, "autoIndex", "a", true, "Whether the server should index warc files automatically") + if err := viper.BindPFlags(cmd.Flags()); err != nil { + log.Fatalf("Failed to bind serve flags, err: %v", err) + } return cmd } -func runE(c *conf) error { - if c.port < 0 { - c.port = viper.GetInt("warcport") - } +func runE(warcDirs []string) error { - dbDir := viper.GetString("indexdir") + dbDir := viper.GetString("indexDir") db, err := index.NewIndexDb(dbDir) if err != nil { log.Fatal(err) } defer db.Close() - if viper.GetBool("autoindex") { - log.Infof("Starting autoindexer") - autoindexer := index.NewAutoIndexer(db, c.warcDirs, c.watchDepth) + if viper.GetBool("autoIndex") { + log.Infof("Starting auto indexer") + watchDepth := viper.GetInt("watchDepth") + autoindexer := index.NewAutoIndexer(db, warcDirs, watchDepth) defer autoindexer.Shutdown() } - log.Infof("Starting web server at http://localhost:%v", c.port) - server.Serve(db, c.port) + port := viper.GetInt("warcPort") + log.Infof("Starting web server at http://localhost:%v", port) + err = server.Serve(db, port) + if err != nil { + log.Warnf("%v", err) + } return nil } diff --git a/exampleconfig.yaml b/exampleconfig.yaml index e69de29..932ba60 100644 --- a/exampleconfig.yaml +++ b/exampleconfig.yaml @@ -0,0 +1 @@ +logLevel: warn \ No newline at end of file diff --git a/pkg/server/server.go b/pkg/server/server.go index 36a742b..75aa3ec 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -31,10 +31,9 @@ import ( "github.com/nlnwa/gowarcserver/pkg/index" "github.com/nlnwa/gowarcserver/pkg/loader" "github.com/nlnwa/gowarcserver/pkg/server/warcserver" - log "github.com/sirupsen/logrus" ) -func Serve(db *index.Db, port int) { +func Serve(db *index.Db, port int) error { l := &loader.Loader{ Resolver: &storageRefResolver{db: db}, Loader: &loader.FileStorageLoader{FilePathResolver: func(fileName string) (filePath string, err error) { @@ -71,7 +70,7 @@ func Serve(db *index.Db, port int) { httpServer.Shutdown(ctx) }() - log.Info(httpServer.ListenAndServe()) + return httpServer.ListenAndServe() } type storageRefResolver struct {