From 5f31cd1dc557ed94a6f5b1b3914940550fbca39c Mon Sep 17 00:00:00 2001 From: Daniel Milde Date: Fri, 16 Jul 2021 23:51:51 +0200 Subject: [PATCH] read analysis from file --- cmd/gdu/app/app.go | 19 +++++++- cmd/gdu/app/app_test.go | 37 ++++++++++++++++ cmd/gdu/main.go | 1 + internal/testdata/test.json | 7 +++ internal/testdata/wrong.json | 1 + report/export.go | 5 +++ report/export_test.go | 10 +++++ report/import.go | 82 ++++++++++++++++++++++++++++++++++ report/import_test.go | 81 ++++++++++++++++++++++++++++++++++ stdout/stdout.go | 85 +++++++++++++++++++++++++++++++++++- stdout/stdout_test.go | 39 +++++++++++++++++ tui/actions.go | 53 ++++++++++++++++++++++ tui/actions_test.go | 41 +++++++++++++++++ 13 files changed, 458 insertions(+), 3 deletions(-) create mode 100644 internal/testdata/test.json create mode 100644 internal/testdata/wrong.json create mode 100644 report/import.go create mode 100644 report/import_test.go diff --git a/cmd/gdu/app/app.go b/cmd/gdu/app/app.go index 0f739c590..4517ee3c4 100644 --- a/cmd/gdu/app/app.go +++ b/cmd/gdu/app/app.go @@ -9,10 +9,10 @@ import ( log "github.com/sirupsen/logrus" "github.com/dundee/gdu/v5/build" - "github.com/dundee/gdu/v5/report" "github.com/dundee/gdu/v5/internal/common" "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" + "github.com/dundee/gdu/v5/report" "github.com/dundee/gdu/v5/stdout" "github.com/dundee/gdu/v5/tui" "github.com/gdamore/tcell/v2" @@ -23,6 +23,7 @@ import ( type UI interface { ListDevices(getter device.DevicesInfoGetter) error AnalyzePath(path string, parentDir *analyze.Dir) error + ReadAnalysis(input io.Reader) error SetIgnoreDirPaths(paths []string) SetIgnoreDirPatterns(paths []string) error SetIgnoreHidden(value bool) @@ -32,6 +33,7 @@ type UI interface { // Flags define flags accepted by Run type Flags struct { LogFile string + InputFile string OutputFile string IgnoreDirs []string IgnoreDirPatterns []string @@ -178,6 +180,21 @@ func (a *App) runAction(ui UI, path string) error { if err := ui.ListDevices(a.Getter); err != nil { return fmt.Errorf("loading mount points: %w", err) } + } else if a.Flags.InputFile != "" { + var input io.Reader + var err error + if a.Flags.InputFile == "-" { + input = os.Stdin + } else { + input, err = os.OpenFile(a.Flags.InputFile, os.O_RDONLY, 0644) + if err != nil { + return fmt.Errorf("opening input file: %w", err) + } + } + + if err := ui.ReadAnalysis(input); err != nil { + return fmt.Errorf("reading analysis: %w", err) + } } else { if err := ui.AnalyzePath(path, nil); err != nil { return fmt.Errorf("scanning dir: %w", err) diff --git a/cmd/gdu/app/app_test.go b/cmd/gdu/app/app_test.go index 4b4681ac8..7c344d830 100644 --- a/cmd/gdu/app/app_test.go +++ b/cmd/gdu/app/app_test.go @@ -118,6 +118,43 @@ func TestAnalyzePathWithExport(t *testing.T) { assert.Nil(t, err) } +func TestReadAnalysisFromFile(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", InputFile: "../../../internal/testdata/test.json"}, + []string{"test_dir"}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.NotEmpty(t, out) + assert.Contains(t, out, "main.go") + assert.Nil(t, err) +} + +func TestReadWrongAnalysisFromFile(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", InputFile: "../../../internal/testdata/wrong.json"}, + []string{"test_dir"}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.Empty(t, out) + assert.Contains(t, err.Error(), "Array of maps not found") +} + +func TestReadWrongAnalysisFromNotExistingFile(t *testing.T) { + out, err := runApp( + &Flags{LogFile: "/dev/null", InputFile: "xxx.json"}, + []string{"test_dir"}, + false, + testdev.DevicesInfoGetterMock{}, + ) + + assert.Empty(t, out) + assert.Contains(t, err.Error(), "no such file or directory") +} + func TestAnalyzePathWithErr(t *testing.T) { fin := testdir.CreateTestDir() defer fin() diff --git a/cmd/gdu/main.go b/cmd/gdu/main.go index a7584d304..ddd01b20c 100644 --- a/cmd/gdu/main.go +++ b/cmd/gdu/main.go @@ -33,6 +33,7 @@ func init() { flags := rootCmd.Flags() flags.StringVarP(&af.LogFile, "log-file", "l", "/dev/null", "Path to a logfile") flags.StringVarP(&af.OutputFile, "output-file", "o", "", "Export analysis into file as JSON") + flags.StringVarP(&af.InputFile, "input-file", "f", "", "Import analysis from JSON file") flags.IntVarP(&af.MaxCores, "max-cores", "m", runtime.NumCPU(), fmt.Sprintf("Set max cores that GDU will use. %d cores available", runtime.NumCPU())) flags.BoolVarP(&af.ShowVersion, "version", "v", false, "Print version") diff --git a/internal/testdata/test.json b/internal/testdata/test.json new file mode 100644 index 000000000..192e1d2ba --- /dev/null +++ b/internal/testdata/test.json @@ -0,0 +1,7 @@ +[1,2,{"progname":"gdu","progver":"development","timestamp":1626807263}, +[{"name":"/home/gdu"}, +[{"name":"app"}, +{"name":"app.go","asize":4638,"dsize":8192}, +{"name":"app_linux_test.go","asize":1410,"dsize":4096}, +{"name":"app_test.go","asize":4974,"dsize":8192}], +{"name":"main.go","asize":3205,"dsize":4096}]] diff --git a/internal/testdata/wrong.json b/internal/testdata/wrong.json new file mode 100644 index 000000000..8adb9bb60 --- /dev/null +++ b/internal/testdata/wrong.json @@ -0,0 +1 @@ +[1,2,3,4] diff --git a/report/export.go b/report/export.go index 6e592059b..48b4b5a25 100644 --- a/report/export.go +++ b/report/export.go @@ -56,6 +56,11 @@ func (ui *UI) ListDevices(getter device.DevicesInfoGetter) error { return errors.New("Exporting devices list is not supported") } +// ReadAnalysis reads analysis report from JSON file +func (ui *UI) ReadAnalysis(input io.Reader) error { + return errors.New("Reading analysis is not possible while exporting") +} + // AnalyzePath analyzes recursively disk usage in given path func (ui *UI) AnalyzePath(path string, _ *analyze.Dir) error { var ( diff --git a/report/export_test.go b/report/export_test.go index 021af608d..7548c161e 100644 --- a/report/export_test.go +++ b/report/export_test.go @@ -60,6 +60,16 @@ func TestShowDevices(t *testing.T) { assert.Contains(t, err.Error(), "not supported") } +func TestReadAnalysisWhileExporting(t *testing.T) { + output := bytes.NewBuffer(make([]byte, 10)) + reportOutput := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateExportUI(output, reportOutput, true) + err := ui.ReadAnalysis(output) + + assert.Contains(t, err.Error(), "not possible while exporting") +} + func TestExportToFile(t *testing.T) { fin := testdir.CreateTestDir() defer fin() diff --git a/report/import.go b/report/import.go new file mode 100644 index 000000000..bab245f81 --- /dev/null +++ b/report/import.go @@ -0,0 +1,82 @@ +package report + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" + + "github.com/dundee/gdu/v5/pkg/analyze" +) + +// ReadAnalysis reads analysis report from JSON file and returns directory item +func ReadAnalysis(input io.Reader) (*analyze.Dir, error) { + var data interface{} + + var buff bytes.Buffer + buff.ReadFrom(input) + json.Unmarshal(buff.Bytes(), &data) + + dataArray, ok := data.([]interface{}) + if !ok { + return nil, errors.New("JSON file does not contain top level array") + } + if len(dataArray) < 4 { + return nil, errors.New("Top level array must have at least 4 items") + } + + items, ok := dataArray[3].([]interface{}) + if !ok { + return nil, errors.New("Array of maps not found in the top level array on 4th position") + } + + return processDir(items) +} + +func processDir(items []interface{}) (*analyze.Dir, error) { + dir := &analyze.Dir{ + File: &analyze.File{ + Flag: ' ', + }, + } + dirMap, ok := items[0].(map[string]interface{}) + if !ok { + return nil, errors.New("Directory item is not a map") + } + name, ok := dirMap["name"].(string) + if !ok { + return nil, errors.New("Directory name is not a string") + } + + slashPos := strings.LastIndex(name, "/") + if slashPos > -1 { + dir.Name = name[slashPos+1:] + dir.BasePath = name[:slashPos+1] + } else { + dir.Name = name + } + + for _, v := range items[1:] { + switch item := v.(type) { + case map[string]interface{}: + file := &analyze.File{} + file.Name = item["name"].(string) + file.Size = int64(item["asize"].(float64)) + file.Usage = int64(item["dsize"].(float64)) + file.Parent = dir + file.Flag = ' ' + + dir.Files.Append(file) + case []interface{}: + subdir, err := processDir(item) + if err != nil { + return nil, err + } + subdir.Parent = dir + dir.Files.Append(subdir) + } + } + + return dir, nil +} diff --git a/report/import_test.go b/report/import_test.go new file mode 100644 index 000000000..c76172c11 --- /dev/null +++ b/report/import_test.go @@ -0,0 +1,81 @@ +package report + +import ( + "bytes" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/assert" +) + +func init() { + log.SetLevel(log.WarnLevel) +} + +func TestReadAnalysis(t *testing.T) { + buff := bytes.NewBuffer([]byte(` + [1,2,{"progname":"gdu","progver":"development","timestamp":1626806293}, + [{"name":"/home/xxx"}, + {"name":"gdu.json","asize":33805233,"dsize":33808384}, + [{"name":"app"}, + {"name":"app.go","asize":4638,"dsize":8192}, + {"name":"app_linux_test.go","asize":1410,"dsize":4096}, + {"name":"app_test.go","asize":4974,"dsize":8192}], + {"name":"main.go","asize":3205,"dsize":4096}]] + `)) + + dir, err := ReadAnalysis(buff) + + assert.Nil(t, err) + assert.Equal(t, "xxx", dir.GetName()) + assert.Equal(t, "/home/xxx", dir.GetPath()) +} + +func TestReadAnalysisWithEmptyInput(t *testing.T) { + buff := bytes.NewBuffer([]byte(``)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "JSON file does not contain top level array", err.Error()) +} + +func TestReadAnalysisWithEmptyArray(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Top level array must have at least 4 items", err.Error()) +} + +func TestReadAnalysisWithWrongContent(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,4]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Array of maps not found in the top level array on 4th position", err.Error()) +} + +func TestReadAnalysisWithEmptyDirContent(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,[{}]]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Directory name is not a string", err.Error()) +} + +func TestReadAnalysisWithWrongDirItem(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,[1, 2, 3]]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Directory item is not a map", err.Error()) +} + +func TestReadAnalysisWithWrongSubdirItem(t *testing.T) { + buff := bytes.NewBuffer([]byte(`[1,2,3,[{"name":"xxx"}, [1,2,3]]]`)) + + _, err := ReadAnalysis(buff) + + assert.Equal(t, "Directory item is not a map", err.Error()) +} diff --git a/stdout/stdout.go b/stdout/stdout.go index 0d246adbf..b572e9a58 100644 --- a/stdout/stdout.go +++ b/stdout/stdout.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "runtime" "sort" "sync" "time" @@ -13,6 +14,7 @@ import ( "github.com/dundee/gdu/v5/internal/common" "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" + "github.com/dundee/gdu/v5/report" "github.com/fatih/color" ) @@ -25,6 +27,8 @@ type UI struct { blue *color.Color } +var progressRunes = []rune(`⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧`) + // CreateStdoutUI creates UI for stdout func CreateStdoutUI(output io.Writer, useColors bool, showProgress bool, showApparentSize bool) *UI { ui := &UI{ @@ -141,6 +145,12 @@ func (ui *UI) AnalyzePath(path string, _ *analyze.Dir) error { wait.Wait() + ui.showDir(dir) + + return nil +} + +func (ui *UI) showDir(dir *analyze.Dir) { sort.Sort(dir.Files) var lineFormat string @@ -173,18 +183,89 @@ func (ui *UI) AnalyzePath(path string, _ *analyze.Dir) error { file.GetName()) } } +} + +// ReadAnalysis reads analysis report from JSON file +func (ui *UI) ReadAnalysis(input io.Reader) error { + var ( + dir *analyze.Dir + wait sync.WaitGroup + err error + doneChan chan struct{} + ) + + if ui.ShowProgress { + wait.Add(1) + doneChan = make(chan struct{}) + go func() { + defer wait.Done() + ui.showReadingProgress(doneChan) + }() + } + + wait.Add(1) + go func() { + defer wait.Done() + dir, err = report.ReadAnalysis(input) + if err != nil { + if ui.ShowProgress { + doneChan <- struct{}{} + } + return + } + runtime.GC() + + links := make(analyze.AlreadyCountedHardlinks, 10) + dir.UpdateStats(links) + + if ui.ShowProgress { + doneChan <- struct{}{} + } + }() + + wait.Wait() + + if err != nil { + return err + } + + ui.showDir(dir) return nil } +func (ui *UI) showReadingProgress(doneChan chan struct{}) { + emptyRow := "\r" + for j := 0; j < 40; j++ { + emptyRow += " " + } + + i := 0 + for { + fmt.Fprint(ui.output, emptyRow) + + select { + case <-doneChan: + fmt.Fprint(ui.output, "\r") + return + default: + } + + fmt.Fprintf(ui.output, "\r %s ", string(progressRunes[i])) + fmt.Fprint(ui.output, "Reading analysis from file...") + + time.Sleep(100 * time.Millisecond) + i++ + i %= 10 + } +} + func (ui *UI) updateProgress() { emptyRow := "\r" for j := 0; j < 100; j++ { emptyRow += " " } - progressRunes := []rune(`⠇⠏⠋⠙⠹⠸⠼⠴⠦⠧`) - progressChan := ui.Analyzer.GetProgressChan() doneChan := ui.Analyzer.GetDoneChan() diff --git a/stdout/stdout_test.go b/stdout/stdout_test.go index 1022285fc..fe514effb 100644 --- a/stdout/stdout_test.go +++ b/stdout/stdout_test.go @@ -2,6 +2,7 @@ package stdout import ( "bytes" + "os" "testing" log "github.com/sirupsen/logrus" @@ -127,6 +128,44 @@ func TestShowDevicesWithColor(t *testing.T) { assert.Contains(t, output.String(), "xxx") } +func TestReadAnalysisWithColor(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/test.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, true, true, true) + err = ui.ReadAnalysis(input) + + assert.Nil(t, err) + assert.Contains(t, output.String(), "main.go") +} + +func TestReadAnalysisBw(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/test.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, false, false, false) + err = ui.ReadAnalysis(input) + + assert.Nil(t, err) + assert.Contains(t, output.String(), "main.go") +} + +func TestReadAnalysisWithWrongFile(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/wrong.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + output := bytes.NewBuffer(make([]byte, 10)) + + ui := CreateStdoutUI(output, true, true, true) + err = ui.ReadAnalysis(input) + + assert.NotNil(t, err) +} + func TestMaxInt(t *testing.T) { assert.Equal(t, 5, maxInt(2, 5)) assert.Equal(t, 4, maxInt(4, 2)) diff --git a/tui/actions.go b/tui/actions.go index 09800ac09..c07c36af6 100644 --- a/tui/actions.go +++ b/tui/actions.go @@ -3,6 +3,7 @@ package tui import ( "bufio" "fmt" + "io" "os" "path/filepath" "runtime" @@ -10,6 +11,7 @@ import ( "github.com/dundee/gdu/v5/pkg/analyze" "github.com/dundee/gdu/v5/pkg/device" + "github.com/dundee/gdu/v5/report" "github.com/gdamore/tcell/v2" "github.com/rivo/tview" ) @@ -113,6 +115,57 @@ func (ui *UI) AnalyzePath(path string, parentDir *analyze.Dir) error { return nil } +// ReadAnalysis reads analysis report from JSON file +func (ui *UI) ReadAnalysis(input io.Reader) error { + ui.progress = tview.NewTextView().SetText("Reading analysis from file...") + ui.progress.SetBorder(true).SetBorderPadding(2, 2, 2, 2) + ui.progress.SetTitle(" Reading... ") + ui.progress.SetDynamicColors(true) + + flex := tview.NewFlex(). + AddItem(nil, 0, 1, false). + AddItem(tview.NewFlex().SetDirection(tview.FlexRow). + AddItem(nil, 10, 1, false). + AddItem(ui.progress, 8, 1, false). + AddItem(nil, 10, 1, false), 0, 50, false). + AddItem(nil, 0, 1, false) + + ui.pages.AddPage("progress", flex, true, true) + + go func() { + var err error + ui.currentDir, err = report.ReadAnalysis(input) + if err != nil { + ui.app.QueueUpdateDraw(func() { + ui.pages.RemovePage("progress") + ui.showErr("Error reading file", err) + }) + if ui.done != nil { + ui.done <- struct{}{} + } + return + } + runtime.GC() + + ui.topDirPath = ui.currentDir.GetPath() + ui.topDir = ui.currentDir + + links := make(analyze.AlreadyCountedHardlinks, 10) + ui.topDir.UpdateStats(links) + + ui.app.QueueUpdateDraw(func() { + ui.showDir() + ui.pages.RemovePage("progress") + }) + + if ui.done != nil { + ui.done <- struct{}{} + } + }() + + return nil +} + func (ui *UI) deleteSelected(shouldEmpty bool) { row, column := ui.table.GetSelection() selectedItem := ui.table.GetCell(row, column).GetReference().(analyze.Item) diff --git a/tui/actions_test.go b/tui/actions_test.go index 3f88ff8e0..401d58162 100644 --- a/tui/actions_test.go +++ b/tui/actions_test.go @@ -1,6 +1,7 @@ package tui import ( + "os" "testing" "github.com/dundee/gdu/v5/internal/testanalyze" @@ -160,6 +161,46 @@ func TestAnalyzePathWithParentDir(t *testing.T) { assert.Contains(t, ui.table.GetCell(1, 0).Text, "aaa") } +func TestReadAnalysis(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/test.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + app := testapp.CreateMockedApp(true) + ui := CreateUI(app, true, true) + ui.done = make(chan struct{}) + + err = ui.ReadAnalysis(input) + assert.Nil(t, err) + + <-ui.done // wait for reading + + assert.Equal(t, "gdu", ui.currentDir.Name) + + for _, f := range ui.app.(*testapp.MockedApp).UpdateDraws { + f() + } +} + +func TestReadAnalysisWithWrongFile(t *testing.T) { + input, err := os.OpenFile("../internal/testdata/wrong.json", os.O_RDONLY, 0644) + assert.Nil(t, err) + + app := testapp.CreateMockedApp(true) + ui := CreateUI(app, true, true) + ui.done = make(chan struct{}) + + err = ui.ReadAnalysis(input) + assert.Nil(t, err) + + <-ui.done // wait for reading + + for _, f := range ui.app.(*testapp.MockedApp).UpdateDraws { + f() + } + + assert.True(t, ui.pages.HasPage("error")) +} + func TestViewDirContents(t *testing.T) { app := testapp.CreateMockedApp(true) ui := CreateUI(app, false, true)