kadai3-2-pei

kadai3-2/pei/Makefile

@@ -0,0 +1,17 @@
+ROOT=github.com/gopherdojo/dojo6/kadai3-2/pei
+BIN=split-download
+MAIN=main.go
+TEST=...
+
+.PHONY: build
+build: ${MAIN}
+	go build -o ${BIN} ${GOPATH}/src/${ROOT}/$?
+
+.PHONY: test
+test:
+	go test -v -cover ${ROOT}/${TEST}
+
+.PHONY: clean
+clean:
+	rm ${BIN}
+	go clean

kadai3-2/pei/README.md

@@ -0,0 +1,33 @@
+# split-download
+
+## Build
+
+```
+$ make build
+```
+
+## Usage
+
+```
+$ ./split-loadload -o [OutputPath] [URL]
+```
+
+## Development
+
+### Build
+
+```
+$ make build
+```
+
+### Test
+
+```
+$ make test
+```
+
+### Clean
+
+```
+$ make clean
+```

kadai3-2/pei/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/gopherdojo/dojo6/kadai3-2/pei/pkg/download"
+)
+
+const (
+	exitCodeOk    = 0
+	exitCodeError = 1
+
+	splitNum = 4
+)
+
+type cliArgs struct {
+	url, outputPath string
+}
+
+func (ca *cliArgs) validate() error {
+	if ca.url == "" {
+		return fmt.Errorf("No URL")
+	}
+
+	if ca.outputPath == "" {
+		ca.outputPath = "./"
+	}
+
+	return nil
+}
+
+func main() {
+	os.Exit(Run())
+}
+
+// Run runs download
+func Run() int {
+	ca := parseArgs()
+	if err := ca.validate(); err != nil {
+		fmt.Fprintln(os.Stderr, "Args error: ", err)
+		return exitCodeError
+	}
+
+	downloader, err := download.NewDownloader(splitNum, ca.url, ca.outputPath)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Create downloader error: ", err)
+		return exitCodeError
+	}
+
+	outputPath, err := downloader.Do()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Download error: ", err)
+		return exitCodeError
+	}
+
+	var downloadType string
+	if reflect.TypeOf(downloader) == reflect.TypeOf(&download.RangeDownloader{}) {
+		downloadType = "Split Download"
+	} else {
+		downloadType = "Download"
+	}
+	fmt.Println("Download Type: ", downloadType)
+	fmt.Println("Download completed. Output: ", outputPath)
+
+	return exitCodeOk
+}
+
+func parseArgs() *cliArgs {
+	var ca cliArgs
+	flag.StringVar(&ca.outputPath, "o", "", "output path")
+	flag.Parse()
+	ca.url = flag.Arg(0)
+
+	return &ca
+}

kadai3-2/pei/pkg/download/download.go

@@ -0,0 +1,171 @@
+package download
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// Downloader Interface
+type Downloader interface {
+	Do() (string, error)
+}
+
+// NonRangeDownloader has info for downloading
+type NonRangeDownloader struct {
+	url, outputPath string
+}
+
+// RangeDownloader has info for split downloading
+type RangeDownloader struct {
+	splitNum        int
+	ranges          []*Range
+	url, outputPath string
+}
+
+// Range has rangestart and rangeend
+type Range struct {
+	start int64
+	end   int64
+}
+
+// NewDownloader creates Downloader
+func NewDownloader(splitNum int, url, outputPath string) (Downloader, error) {
+	dir, fileName := parseDirAndFileName(outputPath)
+	if fileName == "" {
+		outputPath = filepath.Join(dir, parseFileName(url))
+	}
+
+	res, err := http.Head(url)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.Header.Get("Accept-Ranges") != "bytes" {
+		return &NonRangeDownloader{url: url, outputPath: outputPath}, nil
+	}
+
+	contentLength := res.ContentLength
+	unit := contentLength / int64(splitNum)
+	ranges := make([]*Range, splitNum)
+
+	for i := range ranges {
+		var start, end int64
+		if i != 0 {
+			start = int64(i)*unit + 1
+		}
+		end = int64(i+1) * unit
+		if i == splitNum-1 {
+			end = contentLength
+		}
+
+		ranges[i] = &Range{start: start, end: end}
+	}
+
+	return &RangeDownloader{
+		splitNum:   splitNum,
+		ranges:     ranges,
+		url:        url,
+		outputPath: outputPath,
+	}, nil
+}
+
+// Do download
+func (d *NonRangeDownloader) Do() (string, error) {
+	req, err := http.NewRequest(http.MethodGet, d.url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	client := http.DefaultClient
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	return d.outputPath, saveResponseBody(d.outputPath, res)
+}
+
+// Do split download
+func (d *RangeDownloader) Do() (string, error) {
+	eg, ctx := errgroup.WithContext(context.TODO())
+
+	for i := range d.ranges {
+		i := i
+		eg.Go(func() error {
+			return d.do(ctx, i)
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return "", err
+	}
+
+	return d.outputPath, d.mergeFiles()
+}
+
+func (d *RangeDownloader) do(ctx context.Context, idx int) error {
+	req, err := http.NewRequest(http.MethodGet, d.url, nil)
+	if err != nil {
+		return err
+	}
+	req = req.WithContext(ctx)
+
+	ran := d.ranges[idx]
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", ran.start, ran.end))
+
+	client := http.DefaultClient
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	tmpFileName := fmt.Sprintf("%s.%d", d.outputPath, idx)
+	return saveResponseBody(tmpFileName, res)
+}
+
+func (d *RangeDownloader) mergeFiles() error {
+	file, err := os.Create(d.outputPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	for i := range d.ranges {
+		tmpFileName := fmt.Sprintf("%s.%d", d.outputPath, i)
+		tmpFile, err := os.Open(tmpFileName)
+		if err != nil {
+			return err
+		}
+
+		io.Copy(file, tmpFile)
+		tmpFile.Close()
+		if err := os.Remove(tmpFileName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func saveResponseBody(fileName string, response *http.Response) error {
+	file, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	if _, err := io.Copy(file, response.Body); err != nil {
+		return err
+	}
+
+	return nil
+}

kadai3-2/pei/pkg/download/file.go

@@ -0,0 +1,20 @@
+package download
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+func parseDirAndFileName(path string) (dir, file string) {
+	lastSlashIndex := strings.LastIndex(path, "/")
+	dir = path[:lastSlashIndex+1]
+	if len(dir) == len(path) {
+		return dir, ""
+	}
+
+	return dir, path[(len(dir) + 1):]
+}
+
+func parseFileName(url string) string {
+	return filepath.Base(url)
+}