diff --git a/downloader/download_image.go b/downloader/download_image.go index 97e8697..387262b 100644 --- a/downloader/download_image.go +++ b/downloader/download_image.go @@ -2,6 +2,7 @@ package downloader import ( "fmt" + "os" "path" "github.com/samsonmxvi/go-wenku-downloader/util" @@ -15,7 +16,13 @@ func DownloadImage(url string, dirPath string) error { // check file already download if util.CheckFileExist(filePath) { - return nil + // 获取文件信息 + fileInfo, _ := os.Stat(filePath) + // 获取文件大小 + fileSize := fileInfo.Size() + if fileSize > 0 { + return nil + } } // check dir if not exit create diff --git a/downloader/download_volume.go b/downloader/download_volume.go index c787f97..03c3a99 100644 --- a/downloader/download_volume.go +++ b/downloader/download_volume.go @@ -6,6 +6,7 @@ import ( "log" "os" "path" + "strings" "time" "github.com/samsonmxvi/go-wenku-downloader/scraper" @@ -13,7 +14,7 @@ import ( "gopkg.in/cheggaaa/pb.v2" ) -func DownloadVolume(volume *scraper.Volume, dirPath string) error { +func DownloadVolume(volume *scraper.Volume, dirPath string, onlyWenku8Img bool) error { var imageArray []string imageDirPath := path.Join(dirPath, ImageFolderName) @@ -55,14 +56,18 @@ func DownloadVolume(volume *scraper.Volume, dirPath string) error { progressBar.Finish() for _, imageURL := range imageArray { + isWenku8Source := strings.Contains(imageURL, "wenku8.com") + if !isWenku8Source && onlyWenku8Img { + continue + } success := false - for i := 0; i < 3; i++ { + for i := 0; i < RetryTimes; i++ { err := DownloadImage(imageURL, imageDirPath) if err == nil { success = true break } else { - time.Sleep(3 * time.Second) // temp fix rate limit + time.Sleep(RetryTimer) // temp fix rate limit continue } } @@ -75,14 +80,14 @@ func DownloadVolume(volume *scraper.Volume, dirPath string) error { } func getChapterArray(volume *scraper.Volume) ([]*scraper.Chapter, error) { - for i := 0; i < 3; i++ { + for i := 0; i < RetryTimes; i++ { chaterArray, err := scraper.GetChapterArray(volume) if err == nil { - time.Sleep(1 * time.Second) + time.Sleep(DownloadTimer) return chaterArray, nil } else { log.Printf("获取章节列表失败 %v, 重试第%v次 \n", err, i) - time.Sleep(3 * time.Second) // temp fix rate limit + time.Sleep(RetryTimer) // temp fix rate limit continue } } @@ -91,13 +96,13 @@ func getChapterArray(volume *scraper.Volume) ([]*scraper.Chapter, error) { func getChaterContent(chapter *scraper.Chapter) error { var err error - for i := 0; i < 3; i++ { + for i := 0; i < RetryTimes; i++ { err = scraper.GetChapterContent(chapter) if err == nil { - time.Sleep(1 * time.Second) + time.Sleep(DownloadTimer) return nil } else { - time.Sleep(3 * time.Second) // temp fix rate limit + time.Sleep(RetryTimer) // temp fix rate limit log.Printf("获取章节内容失败 %v, 重试第%v次 \n", err, i) continue } diff --git a/downloader/download_volume_test.go b/downloader/download_volume_test.go index 9a434a5..eaec1ab 100644 --- a/downloader/download_volume_test.go +++ b/downloader/download_volume_test.go @@ -11,6 +11,6 @@ func TestDownloadVolume(t *testing.T) { volumeArray, err := scraper.GetNovelVolumeArray("https://www.wenku8.net/novel/1/1973/index.htm") require.NoError(t, err) require.NotEmpty(t, volumeArray) - err = DownloadVolume(volumeArray[0], "./") + err = DownloadVolume(volumeArray[0], "./", true) require.NoError(t, err) } diff --git a/downloader/grab.go b/downloader/grab.go index a3253b8..aa9fc99 100644 --- a/downloader/grab.go +++ b/downloader/grab.go @@ -2,6 +2,7 @@ package downloader import ( "fmt" + "log" "time" "github.com/cavaliergopher/grab/v3" @@ -14,7 +15,7 @@ func Grab(path string, url string) error { req, _ := grab.NewRequest(path, url) // start download - fmt.Printf("Downloading %v...\n", req.URL()) + log.Printf("正在下载 %v...\n", req.URL()) resp := client.Do(req) // start UI loop diff --git a/downloader/variable.go b/downloader/variable.go index 23adb07..5f68800 100644 --- a/downloader/variable.go +++ b/downloader/variable.go @@ -1,6 +1,11 @@ package downloader +import "time" + var ( ErrorJsonName = "error.json" ImageFolderName = "images" + DownloadTimer = time.Second + RetryTimes = 6 + RetryTimer = 6 * time.Second ) diff --git a/prompt/download.go b/prompt/download.go index f998803..a509b5f 100644 --- a/prompt/download.go +++ b/prompt/download.go @@ -56,10 +56,16 @@ func download(novelId int) error { return fmt.Errorf("prompt failed %v", err) } + // get onlyWenku8Img + onlyWenku8Img, err := getInputBool("是否只下载wenku8的图片(推荐使用默认数值, 非文库图片资源大多数情况已失效), 默认:y(y/n)", true) + if err != nil { + return fmt.Errorf("prompt failed %v", err) + } + // download volume for _, volume := range volumeArray { volumePath := path.Join(downloadPath, formatFilename(volume.Name)) - err = downloader.DownloadVolume(volume, volumePath) + err = downloader.DownloadVolume(volume, volumePath, onlyWenku8Img) if err != nil { log.Printf("download volume error %v", err) continue @@ -114,6 +120,9 @@ func createEpub(novel *scraper.Novel, volumeName string, chapterCount int, cover if len(chapter.Content.Images) != 0 { for _, img := range chapter.Content.Images { imgFile := path.Join(imagePath, util.GetUrlLastString(img)) + if !util.CheckFileExist(imgFile) { + continue + } internalPath, _ := util.AddImage(epub, imgFile) xhtml = util.AddImageToXhtml(internalPath, xhtml) imagePathList = append(imagePathList, imgFile) diff --git a/prompt/get_input.go b/prompt/get_input.go index 784bfa0..5a0064e 100644 --- a/prompt/get_input.go +++ b/prompt/get_input.go @@ -2,6 +2,7 @@ package prompt import ( "errors" + "fmt" "regexp" "strconv" @@ -117,3 +118,33 @@ func getSelectedIndex(label string, itmes []string) (int, error) { } return selectedIndex, nil } + +func getInputBool(label string, defaultValue bool) (bool, error) { + validate := func(input string) error { + if input == "y" || input == "n" || input == "" { + return nil + } else { + return fmt.Errorf("只能填写y, n") + } + } + + prompt := promptui.Prompt{ + Label: label, + Validate: validate, + Stdout: &noBellStdout{}, + } + + result, err := prompt.Run() + + if err != nil { + return false, err + } + + if result == "y" { + return true, nil + } else if result == "n" { + return false, nil + } else { + return defaultValue, nil + } +} diff --git a/scraper/get_novel_details.go b/scraper/get_novel_details.go index 14aa3cc..3b290c6 100644 --- a/scraper/get_novel_details.go +++ b/scraper/get_novel_details.go @@ -1,6 +1,7 @@ package scraper import ( + "fmt" "regexp" "strconv" "strings" @@ -21,7 +22,7 @@ func GetNovelDetails(novelId int) (*Novel, error) { } if doc.Find(".blocktitle").First().Text() == "出现错误!" { - return nil, nil + return nil, fmt.Errorf("没有这本小说") } getNovelDetailsDoc(doc, novel)