Upload files remotely for sharing

cmd/wayback/wayback.go

@@ -31,11 +31,11 @@ func archive(cmd *cobra.Command, args []string) {
 			cmd.Println(col.Src, "=>", col.Dst)
 		}
 		for src, bundle := range bundles {
-			for _, path := range bundle.Paths() {
-				if path == "" {
+			for _, asset := range bundle.Asset() {
+				if asset.Local == "" {
 					continue
 				}
-				cmd.Println(src, "=>", path)
+				cmd.Println(src, "=>", asset.Local)
 			}
 		}
 

errors/errors.go

@@ -4,7 +4,11 @@
 
 package errors // import "github.com/wabarc/wayback/errors"
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
 
 // Error represents an error
 type Error struct {
@@ -21,3 +25,7 @@ func (e Error) Error() string {
 func New(message string, args ...interface{}) *Error {
 	return &Error{message: message, args: args}
 }
+
+func Wrap(err error, message string) error {
+	return errors.Wrap(err, message)
+}

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/mattn/go-mastodon v0.0.5-0.20210515144304-86627ec7d635
 	github.com/multiformats/go-multiaddr v0.3.3 // indirect
 	github.com/multiformats/go-multihash v0.0.15 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.30.0
 	github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect
@@ -41,12 +42,14 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/wabarc/archive.is v1.3.0
 	github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
+	github.com/wabarc/go-anonfile v0.1.0
+	github.com/wabarc/go-catbox v0.1.0
 	github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2
 	github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
 	github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933
 	github.com/wabarc/screenshot v1.2.1-0.20210713083422-eee9add4752d
 	github.com/wabarc/telegra.ph v0.0.0-20210718173528-f72fc9cdc365
-	github.com/wabarc/warcraft v0.1.1-0.20210707001544-e897dbede7c3
+	github.com/wabarc/warcraft v0.1.1-0.20210711171056-a5eec617b86c
 	github.com/wabarc/wbipfs v0.2.0
 	github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
 	go.etcd.io/bbolt v1.3.6
@@ -55,6 +58,7 @@ require (
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/tucnak/telebot.v2 v2.3.5
 	maunium.net/go/mautrix v0.9.17
+	mvdan.cc/xurls/v2 v2.3.0 // indirect
 )
 
 replace github.com/go-shiori/obelisk => github.com/wabarc/obelisk v0.0.0-20210420023708-aac2bcc00a78

go.sum

@@ -465,6 +465,7 @@ github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzb
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -508,6 +509,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -587,9 +589,14 @@ github.com/wabarc/archive.is v1.3.0/go.mod h1:01mgwPSUip7WkigtaZypTTHSpv3NVbtWpK
 github.com/wabarc/archive.org v1.2.0/go.mod h1:yEmUMlNO2PPAxIvo/Hf/VxOrCS5SBwL2/vCW8pyTWjA=
 github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896 h1:c3uD+IKXpNhtAHC2/fTt7R9dXnNhw1Q6sQrHUX3xo64=
 github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896/go.mod h1:yEmUMlNO2PPAxIvo/Hf/VxOrCS5SBwL2/vCW8pyTWjA=
+github.com/wabarc/go-anonfile v0.1.0 h1:M4jKUAMROxxVaqLQt30ONmaHA/0YnvyJtX8qg/zI9+I=
+github.com/wabarc/go-anonfile v0.1.0/go.mod h1:CH1LzSKQ0x/RlKpEG2gi+hXaOOZrfeRx6Rp717o+65A=
+github.com/wabarc/go-catbox v0.1.0 h1:/UhV9md3MJrjZtm+EToSyFjawXgPiHSExLNRqsWNisg=
+github.com/wabarc/go-catbox v0.1.0/go.mod h1:Zjs9Y55f2WOwGWwmKSCrUuMfwh+nDktkjub9jgHq4CQ=
 github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8=
 github.com/wabarc/helper v0.0.0-20210407153720-1bfe98b427fe/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210613000316-cc3df6340ab5/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
+github.com/wabarc/helper v0.0.0-20210614160629-1a5ba5e551eb/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210701193643-e0fe0a807cb9/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2 h1:6rMZse2rdD7N6GxHRZqHlkSptBWh/Vf9aHiFVQjlQNo=
 github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ=
@@ -614,8 +621,8 @@ github.com/wabarc/telebot v0.0.0-20210614085950-9479567e0e0a h1:RCrMXolMzoNxmN9X
 github.com/wabarc/telebot v0.0.0-20210614085950-9479567e0e0a/go.mod h1:BgaIIx50PSRS9pG59JH+geT82cfvoJU/IaI5TJdN3v8=
 github.com/wabarc/telegra.ph v0.0.0-20210718173528-f72fc9cdc365 h1:AZeSe84nDyD5bYNpkCVVaOxdvaHIOj5tjVJPqcjjFLI=
 github.com/wabarc/telegra.ph v0.0.0-20210718173528-f72fc9cdc365/go.mod h1:e2nxaKoyr34W/jZefnrTn4HGXKdg0HBygLvIDJKFAiw=
-github.com/wabarc/warcraft v0.1.1-0.20210707001544-e897dbede7c3 h1:P9KKKeXbGdvzJoEEMQ2M9DlusYdS/9hzimS0+rVYrJY=
-github.com/wabarc/warcraft v0.1.1-0.20210707001544-e897dbede7c3/go.mod h1:/BbCwReBjlqHRaw8Yh+7sfAicOesiMYNhiFpuL1x8Rc=
+github.com/wabarc/warcraft v0.1.1-0.20210711171056-a5eec617b86c h1:HPTqoafOgXbqEMD54v3cTTJKKK3fd79UqG1ILR9S640=
+github.com/wabarc/warcraft v0.1.1-0.20210711171056-a5eec617b86c/go.mod h1:/BbCwReBjlqHRaw8Yh+7sfAicOesiMYNhiFpuL1x8Rc=
 github.com/wabarc/wbipfs v0.2.0 h1:TksxONxL/LrBGZ6TTcWD4p+FAzuQPfLQyCL7K64isW8=
 github.com/wabarc/wbipfs v0.2.0/go.mod h1:wh76ZvNbhSrDNxStkKORKcnfwjozfoTtq8XyzU7GKDw=
 github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs=
@@ -1066,8 +1073,9 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 maunium.net/go/maulogger/v2 v2.2.4/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
 maunium.net/go/mautrix v0.9.17 h1:4rRAN11LPQGS0TOGEb4uppUzGet7nSwT9C0jHzSWmtg=
 maunium.net/go/mautrix v0.9.17/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=
-mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A=
 mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
+mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I=
+mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

publish/discord.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"os"
 	"path"
-	"strings"
 
 	"github.com/dustin/go-humanize"
 	"github.com/wabarc/helper"
@@ -53,7 +52,7 @@ func (d *discordBot) Publish(ctx context.Context, cols []wayback.Collect, args .
 	}
 
 	var bnd = bundle(ctx, cols)
-	var txt = render.ForPublish(&render.Discord{Cols: cols}).String()
+	var txt = render.ForPublish(&render.Discord{Cols: cols, Data: bnd}).String()
 	if d.toChannel(ctx, bnd, txt) {
 		metrics.IncrementPublish(metrics.PublishDiscord, metrics.StatusSuccess)
 		return
@@ -64,7 +63,7 @@ func (d *discordBot) Publish(ctx context.Context, cols []wayback.Collect, args .
 
 // toChannel for publish to message to Discord channel,
 // returns boolean as result.
-func (d *discordBot) toChannel(ctx context.Context, bundle *reduxer.Bundle, text string) (ok bool) {
+func (d *discordBot) toChannel(_ context.Context, bundle *reduxer.Bundle, text string) (ok bool) {
 	if text == "" {
 		logger.Warn("post to message to channel failed, text empty")
 		return ok
@@ -78,58 +77,51 @@ func (d *discordBot) toChannel(ctx context.Context, bundle *reduxer.Bundle, text
 		}
 	}
 
-	// TODO: move to render
-	var b strings.Builder
-	if head := title(ctx, bundle); head != "" {
-		b.WriteString(`**`)
-		b.WriteString(head)
-		b.WriteString(`**`)
-		b.WriteString("\n\n")
+	msg, err := d.bot.ChannelMessageSendComplex(config.Opts.DiscordChannel(), &discord.MessageSend{Content: text})
+	if err != nil {
+		logger.Error("post message to channel failed, %v", err)
+		return ok
+	}
+
+	// Send files as reference
+	files := UploadToDiscord(bundle)
+	if len(files) == 0 {
+		logger.Debug("without files, complete.")
+		return true
 	}
-	if dgst := digest(ctx, bundle); dgst != "" {
-		b.WriteString(dgst)
-		b.WriteString("\n\n")
+	ms := &discord.MessageSend{Files: files, Reference: msg.Reference()}
+	if _, err := d.bot.ChannelMessageSendComplex(config.Opts.DiscordChannel(), ms); err != nil {
+		logger.Error("upload files failed, %v", err)
 	}
-	b.WriteString(text)
 
-	msg := &discord.MessageSend{Content: b.String()}
+	return true
+}
+
+func UploadToDiscord(bundle *reduxer.Bundle) (files []*discord.File) {
 	if bundle != nil {
 		var fsize int64
-		var files []*discord.File
 		upper := config.Opts.MaxAttachSize("discord")
-		for _, p := range bundle.Paths() {
-			if p == "" {
+		for _, asset := range bundle.Asset() {
+			if asset.Local == "" {
 				continue
 			}
-			if !helper.Exists(p) {
-				logger.Warn("invalid file %s", p)
+			if !helper.Exists(asset.Local) {
+				logger.Warn("invalid file %s", asset.Local)
 				continue
 			}
-			fsize += helper.FileSize(p)
+			fsize += helper.FileSize(asset.Local)
 			if fsize > upper {
 				logger.Warn("total file size large than %s, skipped", humanize.Bytes(uint64(upper)))
 				continue
 			}
-			logger.Debug("open file: %s", p)
-			rd, err := os.Open(p)
+			logger.Debug("open file: %s", asset.Local)
+			rd, err := os.Open(asset.Local)
 			if err != nil {
 				logger.Error("open file failed: %v", err)
 				continue
 			}
-			files = append(files, &discord.File{Name: path.Base(p), Reader: rd})
-		}
-		if len(files) == 0 {
-			logger.Warn("files empty")
-			return ok
+			files = append(files, &discord.File{Name: path.Base(asset.Local), Reader: rd})
 		}
-		msg.Files = files
 	}
-
-	_, err := d.bot.ChannelMessageSendComplex(config.Opts.DiscordChannel(), msg)
-	if err != nil {
-		logger.Error("post message to channel failed, %v", err)
-		return ok
-	}
-
-	return true
+	return
 }

publish/discord_test.go

@@ -170,8 +170,8 @@ func TestToDiscordChannel(t *testing.T) {
 	}
 	bot.Client = httpClient
 	d := NewDiscord(bot)
-	txt := render.ForPublish(&render.Discord{Cols: collects}).String()
-	got := d.toChannel(context.Background(), nil, txt)
+	txt := render.ForPublish(&render.Discord{Cols: collects, Data: bundleExample}).String()
+	got := d.toChannel(context.Background(), bundleExample, txt)
 	if !got {
 		t.Errorf("Unexpected publish to discord channel got %t instead of %t", got, true)
 	}

publish/github.go

@@ -52,7 +52,7 @@ func (gh *gitHub) Publish(ctx context.Context, cols []wayback.Collect, args ...s
 	}
 
 	var bnd = bundle(ctx, cols)
-	var txt = render.ForPublish(&render.GitHub{Cols: cols}).String()
+	var txt = render.ForPublish(&render.GitHub{Cols: cols, Data: bnd}).String()
 	if gh.toIssues(ctx, bnd, txt) {
 		metrics.IncrementPublish(metrics.PublishGithub, metrics.StatusSuccess)
 		return
@@ -76,19 +76,11 @@ func (gh *gitHub) toIssues(ctx context.Context, bundle *reduxer.Bundle, text str
 		logger.Debug("authorized GitHub user: %v", user)
 	}
 
-	t := strings.TrimSpace(title(ctx, bundle))
+	t := strings.TrimSpace(render.Title(bundle))
 	if t == "" {
 		t = "Published at " + time.Now().Format("2006-01-02T15:04:05")
 	}
 
-	var b strings.Builder
-	if dgst := digest(ctx, bundle); dgst != "" {
-		b.WriteString(dgst)
-		b.WriteString("\n\n")
-	}
-	b.WriteString(text)
-	text = b.String()
-
 	// Create an issue to GitHub
 	ir := &github.IssueRequest{Title: github.String(t), Body: github.String(text)}
 	issue, _, err := gh.client.Issues.Create(ctx, config.Opts.GitHubOwner(), config.Opts.GitHubRepo(), ir)

publish/mastodon.go

@@ -6,7 +6,6 @@ package publish // import "github.com/wabarc/wayback/publish"
 
 import (
 	"context"
-	"strings"
 
 	mstdn "github.com/mattn/go-mastodon"
 	"github.com/wabarc/logger"
@@ -52,7 +51,7 @@ func (m *mastodon) Publish(ctx context.Context, cols []wayback.Collect, args ...
 	}
 
 	var bnd = bundle(ctx, cols)
-	var txt = render.ForPublish(&render.Mastodon{Cols: cols}).String()
+	var txt = render.ForPublish(&render.Mastodon{Cols: cols, Data: bnd}).String()
 	if m.ToMastodon(ctx, bnd, txt, id) {
 		metrics.IncrementPublish(metrics.PublishMstdn, metrics.StatusSuccess)
 		return
@@ -71,15 +70,8 @@ func (m *mastodon) ToMastodon(ctx context.Context, bundle *reduxer.Bundle, text,
 		return false
 	}
 
-	var b strings.Builder
-	if head := title(ctx, bundle); head != "" {
-		b.WriteString(`‹ `)
-		b.WriteString(head)
-		b.WriteString(" ›\n\n")
-	}
-	b.WriteString(text)
 	toot := &mstdn.Toot{
-		Status:     b.String(),
+		Status:     text,
 		Visibility: mstdn.VisibilityPublic,
 	}
 	if id != "" {

publish/matrix.go

@@ -6,13 +6,11 @@ package publish // import "github.com/wabarc/wayback/publish"
 
 import (
 	"context"
-	"strings"
 
 	"github.com/wabarc/logger"
 	"github.com/wabarc/wayback"
 	"github.com/wabarc/wayback/config"
 	"github.com/wabarc/wayback/metrics"
-	"github.com/wabarc/wayback/reduxer"
 	"github.com/wabarc/wayback/template/render"
 	matrix "maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/event"
@@ -59,16 +57,16 @@ func (m *matrixBot) Publish(ctx context.Context, cols []wayback.Collect, args ..
 	}
 
 	var bnd = bundle(ctx, cols)
-	var txt = render.ForPublish(&render.Matrix{Cols: cols}).String()
-	if m.toRoom(ctx, bnd, txt) {
+	var txt = render.ForPublish(&render.Matrix{Cols: cols, Data: bnd}).String()
+	if m.toRoom(txt) {
 		metrics.IncrementPublish(metrics.PublishMatrix, metrics.StatusSuccess)
 		return
 	}
 	metrics.IncrementPublish(metrics.PublishMatrix, metrics.StatusFailure)
 	return
 }
 
-func (m *matrixBot) toRoom(ctx context.Context, bundle *reduxer.Bundle, text string) bool {
+func (m *matrixBot) toRoom(text string) bool {
 	if !config.Opts.PublishToMatrixRoom() || m.client == nil {
 		logger.Warn("publish to Matrix room abort.")
 		return false
@@ -78,19 +76,6 @@ func (m *matrixBot) toRoom(ctx context.Context, bundle *reduxer.Bundle, text str
 		return false
 	}
 
-	var b strings.Builder
-	if head := title(ctx, bundle); head != "" {
-		b.WriteString(`‹ <b>`)
-		b.WriteString(head)
-		b.WriteString(`</b> ›<br><br>`)
-	}
-	if dgst := digest(ctx, bundle); dgst != "" {
-		b.WriteString(dgst)
-		b.WriteString(`<br><br>`)
-	}
-	b.WriteString(text)
-
-	text = b.String()
 	content := &event.MessageEventContent{
 		FormattedBody: text,
 		Format:        event.FormatHTML,

publish/matrix_test.go

@@ -5,7 +5,6 @@
 package publish // import "github.com/wabarc/wayback/publish"
 
 import (
-	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -53,7 +52,7 @@ func TestToMatrixRoom(t *testing.T) {
 
 	mat := NewMatrix(nil)
 	txt := render.ForPublish(&render.Mastodon{Cols: collects}).String()
-	got := mat.toRoom(context.Background(), nil, txt)
+	got := mat.toRoom(txt)
 	if !got {
 		t.Errorf("Unexpected publish room message got %t instead of %t", got, true)
 	}