Add site-specific support for Streamable

internal/pkg/crawl/capture.go

@@ -15,6 +15,7 @@ import (
 	"github.com/clbanning/mxj/v2"
 	"github.com/internetarchive/Zeno/internal/pkg/crawl/sitespecific/cloudflarestream"
 	"github.com/internetarchive/Zeno/internal/pkg/crawl/sitespecific/libsyn"
+	"github.com/internetarchive/Zeno/internal/pkg/crawl/sitespecific/streamable"
 	"github.com/internetarchive/Zeno/internal/pkg/crawl/sitespecific/telegram"
 	"github.com/internetarchive/Zeno/internal/pkg/crawl/sitespecific/tiktok"
 	"github.com/internetarchive/Zeno/internal/pkg/crawl/sitespecific/truthsocial"
@@ -266,7 +267,7 @@ func (c *Crawl) Capture(item *frontier.Item) {
 			// Grab few embeds that are needed for the playback
 			embedURLs, err := truthsocial.EmbedURLs()
 			if err != nil {
-				logError.WithFields(c.genLogFields(err, item.URL, nil)).Error("error while getting embed URLs")
+				logError.WithFields(c.genLogFields(err, item.URL, nil)).Error("error while getting embed URLs for truthsocial")
 			} else {
 				for _, embedURL := range embedURLs {
 					// Create the embed item
@@ -277,6 +278,20 @@ func (c *Crawl) Capture(item *frontier.Item) {
 				}
 			}
 		}
+	} else if streamable.IsStreamableURL(utils.URLToString(item.URL)) {
+		// Get the player URLs
+		embedURLs, err := streamable.EmbedURLs()
+		if err != nil {
+			logError.WithFields(c.genLogFields(err, item.URL, nil)).Error("error while getting embed URLs for streamable")
+		} else {
+			for _, embedURL := range embedURLs {
+				// Create the embed item
+				embedItem := frontier.NewItem(embedURL, item, item.Type, item.Hop, item.ID, false)
+
+				// Capture the embed item
+				c.Capture(embedItem)
+			}
+		}
 	} else if libsyn.IsLibsynURL(utils.URLToString(item.URL)) {
 		// Generate the highwinds URL
 		highwindsURL, err := libsyn.GenerateHighwindsURL(utils.URLToString(item.URL))

internal/pkg/crawl/sitespecific/streamable/streamable.go

@@ -0,0 +1,28 @@
+package streamable
+
+import (
+	"net/url"
+	"strings"
+)
+
+func IsStreamableURL(URL string) bool {
+	return strings.Contains(URL, "//streamable.com/")
+}
+
+func EmbedURLs() (URLs []*url.URL, err error) {
+	URLsString := []string{
+		"https://ui-statics-cf.streamable.com/player/_next/static/chunks/4684.ccb5ae368bff3fcc.js",
+		"https://ui-statics-cf.streamable.com/player/_next/static/chunks/22e7f3a7.05a2bf84ec78694f.js",
+	}
+
+	for _, URL := range URLsString {
+		apiURL, err := url.Parse(URL)
+		if err != nil {
+			return nil, err
+		}
+
+		URLs = append(URLs, apiURL)
+	}
+
+	return URLs, nil
+}