Innertube continuations (#522)

* Search continuations

* add another type of playlists

* Continuation for search and channels working

* Playlist continuation

* Fix pagination - what a fucking mess I've created

* web app fix

* Changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Support for auto-generated channels in Playlet backend
+- Support for continuation/pagination in Playlet backend: search, channels and playlists load more content as you scroll
 
 ### Fixed
 

playlet-lib/src/components/ContentNode/PlaylistContentNode.xml

@@ -7,7 +7,8 @@
         <field id="loadState" type="string" />
         <!-- index in a feed -->
         <field id="feedSourcesIndex" type="integer" value="-1" />
-        <!--             
+        <field id="continuation" type="string" />
+        <!--
             For some strange reason, the "author" field can only be set once.
             If you try to set it again, it will not be updated, with type mismatch warning:
             

playlet-lib/src/components/ContentNode/PlaylistContentTask.bs

@@ -18,7 +18,10 @@ function PlaylistContentTask(input as object) as object
 
     while true
         index = contentNode.getChildCount()
-        response = service.GetPlaylist(contentNode.playlistId, index, m.top.cancellation)
+        continuation = contentNode.continuation
+        hadContinuation = not StringUtils.IsNullOrEmpty(continuation)
+
+        response = service.GetPlaylist(contentNode.playlistId, index, continuation, m.top.cancellation)
 
         if m.top.cancel
             contentNode.loadState = FeedLoadState.None
@@ -59,7 +62,8 @@ function PlaylistContentTask(input as object) as object
 
         videoCount = ValidInt(metadata.videoCount)
 
-        if metadata.videos.Count() = 0 or childCount >= videoCount
+        hasContinuation = not StringUtils.IsNullOrEmpty(metadata.continuation)
+        if (hadContinuation and not hasContinuation) or (metadata.videos.Count() = 0 or childCount >= videoCount)
             contentNode.loadState = FeedLoadState.Loaded
             return {
                 success: true

playlet-lib/src/components/Services/Innertube/BrowseEndpoint.bs

@@ -1,15 +1,19 @@
 namespace Innertube
 
-    function CreateBrowseRequest(browseId as string, clienType as Innertube.ClientType, params as string) as object
+    function CreateBrowseRequest(browseId as string, clienType as Innertube.ClientType, params = "" as string, continuation = "" as string) as object
         deviceInfo = CreateObject("roDeviceInfo")
 
         payload = {
-            "browseId": browseId
             "context": Innertube.CreateContext(clienType, deviceInfo, "")
         }
 
-        if not StringUtils.IsNullOrEmpty(params)
-            payload["params"] = params
+        if not StringUtils.IsNullOrEmpty(continuation)
+            payload["continuation"] = continuation
+        else
+            payload["browseId"] = browseId
+            if not StringUtils.IsNullOrEmpty(params)
+                payload["params"] = params
+            end if
         end if
 
         request = HttpClient.PostJson("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false&alt=json", payload)

playlet-lib/src/components/Services/Innertube/InnertubeService.bs

@@ -84,8 +84,24 @@ namespace InnertubeService
         }
     end function
 
-    function Search(query as string, searchFilters as object, cancellation = invalid as object) as object
-        return Innertube.Search(query, searchFilters, cancellation)
+    function Search(query as string, options = invalid as object) as object
+        searchFilters = invalid
+        continuation = ""
+        cancellation = invalid
+
+        if options <> invalid
+            if options.DoesExist("searchFilters")
+                searchFilters = options.searchFilters
+            end if
+            if options.DoesExist("continuation")
+                continuation = options.continuation
+            end if
+            if options.DoesExist("cancellation")
+                cancellation = options.cancellation
+            end if
+        end if
+
+        return Innertube.Search(query, searchFilters, continuation, cancellation)
     end function
 
     function GetTrending(options = invalid as object) as object
@@ -150,7 +166,7 @@ namespace InnertubeService
             end if
         end if
 
-        request = Innertube.CreateBrowseRequest(browserId, Innertube.ClientType.Tv, "")
+        request = Innertube.CreateBrowseRequest(browserId, Innertube.ClientType.Tv)
         if access_token <> invalid
             request.Header("Authorization", "Bearer " + access_token)
         end if
@@ -166,19 +182,23 @@ namespace InnertubeService
     end function
 
     function GetChannel(channelId as string, options = invalid as object) as object
-        cancellation = invalid
+        continuation = ""
         params = ""
+        cancellation = invalid
 
         if options <> invalid
-            if options.DoesExist("cancellation")
-                cancellation = options.cancellation
+            if options.DoesExist("continuation")
+                continuation = options.continuation
             end if
             if options.DoesExist("params")
                 params = options.params
             end if
+            if options.DoesExist("cancellation")
+                cancellation = options.cancellation
+            end if
         end if
 
-        request = Innertube.CreateBrowseRequest(channelId, Innertube.ClientType.Web, params)
+        request = Innertube.CreateBrowseRequest(channelId, Innertube.ClientType.Web, params, continuation)
         request.Cancellation(cancellation)
 
         response = request.Await()
@@ -191,9 +211,13 @@ namespace InnertubeService
     end function
 
     function GetPlaylist(playlistId as string, options = invalid as object) as object
+        continuation = ""
         cancellation = invalid
 
         if options <> invalid
+            if options.DoesExist("continuation")
+                continuation = options.continuation
+            end if
             if options.DoesExist("cancellation")
                 cancellation = options.cancellation
             end if
@@ -203,7 +227,7 @@ namespace InnertubeService
             playlistId = "VL" + playlistId
         end if
 
-        request = Innertube.CreateBrowseRequest(playlistId, Innertube.ClientType.Web, "")
+        request = Innertube.CreateBrowseRequest(playlistId, Innertube.ClientType.Web, "", continuation)
         request.Cancellation(cancellation)
 
         response = request.Await()

playlet-lib/src/components/Services/Innertube/NodesParser.bs

@@ -48,6 +48,7 @@ namespace Innertube
             "channelRenderer": ParseChannelRenderer
             "gridChannelRenderer": ParseGridChannelRenderer
             "playlistRenderer": ParsePlaylistRenderer
+            "gridPlaylistRenderer": ParseGridPlaylistRenderer
             "lockupViewModel": ParseLockupViewModel
             "shortsLockupViewModel": ParseShortsLockupViewModel
             "tvBrowseRenderer": ParseTvBrowseRenderer
@@ -58,7 +59,8 @@ namespace Innertube
             "gridRenderer": ParseGridRenderer
             "richGridRenderer": ParseRichGridRenderer
             "richItemRenderer": ParseRichItemRenderer
-            "continuationItemRenderer": ParseNotImplemented
+            "continuationItemRenderer": ParseContinuationItemRenderer
+            "appendContinuationItemsAction": ParseAppendContinuationItemsAction
             ' Posts not yet supported
             "postRenderer": ParseNotImplemented
             ' a "View all posts" button
@@ -67,7 +69,7 @@ namespace Innertube
             "horizontalCardListRenderer": ParseNotImplemented
             ' expandableTabRenderer contains the search tab with a channel. Ignored.
             "expandableTabRenderer": ParseNotImplemented
-
+            "clickTrackingParams": ParseNotImplemented
         }
     end function
 
@@ -468,6 +470,69 @@ namespace Innertube
         return invalid
     end function
 
+    function ParseGridPlaylistRenderer(nodeData as object, context as object) as object
+        playlistId = nodeData["playlistId"]
+        if not IsString(playlistId)
+            LogWarn("Invalid playlist ID", nodeData)
+            return invalid
+        end if
+
+        author = invalid
+        authorId = invalid
+        authorPossibleNodes = ["shortBylineText", "ownerText", "longBylineText"]
+        for each authorPossibleNode in authorPossibleNodes
+            node = nodeData[authorPossibleNode]
+            author = ParseText(node)
+            authorId = ObjectUtils.Dig(node, ["runs", 0, "navigationEndpoint", "browseEndpoint", "browseId"])
+            if author <> "" and authorId <> invalid
+                exit for
+            end if
+        end for
+        if StringUtils.IsNullOrEmpty(author) and StringUtils.IsNullOrEmpty(authorId)
+            if IsString(context.currentAuthor) and IsString(context.currentAuthorId)
+                author = context.currentAuthor
+                authorId = context.currentAuthorId
+            end if
+        end if
+
+        videoCountText = ""
+        thumbnailOverlays = nodeData["thumbnailOverlays"]
+        if IsArray(thumbnailOverlays)
+            for each overlay in thumbnailOverlays
+                videoCountText = ParseText(ObjectUtils.Dig(overlay, ["thumbnailOverlayBottomPanelRenderer", "text"]))
+                if videoCountText <> ""
+                    exit for
+                end if
+            end for
+        end if
+        if videoCountText = ""
+            videoCountText = ParseText(nodeData["videoCountText"])
+        end if
+        if videoCountText = ""
+            videoCountText = ParseText(nodeData["videoCountShortText"])
+        end if
+
+        videoCount = ParseText(nodeData["videoCount"])
+        if videoCount = ""
+            videoCount = ParseText(nodeData["videoCountShortText"])
+        end if
+        videoCount = videoCount.ToInt()
+
+        playlist = {
+            "type": "playlist"
+            "playlistId": playlistId
+            "title": ParseText(nodeData["title"])
+            "playlistThumbnail": ObjectUtils.Dig(nodeData, ["thumbnail", "thumbnails", 0, "url"])
+            "author": author
+            "authorId": authorId
+            "videoCount": videoCount
+            "videoCountText": videoCountText
+        }
+
+        PushFeedItem(playlist, context)
+        return invalid
+    end function
+
     function ParseLockupViewModel(nodeData as object, context as object) as object
         contentType = ValidString(nodeData["contentType"])
         if contentType = "LOCKUP_CONTENT_TYPE_PLAYLIST"
@@ -717,6 +782,22 @@ namespace Innertube
         return nodeData["content"]
     end function
 
+    function ParseContinuationItemRenderer(nodeData as object, context as object) as object
+        continuation = ObjectUtils.Dig(nodeData, ["continuationEndpoint", "continuationCommand", "token"])
+        if IsString(continuation)
+            ' In Playlists there are 2 continuation items, and one of them is useless...
+            ' TODO:P1 figure this out
+            if StringUtils.IsNullOrEmpty(context.currentFeed.continuation)
+                context.currentFeed.continuation = continuation
+            end if
+        end if
+        return invalid
+    end function
+
+    function ParseAppendContinuationItemsAction(nodeData as object, context as object) as object
+        return nodeData["continuationItems"]
+    end function
+
     function ParseText(data as object) as string
         if data = invalid
             return ""

playlet-lib/src/components/Services/Innertube/Parser.bs

@@ -9,8 +9,8 @@ namespace Innertube
             return []
         end if
 
-        data = UnwrapContainer(responseData)
-        if data = invalid
+        containers = GetContainers(responseData)
+        if containers.Count() = 0
             return []
         end if
 
@@ -29,7 +29,7 @@ namespace Innertube
             generateVideoThumbnails: true
         })
 
-        Innertube.ParseNode(data, context)
+        Innertube.ParseNode(containers, context)
 
         if context.currentFeed <> invalid and (context.currentFeed.items.Count() > 0 or context.currentFeed.title <> "")
             context.feeds.Push(context.currentFeed)
@@ -127,12 +127,16 @@ namespace Innertube
         end if
 
         items = []
+        continuation = invalid
         feeds = ParseResponse(responseData, { "currentAuthor": author, "currentAuthorId": authorId })
         if feeds.Count() > 0
             items = ValidArray(feeds[0].items)
+            if feeds[0].DoesExist("continuation")
+                continuation = feeds[0].continuation
+            end if
         end if
 
-        return {
+        channel = {
             "author": author
             "authorId": authorId
             "authorThumbnails": thumbnails
@@ -142,6 +146,12 @@ namespace Innertube
             "tabsParams": tabsParams
             "items": items
         }
+
+        if continuation <> invalid
+            channel["continuation"] = continuation
+        end if
+
+        return channel
     end function
 
     function ParsePlaylistPageResponse(responseData as object) as object
@@ -198,23 +208,21 @@ namespace Innertube
         feeds = ParseResponse(responseData)
         if feeds.Count() > 0
             playlist["videos"] = ValidArray(feeds[0].items)
+            if feeds[0].DoesExist("continuation")
+                playlist["continuation"] = feeds[0].continuation
+            end if
         end if
 
         return playlist
     end function
 
-    function UnwrapContainer(responseData as object) as object
-        if responseData.DoesExist("contents")
-            return responseData["contents"]
-        end if
-        if responseData.DoesExist("response")
-            return responseData["response"]
-        end if
-        data = ObjectUtils.Dig(responseData, ["onResponseReceivedActions", 0])
-        if data <> invalid
-            return data
-        end if
-        return responseData
+    function GetContainers(responseData as object) as object
+        return [
+            responseData["contents"]
+            responseData["response"]
+            responseData["onResponseReceivedActions"]
+            responseData["onResponseReceivedCommands"]
+        ]
     end function
 
 end namespace

playlet-lib/src/components/Services/Innertube/Search.bs

@@ -3,18 +3,22 @@ import "pkg:/components/Services/Innertube/Parser.bs"
 
 namespace Innertube
 
-    function Search(query as string, searchFilters as object, cancellation = invalid as object) as object
+    function Search(query as string, searchFilters as object, continuation = "" as string, cancellation = invalid as object) as object
         deviceInfo = CreateObject("roDeviceInfo")
 
         payload = {
-            query: query
             context: Innertube.CreateContext(Innertube.ClientType.WEB, deviceInfo, "")
         }
 
-        if searchFilters <> invalid and searchFilters.Count() > 0
-            params = Innertube.GetEncodedSearchFilters(searchFilters)
-            if params <> ""
-                payload["params"] = Innertube.GetEncodedSearchFilters(searchFilters)
+        if continuation <> ""
+            payload["continuation"] = continuation
+        else
+            payload["query"] = query
+            if searchFilters <> invalid and searchFilters.Count() > 0
+                params = Innertube.GetEncodedSearchFilters(searchFilters)
+                if params <> ""
+                    payload["params"] = Innertube.GetEncodedSearchFilters(searchFilters)
+                end if
             end if
         end if