From 8203196145c30587aa83e0d22c9d1c915356e0c9 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Wed, 18 Oct 2023 02:33:29 +0200 Subject: [PATCH 01/69] [ImgsedBridge] More robust data parsing (#3766) Date Interval with the article "an" or "a" are now handled in a generic way : every "article" is replaced by the number "1" instead of a handling of multiple special case --- bridges/ImgsedBridge.php | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index e605cf4fbb2..12466c6b03b 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -206,14 +206,12 @@ private function parseDate($content) { // Parse date, and transform the date into a timetamp, even in a case of a relative date $date = date_create(); - $dateString = str_replace(' ago', '', $content); - // Special case : 'a day' is not a valid interval in PHP, so replace it with it's PHP equivalenbt : '1 day' - if ($dateString == 'a day') { - $dateString = '1 day'; - } - if ($dateString === 'an hour') { - $dateString = '1 hour'; - } + + // Content trimmed to be sure that the "article" is at the beginning of the string and remove "ago" to make it a valid PHP date interval + $dateString = trim(str_replace(' ago', '', $content)); + + // Replace the article "an" or "a" by the number "1" to be a valid PHP date interval + $dateString = preg_replace('/^((an|a) )/m', '1 ', $dateString); $relativeDate = date_interval_create_from_date_string($dateString); if ($relativeDate) { From a41bb088f816454d9fbd07011e1365d0845fde3d Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 18 Oct 2023 19:10:52 +0200 Subject: [PATCH 02/69] [CssSelectorBridge] Add more metadata tags (#3768) Add og: variants for published/updated time and author --- bridges/CssSelectorBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index f6ab8d15588..8fba52858ef 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -336,9 +336,11 @@ protected function entryHtmlRetrieveMetadata($entry_html) ], 'timestamp' => [ 'article:published_time', + 'og:article:published_time', 'releaseDate', 'releasedate', 'article:modified_time', + 'og:article:modified_time', 'lastModified', 'lastmodified' ], @@ -351,8 +353,9 @@ protected function entryHtmlRetrieveMetadata($entry_html) 'thumbnailimg' ], 'author' => [ - 'author', 'article:author', + 'og:article:author', + 'author', 'article:author:username', 'profile:first_name', 'profile:last_name', From 7533ef12e3348779b1be2a1aa1704f972d3176a3 Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 18 Oct 2023 19:12:19 +0200 Subject: [PATCH 03/69] [html] improve srcset attribute parsing (#3769) Fix commas not being used for splitting, resulting in broken src URL in some cases: srcset="url1.jpg, url2.jpg 2x" would give src="url1.jpg," --- lib/html.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/html.php b/lib/html.php index 505221fc1aa..ba8067a627e 100644 --- a/lib/html.php +++ b/lib/html.php @@ -244,16 +244,26 @@ function convertLazyLoading($dom) $dom = str_get_html($dom); } + // Retrieve image URL from srcset attribute + // https://developer.mozilla.org/en-US/docs/Web/API/HTMLImageElement/srcset + // Example: convert "header640.png 640w, header960.png 960w, header1024.png 1024w" to "header1024.png" + $srcset_to_src = function ($srcset) { + $sources = explode(',', $srcset); + $last_entry = trim(end($sources)); + $url = explode(' ', $last_entry)[0]; + return $url; + }; + // Process standalone images, embeds and picture sources foreach ($dom->find('img, iframe, source') as $img) { if (!empty($img->getAttribute('data-src'))) { $img->src = $img->getAttribute('data-src'); } elseif (!empty($img->getAttribute('data-srcset'))) { - $img->src = explode(' ', $img->getAttribute('data-srcset'))[0]; + $img->src = $srcset_to_src($img->getAttribute('data-srcset')); } elseif (!empty($img->getAttribute('data-lazy-src'))) { $img->src = $img->getAttribute('data-lazy-src'); } elseif (!empty($img->getAttribute('srcset'))) { - $img->src = explode(' ', $img->getAttribute('srcset'))[0]; + $img->src = $srcset_to_src($img->getAttribute('srcset')); } else { continue; // Proceed to next element without removing attributes } From 9056106c2d52991931ba7fc1c41494697396d477 Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 18 Oct 2023 19:13:33 +0200 Subject: [PATCH 04/69] [CNet] Rewrite bridge (#3764) (#3770) Bridge was broken. Full bridge rewrite using Sitemap as source. --- bridges/CNETBridge.php | 160 +++++++++++++++++++------------------- bridges/SitemapBridge.php | 6 +- 2 files changed, 83 insertions(+), 83 deletions(-) diff --git a/bridges/CNETBridge.php b/bridges/CNETBridge.php index 34442abda8f..4a63c84773c 100644 --- a/bridges/CNETBridge.php +++ b/bridges/CNETBridge.php @@ -1,6 +1,6 @@ 'list', 'values' => [ 'All articles' => '', - 'Apple' => 'apple', - 'Google' => 'google', - 'Microsoft' => 'tags-microsoft', - 'Computers' => 'topics-computers', - 'Mobile' => 'topics-mobile', - 'Sci-Tech' => 'topics-sci-tech', - 'Security' => 'topics-security', - 'Internet' => 'topics-internet', - 'Tech Industry' => 'topics-tech-industry' + 'Tech' => 'tech', + 'Money' => 'personal-finance', + 'Home' => 'home', + 'Wellness' => 'health', + 'Energy' => 'home/energy-and-utilities', + 'Deals' => 'deals', + 'Computing' => 'tech/computing', + 'Mobile' => 'tech/mobile', + 'Science' => 'science', + 'Services' => 'tech/services-and-software' ] - ] + ], + 'limit' => self::LIMIT ] ]; - private function cleanArticle($article_html) - { - $offset_p = strpos($article_html, '

'); - $offset_figure = strpos($article_html, '', '', $article_html); - $article_html = str_replace('', '', $article_html); - $article_html = StripWithDelimiters($article_html, ''); - $article_html = stripWithDelimiters($article_html, 'innertext, 'ImageObject","url":"', '"'); + if ($imageObject !== false) { + $enclosure = $imageObject; + } + } - if (is_null($article_thumbnail)) { - $article_thumbnail = extractFromDelimiters($element->innertext, 'find('div.c-shortcodeGallery') as $cleanup) { + $cleanup->outertext = ''; } - if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, self::URI . 'news/') !== false) { - $article_html = getSimpleHTMLDOMCached($article_uri) or $article_html = null; - - if (!is_null($article_html)) { - if (empty($article_thumbnail)) { - $article_thumbnail = $article_html->find('div.originalImage', 0); - } - if (empty($article_thumbnail)) { - $article_thumbnail = $article_html->find('span.imageContainer', 0); - } - if (is_object($article_thumbnail)) { - $article_thumbnail = $article_thumbnail->find('img', 0)->src; - } - - $article_content .= trim( - $this->cleanArticle( - extractFromDelimiters( - $article_html, - 'find('figure') as $figure) { + $img = $figure->find('img', 0); + if ($img) { + $figure->outertext = $img->outertext; } + } + + $content = $content->innertext; + + if ($enclosure) { + $content = "

" . $content; + } + + if ($headline) { + $content = '

' . $headline->plaintext . '


' . $content; + } + + $item = []; + $item['uri'] = $article_uri; + $item['title'] = $title; + $item['author'] = $author; + $item['content'] = $content; - $item = []; - $item['uri'] = $article_uri; - $item['title'] = $article_title; - $item['author'] = $article_author; - $item['timestamp'] = $article_timestamp; - $item['enclosures'] = [$article_thumbnail]; - $item['content'] = $article_content; - $this->items[] = $item; + if (!is_null($date)) { + $item['timestamp'] = $date; } + + if (!is_null($enclosure)) { + $item['enclosures'] = [$enclosure]; + } + + $this->items[] = $item; } } } diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php index bdf662eedd7..bbbb3e16616 100644 --- a/bridges/SitemapBridge.php +++ b/bridges/SitemapBridge.php @@ -131,7 +131,7 @@ protected function sitemapXmlToList($sitemap, $url_pattern = '', $limit = 0, $ke foreach ($sitemap->find('sitemap') as $nested_sitemap) { $url = $nested_sitemap->find('loc'); if (!empty($url)) { - $url = $url[0]->plaintext; + $url = trim($url[0]->plaintext); if (str_ends_with(strtolower($url), '.xml')) { $nested_sitemap_xml = $this->getSitemapXml($url, true); $nested_sitemap_links = $this->sitemapXmlToList($nested_sitemap_xml, $url_pattern, null, true); @@ -148,8 +148,8 @@ protected function sitemapXmlToList($sitemap, $url_pattern = '', $limit = 0, $ke $url = $item->find('loc'); $lastmod = $item->find('lastmod'); if (!empty($url) && !empty($lastmod)) { - $url = $url[0]->plaintext; - $lastmod = $lastmod[0]->plaintext; + $url = trim($url[0]->plaintext); + $lastmod = trim($lastmod[0]->plaintext); $timestamp = strtotime($lastmod); if (empty($url_pattern) || preg_match('/' . $url_pattern . '/', $url) === 1) { $links[$url] = $timestamp; From 658391263ebdbd8614cefeb63eb43e3ea521e5b6 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Thu, 19 Oct 2023 18:02:53 +0300 Subject: [PATCH 05/69] Add 'itunes:duration' tag for items with duration (#3774) * [{Atom,Mrss}Format] Allow itunes tags on items without enclosure * [Arte7Bridge] Add $item['itunes']['duration'] value --- bridges/Arte7Bridge.php | 4 ++++ formats/AtomFormat.php | 12 +++++++----- formats/MrssFormat.php | 12 +++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/bridges/Arte7Bridge.php b/bridges/Arte7Bridge.php index 239fc6ad838..5898e881d49 100644 --- a/bridges/Arte7Bridge.php +++ b/bridges/Arte7Bridge.php @@ -156,6 +156,10 @@ public function collectData() . $element['mainImage']['url'] . '" />'; + $item['itunes'] = [ + 'duration' => $durationSeconds, + ]; + $this->items[] = $item; } } diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index d59e42fea05..07ca7272f5d 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -147,11 +147,13 @@ public function stringify() $entry->appendChild($itunesProperty); $itunesProperty->appendChild($document->createTextNode($itunesValue)); } - $itunesEnclosure = $document->createElement('enclosure'); - $entry->appendChild($itunesEnclosure); - $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); - $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); - $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + if (isset($itemArray['enclosure'])) { + $itunesEnclosure = $document->createElement('enclosure'); + $entry->appendChild($itunesEnclosure); + $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); + $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); + $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + } } elseif (!empty($entryUri)) { $entryLinkAlternate = $document->createElement('link'); $entry->appendChild($entryLinkAlternate); diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 4fd06439775..5b96a6a75a6 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -135,11 +135,13 @@ public function stringify() $entry->appendChild($itunesProperty); $itunesProperty->appendChild($document->createTextNode($itunesValue)); } - $itunesEnclosure = $document->createElement('enclosure'); - $entry->appendChild($itunesEnclosure); - $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); - $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); - $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + if (isset($itemArray['enclosure'])) { + $itunesEnclosure = $document->createElement('enclosure'); + $entry->appendChild($itunesEnclosure); + $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); + $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); + $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + } } if (!empty($itemUri)) { $entryLink = $document->createElement('link'); $entry->appendChild($entryLink); From 8ff39f64f7554de110cb4296e199789d9fa7f5a8 Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 20 Oct 2023 13:31:52 +0200 Subject: [PATCH 06/69] [html] add data-orig-file tag (#3777) Add support for data-orig-file tag in convertLazyLoading() Remplace end() with array_key_last() as discussed in #3769 Fix typo in comment --- lib/html.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/html.php b/lib/html.php index ba8067a627e..1de581d9c95 100644 --- a/lib/html.php +++ b/lib/html.php @@ -249,7 +249,7 @@ function convertLazyLoading($dom) // Example: convert "header640.png 640w, header960.png 960w, header1024.png 1024w" to "header1024.png" $srcset_to_src = function ($srcset) { $sources = explode(',', $srcset); - $last_entry = trim(end($sources)); + $last_entry = trim($sources[array_key_last($sources)]); $url = explode(' ', $last_entry)[0]; return $url; }; @@ -262,12 +262,15 @@ function convertLazyLoading($dom) $img->src = $srcset_to_src($img->getAttribute('data-srcset')); } elseif (!empty($img->getAttribute('data-lazy-src'))) { $img->src = $img->getAttribute('data-lazy-src'); + } elseif (!empty($img->getAttribute('data-orig-file'))) { + $img->src = $img->getAttribute('data-orig-file'); } elseif (!empty($img->getAttribute('srcset'))) { $img->src = $srcset_to_src($img->getAttribute('srcset')); } else { continue; // Proceed to next element without removing attributes } - foreach (['loading', 'decoding', 'srcset', 'data-src', 'data-srcset'] as $attr) { + // Remove attributes that may be processed by the client (data-* are not) + foreach (['loading', 'decoding', 'srcset'] as $attr) { if ($img->hasAttribute($attr)) { $img->removeAttribute($attr); } @@ -284,7 +287,7 @@ function convertLazyLoading($dom) $img->tag = 'img'; } // Adding/removing node would change its position inside the parent element, - // So instead we rewrite the node in-place though the outertext attribute + // So instead we rewrite the node in-place through the outertext attribute $picture->outertext = $img->outertext; } } From 4f7451895bf888e465a9879c466d4067a8c4a17f Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 20 Oct 2023 13:33:07 +0200 Subject: [PATCH 07/69] Fix: content.php: last-modified/if-unmodified-since (#3771) (#3772) * Fix: content.php: last-modified/if-unmodified-since (#3771) Fix exception if server sent invalid Last-Modified header Add support for Unix time instead of standard date string Send back standard RFC7231 date string instead of Unix time * Fix: content.php: if-unmodified-since: cURL API Use getTimestamp() as cURL expects that and will format the If-Modified-Since header appropriately. --- lib/contents.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/contents.php b/lib/contents.php index a3830ca713f..055d6bf3188 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -63,8 +63,13 @@ function getContents( if ($cachedResponse) { $cachedLastModified = $cachedResponse->getHeader('last-modified'); if ($cachedLastModified) { - $cachedLastModified = new \DateTimeImmutable($cachedLastModified); - $config['if_not_modified_since'] = $cachedLastModified->getTimestamp(); + try { + // Some servers send Unix timestamp instead of RFC7231 date. Prepend it with @ to allow parsing as DateTime + $cachedLastModified = new \DateTimeImmutable((is_numeric($cachedLastModified) ? '@' : '') . $cachedLastModified); + $config['if_not_modified_since'] = $cachedLastModified->getTimestamp(); + } catch (Exception $dateTimeParseFailue) { + // Ignore invalid 'Last-Modified' HTTP header value + } } } From 4722201281f3e62c3f8067d2721e4b771a6ba5e0 Mon Sep 17 00:00:00 2001 From: Manu <3916435+m3nu@users.noreply.github.com> Date: Fri, 20 Oct 2023 19:29:28 +0100 Subject: [PATCH 08/69] Add one-click install to PikaPods (#3778) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7037095ec19..570fb87d89a 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ Browse http://localhost:3000/ [![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge) [![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) [![Deploy to Cloudron](https://cloudron.io/img/button.svg)](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html) +[![Run on PikaPods](https://www.pikapods.com/static/run-button.svg)](https://www.pikapods.com/pods?run=rssbridge) The Heroku quick deploy currently does not work. It might possibly work if you fork this repo and modify the `repository` in `scalingo.json`. See https://github.com/RSS-Bridge/rss-bridge/issues/2688 From a6a450220991a42f34b3dfa3219b0cafbaba11eb Mon Sep 17 00:00:00 2001 From: mruac Date: Sat, 21 Oct 2023 20:24:50 +1030 Subject: [PATCH 09/69] [Itaku] extend the number of images shown in a post (#3780) * minor fixes - extended itaku post if post does not have all images * phpcbf * . * resolve deprecated explode param yay null coalesces --- bridges/ItakuBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 62a130ff8b6..149757f5c4e 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -201,7 +201,7 @@ public function collectData() 'rating_e' => $this->getInput('rating_e') ]; - $tag_arr = explode(' ', $this->getInput('tags')); + $tag_arr = explode(' ', $this->getInput('tags') ?? ''); foreach ($tag_arr as $str) { switch ($str[0]) { case '-': @@ -446,6 +446,9 @@ private function getOwnerID($username) private function getPost($id, array $metadata = null) { + if (isset($metadata) && sizeof($metadata['gallery_images']) < $metadata['num_images']) { + $metadata = null; //force re-fetch of metadata + } $uri = self::URI . '/posts/' . $id; $url = self::URI . '/api/posts/' . $id . '/?format=json'; $data = $metadata ?? $this->getData($url, true, true) From f134808a268065e5000ef694149f62bb0f263b16 Mon Sep 17 00:00:00 2001 From: Park0 Date: Sun, 22 Oct 2023 17:36:36 +0200 Subject: [PATCH 10/69] Marktplaats categories added (#3761) * Update MarktplaatsBridge.php * Update MarktplaatsBridge.php only main categories As the whole list is too big only main categories are used for now. * Renamed parameter 2 to sc Renamed unused method to better reflect it usage * Update MarktplaatsBridge.php Several fixed Categories completed Added a default empty one Check if the input is not empty before using Added helper methods to generate the categorylist * Update MarktplaatsBridge.php Set the methods to private for the CI --- bridges/MarktplaatsBridge.php | 143 +++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/bridges/MarktplaatsBridge.php b/bridges/MarktplaatsBridge.php index 70a369d9542..6ba993e7885 100644 --- a/bridges/MarktplaatsBridge.php +++ b/bridges/MarktplaatsBridge.php @@ -14,6 +14,51 @@ class MarktplaatsBridge extends BridgeAbstract 'required' => true, 'title' => 'The search string for marktplaats', ], + 'c' => [ + 'name' => 'Category', + 'type' => 'list', + 'values' => [ + 'Select a category' => '', + 'Antiek en Kunst' => '1', + 'Audio, Tv en Foto' => '31', + 'Auto's' => '91', + 'Auto-onderdelen' => '2600', + 'Auto diversen' => '48', + 'Boeken' => '201', + 'Caravans en Kamperen' => '289', + 'Cd's en Dvd's' => '1744', + 'Computers en Software' => '322', + 'Contacten en Berichten' => '378', + 'Diensten en Vakmensen' => '1098', + 'Dieren en Toebehoren' => '395', + 'Doe-het-zelf en Verbouw' => '239', + 'Fietsen en Brommers' => '445', + 'Hobby en Vrije tijd' => '1099', + 'Huis en Inrichting' => '504', + 'Huizen en Kamers' => '1032', + 'Kinderen en Baby's' => '565', + 'Kleding | Dames' => '621', + 'Kleding | Heren' => '1776', + 'Motoren' => '678', + 'Muziek en Instrumenten' => '728', + 'Postzegels en Munten' => '1784', + 'Sieraden, Tassen en Uiterlijk' => '1826', + 'Spelcomputers en Games' => '356', + 'Sport en Fitness' => '784', + 'Telecommunicatie' => '820', + 'Tickets en Kaartjes' => '1984', + 'Tuin en Terras' => '1847', + 'Vacatures' => '167', + 'Vakantie' => '856', + 'Verzamelen' => '895', + 'Watersport en Boten' => '976', + 'Witgoed en Apparatuur' => '537', + 'Zakelijke goederen' => '1085', + 'Diversen' => '428', + ], + 'required' => false, + 'title' => 'The category to search in', + ], 'z' => [ 'name' => 'zipcode', 'type' => 'text', @@ -57,7 +102,15 @@ class MarktplaatsBridge extends BridgeAbstract 'type' => 'checkbox', 'required' => false, 'title' => 'Include the raw data behind the content', - ] + ], + 'sc' => [ + 'name' => 'Sub category', + 'type' => 'number', + 'required' => false, + 'exampleValue' => '12345', + 'title' => 'Sub category has to be given by id as the list is too big to show here. + Only use subcategories that belong to the main category. Both have to be correct', + ], ] ]; const CACHE_TIMEOUT = 900; @@ -80,6 +133,12 @@ public function collectData() $excludeGlobal = true; } } + if (!empty($this->getInput('c'))) { + $query .= '&l1CategoryId=' . $this->getInput('c'); + } + if (!is_null($this->getInput('sc'))) { + $query .= '&l2CategoryId=' . $this->getInput('sc'); + } $url = 'https://www.marktplaats.nl/lrp/api/search?query=' . urlencode($this->getInput('q')) . $query; $jsonString = getSimpleHTMLDOM($url); $jsonObj = json_decode($jsonString); @@ -97,15 +156,15 @@ public function collectData() $item['enclosures'] = $listing->imageUrls; if (is_array($listing->imageUrls)) { foreach ($listing->imageUrls as $imgurl) { - $item['content'] .= "
\n"; + $item['content'] .= "
\n"; } } else { - $item['content'] .= "
\n"; + $item['content'] .= "
\n"; } } if (!is_null($this->getInput('r'))) { if ($this->getInput('r')) { - $item['content'] .= "
\n
\n
\n" . json_encode($listing); + $item['content'] .= "
\n
\n
\n" . json_encode($listing) . "
$url"; } } $item['content'] .= "
\n
\nPrice: " . $listing->priceInfo->priceCents / 100; @@ -130,4 +189,80 @@ public function getName() } return parent::getName(); } + + /** + * Method can be used to scrape the subcategories from marktplaats + */ + private static function scrapeSubCategories() + { + $main = []; + $main['Select a category'] = ''; + $marktplaatsHTML = file_get_html('https://www.marktplaats.nl'); + foreach ($marktplaatsHTML->find('select[id=categoryId] option') as $opt) { + if (!str_contains($opt->innertext, 'categorie')) { + $main[$opt->innertext] = $opt->value; + $ids[] = $opt->value; + } + } + + $result = []; + foreach ($ids as $id) { + $url = 'https://www.marktplaats.nl/lrp/api/search?l1CategoryId=' . $id; + $jsonstring = getContents($url); + $jsondata = json_decode((string)$jsonstring); + if (isset($jsondata->searchCategoryOptions)) { + $categories = $jsondata->searchCategoryOptions; + if (isset($jsondata->categoriesById->$id)) { + $maincategory = $jsondata->categoriesById->$id; + $array = []; + foreach ($categories as $categorie) { + $array[$categorie->fullName] = $categorie->id; + } + $result[$maincategory->fullName] = $array; + } + } else { + print($jsonstring); + } + } + $combinedResult = [ + 'main' => $main, + 'sub' => $result + ]; + return $combinedResult; + } + + /** + * Helper method to construct the array that could be used for categories + * + * @param $array + * @param $indent + * @return void + */ + private static function printArrayAsCode($array, $indent = 0) + { + foreach ($array as $key => $value) { + if (is_array($value)) { + echo str_repeat(' ', $indent) . "'$key' => [" . PHP_EOL; + self::printArrayAsCode($value, $indent + 1); + echo str_repeat(' ', $indent) . '],' . PHP_EOL; + } else { + $value = str_replace('\'', '\\\'', $value); + $key = str_replace('\'', '\\\'', $key); + echo str_repeat(' ', $indent) . "'$key' => '$value'," . PHP_EOL; + } + } + } + + private static function printScrapeArray() + { + $array = (MarktplaatsBridge::scrapeSubCategories()); + + echo '$myArray = [' . PHP_EOL; + self::printArrayAsCode($array['main'], 1); + echo '];' . PHP_EOL; + + echo '$myArray = [' . PHP_EOL; + self::printArrayAsCode($array['sub'], 1); + echo '];' . PHP_EOL; + } } From d4e4c3e89ac148d4943c8c1669322649e1d0ae7b Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Mon, 23 Oct 2023 17:12:05 -0400 Subject: [PATCH 11/69] [FarsideNitterBridge] New twitter bridge (#3781) * [FarsideNitterBridge] New twitter bridge * example value * lint fix --- bridges/FarsideNitterBridge.php | 103 ++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 bridges/FarsideNitterBridge.php diff --git a/bridges/FarsideNitterBridge.php b/bridges/FarsideNitterBridge.php new file mode 100644 index 00000000000..b167347acf8 --- /dev/null +++ b/bridges/FarsideNitterBridge.php @@ -0,0 +1,103 @@ + [ + 'name' => 'username', + 'required' => true, + 'exampleValue' => 'NASA' + ], + 'noreply' => [ + 'name' => 'Without replies', + 'type' => 'checkbox', + 'title' => 'Only return initial tweets' + ], + 'noretweet' => [ + 'name' => 'Without retweets', + 'required' => false, + 'type' => 'checkbox', + 'title' => 'Hide retweets' + ], + 'linkbacktotwitter' => [ + 'name' => 'Link back to twitter', + 'required' => false, + 'type' => 'checkbox', + 'title' => 'Rewrite links back to twitter.com' + ] + ], + ]; + + public function detectParameters($url) + { + if (preg_match('/^(https?:\/\/)?(www\.)?(nitter\.net|twitter\.com)\/([^\/?\n]+)/', $url, $matches) > 0) { + return [ + 'username' => $matches[4], + 'noreply' => true, + 'noretweet' => true, + 'linkbacktotwitter' => true + ]; + } + return null; + } + + public function collectData() + { + $this->getRSS(); + } + + private function getRSS($attempt = 0) + { + try { + $this->collectExpandableDatas(self::URI . $this->getInput('username') . '/rss'); + } catch (\Exception $e) { + if ($attempt >= self::MAX_RETRIES) { + throw $e; + } else { + $this->getRSS($attempt++); + } + } + } + + protected function parseItem(array $item) + { + if ($this->getInput('noreply') && substr($item['title'], 0, 5) == 'R to ') { + return; + } + if ($this->getInput('noretweet') && substr($item['title'], 0, 6) == 'RT by ') { + return; + } + $item['title'] = truncate($item['title']); + if (preg_match('/(\/status\/.+)/', $item['uri'], $matches) > 0) { + if ($this->getInput('linkbacktotwitter')) { + $item['uri'] = self::HOST . $this->getInput('username') . $matches[1]; + } else { + $item['uri'] = self::URI . $this->getInput('username') . $matches[1]; + } + } + return $item; + } + + public function getName() + { + if (preg_match('/(.+) \//', parent::getName(), $matches) > 0) { + return $matches[1]; + } + return parent::getName(); + } + + public function getURI() + { + if ($this->getInput('linkbacktotwitter')) { + return self::HOST . $this->getInput('username'); + } else { + return self::URI . $this->getInput('username'); + } + } +} From cee25d862d71a1f2484135f31b0bbbd5017fcd8d Mon Sep 17 00:00:00 2001 From: ORelio Date: Tue, 24 Oct 2023 19:57:25 +0200 Subject: [PATCH 12/69] [html] clean data attributes (#3782) Some feed readers had difficulties with attributes containing html tags --- lib/html.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/html.php b/lib/html.php index 1de581d9c95..d65d1b20440 100644 --- a/lib/html.php +++ b/lib/html.php @@ -269,7 +269,15 @@ function convertLazyLoading($dom) } else { continue; // Proceed to next element without removing attributes } - // Remove attributes that may be processed by the client (data-* are not) + + // Remove data attributes, no longer necessary + foreach ($img->getAllAttributes() as $attr => $val) { + if (str_starts_with($attr, 'data-')) { + $img->removeAttribute($attr); + } + } + + // Remove other attributes that may be processed by the client foreach (['loading', 'decoding', 'srcset'] as $attr) { if ($img->hasAttribute($attr)) { $img->removeAttribute($attr); From 1dabd10e25ca6ac1ab7cd19742707380104d2642 Mon Sep 17 00:00:00 2001 From: Niehztog Date: Mon, 30 Oct 2023 11:47:25 +0100 Subject: [PATCH 13/69] [NintendoBridge] Add new bridge (#3784) * Adds new NintendoBridge * fix item uids, fix feed title * fix feed icon, adds item categories * fix feed source uri * make currentCatgory property nullable * fix linter errors * fix linter errors * attempt to fix unit tests by assigning default category --- bridges/NintendoBridge.php | 466 +++++++++++++++++++++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 bridges/NintendoBridge.php diff --git a/bridges/NintendoBridge.php b/bridges/NintendoBridge.php new file mode 100644 index 00000000000..3455073776d --- /dev/null +++ b/bridges/NintendoBridge.php @@ -0,0 +1,466 @@ + [ + 'category' => [ + 'name' => 'Category', + 'type' => 'list', + 'values' => [ + 'All' => 'all', + 'Mario Kart 8 Deluxe' => 'mk8d', + 'Splatoon 2' => 's2', + 'Super Mario 3D All-Stars' => 'sm3as', + 'Super Mario 3D World + Bowser’s Fury' => 'sm3wbf', + 'Super Mario Maker 2' => 'smm2', + 'Super Mario Odyssey' => 'smo', + 'Super Smash Bros. Ultimate' => 'ssbu', + 'Switch Firmware' => 'sf', + 'The Legend of Zelda: Link’s Awakening' => 'tlozla', + 'The Legend of Zelda: Skyward Sword HD' => 'tlozss', + 'The Legend of Zelda: Tears of the Kingdom' => 'tloztotk', + 'Xenoblade Chronicles 2' => 'xc2', + ], + 'defaultValue' => 'mk8d', + 'title' => 'Select category' + ], + 'country' => [ + 'name' => 'Country', + 'type' => 'list', + 'values' => [ + 'België' => 'be/nl', + 'Belgique' => 'be/fr', + 'Deutschland' => 'de', + 'España' => 'es', + 'France' => 'fr', + 'Italia' => 'it', + 'Nederland' => 'nl', + 'Österreich' => 'at', + 'Portugal' => 'pt', + 'Schweiz' => 'ch/de', + 'Suisse' => 'ch/fr', + 'Svizzera' => 'ch/it', + 'UK & Ireland' => 'co.uk', + 'South Africa' => 'co.za' + ], + 'defaultValue' => 'co.uk', + 'title' => 'Select your country' + ] + ] + ]; + + const CACHE_TIMEOUT = 3600; + + const FEED_SOURCE_URL = [ + 'mk8d' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Mario-Kart-8-Deluxe-1482895.html', + 's2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Splatoon-2-1482897.html', + 'sm3as' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-All-Stars-1844226.html', + 'sm3wbf' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-World-Bowser-s-Fury-1920668.html', + 'smm2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Maker-2-1586745.html', + 'smo' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Odyssey-1482901.html', + 'ssbu' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Smash-Bros-Ultimate-1484130.html', + 'sf' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/System-Updates/Nintendo-Switch-System-Updates-and-Change-History-1445507.html', + 'tlozla' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-The-Legend-of-Zelda-Link-s-Awakening-1666739.html', + 'tlozss' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-The-Legend-of-Zelda-Skyward-Sword-HD-2022801.html', + 'tloztotk' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-The-Legend-of-Zelda-Tears-of-the-Kingdom-2388231.html', + 'xc2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/Xenoblade-Chronicles-2-Update-History-1482911.html', + ]; + const XPATH_EXPRESSION_ITEM = '//div[@class="col-xs-12 content"]/div[starts-with(@id,"v") and @class="collapse"]'; + const XPATH_EXPRESSION_ITEM_FIRMWARE = '//div[@id="latest" and @class="collapse" and @rel="1"]'; + const XPATH_EXPRESSION_ITEM_TITLE = './/h2[1]/node()'; + const XPATH_EXPRESSION_ITEM_CONTENT = '.'; + const XPATH_EXPRESSION_ITEM_URI = '//link[@rel="canonical"]/@href'; + + //const XPATH_EXPRESSION_ITEM_AUTHOR = ''; + const XPATH_EXPRESSION_ITEM_TIMESTAMP_PART = 'substring-after(//a[@class="collapse_link collapsed" and @data-target="#{{id_here}}"]/text(), "{{label_here}}")'; + const XPATH_EXPRESSION_ITEM_TIMESTAMP = 'substring(' . self::XPATH_EXPRESSION_ITEM_TIMESTAMP_PART . ', 1, string-length(' + . self::XPATH_EXPRESSION_ITEM_TIMESTAMP_PART . ') - 1)'; + + //const XPATH_EXPRESSION_ITEM_ENCLOSURES = ''; + //const XPATH_EXPRESSION_ITEM_CATEGORIES = ''; + const SETTING_FIX_ENCODING = false; + const SETTING_USE_RAW_ITEM_CONTENT = true; + + private const GAME_COUNTRY_DATE_SUBSTRING_PART = [ + 'mk8d' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'ubblicata il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 's2' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'ubblicata il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'sm3as' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ubliée le ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'sm3wbf' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'smm2' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ubliée le ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'smo' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'ssbu' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'sf' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ise en ligne le ', + 'it' => 'ubblicata il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'istributed ', + ], + 'tlozla' => [ + 'de' => 'eröffentlicht ', + 'es' => 'ublicada el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgegeven op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'tlozss' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'ubblicata l\'', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'tloztotk' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ubliée le ', + 'it' => 'ubblicata il ', + 'nl' => 'erschenen op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'xc2' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + ]; + + private const GAME_COUNTRY_DATE_FORMAT = [ + 'mk8d' => [ + 'de' => 'd.m.y', + 'es' => 'd-m-y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + 's2' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-Y', + 'fr' => 'd/m/y', + 'it' => 'd/m/y', + 'nl' => 'd/m/y', + 'pt' => 'd/m/y', + 'en' => 'd F Y', + ], + 'sm3as' => [ + 'de' => 'j. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'j m Y', + 'it' => 'j m Y', + 'nl' => 'j m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], + 'sm3wbf' => [ + 'de' => 'd.m.y', + 'es' => 'd-m-y', + 'fr' => 'd/m/y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'F j, Y', + ], + 'smm2' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-Y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/Y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + 'smo' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-Y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + 'ssbu' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'j m Y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'd/m/Y', + 'en' => 'j F Y', + ], + 'sf' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/Y', + 'nl' => 'd m Y', + 'pt' => 'd/m/Y', + 'en' => 'd/m/Y', + ], + 'tlozla' => [ + 'de' => 'd. m Y', + 'es' => 'j m \d\e Y', + 'fr' => 'd/m/y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F y', + ], + 'tlozss' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'd/m/y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], + 'tloztotk' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'j m Y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], + 'xc2' => [ + 'de' => 'd.m.y', + 'es' => 'd-m-y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + ]; + + private const FOREIGN_MONTH_NAMES = [ + 'nl' => ['01' => 'januari', '02' => 'februari', '03' => 'maart', '04' => 'april', '05' => 'mei', '06' => 'juni', '07' => 'juli', '08' => 'augustus', + '09' => 'september', '10' => 'oktober', '11' => 'november', '12' => 'december'], + 'fr' => ['01' => 'janvier', '02' => 'février', '03' => 'mars', '04' => 'avril', '05' => 'mai', '06' => 'juin', '07' => 'juillet', '08' => 'août', + '09' => 'septembre', '10' => 'octobre', '11' => 'novembre', '12' => 'décembre'], + 'de' => ['01' => 'Januar', '02' => 'Februar', '03' => 'März', '04' => 'April', '05' => 'Mai', '06' => 'Juni', '07' => 'Juli', '08' => 'August', + '09' => 'September', '10' => 'Oktober', '11' => 'November', '12' => 'Dezember'], + 'es' => ['01' => 'enero', '02' => 'febrero', '03' => 'marzo', '04' => 'abril', '05' => 'mayo', '06' => 'junio', '07' => 'julio', '08' => 'agosto', + '09' => 'septiembre', '10' => 'octubre', '11' => 'noviembre', '12' => 'diciembre'], + 'it' => ['01' => 'gennaio', '02' => 'febbraio', '03' => 'marzo', '04' => 'aprile', '05' => 'maggio', '06' => 'giugno', '07' => 'luglio', '08' => 'agosto', + '09' => 'settembre', '10' => 'ottobre', '11' => 'novembre', '12' => 'dicembre'], + 'pt' => ['01' => 'janeiro', '02' => 'fevereiro', '03' => 'março', '04' => 'abril', '05' => 'maio', '06' => 'junho', '07' => 'julho', '08' => 'agosto', + '09' => 'setembro', '10' => 'outubro', '11' => 'novembro', '12' => 'dezembro'], + ]; + const LANGUAGE_REWRITE = ['co.uk' => 'en', 'co.za' => 'en', 'at' => 'de']; + + private string $lastId = ''; + private ?string $currentCategory = ''; + + private function getCurrentCategory() + { + if (empty($this->currentCategory)) { + $category = $this->getInput('category'); + $this->currentCategory = empty($category) ? self::PARAMETERS['']['category']['defaultValue'] : $category; + } + return $this->currentCategory; + } + + public function getIcon() + { + return 'https://www.nintendo.co.uk/favicon.ico'; + } + + public function getURI() + { + $category = $this->getInput('category'); + return 'all' === $category ? self::URI : $this->getSourceUrl(); + } + + protected function provideFeedTitle(\DOMXPath $xpath) + { + $category = $this->getInput('category'); + $categoryName = array_search($category, self::PARAMETERS['']['category']['values']); + return 'all' === $category ? self::NAME : $categoryName . ' Software-Updates'; + } + + protected function getSourceUrl() + { + $country = $this->getInput('country'); + $category = $this->getCurrentCategory(); + return str_replace(self::PARAMETERS['']['country']['defaultValue'], $country, self::FEED_SOURCE_URL[$category]); + } + + protected function getExpressionItem() + { + $category = $this->getCurrentCategory(); + return 'sf' === $category ? self::XPATH_EXPRESSION_ITEM_FIRMWARE : self::XPATH_EXPRESSION_ITEM; + } + + protected function getExpressionItemTimestamp() + { + if (empty($this->lastId)) { + return null; + } + $country = $this->getInput('country'); + $category = $this->getCurrentCategory(); + $language = $this->getLanguageFromCountry($country); + return str_replace( + ['{{id_here}}', '{{label_here}}'], + [$this->lastId, static::GAME_COUNTRY_DATE_SUBSTRING_PART[$category][$language]], + static::XPATH_EXPRESSION_ITEM_TIMESTAMP + ); + } + + protected function getExpressionItemCategories() + { + $category = $this->getCurrentCategory(); + $categoryName = array_search($category, self::PARAMETERS['']['category']['values']); + return 'string("' . $categoryName . '")'; + } + + public function collectData() + { + $category = $this->getCurrentCategory(); + if ('all' === $category) { + $allItems = []; + foreach (self::PARAMETERS['']['category']['values'] as $catKey) { + if ('all' === $catKey) { + continue; + } + $this->currentCategory = $catKey; + $this->items = []; + parent::collectData(); + $allItems = [...$allItems, ...$this->items]; + } + $this->currentCategory = 'all'; + $this->items = $allItems; + } else { + parent::collectData(); + } + } + + protected function formatItemTitle($value) + { + if (false !== strpos($value, ' (')) { + $value = substr($value, 0, strpos($value, ' (')); + } + if ('all' === $this->getInput('category')) { + $category = $this->getCurrentCategory(); + $categoryName = array_search($category, self::PARAMETERS['']['category']['values']); + return $categoryName . ' ' . $value; + } + return $value; + } + + protected function formatItemContent($value) + { + $result = preg_match('~
(.*)
~', $value, $matches); + if (1 === $result) { + $this->lastId = $matches[1]; + return trim($matches[2]); + } + return $value; + } + + protected function formatItemTimestamp($value) + { + $country = $this->getInput('country'); + $category = $this->getCurrentCategory(); + $language = $this->getLanguageFromCountry($country); + + $aMonthNames = self::FOREIGN_MONTH_NAMES[$language] ?? null; + if (null !== $aMonthNames) { + $value = str_replace(array_values($aMonthNames), array_keys($aMonthNames), $value); + } + $value = str_replace('­', '-', $value); + $value = str_replace('--', '-', $value); + + $date = \DateTime::createFromFormat(self::GAME_COUNTRY_DATE_FORMAT[$category][$language], $value); + if (false === $date) { + $date = new \DateTime('now'); + } + return $date->getTimestamp(); + } + + protected function generateItemId(FeedItem $item) + { + return $this->getCurrentCategory() . '-' . $this->lastId; + } + + private function getLanguageFromCountry($country) + { + return (strpos($country, '/') !== false) ? substr($country, strpos($country, '/') + 1) : (self::LANGUAGE_REWRITE[$country] ?? $country); + } +} From 8d0ddb579fd4e15ec02ea17a11d4337b19fc2424 Mon Sep 17 00:00:00 2001 From: Evgeny <76707795+itsLameni@users.noreply.github.com> Date: Fri, 3 Nov 2023 16:42:34 +0000 Subject: [PATCH 14/69] Adding rss.m3wz.su instance to list of public hosts (#3789) --- docs/01_General/06_Public_Hosts.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index 9aa292a5744..de538cf15b2 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -20,6 +20,8 @@ | ![](https://iplookup.flagfox.net/images/h16/NL.png) | https://feed.eugenemolotov.ru | ![](https://img.shields.io/website/https/feed.eugenemolotov.ru.svg) | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss-bridge.mediani.de | ![](https://img.shields.io/website/https/rss-bridge.mediani.de.svg) | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany | | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | +| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud + ## Inactive instances From 84b5ffcc7c6ee42d323bb897f61e523127eb7595 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 7 Nov 2023 05:02:34 +0100 Subject: [PATCH 15/69] [PepperBridgeAbstract] Fix Deal Origin and Shipping cost (#3790) - Deal Origin was changed by the website : fixed the CSS class to get it - Shipping cost had an extra SVG image in the content : removed the whole HTML tags from the content --- bridges/PepperBridgeAbstract.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 5d2e552b844..875ed8f647c 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -356,11 +356,11 @@ private function getShippingCost($deal) if ($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0) != null) { if ($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1) != null) { return '
' . $this->i8n('shipping') . ' : ' - . $deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1)->innertext + . strip_tags($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1)->innertext) . '
'; } else { return '
' . $this->i8n('shipping') . ' : ' - . $deal->find('span[class*=text--color-greyShade flex--inline]', 0)->innertext + . strip_tags($deal->find('span[class*=text--color-greyShade flex--inline]', 0)->innertext) . '
'; } } else { @@ -376,7 +376,7 @@ private function getSource($deal) { if (($origin = $deal->find('button[class*=text--color-greyShade]', 0)) != null) { $path = str_replace(' ', '/', trim(Json::decode($origin->{'data-cloak-link'})['path'])); - $text = $origin->find('span[class*=cept-merchant-name]', 0); + $text = $origin->find('span[class*=link]', 0); return '
' . $this->i8n('origin') . ' : ' . $text . '
'; } else { return ''; From a6310cff1ab77ebb9be4b9aadbf3f6a007f2b09c Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 7 Nov 2023 21:32:46 +0100 Subject: [PATCH 16/69] [GreatFonBridge] Add new Instagram Viewer Bridge (#3791) Add a new Instagram Bridge not using Cloudflare DDoS Protection --- bridges/GreatFonBridge.php | 140 +++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 bridges/GreatFonBridge.php diff --git a/bridges/GreatFonBridge.php b/bridges/GreatFonBridge.php new file mode 100644 index 00000000000..2951634c15f --- /dev/null +++ b/bridges/GreatFonBridge.php @@ -0,0 +1,140 @@ + [ + 'u' => [ + 'name' => 'username', + 'type' => 'text', + 'title' => 'Instagram username you want to follow', + 'exampleValue' => 'aesoprockwins', + 'required' => true, + ], + ] + ]; + const TEST_DETECT_PARAMETERS = [ + 'https://www.instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], + 'https://instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], + 'https://greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], + 'https://www.greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], + ]; + + public function collectData() + { + $username = $this->getInput('u'); + $html = getSimpleHTMLDOMCached(self::URI . '/v/' . $username); + $html = defaultLinkTo($html, self::URI); + + foreach ($html->find('div[class*=content__item]') as $post) { + // Skip the ads + if (!str_contains($post->class, 'ads')) { + $url = $post->find('a[href^=https://greatfon.com/c/]', 0)->href; + $date = $this->parseDate($post->find('div[class=content__time-text]', 0)->plaintext); + $description = $post->find('img', 0)->alt; + $imageUrl = $post->find('img', 0)->src; + $author = $username; + $uid = $url; + $title = 'Post - ' . $username . ' - ' . $this->descriptionToTitle($description); + + // Checking post type + $isVideo = (bool) $post->find('div[class=content__camera]', 0); + $videoNote = $isVideo ? '

(video)

' : ''; + + $this->items[] = [ + 'uri' => $url, + 'author' => $author, + 'timestamp' => $date, + 'title' => $title, + 'thumbnail' => $imageUrl, + 'enclosures' => [$imageUrl], + 'content' => << + {$description} + +{$videoNote} +

{$description}

+HTML, + 'uid' => $uid + ]; + } + } + } + + private function parseDate($content) + { + // Parse date, and transform the date into a timetamp, even in a case of a relative date + $date = date_create(); + + // Content trimmed to be sure that the "article" is at the beginning of the string and remove "ago" to make it a valid PHP date interval + $dateString = trim(str_replace(' ago', '', $content)); + + // Replace the article "an" or "a" by the number "1" to be a valid PHP date interval + $dateString = preg_replace('/^((an|a) )/m', '1 ', $dateString); + + $relativeDate = date_interval_create_from_date_string($dateString); + if ($relativeDate) { + date_sub($date, $relativeDate); + // As the relative interval has the precision of a day for date older than 24 hours, we can remove the hour of the date, as it is not relevant + date_time_set($date, 0, 0, 0, 0); + } else { + $this->logger->info(sprintf('Unable to parse date string: %s', $dateString)); + } + return date_format($date, 'r'); + } + + public function getURI() + { + if (!is_null($this->getInput('u'))) { + return urljoin(self::URI, '/v/' . $this->getInput('u')); + } + + return parent::getURI(); + } + + public function getIcon() + { + return static::URI . '/images/favicon-hub-3ede543aa6d1225e8dc016ccff6879c8.ico?vsn=d'; + } + + private function descriptionToTitle($description) + { + return strlen($description) > 60 ? mb_substr($description, 0, 57) . '...' : $description; + } + + public function getName() + { + if (!is_null($this->getInput('u'))) { + return 'Username ' . $this->getInput('u') . ' - GreatFon Bridge'; + } + return parent::getName(); + } + + public function detectParameters($url) + { + $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})(\/reels\/|\/tagged\/|\/|)|(www\.|)(greatfon.com)\/v\/([a-zA-Z0-9_\.]{1,30}))/'; + if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Username'; + // Extract detected domain using the regex + $domain = $matches[8] ?? $matches[4]; + if ($domain == 'greatfon.com') { + $params['u'] = $matches[9]; + return $params; + } elseif ($domain == 'instagram.com') { + $params['u'] = $matches[5]; + return $params; + } else { + return null; + } + } else { + return null; + } + } +} From 7a7fa876d2a07526f0c57610b3d83c261b99e0ed Mon Sep 17 00:00:00 2001 From: wpdevelopment11 <85058595+wpdevelopment11@users.noreply.github.com> Date: Wed, 8 Nov 2023 18:40:24 +0300 Subject: [PATCH 17/69] [VkBridge] Fix regex that extracts page name (#3793) Dot should be allowed in page names. Precise rules for page names are available here: https://vk.com/faq19715 (in Russian) --- bridges/VkBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 0d47692da33..60e4315bcc6 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -29,11 +29,12 @@ class VkBridge extends BridgeAbstract 'https://vk.com/groupname/anythingelse' => ['u' => 'groupname'], 'https://vk.com/groupname?w=somethingelse' => ['u' => 'groupname'], 'https://vk.com/with_underscore' => ['u' => 'with_underscore'], + 'https://vk.com/vk.cats' => ['u' => 'vk.cats'], ]; protected $pageName; protected $tz = 0; - private $urlRegex = '/vk\.com\/([\w]+)/'; + private $urlRegex = '/vk\.com\/([\w.]+)/'; public function getURI() { From 57b61c8787b7946c674a06e76699baff94931f7e Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Thu, 9 Nov 2023 10:16:34 +0100 Subject: [PATCH 18/69] [MydealsBridge] Fix keyword seatch (#3794) When no result were found using the keyword search, some random deals were displayed because the "not found" text has been modified : the text is now up to date. Some type in the textual name of the Bridge and texte about the website name was fixed --- bridges/MydealsBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 0ef9c201701..de702583b7a 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2,9 +2,9 @@ class MydealsBridge extends PepperBridgeAbstract { - const NAME = 'Mydeals bridge'; + const NAME = 'Mydealz bridge'; const URI = 'https://www.mydealz.de/'; - const DESCRIPTION = 'Zeigt die Deals von mydeals.de'; + const DESCRIPTION = 'Zeigt die Deals von mydealz.de'; const MAINTAINER = 'sysadminstory'; const PARAMETERS = [ 'Suche nach Stichworten' => [ @@ -2023,7 +2023,7 @@ class MydealsBridge extends PepperBridgeAbstract 'uri-deal' => 'deals/', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', - 'no-results' => 'Ups, wir konnten keine Deals zu', + 'no-results' => 'Ups, wir konnten nichts', 'relative-date-indicator' => [ 'vor', 'seit' From e76b0601b3312d957b6bf6e46ddeb3df6cd70a01 Mon Sep 17 00:00:00 2001 From: SebLaus <97241865+SebLaus@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:55:56 +0100 Subject: [PATCH 19/69] [IdealoBridge] New Bridge to track prices on idealo.de (#3786) * [IdealoBridge] Created Checks the price of a given item on idealo.de. Can create an Alarm Message if a the price is lower than set or an Priceupdate if the price has changed. * Changed Exec and syntax * last fixes for remaining warning --- bridges/IdealoBridge.php | 180 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 bridges/IdealoBridge.php diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php new file mode 100644 index 00000000000..89c5f87df90 --- /dev/null +++ b/bridges/IdealoBridge.php @@ -0,0 +1,180 @@ + [ + 'name' => 'Idealo.de Link to productpage', + 'required' => true, + 'exampleValue' => 'https://www.idealo.de/preisvergleich/OffersOfProduct/202007367_-s7-pro-ultra-roborock.html' + ], + 'ExcludeNew' => [ + 'name' => 'Priceupdate: Do not track new items', + 'type' => 'checkbox', + 'value' => 'c' + ], + 'ExcludeUsed' => [ + 'name' => 'Priceupdate: Do not track used items', + 'type' => 'checkbox', + 'value' => 'uc' + ], + 'MaxPriceNew' => [ + 'name' => 'Pricealarm: Maximum price for new Product', + 'type' => 'number' + ], + 'MaxPriceUsed' => [ + 'name' => 'Pricealarm: Maximum price for used Product', + 'type' => 'number' + ], + ] + ]; + + public function getIcon() + { + return 'https://cdn.idealo.com/storage/ids-assets/ico/favicon.ico'; + } + + public function collectData() + { + $link = $this->getInput('Link'); + $html = getSimpleHTMLDOM($link); + + // Get Productname + $titleobj = $html->find('.oopStage-title', 0); + $Productname = $titleobj->find('span', 0)->plaintext; + + // Create product specific Cache Keys with the link + $KeyNEW = $link; + $KeyNEW .= 'NEW'; + + $KeyUSED = $link; + $KeyUSED .= 'USED'; + + // Load previous Price + $OldPriceNew = $this->loadCacheValue($KeyNEW); + $OldPriceUsed = $this->loadCacheValue($KeyUSED); + + // First button is new. Found at oopStage-conditionButton-wrapper-text class (.) + $FirstButton = $html->find('.oopStage-conditionButton-wrapper-text', 0); + if ($FirstButton) { + $PriceNew = $FirstButton->find('strong', 0)->plaintext; + } + + // Second Button is used + $SecondButton = $html->find('.oopStage-conditionButton-wrapper-text', 1); + if ($SecondButton) { + $PriceUsed = $SecondButton->find('strong', 0)->plaintext; + } + + // Only continue if a price has changed + if ($PriceNew != $OldPriceNew || $PriceUsed != $OldPriceUsed) { + // Get Product Image + $image = $html->find('.datasheet-cover-image', 0)->src; + + // Generate Content + if ($PriceNew > 1) { + $content = "

Price New:
$PriceNew

"; + $content .= "

Price Newbefore:
$OldPriceNew

"; + } + + if ($this->getInput('MaxPriceNew') != '') { + $content .= sprintf('

Max Price Used:
%s,00 €

', $this->getInput('MaxPriceNew')); + } + + if ($PriceUsed > 1) { + $content .= "

Price Used:
$PriceUsed

"; + $content .= "

Price Used before:
$OldPriceUsed

"; + } + + if ($this->getInput('MaxPriceUsed') != '') { + $content .= sprintf('

Max Price Used:
%s,00 €

', $this->getInput('MaxPriceUsed')); + } + + $content .= ""; + + + $now = date('d.m.j H:m'); + + $Pricealarm = 'Pricealarm %s: %s %s %s'; + + // Currently under Max new price + if ($this->getInput('MaxPriceNew') != '') { + if ($PriceNew < $this->getInput('MaxPriceNew')) { + $title = sprintf($Pricealarm, 'Used', $PriceNew, $Productname, $now); + $item = [ + 'title' => $title, + 'uri' => $link, + 'content' => $content, + 'uid' => md5($title) + ]; + $this->items[] = $item; + } + } + + // Currently under Max used price + if ($this->getInput('MaxPriceUsed') != '') { + if ($PriceUsed < $this->getInput('MaxPriceUsed')) { + $title = sprintf($Pricealarm, 'Used', $PriceUsed, $Productname, $now); + $item = [ + 'title' => $title, + 'uri' => $link, + 'content' => $content, + 'uid' => md5($title) + ]; + $this->items[] = $item; + } + } + + // General Priceupdate + if ($this->getInput('MaxPriceUsed') == '' && $this->getInput('MaxPriceNew') == '') { + // check if a relevant pricechange happened + if ( + (!$this->getInput('ExcludeNew') && $PriceNew != $OldPriceNew ) || + (!$this->getInput('ExcludeUsed') && $PriceUsed != $OldPriceUsed ) + ) { + $title .= 'Priceupdate! '; + + if (!$this->getInput('ExcludeNew')) { + if ($PriceNew < $OldPriceNew) { + $title .= 'NEW:⬇ '; // Arrow Down Emoji + } + if ($PriceNew > $OldPriceNew) { + $title .= 'NEW:⬆ '; // Arrow Up Emoji + } + } + + + if (!$this->getInput('ExcludeUsed')) { + if ($PriceUsed < $OldPriceUsed) { + $title .= 'USED:⬇ '; // Arrow Down Emoji + } + if ($PriceUsed > $OldPriceUsed) { + $title .= 'USED:⬆ '; // Arrow Up Emoji + } + } + $title .= $Productname; + $title .= ' '; + $title .= $now; + + $item = [ + 'title' => $title, + 'uri' => $link, + 'content' => $content, + 'uid' => md5($title) + ]; + $this->items[] = $item; + } + } + } + + // Save current price + $this->saveCacheValue($KeyNEW, $PriceNew); + $this->saveCacheValue($KeyUSED, $PriceUsed); + } +} From b347a9268a7d6bc34442509e2cf7dcd793c39853 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 10 Nov 2023 12:56:11 +0100 Subject: [PATCH 20/69] feat: new bridge MangaReader (#3795) --- bridges/MangaReaderBridge.php | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 bridges/MangaReaderBridge.php diff --git a/bridges/MangaReaderBridge.php b/bridges/MangaReaderBridge.php new file mode 100644 index 00000000000..1fa0c62dc54 --- /dev/null +++ b/bridges/MangaReaderBridge.php @@ -0,0 +1,44 @@ + [ + 'name' => 'Manga URL', + 'type' => 'text', + 'required' => true, + 'title' => 'The URL of the manga on MangaReader', + 'pattern' => '^https:\/\/mangareader\.to\/[^\/]+$', + 'exampleValue' => 'https://mangareader.to/bleach-1623', + ], + 'lang' => [ + 'name' => 'Chapter Language', + 'title' => 'two-letter language code (example "en", "jp", "fr")', + 'exampleValue' => 'en', + 'required' => true, + 'pattern' => '^[a-z][a-z]$', + ] + ] + ]; + + public function collectData() + { + $url = $this->getInput('url'); + $lang = $this->getInput('lang'); + $dom = getSimpleHTMLDOM($url); + $chapters = $dom->getElementById($lang . '-chapters'); + + foreach ($chapters->getElementsByTagName('li') as $chapter) { + $a = $chapter->getElementsByTagName('a')[0]; + $item = []; + $item['title'] = $a->getAttribute('title'); + $item['uri'] = self::URI . $a->getAttribute('href'); + $this->items[] = $item; + } + } +} From 4919c53c108a36a7969b027bc6e6653415096353 Mon Sep 17 00:00:00 2001 From: knrdl <35548889+knrdl@users.noreply.github.com> Date: Mon, 13 Nov 2023 00:11:19 +0100 Subject: [PATCH 21/69] [DemosBerlinBridge] add bridge (#3800) --- bridges/DemosBerlinBridge.php | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 bridges/DemosBerlinBridge.php diff --git a/bridges/DemosBerlinBridge.php b/bridges/DemosBerlinBridge.php new file mode 100644 index 00000000000..05fd2335d45 --- /dev/null +++ b/bridges/DemosBerlinBridge.php @@ -0,0 +1,62 @@ + [ + 'name' => 'Tage', + 'type' => 'number', + 'title' => 'Einträge für die nächsten Tage zurückgeben', + 'required' => true, + 'defaultValue' => 7, + ] + ]]; + + public function getIcon() + { + return 'https://www.berlin.de/i9f/r1/images/favicon/favicon.ico'; + } + + public function collectData() + { + $json = getContents('https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/index/all.json'); + $jsonFile = json_decode($json, true); + + $daysInterval = DateInterval::createFromDateString($this->getInput('days') . ' day'); + $maxTargetDate = date_add(new DateTime('now'), $daysInterval); + + foreach ($jsonFile['index'] as $entry) { + $entryDay = implode('-', array_reverse(explode('.', $entry['datum']))); // dd.mm.yyyy to yyyy-mm-dd + $ts = (new DateTime())->setTimestamp(strtotime($entryDay)); + if ($ts <= $maxTargetDate) { + $item = []; + $item['uri'] = 'https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/detail/' . $entry['id']; + $item['timestamp'] = $entryDay . ' ' . $entry['von']; + $item['title'] = $entry['thema']; + $location = $entry['strasse_nr'] . ' ' . $entry['plz']; + $locationQuery = http_build_query(['query' => $location]); + $item['content'] = <<{$entry['thema']} +

📅

+ + 📍 {$location} + +

{$entry['aufzugsstrecke']}

+ HTML; + $item['uid'] = $this->getSanitizedHash($entry['datum'] . '-' . $entry['von'] . '-' . $entry['bis'] . '-' . $entry['thema']); + + $this->items[] = $item; + } + } + } + + private function getSanitizedHash($string) + { + return hash('sha1', preg_replace('/[^a-zA-Z0-9]/', '', strtolower($string))); + } +} From ef711cb30b026b581573c5dc5f4de88b47f105b9 Mon Sep 17 00:00:00 2001 From: knrdl <35548889+knrdl@users.noreply.github.com> Date: Mon, 13 Nov 2023 00:12:39 +0100 Subject: [PATCH 22/69] [KleinanzeigenBridge] add new bridge (#3798) * [KleinanzeigenBridge] add new bridge * [KleinanzeigenBridge] fix missing timestamp * [KleinanzeigenBridge] linting * [KleinanzeigenBridge] fix end of list detection --- bridges/KleinanzeigenBridge.php | 150 ++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 bridges/KleinanzeigenBridge.php diff --git a/bridges/KleinanzeigenBridge.php b/bridges/KleinanzeigenBridge.php new file mode 100644 index 00000000000..e0535b59c9c --- /dev/null +++ b/bridges/KleinanzeigenBridge.php @@ -0,0 +1,150 @@ + [ + 'query' => [ + 'name' => 'query', + 'required' => false, + 'title' => 'query term', + ], + 'location' => [ + 'name' => 'location', + 'required' => false, + 'title' => 'e.g. Berlin', + ], + 'radius' => [ + 'name' => 'radius', + 'required' => false, + 'type' => 'number', + 'title' => 'search radius in kilometers', + 'defaultValue' => 10, + ], + 'pages' => [ + 'name' => 'pages', + 'required' => true, + 'type' => 'number', + 'title' => 'how many pages to fetch', + 'defaultValue' => 2, + ] + ], + 'By profile' => [ + 'userid' => [ + 'name' => 'user id', + 'required' => true, + 'type' => 'number', + 'exampleValue' => 12345678 + ], + 'pages' => [ + 'name' => 'pages', + 'required' => true, + 'type' => 'number', + 'title' => 'how many pages to fetch', + 'defaultValue' => 2, + ] + ], + ]; + + public function getIcon() + { + return 'https://www.kleinanzeigen.de/favicon.ico'; + } + + public function getName() + { + switch ($this->queriedContext) { + case 'By profile': + return 'Kleinanzeigen Profil'; + case 'By search': + return 'Kleinanzeigen ' . $this->getInput('query') . ' / ' . $this->getInput('location'); + default: + return parent::getName(); + } + } + + public function collectData() + { + if ($this->queriedContext === 'By profile') { + for ($i = 1; $i <= $this->getInput('pages'); $i++) { + $html = getSimpleHTMLDOM(self::URI . '/s-bestandsliste.html?userId=' . $this->getInput('userid') . '&pageNum=' . $i . '&sortingField=SORTING_DATE'); + + $foundItem = false; + foreach ($html->find('article.aditem') as $element) { + $this->addItem($element); + $foundItem = true; + } + if (!$foundItem) { + break; + } + } + } + + if ($this->queriedContext === 'By search') { + $locationID = ''; + if ($this->getInput('location')) { + $json = getContents(self::URI . '/s-ort-empfehlungen.json?' . http_build_query(['query' => $this->getInput('location')])); + $jsonFile = json_decode($json, true); + $locationID = str_replace('_', '', array_key_first($jsonFile)); + } + for ($i = 1; $i <= $this->getInput('pages'); $i++) { + $searchUrl = self::URI . '/s-walled-garden/'; + if ($i != 1) { + $searchUrl .= 'seite:' . $i . '/'; + } + if ($this->getInput('query')) { + $searchUrl .= urlencode($this->getInput('query')) . '/k0'; + } + if ($locationID) { + $searchUrl .= 'l' . $locationID; + } + if ($this->getInput('radius')) { + $searchUrl .= 'r' . $this->getInput('radius'); + } + + $html = getSimpleHTMLDOM($searchUrl); + + // end of list if returned page is not the expected one + if ($html->find('.pagination-current', 0)->plaintext != $i) { + break; + } + + foreach ($html->find('ul#srchrslt-adtable article.aditem') as $element) { + $this->addItem($element); + } + } + } + } + + private function addItem($element) + { + $item = []; + + $item['uid'] = $element->getAttribute('data-adid'); + $item['uri'] = self::URI . $element->getAttribute('data-href'); + + $item['title'] = $element->find('h2', 0)->plaintext; + $item['timestamp'] = $element->find('div.aditem-main--top--right', 0)->plaintext; + $imgUrl = str_replace( + 'rule=$_2.JPG', + 'rule=$_57.JPG', + str_replace( + 'rule=$_35.JPG', + 'rule=$_57.JPG', + $element->find('img', 0) ? $element->find('img', 0)->getAttribute('src') : '' + ) + ); //enhance img quality + $textContainer = $element->find('div.aditem-main', 0); + $textContainer->find('a', 0)->href = self::URI . $textContainer->find('a', 0)->href; // add domain to url + $item['content'] = '' . + $textContainer->outertext; + + $this->items[] = $item; + } +} From 2b741b1c1bf3ef352cea0903bd1561bc00ff45b0 Mon Sep 17 00:00:00 2001 From: joaomqc Date: Wed, 15 Nov 2023 15:26:25 +0000 Subject: [PATCH 23/69] [SongkickBridge] add new bridge (#3803) * [SongkickBridge] add new bridge * [SongkickBridge] fix var reference and outdoor category * [SongkickBridge] remove unnecessary string concat * [SongkickBridge] fix if clause formatting * [SongkickBridge] fix formatting and event title --- bridges/SongkickBridge.php | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 bridges/SongkickBridge.php diff --git a/bridges/SongkickBridge.php b/bridges/SongkickBridge.php new file mode 100644 index 00000000000..bfe29865300 --- /dev/null +++ b/bridges/SongkickBridge.php @@ -0,0 +1,92 @@ + [ + 'name' => 'Artist ID', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '2506696-imagine-dragons', + ] + ] ]; + + const ARTIST_URI = 'https://www.songkick.com/artists/%s/'; + const CALENDAR_URI = self::ARTIST_URI . 'calendar'; + + private $name = ''; + + public function getURI() + { + return sprintf(self::ARTIST_URI, $this->getInput('artistid')); + } + + public function getName() + { + if (!empty($this->name)) { + return $this->name . ' - ' . parent::getName(); + } + return parent::getName(); + } + + public function getIcon() + { + return 'https://assets.sk-static.com/images/nw/furniture/songkick-logo.svg'; + } + + public function collectData() + { + $url = sprintf(self::CALENDAR_URI, $this->getInput('artistid')); + + $dom = getSimpleHTMLDOM($url); + + $jsonscript = $dom->find('div.microformat > script', 0); + + if (empty($this->name) && $jsonscript) { + $this->name = json_decode($jsonscript->innertext)[0]->name; + } + + $dom = $dom->find('div.container > div.row > div.primary', 0); + + if (!$dom) { + throw new Exception(sprintf('Unable to find css selector on `%s`', $url)); + } + $dom = defaultLinkTo($dom, $this->getURI()); + + foreach ($dom->find('div[@id="calendar-summary"] > ol > li') as $article) { + $detailsobj = json_decode($article->find('div.microformat > script', 0)->innertext)[0]; + + $a = $article->find('a', 0); + + $details = $a->find('div.event-details', 0); + $title = $details->find('.secondary-detail', 0)->plaintext; + $city = $details->find('.primary-detail', 0)->plaintext; + $event = $detailsobj->location->name; + + $content = 'City: ' . $city . '
Event: ' . $event . '
Date: ' . $article->title; + + $categories = []; + if ($details->hasClass('concert')) { + $categories[] = 'concert'; + } + if ($details->hasClass('festival')) { + $categories[] = 'festival'; + } + if (!is_null($details->find('.outdoor', 0))) { + $categories[] = 'outdoor'; + } + + $this->items[] = [ + 'title' => $title, + 'uri' => $a->href, + 'content' => $content, + 'categories' => $categories, + ]; + } + } +} From b037d1b4d1f0b0f422e21125ddef00a58e185ed1 Mon Sep 17 00:00:00 2001 From: Matt DeMoss Date: Tue, 21 Nov 2023 11:00:02 -0500 Subject: [PATCH 24/69] [Threads] add bridge (#3805) * initial working Threads bridge * properly specify a default limit * phpcs formatted --- bridges/ThreadsBridge.php | 120 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 bridges/ThreadsBridge.php diff --git a/bridges/ThreadsBridge.php b/bridges/ThreadsBridge.php new file mode 100644 index 00000000000..b7e5cd1abff --- /dev/null +++ b/bridges/ThreadsBridge.php @@ -0,0 +1,120 @@ + [ + 'u' => [ + 'name' => 'username', + 'required' => true, + 'exampleValue' => 'zuck', + 'title' => 'Insert a user name' + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of posts to fetch', + 'defaultValue' => 5 + ] + ] + ]; + + protected $feedName = self::NAME; + public function getName() + { + return $this->feedName; + } + + public function detectParameters($url) + { + // By username + $regex = '/^(https?:\/\/)?(www\.)?threads\.net\/(@)?([^\/?\n]+)/'; + if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By username'; + $params['u'] = urldecode($matches[3]); + return $params; + } + return null; + } + + public function getURI() + { + return self::URI . '@' . $this->getInput('u'); + } + + // https://stackoverflow.com/a/3975706/421140 + // Found this in FlaschenpostBridge, modified to return an array and take an object. + private function recursiveFind($haystack, $needle) + { + $found = []; + $iterator = new \RecursiveArrayIterator($haystack); + $recursive = new \RecursiveIteratorIterator( + $iterator, + \RecursiveIteratorIterator::SELF_FIRST + ); + foreach ($recursive as $key => $value) { + if ($key === $needle) { + $found[] = $value; + } + } + return $found; + } + + public function collectData() + { + $html = getSimpleHTMLDOMCached($this->getURI(), static::CACHE_TIMEOUT); + Debug::log(sprintf('Fetched: %s', $this->getURI())); + $jsonBlobs = $html->find('script[type="application/json"]'); + Debug::log(sprintf('%d JSON blobs found.', count($jsonBlobs))); + $gatheredCodes = []; + $limit = $this->getInput('limit'); + foreach ($jsonBlobs as $jsonBlob) { + // The structure of the JSON document is likely to change, but we're looking for a "code" inside a "post" + foreach ($this->recursiveFind($this->recursiveFind(json_decode($jsonBlob->innertext), 'post'), 'code') as $candidateCode) { + // code should be like CzZk4-USq1O or Cy3m1VnRiwP or Cywjyrdv9T6 or CzZk4-USq1O + if (grapheme_strlen($candidateCode) == 11 and !in_array($candidateCode, $gatheredCodes)) { + $gatheredCodes[] = $candidateCode; + if (count($gatheredCodes) >= $limit) { + break 2; + } + } + } + } + Debug::log(sprintf('Candidate codes found in JSON in script tags: %s', print_r($gatheredCodes, true))); + + $this->feedName = html_entity_decode($html->find('meta[property=og:title]', 0)->content); + // todo: meta[property=og:description] could populate the feed description + + foreach ($gatheredCodes as $postCode) { + $item = []; + // post URL is like: https://www.threads.net/@zuck/post/Czrr520PZfh + $item['uri'] = $this->getURI() . '/post/' . $postCode; + $articleHtml = getSimpleHTMLDOMCached($item['uri'], 15778800); // cache time: six months + + // Relying on meta tags ought to be more reliable. + if ($articleHtml->find('meta[property=og:type]', 0)->content != 'article') { + continue; + } + $item['title'] = $articleHtml->find('meta[property=og:description]', 0)->content; + $item['content'] = $articleHtml->find('meta[property=og:description]', 0)->content; + $item['author'] = html_entity_decode($articleHtml->find('meta[property=og:title]', 0)->content); + + $imageUrl = $articleHtml->find('meta[property=og:image]', 0); + if ($imageUrl) { + $item['enclosures'][] = html_entity_decode($imageUrl->content); + } + + // todo: parse hashtags out of content for $item['categories'] + // todo: try to scrape out a timestamp for $item['timestamp'], it's not in the meta tags + + $this->items[] = $item; + } + } +} From 609eed1791598a798e7f94ac694d014605ec11ee Mon Sep 17 00:00:00 2001 From: George Sokianos Date: Tue, 28 Nov 2023 21:54:39 +0000 Subject: [PATCH 25/69] KoFiBridge fix the "Call to a member function find() on null" line 39 (#3807) --- bridges/KoFiBridge.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bridges/KoFiBridge.php b/bridges/KoFiBridge.php index c16005907fc..da8f1e7da34 100644 --- a/bridges/KoFiBridge.php +++ b/bridges/KoFiBridge.php @@ -27,12 +27,15 @@ public function collectData() if (isset($titleWrapper[0])) { $item = []; $item['title'] = $element->find('div.content-link-text div')[0]->plaintext; - // $item['timestamp'] = strtotime($element->find('div.feeditem-time', 0)->plaintext); - $item['uri'] = self::URI . $element->find('div.fi-post-item-large a')[0]->href; + $uri = $element->find('div.content-link-text div')[2]->find('a')[0]->onclick; + $uri = trim(str_replace('window.location =', '', $uri)); + $uri = trim(str_replace(''', '', $uri)); + $uri = trim(str_replace(';', '', $uri)); + $item['uri'] = self::URI . $uri; + if (isset($element->find('div.fi-post-item-large div.content-link-post img')[0])) { $item['enclosures'][] = $element->find('div.fi-post-item-large div.content-link-post img')[0]->src; } - // $item['content'] = $element->find('div.content-link-text div#content-link', 0)->plaintext; $html = getSimpleHTMLDOM($item['uri']); $feedItemTime = $html->find('div.feeditem-time', 0); From ccc20849ffe297e09e046a086734c2c90e94420a Mon Sep 17 00:00:00 2001 From: Michael Bemmerl Date: Thu, 30 Nov 2023 16:52:51 +0000 Subject: [PATCH 26/69] [SchweinfurtBuergerinformationenBridge] Don't include images with data URIs as enclosures. (#3811) See also setEnclosures() in FeedItem.php: URIs with a path are required. --- bridges/SchweinfurtBuergerinformationenBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/SchweinfurtBuergerinformationenBridge.php b/bridges/SchweinfurtBuergerinformationenBridge.php index 349a9d8a84e..d1f5db158f5 100644 --- a/bridges/SchweinfurtBuergerinformationenBridge.php +++ b/bridges/SchweinfurtBuergerinformationenBridge.php @@ -107,9 +107,9 @@ private function generateItemFromArticle($id) ]; // Let's see if there are images in the content, and if yes, attach - // them as enclosures, but not images which are used for linking to an external site. + // them as enclosures, but not images which are used for linking to an external site and data URIs. foreach ($images as $image) { - if ($image->class != 'imgextlink') { + if ($image->class != 'imgextlink' && parse_url($image->src, PHP_URL_SCHEME) != 'data') { $item['enclosures'][] = $image->src; } } From 44ff2f2cf8cc4403c7fd2af182607c59c74e14ba Mon Sep 17 00:00:00 2001 From: Niehztog Date: Thu, 30 Nov 2023 17:53:47 +0100 Subject: [PATCH 27/69] adds Super Mario Bros. Wonder to NintendoBridge (#3810) --- bridges/NintendoBridge.php | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/bridges/NintendoBridge.php b/bridges/NintendoBridge.php index 3455073776d..1f463e91a00 100644 --- a/bridges/NintendoBridge.php +++ b/bridges/NintendoBridge.php @@ -18,6 +18,7 @@ class NintendoBridge extends XPathAbstract 'Splatoon 2' => 's2', 'Super Mario 3D All-Stars' => 'sm3as', 'Super Mario 3D World + Bowser’s Fury' => 'sm3wbf', + 'Super Mario Bros. Wonder' => 'smbw', 'Super Mario Maker 2' => 'smm2', 'Super Mario Odyssey' => 'smo', 'Super Smash Bros. Ultimate' => 'ssbu', @@ -62,6 +63,7 @@ class NintendoBridge extends XPathAbstract 's2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Splatoon-2-1482897.html', 'sm3as' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-All-Stars-1844226.html', 'sm3wbf' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-World-Bowser-s-Fury-1920668.html', + 'smbw' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Bros-Wonder-2485410.html', 'smm2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Maker-2-1586745.html', 'smo' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Odyssey-1482901.html', 'ssbu' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Smash-Bros-Ultimate-1484130.html', @@ -73,7 +75,7 @@ class NintendoBridge extends XPathAbstract ]; const XPATH_EXPRESSION_ITEM = '//div[@class="col-xs-12 content"]/div[starts-with(@id,"v") and @class="collapse"]'; const XPATH_EXPRESSION_ITEM_FIRMWARE = '//div[@id="latest" and @class="collapse" and @rel="1"]'; - const XPATH_EXPRESSION_ITEM_TITLE = './/h2[1]/node()'; + const XPATH_EXPRESSION_ITEM_TITLE = '(.//h2[1] | .//strong[1])[1]/node()'; const XPATH_EXPRESSION_ITEM_CONTENT = '.'; const XPATH_EXPRESSION_ITEM_URI = '//link[@rel="canonical"]/@href'; @@ -124,6 +126,15 @@ class NintendoBridge extends XPathAbstract 'pt' => 'ançada no dia ', 'en' => 'eleased ', ], + 'smbw' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], 'smm2' => [ 'de' => 'eröffentlicht am ', 'es' => 'isponible desde el ', @@ -235,6 +246,15 @@ class NintendoBridge extends XPathAbstract 'pt' => 'd/m/y', 'en' => 'F j, Y', ], + 'smbw' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'd/m/Y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], 'smm2' => [ 'de' => 'd.m.Y', 'es' => 'd-m-Y', From 206edaedf5397aee35848002b3274417007a86c1 Mon Sep 17 00:00:00 2001 From: Nick McCarthy Date: Fri, 1 Dec 2023 21:36:26 +0000 Subject: [PATCH 28/69] [GoogleScholarBridge] Minor patch (#3814) * Do not add RSS entry if Check for updates is found in the article title - avoids repeat entries --- bridges/GoogleScholarBridge.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bridges/GoogleScholarBridge.php b/bridges/GoogleScholarBridge.php index 981355dd32a..11dc123b22b 100644 --- a/bridges/GoogleScholarBridge.php +++ b/bridges/GoogleScholarBridge.php @@ -2,7 +2,7 @@ class GoogleScholarBridge extends BridgeAbstract { - const NAME = 'Google Scholar v2'; + const NAME = 'Google Scholar'; const URI = 'https://scholar.google.com/'; const DESCRIPTION = 'Search for publications or follow authors on Google Scholar.'; const MAINTAINER = 'nicholasmccarthy'; @@ -193,6 +193,11 @@ public function collectData() $articleUrl = $articleTitleElement->find('a', 0)->href; $articleTitle = $articleTitleElement->plaintext; + // Break the loop if 'Check for Updates' is found in the article title + if (strpos($articleTitle, 'Check for updates') !== false) { + break; + } + $articleDateElement = $publication->find('div[class="gs_a"]', 0); $articleDate = $articleDateElement ? $articleDateElement->plaintext : ''; From f3df283c4d93a90f81bda7c466e8b9937f178acf Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Sun, 3 Dec 2023 22:54:23 +0500 Subject: [PATCH 29/69] [VkBridge] Fix single photo duplication (#3816) --- bridges/VkBridge.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 60e4315bcc6..503bc4d0ed1 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -315,6 +315,13 @@ public function collectData() $copy_quote->outertext = "
Reposted ($copy_quote_author):
$copy_quote_content"; } + foreach ($post->find('.PrimaryAttachment .PhotoPrimaryAttachment') as $pa) { + $img = $pa->find('.PhotoPrimaryAttachment__imageElement', 0); + if (is_object($img)) { + $pa->outertext = $img->outertext; + } + } + foreach ($post->find('.SecondaryAttachment') as $sa) { $sa_href = $sa->getAttribute('href'); if (!$sa_href) { From deb9a7269e166c74dfd9140da3da74923bb67608 Mon Sep 17 00:00:00 2001 From: knrdl <35548889+knrdl@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:07:22 +0100 Subject: [PATCH 30/69] [MotatosBridge] add bridge (#3799) * [MotatosBridge] add bridge * [MotatosBridge] fix uid as string * [MotatosBridge] add support for all regions * [MotatosBridge] fix: region: "required" attribute not supported for list --- bridges/MotatosBridge.php | 102 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 bridges/MotatosBridge.php diff --git a/bridges/MotatosBridge.php b/bridges/MotatosBridge.php new file mode 100644 index 00000000000..6833521a794 --- /dev/null +++ b/bridges/MotatosBridge.php @@ -0,0 +1,102 @@ + [ + 'name' => 'Region', + 'type' => 'list', + 'title' => 'Choose country', + 'values' => [ + 'Austria' => 'at', + 'Denmark' => 'dk', + 'Finland' => 'fi', + 'Germany' => 'de', + 'Sweden' => 'se', + ], + ], + ]]; + + public function getName() + { + switch ($this->getInput('region')) { + case 'at': + return 'Motatos'; + case 'dk': + return 'Motatos'; + case 'de': + return 'Motatos'; + case 'fi': + return 'Matsmart'; + case 'se': + return 'Matsmart'; + default: + return self::NAME; + } + } + + public function getURI() + { + switch ($this->getInput('region')) { + case 'at': + return 'https://www.motatos.at/neu-im-shop'; + case 'dk': + return 'https://www.motatos.dk/nye-varer'; + case 'de': + return 'https://www.motatos.de/neu-im-shop'; + case 'fi': + return 'https://www.matsmart.fi/uusimmat'; + case 'se': + return 'https://www.matsmart.se/nyinkommet'; + default: + return self::URI; + } + } + + public function getIcon() + { + return 'https://www.motatos.de/favicon.ico'; + } + + private function getApiUrl() + { + switch ($this->getInput('region')) { + case 'at': + return 'https://api.findify.io/v4/4359f7b3-17e0-4f74-9fdb-e6606dfed25c/smart-collection/new-arrivals'; + case 'dk': + return 'https://api.findify.io/v4/3709426e-621a-49df-bd61-ac8543452022/smart-collection/new-arrivals'; + case 'de': + return 'https://api.findify.io/v4/2a044754-6cda-4541-b159-39133b75386c/smart-collection/new-arrivals'; + case 'fi': + return 'https://api.findify.io/v4/63946f89-2a82-4839-a412-883b79144f7b/smart-collection/new-arrivals'; + case 'se': + return 'https://api.findify.io/v4/3ae86b36-a1bd-4442-a3d9-2af6845908e6/smart-collection/new-arrivals'; + } + } + + public function collectData() + { + // motatos uses this api to dynamically load more items on page scroll + $json = getContents($this->getApiUrl() . '?t_client=0&user={%22uid%22:%220%22,%22sid%22:%220%22}'); + $jsonFile = json_decode($json, true); + + foreach ($jsonFile['items'] as $entry) { + $item = []; + $item['uid'] = $entry['custom_fields']['uuid'][0]; + $item['uri'] = $entry['product_url']; + $item['timestamp'] = $entry['created_at'] / 1000; + $item['title'] = $entry['title']; + $item['content'] = <<{$entry['title']} + +

{$entry['price'][0]}€

+ HTML; + $this->items[] = $item; + } + } +} From c3d93835239fe6b5d70033340d450cd9d88c477f Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 8 Dec 2023 06:24:43 +0100 Subject: [PATCH 31/69] [FindfeedAction.php] Use relative URL in Feed Link (#3820) The FindFeed action used absolute URL. This breaks the usage of RSS Bridge behind a reverse proxy in a container. Fixes #3801 for the Find Feed action --- actions/FindfeedAction.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/FindfeedAction.php b/actions/FindfeedAction.php index 25fe4714f8c..fe5ceef99f1 100644 --- a/actions/FindfeedAction.php +++ b/actions/FindfeedAction.php @@ -56,7 +56,7 @@ public function execute(array $request) $bridgeParams['bridge'] = $bridgeClassName; $bridgeParams['format'] = $format; $content = [ - 'url' => get_home_page_url() . '?action=display&' . http_build_query($bridgeParams), + 'url' => './?action=display&' . http_build_query($bridgeParams), 'bridgeParams' => $bridgeParams, 'bridgeData' => $bridgeData, 'bridgeMeta' => [ From 3ef0226a087fadfa565b70b0868fc988f927cfac Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 8 Dec 2023 06:25:39 +0100 Subject: [PATCH 32/69] [PepperBridgeAbstract] Fix Detection of "no deals found" and more (#3821) - CSS styles showing there were no deals found has changed : CSS class was updated - Relative Date handling : the minimum granularity of a relative date is the minute on the site. Seconds are therefore meaningless, and are now deleted. MydealsBridge was missing one relateve date prefix : now every date is parsed (I hope so !) --- bridges/MydealsBridge.php | 4 +++- bridges/PepperBridgeAbstract.php | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index de702583b7a..22b4641305d 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2068,7 +2068,9 @@ class MydealsBridge extends PepperBridgeAbstract 'relative-date-alt-prefixes' => [ 'aktualisiert vor ', 'kommentiert vor ', - 'heiß seit ' + 'eingestellt vor ', + 'heiß seit ', + 'vor ' ], 'relative-date-ignore-suffix' => [ '/von.*$/' diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 875ed8f647c..8e8a2e8d575 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -94,7 +94,7 @@ protected function collectDeals($url) ); // If there is no results, we don't parse the content because it display some random deals - $noresult = $html->find('h3[class=size--all-l]', 0); + $noresult = $html->find('h3[class*=text--b]', 0); if ($noresult != null && strpos($noresult->plaintext, $this->i8n('no-results')) !== false) { $this->items = []; } else { @@ -542,6 +542,10 @@ private function relativeDateToTimestamp($str) { $date = new DateTime(); + // The minimal amount of time substracted is a minute : the seconds in the resulting date would be related to the execution time of the script. + // This make no sense, so we set the seconds manually to "00". + $date->setTime($date->format('H'), $date->format('i'), 0); + // In case of update date, replace it by the regular relative date first word $str = str_replace($this->i8n('relative-date-alt-prefixes'), $this->i8n('local-time-relative')[0], $str); @@ -559,6 +563,8 @@ private function relativeDateToTimestamp($str) '' ]; $date->modify(str_replace($search, $replace, $str)); + + return $date->getTimestamp(); } From 4a398a5b14d6f5c2bb12bf0156348798f4e4fa37 Mon Sep 17 00:00:00 2001 From: Raymond Berger Date: Sat, 9 Dec 2023 11:52:57 +0100 Subject: [PATCH 33/69] Update FacebookBridge.md - not working (#3823) --- docs/10_Bridge_Specific/FacebookBridge.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/10_Bridge_Specific/FacebookBridge.md b/docs/10_Bridge_Specific/FacebookBridge.md index c2a1fd0eb03..f24f8aa86a6 100644 --- a/docs/10_Bridge_Specific/FacebookBridge.md +++ b/docs/10_Bridge_Specific/FacebookBridge.md @@ -1,18 +1,18 @@ FacebookBridge =============== -Resume of the actual state of this bridge: +State of this bridge: +- Facebook Groups (and probably other sections too) do not work at all - No maintainer -- Need Cookies consent -- New design architecture deployed +- Needs cookie consent support for public pages +- Needs login support (see [this example]([url](https://github.com/RSS-Bridge/rss-bridge/issues/1891)) for Instagram) for private groups -Due [facebook-redesing](https://engineering.fb.com/2020/05/08/web/facebook-redesign/) +Due to the 2020 [Facebook redesign](https://engineering.fb.com/2020/05/08/web/facebook-redesign/) and the requirement to [accept cookies](https://www.facebook.com/business/help/348535683460989) -users start getting [Problems with Facebook on public RSS-Bridge instances](https://github.com/RSS-Bridge/rss-bridge/issues/2047 ) +users are getting [problems with Facebook on public RSS-Bridge instances](https://github.com/RSS-Bridge/rss-bridge/issues/2047). +Relevant Info +-------------- -[Facebook Cookies](https://www.facebook.com/policy/cookies/) - -"Datr" is a unique identifier for your browser and it has a lifespan of two years. - -"c_user" and "xs" cookies to verify the account and have a lifespan of 365 days - +- [Facebook Cookies](https://www.facebook.com/policy/cookies/) +- "Datr" is a unique identifier for your browser and it has a lifespan of two years. +- "c_user" and "xs" cookies to verify the account and have a lifespan of 365 days From a3b064f4eef00ca5559cd23f2c7a47ded8083f21 Mon Sep 17 00:00:00 2001 From: Guillaume Lacasa Date: Mon, 11 Dec 2023 17:38:39 +0100 Subject: [PATCH 34/69] Find PanneauPocket city id from page URL (#3825) Co-authored-by: Guillaume Lacasa --- bridges/PanneauPocketBridge.php | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/bridges/PanneauPocketBridge.php b/bridges/PanneauPocketBridge.php index 8547a500c8c..464d56c5d92 100644 --- a/bridges/PanneauPocketBridge.php +++ b/bridges/PanneauPocketBridge.php @@ -12,6 +12,12 @@ class PanneauPocketBridge extends BridgeAbstract 'name' => 'Choisir une ville', 'type' => 'list', 'values' => self::CITIES, + ], + 'cityName' => [ + 'name' => 'Ville', + ], + 'cityId' => [ + 'name' => 'Identifiant', ] ] ]; @@ -113,8 +119,14 @@ class PanneauPocketBridge extends BridgeAbstract public function collectData() { - $matchedCity = array_search($this->getInput('cities'), self::CITIES); - $city = strtolower($this->getInput('cities') . '-' . $matchedCity); + $cityId = $this->getInput('cityId'); + if ($cityId != null) { + $cityName = $this->getInput('cityName'); + $city = strtolower($cityId . '-' . $cityName); + } else { + $matchedCity = array_search($this->getInput('cities'), self::CITIES); + $city = strtolower($this->getInput('cities') . '-' . $matchedCity); + } $url = sprintf('https://app.panneaupocket.com/ville/%s', urlencode($city)); $html = getSimpleHTMLDOM($url); @@ -136,6 +148,18 @@ public function collectData() } } + public function detectParameters($url) + { + $params = []; + $regex = '/\/ville\/(\d+)-([a-z0-9-]+)/'; + if (preg_match($regex, $url, $matches)) { + $params['cityId'] = $matches[1]; + $params['cityName'] = $matches[2]; + return $params; + } + return null; + } + /** * Produce self::CITIES array */ From 0b67544f86e887d0bf4544b452fef816ec262928 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Wed, 13 Dec 2023 21:09:48 +0100 Subject: [PATCH 35/69] [PepperBridgeAbstract] Fix temperature handling (#3828) Website has changed how the temperature is renderd : the bridge does follow the new website structure --- bridges/PepperBridgeAbstract.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 8e8a2e8d575..6cb0f3024a1 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -117,8 +117,7 @@ protected function collectDeals($url) . $this->getSource($deal) . $deal->find('div[class*=' . $selectorDescription . ']', 0)->innertext . '' - . $deal->find('div[class*=' . $selectorHot . ']', 0) - ->find('span', 0)->outertext + . $this->getTemperature($deal) . ''; // Check if a clock icon is displayed on the deal @@ -368,6 +367,16 @@ private function getShippingCost($deal) } } + /** + * Get the temperature from a Deal if it exists + * @return string String of the deal temperature + */ + private function getTemperature($deal) + { + $data = Json::decode($deal->find('div[class=js-vue2]', 0)->getAttribute('data-vue2')); + return $data['props']['thread']['temperature'] . '°'; + } + /** * Get the source of a Deal if it exists * @return string String of the deal source From f01729c86f29b61d4a50ea8f76c639cd1fc19f5a Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 21:40:13 +0100 Subject: [PATCH 36/69] fix(arstechnica): plus a few unrelated tweaks (#3829) --- actions/FrontpageAction.php | 1 + bridges/ArsTechnicaBridge.php | 9 ++++++++- bridges/VkBridge.php | 2 +- caches/FileCache.php | 2 +- index.php | 6 +++++- lib/Configuration.php | 15 ++------------- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php index 64281b1e9e2..ad48927d731 100644 --- a/actions/FrontpageAction.php +++ b/actions/FrontpageAction.php @@ -31,6 +31,7 @@ public function execute(array $request) } } + // todo: cache this renderered template return render(__DIR__ . '/../templates/frontpage.html.php', [ 'messages' => $messages, 'admin_email' => Configuration::getConfig('admin', 'email'), diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 5b3283b519b..613c1c58ca4 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -37,7 +37,14 @@ protected function parseItem(array $item) { $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); $item_html = defaultLinkTo($item_html, self::URI); - $item['content'] = $item_html->find('.amp-wp-article-content', 0); + + $item_content = $item_html->find('.article-content.post-page', 0); + if (!$item_content) { + // The dom selector probably broke. Let's just return the item as-is + return $item; + } + + $item['content'] = $item_content; // remove various ars advertising $item['content']->find('#social-left', 0)->remove(); diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 503bc4d0ed1..980b4154877 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -523,7 +523,7 @@ private function getContents() } if (!preg_match('#^https?://vk.com/#', $uri)) { - returnServerError('Unexpected redirect location'); + returnServerError('Unexpected redirect location: ' . $uri); } $redirects++; diff --git a/caches/FileCache.php b/caches/FileCache.php index 2f4b3ad5ec6..09d127910ac 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -49,8 +49,8 @@ public function set($key, $value, int $ttl = null): void { $item = [ 'key' => $key, - 'value' => $value, 'expiration' => $ttl === null ? 0 : time() + $ttl, + 'value' => $value, ]; $cacheFile = $this->createCacheFile($key); $bytes = file_put_contents($cacheFile, serialize($item), LOCK_EX); diff --git a/index.php b/index.php index 123f6ecdb88..14713e06f75 100644 --- a/index.php +++ b/index.php @@ -6,7 +6,11 @@ require_once __DIR__ . '/lib/bootstrap.php'; -Configuration::verifyInstallation(); +$errors = Configuration::checkInstallation(); +if ($errors) { + die('
' . implode("\n", $errors) . '
'); +} + $customConfig = []; if (file_exists(__DIR__ . '/config.ini.php')) { $customConfig = parse_ini_file(__DIR__ . '/config.ini.php', true, INI_SCANNER_TYPED); diff --git a/lib/Configuration.php b/lib/Configuration.php index d699178fac0..ac7d29bfbdc 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -15,15 +15,7 @@ private function __construct() { } - /** - * Verifies the current installation of RSS-Bridge and PHP. - * - * Returns an error message and aborts execution if the installation does - * not satisfy the requirements of RSS-Bridge. - * - * @return void - */ - public static function verifyInstallation() + public static function checkInstallation(): array { $errors = []; @@ -57,10 +49,7 @@ public static function verifyInstallation() if (!extension_loaded('json')) { $errors[] = 'json extension not loaded'; } - - if ($errors) { - throw new \Exception(sprintf('Configuration error: %s', implode(', ', $errors))); - } + return $errors; } public static function loadConfiguration(array $customConfig = [], array $env = []) From d157816e07e47dfdc8583d5fcee1925031aa6496 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 21:56:14 +0100 Subject: [PATCH 37/69] fix(reddit): cache tweak for 403 forbidden (#3830) --- actions/DisplayAction.php | 9 +++++++-- bridges/RedditBridge.php | 5 +++++ lib/contents.php | 1 + lib/http.php | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index e3b25fef8a2..435639966fd 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -19,6 +19,7 @@ public function execute(array $request) 'message' => 'RSS-Bridge is down for maintenance.', ]), 503); } + $cacheKey = 'http_' . json_encode($request); /** @var Response $cachedResponse */ $cachedResponse = $this->cache->get($cacheKey); @@ -80,16 +81,19 @@ public function execute(array $request) $this->cache->set($cacheKey, $response, $ttl); } - if (in_array($response->getCode(), [429, 503])) { - $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); // average 20m + if (in_array($response->getCode(), [403, 429, 503])) { + // Cache these responses for about ~20 mins on average + $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); } if ($response->getCode() === 500) { $this->cache->set($cacheKey, $response, 60 * 15); } + if (rand(1, 100) === 2) { $this->cache->prune(); } + return $response; } @@ -187,6 +191,7 @@ private function createFeedItemFromException($e, BridgeAbstract $bridge): FeedIt private function logBridgeError($bridgeName, $code) { + // todo: it's not really necessary to json encode $report $cacheKey = 'error_reporting_' . $bridgeName . '_' . $code; $report = $this->cache->get($cacheKey); if ($report) { diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index f761afaa378..c393c146b19 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -85,6 +85,11 @@ public function collectData() if ($e->getCode() === 429) { $this->cache->set($cacheKey, true, 60 * 16); } + if ($e->getCode() === 403) { + // 403 Forbidden + // This can possibly mean that reddit has permanently blocked this server's ip address + $this->cache->set($cacheKey, true, 60 * 61); + } throw $e; } } diff --git a/lib/contents.php b/lib/contents.php index 055d6bf3188..a4def21ae68 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -71,6 +71,7 @@ function getContents( // Ignore invalid 'Last-Modified' HTTP header value } } + // todo: to be nice nice citizen we should also check for Etag } $response = $httpClient->request($url, $config); diff --git a/lib/http.php b/lib/http.php index c5c57d05c6b..eb70705f600 100644 --- a/lib/http.php +++ b/lib/http.php @@ -2,6 +2,7 @@ class HttpException extends \Exception { + // todo: should include the failing http response (if present) } final class CloudFlareException extends HttpException From 0c4b498d4f41d8402bbc33cbf2864e13c0d76ba2 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 22:06:47 +0100 Subject: [PATCH 38/69] fix(reddit): tweak internal cache logic (#3831) --- bridges/RedditBridge.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index c393c146b19..2b7fe84f9a5 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -75,20 +75,26 @@ class RedditBridge extends BridgeAbstract public function collectData() { - $cacheKey = 'reddit_rate_limit'; - if ($this->cache->get($cacheKey)) { + $forbiddenKey = 'reddit_forbidden'; + if ($this->cache->get($forbiddenKey)) { + throw new HttpException('403 Forbidden', 403); + } + + $rateLimitKey = 'reddit_rate_limit'; + if ($this->cache->get($rateLimitKey)) { throw new HttpException('429 Too Many Requests', 429); } + try { $this->collectDataInternal(); } catch (HttpException $e) { - if ($e->getCode() === 429) { - $this->cache->set($cacheKey, true, 60 * 16); - } if ($e->getCode() === 403) { // 403 Forbidden // This can possibly mean that reddit has permanently blocked this server's ip address - $this->cache->set($cacheKey, true, 60 * 61); + $this->cache->set($forbiddenKey, true, 60 * 61); + } + if ($e->getCode() === 429) { + $this->cache->set($rateLimitKey, true, 60 * 16); } throw $e; } From 38e9c396cfe6b933f1752942385dcf5ee05730d6 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 22:20:21 +0100 Subject: [PATCH 39/69] fix(codeberg): css selector tweak (#3832) * fix(codeberg): css selector tweak * yup --- bridges/CodebergBridge.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bridges/CodebergBridge.php b/bridges/CodebergBridge.php index 2a450477340..79dd706cdd9 100644 --- a/bridges/CodebergBridge.php +++ b/bridges/CodebergBridge.php @@ -79,9 +79,9 @@ class CodebergBridge extends BridgeAbstract public function collectData() { - $html = getSimpleHTMLDOM($this->getURI()); - - $html = defaultLinkTo($html, $this->getURI()); + $url = $this->getURI(); + $html = getSimpleHTMLDOM($url); + $html = defaultLinkTo($html, $url); switch ($this->queriedContext) { case 'Commits': @@ -205,22 +205,22 @@ private function extractCommits($html) */ private function extractIssues($html) { - $div = $html->find('div.issue.list', 0); + $issueList = $html->find('div#issue-list', 0); - foreach ($div->find('li.item') as $li) { + foreach ($issueList->find('div.flex-item') as $div) { $item = []; - $number = trim($li->find('a.index,ml-0.mr-2', 0)->plaintext); + $number = trim($div->find('a.index,ml-0.mr-2', 0)->plaintext); - $item['title'] = $li->find('a.title', 0)->plaintext . ' (' . $number . ')'; - $item['uri'] = $li->find('a.title', 0)->href; + $item['title'] = $div->find('a.issue-title', 0)->plaintext . ' (' . $number . ')'; + $item['uri'] = $div->find('a.issue-title', 0)->href; - $time = $li->find('relative-time.time-since', 0); + $time = $div->find('relative-time.time-since', 0); if ($time) { $item['timestamp'] = $time->datetime; } - $item['author'] = $li->find('div.desc', 0)->find('a', 1)->plaintext; + //$item['author'] = $li->find('div.desc', 0)->find('a', 1)->plaintext; // Fetch issue page $issuePage = getSimpleHTMLDOMCached($item['uri'], 3600); @@ -228,7 +228,7 @@ private function extractIssues($html) $item['content'] = $issuePage->find('div.timeline-item.comment.first', 0)->find('div.render-content.markup', 0); - foreach ($li->find('a.ui.label') as $label) { + foreach ($div->find('a.ui.label') as $label) { $item['categories'][] = $label->plaintext; } From d127bf6e009318914f9b35222fcd97ff137dad57 Mon Sep 17 00:00:00 2001 From: Arnav Jain Date: Fri, 15 Dec 2023 23:36:50 +0100 Subject: [PATCH 40/69] [DagensNyheterDirektBridge] New bridge (#3834) * [DagensNyheterDirektBridge] New bridge * [DagensNyheterDirektBridge] Lint: Replace all tabs with space * [DagensNyheterDirektBridge] Lint: Lines Add empty lines and move start brace to new line * [DagensNyheterDirektBridge] Lint: short- array syntax * [DagensNyheterDirektBridge] Lint: short array syntax Fix incorrect line ending * [DagensNyheterDirektBridge] Lint: further lint fixes * [DagensNyheterDirektBridge] Lint: final fixes --- bridges/DagensNyheterDirektBridge.php | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 bridges/DagensNyheterDirektBridge.php diff --git a/bridges/DagensNyheterDirektBridge.php b/bridges/DagensNyheterDirektBridge.php new file mode 100644 index 00000000000..4d1629fbbd5 --- /dev/null +++ b/bridges/DagensNyheterDirektBridge.php @@ -0,0 +1,62 @@ +find('article') as $element) { + $link = $element->find('button', 0)->getAttribute('data-link'); + $datetime = $element->getAttribute('data-publication-time'); + $url = self::BASEURL . $link; + $title = $element->find('h2', 0)->plaintext; + $author = $element->find('div.ds-byline__titles', 0)->plaintext; + // Debug::log($link); + // Debug::log($datetime); + // Debug::log($title); + // Debug::log($url); + // Debug::log($author); + + $article_content = $element->find('div.direkt-post__content', 0); + $article_html = ''; + + $figure = $element->find('figure', 0); + + if ($figure) { + $article_html = $figure->find('img', 0) . '

' . $figure->find('figcaption', 0) . '

'; + } + + foreach ($article_content->find('p') as $p) { + $article_html = $article_html . $p; + } + + $this->items[] = [ + 'uri' => $url, + 'title' => $title, + 'author' => trim($author), + 'timestamp' => $datetime, + 'content' => trim($article_html), + ]; + + if (count($this->items) > self::LIMIT) { + break; + } + } + } +} From 4e1fa946b4915164c0ec2f09090999597b9d69ad Mon Sep 17 00:00:00 2001 From: ash <153942603+xz47sv@users.noreply.github.com> Date: Fri, 15 Dec 2023 23:39:04 +0100 Subject: [PATCH 41/69] add rb.ash.fail to list of public hosts (#3835) --- docs/01_General/06_Public_Hosts.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index de538cf15b2..c9572824844 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -20,7 +20,8 @@ | ![](https://iplookup.flagfox.net/images/h16/NL.png) | https://feed.eugenemolotov.ru | ![](https://img.shields.io/website/https/feed.eugenemolotov.ru.svg) | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss-bridge.mediani.de | ![](https://img.shields.io/website/https/rss-bridge.mediani.de.svg) | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany | | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | -| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud +| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud | +| ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rb.ash.fail | ![](https://img.shields.io/website/https/rb.ash.fail.svg) | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany ## Inactive instances From d4ae55733b6f7684de0b878d91e53c8a5f917d41 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Fri, 15 Dec 2023 23:39:27 +0100 Subject: [PATCH 42/69] Update GolemBridge.php (#3836) deleted the code which adds the author to the feed, because the author is already in the original feed, so it is not needed. --- bridges/GolemBridge.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index 6699e433617..debf5b297fc 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -82,11 +82,6 @@ protected function parseItem(array $item) // URI without RSS feed reference $item['uri'] = $articlePage->find('head meta[name="twitter:url"]', 0)->content; - $author = $articlePage->find('article header .authors .authors__name', 0); - if ($author) { - $item['author'] = $author->plaintext; - } - $categories = $articlePage->find('ul.tags__list li'); foreach ($categories as $category) { $trimmedcategories[] = trim(html_entity_decode($category->plaintext)); From 0116dde27549fcdaf5600bc255173047e1fce4f9 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Sat, 16 Dec 2023 10:43:27 +0100 Subject: [PATCH 43/69] [GolemBridge] Add h2 elements from article content Else some headers are just missing. Example article with previously missing movie names: https://www.golem.de/news/science-fiction-die-zehn-besten-filme-aus-den-spannenden-70ern-2312-179557.html --- bridges/GolemBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index debf5b297fc..c1b03433af9 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -132,7 +132,7 @@ private function extractContent($page) $img->src = $img->getAttribute('data-src-full'); } - foreach ($content->find('p, h1, h3, img[src*="."]') as $element) { + foreach ($content->find('p, h1, h2, h3, img[src*="."]') as $element) { $item .= $element; } From c5f586497f3d23be61a6e8a5fe0f948f98a5b2f6 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Sat, 16 Dec 2023 11:21:19 +0100 Subject: [PATCH 44/69] [GolemBridge] Remove multi-page page headers On multi-page articles like [1], all the pages after the first one have a page header that we add in the article content. When we tack the pages together again, we don't need those extra page headers. [1] https://www.golem.de/news/science-fiction-die-zehn-besten-filme-aus-den-spannenden-70ern-2312-179557.html --- bridges/GolemBridge.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index c1b03433af9..599d713a0ee 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -116,9 +116,6 @@ private function extractContent($page) // reload html, as remove() is buggy $article = str_get_html($article->outertext); - if ($pageHeader = $article->find('header.paged-cluster-header h1', 0)) { - $item .= $pageHeader; - } $header = $article->find('header', 0); foreach ($header->find('p, figure') as $element) { From b34fa2d278eb14172cbeac4ddf34dca90716b5cb Mon Sep 17 00:00:00 2001 From: Brendan Kidwell Date: Sun, 17 Dec 2023 11:08:40 -0500 Subject: [PATCH 45/69] RumbleBridge - new selector needed on user/channel page (#3843) --- bridges/RumbleBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php index 08b416bfe22..d5b82136ad1 100644 --- a/bridges/RumbleBridge.php +++ b/bridges/RumbleBridge.php @@ -39,7 +39,7 @@ public function collectData() } $dom = getSimpleHTMLDOM($url); - foreach ($dom->find('li.video-listing-entry') as $video) { + foreach ($dom->find('ol.thumbnail__grid div.thumbnail__grid--item') as $video) { $datetime = $video->find('time', 0)->getAttribute('datetime'); $this->items[] = [ From 3944ae68cbe8b8dd4fd653a288cffdb42cd3802e Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 19 Dec 2023 07:53:25 +0100 Subject: [PATCH 46/69] fix(reddit): use old.reddit.com instead of www.reddit.com (#3848) --- bridges/RedditBridge.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 2b7fe84f9a5..bb3e7afcf38 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -1,10 +1,15 @@ Date: Tue, 19 Dec 2023 08:46:37 +0100 Subject: [PATCH 47/69] fix(gatesnotes): the unfucked their json (#3849) --- bridges/GatesNotesBridge.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bridges/GatesNotesBridge.php b/bridges/GatesNotesBridge.php index 24ba9b2ec17..0d9199680f2 100644 --- a/bridges/GatesNotesBridge.php +++ b/bridges/GatesNotesBridge.php @@ -23,12 +23,14 @@ public function collectData() $cleanedContent = str_replace([ '', '', - '\r\n', ], '', $rawContent); - $cleanedContent = str_replace('\"', '"', $cleanedContent); - $cleanedContent = trim($cleanedContent, '"'); + // $cleanedContent = str_replace('\"', '"', $cleanedContent); + // $cleanedContent = trim($cleanedContent, '"'); $json = Json::decode($cleanedContent, false); + if (is_string($json)) { + throw new \Exception('wtf? ' . $json); + } foreach ($json as $article) { $item = []; From 98a94855dc6b909b75629c6630c3795c68e7d560 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 20 Dec 2023 03:16:25 +0100 Subject: [PATCH 48/69] feat: embed response in http exception (#3847) --- bridges/GettrBridge.php | 10 +++++++++- config.default.ini.php | 3 ++- lib/contents.php | 15 ++------------- lib/http.php | 24 +++++++++++++++++++++++- templates/exception.html.php | 23 +++++++++++++++++++++++ 5 files changed, 59 insertions(+), 16 deletions(-) diff --git a/bridges/GettrBridge.php b/bridges/GettrBridge.php index 74804043049..d3b9b899aa3 100644 --- a/bridges/GettrBridge.php +++ b/bridges/GettrBridge.php @@ -33,7 +33,15 @@ public function collectData() $user, min($this->getInput('limit'), 20) ); - $data = json_decode(getContents($api), false); + try { + $json = getContents($api); + } catch (HttpException $e) { + if ($e->getCode() === 400 && str_contains($e->response->getBody(), 'E_USER_NOTFOUND')) { + throw new \Exception('User not found: ' . $user); + } + throw $e; + } + $data = json_decode($json, false); foreach ($data->result->aux->post as $post) { $this->items[] = [ diff --git a/config.default.ini.php b/config.default.ini.php index 52786aefbe4..201b1414fcd 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -47,7 +47,8 @@ enable_maintenance_mode = false [http] -timeout = 60 +; Operation timeout in seconds +timeout = 30 useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0" ; Max http response size in MB diff --git a/lib/contents.php b/lib/contents.php index a4def21ae68..8676a2a8df8 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -101,19 +101,8 @@ function getContents( $response = $response->withBody($cachedResponse->getBody()); break; default: - $exceptionMessage = sprintf( - '%s resulted in %s %s %s', - $url, - $response->getCode(), - $response->getStatusLine(), - // If debug, include a part of the response body in the exception message - Debug::isEnabled() ? mb_substr($response->getBody(), 0, 500) : '', - ); - - if (CloudFlareException::isCloudFlareResponse($response)) { - throw new CloudFlareException($exceptionMessage, $response->getCode()); - } - throw new HttpException(trim($exceptionMessage), $response->getCode()); + $e = HttpException::fromResponse($response, $url); + throw $e; } if ($returnFull === true) { // todo: return the actual response object diff --git a/lib/http.php b/lib/http.php index eb70705f600..bfa6b6bff7f 100644 --- a/lib/http.php +++ b/lib/http.php @@ -2,7 +2,29 @@ class HttpException extends \Exception { - // todo: should include the failing http response (if present) + public ?Response $response; + + public function __construct(string $message = '', int $statusCode = 0, ?Response $response = null) + { + parent::__construct($message, $statusCode); + $this->response = $response ?? new Response('', 0); + } + + public static function fromResponse(Response $response, string $url): HttpException + { + $message = sprintf( + '%s resulted in %s %s %s', + $url, + $response->getCode(), + $response->getStatusLine(), + // If debug, include a part of the response body in the exception message + Debug::isEnabled() ? mb_substr($response->getBody(), 0, 500) : '', + ); + if (CloudFlareException::isCloudFlareResponse($response)) { + return new CloudFlareException($message, $response->getCode(), $response); + } + return new HttpException(trim($message), $response->getCode(), $response); + } } final class CloudFlareException extends HttpException diff --git a/templates/exception.html.php b/templates/exception.html.php index dac0ad26a7a..e1dd97c112e 100644 --- a/templates/exception.html.php +++ b/templates/exception.html.php @@ -16,6 +16,13 @@

+ getCode() === 400): ?> +

400 Bad Request

+

+ This is usually caused by an incorrectly constructed http request. +

+ + getCode() === 404): ?>

404 Page Not Found

@@ -40,6 +47,22 @@

+ getCode() === 0): ?> +

+ See + + https://curl.haxx.se/libcurl/c/libcurl-errors.html + + for description of the curl error code. +

+ +

+ + https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/getCode()) ?> + +

+ + getCode() === 10): ?>

The rss feed is completely empty

From 4e40e032b0fcac52bc74ba5994cefe1d00debf45 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Wed, 20 Dec 2023 22:18:10 +0100 Subject: [PATCH 49/69] Remove matrix reference The main communications platform is still Libera.chat, matrix was only provided by the hosted IRC-Matrix bridge. The bridge was turned off already and won't come back. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 570fb87d89a..2a762d45763 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ Officially hosted instance: https://rss-bridge.org/bridge01/ [![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest) [![irc.libera.chat](https://img.shields.io/badge/irc.libera.chat-%23rssbridge-blue.svg)](https://web.libera.chat/#rssbridge) -[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/#rssbridge:libera.chat) [![Actions Status](https://img.shields.io/github/actions/workflow/status/RSS-Bridge/rss-bridge/tests.yml?branch=master&label=GitHub%20Actions&logo=github)](https://github.com/RSS-Bridge/rss-bridge/actions) ||| From 4c5cf89725e7ebd975eb6ec5136b5e3927df07fe Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 21 Dec 2023 09:18:21 +0100 Subject: [PATCH 50/69] fix(rumble): not all videos have a datetime (#3852) --- bridges/RumbleBridge.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php index d5b82136ad1..f6bfca7d193 100644 --- a/bridges/RumbleBridge.php +++ b/bridges/RumbleBridge.php @@ -40,15 +40,18 @@ public function collectData() $dom = getSimpleHTMLDOM($url); foreach ($dom->find('ol.thumbnail__grid div.thumbnail__grid--item') as $video) { - $datetime = $video->find('time', 0)->getAttribute('datetime'); - - $this->items[] = [ + $item = [ 'title' => $video->find('h3', 0)->plaintext, 'uri' => self::URI . $video->find('a', 0)->href, - 'timestamp' => (new \DateTimeImmutable($datetime))->getTimestamp(), 'author' => $account . '@rumble.com', 'content' => defaultLinkTo($video, self::URI)->innertext, ]; + $time = $video->find('time', 0); + if ($time) { + $publishedAt = new \DateTimeImmutable($time->getAttribute('datetime')); + $item['timestamp'] = $publishedAt->getTimestamp(); + } + $this->items[] = $item; } } From f40f99740588b09033917fd38132a99875495540 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 21 Dec 2023 09:24:22 +0100 Subject: [PATCH 51/69] fix: various small fixes (#3853) --- bridges/ARDAudiothekBridge.php | 20 +++++++++++++------- bridges/CarThrottleBridge.php | 6 ++---- bridges/EZTVBridge.php | 2 +- bridges/TrelloBridge.php | 2 +- bridges/YoutubeBridge.php | 6 +++++- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/bridges/ARDAudiothekBridge.php b/bridges/ARDAudiothekBridge.php index 2c1958f3d0e..619c0911f06 100644 --- a/bridges/ARDAudiothekBridge.php +++ b/bridges/ARDAudiothekBridge.php @@ -63,11 +63,13 @@ class ARDAudiothekBridge extends BridgeAbstract public function collectData() { - $oldTz = date_default_timezone_get(); + $path = $this->getInput('path'); + $limit = $this->getInput('limit'); + $oldTz = date_default_timezone_get(); date_default_timezone_set('Europe/Berlin'); - $pathComponents = explode('/', $this->getInput('path')); + $pathComponents = explode('/', $path); if (empty($pathComponents)) { returnClientError('Path may not be empty'); } @@ -82,17 +84,21 @@ public function collectData() } $url = self::APIENDPOINT . 'programsets/' . $showID . '/'; - $rawJSON = getContents($url); - $processedJSON = json_decode($rawJSON)->data->programSet; + $json1 = getContents($url); + $data1 = Json::decode($json1, false); + $processedJSON = $data1->data->programSet; + if (!$processedJSON) { + throw new \Exception('Unable to find show id: ' . $showID); + } - $limit = $this->getInput('limit'); $answerLength = 1; $offset = 0; $numberOfElements = 1; while ($answerLength != 0 && $offset < $numberOfElements && (is_null($limit) || $offset < $limit)) { - $rawJSON = getContents($url . '?offset=' . $offset); - $processedJSON = json_decode($rawJSON)->data->programSet; + $json2 = getContents($url . '?offset=' . $offset); + $data2 = Json::decode($json2, false); + $processedJSON = $data2->data->programSet; $answerLength = count($processedJSON->items->nodes); $offset = $offset + $answerLength; diff --git a/bridges/CarThrottleBridge.php b/bridges/CarThrottleBridge.php index 913b686caec..70d7b54e140 100644 --- a/bridges/CarThrottleBridge.php +++ b/bridges/CarThrottleBridge.php @@ -9,8 +9,7 @@ class CarThrottleBridge extends BridgeAbstract public function collectData() { - $news = getSimpleHTMLDOMCached(self::URI . 'news') - or returnServerError('could not retrieve page'); + $news = getSimpleHTMLDOMCached(self::URI . 'news'); $this->items[] = []; @@ -22,8 +21,7 @@ public function collectData() $item['uri'] = self::URI . $titleElement->getAttribute('href'); $item['title'] = $titleElement->innertext; - $articlePage = getSimpleHTMLDOMCached($item['uri']) - or returnServerError('could not retrieve page'); + $articlePage = getSimpleHTMLDOMCached($item['uri']); $authorDiv = $articlePage->find('div.author div'); if ($authorDiv) { diff --git a/bridges/EZTVBridge.php b/bridges/EZTVBridge.php index 73318f0c713..25a88124266 100644 --- a/bridges/EZTVBridge.php +++ b/bridges/EZTVBridge.php @@ -96,7 +96,7 @@ protected function getEztvUri() protected function getItemFromTorrent($torrent) { $item = []; - $item['uri'] = $torrent->episode_url; + $item['uri'] = $torrent->episode_url ?? $torrent->torrent_url; $item['author'] = $torrent->imdb_id; $item['timestamp'] = $torrent->date_released_unix; $item['title'] = $torrent->title; diff --git a/bridges/TrelloBridge.php b/bridges/TrelloBridge.php index a1b5cfb8567..cab2bde2880 100644 --- a/bridges/TrelloBridge.php +++ b/bridges/TrelloBridge.php @@ -648,7 +648,7 @@ public function collectData() $action->type ]; if (isset($action->data->card)) { - $item['categories'][] = $action->data->card->name; + $item['categories'][] = $action->data->card->name ?? $action->data->card->id; $item['uri'] = 'https://trello.com/c/' . $action->data->card->shortLink . '#action-' diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 993f8c90663..6a29e387158 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -164,7 +164,11 @@ private function collectDataInternal() $jsonData = $this->extractJsonFromHtml($html); // TODO: this method returns only first 100 video items // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0] ?? null; + if (!$jsonData) { + // playlist probably doesnt exists + throw new \Exception('Unable to find playlist: ' . $url_listing); + } $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; $item_count = count($jsonData); From ea2b4d7506f0feded2899cb0aab351fa7dca3194 Mon Sep 17 00:00:00 2001 From: July Date: Sat, 23 Dec 2023 03:42:37 -0500 Subject: [PATCH 52/69] [ArsTechnicaBridge] Properly handle paged content (#3855) * [ArsTechnicaBridge] Properly handle paged content * [ArsTechnicaBridge] Remove normal site ad wrapper --- bridges/ArsTechnicaBridge.php | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 613c1c58ca4..2c631871caf 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -35,39 +35,34 @@ public function collectData() protected function parseItem(array $item) { - $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); + $item_html = getSimpleHTMLDOMCached($item['uri']); $item_html = defaultLinkTo($item_html, self::URI); + $item['content'] = $item_html->find('.article-content', 0); - $item_content = $item_html->find('.article-content.post-page', 0); - if (!$item_content) { - // The dom selector probably broke. Let's just return the item as-is - return $item; + $pages = $item_html->find('nav.page-numbers > .numbers > a', -2); + if (null !== $pages) { + for ($i = 2; $i <= $pages->innertext; $i++) { + $page_url = $item['uri'] . '&page=' . $i; + $page_html = getSimpleHTMLDOMCached($page_url); + $page_html = defaultLinkTo($page_html, self::URI); + $item['content'] .= $page_html->find('.article-content', 0); + } + $item['content'] = str_get_html($item['content']); } - $item['content'] = $item_content; - // remove various ars advertising $item['content']->find('#social-left', 0)->remove(); foreach ($item['content']->find('.ars-component-buy-box') as $ad) { $ad->remove(); } - foreach ($item['content']->find('i-amphtml-sizer') as $ad) { + foreach ($item['content']->find('.ad_wrapper') as $ad) { $ad->remove(); } foreach ($item['content']->find('.sidebar') as $ad) { $ad->remove(); } - foreach ($item['content']->find('a') as $link) { //remove amp redirect links - $url = $link->getAttribute('href'); - if (str_contains($url, 'go.redirectingat.com')) { - $url = extractFromDelimiters($url, 'url=', '&'); - $url = urldecode($url); - $link->setAttribute('href', $url); - } - } - - $item['content'] = backgroundToImg(str_replace('data-amp-original-style="background-image', 'style="background-image', $item['content'])); + $item['content'] = backgroundToImg($item['content']); $item['uid'] = explode('=', $item['uri'])[1]; From 98dafb61ae5519b7c6c4be2d7dd4d66b6bd6a4eb Mon Sep 17 00:00:00 2001 From: xduugu Date: Sat, 23 Dec 2023 08:43:01 +0000 Subject: [PATCH 53/69] [ARDAudiothekBridge] add duration to feed items (#3854) --- bridges/ARDAudiothekBridge.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bridges/ARDAudiothekBridge.php b/bridges/ARDAudiothekBridge.php index 619c0911f06..02b6b00778d 100644 --- a/bridges/ARDAudiothekBridge.php +++ b/bridges/ARDAudiothekBridge.php @@ -125,6 +125,10 @@ public function collectData() $item['categories'] = [$category]; } + $item['itunes'] = [ + 'duration' => $audio->duration, + ]; + $this->items[] = $item; } } From 9f163ab7c651f44c1d6266ca817aca2c0f208f51 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Mon, 25 Dec 2023 14:51:51 +0100 Subject: [PATCH 54/69] [FreeTelechargerBridge] Update to the new URL (#3856) * [FreeTelechargerBridge] Update to the new URL Website has changed URL and some design : this bridge is now adapted to thoses changes * [FreeTelechargerBridge] Fix example value Example valuse seems to use an "old" template, switch to a newer example that use the new template * [FreeTelechargerBridge] Fix notice Fix notice --- bridges/FreeTelechargerBridge.php | 61 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/bridges/FreeTelechargerBridge.php b/bridges/FreeTelechargerBridge.php index 8362b4ff74c..f0e5d35a5bb 100644 --- a/bridges/FreeTelechargerBridge.php +++ b/bridges/FreeTelechargerBridge.php @@ -3,7 +3,7 @@ class FreeTelechargerBridge extends BridgeAbstract { const NAME = 'Free-Telecharger'; - const URI = 'https://www.free-telecharger.live/'; + const URI = 'https://www.free-telecharger.art/'; const DESCRIPTION = 'Suivi de série sur Free-Telecharger'; const MAINTAINER = 'sysadminstory'; const PARAMETERS = [ @@ -12,43 +12,46 @@ class FreeTelechargerBridge extends BridgeAbstract 'name' => 'URL de la série', 'type' => 'text', 'required' => true, - 'title' => 'URL d\'une série sans le https://www.free-telecharger.live/', + 'title' => 'URL d\'une série sans le https://www.free-telecharger.art/', 'pattern' => 'series.*\.html', - 'exampleValue' => 'series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html' + 'exampleValue' => 'series-vf-hd/151432-wolf-saison-1-complete-web-dl-720p.html' ], ] ]; const CACHE_TIMEOUT = 3600; + private string $showTitle; + private string $showTechDetails; + public function collectData() { - $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')); + $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')); - // Find all block content of the page - $blocks = $html->find('div[class=block1]'); + // Find all block content of the page + $blocks = $html->find('div[class=block1]'); - // Global Infos block - $infosBlock = $blocks[0]; - // Links block - $linksBlock = $blocks[2]; + // Global Infos block + $infosBlock = $blocks[0]; + // Links block + $linksBlock = $blocks[2]; - // Extract Global Show infos - $this->showTitle = trim($infosBlock->find('div[class=titre1]', 0)->find('font', 0)->plaintext); - $this->showTechDetails = trim($infosBlock->find('div[align=center]', 0)->find('b', 0)->plaintext); + // Extract Global Show infos + $this->showTitle = trim($infosBlock->find('div[class=titre1]', 0)->find('font', 0)->plaintext); + $this->showTechDetails = trim($infosBlock->find('div[align=center]', 0)->find('b', 0)->plaintext); - // Get Episodes names and links - $episodes = $linksBlock->find('div[id=link]', 0)->find('font[color=#ff6600]'); - $links = $linksBlock->find('div[id=link]', 0)->find('a'); + // Get Episodes names and links + $episodes = $linksBlock->find('div[id=link]', 0)->find('font[color=#e93100]'); + $links = $linksBlock->find('div[id=link]', 0)->find('a'); foreach ($episodes as $index => $episode) { - $item = []; // Create an empty item - $item['title'] = $this->showTitle . ' ' . $this->showTechDetails . ' - ' . ltrim(trim($episode->plaintext), '-'); - $item['uri'] = $links[$index]->href; - $item['content'] = '' . $item['title'] . ''; - $item['uid'] = hash('md5', $item['uri']); + $item = []; // Create an empty item + $item['title'] = $this->showTitle . ' ' . $this->showTechDetails . ' - ' . ltrim(trim($episode->plaintext), '-'); + $item['uri'] = $links[$index]->href; + $item['content'] = '' . $item['title'] . ''; + $item['uid'] = hash('md5', $item['uri']); - $this->items[] = $item; // Add this item to the list + $this->items[] = $item; // Add this item to the list } } @@ -57,7 +60,7 @@ public function getName() switch ($this->queriedContext) { case 'Suivi de publication de série': return $this->showTitle . ' ' . $this->showTechDetails . ' - ' . self::NAME; - break; + break; default: return self::NAME; } @@ -68,7 +71,7 @@ public function getURI() switch ($this->queriedContext) { case 'Suivi de publication de série': return self::URI . $this->getInput('url'); - break; + break; default: return self::URI; } @@ -76,14 +79,14 @@ public function getURI() public function detectParameters($url) { - // Example: https://www.free-telecharger.live/series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html + // Example: https://www.free-telecharger.art/series-vf-hd/151432-wolf-saison-1-complete-web-dl-720p.html $params = []; - $regex = '/^https:\/\/www.*\.free-telecharger\.live\/(series.*\.html)/'; + $regex = '/^https:\/\/www.*\.free-telecharger\.art\/(series.*\.html)/'; if (preg_match($regex, $url, $matches) > 0) { - $params['context'] = 'Suivi de publication de série'; - $params['url'] = urldecode($matches[1]); - return $params; + $params['context'] = 'Suivi de publication de série'; + $params['url'] = urldecode($matches[1]); + return $params; } return null; From c9074facfed51371a59dd189648c5a80751feb4e Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 26 Dec 2023 12:18:42 +0100 Subject: [PATCH 55/69] [GreatFonBridge] Remove bridge (#3857) Website is unreliable, it's not useful to keep this bridge. --- bridges/GreatFonBridge.php | 140 ------------------------------------- 1 file changed, 140 deletions(-) delete mode 100644 bridges/GreatFonBridge.php diff --git a/bridges/GreatFonBridge.php b/bridges/GreatFonBridge.php deleted file mode 100644 index 2951634c15f..00000000000 --- a/bridges/GreatFonBridge.php +++ /dev/null @@ -1,140 +0,0 @@ - [ - 'u' => [ - 'name' => 'username', - 'type' => 'text', - 'title' => 'Instagram username you want to follow', - 'exampleValue' => 'aesoprockwins', - 'required' => true, - ], - ] - ]; - const TEST_DETECT_PARAMETERS = [ - 'https://www.instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], - 'https://instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], - 'https://greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], - 'https://www.greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], - ]; - - public function collectData() - { - $username = $this->getInput('u'); - $html = getSimpleHTMLDOMCached(self::URI . '/v/' . $username); - $html = defaultLinkTo($html, self::URI); - - foreach ($html->find('div[class*=content__item]') as $post) { - // Skip the ads - if (!str_contains($post->class, 'ads')) { - $url = $post->find('a[href^=https://greatfon.com/c/]', 0)->href; - $date = $this->parseDate($post->find('div[class=content__time-text]', 0)->plaintext); - $description = $post->find('img', 0)->alt; - $imageUrl = $post->find('img', 0)->src; - $author = $username; - $uid = $url; - $title = 'Post - ' . $username . ' - ' . $this->descriptionToTitle($description); - - // Checking post type - $isVideo = (bool) $post->find('div[class=content__camera]', 0); - $videoNote = $isVideo ? '

(video)

' : ''; - - $this->items[] = [ - 'uri' => $url, - 'author' => $author, - 'timestamp' => $date, - 'title' => $title, - 'thumbnail' => $imageUrl, - 'enclosures' => [$imageUrl], - 'content' => << - {$description} - -{$videoNote} -

{$description}

-HTML, - 'uid' => $uid - ]; - } - } - } - - private function parseDate($content) - { - // Parse date, and transform the date into a timetamp, even in a case of a relative date - $date = date_create(); - - // Content trimmed to be sure that the "article" is at the beginning of the string and remove "ago" to make it a valid PHP date interval - $dateString = trim(str_replace(' ago', '', $content)); - - // Replace the article "an" or "a" by the number "1" to be a valid PHP date interval - $dateString = preg_replace('/^((an|a) )/m', '1 ', $dateString); - - $relativeDate = date_interval_create_from_date_string($dateString); - if ($relativeDate) { - date_sub($date, $relativeDate); - // As the relative interval has the precision of a day for date older than 24 hours, we can remove the hour of the date, as it is not relevant - date_time_set($date, 0, 0, 0, 0); - } else { - $this->logger->info(sprintf('Unable to parse date string: %s', $dateString)); - } - return date_format($date, 'r'); - } - - public function getURI() - { - if (!is_null($this->getInput('u'))) { - return urljoin(self::URI, '/v/' . $this->getInput('u')); - } - - return parent::getURI(); - } - - public function getIcon() - { - return static::URI . '/images/favicon-hub-3ede543aa6d1225e8dc016ccff6879c8.ico?vsn=d'; - } - - private function descriptionToTitle($description) - { - return strlen($description) > 60 ? mb_substr($description, 0, 57) . '...' : $description; - } - - public function getName() - { - if (!is_null($this->getInput('u'))) { - return 'Username ' . $this->getInput('u') . ' - GreatFon Bridge'; - } - return parent::getName(); - } - - public function detectParameters($url) - { - $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})(\/reels\/|\/tagged\/|\/|)|(www\.|)(greatfon.com)\/v\/([a-zA-Z0-9_\.]{1,30}))/'; - if (preg_match($regex, $url, $matches) > 0) { - $params['context'] = 'Username'; - // Extract detected domain using the regex - $domain = $matches[8] ?? $matches[4]; - if ($domain == 'greatfon.com') { - $params['u'] = $matches[9]; - return $params; - } elseif ($domain == 'instagram.com') { - $params['u'] = $matches[5]; - return $params; - } else { - return null; - } - } else { - return null; - } - } -} From 19384463857c35b1d3ef0a7dbbbcc40d2f0cba0c Mon Sep 17 00:00:00 2001 From: Florent V Date: Tue, 26 Dec 2023 12:19:08 +0100 Subject: [PATCH 56/69] [EdfPricesBridge] add new bridge (#3846) * [EdfPricesBridge] add new brige * [EdfPricesBridge] bad refactor * [EdfPricesBridge] support php 7.4 --------- Co-authored-by: Florent VIOLLEAU --- bridges/EdfPricesBridge.php | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 bridges/EdfPricesBridge.php diff --git a/bridges/EdfPricesBridge.php b/bridges/EdfPricesBridge.php new file mode 100644 index 00000000000..f67ed30b1c7 --- /dev/null +++ b/bridges/EdfPricesBridge.php @@ -0,0 +1,106 @@ + [ + 'name' => 'Choisir un contrat', + 'type' => 'list', + // we can add later HCHP, EJP, base + 'values' => ['Tempo' => '/energie/edf/tarifs/tempo'], + ] + ] + ]; + const CACHE_TIMEOUT = 7200; // 2h + + /** + * @param simple_html_dom $html + * @param string $contractUri + * @return void + */ + private function tempo(simple_html_dom $html, string $contractUri): void + { + // current color and next + $daysDom = $html->find('#calendrier', 0)->nextSibling()->find('.card--ejp'); + if ($daysDom && count($daysDom) === 2) { + foreach ($daysDom as $dayDom) { + $day = trim($dayDom->find('.card__title', 0)->innertext) . '/' . (new \DateTime('now'))->format(('Y')); + $dayColor = $dayDom->find('.card-ejp__icon span', 0)->innertext; + + $text = $day . ' - ' . $dayColor; + $item['uri'] = self::URI . $contractUri; + $item['title'] = $text; + $item['author'] = self::MAINTAINER; + $item['content'] = $text; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + } + + // colors + $ulDom = $html->find('#tarif-de-l-offre-edf-tempo-current-date-html-year', 0)->nextSibling()->nextSibling()->nextSibling(); + $elementsDom = $ulDom->find('li'); + if ($elementsDom && count($elementsDom) === 3) { + foreach ($elementsDom as $elementDom) { + $item = []; + + $matches = []; + preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0); + + if ($matches && count($matches[0]) === 6) { + for ($i = 0; $i < 2; $i++) { + $text = 'Jour ' . $matches[0][1] . ' - Heures ' . $matches[0][2 + 2 * $i] . ' : ' . $matches[0][3 + 2 * $i] . '€'; + $item['uri'] = self::URI . $contractUri; + $item['title'] = $text; + $item['author'] = self::MAINTAINER; + $item['content'] = $text; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + } + } + } + + // powers + $ulPowerContract = $ulDom->nextSibling()->nextSibling(); + $elementsPowerContractDom = $ulPowerContract->find('li'); + if ($elementsPowerContractDom && count($elementsPowerContractDom) === 4) { + foreach ($elementsPowerContractDom as $elementPowerContractDom) { + $item = []; + + $matches = []; + preg_match_all('/(.*) kVA : (.*) €/um', $elementPowerContractDom->innertext, $matches, PREG_SET_ORDER, 0); + + if ($matches && count($matches[0]) === 3) { + $text = $matches[0][1] . ' kVA : ' . $matches[0][2] . '€'; + $item['uri'] = self::URI . $contractUri; + $item['title'] = $text; + $item['author'] = self::MAINTAINER; + $item['content'] = $text; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + } + } + } + + public function collectData() + { + $contract = $this->getKey('contract'); + $contractUri = $this->getInput('contract'); + $html = getSimpleHTMLDOM(self::URI . $contractUri); + + if ($contract === 'Tempo') { + $this->tempo($html, $contractUri); + } + } +} From ad2d4c7b1b538868070e0264f3692542883cac50 Mon Sep 17 00:00:00 2001 From: Florent V Date: Tue, 26 Dec 2023 12:20:49 +0100 Subject: [PATCH 57/69] [BridgeAbstract] use getParameters instead of static to allow overriding it from bridges (#3858) --- lib/BridgeAbstract.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index a7b811a84da..0f86f454c0d 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -154,8 +154,8 @@ private function setInputWithContext(array $input, $queriedContext) { // Import and assign all inputs to their context foreach ($input as $name => $value) { - foreach (static::PARAMETERS as $context => $set) { - if (array_key_exists($name, static::PARAMETERS[$context])) { + foreach ($this->getParameters() as $context => $set) { + if (array_key_exists($name, $this->getParameters()[$context])) { $this->inputs[$context][$name]['value'] = $value; } } @@ -163,16 +163,16 @@ private function setInputWithContext(array $input, $queriedContext) // Apply default values to missing data $contexts = [$queriedContext]; - if (array_key_exists('global', static::PARAMETERS)) { + if (array_key_exists('global', $this->getParameters())) { $contexts[] = 'global'; } foreach ($contexts as $context) { - if (!isset(static::PARAMETERS[$context])) { + if (!isset($this->getParameters()[$context])) { // unknown context provided by client, throw exception here? or continue? } - foreach (static::PARAMETERS[$context] as $name => $properties) { + foreach ($this->getParameters()[$context] as $name => $properties) { if (isset($this->inputs[$context][$name]['value'])) { continue; } @@ -204,8 +204,8 @@ private function setInputWithContext(array $input, $queriedContext) } // Copy global parameter values to the guessed context - if (array_key_exists('global', static::PARAMETERS)) { - foreach (static::PARAMETERS['global'] as $name => $properties) { + if (array_key_exists('global', $this->getParameters())) { + foreach ($this->getParameters()['global'] as $name => $properties) { if (isset($input[$name])) { $value = $input[$name]; } else { @@ -246,8 +246,8 @@ public function getKey($input) if (!isset($this->inputs[$this->queriedContext][$input]['value'])) { return null; } - if (array_key_exists('global', static::PARAMETERS)) { - if (array_key_exists($input, static::PARAMETERS['global'])) { + if (array_key_exists('global', $this->getParameters())) { + if (array_key_exists($input, $this->getParameters()['global'])) { $context = 'global'; } } @@ -256,7 +256,7 @@ public function getKey($input) } $needle = $this->inputs[$this->queriedContext][$input]['value']; - foreach (static::PARAMETERS[$context][$input]['values'] as $first_level_key => $first_level_value) { + foreach ($this->getParameters()[$context][$input]['values'] as $first_level_key => $first_level_value) { if (!is_array($first_level_value) && $needle === (string)$first_level_value) { return $first_level_key; } elseif (is_array($first_level_value)) { @@ -273,7 +273,7 @@ public function detectParameters($url) { $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; if ( - empty(static::PARAMETERS) + empty($this->getParameters()) && preg_match($regex, $url, $urlMatches) > 0 && preg_match($regex, static::URI, $bridgeUriMatches) > 0 && $urlMatches[3] === $bridgeUriMatches[3] From c8178e1fc409635af1a40167c4f511feb8d3df7f Mon Sep 17 00:00:00 2001 From: Damien Calesse <2787828+kranack@users.noreply.github.com> Date: Wed, 27 Dec 2023 13:17:49 +0100 Subject: [PATCH 58/69] [SensCritique] Fix bridge (#3860) --- bridges/SensCritiqueBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php index b823b55c23a..005704e169e 100644 --- a/bridges/SensCritiqueBridge.php +++ b/bridges/SensCritiqueBridge.php @@ -57,7 +57,7 @@ public function collectData() } $html = getSimpleHTMLDOM($uri); // This selector name looks like it's automatically generated - $list = $html->find('div.Universes__WrapperProducts-sc-1qa2w66-0.eVdcAv', 0); + $list = $html->find('div[data-testid="row"]', 0); $this->extractDataFromList($list); } @@ -69,6 +69,7 @@ private function extractDataFromList($list) if ($list === null) { returnClientError('Cannot extract data from list'); } + foreach ($list->find('div[data-testid="product-list-item"]') as $movie) { $item = []; $item['title'] = $movie->find('h2 a', 0)->plaintext; From 5ab1924c4f96937885e12bcbd16b7bfb83a3c15b Mon Sep 17 00:00:00 2001 From: tillcash Date: Thu, 28 Dec 2023 18:20:34 +0530 Subject: [PATCH 59/69] Add WorldbankBridge and OglafBridge (#3862) * Add WorldbankBridge and OglafBridge * Update OglafBridge.php Remove redundant parent call to parseItem and rename formal argument to improve code clarity. * Update WorldbankBridge.php fix lint --- bridges/OglafBridge.php | 35 +++++++++++++++++++++++++ bridges/WorldbankBridge.php | 52 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 bridges/OglafBridge.php create mode 100644 bridges/WorldbankBridge.php diff --git a/bridges/OglafBridge.php b/bridges/OglafBridge.php new file mode 100644 index 00000000000..1f4bc1aff9e --- /dev/null +++ b/bridges/OglafBridge.php @@ -0,0 +1,35 @@ + [ + 'name' => 'limit (max 20)', + 'type' => 'number', + 'defaultValue' => 10, + 'required' => true, + ] + ] + ]; + + public function collectData() + { + $url = self::URI . 'feeds/rss/'; + $limit = min(20, $this->getInput('limit')); + $this->collectExpandableDatas($url, $limit); + } + + protected function parseItem($item) + { + $html = getSimpleHTMLDOMCached($item['uri']); + $comicImage = $html->find('img[id="strip"]', 0); + $item['content'] = $comicImage; + + return $item; + } +} diff --git a/bridges/WorldbankBridge.php b/bridges/WorldbankBridge.php new file mode 100644 index 00000000000..9b40e86e5da --- /dev/null +++ b/bridges/WorldbankBridge.php @@ -0,0 +1,52 @@ + [ + 'name' => 'Language', + 'type' => 'list', + 'defaultValue' => 'English', + 'values' => [ + 'English' => 'English', + 'French' => 'French', + ] + ], + 'limit' => [ + 'name' => 'limit (max 100)', + 'type' => 'number', + 'defaultValue' => 5, + 'required' => true, + ] + ] + ]; + + public function collectData() + { + $apiUrl = 'https://search.worldbank.org/api/v2/news?format=json&rows=' + . min(100, $this->getInput('limit')) + . '&lang_exact=' . $this->getInput('lang'); + + $jsonData = json_decode(getContents($apiUrl)); + + // Remove unnecessary data from the original object + if (isset($jsonData->documents->facets)) { + unset($jsonData->documents->facets); + } + + foreach ($jsonData->documents as $element) { + $this->items[] = [ + 'uid' => $element->id, + 'timestamp' => $element->lnchdt, + 'title' => $element->title->{'cdata!'}, + 'uri' => $element->url, + 'content' => $element->descr->{'cdata!'}, + ]; + } + } +} From f67d2eb88adc597cc57fbfc402c28725b671e5a3 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Thu, 28 Dec 2023 13:53:06 +0100 Subject: [PATCH 60/69] [TikTokBridge] Use embed iframe to bypass scraping protection (#3864) The Tiktok Website was totally changed using some "scraping" protection (passing as parameter value generated somewhere in the bunch of javascript to the "API URL" that was before). The iframe embed does not have such protection. It has less information (no date, ...) but it's better than nothing ! --- bridges/TikTokBridge.php | 66 ++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index 73a18b0468c..6590df66808 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -8,12 +8,12 @@ class TikTokBridge extends BridgeAbstract const MAINTAINER = 'VerifiedJoseph'; const PARAMETERS = [ 'By user' => [ - 'username' => [ - 'name' => 'Username', - 'type' => 'text', - 'required' => true, - 'exampleValue' => '@tiktok', - ] + 'username' => [ + 'name' => 'Username', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '@tiktok', + ] ]]; const TEST_DETECT_PARAMETERS = [ @@ -24,53 +24,33 @@ class TikTokBridge extends BridgeAbstract const CACHE_TIMEOUT = 900; // 15 minutes - private $feedName = ''; - public function collectData() { - $html = getSimpleHTMLDOM($this->getURI()); + $html = getSimpleHTMLDOMCached('https://www.tiktok.com/embed/' . $this->processUsername()); - $title = $html->find('h1', 0)->plaintext ?? self::NAME; - $this->feedName = htmlspecialchars_decode($title); + $author = $html->find('span[data-e2e=creator-profile-userInfo-TUXText]', 0)->plaintext ?? self::NAME; - $var = $html->find('script[id=SIGI_STATE]', 0); - if (!$var) { - throw new \Exception('Unable to find tiktok user data for ' . $this->processUsername()); - } - $SIGI_STATE_RAW = $var->innertext; - $SIGI_STATE = Json::decode($SIGI_STATE_RAW, false); + $videos = $html->find('div[data-e2e=common-videoList-VideoContainer]'); - if (!isset($SIGI_STATE->ItemModule)) { - return; - } - - foreach ($SIGI_STATE->ItemModule as $key => $value) { + foreach ($videos as $video) { $item = []; - $link = 'https://www.tiktok.com/@' . $value->author . '/video/' . $value->id; - $image = $value->video->dynamicCover; - if (empty($image)) { - $image = $value->video->cover; - } - $views = $value->stats->playCount; - $hastags = []; - foreach ($value->textExtra as $tag) { - $hastags[] = $tag->hashtagName; - } - $hastags_str = ''; - foreach ($hastags as $tag) { - $hastags_str .= '#' . $tag . ' '; - } + // Handle link "untracking" + $linkParts = parse_url($video->find('a', 0)->href); + $link = $linkParts['scheme'] . '://' . $linkParts['host'] . '/' . $linkParts['path']; + + $image = $video->find('video', 0)->poster; + $views = $video->find('div[data-e2e=common-Video-Count]', 0)->plaintext; + + $enclosures = [$image]; $item['uri'] = $link; - $item['title'] = $value->desc; - $item['timestamp'] = $value->createTime; - $item['author'] = '@' . $value->author; - $item['enclosures'][] = $image; - $item['categories'] = $hastags; + $item['title'] = 'Video'; + $item['author'] = '@' . $author; + $item['enclosures'] = $enclosures; $item['content'] = << -

{$views} views


Hashtags: {$hastags_str} +

{$views} views


EOD; $this->items[] = $item; @@ -91,7 +71,7 @@ public function getName() { switch ($this->queriedContext) { case 'By user': - return $this->feedName . ' (' . $this->processUsername() . ') - TikTok'; + return $this->processUsername() . ' - TikTok'; default: return parent::getName(); } From 2032ed18c49a82fc2e634dfa6f2b91e652228876 Mon Sep 17 00:00:00 2001 From: Damien Calesse <2787828+kranack@users.noreply.github.com> Date: Thu, 28 Dec 2023 19:51:15 +0100 Subject: [PATCH 61/69] [SensCritique] Update the content to add the image (#3865) --- bridges/SensCritiqueBridge.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php index 005704e169e..f6a2ea16142 100644 --- a/bridges/SensCritiqueBridge.php +++ b/bridges/SensCritiqueBridge.php @@ -71,10 +71,17 @@ private function extractDataFromList($list) } foreach ($list->find('div[data-testid="product-list-item"]') as $movie) { + $synopsis = $movie->find('p[data-testid="synopsis"]', 0); + $item = []; $item['title'] = $movie->find('h2 a', 0)->plaintext; - // todo: fix image - $item['content'] = $movie->innertext; + $item['content'] = sprintf( + '

%s

%s

%s', + $movie->find('span[data-testid="poster-img-wrapper"]', 0)->{'data-srcname'}, + $movie->find('p[data-testid="other-infos"]', 0)->innertext, + $movie->find('p[data-testid="creators"]', 0)->innertext, + $synopsis ? sprintf('

%s

', $synopsis->innertext) : '' + ); $item['id'] = $this->getURI() . ltrim($movie->find('a', 0)->href, '/'); $item['uri'] = $this->getURI() . ltrim($movie->find('a', 0)->href, '/'); $this->items[] = $item; From 7dbe10658213e165c07faac01a8c79771b4917c8 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 28 Dec 2023 23:26:14 +0100 Subject: [PATCH 62/69] docs(nginx, phpfpm): improve install and config instructions (#3866) --- README.md | 162 ++++++++++++++++++++++++++++++++++++------- caches/FileCache.php | 1 + index.php | 3 +- 3 files changed, 140 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 2a762d45763..34efc8de3e7 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,15 @@ ![RSS-Bridge](static/logo_600px.png) -RSS-Bridge is a web application. +RSS-Bridge is a PHP web application. It generates web feeds for websites that don't have one. Officially hosted instance: https://rss-bridge.org/bridge01/ +IRC channel #rssbridge at https://libera.chat/ + + [![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest) [![irc.libera.chat](https://img.shields.io/badge/irc.libera.chat-%23rssbridge-blue.svg)](https://web.libera.chat/#rssbridge) @@ -48,54 +51,147 @@ Check out RSS-Bridge right now on https://rss-bridge.org/bridge01/ Alternatively find another [public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html). +Requires minimum PHP 7.4. + ## Tutorial -### Install with composer or git +### How to install on traditional shared web hosting -Requires minimum PHP 7.4. +RSS-Bridge can basically be unzipped in a web folder. Should be working instantly. -```shell -apt install nginx php-fpm php-mbstring php-simplexml php-curl -``` +Latest zip as of Sep 2023: https://github.com/RSS-Bridge/rss-bridge/archive/refs/tags/2023-09-24.zip -```shell -cd /var/www -composer create-project -v --no-dev rss-bridge/rss-bridge -``` +### How to install on Debian 12 (nginx + php-fpm) + +These instructions have been tested on a fresh Debian 12 VM from Digital Ocean (1vcpu-512mb-10gb, 5 USD/month). ```shell +timedatectl set-timezone Europe/Oslo + +apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl + +# Create a new user account +useradd --shell /bin/bash --create-home rss-bridge + cd /var/www -git clone https://github.com/RSS-Bridge/rss-bridge.git -``` -Config: +# Create folder and change ownership +mkdir rss-bridge && chown rss-bridge:rss-bridge rss-bridge/ -```shell -# Give the http user write permission to the cache folder -chown www-data:www-data /var/www/rss-bridge/cache +# Become user +su rss-bridge + +# Fetch latest master +git clone https://github.com/RSS-Bridge/rss-bridge.git rss-bridge/ +cd rss-bridge + +# Copy over the default config +cp -v config.default.ini.php config.ini.php -# Optionally copy over the default config file -cp config.default.ini.php config.ini.php +# Give full permissions only to owner (rss-bridge) +chmod 700 -R ./ + +# Give read and execute to others (nginx and php-fpm) +chmod o+rx ./ ./static + +# Give read to others (nginx) +chmod o+r -R ./static ``` -Example config for nginx: +Nginx config: ```nginx -# /etc/nginx/sites-enabled/rssbridge +# /etc/nginx/sites-enabled/rss-bridge.conf + server { listen 80; server_name example.com; - root /var/www/rss-bridge; - index index.php; + access_log /var/log/nginx/rss-bridge.access.log; + error_log /var/log/nginx/rss-bridge.error.log; + + # Intentionally not setting a root folder here + + # autoindex is off by default but feels good to explicitly turn off + autoindex off; - location ~ \.php$ { + # Static content only served here + location /static/ { + alias /var/www/rss-bridge/static/; + } + + # Pass off to php-fpm only when location is exactly / + location = / { + root /var/www/rss-bridge/; include snippets/fastcgi-php.conf; - fastcgi_read_timeout 60s; - fastcgi_pass unix:/run/php/php-fpm.sock; + fastcgi_pass unix:/run/php/rss-bridge.sock; + } + + # Reduce spam + location = /favicon.ico { + access_log off; + log_not_found off; + } + + # Reduce spam + location = /robots.txt { + access_log off; + log_not_found off; } } ``` +PHP FPM pool config: +```ini +; /etc/php/8.2/fpm/pool.d/rss-bridge.conf + +[rss-bridge] + +user = rss-bridge +group = rss-bridge + +listen = /run/php/rss-bridge.sock + +listen.owner = www-data +listen.group = www-data + +pm = static +pm.max_children = 10 +pm.max_requests = 500 +``` + +PHP ini config: +```ini +; /etc/php/8.2/fpm/conf.d/30-rss-bridge.ini + +max_execution_time = 20 +memory_limit = 64M +``` + +Restart fpm and nginx: + +```shell +# Lint and restart php-fpm +php-fpm8.2 -t +systemctl restart php8.2-fpm + +# Lint and restart nginx +nginx -t +systemctl restart nginx +``` + +### How to install from Composer + +Install the latest release. + +```shell +cd /var/www +composer create-project -v --no-dev rss-bridge/rss-bridge +``` + +### How to install with Caddy + +TODO. See https://github.com/RSS-Bridge/rss-bridge/issues/3785 + ### Install from Docker Hub: Install by downloading the docker image from Docker Hub: @@ -163,6 +259,22 @@ Learn more in ## How-to +### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable" + +```shell +# Give rssbridge ownership +chown rssbridge:rssbridge -R /var/www/rss-bridge/cache + +# Or, give www-data ownership +chown www-data:www-data -R /var/www/rss-bridge/cache + +# Or, give everyone write permission +chmod 777 -R /var/www/rss-bridge/cache + +# Or last ditch effort (CAREFUL) +rm -rf /var/www/rss-bridge/cache/ && mkdir /var/www/rss-bridge/cache/ +``` + ### How to create a new bridge from scratch Create the new bridge in e.g. `bridges/BearBlogBridge.php`: diff --git a/caches/FileCache.php b/caches/FileCache.php index 09d127910ac..7a0eb81d95e 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -54,6 +54,7 @@ public function set($key, $value, int $ttl = null): void ]; $cacheFile = $this->createCacheFile($key); $bytes = file_put_contents($cacheFile, serialize($item), LOCK_EX); + // todo: Consider tightening the permissions of the created file. It usually allow others to read, depending on umask if ($bytes === false) { // Consider just logging the error here throw new \Exception(sprintf('Failed to write to: %s', $cacheFile)); diff --git a/index.php b/index.php index 14713e06f75..c2c546a184e 100644 --- a/index.php +++ b/index.php @@ -8,7 +8,8 @@ $errors = Configuration::checkInstallation(); if ($errors) { - die('
' . implode("\n", $errors) . '
'); + print '
' . implode("\n", $errors) . '
'; + exit(1); } $customConfig = []; From fac1f5cd88f04855a891aeb7341f783e57ce5b3c Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 30 Dec 2023 01:33:31 +0100 Subject: [PATCH 63/69] refactor(reddit) (#3869) * refactor * yup * fix also reporterre --- bridges/RedditBridge.php | 66 +++++++++++------------------------- bridges/ReporterreBridge.php | 44 +++++++++++++----------- 2 files changed, 44 insertions(+), 66 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index bb3e7afcf38..618463a642d 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -173,7 +173,7 @@ private function collectDataInternal(): void $item['author'] = $data->author; $item['uid'] = $data->id; $item['timestamp'] = $data->created_utc; - $item['uri'] = $this->encodePermalink($data->permalink); + $item['uri'] = $this->urlEncodePathParts($data->permalink); $item['categories'] = []; @@ -193,13 +193,11 @@ private function collectDataInternal(): void if ($post->kind == 't1') { // Comment - $item['content'] - = htmlspecialchars_decode($data->body_html); + $item['content'] = htmlspecialchars_decode($data->body_html); } elseif ($data->is_self) { // Text post - $item['content'] - = htmlspecialchars_decode($data->selftext_html); + $item['content'] = htmlspecialchars_decode($data->selftext_html); } elseif (isset($data->post_hint) && $data->post_hint == 'link') { // Link with preview @@ -215,18 +213,11 @@ private function collectDataInternal(): void $embed = ''; } - $item['content'] = $this->template( - $data->url, - $data->thumbnail, - $data->domain - ) . $embed; - } elseif (isset($data->post_hint) ? $data->post_hint == 'image' : false) { + $item['content'] = $this->createFigureLink($data->url, $data->thumbnail, $data->domain) . $embed; + } elseif (isset($data->post_hint) && $data->post_hint == 'image') { // Single image - $item['content'] = $this->link( - $this->encodePermalink($data->permalink), - '' - ); + $item['content'] = $this->createLink($this->urlEncodePathParts($data->permalink), ''); } elseif ($data->is_gallery ?? false) { // Multiple images @@ -246,32 +237,18 @@ private function collectDataInternal(): void end($data->preview->images[0]->resolutions); $index = key($data->preview->images[0]->resolutions); - $item['content'] = $this->template( - $data->url, - $data->preview->images[0]->resolutions[$index]->url, - 'Video' - ); - } elseif (isset($data->media) ? $data->media->type == 'youtube.com' : false) { + $item['content'] = $this->createFigureLink($data->url, $data->preview->images[0]->resolutions[$index]->url, 'Video'); + } elseif (isset($data->media) && $data->media->type == 'youtube.com') { // Youtube link - - $item['content'] = $this->template( - $data->url, - $data->media->oembed->thumbnail_url, - 'YouTube' - ); + $item['content'] = $this->createFigureLink($data->url, $data->media->oembed->thumbnail_url, 'YouTube'); + //$item['content'] = htmlspecialchars_decode($data->media->oembed->html); } elseif (explode('.', $data->domain)[0] == 'self') { // Crossposted text post // TODO (optionally?) Fetch content of the original post. - - $item['content'] = $this->link( - $this->encodePermalink($data->permalink), - 'Crossposted from r/' - . explode('.', $data->domain)[1] - ); + $item['content'] = $this->createLink($this->urlEncodePathParts($data->permalink), 'Crossposted from r/' . explode('.', $data->domain)[1]); } else { // Link WITHOUT preview - - $item['content'] = $this->link($data->url, $data->domain); + $item['content'] = $this->createLink($data->url, $data->domain); } $this->items[] = $item; @@ -279,7 +256,7 @@ private function collectDataInternal(): void } // Sort the order to put the latest posts first, even for mixed subreddits usort($this->items, function ($a, $b) { - return $a['timestamp'] < $b['timestamp']; + return $b['timestamp'] <=> $a['timestamp']; }); } @@ -299,24 +276,19 @@ public function getName() } } - private function encodePermalink($link) + private function urlEncodePathParts($link) { - return self::URI . implode( - '/', - array_map('urlencode', explode('/', $link)) - ); + return self::URI . implode('/', array_map('urlencode', explode('/', $link))); } - private function template($href, $src, $caption) + private function createFigureLink($href, $src, $caption) { - return '
' - . $caption . '
'; + return sprintf('
%s
', $href, $caption, $src); } - private function link($href, $text) + private function createLink($href, $text) { - return '' . $text . ''; + return sprintf('%s', $href, $text); } public function detectParameters($url) diff --git a/bridges/ReporterreBridge.php b/bridges/ReporterreBridge.php index 18378d2480d..78c60d5f599 100644 --- a/bridges/ReporterreBridge.php +++ b/bridges/ReporterreBridge.php @@ -1,31 +1,20 @@ find('div[style=text-align:justify]') as $e) { - $text = $e->outertext; - } - - $html2->clear(); - unset($html2); - - $text = strip_tags($text, '


'); - return $text; - } + const DESCRIPTION = 'Returns the newest articles. See also their official feed https://reporterre.net/spip.php?page=backend-simple'; public function collectData() { - $html = getSimpleHTMLDOM(self::URI . 'spip.php?page=backend'); + //$url = self::URI . 'spip.php?page=backend'; + $url = self::URI . 'spip.php?page=backend-simple'; + $html = getSimpleHTMLDOM($url); $limit = 0; foreach ($html->find('item') as $element) { @@ -34,10 +23,27 @@ public function collectData() $item['title'] = html_entity_decode($element->find('title', 0)->plaintext); $item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext); $item['uri'] = $element->find('guid', 0)->innertext; - $item['content'] = html_entity_decode($this->extractContent($item['uri'])); + //$item['content'] = html_entity_decode($this->extractContent($item['uri'])); + $item['content'] = htmlspecialchars_decode($element->find('description', 0)->plaintext); $this->items[] = $item; $limit++; } } } + + private function extractContent($url) + { + $html2 = getSimpleHTMLDOM($url); + $html2 = defaultLinkTo($html2, self::URI); + + foreach ($html2->find('div[style=text-align:justify]') as $e) { + $text = $e->outertext; + } + + $html2->clear(); + unset($html2); + + $text = strip_tags($text, '


'); + return $text; + } } From ef378663aaa98ef54c7145781e8ab1e35fe50e7d Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 2 Jan 2024 16:21:52 +0100 Subject: [PATCH 64/69] test: happy new year (#3873) * test: happy new year * yup --- tests/FeedItemTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/FeedItemTest.php b/tests/FeedItemTest.php index 0e7af222e06..3390e7b3534 100644 --- a/tests/FeedItemTest.php +++ b/tests/FeedItemTest.php @@ -41,7 +41,8 @@ public function testTimestamp() $this->assertSame(64800, $item->getTimestamp()); $item->setTimestamp('1st jan last year'); - // This will fail at 2024-01-01 hehe - $this->assertSame(1640995200, $item->getTimestamp()); + + // This will fail at 2025-01-01 hehe + $this->assertSame(1672531200, $item->getTimestamp()); } } From e904de2dc987d6578f9fd5f527aa736801c2185c Mon Sep 17 00:00:00 2001 From: Damien Calesse <2787828+kranack@users.noreply.github.com> Date: Tue, 2 Jan 2024 16:22:39 +0100 Subject: [PATCH 65/69] [YGGTorrent] Update URI (#3871) --- bridges/YGGTorrentBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/YGGTorrentBridge.php b/bridges/YGGTorrentBridge.php index f0c31f11dd5..018bcfc4f02 100644 --- a/bridges/YGGTorrentBridge.php +++ b/bridges/YGGTorrentBridge.php @@ -7,7 +7,7 @@ class YGGTorrentBridge extends BridgeAbstract { const MAINTAINER = 'teromene'; const NAME = 'Yggtorrent Bridge'; - const URI = 'https://www5.yggtorrent.fi'; + const URI = 'https://www3.yggtorrent.qa'; const DESCRIPTION = 'Returns torrent search from Yggtorrent'; const PARAMETERS = [ From 0f6fa8034b04e1e007158ef0c5cc784bf8d7ef45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kol=C3=A1=C5=99?= Date: Tue, 2 Jan 2024 16:23:13 +0100 Subject: [PATCH 66/69] Fixed selector in CeskaTelevizeBridge (#3872) * Fixed selector in CeskaTelevizeBridge * Fixed also description selector --- bridges/CeskaTelevizeBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/CeskaTelevizeBridge.php b/bridges/CeskaTelevizeBridge.php index 003cd4c76f0..be00d6640e7 100644 --- a/bridges/CeskaTelevizeBridge.php +++ b/bridges/CeskaTelevizeBridge.php @@ -57,9 +57,9 @@ public function collectData() $this->feedName .= " ({$category})"; } - foreach ($html->find('#episodeListSection a[data-testid=next-link]') as $element) { + foreach ($html->find('#episodeListSection a[data-testid=card]') as $element) { $itemTitle = $element->find('h3', 0); - $itemContent = $element->find('div[class^=content-]', 0); + $itemContent = $element->find('p[class^=content-]', 0); $itemDate = $element->find('div[class^=playTime-] span', 0); $itemThumbnail = $element->find('img', 0); $itemUri = self::URI . $element->getAttribute('href'); From 12395fcf2d87939a8a95d8bbc95e188e171bfbca Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 5 Jan 2024 07:22:16 +0100 Subject: [PATCH 67/69] Docker fix default fastcgi.logging (#3875) Mistake from https://github.com/RSS-Bridge/rss-bridge/pull/3500 Wrong file extension: should have been `.ini` and not `.conf` otherwise it has no effect. See https://github.com/docker-library/php/pull/1360 and https://github.com/docker-library/php/issues/878#issuecomment-938595965 --- Dockerfile | 2 +- config/php.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f504b51f138..2f1f4f3d93a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ ENV CURL_IMPERSONATE ff91esr COPY ./config/nginx.conf /etc/nginx/sites-available/default COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf -COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.conf +COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini COPY --chown=www-data:www-data ./ /app/ diff --git a/config/php.ini b/config/php.ini index 115f1c89f37..383afffb0b6 100644 --- a/config/php.ini +++ b/config/php.ini @@ -1,4 +1,4 @@ ; Inspired by https://github.com/docker-library/php/blob/master/8.2/bookworm/fpm/Dockerfile -; https://github.com/docker-library/php/issues/878#issuecomment-938595965' +; https://github.com/docker-library/php/issues/878#issuecomment-938595965 fastcgi.logging = Off From 55ffac5bae8d84ff1b42339d1114117cf32a6854 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 5 Jan 2024 07:23:40 +0100 Subject: [PATCH 68/69] [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] (#3876) Fix the Deal source link The HTML does not contain the link to the "Deal source anymore", now only an attribute does contain the information about the Deal Source. The JSON data is now extraced for each Deal, and used to get the Temperature and Deal Source. --- bridges/DealabsBridge.php | 1 + bridges/HotUKDealsBridge.php | 1 + bridges/MydealsBridge.php | 1 + bridges/PepperBridgeAbstract.php | 29 +++++++++++++++++++++-------- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index a904c3ff495..4d39502ca9a 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1910,6 +1910,7 @@ class DealabsBridge extends PepperBridgeAbstract 'context-talk' => 'Surveillance Discussion', 'uri-group' => 'groupe/', 'uri-deal' => 'bons-plans/', + 'uri-merchant' => 'search/bons-plans?merchant-id=', 'request-error' => 'Impossible de joindre Dealabs', 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', 'no-results' => 'Il n'y a rien à afficher pour le moment :(', diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 69301c42ae2..a7e622500e7 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3274,6 +3274,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'context-talk' => 'Discussion Monitoring', 'uri-group' => 'tag/', 'uri-deal' => 'deals/', + 'uri-merchant' => 'search/deals?merchant-id=', 'request-error' => 'Could not request HotUKDeals', 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', 'no-results' => 'Ooops, looks like we could', diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 22b4641305d..d7e074a9aac 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2021,6 +2021,7 @@ class MydealsBridge extends PepperBridgeAbstract 'context-talk' => 'Überwachung Diskussion', 'uri-group' => 'gruppe/', 'uri-deal' => 'deals/', + 'uri-merchant' => 'search/gutscheine?merchant-id=', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', 'no-results' => 'Ups, wir konnten nichts', diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 6cb0f3024a1..73bd194da8e 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -104,6 +104,9 @@ protected function collectDeals($url) $item['title'] = $this->getTitle($deal); $item['author'] = $deal->find('span.thread-username', 0)->plaintext; + // Get the JSON Data stored as vue + $jsonDealData = $this->getDealJsonData($deal); + $item['content'] = '
find('div[class=js-vue2]', 0)->getAttribute('data-vue2')); + return $data; + } + /** * Get the source of a Deal if it exists * @return string String of the deal source */ - private function getSource($deal) + private function getSource($jsonData) { - if (($origin = $deal->find('button[class*=text--color-greyShade]', 0)) != null) { - $path = str_replace(' ', '/', trim(Json::decode($origin->{'data-cloak-link'})['path'])); - $text = $origin->find('span[class*=link]', 0); + if ($jsonData['props']['thread']['merchant'] != null) { + $path = $this->i8n('uri-merchant') . $jsonData['props']['thread']['merchant']['merchantId']; + $text = $jsonData['props']['thread']['merchant']['merchantName']; return ''; } else { return ''; From ea58c8d2bcd17b09e7d9dea64297ea44885a3933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B5=D0=B7=D0=B4=D0=B0=D0=BB=D0=B8=D1=81=D1=8C?= =?UTF-8?q?=D0=BA=D0=BE?= <105280814+uandreew@users.noreply.github.com> Date: Sat, 6 Jan 2024 19:13:50 +0200 Subject: [PATCH 69/69] Update 06_Public_Hosts.md (#3877) --- docs/01_General/06_Public_Hosts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index c9572824844..4aa905dad49 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -22,6 +22,7 @@ | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rb.ash.fail | ![](https://img.shields.io/website/https/rb.ash.fail.svg) | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany +| ![](https://iplookup.flagfox.net/images/h16/UA.png) | https://rss.noleron.com | ![](https://img.shields.io/website/https/rss.noleron.com) | [@ihor](https://noleron.com/about) | Hosted with Hosting Ukraine, Ukraine ## Inactive instances