- Add support for grabbing youtube resources via proxies

web2py/applications/smc/controllers/test.py

@@ -31,6 +31,14 @@
 ret = module_reload.ReloadModules()
 
 
+@auth.requires_membership("Administrators")
+def test_yt_proxies():
+    response.view = 'generic.json'
+
+    ret = get_youtube_proxies()
+
+    return ret
+
 @auth.requires_permission("credential")
 def credential():
     return dict(message="It Worked!")

web2py/applications/smc/languages/plural-en.py

@@ -1,12 +1,15 @@
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 {
+# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],
 'account': ['accounts'],
 'book': ['books'],
 'is': ['are'],
 'man': ['men'],
 'miss': ['misses'],
 'person': ['people'],
 'quark': ['quarks'],
+'row': ['rows'],
 'shop': ['shops'],
 'this': ['these'],
 'was': ['were'],

web2py/applications/smc/models/b_media_code.py

@@ -21,6 +21,19 @@
 from pytube import YouTube
 #import pytube
 
+def get_youtube_proxies():
+    query = (db.youtube_proxy_list.enabled==True)
+    # Random order so you get a possible different item from the list each time
+    rows = db(query).select(orderby='<random>')
+    ret = None
+    for row in rows:
+        if ret is None:
+            ret = {}
+
+        ret[row["protocol"]] = row["proxy_url"]
+
+    return ret
+
 def get_flashcards_folder_path(flashcard_id):
     (w2py_folder, applications_folder, app_folder) = get_app_folders()
     flashcards_folder = os.path.join(app_folder, "static/media/flashcards")

web2py/applications/smc/models/x_app_models.py

@@ -596,6 +596,13 @@
       auth.signature,
 )
 
+db.define_table("youtube_proxy_list",
+      Field("protocol", "string", default="https"),
+      Field("proxy_url", "string", default=""),
+      Field("enabled", "boolean", default=True),
+      auth.signature,
+)
+
 # Adjust the app logo if it is set
 app_logo = AppSettings.GetValue('app_logo', '<none>')
 if app_logo != "<none>":

web2py/applications/smc/models/x_scheduler.py

@@ -60,9 +60,9 @@ def unicode(s):
     #from urllib import request
 
     def install_proxy(proxy_handler):
-        proxy_support = request.ProxyHandler(proxy_handler)
-        opener = request.build_opener(proxy_support)
-        request.install_opener(opener)
+        proxy_support = urllib.request.ProxyHandler(proxy_handler)
+        opener = urllib.request.build_opener(proxy_support)
+        urllib.request.install_opener(opener)
 
     def unicode(s):
         """No-op."""
@@ -326,7 +326,9 @@ def find_best_yt_stream(yt_url):
 
     # Change out embed for watch so the link works properly
     try:
-        yt = YouTube(yt_url.replace("/embed/", "/watch?v="))
+        proxies = get_youtube_proxies()
+        print("- YT - Using proxies: " + str(proxies))
+        yt = YouTube(yt_url.replace("/embed/", "/watch?v="), proxies=proxies)
     except HTTPError as ex:
         if ex.code == 429:
             # Need to try again later
@@ -380,7 +382,7 @@ def pull_youtube_caption(yt_url, media_guid):
     target_file = get_media_file_path(media_guid, "srt")
     from pytube import YouTube
     try:
-        yt = YouTube(yt_url.replace("/embed/", "/watch?v="))
+        yt = YouTube(yt_url.replace("/embed/", "/watch?v="), proxies=get_youtube_proxies())
     except HTTPError as ex:
         if ex.code == 429:
             # Need to try again later
@@ -473,7 +475,7 @@ def pull_youtube_video(yt_url, media_guid):
         if ex.code == 429:
             # Too many requests - have this try again later...
 
-            sleep_time = 300  # sleep for 5 mins?
+            sleep_time = 60  # sleep for 5 mins?
             if "Retry-After" in ex.headers:
                 sleep_time = int(ex.headers["Retry-After"])
             print("Too many requests - sleeping for a few... " + str(sleep_time))
@@ -493,7 +495,7 @@ def pull_youtube_video(yt_url, media_guid):
         if ex.code == 429:
             # Too many requests - have this try again later...
 
-            sleep_time = 300  # sleep for 5 mins?
+            sleep_time = 60  # sleep for 5 mins?
             if "Retry-After" in ex.headers:
                 sleep_time = int(ex.headers["Retry-After"])
             print("Too many requests - sleeping for a few... " + str(sleep_time))
@@ -592,7 +594,7 @@ def pull_youtube_video(yt_url, media_guid):
     db.commit()
 
     # Throw a little delay in here to help keep from getting blocked by youtube
-    time.sleep(30)
+    time.sleep(15)
     return True
 
 
@@ -712,7 +714,7 @@ def process_wamap_video_links():
                     f.close()
 
                     # Download the video from the internet
-                    yt = YouTube()
+                    yt = YouTube(proxies=get_youtube_proxies())
                     yt_url = yt_url.replace("!!1", "").replace("!!0", "")  # because some urls end up with the
                     # field separator still in it
                     if "/embed/" in yt_url: