Upgrades bleach and simplejson (oppia#6797)

heyimShivam · Jun 12, 2019 · 5803780 · 5803780
1 parent 2b65987
commit 5803780
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 30 deletions.
diff --git a/appengine_config.py b/appengine_config.py
@@ -88,17 +88,20 @@ def save(self):
 
 THIRD_PARTY_LIBS = [
     os.path.join(ROOT_PATH, 'third_party', 'backports.functools_lru_cache-1.5'),
-    os.path.join(ROOT_PATH, 'third_party', 'bleach-1.2.2'),
-    os.path.join(ROOT_PATH, 'third_party', 'html5lib-python-0.95'),
-    os.path.join(ROOT_PATH, 'third_party', 'gae-mapreduce-1.9.17.0'),
+    os.path.join(ROOT_PATH, 'third_party', 'beautifulsoup4-4.7.1'),
+    os.path.join(ROOT_PATH, 'third_party', 'bleach-3.1.0'),
+    os.path.join(ROOT_PATH, 'third_party', 'callbacks-0.3.0'),
     os.path.join(ROOT_PATH, 'third_party', 'gae-cloud-storage-1.9.15.0'),
+    os.path.join(ROOT_PATH, 'third_party', 'gae-mapreduce-1.9.17.0'),
     os.path.join(ROOT_PATH, 'third_party', 'gae-pipeline-1.9.17.0'),
     os.path.join(ROOT_PATH, 'third_party', 'graphy-1.0.0'),
-    os.path.join(ROOT_PATH, 'third_party', 'requests-2.10.0'),
-    os.path.join(ROOT_PATH, 'third_party', 'simplejson-3.7.1'),
-    os.path.join(ROOT_PATH, 'third_party', 'beautifulsoup4-4.7.1'),
+    os.path.join(ROOT_PATH, 'third_party', 'html5lib-python-1.0.1'),
     os.path.join(ROOT_PATH, 'third_party', 'mutagen-1.38'),
-    os.path.join(ROOT_PATH, 'third_party', 'soupsieve-1.8'),
+    os.path.join(ROOT_PATH, 'third_party', 'requests-2.10.0'),
+    os.path.join(ROOT_PATH, 'third_party', 'simplejson-3.16.0'),
+    os.path.join(ROOT_PATH, 'third_party', 'six-1.12.0'),
+    os.path.join(ROOT_PATH, 'third_party', 'soupsieve-1.9.1'),    
+    os.path.join(ROOT_PATH, 'third_party', 'webencodings-0.5.1'),
 ]
 
 for lib_path in THIRD_PARTY_LIBS:

diff --git a/core/domain/exp_domain_test.py b/core/domain/exp_domain_test.py
@@ -333,7 +333,7 @@ def test_validation(self):
 
         outcome.dest = destination
 
-        outcome.feedback = state_domain.SubtitledHtml('feedback_1', {})
+        outcome.feedback = state_domain.SubtitledHtml('feedback_1', '')
         exploration.validate()
 
         outcome.labelled_as_correct = 'hello'

diff --git a/core/domain/html_cleaner.py b/core/domain/html_cleaner.py
@@ -26,17 +26,20 @@
 from core.domain import rte_component_registry
 
 
-def filter_a(name, value):
-    """Returns whether the described attribute of an anchor ('a') tag should be
+def filter_a(tag, name, value):
+    """Returns whether the described attribute of a tag should be
     whitelisted.
 
     Args:
+        tag: str. The name of the tag passed.
         name: str. The name of the attribute.
         value: str. The value of the attribute.
 
     Returns:
         bool. Whether the given attribute should be whitelisted.
     """
+    if tag != 'a':
+        raise Exception('The filter_a method should only be used for a tags.')
     if name in ('title', 'target'):
         return True
     if name == 'href':

diff --git a/core/domain/html_cleaner_test.py b/core/domain/html_cleaner_test.py
@@ -27,6 +27,20 @@ def setUp(self):
         super(HtmlCleanerUnitTests, self).setUp()
         self.longMessage = True
 
+    def test_whitelisted_tags(self):
+
+        self.assertTrue(
+            html_cleaner.filter_a('a', 'href', 'http://www.oppia.com'))
+
+        self.assertFalse(
+            html_cleaner.filter_a('a', 'href', '<code>http://www.oppia.com'))
+
+        self.assertTrue(
+            html_cleaner.filter_a('a', 'title', 'http://www.oppia.com'))
+
+        with self.assertRaises(Exception):
+            html_cleaner.filter_a('link', 'href', 'http://www.oppia.com')
+
     def test_good_tags_allowed(self):
         test_data = [(
             '<a href="http://www.google.com">Hello</a>',

diff --git a/core/domain/state_domain_test.py b/core/domain/state_domain_test.py
@@ -515,7 +515,7 @@ def test_solution_validation(self):
         hints_list.append({
             'hint_content': {
                 'content_id': 'hint_1',
-                'html': {}
+                'html': ''
             },
         })
         init_state.update_interaction_hints(hints_list)

diff --git a/core/tests/gae_suite.py b/core/tests/gae_suite.py
@@ -43,17 +43,20 @@
     os.path.join(OPPIA_TOOLS_DIR, 'Pillow-6.0.0'),
     CURR_DIR,
     os.path.join(THIRD_PARTY_DIR, 'backports.functools_lru_cache-1.5'),
-    os.path.join(THIRD_PARTY_DIR, 'bleach-1.2.2'),
+    os.path.join(THIRD_PARTY_DIR, 'beautifulsoup4-4.7.1'),
+    os.path.join(THIRD_PARTY_DIR, 'bleach-3.1.0'),
+    os.path.join(THIRD_PARTY_DIR, 'callbacks-0.3.0'),
     os.path.join(THIRD_PARTY_DIR, 'gae-cloud-storage-1.9.15.0'),
     os.path.join(THIRD_PARTY_DIR, 'gae-mapreduce-1.9.17.0'),
     os.path.join(THIRD_PARTY_DIR, 'gae-pipeline-1.9.17.0'),
     os.path.join(THIRD_PARTY_DIR, 'graphy-1.0.0'),
-    os.path.join(THIRD_PARTY_DIR, 'html5lib-python-0.95'),
-    os.path.join(THIRD_PARTY_DIR, 'requests-2.10.0'),
-    os.path.join(THIRD_PARTY_DIR, 'simplejson-3.7.1'),
-    os.path.join(THIRD_PARTY_DIR, 'beautifulsoup4-4.7.1'),
+    os.path.join(THIRD_PARTY_DIR, 'html5lib-python-1.0.1'),
     os.path.join(THIRD_PARTY_DIR, 'mutagen-1.38'),
-    os.path.join(THIRD_PARTY_DIR, 'soupsieve-1.8'),
+    os.path.join(THIRD_PARTY_DIR, 'requests-2.10.0'),
+    os.path.join(THIRD_PARTY_DIR, 'simplejson-3.16.0'),
+    os.path.join(THIRD_PARTY_DIR, 'six-1.12.0'),
+    os.path.join(THIRD_PARTY_DIR, 'soupsieve-1.9.1'),
+    os.path.join(THIRD_PARTY_DIR, 'webencodings-0.5.1'),
 ]
 
 _PARSER = argparse.ArgumentParser()

diff --git a/manifest.json b/manifest.json
@@ -10,9 +10,9 @@
         "targetDirPrefix": "backports.functools_lru_cache-"
       },
       "bleach": {
-        "version": "1.2.2",
+        "version": "3.1.0",
         "downloadFormat": "zip",
-        "url": "https://github.com/jsocol/bleach/archive/v1.2.2.zip",
+        "url": "https://github.com/jsocol/bleach/archive/v3.1.0.zip",
         "rootDirPrefix": "bleach-",
         "targetDirPrefix": "bleach-"
       },
@@ -49,9 +49,9 @@
         "targetDirPrefix": "graphy-"
       },
       "html5Lib": {
-        "version": "0.95",
+        "version": "1.0.1",
         "downloadFormat": "zip",
-        "url": "https://github.com/html5lib/html5lib-python/archive/0.95.zip",
+        "url": "https://github.com/html5lib/html5lib-python/archive/1.0.1.zip",
         "rootDirPrefix": "html5lib-python-",
         "targetDirPrefix": "html5lib-python-"
       },
@@ -64,12 +64,36 @@
         "targetDirPrefix": "requests-"
       },
       "simpleJson": {
-        "version": "3.7.1",
+        "version": "3.16.0",
         "downloadFormat": "tar",
-        "url": "https://pypi.python.org/packages/source/s/simplejson/simplejson-3.7.1.tar.gz#md5=c76c2d11b87e9fb501bd0b2b72091653",
+        "url": "https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz",
         "tarRootDirPrefix": "simplejson-",
         "rootDirPrefix": "simplejson-",
         "targetDirPrefix": "simplejson-"
+      }, 
+      "six": {
+        "version": "1.12.0",
+        "downloadFormat": "tar",
+        "url": "https://files.pythonhosted.org/packages/dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca/six-1.12.0.tar.gz",
+        "tarRootDirPrefix": "six-",
+        "rootDirPrefix": "six-",
+        "targetDirPrefix": "six-"
+      },
+      "webencodings": {
+        "version": "0.5.1",
+        "downloadFormat": "tar",
+        "url": "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz",
+        "tarRootDirPrefix": "webencodings-",
+        "rootDirPrefix": "webencodings-",
+        "targetDirPrefix": "webencodings-"
+      },
+      "callbacks": {
+        "version": "0.3.0",
+        "downloadFormat": "tar",
+        "url": "https://files.pythonhosted.org/packages/eb/15/c71150189ea1cfd04a5853a40e2e074ec777755bd3924980fe51e5c6dd44/callbacks-0.3.0.tar.gz",
+        "tarRootDirPrefix": "callbacks-",
+        "rootDirPrefix": "callbacks-",
+        "targetDirPrefix": "callbacks-"
       },
       "beautifulSoup": {
         "version": "4.7.1",
@@ -88,9 +112,9 @@
         "targetDirPrefix": "mutagen-"
       },
       "soupsieve": {
-        "version": "1.8",
+        "version": "1.9.1",
         "downloadFormat": "tar",
-        "url": "https://pypi.python.org/packages/0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f/soupsieve-1.8.tar.gz#md5=3b3d830576e5fa5148bc154e38e63ebc",
+        "url": "https://files.pythonhosted.org/packages/fb/9e/2e236603b058daa6820193d4d95f4dcfbbbd0d3c709bec8c6ef1b1902501/soupsieve-1.9.1.tar.gz",
         "tarRootDirPrefix": "soupsieve-",
         "rootDirPrefix": "soupsieve-",
         "targetDirPrefix": "soupsieve-"

diff --git a/schema_utils.py b/schema_utils.py
@@ -250,7 +250,7 @@ def sanitize_url(obj):
             urllib.quote(component) for component in url_components)
         raw = urlparse.urlunsplit(quoted_url_components)
 
-        acceptable = html_cleaner.filter_a('href', obj)
+        acceptable = html_cleaner.filter_a('a', 'href', obj)
         assert acceptable, (
             'Invalid URL: Sanitized URL should start with '
             '\'http://\' or \'https://\'; received %s' % raw)

diff --git a/scripts/pre_commit_linter.py b/scripts/pre_commit_linter.py
@@ -356,13 +356,16 @@
     os.path.join(_PARENT_DIR, 'oppia_tools', 'PyGithub-1.43.5'),
     os.path.join(_PARENT_DIR, 'oppia_tools', 'Pillow-6.0.0'),
     os.path.join('third_party', 'backports.functools_lru_cache-1.5'),
-    os.path.join('third_party', 'gae-pipeline-1.9.17.0'),
-    os.path.join('third_party', 'bleach-1.2.2'),
     os.path.join('third_party', 'beautifulsoup4-4.7.1'),
+    os.path.join('third_party', 'bleach-3.1.0'),
+    os.path.join('third_party', 'callbacks-0.3.0'),
+    os.path.join('third_party', 'gae-cloud-storage-1.9.15.0'),
     os.path.join('third_party', 'gae-mapreduce-1.9.17.0'),
+    os.path.join('third_party', 'gae-pipeline-1.9.17.0'),
     os.path.join('third_party', 'mutagen-1.38'),
-    os.path.join('third_party', 'soupsieve-1.8'),
-    os.path.join('third_party', 'gae-cloud-storage-1.9.15.0'),
+    os.path.join('third_party', 'soupsieve-1.9.1'),
+    os.path.join('third_party', 'six-1.12.0'),
+    os.path.join('third_party', 'webencodings-0.5.1'),
 ]
 for path in _PATHS_TO_INSERT:
     sys.path.insert(0, path)