Update superset to v0.34.0rc1

.travis.yml

@@ -1,5 +1,3 @@
-dist: trusty
-
 branches:
   only:
     - master
@@ -53,14 +51,17 @@ jobs:
       python: 3.6
       env: TOXENV=py36-postgres
       services:
-        - postgres
+        - postgresql
         - redis-server
       before_script:
         - psql -U postgres -c "CREATE DATABASE superset;"
         - psql -U postgres -c "CREATE USER postgresuser WITH PASSWORD 'pguserpassword';"
     - language: python
       python: 3.6
       env: TOXENV=pylint
+    - language: python
+      python: 3.6
+      env: TOXENV=black
     - stage: release
       name: 'Release to Docker Hub'
       services:

CHANGELOG.md

@@ -7,6 +7,10 @@
     Click to see more.
   </summary>
 
+### Added
+
+- Superset updated to v0.34.0rc1 which brings many improvements and bug fixes ([#250](https://github.com/src-d/sourced-ui/issues/250))
+
 </details>
 
 ## [v0.5.0](https://github.com/src-d/sourced-ui/releases/tag/v0.5.0) - 2019-08-20

CONTRIBUTING.md

@@ -142,3 +142,74 @@ if needed)
     ```shell
     $ sourced logs -f sourced-ui
     ```
+
+## Merging upstream
+
+We base sourced-ui on release tags of superset instead of master.
+
+### Revert release branch changes (optional)
+
+Release branches might contain cherry-picks or other commits that aren't presented in master branch or the next release branch.
+In such case it makes sense to revert them before applying changes from newer release.
+
+All the releases have a topic in [mailing list](https://lists.apache.org/[email protected]) which contain common ancestor sha and list of commits for each release tag.
+
+1. Find common ancestor in the mailing list
+1. Find the last split commit
+    ```
+    $ git log | grep git-subtree-split
+    ```
+1. Checkout to split commit
+1. Revert commits
+    ```
+    $ git revert --no-commit <common ancestor sha>...<release tag>
+    ```
+1. Commit the revert as single commit
+    ```
+    $ git commit -s -m "revert to upstream master from <release tag>"
+    ```
+1. Merge revert into sourced-ui tree
+    ```
+    $ git co master
+    $ git subtree merge -P superset HEAD@{1} --squash
+    ```
+
+### Merge new release
+
+```
+$ git subtree merge -P superset <release-tag> --squash
+```
+
+### After merge checklist
+
+- Re-create venv and re-install dependencies
+- Run linters
+    ```
+    $ make patch
+    $ cd superset
+    $ TOXENV=black tox
+    $ TOXENV=flake8 tox
+    $ TOXENV=pylint tox
+    ```
+- Merge migrations (if needed)
+    ```
+    $ pip install -e .
+    $ superset db heads
+    $ superset db merge <head1> <head2>
+    ```
+- Run tests
+    ```
+    $ docker run --rm -p 6379:6379 redis
+    $ TOXENV=py36-sqlite tox
+    ```
+- Build & check it with srcd-ce
+    ```
+    $ make build
+    <run srcd-ce with this image>
+    ```
+- Check if some changes to configuration should be ported, files to check:
+  - superset/contib/docker
+  - superset/superset/config.py
+  - superset/.travis.yml
+- Update `Makefile`
+- Test upgrade

Makefile

@@ -1,6 +1,6 @@
 # Superset upstream configuration
 SUPERSET_REPO = https://github.com/apache/incubator-superset.git
-SUPERSET_VERSION = 0.33.0rc1
+SUPERSET_VERSION = 0.34.0rc1
 SUPERSET_REMOTE = superset
 # Directory to sync superset upstream with
 SUPERSET_DIR = superset

srcd/contrib/docker/bootstrap.py

@@ -3,7 +3,7 @@
 from flask import g
 from flask_migrate import upgrade as db_upgrade
 
-from superset import app, conf, db, security_manager
+from superset import app, appbuilder, conf, db, security_manager
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.models import core as models
 from superset.models.user_attributes import UserAttribute
@@ -27,7 +27,9 @@ def get_or_create_datasource(name, uri, **kwargs):
 
 def create_datasource_tables(dbobj, schema):
     TBL = ConnectorRegistry.sources['table']
-    for table in dbobj.all_table_names_in_schema(schema):
+    for ds in dbobj.get_all_table_names_in_schema(schema):
+        table = ds.table
+
         # table_name should match the one in the datasource for fetch_metadata to work
         if db.session.query(TBL).filter_by(table_name=table).first():
             continue
@@ -96,6 +98,7 @@ def bootstrap():
                                                os.environ['ADMIN_PASSWORD'])
         # Create default roles and permissions
         utils.get_or_create_main_db()
+        appbuilder.add_permissions(update_perms=True)
         security_manager.sync_role_definitions()
 
         # set admin user as a current user

srcd/dashboards/metadata/welcome.json

@@ -14,7 +14,7 @@
                             "datasource_name": "Untitled Query-yRPMUzfQC",
                             "datasource_type": "table",
                             "id": 21,
-                            "params": "{\"adhoc_filters\": [], \"align_pn\": false, \"all_columns\": [\"org\", \"entity\", \"done\", \"total\", \"progress\", \"status\"], \"color_pn\": true, \"datasource\": \"39__table\", \"granularity_sqla\": null, \"groupby\": [], \"include_search\": true, \"include_time\": false, \"metrics\": [], \"order_by_cols\": [\"[\\\"org\\\", true]\", \"[\\\"entity\\\", true]\"], \"order_desc\": true, \"page_length\": 0, \"percent_metrics\": [], \"row_limit\": null, \"slice_id\": 29, \"table_filter\": false, \"table_timestamp_format\": \"%Y-%m-%d %H:%M:%S\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"100 years ago : \", \"timeseries_limit_metric\": null, \"viz_type\": \"table\", \"remote_id\": \"144a891f-282f-42ca-99f1-8437e1aeb8d9\", \"datasource_name\": \"Untitled Query-yRPMUzfQC\", \"schema\": \"Untitled Query-yRPMUzfQC\", \"database_name\": \"metadata\"}",
+                            "params": "{\"adhoc_filters\": [], \"align_pn\": false, \"all_columns\": [\"org\", \"entity\", \"done\", \"total\", \"progress\", \"status\"], \"color_pn\": true, \"datasource\": \"39__table\", \"granularity_sqla\": null, \"groupby\": [], \"include_search\": true, \"include_time\": false, \"metrics\": [], \"order_by_cols\": [\"[\\\"org\\\", true]\", \"[\\\"entity\\\", true]\"], \"order_desc\": true, \"page_length\": 0, \"percent_metrics\": [], \"row_limit\": null, \"slice_id\": 29, \"table_filter\": false, \"table_timestamp_format\": \"%Y-%m-%d %H:%M:%S\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"100 years ago : \", \"timeseries_limit_metric\": null, \"viz_type\": \"table\", \"remote_id\": \"144a891f-282f-42ca-99f1-8437e1aeb8d9\", \"datasource_name\": \"Untitled Query-yRPMUzfQC\", \"schema\": null, \"database_name\": \"metadata\"}",
                             "slice_name": "Metadata collection progress",
                             "viz_type": "table"
                         }
@@ -25,7 +25,7 @@
                             "datasource_name": "public.Untitled Query-Xwa4Amapg",
                             "datasource_type": "table",
                             "id": 22,
-                            "params": "{\"adhoc_filters\": [], \"align_pn\": false, \"all_columns\": [\"org\", \"discovered\", \"downloaded\", \"updated\", \"failed\"], \"color_pn\": true, \"datasource\": \"40__table\", \"granularity_sqla\": null, \"groupby\": [], \"include_search\": true, \"include_time\": false, \"metrics\": [], \"order_by_cols\": [], \"order_desc\": true, \"page_length\": 0, \"percent_metrics\": [], \"row_limit\": 1000, \"slice_id\": 30, \"table_filter\": false, \"table_timestamp_format\": \"%Y-%m-%d %H:%M:%S\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"100 years ago : \", \"timeseries_limit_metric\": null, \"viz_type\": \"table\", \"remote_id\": \"77406f30-3421-4d90-94e1-d8056b56bed7\", \"datasource_name\": \"public.Untitled Query-Xwa4Amapg\", \"schema\": \"public.Untitled Query-Xwa4Amapg\", \"database_name\": \"main\"}",
+                            "params": "{\"adhoc_filters\": [], \"align_pn\": false, \"all_columns\": [\"org\", \"discovered\", \"downloaded\", \"updated\", \"failed\"], \"color_pn\": true, \"datasource\": \"40__table\", \"granularity_sqla\": null, \"groupby\": [], \"include_search\": true, \"include_time\": false, \"metrics\": [], \"order_by_cols\": [], \"order_desc\": true, \"page_length\": 0, \"percent_metrics\": [], \"row_limit\": 1000, \"slice_id\": 30, \"table_filter\": false, \"table_timestamp_format\": \"%Y-%m-%d %H:%M:%S\", \"time_grain_sqla\": \"P1D\", \"time_range\": \"100 years ago : \", \"timeseries_limit_metric\": null, \"viz_type\": \"table\", \"remote_id\": \"77406f30-3421-4d90-94e1-d8056b56bed7\", \"datasource_name\": \"Untitled Query-Xwa4Amapg\", \"schema\": \"public\", \"database_name\": \"main\"}",
                             "slice_name": "Repository collection progress",
                             "viz_type": "table"
                         }
@@ -48,7 +48,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:31"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -70,7 +69,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:31"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -92,7 +90,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:31"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -114,7 +111,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:31"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -136,7 +132,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:31"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -158,7 +153,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:31"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -221,7 +215,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:32"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -243,7 +236,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:32"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -265,7 +257,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:32"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -287,7 +278,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:32"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,
@@ -309,7 +299,6 @@
                             "created_on": {
                                 "__datetime__": "2019-08-02T11:03:32"
                             },
-                            "database_expression": null,
                             "description": null,
                             "expression": "",
                             "filterable": true,

srcd/superset/assets/src/components/FilterableTable/isUAST.jsx

@@ -0,0 +1,17 @@
+// validation that cellData is JSON happens outside of this function
+function isUAST(cellData) {
+  // First perform a cheap proxy to avoid calling JSON.parse on data that is clearly not a
+  // JSON object or array
+  if (typeof cellData !== 'string' || ['{', '['].indexOf(cellData.substring(0, 1)) === -1) {
+    return false;
+  }
+
+  try {
+    JSON.parse(cellData);
+    return cellData.includes('"@pos"');
+  } catch (_) {
+    return false;
+  }
+}
+
+export default isUAST;

srcd/superset/bblfsh/views.py

@@ -12,42 +12,46 @@
 
 
 class Bblfsh(BaseSupersetView):
-    default_view = 'explore'
+    default_view = "explore"
 
     @has_access
-    @expose('/explore/')
+    @expose("/explore/")
     def explore(self):
         """UAST Viewer"""
-        d = {
-            'common': self.common_bootsrap_payload(),
-        }
+        d = {"common": self.common_bootstrap_payload()}
         return self.render_template(
-            'superset/basic.html',
-            entry='uast',
-            bootstrap_data=json.dumps(d),
+            "superset/basic.html", entry="uast", bootstrap_data=json.dumps(d)
         )
 
     @has_access
-    @expose('/api/<path>', methods=['GET', 'POST'])
+    @expose("/api/<path>", methods=["GET", "POST"])
     def api(self, path):
         """ Proxy frontend request to bblfsh-web server """
         resp = requests.request(
             method=request.method,
-            url='%s/api/%s' % (BBLFSH_WEB_ADDRESS, path),
-            headers={key: value for (key, value)
-                     in request.headers if key != 'Host'},
+            url="%s/api/%s" % (BBLFSH_WEB_ADDRESS, path),
+            headers={key: value for (key, value) in request.headers if key != "Host"},
             data=request.get_data(),
             cookies=request.cookies,
-            allow_redirects=False)
+            allow_redirects=False,
+        )
 
-        excluded_headers = ['content-encoding',
-                            'content-length', 'transfer-encoding', 'connection']
-        headers = [(name, value) for (name, value) in resp.raw.headers.items()
-                   if name.lower() not in excluded_headers]
+        excluded_headers = [
+            "content-encoding",
+            "content-length",
+            "transfer-encoding",
+            "connection",
+        ]
+        headers = [
+            (name, value)
+            for (name, value) in resp.raw.headers.items()
+            if name.lower() not in excluded_headers
+        ]
 
         response = Response(resp.content, resp.status_code, headers)
         return response
 
 
-appbuilder.add_view(Bblfsh, 'UAST', label=_('UAST'),
-                    category_icon='fa-tree', icon='fa-tree')
+appbuilder.add_view(
+    Bblfsh, "UAST", label=_("UAST"), category_icon="fa-tree", icon="fa-tree"
+)

superset/.codecov.yml

@@ -0,0 +1,11 @@
+coverage:
+  status:
+    project:
+      default:
+        # Commits pushed to master should not make the overall
+        # project coverage decrease:
+        target: auto
+        threshold: 0%
+    patch:
+      default:
+        threshold: 0%