import-io/improvement

import-io/Makefile

@@ -1,4 +1,4 @@
-PLUGIN_VERSION=1.0.1
+PLUGIN_VERSION=1.0.2
 PLUGIN_ID=import-io
 
 plugin:

import-io/code-env/python/desc.json

@@ -0,0 +1,6 @@
+{
+  "acceptedPythonInterpreters": ["PYTHON27","PYTHON35","PYTHON36"],
+  "forceConda": false,
+  "installCorePackages": true,
+  "installJupyterSupport": false
+}

import-io/custom-recipes/import-io-enrich-connector/recipe.py

@@ -1,4 +1,5 @@
-import urllib, json
+import urllib
+import json
 from dataiku.customrecipe import *
 import importio_utils
 

import-io/custom-recipes/import-io-enrich-extractor/recipe.py

@@ -1,4 +1,4 @@
-import dataiku, urllib, urlparse, requests, sys
+import urllib
 from dataiku.customrecipe import *
 import importio_utils
 
@@ -7,6 +7,5 @@
 def build_query(in_row, apikey):
     url = in_row[url_field]
     return 'input=webpage/url:' + urllib.quote(safe='',s=url) + '&_apikey=' + apikey
-          # input/webpage/url=
 
 importio_utils.run(build_query)

import-io/plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "import-io",
-    "version": "1.0.1",
+    "version": "1.0.2",
     "meta": {
         "label": "import.io",
         "description": "Downloads data from import.io's API & enriches datasets",

import-io/python-connectors/import-io-simple-api/connector.py

@@ -1,7 +1,12 @@
-import requests, json
+import requests
+import json
 import pandas as pd
 from dataiku.connector import Connector
 import importio_utils
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 class ImportIOConnector(Connector):
 
@@ -13,24 +18,23 @@ def __init__(self, config):
         elif self.config['api_url'].startswith('https://extraction.import.io/'):
             self.api_version = 'extraction'
         else:
-            raise Exception(
-                'It looks like this URL is not an API URL. URLs to call the API (and get a json response) start with "https://api.import.io" .')
-        print '[import.io connector] calling API...'
+            raise Exception('It looks like this URL is not an API URL. URLs to call the API (and get a json response) start with "https://api.import.io" .')
+        logger.info('[import.io connector] calling API...')
         response = requests.get(self.config['api_url'])
-        print '[import.io connector] got response'
+        logger.info('[import.io connector] got response')
         try:
             self.json = response.json()
         except Exception as e:
-            print e
-            print 'response was:\n', response.text
-            raise
+            logger.error(e)
+            logger.error('response was:{}'.format(response.text))
+            raise ValueError
 
     def get_read_schema(self):
         if self.api_version == 'api':
             columns = importio_utils.convert_schema(self.json['outputProperties'])
-            return {"columns":columns}
+            return {"columns": columns}
         else:
-             return None
+            return None
 
     def generate_rows(self, dataset_schema=None, dataset_partitioning=None, partition_id=None, records_limit = -1):
         if self.api_version == 'api':

import-io/python-lib/importio_utils.py

@@ -1,6 +1,13 @@
 # coding: UTF8
-import urlparse, requests, sys, dataiku, time
+import urlparse
+import requests
+import sys
+import dataiku
+import time
 from dataiku.customrecipe import *
+import logging
+
+logger = logging.getLogger(__name__)
 
 # See http://api.docs.import.io/#DataTypes
 importIO_subfields = {
@@ -24,9 +31,7 @@ def convert_type(importIO_type):
 def convert_schema(import_io_schema):
     result = []
     for col in import_io_schema:
-        result.append({
-            'name':col['name'],
-            'type':convert_type(col['type'])})
+        result.append({'name':col['name'],'type':convert_type(col['type'])})
         for subfield in importIO_subfields[col['type']]:
             result.append({'name':col['name']+'/'+subfield, 'type':'string'})
     return result
@@ -54,21 +59,24 @@ def run(build_query):
             response = requests.get(request_url)
             json = response.json()
         except Exception as e:
-            print 'request to import.io failed'
-            print e
-            print 'response was:\n',response
-            raise
+            logger.error('request to import.io failed')
+            logger.error(e)
+            logger.error('response was: {}'.format(response))
+            raise ValueError
         if 'error' in json:
-            print "response: ", json
+            logger.error("response: {}".format(json))
             raise Exception(json['error'])
         for result_line in json['results']:
             if not output_schema:
-                print "Setting schema"
+                logger.error("Setting schema")
                 input_schema_names = frozenset([e['name'] for e in input.read_schema()])
                 output_schema = input.read_schema()
+                print(')))))))))')
+                print(json)
+                print(json.keys())
                 for col in convert_schema(json['outputProperties']):
                     if col['name'] in input_schema_names:
-                        print "Warning: input col "+col['name']+" will be overwritten by output col with same name."
+                        logger.error("Warning: input col "+col['name']+" will be overwritten by output col with same name.")
                         input_cols_to_drop.append(col['name'])
                     else:
                         output_schema.append(col)