Merge branch 'release_16.04' into dev

lib/galaxy/jobs/runners/drmaa.py

@@ -15,7 +15,6 @@
 from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE
 from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
 from galaxy.util import asbool
-from pulsar.managers.util.drmaa import DrmaaSessionFactory
 
 drmaa = None
 
@@ -62,6 +61,7 @@ def __init__( self, app, nworkers, **kwargs ):
                                 'feature, please install it or correct the '
                                 'following error:\n%s: %s' %
                                 (exc.__class__.__name__, str(exc)))
+        from pulsar.managers.util.drmaa import DrmaaSessionFactory
 
         # Subclasses may need access to state constants
         self.drmaa_job_states = drmaa.JobState

lib/galaxy/tools/actions/__init__.py

@@ -7,7 +7,7 @@
 from galaxy import model
 from galaxy.exceptions import ObjectInvalid
 from galaxy.model import LibraryDatasetDatasetAssociation
-from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter
+from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue
 from galaxy.tools.parameters.wrapped import WrappedParameters
 from galaxy.tools.parameters import update_param
 from galaxy.util import ExecutionTimer
@@ -54,8 +54,8 @@ def collect_input_datasets( self, tool, param_values, trans, current_user_roles=
         def visitor( input, value, prefix, parent=None, **kwargs ):
 
             def process_dataset( data, formats=None ):
-                if not data:
-                    return data
+                if not data or isinstance( data, RuntimeValue ):
+                    return None
                 if formats is None:
                     formats = input.formats
                 if not data.datatype.matches_any( formats ):

lib/galaxy/web/security/__init__.py

@@ -54,13 +54,13 @@ def encode_id( self, obj_id, kind=None ):
         # Encrypt
         return id_cipher.encrypt( s ).encode( 'hex' )
 
-    def encode_dict_ids( self, a_dict, kind=None ):
+    def encode_dict_ids( self, a_dict, kind=None, skip_startswith=None ):
         """
         Encode all ids in dictionary. Ids are identified by (a) an 'id' key or
         (b) a key that ends with '_id'
         """
         for key, val in a_dict.items():
-            if key == 'id' or key.endswith('_id'):
+            if key == 'id' or key.endswith('_id') and ( skip_startswith is None or not key.startswith( skip_startswith ) ):
                 a_dict[ key ] = self.encode_id( val, kind=kind )
 
         return a_dict

lib/galaxy/webapps/galaxy/api/tools.py

@@ -284,7 +284,7 @@ def create( self, trans, payload, **kwd ):
             # so it's possible to figure out which newly created elements
             # correspond with which tool file outputs
             output_dict[ 'output_name' ] = output_name
-            outputs.append( trans.security.encode_dict_ids( output_dict ) )
+            outputs.append( trans.security.encode_dict_ids( output_dict, skip_startswith="metadata_" ) )
 
         for job in vars.get('jobs', []):
             rval[ 'jobs' ].append( self.encode_all_ids( trans, job.to_dict( view='collection' ), recursive=True ) )

test-data/biom1_metadata_test.txt

@@ -0,0 +1,11 @@
+table_rows: __ob__u__sq__2__sq__, u__sq__3__sq__, u__sq__4__sq__, u__sq__5__sq__, u__sq__8__sq__, u__sq__9__sq__, u__sq__10__sq__, u__sq__11__sq__, u__sq__12__sq__, u__sq__14__sq__, u__sq__15__sq__, u__sq__16__sq__, u__sq__17__sq__, u__sq__18__sq__, u__sq__22__sq__, u__sq__23__sq__, u__sq__25__sq__, u__sq__28__sq__, u__sq__29__sq____cb__
+table_matrix_element_type: float
+table_format: Biological Observation Matrix 1.0.0
+table_generated_by: BIOM-Format 2.1.5
+table_matrix_type: sparse
+table_shape: __ob__19, 2__cb__
+table_format_url: http://biom-format.org
+table_date: 2016-05-26T16:43:45.614267
+table_type: OTU table
+table_id: None
+table_columns: __ob__u__sq__SAMPLE_1__sq__, u__sq__SAMPLE_2__sq____cb__

test-data/input_taxonomy.biom1

@@ -0,0 +1 @@
+{"id": "None","format": "Biological Observation Matrix 1.0.0","format_url": "http://biom-format.org","matrix_type": "sparse","generated_by": "BIOM-Format 2.1.5","date": "2016-05-26T16:43:45.614267","type": "OTU table","matrix_element_type": "float","shape": [19, 2],"data": [[1,0,160.0],[1,1,242.0],[6,0,1.0],[6,1,1.0],[7,0,3.0],[7,1,4.0],[12,0,13.0],[12,1,36.0],[14,0,1.0],[14,1,5.0],[15,0,1.0],[16,0,1.0],[16,1,3.0]],"rows": [{"id": "2", "metadata": {"taxonomy": ["d__Archaea"]}},{"id": "3", "metadata": {"taxonomy": ["d__Bacteria"]}},{"id": "4", "metadata": {"taxonomy": ["d__Archaea", "p__Crenarchaeota"]}},{"id": "5", "metadata": {"taxonomy": ["d__Archaea", "p__Euryarchaeota"]}},{"id": "8", "metadata": {"taxonomy": ["d__Bacteria", "p__AC1"]}},{"id": "9", "metadata": {"taxonomy": ["d__Bacteria", "p__AD3"]}},{"id": "10", "metadata": {"taxonomy": ["d__Bacteria", "p__Acidobacteria"]}},{"id": "11", "metadata": {"taxonomy": ["d__Bacteria", "p__Actinobacteria"]}},{"id": "12", "metadata": {"taxonomy": ["d__Bacteria", "p__AncK6"]}},{"id": "14", "metadata": {"taxonomy": ["d__Bacteria", "p__Armatimonadetes"]}},{"id": "15", "metadata": {"taxonomy": ["d__Bacteria", "p__BHI80-139"]}},{"id": "16", "metadata": {"taxonomy": ["d__Bacteria", "p__BRC1"]}},{"id": "17", "metadata": {"taxonomy": ["d__Bacteria", "p__Bacteroidetes"]}},{"id": "18", "metadata": {"taxonomy": ["d__Bacteria", "p__CD12"]}},{"id": "22", "metadata": {"taxonomy": ["d__Bacteria", "p__Chlorobi"]}},{"id": "23", "metadata": {"taxonomy": ["d__Bacteria", "p__Chloroflexi"]}},{"id": "25", "metadata": {"taxonomy": ["d__Bacteria", "p__Cyanobacteria"]}},{"id": "28", "metadata": {"taxonomy": ["d__Bacteria", "p__EM19"]}},{"id": "29", "metadata": {"taxonomy": ["d__Bacteria", "p__EM3"]}}],"columns": [{"id": "SAMPLE_1", "metadata": null},{"id": "SAMPLE_2", "metadata": null}]}

test/functional/tools/metadata_biom1.xml

@@ -0,0 +1,28 @@
+<tool id="metadata_biom1" name="metadata_BIOM1" version="1.0.0">
+    <command>cp "${input_metadata_values}" "${output_of_input_metadata}"</command>
+    <configfiles>
+        <configfile name="input_metadata_values">table_rows: ${input_biom1.metadata.table_rows}
+table_matrix_element_type: ${input_biom1.metadata.table_matrix_element_type}
+table_format: ${input_biom1.metadata.table_format}
+table_generated_by: ${input_biom1.metadata.table_generated_by}
+table_matrix_type: ${input_biom1.metadata.table_matrix_type}
+table_shape: ${input_biom1.metadata.table_shape}
+table_format_url: ${input_biom1.metadata.table_format_url}
+table_date: ${input_biom1.metadata.table_date}
+table_type: ${input_biom1.metadata.table_type}
+table_id: ${input_biom1.metadata.table_id}
+table_columns: ${input_biom1.metadata.table_columns}</configfile>
+    </configfiles>
+    <inputs>
+      <param name="input_biom1" type="data" format="biom1" label="BIOM1 File"/>
+    </inputs>
+    <outputs>
+      <data format="txt" name="output_of_input_metadata" />
+    </outputs>
+    <tests>
+      <test>
+        <param name="input_biom1" value="input_taxonomy.biom1" ftype="biom1" />
+        <output name="output_of_input_metadata" ftype="txt" file="biom1_metadata_test.txt"/>
+      </test>
+    </tests>
+</tool>

test/functional/tools/sample_datatypes_conf.xml

@@ -13,5 +13,6 @@
     <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true" />
     <datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM" />
     <datatype extension="bcf" type="galaxy.datatypes.binary:Bcf" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bcf' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BCF" />
+    <datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="True" subclass="True" mimetype="application/json"/>
   </registration>
-</datatypes>
+</datatypes>

test/functional/tools/samples_tool_conf.xml

@@ -24,6 +24,7 @@
   <tool file="metadata.xml" />
   <tool file="metadata_bam.xml" />
   <tool file="metadata_bcf.xml" />
+  <tool file="metadata_biom1.xml" />
   <tool file="strict_shell.xml" />
   <tool file="strict_shell_default_off.xml" />
   <tool file="detect_errors_aggressive.xml" />