From 2b6bc653a6b097dd2b30d858061e3bd7b7cc6eb2 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 12 Jun 2017 17:56:45 -0700
Subject: [PATCH 01/76] Add button to delete selected genomes

---
 web/dist/js/islandcompare-rest-urls.js |  1 +
 web/pages/home.html                    | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/web/dist/js/islandcompare-rest-urls.js b/web/dist/js/islandcompare-rest-urls.js
index 1b66ae35..2846a256 100644
--- a/web/dist/js/islandcompare-rest-urls.js
+++ b/web/dist/js/islandcompare-rest-urls.js
@@ -4,6 +4,7 @@ var authUrl = rootUrl + "accounts/auth-token/";
 var registerUrl = rootUrl + "accounts/register/";
 var genomesUrl = rootUrl + "genomes/";
 var genomesUploadUrl = rootUrl + "genomes/upload/";
+var genomesUpdateUrl = rootUrl + "genomes/details/";
 var genomesGenesUrl = rootUrl + "genomes/genes/";
 var analysisUrl = rootUrl + "analysis/";
 var analysisRunUrl = rootUrl + "analysis/run/";
diff --git a/web/pages/home.html b/web/pages/home.html
index bb465f0a..d267e8b2 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -49,6 +49,9 @@ <h3 class="box-title">Upload Genomes</h3>
             <div class="box">
                 <div class="box-header with-border">
                     <h3 class="box-title">Uploaded Genomes</h3>
+                    <button class="btn btn-primary btn-flat pull-right" id="genome-delete">
+                        Delete Selected Genomes
+                    </button>
                 </div>
                 <!-- /.box-header -->
                 <div class="box-body">
@@ -207,6 +210,27 @@ <h3 class="box-title">Run Analysis</h3>
             return false;
         });
 
+        $("#genome-delete").click(function() {
+            var selected_genomes = $("select#selected-genomes-field option").map(function () {
+                return $(this).val();
+            }).get();
+            for (var i = 0; i < selected_genomes.length; i++) {
+                $.ajax({
+                    type: "DELETE",
+                    url: genomesUpdateUrl + selected_genomes[i],
+                    headers: {
+                        "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+                    },
+                    success: function() {
+                        genomesTable.row('.selected').remove().draw();
+                    },
+                    error: function(message) {
+                        BootstrapDialog.warning(message.responseText);
+                    }
+                });
+            }
+        });
+
         $.ajax({
             type: "GET",
             url: analysisUrl,
@@ -234,7 +258,7 @@ <h3 class="box-title">Run Analysis</h3>
             "Authorization": 'Token'+' '+getAuthenticationCookie()
         },
         success: function(){
-            ReloadGenomesTable();
+            ReloadGenomesTable(); //TODO
         }
     });
 

From 3641181d5c040867bfc14c01b4915df8d652186f Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 13 Jun 2017 11:37:41 -0700
Subject: [PATCH 02/76] Uploaded genomes now appear in table immediately

---
 web/pages/home.html | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/web/pages/home.html b/web/pages/home.html
index d267e8b2..5fa95315 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -210,6 +210,8 @@ <h3 class="box-title">Run Analysis</h3>
             return false;
         });
 
+        // DELETE on the url of each selected genome
+        // Genome is then removed from table and from the selected genomes area
         $("#genome-delete").click(function() {
             var selected_genomes = $("select#selected-genomes-field option").map(function () {
                 return $(this).val();
@@ -223,6 +225,7 @@ <h3 class="box-title">Run Analysis</h3>
                     },
                     success: function() {
                         genomesTable.row('.selected').remove().draw();
+                        $("#selected-genomes-field option").remove();
                     },
                     error: function(message) {
                         BootstrapDialog.warning(message.responseText);
@@ -243,7 +246,7 @@ <h3 class="box-title">Run Analysis</h3>
             success: function(data){
                 ReloadRecentAnalysisTable(data);
             }
-        })
+        });
 
 
     });
@@ -257,8 +260,24 @@ <h3 class="box-title">Run Analysis</h3>
             "Cache-Control": "",
             "Authorization": 'Token'+' '+getAuthenticationCookie()
         },
-        success: function(){
-            ReloadGenomesTable(); //TODO
+        // If upload was successful, another GET call is made to find the id of the last genome - the one that was just
+        // uploaded. This id is needed to add a row for the new genome into the DataTable.
+        // There must be a better way to do this. While the data returned by the upload (up_data) contains the
+        // file name (up_data.name), it doesn't seem to contain the id.
+        success: function(up_data){
+            var new_row_id;
+            $.ajax({
+                type: "GET",
+                url: genomesUrl,
+                headers: {
+                    "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+                },
+                success: function(data) {
+                    var new_row_id = data[data.length - 1].id;
+                    var genomesTable = $("#uploaded-genomes").DataTable();
+                    genomesTable.row.add({ name: up_data.name, id: new_row_id }).draw();
+                }
+            });
         }
     });
 

From af4a040ef0d2489da7e5a9844ad7a7ba61256855 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 14 Jun 2017 16:39:30 -0700
Subject: [PATCH 03/76] Deleting genome now deletes related analyses on
 confirmation from user

---
 IslandCompare/analysis/urls.py  |  1 +
 IslandCompare/analysis/views.py |  9 +++++
 IslandCompare/genomes/views.py  |  9 +++++
 web/pages/home.html             | 67 +++++++++++++++++++++++++--------
 4 files changed, 70 insertions(+), 16 deletions(-)

diff --git a/IslandCompare/analysis/urls.py b/IslandCompare/analysis/urls.py
index 7d6ea063..628756bc 100644
--- a/IslandCompare/analysis/urls.py
+++ b/IslandCompare/analysis/urls.py
@@ -7,4 +7,5 @@
     url(r'^export/(?P<pk>[0-9]+)$', views.ExportAnalysisResultView.as_view(), name="analysis_export"),
     url(r'^results/(?P<pk>[0-9]+)$', views.AnalysisResultsView.as_view(), name="analysis_results"),
     url(r'^run/', views.AnalysisRunView.as_view(), name="analysis_run"),
+    url(r'^delete/(?P<pk>[0-9]+)$', views.AnalysisDestroyView.as_view(), name="analysis_destroy"),
 ]
diff --git a/IslandCompare/analysis/views.py b/IslandCompare/analysis/views.py
index 5aae6bfd..2f843347 100644
--- a/IslandCompare/analysis/views.py
+++ b/IslandCompare/analysis/views.py
@@ -40,6 +40,15 @@ class AnalysisRetrieveUpdateView(generics.RetrieveUpdateAPIView):
     def get_queryset(self):
         return Analysis.objects.filter(owner=self.request.user)
 
+class AnalysisDestroyView(generics.RetrieveDestroyAPIView):
+    """
+    Destroy Analysis
+    """
+    permission_classes = [IsAuthenticated]
+    serializer_class = AnalysisSerializer
+
+    def get_queryset(self):
+        return Analysis.objects.filter(owner=self.request.user)
 
 class AnalysisRunView(APIView):
     """
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index d0813d96..b597baa7 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -2,6 +2,7 @@
 from rest_framework.permissions import IsAuthenticated
 from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GenomeGenesSerializer
 from genomes.models import Genome
+from analysis.models import Analysis
 from rest_framework.parsers import MultiPartParser, FormParser
 from rest_framework.response import Response
 
@@ -54,6 +55,14 @@ class GenomeRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView):
     def get_queryset(self):
         return Genome.objects.filter(owner=self.request.user)
 
+    def delete(self, request, *args, **kwargs):
+        genome = self.get_object()
+        for a in Analysis.objects.filter(genomes__id__contains=genome.id):
+            # print(a.name)
+            a.delete()
+        genome.delete()
+        return response.Response(status=204)
+
 
 class GenomeGeneRetrieveView(generics.RetrieveAPIView):
     """
diff --git a/web/pages/home.html b/web/pages/home.html
index 5fa95315..09de3369 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -216,22 +216,37 @@ <h3 class="box-title">Run Analysis</h3>
             var selected_genomes = $("select#selected-genomes-field option").map(function () {
                 return $(this).val();
             }).get();
-            for (var i = 0; i < selected_genomes.length; i++) {
-                $.ajax({
-                    type: "DELETE",
-                    url: genomesUpdateUrl + selected_genomes[i],
-                    headers: {
-                        "Authorization": 'Token' + ' ' + getAuthenticationCookie()
-                    },
-                    success: function() {
-                        genomesTable.row('.selected').remove().draw();
-                        $("#selected-genomes-field option").remove();
-                    },
-                    error: function(message) {
-                        BootstrapDialog.warning(message.responseText);
+            var names = $("select#selected-genomes-field option").map(function () {
+                return $(this).text();
+            }).get();
+            $.ajax({
+                type: "GET",
+                url: analysisUrl,
+                headers: {
+                    "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+                },
+                success: function(data) {
+                    var corresponding_analyses = [];
+                    for (var analysis_i = 0; analysis_i < data.length; analysis_i++) {
+                        for (var genome_i = 0; genome_i < selected_genomes.length; genome_i++) {
+                            var gen_id = parseInt(selected_genomes[genome_i]);
+                            if ($.inArray(gen_id, data[analysis_i].genomes) !== -1) {
+                                if ($.inArray(data[analysis_i].name, corresponding_analyses) == -1) {
+                                    corresponding_analyses.push(data[analysis_i].name);
+                                }
+                            }
+                        }
                     }
-                });
-            }
+                    if (corresponding_analyses.length) {
+                        var ask = confirm("Deleting the genomes " + names + " will delete corresponding analyses:\n" + corresponding_analyses + "\nAre you sure?");
+                        if (ask) {
+                            deleteGenomes(selected_genomes);
+                        }
+                    } else {
+                        deleteGenomes(selected_genomes);
+                    }
+                }
+            });
         });
 
         $.ajax({
@@ -273,7 +288,7 @@ <h3 class="box-title">Run Analysis</h3>
                     "Authorization": 'Token' + ' ' + getAuthenticationCookie()
                 },
                 success: function(data) {
-                    var new_row_id = data[data.length - 1].id;
+                    var new_row_id = data[data.length - 1].id; // TODO validate this is correct genome
                     var genomesTable = $("#uploaded-genomes").DataTable();
                     genomesTable.row.add({ name: up_data.name, id: new_row_id }).draw();
                 }
@@ -349,6 +364,26 @@ <h3 class="box-title">Run Analysis</h3>
         });
     }
 
+    function deleteGenomes(selected_genomes) {
+        for (var i = 0; i < selected_genomes.length; i++) {
+                $.ajax({
+                    type: "DELETE",
+                    url: genomesUpdateUrl + selected_genomes[i],
+                    headers: {
+                        "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+                    },
+                    success: function() {
+                        var genomesTable = $("#uploaded-genomes").DataTable();
+                        genomesTable.row('.selected').remove().draw();
+                        $("#selected-genomes-field option").remove();
+                    },
+                    error: function(message) {
+                        BootstrapDialog.warning(message.responseText);
+                    }
+                });
+            }
+    }
+
 </script>
 </body>
 </html>

From 055d64bb7978b69df1b161e1f5d76de266e786a7 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 14 Jun 2017 17:47:25 -0700
Subject: [PATCH 04/76] Add option to remove analysis on job history page

---
 web/dist/js/islandcompare-rest-urls.js |  3 ++-
 web/pages/history.html                 | 29 ++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/web/dist/js/islandcompare-rest-urls.js b/web/dist/js/islandcompare-rest-urls.js
index 2846a256..84dfd2db 100644
--- a/web/dist/js/islandcompare-rest-urls.js
+++ b/web/dist/js/islandcompare-rest-urls.js
@@ -9,4 +9,5 @@ var genomesGenesUrl = rootUrl + "genomes/genes/";
 var analysisUrl = rootUrl + "analysis/";
 var analysisRunUrl = rootUrl + "analysis/run/";
 var analysisResultsUrl = rootUrl + "analysis/results/";
-var analysisExportUrl = rootUrl + "analysis/export/";
\ No newline at end of file
+var analysisExportUrl = rootUrl + "analysis/export/";
+var analysisDeleteUrl = rootUrl + "analysis/delete/";
\ No newline at end of file
diff --git a/web/pages/history.html b/web/pages/history.html
index 57348845..c8764ee9 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -33,6 +33,7 @@ <h3 class="box-title">Job History</h3>
                             <th>Mash-MCL Status</th>
                             <th>Results</th>
                             <th>Export</th>
+                            <th>Delete</th>
                         </tr>
                         </thead>
                         <tbody>
@@ -87,6 +88,26 @@ <h3 class="box-title">Job History</h3>
         });
     }
 
+    function delete_job(jobid) {
+        var ask = confirm("Are you sure you want to delete analysis " + jobid + "?");
+        if (ask) {
+            $.ajax({
+                type: "DELETE",
+                url: analysisDeleteUrl + jobid,
+                beforeSend: function (request) {
+                    request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
+                },
+                success: function(data){
+                    var jobtable = $("#job-history").DataTable();
+                    jobtable.row('#' + jobid).remove().draw();
+                },
+                error: function(message) {
+                    BootstrapDialog.warning(message.responseText);
+                }
+            });
+        }
+    }
+
     $(document).ready(function () {
         jobtable = $("#job-history").DataTable({
             ajax: {
@@ -190,6 +211,14 @@ <h3 class="box-title">Job History</h3>
                             return '<button class="btn btn-block btn-flat disabled recent-job-item"><i class="fa fa-download" aria-hidden="true"></button>';
                         }
                     }
+                },
+                {
+                    "aTargets": [9],
+                    "mData": null,
+                    "mRender": function (data, type, full) {
+                        var analysis_id = full['id'];
+                        return '<a href="#" onclick="javascript:delete_job(' + analysis_id + ');"><button class="btn btn-block btn-flat btn-danger recent-job-item">Delete</button></a>';
+                    }
                 }
             ],
             rowId: 'id'

From efd437397b9541094862bab002c2be4af0bdaaaa Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 15 Jun 2017 15:29:23 -0700
Subject: [PATCH 05/76] Add rename option to job history page

---
 web/dist/js/islandcompare-rest-urls.js |  3 ++-
 web/pages/history.html                 | 30 ++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/web/dist/js/islandcompare-rest-urls.js b/web/dist/js/islandcompare-rest-urls.js
index 84dfd2db..baa37b36 100644
--- a/web/dist/js/islandcompare-rest-urls.js
+++ b/web/dist/js/islandcompare-rest-urls.js
@@ -10,4 +10,5 @@ var analysisUrl = rootUrl + "analysis/";
 var analysisRunUrl = rootUrl + "analysis/run/";
 var analysisResultsUrl = rootUrl + "analysis/results/";
 var analysisExportUrl = rootUrl + "analysis/export/";
-var analysisDeleteUrl = rootUrl + "analysis/delete/";
\ No newline at end of file
+var analysisDeleteUrl = rootUrl + "analysis/delete/";
+var analysisUpdateUrl = rootUrl + "analysis/details/";
\ No newline at end of file
diff --git a/web/pages/history.html b/web/pages/history.html
index c8764ee9..35a77602 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -33,6 +33,7 @@ <h3 class="box-title">Job History</h3>
                             <th>Mash-MCL Status</th>
                             <th>Results</th>
                             <th>Export</th>
+                            <th>Rename</th>
                             <th>Delete</th>
                         </tr>
                         </thead>
@@ -88,6 +89,27 @@ <h3 class="box-title">Job History</h3>
         });
     }
 
+    function rename_job(jobid) {
+        var new_name = prompt("Please enter new name");
+        $.ajax({
+            type: "PATCH",
+            url: analysisUpdateUrl + jobid,
+            data: {name: new_name},
+            beforeSend: function (request) {
+                request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
+            },
+            success: function() {
+                var jobtable = $("#job-history").DataTable();
+                var row_values = jobtable.row('#'+jobid).data();
+                row_values.name = new_name;
+                jobtable.row('#'+jobid).data(row_values).draw();
+            },
+            error: function(message) {
+                BootstrapDialog.warning(message.responseText);
+            }
+        });
+    }
+
     function delete_job(jobid) {
         var ask = confirm("Are you sure you want to delete analysis " + jobid + "?");
         if (ask) {
@@ -215,6 +237,14 @@ <h3 class="box-title">Job History</h3>
                 {
                     "aTargets": [9],
                     "mData": null,
+                    "mRender": function (data, type, full) {
+                        var analysis_id = full['id'];
+                        return '<a href="#" onclick="javascript:rename_job(' + analysis_id + ');"><button class="btn btn-block btn-flat btn-success recent-job-item">Rename</button></a>'
+                    }
+                },
+                {
+                    "aTargets": [10],
+                    "mData": null,
                     "mRender": function (data, type, full) {
                         var analysis_id = full['id'];
                         return '<a href="#" onclick="javascript:delete_job(' + analysis_id + ');"><button class="btn btn-block btn-flat btn-danger recent-job-item">Delete</button></a>';

From 12944fb0cd3d59c3ae6b1a705e8e6b3edc74b431 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 16 Jun 2017 16:40:34 -0700
Subject: [PATCH 06/76] Add RGI status and bootstrap dialog to history. Recent
 jobs table improved

---
 web/pages/history.html | 108 ++++++++++++++++++++++------------
 web/pages/home.html    | 129 ++++++++++++++++++++++-------------------
 2 files changed, 139 insertions(+), 98 deletions(-)

diff --git a/web/pages/history.html b/web/pages/history.html
index 35a77602..ae30ca07 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -26,6 +26,7 @@ <h3 class="box-title">Job History</h3>
                         <tr>
                             <th>Analysis ID</th>
                             <th>Analysis Name</th>
+                            <th>RGI Status</th>
                             <th>Parsnp Status</th>
                             <th>Mauve Status</th>
                             <th>SigiHMM Status</th>
@@ -90,44 +91,67 @@ <h3 class="box-title">Job History</h3>
     }
 
     function rename_job(jobid) {
-        var new_name = prompt("Please enter new name");
-        $.ajax({
-            type: "PATCH",
-            url: analysisUpdateUrl + jobid,
-            data: {name: new_name},
-            beforeSend: function (request) {
-                request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
-            },
-            success: function() {
-                var jobtable = $("#job-history").DataTable();
-                var row_values = jobtable.row('#'+jobid).data();
-                row_values.name = new_name;
-                jobtable.row('#'+jobid).data(row_values).draw();
-            },
-            error: function(message) {
-                BootstrapDialog.warning(message.responseText);
+        //var new_name = prompt("Please enter new name");
+        BootstrapDialog.confirm({
+            title: 'Please enter new name',
+            message: $("<input class='form-control' id='new_name' maxlength='100'/>"),
+            callback: function(result) {
+                if (result) {
+                    var new_name = $("#new_name").val();
+                    $.ajax({
+                        type: "PATCH",
+                        url: analysisUpdateUrl + jobid,
+                        data: {name: new_name},
+                        beforeSend: function (request) {
+                            request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
+                        },
+                        success: function() {
+                            var jobtable = $("#job-history").DataTable();
+                            var row_values = jobtable.row('#'+jobid).data();
+                            row_values.name = new_name;
+                            jobtable.row('#'+jobid).data(row_values).draw();
+                        },
+                        error: function(message) {
+                            BootstrapDialog.warning(message.responseText);
+                        }
+                    });
+                }
             }
         });
+
     }
 
     function delete_job(jobid) {
-        var ask = confirm("Are you sure you want to delete analysis " + jobid + "?");
-        if (ask) {
-            $.ajax({
-                type: "DELETE",
-                url: analysisDeleteUrl + jobid,
-                beforeSend: function (request) {
-                    request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
-                },
-                success: function(data){
-                    var jobtable = $("#job-history").DataTable();
-                    jobtable.row('#' + jobid).remove().draw();
-                },
-                error: function(message) {
-                    BootstrapDialog.warning(message.responseText);
+        BootstrapDialog.show({
+            title: 'Warning',
+            message: 'Are you sure you want to delete analysis ' + jobid + '?',
+            type: BootstrapDialog.TYPE_WARNING,
+            buttons: [{
+                label: 'Delete',
+                action: function(dialog) {
+                    $.ajax({
+                        type: "DELETE",
+                        url: analysisDeleteUrl + jobid,
+                        beforeSend: function (request) {
+                            request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
+                        },
+                        success: function(data){
+                            var jobtable = $("#job-history").DataTable();
+                            jobtable.row('#' + jobid).remove().draw();
+                        },
+                        error: function(message) {
+                            BootstrapDialog.warning(message.responseText);
+                        }
+                    });
+                    dialog.close();
                 }
-            });
-        }
+            },{
+                label: 'Cancel',
+                action: function(dialog) {
+                    dialog.close();
+                }
+            }]
+        });
     }
 
     $(document).ready(function () {
@@ -142,6 +166,7 @@ <h3 class="box-title">Job History</h3>
             columns: [
                 { data: 'id'},
                 { data: 'name' },
+                { data: 'analysiscomponent_set.rgi.status'},
                 { data: 'analysiscomponent_set.parsnp.status', defaultContent: "USER DEFINED TREE"},
                 { data: 'analysiscomponent_set.mauve.status'},
                 { data: 'analysiscomponent_set.sigi.status', defaultContent: "USER DEFINED GIS"},
@@ -150,11 +175,12 @@ <h3 class="box-title">Job History</h3>
             ],
             aoColumnDefs: [
                 {
-                    "aTargets": [7],
+                    "aTargets": [8],
                     "mData": null,
                     "mRender": function (data, type, full) {
                         var analysis_id = full['id'];
                         var end_status = null;
+                        var rgi_status = null;
                         var parsnp_status = null;
                         var mauve_status = null;
                         var sigi_status = null;
@@ -167,6 +193,13 @@ <h3 class="box-title">Job History</h3>
                             end_status = "UNKNOWN"
                         }
 
+                        if( typeof full['analysiscomponent_set']['rgi'] !== 'undefined' ) {
+                            rgi_status = full['analysiscomponent_set']['rgi']['status'];
+                        }
+                        else{
+                            rgi_status = "UNKNOWN"
+                        }
+
                         if( typeof full['analysiscomponent_set']['parsnp'] !== 'undefined' ) {
                             parsnp_status = full['analysiscomponent_set']['parsnp']['status'];
                         }
@@ -198,7 +231,8 @@ <h3 class="box-title">Job History</h3>
                             island_path_status = "UNKNOWN"
                         }
 
-                        if (parsnp_status === "FAILURE" ||
+                        if (rgi_status === "FAILURE" ||
+                            parsnp_status === "FAILURE" ||
                             mauve_status === "FAILURE" ||
                             sigi_status === "FAILURE" ||
                             island_path_status === "FAILURE") {
@@ -213,7 +247,7 @@ <h3 class="box-title">Job History</h3>
                     }
                 },
                 {
-                    "aTargets": [8],
+                    "aTargets": [9],
                     "mData": null,
                     "mRender": function (data, type, full) {
                         var analysis_id = full['id'];
@@ -235,7 +269,7 @@ <h3 class="box-title">Job History</h3>
                     }
                 },
                 {
-                    "aTargets": [9],
+                    "aTargets": [10],
                     "mData": null,
                     "mRender": function (data, type, full) {
                         var analysis_id = full['id'];
@@ -243,7 +277,7 @@ <h3 class="box-title">Job History</h3>
                     }
                 },
                 {
-                    "aTargets": [10],
+                    "aTargets": [11],
                     "mData": null,
                     "mRender": function (data, type, full) {
                         var analysis_id = full['id'];
diff --git a/web/pages/home.html b/web/pages/home.html
index 09de3369..af439dc5 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -20,6 +20,9 @@ <h1>
             <div class="box">
                 <div class="box-header with-border">
                     <h3 class="box-title">Recent Jobs</h3>
+                    <button class="btn btn-primary btn-flat pull-right" id="recent-job-refresh">
+                        <i class="fa fa-refresh" aria-hidden="true"></i>
+                    </button>
                 </div><!-- /.box-header -->
                 <div class="box-body" id="recent-jobs-body">
 
@@ -198,6 +201,7 @@ <h3 class="box-title">Run Analysis</h3>
                     $("#selected-genomes-field option").remove();
                     genomesTable.rows( { selected: true } ).deselect();
                     BootstrapDialog.alert("Analysis has been submitted.");
+                    ReloadRecentAnalysisTable();
                 },
                 error: function(message) {
                     BootstrapDialog.warning(message.responseText);
@@ -249,20 +253,11 @@ <h3 class="box-title">Run Analysis</h3>
             });
         });
 
-        $.ajax({
-            type: "GET",
-            url: analysisUrl,
-            data: {
-                "num": 3
-            },
-            headers: {
-                "Authorization": 'Token' + ' ' + getAuthenticationCookie()
-            },
-            success: function(data){
-                ReloadRecentAnalysisTable(data);
-            }
+        $("#recent-job-refresh").click(function() {
+            ReloadRecentAnalysisTable();
         });
 
+        ReloadRecentAnalysisTable();
 
     });
 
@@ -296,60 +291,72 @@ <h3 class="box-title">Run Analysis</h3>
         }
     });
 
-    function ReloadRecentAnalysisTable(data) {
-        var table = '<ul class ="list-group">';
+    function ReloadRecentAnalysisTable() {
+        $.ajax({
+            type: "GET",
+            url: analysisUrl,
+            data: {
+                "num": 3
+            },
+            headers: {
+                "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+            },
+            success: function(data){
+                var table = '<ul class ="list-group">';
 
-        for (var i = 0; i < data.length; i++){
-            table = table.concat(
-                '<li class="list-group-item"><a href="#" class="recent-job-item" data-jobid="' +
-                data[i]["id"] + '"><h5>' + data[i]["name"] + '</h5>'
-            );
+                for (var i = 0; i < data.length; i++){
+                    table = table.concat(
+                        '<li class="list-group-item"><a href="#" class="recent-job-item" data-jobid="' +
+                        data[i]["id"] + '"><h5>' + data[i]["name"] + '</h5>'
+                    );
 
-            var success_count = 0;
-            var state = 0; //1 = Success, Fail = -1, 0 = Pending
-            for (var j in Object.keys(data[i]["analysiscomponent_set"])) {
-                var status = data[i]["analysiscomponent_set"][Object.keys(data[i]["analysiscomponent_set"])[j]]["status"];
-                if (status === "FAILURE"){
-                    //Fail
-                    state = -1;
-                }
-                else if (status === "SUCCESS"){
-                    success_count += 1;
-                }
-                else{
-                    //Assume pending, do nothing
+                    var success_count = 0;
+                    var state = 0; //1 = Success, Fail = -1, 0 = Pending
+                    for (var j in Object.keys(data[i]["analysiscomponent_set"])) {
+                        var status = data[i]["analysiscomponent_set"][Object.keys(data[i]["analysiscomponent_set"])[j]]["status"];
+                        if (status === "FAILURE"){
+                            //Fail
+                            state = -1;
+                        }
+                        else if (status === "SUCCESS"){
+                            success_count += 1;
+                        }
+                        else{
+                            //Assume pending, do nothing
+                        }
+                    }
+
+                    if (success_count === Object.keys(data[i]["analysiscomponent_set"]).length){
+                        table = table.concat(
+                            '<div class="progress sm">' +
+                            '<div class="progress-bar progress-bar-success progress-bar-striped active" style="width: 100%"></div>' +
+                            '</div>'
+                        )
+                    }
+                    else if (state === -1){
+                        var percent = (success_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
+                        table = table.concat(
+                            '<div class="progress sm">' +
+                            '<div class="progress-bar progress-bar-danger progress-bar-striped active" style="width: ' + percent + '%"></div>' +
+                            '</div>'
+                        )
+                    }
+                    else{
+                        var percent = (success_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
+                        table = table.concat(
+                            '<div class="progress sm">' +
+                            '<div class="progress-bar progress-bar-info progress-bar-striped active" style="width: ' + percent + '%"></div>' +
+                            '</div>'
+                        )
+                    }
+                    table = table.concat('</a></li>');
                 }
-            }
+                table = table.concat('</ul>');
+                $("#recent-jobs-body").html(table);
 
-            if (success_count === Object.keys(data[i]["analysiscomponent_set"]).length){
-                table = table.concat(
-                    '<div class="progress sm">' +
-                        '<div class="progress-bar progress-bar-success progress-bar-striped active" style="width: 100%"></div>' +
-                    '</div>'
-                )
-            }
-            else if (state === -1){
-                var percent = (success_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
-                table = table.concat(
-                    '<div class="progress sm">' +
-                    '<div class="progress-bar progress-bar-danger progress-bar-striped active" style="width: ' + percent + '%"></div>' +
-                    '</div>'
-                )
+                loadRecentJobFunction();
             }
-            else{
-                var percent = (success_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
-                table = table.concat(
-                    '<div class="progress sm">' +
-                    '<div class="progress-bar progress-bar-info progress-bar-striped active" style="width: ' + percent + '%"></div>' +
-                    '</div>'
-                )
-            }
-            table = table.concat('</a></li>');
-        }
-        table = table.concat('</ul>');
-        $("#recent-jobs-body").html(table);
-
-        loadRecentJobFunction();
+        });
     }
 
     function loadRecentJobFunction() {

From 8850d0024296b0ff9eb00aaaec796fb5cd33a4aa Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 19 Jun 2017 11:45:51 -0700
Subject: [PATCH 07/76] Improved delete genome warning messages

---
 web/pages/home.html | 53 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 6 deletions(-)

diff --git a/web/pages/home.html b/web/pages/home.html
index af439dc5..4ff4e0cc 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -214,8 +214,8 @@ <h3 class="box-title">Run Analysis</h3>
             return false;
         });
 
-        // DELETE on the url of each selected genome
-        // Genome is then removed from table and from the selected genomes area
+        // If selected genomes are involved in any analyses, a warning is displayed to the user that these analyses will
+        // be removed along with the genomes. If the user confirms or there are no related analyses, deleteGenomes is called
         $("#genome-delete").click(function() {
             var selected_genomes = $("select#selected-genomes-field option").map(function () {
                 return $(this).val();
@@ -242,12 +242,52 @@ <h3 class="box-title">Run Analysis</h3>
                         }
                     }
                     if (corresponding_analyses.length) {
-                        var ask = confirm("Deleting the genomes " + names + " will delete corresponding analyses:\n" + corresponding_analyses + "\nAre you sure?");
-                        if (ask) {
-                            deleteGenomes(selected_genomes);
+                        var message = "Deleting the genome";
+                        if (names.length > 1) {
+                            message += "s";
                         }
+                        message += ": '" + names.join("', '") + "' will delete corresponding ";
+                        if (corresponding_analyses.length > 1) {
+                            message += "analyses:\n'";
+                        } else {
+                            message += "analysis:\n'";
+                        }
+                        message += corresponding_analyses.join("'\n'") + "'\n";
+                        BootstrapDialog.show({
+                            title: 'Warning',
+                            message: message + "<br /><p><b>Are you sure?</b></p>",
+                            type: BootstrapDialog.TYPE_WARNING,
+                            buttons: [{
+                                label: 'Delete',
+                                action: function(dialog) {
+                                    deleteGenomes(selected_genomes);
+                                    dialog.close();
+                                }
+                            },{
+                                label: 'Cancel',
+                                action: function(dialog) {
+                                    dialog.close();
+                                }
+                            }]
+                        });
                     } else {
-                        deleteGenomes(selected_genomes);
+                        BootstrapDialog.show({
+                            title: 'Warning',
+                            message: "Delete '" + names.join("', ") + "'\n<br /><p><b>Are you sure?</b></p>",
+                            type: BootstrapDialog.TYPE_WARNING,
+                            buttons: [{
+                                label: 'Delete',
+                                action: function(dialog) {
+                                    deleteGenomes(selected_genomes);
+                                    dialog.close();
+                                }
+                            },{
+                                label: 'Cancel',
+                                action: function(dialog) {
+                                    dialog.close();
+                                }
+                            }]
+                        });
                     }
                 }
             });
@@ -383,6 +423,7 @@ <h3 class="box-title">Run Analysis</h3>
                         var genomesTable = $("#uploaded-genomes").DataTable();
                         genomesTable.row('.selected').remove().draw();
                         $("#selected-genomes-field option").remove();
+                        ReloadRecentAnalysisTable();
                     },
                     error: function(message) {
                         BootstrapDialog.warning(message.responseText);

From eba2c752a722701b159833a4349ff41c52b480d6 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 22 Jun 2017 18:00:36 -0700
Subject: [PATCH 08/76] SigiHMM and islandpath failures do not crash pipeline,
 merge_gis and mash will get skipped. Status reflected in job history

---
 IslandCompare/analysis/tasks.py | 54 ++++++++++++++++++---------------
 web/pages/history.html          | 27 ++++++++++++++---
 2 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index 70905d68..b6213904 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -1,5 +1,5 @@
 from __future__ import absolute_import, unicode_literals
-from celery import shared_task
+from celery import shared_task, chain
 from analysis.models import AnalysisComponent
 from analysis import components
 from analysis.pipeline import PipelineSerializer
@@ -75,26 +75,18 @@ def run_pipeline(self, serialized_pipeline):
     pipeline.analysis.celery_task_id = self.request.id
     pipeline.analysis.save()
 
-    index = len(pipeline.pipeline_components) - 1
-    next_component = None
-
-    while index != 0:
-        if next_component is None:
-            next_component = run_pipeline_component.s(pipeline.analysis.id,
-                                                      pipeline.pipeline_components[index].name,
-                                                      pipeline.pipeline_components[index].param)
+    component_list = serialized_pipeline['pipeline_components']
+    tasks = [run_pipeline_component.s(serialized_pipeline, pipeline.analysis.id, "start_pipeline")]
+    for comp in component_list[1:]:
+        if comp == "sigi":
+            tasks.append(run_pipeline_component.s(pipeline.analysis.id, comp)
+                         .on_error(sigi_error_handler.s(pipeline.analysis.id)))
+        elif comp == "islandpath":
+            tasks.append(run_pipeline_component.s(pipeline.analysis.id, comp)
+                         .on_error(ipath_error_handler.s(pipeline.analysis.id)))
         else:
-            current_component = run_pipeline_component.s(pipeline.analysis.id,
-                                                         pipeline.pipeline_components[index].name,
-                                                         pipeline.pipeline_components[index].param)
-            current_component.link(next_component)
-            next_component = current_component
-        index -= 1
-
-    task_chain = run_pipeline_component.s(serialized_pipeline,
-                                          pipeline.analysis.id,
-                                          pipeline.pipeline_components[index].name)
-    task_chain.link(next_component)
+            tasks.append(run_pipeline_component.s(pipeline.analysis.id, comp))
+    task_chain = chain(*tasks)
 
     logger.info("End scheduling of pipeline components with celery task id: {}".format(self.request.id))
     return task_chain.apply_async()
@@ -103,7 +95,7 @@ def run_pipeline(self, serialized_pipeline):
 @shared_task(bind=True)
 def run_pipeline_component(self, report, analysis_id, pipeline_component_name, pipeline_components_param=None):
     """
-    Runs a pipleine component
+    Runs a pipeline component
     :param self:
     :param report:
     :param analysis_id:
@@ -112,12 +104,24 @@ def run_pipeline_component(self, report, analysis_id, pipeline_component_name, p
     :return:
     """
     pipeline_component = PipelineComponentFactory.create_component(pipeline_component_name, pipeline_components_param)
-    logger.info("Attempting to run component: {} for analysis with id: {}".format(pipeline_component_name,
-                                                                                   analysis_id))
+    logger.info("Attempting to run component: {} for analysis with id: {}".format(pipeline_component_name, analysis_id))
 
-    analysis_component = AnalysisComponent.objects.get(analysis__id=analysis_id,
-                                                       type__name=pipeline_component.name)
+    analysis_component = AnalysisComponent.objects.get(analysis__id=analysis_id, type__name=pipeline_component.name)
     analysis_component.celery_task_id = self.request.id
     analysis_component.save()
 
     return pipeline_component.run(report)
+
+
+@shared_task()
+def sigi_error_handler(context, exc, traceback, pipeline_id):
+    logger.info("Sigi HMM Failed! Merge_gis and mash_mcl will be skipped. analysis id: {}".format(pipeline_id))
+    task_chain = chain(run_pipeline_component.s(context.args[0], pipeline_id, "islandpath")
+                           .on_error(ipath_error_handler.s(pipeline_id)),
+                       run_pipeline_component.s(pipeline_id, "end_pipeline"))
+    return task_chain.apply_async()
+
+@shared_task()
+def ipath_error_handler(context, exc, traceback, pipeline_id):
+    logger.info("Islandpath Failed! Merge_gis and mash_mcl will be skipped. analysis id: {}".format(pipeline_id))
+    return run_pipeline_component.s(context.args[0], pipeline_id, "end_pipeline").apply_async()
diff --git a/web/pages/history.html b/web/pages/history.html
index ae30ca07..cfcaa2e6 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -170,10 +170,29 @@ <h3 class="box-title">Job History</h3>
                 { data: 'analysiscomponent_set.parsnp.status', defaultContent: "USER DEFINED TREE"},
                 { data: 'analysiscomponent_set.mauve.status'},
                 { data: 'analysiscomponent_set.sigi.status', defaultContent: "USER DEFINED GIS"},
-                { data: 'analysiscomponent_set.islandpath.status', defaultContent: "USER DEFINED GIS"},
-                { data: 'analysiscomponent_set.mash_mcl.status', defaultContent: "USER DEFINED GIS"}
+                { data: 'analysiscomponent_set.islandpath.status', defaultContent: "USER DEFINED GIS"}
             ],
             aoColumnDefs: [
+                {
+                    "aTargets": [7],
+                    "mData": null,
+                    "mRender": function (data, type, full) {
+                        if ( typeof full['analysiscomponent_set']['mash_mcl'] !== 'undefined') {
+                            if (full['analysiscomponent_set']['mash_mcl']['status'] === "PENDING") {
+                                if (full['analysiscomponent_set']['sigi']['status'] === "FAILURE" ||
+                                    full['analysiscomponent_set']['islandpath']['status'] === "FAILURE") {
+                                    return "SKIPPED"
+                                }
+                                return "PENDING"
+                            } else {
+                                return full['analysiscomponent_set']['mash_mcl']['status']
+                            }
+                        } else {
+                            console.log("mash_mcl undefined")
+                            return "UNKNOWN"
+                        }
+                    }
+                },
                 {
                     "aTargets": [8],
                     "mData": null,
@@ -233,9 +252,7 @@ <h3 class="box-title">Job History</h3>
 
                         if (rgi_status === "FAILURE" ||
                             parsnp_status === "FAILURE" ||
-                            mauve_status === "FAILURE" ||
-                            sigi_status === "FAILURE" ||
-                            island_path_status === "FAILURE") {
+                            mauve_status === "FAILURE") {
                             return '<button class="btn btn-block btn-danger btn-flat disabled recent-job-item">Results</button>';
                         }
                         else if (end_status === "SUCCESS") {

From 6e4ddf31cc0d90a5a8dbff4931757c2c5aa01ba3 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 23 Jun 2017 16:22:39 -0700
Subject: [PATCH 09/76] If only one GI predictor passes, merge_gis will be
 assigned those results so mash can still be run.
 ReportVisualizationOverviewSerializer does not assume all components passed

---
 IslandCompare/analysis/components.py  | 12 ++++++++----
 IslandCompare/analysis/serializers.py | 28 +++++++++++----------------
 IslandCompare/analysis/tasks.py       | 12 ++++++++++--
 web/pages/history.html                |  3 +--
 4 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index a9569e29..d2b4ed5e 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -563,11 +563,15 @@ def set_threshold(self, threshold):
     def analysis(self, report):
         merged_gi_dict = dict()
 
-        for genome_id in report["islandpath_gis"]:
-            merged_gi_dict[genome_id] = self.merge_gi_list(report["islandpath_gis"][str(genome_id)],
-                                                           report["sigi_gis"][str(genome_id)])
+        # MergeIslands is only called when islandpath is successful
+        if "sigi_gis" in report:
+            for genome_id in report["islandpath_gis"]:
+                merged_gi_dict[genome_id] = self.merge_gi_list(report["islandpath_gis"][str(genome_id)],
+                                                               report["sigi_gis"][str(genome_id)])
 
-        report["merge_gis"] = merged_gi_dict
+            report["merge_gis"] = merged_gi_dict
+        else:
+            report["merge_gis"] = report["islandpath_gis"]
 
 
 class MashMclClusterPipelineComponent(PipelineComponent):
diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index a8a99349..a94551f9 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -82,7 +82,7 @@ class RunAnalysisSerializer(serializers.Serializer):
 
     def validate_genomes(self, value):
         if len(value) <= 1:
-            raise serializers.ValidationError("Need 2 or more genomes to create an analysis. Only received {}"
+            raise serializers.ValidationError("Need 2 or more genomes to create an analysis. Received {}"
                                               .format(len(value)))
         return value
 
@@ -128,29 +128,23 @@ def to_representation(self, instance):
         output = dict()
 
         output["genomes"] = dict()
+        gi_types = {"sigi_gis", "islandpath_gis", "merge_gis", "user_gis"}.intersection(instance)
+        if "cluster_gis" in instance:
+            number_clusters = instance["cluster_gis"]["numberClusters"]
+            color_index = self.get_spaced_colors(number_clusters)
         for genome in genomes.all():
             output["genomes"][genome.id] = dict()
             output["genomes"][genome.id]["name"] = genome.name
             output["genomes"][genome.id]["length"] = instance["gbk_metadata"][str(genome.id)]['size']
             output["genomes"][genome.id]["amr_genes"] = [{'start': amr['orf_start'], 'end': amr['orf_end'], 'strand': amr['orf_strand']} for amr in instance["amr_genes"][str(genome.id)]]
             output["genomes"][genome.id]["genomic_islands"] = dict()
-
-            if "user_gis" in instance.keys():
-                output["genomes"][genome.id]["genomic_islands"]["user"] = instance["user_gis"][str(genome.id)]
-            else:
-                output["genomes"][genome.id]["genomic_islands"]["sigi"] = [{'start': island[0], 'end': island[1]} for island in instance["sigi_gis"][str(genome.id)]]
-                output["genomes"][genome.id]["genomic_islands"]["islandpath"] = [{'start': island[0], 'end':island[1]} for island in instance["islandpath_gis"][str(genome.id)]]
-                output["genomes"][genome.id]["genomic_islands"]["merged"] = [{'start': island[0], 'end':island[1]} for island in instance["merge_gis"][str(genome.id)]]
-
-        if "user_gis" not in instance.keys():
-            number_clusters = instance["cluster_gis"]["numberClusters"]
-            color_index = self.get_spaced_colors(number_clusters)
-
-            for genome_id in instance["gbk_paths"].keys():
-                for gi_index in range(len(output["genomes"][int(genome_id)]["genomic_islands"]["merged"])):
-                    clusters = instance['cluster_gis'][str(genome_id)]
+            for gi_type in gi_types:
+                output["genomes"][genome.id]["genomic_islands"][gi_type[:-4].replace("merge", "merged")] = [{'start': island[0], 'end': island[1]} for island in instance[gi_type][str(genome.id)]]
+            if "cluster_gis" in instance:
+                for gi_index in range(len(instance["merge_gis"][str(genome.id)])):
+                    clusters = instance['cluster_gis'][str(genome.id)]
                     cluster_index = int(clusters[str(gi_index)])
-                    output["genomes"][int(genome_id)]["genomic_islands"]["merged"][gi_index]['color'] = color_index[cluster_index]
+                    output["genomes"][genome.id]["genomic_islands"]["merged"][gi_index]['color'] = color_index[cluster_index]
 
         output["newick"] = instance["newick"]
         output["alignment"] = instance["alignment"]
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index b6213904..b6206f89 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -115,13 +115,21 @@ def run_pipeline_component(self, report, analysis_id, pipeline_component_name, p
 
 @shared_task()
 def sigi_error_handler(context, exc, traceback, pipeline_id):
-    logger.info("Sigi HMM Failed! Merge_gis and mash_mcl will be skipped. analysis id: {}".format(pipeline_id))
+    logger.info("Sigi HMM Failed! analysis id: {}".format(pipeline_id))
+    del context.args[0]["sigi_gis"]
     task_chain = chain(run_pipeline_component.s(context.args[0], pipeline_id, "islandpath")
                            .on_error(ipath_error_handler.s(pipeline_id)),
+                       run_pipeline_component.s(pipeline_id, "merge_gis"),
+                       run_pipeline_component.s(pipeline_id, "mash_mcl"),
                        run_pipeline_component.s(pipeline_id, "end_pipeline"))
     return task_chain.apply_async()
 
 @shared_task()
 def ipath_error_handler(context, exc, traceback, pipeline_id):
-    logger.info("Islandpath Failed! Merge_gis and mash_mcl will be skipped. analysis id: {}".format(pipeline_id))
+    logger.info("Islandpath Failed! analysis id: {}".format(pipeline_id))
+    del context.args[0]["islandpath_gis"]
+    if "sigi_gis" in context.args[0]:
+        context.args[0]["merge_gis"] = context.args[0]["sigi_gis"]
+        return chain(run_pipeline_component.s(context.args[0], pipeline_id, "mash_mcl"),
+                     run_pipeline_component.s(pipeline_id, "end_pipeline")).apply_async()
     return run_pipeline_component.s(context.args[0], pipeline_id, "end_pipeline").apply_async()
diff --git a/web/pages/history.html b/web/pages/history.html
index cfcaa2e6..554a505e 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -179,7 +179,7 @@ <h3 class="box-title">Job History</h3>
                     "mRender": function (data, type, full) {
                         if ( typeof full['analysiscomponent_set']['mash_mcl'] !== 'undefined') {
                             if (full['analysiscomponent_set']['mash_mcl']['status'] === "PENDING") {
-                                if (full['analysiscomponent_set']['sigi']['status'] === "FAILURE" ||
+                                if (full['analysiscomponent_set']['sigi']['status'] === "FAILURE" &&
                                     full['analysiscomponent_set']['islandpath']['status'] === "FAILURE") {
                                     return "SKIPPED"
                                 }
@@ -188,7 +188,6 @@ <h3 class="box-title">Job History</h3>
                                 return full['analysiscomponent_set']['mash_mcl']['status']
                             }
                         } else {
-                            console.log("mash_mcl undefined")
                             return "UNKNOWN"
                         }
                     }

From aa00f63dff0fcb84665fcefcb78f845040f75a3d Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 23 Jun 2017 16:51:11 -0700
Subject: [PATCH 10/76] Visualization working with optional GI arrays

---
 web/pages/result.html | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/web/pages/result.html b/web/pages/result.html
index c952c524..dd5b6de5 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -101,12 +101,18 @@ <h1>
                     currentSeq.addAMR(data["genomes"][genomeId]["amr_genes"]);
                   
                     if ("user" in data["genomes"][genomeId]["genomic_islands"]){
-                        currentSeq.addGI("user", data["genomes"][genomeId]["genomic_islands"]["user"])
+                        currentSeq.addGI("user", data["genomes"][genomeId]["genomic_islands"]["user"]);
                     }
                     else {
-                        currentSeq.addGI("islandpath", data["genomes"][genomeId]["genomic_islands"]["islandpath"]);
-                        currentSeq.addGI("sigi", data["genomes"][genomeId]["genomic_islands"]["sigi"]);
-                        currentSeq.addGI("merged", data["genomes"][genomeId]["genomic_islands"]["merged"]);
+                        if ("islandpath" in data["genomes"][genomeId]["genomic_islands"]) {
+                            currentSeq.addGI("islandpath", data["genomes"][genomeId]["genomic_islands"]["islandpath"]);
+                        }
+                        if ("sigi" in data["genomes"][genomeId]["genomic_islands"]) {
+                            currentSeq.addGI("sigi", data["genomes"][genomeId]["genomic_islands"]["sigi"]);
+                        }
+                        if ("merged" in data["genomes"][genomeId]["genomic_islands"]) {
+                            currentSeq.addGI("merged", data["genomes"][genomeId]["genomic_islands"]["merged"]);
+                        }
                     }
 
                     // at this scale, individual scaling for sequences may not be usable...so used fixed scale

From d3b9a5b95aaf1b0280cbe9fd763857be8df39747 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 26 Jun 2017 11:52:30 -0700
Subject: [PATCH 11/76] Add warning colour to recent jobs table in case of
 sigi/islandpath fail. Finished jobs with sigi/islandpath fail will have full
 progress bar

---
 web/pages/home.html | 61 ++++++++++++++++++---------------------------
 1 file changed, 24 insertions(+), 37 deletions(-)

diff --git a/web/pages/home.html b/web/pages/home.html
index 4ff4e0cc..311566a5 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -350,46 +350,33 @@ <h3 class="box-title">Run Analysis</h3>
                         data[i]["id"] + '"><h5>' + data[i]["name"] + '</h5>'
                     );
 
-                    var success_count = 0;
-                    var state = 0; //1 = Success, Fail = -1, 0 = Pending
-                    for (var j in Object.keys(data[i]["analysiscomponent_set"])) {
-                        var status = data[i]["analysiscomponent_set"][Object.keys(data[i]["analysiscomponent_set"])[j]]["status"];
-                        if (status === "FAILURE"){
-                            //Fail
-                            state = -1;
-                        }
-                        else if (status === "SUCCESS"){
-                            success_count += 1;
-                        }
-                        else{
-                            //Assume pending, do nothing
+                    var done_count = 0;
+                    var state = "info"; //"info" = pending, "warning" = allowed failure, "danger" = disallowed failure
+                    for (var j in data[i]["analysiscomponent_set"]) {
+                        var status = data[i]["analysiscomponent_set"][j]["status"];
+                        if (status !== "PENDING") {
+                            done_count += 1;
+                            if (status === "FAILURE" && state !== "danger") {
+                                if (j === "sigi" || j === "islandpath") {
+                                    state = "warning";
+                                } else {
+                                    state = "danger";
+                                }
+                            }
                         }
                     }
-
-                    if (success_count === Object.keys(data[i]["analysiscomponent_set"]).length){
-                        table = table.concat(
-                            '<div class="progress sm">' +
-                            '<div class="progress-bar progress-bar-success progress-bar-striped active" style="width: 100%"></div>' +
-                            '</div>'
-                        )
-                    }
-                    else if (state === -1){
-                        var percent = (success_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
-                        table = table.concat(
-                            '<div class="progress sm">' +
-                            '<div class="progress-bar progress-bar-danger progress-bar-striped active" style="width: ' + percent + '%"></div>' +
-                            '</div>'
-                        )
-                    }
-                    else{
-                        var percent = (success_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
-                        table = table.concat(
-                            '<div class="progress sm">' +
-                            '<div class="progress-bar progress-bar-info progress-bar-striped active" style="width: ' + percent + '%"></div>' +
-                            '</div>'
-                        )
+                    var percent = (done_count / Object.keys(data[i]["analysiscomponent_set"]).length) * 100;
+                    if (data[i]["analysiscomponent_set"]["end_pipeline"]["status"] === "SUCCESS") {
+                        percent = 100;
+                        if (state === "info") {
+                            state = "success";
+                        }
                     }
-                    table = table.concat('</a></li>');
+                    table = table.concat(
+                        '<div class="progress sm">' +
+                        '<div class="progress-bar progress-bar-' + state + ' progress-bar-striped active" style="width: ' + percent + '%"></div>' +
+                        '</div></a></li>'
+                    );
                 }
                 table = table.concat('</ul>');
                 $("#recent-jobs-body").html(table);

From 954329bc5baf1ba5ee45be0c8231308db13c05a9 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 26 Jun 2017 16:37:58 -0700
Subject: [PATCH 12/76] Add warning message and removed checkboxes for failed
 GI predictors. Fixes #89

---
 IslandCompare/analysis/pipeline.py    |  7 ++++++-
 IslandCompare/analysis/serializers.py |  1 +
 IslandCompare/analysis/tasks.py       |  2 ++
 web/pages/result.html                 | 27 +++++++++++++++++++++++++--
 4 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/IslandCompare/analysis/pipeline.py b/IslandCompare/analysis/pipeline.py
index b1bdcf66..f3fb8c53 100644
--- a/IslandCompare/analysis/pipeline.py
+++ b/IslandCompare/analysis/pipeline.py
@@ -133,6 +133,8 @@ class Pipeline(object):
     """A list of available dependencies"""
     analysis = None
     """The analysis"""
+    failed_components = None
+    """A list of failed components"""
 
     def __init__(self):
         """
@@ -141,6 +143,7 @@ def __init__(self):
         self.pipeline_components = []
         self.available_dependencies = []
         self.analysis = None
+        self.failed_components = []
 
     def create_database_entry(self, name, genomes, owner):
         """
@@ -188,7 +191,8 @@ def serialize(pipeline):
             'analysis': pipeline.analysis.id,
             'available_dependencies': pipeline.available_dependencies,
             'pipeline_components': serialized_pipeline_components,
-            'component_param': serialized_pipeline_param
+            'component_param': serialized_pipeline_param,
+            'failed_components': pipeline.failed_components
         }
         return serialized_dict
 
@@ -208,5 +212,6 @@ def deserialize(serialized_dict, pipeline_component_factory):
             )
         pipeline.available_dependencies = serialized_dict['available_dependencies']
         pipeline.analysis = Analysis.objects.get(id=serialized_dict['analysis'])
+        pipeline.failed_components = serialized_dict['failed_components']
 
         return pipeline
diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index a94551f9..6a19a04b 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -148,6 +148,7 @@ def to_representation(self, instance):
 
         output["newick"] = instance["newick"]
         output["alignment"] = instance["alignment"]
+        output["failed_components"] = instance["failed_components"]
 
         return output
 
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index b6206f89..c8af12d7 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -117,6 +117,7 @@ def run_pipeline_component(self, report, analysis_id, pipeline_component_name, p
 def sigi_error_handler(context, exc, traceback, pipeline_id):
     logger.info("Sigi HMM Failed! analysis id: {}".format(pipeline_id))
     del context.args[0]["sigi_gis"]
+    context.args[0]["failed_components"].append("sigi")
     task_chain = chain(run_pipeline_component.s(context.args[0], pipeline_id, "islandpath")
                            .on_error(ipath_error_handler.s(pipeline_id)),
                        run_pipeline_component.s(pipeline_id, "merge_gis"),
@@ -128,6 +129,7 @@ def sigi_error_handler(context, exc, traceback, pipeline_id):
 def ipath_error_handler(context, exc, traceback, pipeline_id):
     logger.info("Islandpath Failed! analysis id: {}".format(pipeline_id))
     del context.args[0]["islandpath_gis"]
+    context.args[0]["failed_components"].append("islandpath")
     if "sigi_gis" in context.args[0]:
         context.args[0]["merge_gis"] = context.args[0]["sigi_gis"]
         return chain(run_pipeline_component.s(context.args[0], pipeline_id, "mash_mcl"),
diff --git a/web/pages/result.html b/web/pages/result.html
index dd5b6de5..a0601a76 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -8,6 +8,11 @@
 
 <!-- Content Header (Page header) -->
 <section class="content-header">
+    <div class="row">
+        <div class="col-xs-3"></div>
+        <div hidden id="warning-box" class="alert alert-warning alert-dismissable col-xs-6 text-center"></div>
+        <div class="col-xs-3"></div>
+    </div>
     <h1>
         Results
     </h1>
@@ -49,8 +54,12 @@ <h1>
                             </svg>
                             <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
 
-                            <input type="checkbox" name="sigi" value="sigi" id="sigi-box" checked>Sigi-HMM
-                            <input type="checkbox" name="islandpath" value="islandpath" id="islandpath-box" checked>IslandPath
+                            <span id="sigi-toggle">
+                                <input type="checkbox" name="sigi" value="sigi" id="sigi-box" checked>Sigi-HMM
+                            </span>
+                            <span id="islandpath-toggle">
+                                <input type="checkbox" name="islandpath" value="islandpath" id="islandpath-box" checked>IslandPath
+                            </span>
                         </div>
                         <!--
                         --Currently Not used
@@ -83,6 +92,20 @@ <h1>
                 request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
             },
             success: function(data) {
+                console.log(data);
+                if (data["failed_components"].length !== 0) {
+                    var warning_message = "<h4>Warning:</h4>";
+                    if ($.inArray("sigi",data["failed_components"]) !== -1) {
+                        warning_message = warning_message.concat("<p>Sigi-HMM Failed</p>");
+                        $("#sigi-toggle").hide();
+                    }
+                    if ($.inArray("islandpath",data["failed_components"]) !== -1) {
+                        warning_message = warning_message.concat("<p>IslandPath Failed</p>");
+                        $("#islandpath-toggle").hide();
+                    }
+                    $("#warning-box").html(warning_message)
+                        .show();
+                }
                 // Add the newick data to generate a tree
                 container.newickData = Newick.parse(data["newick"]);
                 // Set the root of the tree

From f6e44a76bac7fe86d9fc9aa39bde5dcd9ae93288 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 29 Jun 2017 12:57:40 -0700
Subject: [PATCH 13/76] Fixes #107 by retrying parsnp component when newick is
 incorrect due to file name bug

---
 IslandCompare/analysis/components.py | 14 ++++++++++++--
 IslandCompare/analysis/tasks.py      | 10 ++++++++--
 web/pages/result.html                |  1 -
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index d2b4ed5e..00110fde 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -164,10 +164,13 @@ def convert_gbk_to_fna(input_path, output_path):
         input_handle.close()
 
     @staticmethod
-    def parse_newick(input_path):
+    def parse_newick(input_path, num_genomes):
         tree = Phylo.read(input_path, 'newick')
         terminals = tree.get_terminals()
 
+        # Parsnp may fail to include all genomes in the newick in specific cases
+        assert (len(terminals) == num_genomes), "Newick produced by Parsnp does not contain all genomes"
+
         for leaf in terminals:
             leaf.name = leaf.name.split('.')[0]
 
@@ -201,7 +204,14 @@ def analysis(self, report):
             subprocess.check_call(script_file.name, stdout=logs)
         script_file.close()
 
-        report["newick"] = self.parse_newick(self.temp_results_dir + "/parsnp.tree")
+        try:
+            # parse_newick also validates that newick contains all genomes
+            report["newick"] = self.parse_newick(self.temp_results_dir + "/parsnp.tree", len(report["gbk_paths"]))
+        except AssertionError:
+            # temp dir with incorrect newick file is deleted so parsnp can remake on retry
+            if self.temp_results_dir is not None and os.path.exists(self.temp_results_dir):
+                rmtree(self.temp_results_dir)
+            raise
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index c8af12d7..784b86e8 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -110,8 +110,14 @@ def run_pipeline_component(self, report, analysis_id, pipeline_component_name, p
     analysis_component.celery_task_id = self.request.id
     analysis_component.save()
 
-    return pipeline_component.run(report)
-
+    try:
+        return pipeline_component.run(report)
+    except (AssertionError, FileNotFoundError) as exc:
+        # Parsnp may fail to include all genomes in the newick in specific cases - retry until success or 6 failures
+        if pipeline_component_name == "parsnp":
+            raise self.retry(exc=exc, countdown=10, max_retries=5)
+        else:
+            raise
 
 @shared_task()
 def sigi_error_handler(context, exc, traceback, pipeline_id):
diff --git a/web/pages/result.html b/web/pages/result.html
index a0601a76..32766009 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -92,7 +92,6 @@ <h1>
                 request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
             },
             success: function(data) {
-                console.log(data);
                 if (data["failed_components"].length !== 0) {
                     var warning_message = "<h4>Warning:</h4>";
                     if ($.inArray("sigi",data["failed_components"]) !== -1) {

From 8b44236c12e6f42b2f099e23a3e3f239a8bbc6f6 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 30 Jun 2017 10:21:00 -0700
Subject: [PATCH 14/76] Fixed #109

---
 web/dist/js/genomecomparevis.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 9836ec26..396c6cd7 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -195,9 +195,7 @@ function MultiVis(targetNode){
     //Renders the graph
     this.render = function (){
         this.container.select("svg").remove();
-        if (self.scale == null){
-            self.setScale(0,this.getLargestSequenceSize());
-        }
+
         // Gets the sequenceOrder of the graph
         var seqOrder = self.getSequenceOrder();
 
@@ -209,6 +207,11 @@ function MultiVis(targetNode){
             .attr("width",this.containerWidth())
             .attr("height",(this.containerHeight()+this.getSequenceModHeight()*2)+TOPPADDING);
 
+        //Scale is set after adding svg as adding svg changes container width
+        if (self.scale == null){
+            self.setScale(0,this.getLargestSequenceSize());
+        }
+
         //Add the visualization container
         var visContainer = svg.append("g")
             .attr("class","visContainer");

From 63912fb4f72375a067cca9079b482c526971fe4f Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 4 Jul 2017 16:07:44 -0700
Subject: [PATCH 15/76] Fixed #108. Homologous regions are now indexed by
 genome ids rather than sequential index

---
 web/dist/js/genomecomparevis.js | 110 +++++++++++++++++---------------
 web/pages/result.html           |  15 +++--
 2 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 396c6cd7..ab8904e5 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -192,6 +192,21 @@ function MultiVis(targetNode){
         this.transition();
     };
 
+    // Post-order tree traversal to get the order of the sequences
+    this.traverseTreeForOrder = function (node) {
+        var tree = [];
+        if (node.branchset != null) {
+            for (var nodeIndex = 0; nodeIndex < node.branchset.length; nodeIndex++) {
+                var output = this.traverseTreeForOrder(node.branchset[nodeIndex]);
+                Array.prototype.push.apply(tree, output);
+            }
+        }
+        else {
+            return [node['name']];
+        }
+        return tree;
+    };
+
     //Renders the graph
     this.render = function (){
         this.container.select("svg").remove();
@@ -236,26 +251,8 @@ function MultiVis(targetNode){
                     self.newickRoot = d;
                     //Set the order of the sequences (as linear plot and tree should match)
                     //Do a post-order traversal on tree looking for leaves.
-                    var tree = [];
-                    traverseTreeForOrder = function (node) {
-                        if (node['children'] != null) {
-                            for (var nodeIndex = 0; nodeIndex < node['children'].length; nodeIndex++) {
-                                output = traverseTreeForOrder(node['children'][nodeIndex]);
-                                if (output != null) {
-                                    tree.push(output);
-                                }
-                            }
-                        }
-                        else {
-                            var memory = (node['name'].replace(/'/g, "").split(/[\\/]/).pop());
-                            memory = memory.split(".")[0];
-                            return self.backbone.getSequenceIdFromName(memory);
-                        }
-                        return null;
-                    };
-                    traverseTreeForOrder(self.newickRoot);
-                    //Set the sequenceOrder of the tree to the sequences obtained from traversal of the new treeRoot
-                    self.sequenceOrder = tree;
+                    var seqIds = self.traverseTreeForOrder(self.newickRoot);
+                    self.sequenceOrder = self.backbone.getIndicesFromIds(seqIds);
                     //Re-renders the graph
                     self.transition();
                 }
@@ -270,25 +267,24 @@ function MultiVis(targetNode){
 
         //Draw Homologous Region Lines
         var lines = [];
-
-        for (var i=0; i<this.sequences.length-1; i++){
+        for (var i = 0; i < seqOrder.length - 1; i++){
             var seqlines = sequenceHolder.append("g")
                 .attr("class","all-homolous-regions");
-            //Find a way to clean this line up
-            var homologousRegions = (this.backbone.retrieveHomologousRegions(seqOrder[i],seqOrder[i+1]));
+
+            var seqId = this.backbone.sequences[seqOrder[i]].sequenceId;
+            var homologousRegions = this.backbone.retrieveHomologousRegions(seqId);
 
             //Homologous Region polygons (shaded regions)
-            for (var j=0;j<homologousRegions.length;j++){
+            for (var j = 0; j < homologousRegions.length; j++){
                 var homolousRegion = seqlines.append("g")
                     .attr("class", "homologous-region")
-                    .attr("transform","translate("+ 0 +","
-                        +SEQUENCEWIDTH/2+")");
+                    .attr("transform","translate("+ 0 +","+ SEQUENCEWIDTH/2 +")");
 
                 //Build Shaded Polygon For Homologous Region
-                var points = self.scale(this.sequences[i].scale(homologousRegions[j].start1))+","+(this.getSequenceModHeight()*i+SEQUENCEWIDTH/2)+" ";
-                points += self.scale(this.sequences[i].scale(homologousRegions[j].end1))+","+(this.getSequenceModHeight()*i+SEQUENCEWIDTH/2)+" ";
-                points += self.scale(this.sequences[i+1].scale(homologousRegions[j].end2))+","+(this.getSequenceModHeight()*(i+1)-SEQUENCEWIDTH/2)+" ";
-                points += self.scale(this.sequences[i+1].scale(homologousRegions[j].start2))+","+(this.getSequenceModHeight()*(i+1)-SEQUENCEWIDTH/2)+" ";
+                var points = self.scale(homologousRegions[j].start1)+","+(this.getSequenceModHeight()*i+SEQUENCEWIDTH/2)+" ";
+                points += self.scale(homologousRegions[j].end1)+","+(this.getSequenceModHeight()*i+SEQUENCEWIDTH/2)+" ";
+                points += self.scale(homologousRegions[j].end2)+","+(this.getSequenceModHeight()*(i+1)-SEQUENCEWIDTH/2)+" ";
+                points += self.scale(homologousRegions[j].start2)+","+(this.getSequenceModHeight()*(i+1)-SEQUENCEWIDTH/2)+" ";
 
                 homolousRegion.append("polygon")
                     .attr("points",points)
@@ -528,7 +524,7 @@ function MultiVis(targetNode){
 function Backbone() {
     var self = this;
     this.sequences = [];
-    this.backbone = [[]];
+    this.backbone = {};
 
     // Retrieves the sequenceId from sequences given the name of a sequence.
     // Will return -1 if no sequence is found
@@ -541,6 +537,31 @@ function Backbone() {
         return -1;
     };
 
+    // Remove if unused
+    this.getIndexFromId = function (id) {
+        id = id.toString();
+        for (var index = 0; index < this.sequences.length; index++) {
+            if (this.sequences[index]['sequenceId'] == id) {
+                return index;
+            }
+        }
+        return -1;
+    };
+
+    this.getIndicesFromIds = function(ids) {
+        var adjuster = 0;
+        var indices = [];
+        for (var seq_i = 0; seq_i < this.sequences.length; seq_i++) {
+            var index = ids.indexOf(this.sequences[seq_i].sequenceId)
+            if (index != -1) {
+                indices.push(index + adjuster);
+            } else {
+                adjuster++;
+            }
+        }
+        return indices;
+    };
+
     this.addSequence = function (sequenceId, sequenceSize, sequenceName, givenName) {
         sequenceName = sequenceName || null;
         givenName = givenName || null;
@@ -553,29 +574,16 @@ function Backbone() {
         return self.sequences;
     };
 
-    this.addHomologousRegion = function (seqId1, seqId2, start1, end1, start2, end2) {
-        if (this.backbone[seqId1] === undefined) {
-            this.backbone[seqId1] = [];
-        }
-
-        if (this.backbone[seqId1][seqId2] === undefined) {
-            this.backbone[seqId1][seqId2] = [];
-        }
-
-        if (this.backbone[seqId2] === undefined) {
-            this.backbone[seqId2] = [];
-        }
-
-        if (this.backbone[seqId2][seqId1] === undefined) {
-            this.backbone[seqId2][seqId1] = [];
+    this.addHomologousRegion = function (seqId, start1, end1, start2, end2) {
+        if (this.backbone[seqId] === undefined) {
+            this.backbone[seqId] = [];
         }
-        this.backbone[seqId1][seqId2].push(new HomologousRegion(start1, end1, start2, end2));
-        this.backbone[seqId2][seqId1].push(new HomologousRegion(start2, end2, start1, end1));
+        this.backbone[seqId].push(new HomologousRegion(start1, end1, start2, end2));
     };
 
-    this.retrieveHomologousRegions = function (seqId1, seqId2) {
+    this.retrieveHomologousRegions = function (seqId) {
         try {
-            var homologousRegions = this.backbone[seqId1][seqId2];
+            var homologousRegions = this.backbone[seqId];
         } catch (e) {
             return [];
         }
diff --git a/web/pages/result.html b/web/pages/result.html
index 32766009..985d02db 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -110,9 +110,12 @@ <h1>
                 // Set the root of the tree
                 container.newickRoot = container.newickData;
 
+                // Traverse tree to find sequence id order
+                var treeOrder = container.traverseTreeForOrder(container.newickRoot);
+
                 // Add Genomes to Visualization
-                for (var genomeIndex in container.newickData["branchset"]){
-                    var genomeId = container.newickData["branchset"][genomeIndex]["name"];
+                for (var i = 0; i < treeOrder.length; i++) {
+                    var genomeId = treeOrder[i];
                     var currentSeq = container.backbone.addSequence(
                         genomeId,
                         data["genomes"][genomeId]["length"],
@@ -141,13 +144,11 @@ <h1>
                     currentSeq.updateScale(0, container.getLargestSequenceSize(), container.getLargestSequenceSize());
                 }
 
-                // Add homolous regions to Visualization
-                // TODO Fix ids for homologous region, doesnt use genome ids but rather goes from 0 -> N
-                for (var i = 0; i < container.newickData["branchset"].length - 1; i++){
+                // Add homologous regions to Visualization
+                for (var i = 0; i < data["alignment"].length; i++){
                     for (var j = 0; j < data["alignment"][i].length; j++){
                         container.backbone.addHomologousRegion(
-                            i,
-                            i + 1,
+                            treeOrder[i],
                             data["alignment"][i][j][0],
                             data["alignment"][i][j][1],
                             data["alignment"][i][j][2],

From 55ec05cb7217f48276337ab07d08421ada454cac Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 4 Jul 2017 16:43:11 -0700
Subject: [PATCH 16/76] Adjusted downloadable csv files for optional GI
 predictor success

---
 IslandCompare/analysis/components.py  |  2 +-
 IslandCompare/analysis/serializers.py | 32 ++++++++++++++-------------
 web/pages/history.html                | 26 +++++++++++++++++-----
 3 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 00110fde..f66c7184 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -93,7 +93,7 @@ def parse_rgi_json(json_file):
             for entry_key in json_dict[key]:
                 entry = json_dict[key][entry_key]
                 if type(entry) is dict and "orf_start" in entry:
-                    amr_genes.append({k : entry.get(k) for k in ("orf_start", "orf_end", "orf_strand", "type_match")})
+                    amr_genes.append({k : entry.get(k) for k in ("orf_start", "orf_end", "orf_strand")})
         # Keep only unique entries
         amr_genes = [dict(y) for y in set(tuple(x.items()) for x in amr_genes)]
         amr_sorted = sorted(amr_genes, key=lambda gene: gene['orf_start'])
diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 6a19a04b..30cfe5d9 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -174,26 +174,28 @@ def to_representation(self, instance):
 
         method_keys = ["islandpath_gis", "sigi_gis"]
 
-        for key in instance["gbk_paths"]:
-            counter = 0
-            for island in instance["merge_gis"][key]:
-                genome = Genome.objects.get(id=key)
-                writer.writerow({'name': genome.name,
-                                 'start': island[0],
-                                 'end': island[1],
-                                 'method': 'merge_gis',
-                                 'cluster_id': instance["cluster_gis"][str(genome.id)][str(counter)]
-                                 })
-                counter += 1
-
-        for method in method_keys:
+        if "merge_gis" in instance:
             for key in instance["gbk_paths"]:
-                for island in instance[method][key]:
+                counter = 0
+                for island in instance["merge_gis"][key]:
                     genome = Genome.objects.get(id=key)
                     writer.writerow({'name': genome.name,
                                      'start': island[0],
                                      'end': island[1],
-                                     'method': method})
+                                     'method': 'merge_gis',
+                                     'cluster_id': instance["cluster_gis"][str(genome.id)][str(counter)]
+                                     })
+                    counter += 1
+
+        for method in method_keys:
+            if method in instance:
+                for key in instance["gbk_paths"]:
+                    for island in instance[method][key]:
+                        genome = Genome.objects.get(id=key)
+                        writer.writerow({'name': genome.name,
+                                         'start': island[0],
+                                         'end': island[1],
+                                         'method': method})
 
         contents = output.getvalue()
         output.close()
diff --git a/web/pages/history.html b/web/pages/history.html
index 554a505e..2c8d03d2 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -202,7 +202,7 @@ <h3 class="box-title">Job History</h3>
                         var parsnp_status = null;
                         var mauve_status = null;
                         var sigi_status = null;
-                        var island_path_status = null;
+                        var islandpath_status = null;
 
                         if( typeof full['analysiscomponent_set']['end_pipeline'] !== 'undefined' ) {
                             end_status = full['analysiscomponent_set']['end_pipeline']['status'];
@@ -241,12 +241,11 @@ <h3 class="box-title">Job History</h3>
                         else{
                             sigi_status = "UNKNOWN"
                         }
-
-                        if( typeof full['analysiscomponent_set']['island_path'] !== 'undefined' ) {
-                            island_path_status = full['analysiscomponent_set']['island_path']['status'];
+                        if( typeof full['analysiscomponent_set']['islandpath'] !== 'undefined' ) {
+                            islandpath_status = full['analysiscomponent_set']['islandpath']['status'];
                         }
                         else{
-                            island_path_status = "UNKNOWN"
+                            islandpath_status = "UNKNOWN"
                         }
 
                         if (rgi_status === "FAILURE" ||
@@ -267,13 +266,28 @@ <h3 class="box-title">Job History</h3>
                     "mData": null,
                     "mRender": function (data, type, full) {
                         var analysis_id = full['id'];
+                        var sigi_status = null;
+                        var islandpath_status = null;
+                        var end_status = null;
+                        if( typeof full['analysiscomponent_set']['sigi'] !== 'undefined' ) {
+                            sigi_status = full['analysiscomponent_set']['sigi']['status'];
+                        }
+                        else{
+                            sigi_status = "UNKNOWN"
+                        }
+                        if( typeof full['analysiscomponent_set']['islandpath'] !== 'undefined' ) {
+                            islandpath_status = full['analysiscomponent_set']['islandpath']['status'];
+                        }
+                        else{
+                            islandpath_status = "UNKNOWN"
+                        }
                         if( typeof full['analysiscomponent_set']['end_pipeline'] !== 'undefined' ) {
                             end_status = full['analysiscomponent_set']['end_pipeline']['status'];
                         }
                         else{
                             end_status = "UNKNOWN"
                         }
-                        if (end_status === "FAILURE") {
+                        if (end_status === "FAILURE" || (sigi_status === "FAILURE" && islandpath_status === "FAILURE")) {
                             return '<button class="btn btn-block btn-flat disabled recent-job-item"><i class="fa fa-download" aria-hidden="true"></button>';
                         }
                         else if (end_status === "SUCCESS") {

From d37f3a6082485875414d45eef846edbf69914f20 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 5 Jul 2017 13:59:04 -0700
Subject: [PATCH 17/76] Fixed duplication of AMR polygon elements. Filter out
 AMRs that are not within the current scale domain

---
 web/dist/js/genomecomparevis.js | 75 ++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 33 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index ab8904e5..9b3d82f0 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -321,6 +321,7 @@ function MultiVis(targetNode){
             });
 
         //Add the gis to the SVG
+        //TODO each sequence element contains all GIs from all sequences - fix
         var giFiltervalue = self.getGIFilterValue();
         var gis = seq.each(function(d, i){
             var genomicIslandcontainer = seq.append("g")
@@ -359,39 +360,47 @@ function MultiVis(targetNode){
         //Add AMR genes to the SVG
         var amrcontainer = seq.append("g")
             .attr("class", "amrs");
-        seq.each(function(d, i) {
-            for (var amrIndex = 0; amrIndex < d.amr.length; amrIndex++) {
-                var amr = d.amr[amrIndex];
-                var startPosition = parseInt(amr['start']);
-                var endPosition = parseInt(amr['end']);
-                var plus_strand = (amr['strand'] == "+") ? true : false;
-                var adjust = GISIZE * 2;
-                var polypoints = "";
-
-                // If scale is close, AMRs will be rendered as trapezoids
-                if ((self.scale.domain()[1] - self.scale.domain()[0]) < self.getGeneFilterValue() * 3) {
-                    var width = Math.abs(self.scale(endPosition) - self.scale(startPosition));
-                    // min ensures the angles on the trapezoids are at most 45 degrees
-                    polypoints = self.scale(startPosition) + Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
-                    polypoints += self.scale(endPosition) - Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
-                    polypoints += self.scale(endPosition) - Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE) + " ";
-                    polypoints += self.scale(startPosition) + Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE);
-                // Else AMRs will be rectangles of reduced height to improve appearance from a wider view
-                } else {
-                    // Each rectangle is given a min width of 1 so they will be visible
-                    polypoints = self.scale(startPosition) + "," + (self.getSequenceModHeight() * i) + " ";
-                    polypoints += Math.max(self.scale(endPosition), self.scale(startPosition) + 1) + "," + (self.getSequenceModHeight() * i) + " ";
-                    polypoints += Math.max(self.scale(endPosition), self.scale(startPosition) + 1) + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
-                    polypoints += self.scale(startPosition) + "," + (self.getSequenceModHeight() * i - GISIZE / 2);
-                    adjust = 3 * GISIZE / 2;
-                }
-                var p = amrcontainer.append("polygon")
-                    .attr("points", polypoints);
-                // Positive strand AMRs are in place, negative strand AMRs must be moved down to the other side
-                if (!plus_strand) {
-                    p.attr("transform", "translate(0," + ( adjust ) +")");
-                }
-            }
+        amrcontainer.each(function(d, i) {
+            // AMRs are given more detail at a smaller scale
+            var details = ((self.scale.domain()[1] - self.scale.domain()[0]) < self.getGeneFilterValue() * 3);
+            d3.select(this).selectAll("polygon")
+                .data(d.amr.filter(function(d) {
+                    // Filter out the AMRs that are not within the current view
+                    return (d.start < self.scale.domain()[1] && d.end > self.scale.domain()[0]);
+                }))
+                .enter().append("polygon")
+                .attr("points", function(d) {
+                    var startPosition = self.scale(parseInt(d.start));
+                    var endPosition = self.scale(parseInt(d.end));
+                    var plus_strand = (d.strand === "+");
+                    var polypoints = "";
+                    // If scale is close, AMRs will be rendered as trapezoids
+                    if (details) {
+                        var width = Math.abs(endPosition - startPosition);
+                        // min ensures the angles on the trapezoids are at most 45 degrees
+                        polypoints = startPosition + Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += endPosition - Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += endPosition - Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE) + " ";
+                        polypoints += startPosition + Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE);
+                    // Else AMRs will be rectangles of reduced height to improve appearance from a wider view
+                    } else {
+                        // Each rectangle is given a min width of 1 so they will be visible
+                        polypoints = startPosition + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += Math.max(endPosition, startPosition + 1) + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += Math.max(endPosition, startPosition + 1) + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
+                        polypoints += startPosition + "," + (self.getSequenceModHeight() * i - GISIZE / 2);
+                    }
+                    return polypoints;
+                })
+                .attr("transform", function(d) {
+                    var plus_strand = (d.strand === "+");
+                    // The rectangles are smaller so they require a smaller adjustment
+                    var adjust = GISIZE * 2 - (+!details * GISIZE / 2);
+                    // Positive strand AMRs are in place, negative strand AMRs need to be moved to the other side
+                    if (!plus_strand) {
+                        return "translate(0," + adjust + ")";
+                    }
+                });
         });
 
         //Add the brush for zooming and focusing

From b6ca82345d1a499450f92f7785bf74347325fd90 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 5 Jul 2017 17:09:57 -0700
Subject: [PATCH 18/76] Fixed duplication of GI polygon elements. Filter out
 GIs that are not within the current scale domain

---
 web/dist/js/genomecomparevis.js | 96 ++++++++++++++++-----------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 9b3d82f0..c467b190 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -295,14 +295,14 @@ function MultiVis(targetNode){
             }
         }
         //Create the sequence list from the ordered sequence list
-        var sequencedata = [];
+        var sequenceData = [];
         for (var index=0;index<seqOrder.length;index++){
-            sequencedata.push(this.sequences[seqOrder[index]]);
+            sequenceData.push(this.sequences[seqOrder[index]]);
         }
 
         //Create the sequences container on the svg
         var seq = sequenceHolder.selectAll("sequencesAxis")
-            .data(sequencedata)
+            .data(sequenceData)
             .enter()
             .append("g")
             .attr("class", "sequences");
@@ -320,52 +320,53 @@ function MultiVis(targetNode){
                 return self.scale(d.scale(d.getSequenceSize()));
             });
 
-        //Add the gis to the SVG
-        //TODO each sequence element contains all GIs from all sequences - fix
-        var giFiltervalue = self.getGIFilterValue();
-        var gis = seq.each(function(d, i){
-            var genomicIslandcontainer = seq.append("g")
-                .attr("class","genomicIslands")
-                .attr("transform","translate("+ 0 +","
-                    +0+")");
-            for (var prediction_name in d.gi){
-                var giClassContainer = genomicIslandcontainer.append("g")
-                    .attr("class", prediction_name);
-
-                for (var giIndex=0;giIndex< d.gi[prediction_name].length;giIndex++){
-                    var startPosition = parseInt(d.gi[prediction_name][giIndex]['start']);
-                    var endPosition = parseInt(d.gi[prediction_name][giIndex]['end']);
-
-                    if ((endPosition - startPosition)>giFiltervalue) {
-                        var rectpoints = self.scale(startPosition) + "," + (self.getSequenceModHeight() * i + GISIZE / 2) + " ";
-                        rectpoints += self.scale(endPosition) + "," + (self.getSequenceModHeight() * i + GISIZE / 2) + " ";
-                        rectpoints += self.scale(endPosition) + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
-                        rectpoints += self.scale(startPosition) + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
-
-                        var gi = giClassContainer.append("polygon")
-                            .attr("points", rectpoints)
-                            .attr("stroke-width", 1)
-                            .attr("transform", "translate(" + 0 + "," + (GISIZE / 2) + ")");
-
-                        // if color was given for this gi, then color the fill and stroke of this gi to the given color
-                        if (d.gi[prediction_name][giIndex]['color'] != null) {
-                            gi.attr("fill", d.gi[prediction_name][giIndex]['color'])
-                                .attr("stroke", d.gi[prediction_name][giIndex]['color']);
+        //Add GIs to each sequence
+        seq.append("g").attr("class","genomicIslands").each(function(d, i){
+            var methods = d3.select(this).selectAll("g")
+                .data(Object.keys(d.gi))
+                .enter().append("g")
+                .attr("class", function(d) {
+                    return d;
+                });
+            methods.each(function(method) {
+                d3.select(this).selectAll("polygon")
+                    .data(d.gi[method].filter(function(d) {
+                        // Filter out the GIs that aren't within the current scale or are too small
+                        var withinScale = d.start < self.scale.domain()[1] && d.end > self.scale.domain()[0];
+                        var withinLength = d.end - d.start > self.getGIFilterValue();
+                        return withinScale && withinLength;
+                    }))
+                    .enter().append("polygon")
+                    .attr("points", function(d) {
+                        var startPosition = self.scale(parseInt(d.start));
+                        var endPosition = self.scale(parseInt(d.end));
+                        return startPosition + "," + (self.getSequenceModHeight() * i + GISIZE / 2) + " " +
+                                 endPosition + "," + (self.getSequenceModHeight() * i + GISIZE / 2) + " " +
+                                 endPosition + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " " +
+                               startPosition + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
+                    })
+                    .attr("stroke-width", 1)
+                    .attr("transform", "translate(0," + (GISIZE / 2) + ")")
+                    .attr("fill", function(d) {
+                        if (d.color != null) {
+                            return d.color;
                         }
-                    }
-                }
-            }
+                    })
+                    .attr("stroke", function(d) {
+                        if (d.color != null) {
+                            return d.color;
+                        }
+                    });
+            });
         });
 
-        //Add AMR genes to the SVG
-        var amrcontainer = seq.append("g")
-            .attr("class", "amrs");
-        amrcontainer.each(function(d, i) {
+        //Add AMR genes to each sequence
+        seq.append("g").attr("class", "amrs").each(function(d, i) {
             // AMRs are given more detail at a smaller scale
             var details = ((self.scale.domain()[1] - self.scale.domain()[0]) < self.getGeneFilterValue() * 3);
             d3.select(this).selectAll("polygon")
                 .data(d.amr.filter(function(d) {
-                    // Filter out the AMRs that are not within the current view
+                    // Filter out the AMRs that aren't within the current scale
                     return (d.start < self.scale.domain()[1] && d.end > self.scale.domain()[0]);
                 }))
                 .enter().append("polygon")
@@ -378,16 +379,15 @@ function MultiVis(targetNode){
                     if (details) {
                         var width = Math.abs(endPosition - startPosition);
                         // min ensures the angles on the trapezoids are at most 45 degrees
-                        polypoints = startPosition + Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += startPosition + Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
                         polypoints += endPosition - Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
                         polypoints += endPosition - Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE) + " ";
                         polypoints += startPosition + Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE);
                     // Else AMRs will be rectangles of reduced height to improve appearance from a wider view
                     } else {
-                        // Each rectangle is given a min width of 1 so they will be visible
-                        polypoints = startPosition + "," + (self.getSequenceModHeight() * i) + " ";
-                        polypoints += Math.max(endPosition, startPosition + 1) + "," + (self.getSequenceModHeight() * i) + " ";
-                        polypoints += Math.max(endPosition, startPosition + 1) + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
+                        polypoints += startPosition + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += startPosition + 1 + "," + (self.getSequenceModHeight() * i) + " ";
+                        polypoints += startPosition + 1 + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
                         polypoints += startPosition + "," + (self.getSequenceModHeight() * i - GISIZE / 2);
                     }
                     return polypoints;
@@ -495,7 +495,7 @@ function MultiVis(targetNode){
             .attr("class","sequenceLabels");
 
         var text = textContainer.selectAll("text")
-            .data(sequencedata)
+            .data(sequenceData)
             .enter()
             .append("text");
 

From 9081ea08cfc73d923032bfbe3aa2f6a5d5eb38ea Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 7 Jul 2017 12:34:10 -0700
Subject: [PATCH 19/76] Remove CDS from genes

---
 IslandCompare/genomes/serializers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index 03012227..15058df7 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -2,7 +2,7 @@
 from rest_framework import serializers
 from Bio import SeqIO
 from Bio.Seq import UnknownSeq
-import logging
+import logging, time
 from io import StringIO
 
 class GenomeSerializer(serializers.ModelSerializer):
@@ -93,12 +93,12 @@ def get_genes_from_gbk(self, filePath, start_cut_off=None, end_cut_off=None):
         if start_cut_off is None and end_cut_off is not None:
             self.logger.warning("End cut off is set but start cut off is not. Not using specified cut offs")
 
-        # Given a path to a gbk file, this will return all CDS
+        # Given a path to a gbk file, this will return all genes
         geneList = []
         for record in SeqIO.parse(open(filePath), "genbank"):
             for feature in record.features:
                 geneInfo = {}
-                if feature.type == 'gene' or feature.type == 'CDS':
+                if feature.type == 'gene':
                     # Bio.SeqIO returns 1 for (+) and  -1 for (-)
                     geneInfo['strand'] = feature.location.strand
                     geneInfo['start'] = feature.location.start
@@ -112,7 +112,7 @@ def get_genes_from_gbk(self, filePath, start_cut_off=None, end_cut_off=None):
                     except:
                         logging.info("No Name Found For This Gene")
                         try:
-                            geneInfo['name'] = feature.qualifiers['locus_tag']
+                            geneInfo['name'] = feature.qualifiers['locus_tag'][0]
                         except:
                             logging.info("No Locus Found For This Gene")
                     if start_cut_off is not None and end_cut_off is not None:

From 268b3d288965fa2a42dcc3bdcb5113dfbc52f2e8 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 7 Jul 2017 14:52:46 -0700
Subject: [PATCH 20/76] Merge component will now merge GIs when only one
 predictor passed

---
 IslandCompare/analysis/components.py | 17 ++++++++++-------
 IslandCompare/analysis/tasks.py      |  4 ++--
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index f66c7184..f933cd7b 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -573,15 +573,18 @@ def set_threshold(self, threshold):
     def analysis(self, report):
         merged_gi_dict = dict()
 
-        # MergeIslands is only called when islandpath is successful
         if "sigi_gis" in report:
-            for genome_id in report["islandpath_gis"]:
-                merged_gi_dict[genome_id] = self.merge_gi_list(report["islandpath_gis"][str(genome_id)],
-                                                               report["sigi_gis"][str(genome_id)])
-
-            report["merge_gis"] = merged_gi_dict
+            sigi_gis = report["sigi_gis"]
+        else:
+            sigi_gis = {genome_id: [] for genome_id in report["islandpath_gis"]}
+        if "islandpath_gis" in report:
+            islandpath_gis = report["islandpath_gis"]
         else:
-            report["merge_gis"] = report["islandpath_gis"]
+            islandpath_gis = {genome_id: [] for genome_id in report["sigi_gis"]}
+
+        for genome_id in sigi_gis:
+            merged_gi_dict[genome_id] = self.merge_gi_list(islandpath_gis[str(genome_id)], sigi_gis[str(genome_id)])
+        report["merge_gis"] = merged_gi_dict
 
 
 class MashMclClusterPipelineComponent(PipelineComponent):
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index 784b86e8..186ba781 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -137,7 +137,7 @@ def ipath_error_handler(context, exc, traceback, pipeline_id):
     del context.args[0]["islandpath_gis"]
     context.args[0]["failed_components"].append("islandpath")
     if "sigi_gis" in context.args[0]:
-        context.args[0]["merge_gis"] = context.args[0]["sigi_gis"]
-        return chain(run_pipeline_component.s(context.args[0], pipeline_id, "mash_mcl"),
+        return chain(run_pipeline_component.s(context.args[0], pipeline_id, "merge_gis"),
+                     run_pipeline_component.s(pipeline_id, "mash_mcl"),
                      run_pipeline_component.s(pipeline_id, "end_pipeline")).apply_async()
     return run_pipeline_component.s(context.args[0], pipeline_id, "end_pipeline").apply_async()

From 00a20cc8dc3644d53db496ce7f925c9cced59254 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 7 Jul 2017 17:00:30 -0700
Subject: [PATCH 21/76] Merge component will merge GIs that are within the
 boundaries of another

---
 IslandCompare/analysis/components.py       | 23 +++++++++-------------
 IslandCompare/analysis/tests_components.py | 12 +++++++++++
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index f933cd7b..e2fe6b16 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -550,22 +550,17 @@ class MergeIslandsPipelineComponent(PipelineComponent):
     def merge_gi_list(self, first_list, second_list):
         merged_gis = first_list + second_list
         merged_gis.sort(key=lambda x: int(x[0]))
-        output_gis = []
-
-        current_gi = None
-        for gi in merged_gis:
-            if current_gi is None:
-                current_gi = copy.deepcopy(gi)
-            elif int(gi[0]) < (int(current_gi[1]) + self.threshold) and int(gi[1]) > int(current_gi[1]):
-                current_gi[1] = gi[1]
-            else:
-                output_gis.append(current_gi)
-                current_gi = copy.deepcopy(gi)
 
-        if current_gi is not None:
-            output_gis.append(current_gi)
+        i = 0
+        while i < len(merged_gis) - 1:
+            if int(merged_gis[i][1]) + self.threshold >= int(merged_gis[i + 1][0]):
+                if int(merged_gis[i][1]) < int(merged_gis[i + 1][1]):
+                    merged_gis[i][1] = merged_gis[i + 1][1]
+                del merged_gis[i + 1]
+            else:
+                i += 1
 
-        return output_gis
+        return merged_gis
 
     def set_threshold(self, threshold):
         self.threshold = threshold
diff --git a/IslandCompare/analysis/tests_components.py b/IslandCompare/analysis/tests_components.py
index e1bff010..851542e4 100644
--- a/IslandCompare/analysis/tests_components.py
+++ b/IslandCompare/analysis/tests_components.py
@@ -334,3 +334,15 @@ def test_merge_secondlist_gi_list(self):
         component.cleanup()
 
         self.assertListEqual([["0", "1200"]], self.report["merge_gis"]["1"])
+
+    def test_merge_encompassed_gi(self):
+        self.report["sigi_gis"] = {"1": [["1000", "2000"], ["3000", "4000"]]}
+        self.report["islandpath_gis"] = {"1": [["100", "5000"]]}
+
+        component = MergeIslandsPipelineComponent()
+
+        component.setup(self.report)
+        component.analysis(self.report)
+        component.cleanup()
+
+        self.assertListEqual([["100", "5000"]], self.report["merge_gis"]["1"])
\ No newline at end of file

From 8546ced8d3aed8e45a2462101e85cbb9ffa28e5f Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 11 Jul 2017 18:01:37 -0700
Subject: [PATCH 22/76] Add tRNA and rRNA to visualization. Legend changes
 depending on scale

---
 IslandCompare/genomes/serializers.py |   5 +-
 web/dist/css/linearplot.css          |  30 ++++++--
 web/dist/js/genomecomparevis.js      |  39 +++++-----
 web/pages/result.html                | 104 ++++++++++++++++-----------
 4 files changed, 111 insertions(+), 67 deletions(-)

diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index 15058df7..ca5ea78d 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -2,7 +2,7 @@
 from rest_framework import serializers
 from Bio import SeqIO
 from Bio.Seq import UnknownSeq
-import logging, time
+import logging
 from io import StringIO
 
 class GenomeSerializer(serializers.ModelSerializer):
@@ -98,11 +98,12 @@ def get_genes_from_gbk(self, filePath, start_cut_off=None, end_cut_off=None):
         for record in SeqIO.parse(open(filePath), "genbank"):
             for feature in record.features:
                 geneInfo = {}
-                if feature.type == 'gene':
+                if feature.type in ["gene", "rRNA", "tRNA"]:
                     # Bio.SeqIO returns 1 for (+) and  -1 for (-)
                     geneInfo['strand'] = feature.location.strand
                     geneInfo['start'] = feature.location.start
                     geneInfo['end'] = feature.location.end
+                    geneInfo['type'] = feature.type
                     try:
                         geneInfo['note'] = feature.qualifiers['note']
                     except:
diff --git a/web/dist/css/linearplot.css b/web/dist/css/linearplot.css
index 470080ce..1385de68 100644
--- a/web/dist/css/linearplot.css
+++ b/web/dist/css/linearplot.css
@@ -67,16 +67,26 @@ g.root.node circle{
                                  stroke:#ff9933;
                              }
 
-.genes.print polygon, #genesLegend.print{
-    fill: #0d0d0d;
-    stroke: #0d0d0d;
-}
-
-.genes polygon, #genesLegend{
+.genes polygon.gene, #genesLegend{
     fill: #6acbff;
     stroke: #6acbff;
 }
 
+.genes polygon.tRNA, #tRNALegend{
+    fill: #c26dff;
+    stroke: #c26dff;
+}
+
+.genes polygon.rRNA, #rRNALegend{
+    fill: #7ceeb2;
+    stroke: #7ceeb2;
+}
+
+.genes.print polygon, #genesLegend.print, #tRNALegend.print, #rRNALegend.print{
+    fill: #0d0d0d;
+    stroke: #0d0d0d;
+}
+
 .amrs.print polygon, #amrLegend.print{
     fill: white;
     stroke: black;
@@ -105,6 +115,14 @@ g.root.node circle{
     margin-right: 8px;
 }
 
+.legendBody{
+    padding: 6px;
+}
+
+.legendBody label{
+    font-weight: normal;
+}
+
 .controlSide{
     display: table-cell;
 }
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index c467b190..fd0b075e 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -175,7 +175,7 @@ function MultiVis(targetNode){
 
     //Readjusts the graph for updated sequence domains, TODO (improve later, currently just re-renders graph)
     this.transition = function(){
-        this.container.select("svg").remove();
+        this.container.select("svg").remove(); //Redundant?
         this.render();
     };
 
@@ -428,7 +428,8 @@ function MultiVis(targetNode){
 
         //Add the genes to the plot
         var geneFilterValue = self.getGeneFilterValue();
-        if((self.scale.domain()[1]-self.scale.domain()[0])<geneFilterValue) {
+        if((self.scale.domain()[1] - self.scale.domain()[0]) < geneFilterValue) {
+            $(".genesLegend-toggle").show();
             var geneContainer = sequenceHolder.append("g")
                 .attr("class", "genes")
                 .attr("transform", "translate(0," + (GENESIZE / 4 + GENESIZE/2) + ")");
@@ -438,33 +439,35 @@ function MultiVis(targetNode){
                     async: false,
                     success: function(data){
                         for (var geneIndex = 0; geneIndex < data['genes'].length; geneIndex++){
-                            if (data['genes'][geneIndex]['strand'] == 1) {
-                                var rectpoints = self.scale((data['genes'][geneIndex]['start'])) + "," + (self.getSequenceModHeight() * i) + " ";
-                                rectpoints += self.scale((data['genes'][geneIndex]['end'])) + "," + (self.getSequenceModHeight() * i) + " ";
-                                rectpoints += self.scale((data['genes'][geneIndex]['end'])) + "," + (self.getSequenceModHeight() * i - GENESIZE / 2 - 1) + " ";
-                                rectpoints += self.scale((data['genes'][geneIndex]['start'])) + "," + (self.getSequenceModHeight() * i - GENESIZE / 2 - 1) + " ";
+                            var gene = data['genes'][geneIndex];
+                            if (gene['strand'] == 1) {
+                                var rectpoints = self.scale((gene['start'])) + "," + (self.getSequenceModHeight() * i) + " ";
+                                rectpoints += self.scale((gene['end'])) + "," + (self.getSequenceModHeight() * i) + " ";
+                                rectpoints += self.scale((gene['end'])) + "," + (self.getSequenceModHeight() * i - GENESIZE / 2 - 1) + " ";
+                                rectpoints += self.scale((gene['start'])) + "," + (self.getSequenceModHeight() * i - GENESIZE / 2 - 1) + " ";
                             }
 
-                            else if (data['genes'][geneIndex]['strand'] == -1) {
-                                var rectpoints = self.scale((data['genes'][geneIndex]['start'])) + "," + (self.getSequenceModHeight() * i + GENESIZE / 2 - 1) + " ";
-                                rectpoints += self.scale((data['genes'][geneIndex]['end'])) + "," + (self.getSequenceModHeight() * i + GENESIZE / 2 - 1) + " ";
-                                rectpoints += self.scale((data['genes'][geneIndex]['end'])) + "," + (self.getSequenceModHeight() * i + GENESIZE - 1) + " ";
-                                rectpoints += self.scale((data['genes'][geneIndex]['start'])) + "," + (self.getSequenceModHeight() * i + GENESIZE - 1) + " ";
+                            else if (gene['strand'] == -1) {
+                                var rectpoints = self.scale((gene['start'])) + "," + (self.getSequenceModHeight() * i + GENESIZE / 2 - 1) + " ";
+                                rectpoints += self.scale((gene['end'])) + "," + (self.getSequenceModHeight() * i + GENESIZE / 2 - 1) + " ";
+                                rectpoints += self.scale((gene['end'])) + "," + (self.getSequenceModHeight() * i + GENESIZE - 1) + " ";
+                                rectpoints += self.scale((gene['start'])) + "," + (self.getSequenceModHeight() * i + GENESIZE - 1) + " ";
                             }
                             else {
-                                console.log("An error occured while trying to draw: " + d.genes[geneIndex['name']]);
+                                console.log("An error occurred while trying to draw: " + d.genes[geneIndex['name']]);
                                 continue;
                             }
 
-                            var genename = data['genes'][geneIndex]['name'];
+                            var geneName = gene['name'];
 
                             geneContainer.append("polygon")
                                 .attr("points", rectpoints)
                                 .attr("stroke-width", 1)
+                                .attr("class", gene['type'])
                                 .append("title")
-                                .text(function (d, i) {
-                                    return genename;
-                                });
+                                    .text(function (d, i) {
+                                        return geneName;
+                                    });
                         }
                     },
                     headers: {
@@ -472,6 +475,8 @@ function MultiVis(targetNode){
                     }
                 })
             });
+        } else {
+            $(".genesLegend-toggle").hide();
         }
 
         //Adds the xAvis TODO Need a different implementation for IslandViewer
diff --git a/web/pages/result.html b/web/pages/result.html
index 985d02db..d4512c77 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -23,63 +23,79 @@ <h1>
 
 <!-- Main content -->
 <section class="content">
-    <div class="row">
-        <div class="col-xs-12">
-            <div class="box">
-                <div class="box-header with-border">
-                    <div class="controlRow">
-                        <div class="controlBody">
-                            <span id="#controlTitle">Controls: </span>
-                            <button id="resetTree">Reset Tree</button>
-                            <button id="resetRange">Reset Zoom</button>
-                            <button id="toggleBranchLength">Toggle Branches</button>
-                            <button id="printerColors">Toggle Grayscale</button>
-                        </div>
-                        <div class="legendBody">
-                            <span id="legendTitle">Legend: </span>
+    <div class="box">
+        <div class="box-header with-border">
+            <div class="controlRow row">
+                <div class="col-xs-6">
+                    <div class="clearfix">
+                        <span id="#controlTitle">Controls: </span>
+                    </div>
+                    <div class="clearfix">
+                        <button id="resetTree" class="btn btn-primary btn-flat">Reset Tree</button>
+                    <button id="resetRange" class="btn btn-primary btn-flat">Reset Zoom</button>
+                    <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
+                    <button id="printerColors" class="btn btn-primary btn-flat">Toggle Grayscale</button>
+                    </div>
+                </div>
+                <div class="col-xs-6">
+                    <div class="clearfix">
+                        <span id="legendTitle">Legend: </span>
+                    </div>
+                    <div class="legendBody clearfix">
+                        <span hidden class="genesLegend-toggle">
                             <svg id="genesLegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
                             <span class="legendText">Genes</span>
-                            <svg id="homologosRegionLegend" height="10" width="10">
+                            <svg id="tRNALegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
-                            <span class="legendText">Homologous Region</span>
-                            <svg id="genomeLegend" height="10" width="10">
+                            <span class="legendText">tRNA</span>
+                            <svg id="rRNALegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
-                            <span class="legendText">Genome</span>
-                            <svg id="amrLegend" height="10" width="10">
-                                <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Genes</title></polygon>
-                            </svg>
-                            <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
+                            <span class="legendText">rRNA</span>
+                        </span>
+                        <svg id="homologosRegionLegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span class="legendText">Homologous Region</span>
+                        <svg id="genomeLegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span class="legendText">Genome</span>
+                        <svg id="amrLegend" height="10" width="10">
+                            <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Genes</title></polygon>
+                        </svg>
+                        <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
 
-                            <span id="sigi-toggle">
+                        <span id="sigiLegend">
+                            <label>
                                 <input type="checkbox" name="sigi" value="sigi" id="sigi-box" checked>Sigi-HMM
-                            </span>
-                            <span id="islandpath-toggle">
+                            </label>
+                        </span>
+                        <span id="islandpathLegend">
+                            <label>
                                 <input type="checkbox" name="islandpath" value="islandpath" id="islandpath-box" checked>IslandPath
-                            </span>
-                        </div>
-                        <!--
-                        --Currently Not used
-                        <div class="controlSide">
-                            <button id="toggleControls">(<<)</button>
-                        </div>
-                        -->
+                            </label>
+                        </span>
                     </div>
                 </div>
-                <!-- /.box-header -->
-                <div class="box-body">
-                    <div id="visualization-body"></div>
+                <!--
+                --Currently Not used
+                <div class="controlSide">
+                    <button id="toggleControls">(<<)</button>
                 </div>
-                <!-- /.box-body -->
+                -->
             </div>
-            <!-- /.box -->
         </div>
-        <!-- /.col -->
+        <!-- /.box-header -->
+        <div class="box-body">
+            <div id="visualization-body"></div>
+        </div>
+        <!-- /.box-body -->
     </div>
-    <!-- /.row -->
+    <!-- /.box -->
 </section>
 <!-- /.content -->
 
@@ -96,11 +112,11 @@ <h1>
                     var warning_message = "<h4>Warning:</h4>";
                     if ($.inArray("sigi",data["failed_components"]) !== -1) {
                         warning_message = warning_message.concat("<p>Sigi-HMM Failed</p>");
-                        $("#sigi-toggle").hide();
+                        $("#sigiLegend").hide();
                     }
                     if ($.inArray("islandpath",data["failed_components"]) !== -1) {
                         warning_message = warning_message.concat("<p>IslandPath Failed</p>");
-                        $("#islandpath-toggle").hide();
+                        $("#islandpathLegend").hide();
                     }
                     $("#warning-box").html(warning_message)
                         .show();
@@ -188,12 +204,16 @@ <h1>
                 if (isPrinterColors) {
                     $("#genomicIslandLegend").attr("class", "#genomicIslandsLegend print");
                     $("#genesLegend").attr("class", "#genesLegend print");
+                    $("#tRNALegend").attr("class", "#tRNALegend print");
+                    $("#rRNALegend").attr("class", "#rRNALegend print");
                     $("#genomeLegend").attr("class", "#genomeLegend print");
                     $("#amrLegend").attr("class", "#amrLegend print");
                 }
                 else {
                     $("#genomicIslandLegend").attr("class", "#genomicIslandsLegend");
                     $("#genesLegend").attr("class", "#genesLegend");
+                    $("#tRNALegend").attr("class", "#tRNALegend");
+                    $("#rRNALegend").attr("class", "#rRNALegend");
                     $("#genomeLegend").attr("class", "#genomeLegend");
                     $("#amrLegend").attr("class", "#amrLegend");
                 }

From 945a0be8724f21b9731dbf86afd34c8b39cbdbc1 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 12 Jul 2017 18:33:37 -0700
Subject: [PATCH 23/76] Add controls for GI colouring method

---
 web/dist/css/linearplot.css     | 14 ++---
 web/dist/js/genomecomparevis.js | 29 ++++++++++
 web/pages/result.html           | 99 +++++++++++++++++++++------------
 3 files changed, 100 insertions(+), 42 deletions(-)

diff --git a/web/dist/css/linearplot.css b/web/dist/css/linearplot.css
index 1385de68..30b2cc5b 100644
--- a/web/dist/css/linearplot.css
+++ b/web/dist/css/linearplot.css
@@ -49,7 +49,7 @@ g.root.node circle{
     stroke: #0d0d0d;
 }
 
-.homologous-region polygon, #homologosRegionLegend{
+.homologous-region polygon, #homologousRegionLegend{
     fill: #d1d1d1;
     stroke: #A4A4A4;
 }
@@ -93,7 +93,7 @@ g.root.node circle{
 }
 
 .amrs polygon, #amrLegend{
-    fill: red;
+    fill: #ff34f3;
 }
 
 #alignmentControls{
@@ -115,11 +115,11 @@ g.root.node circle{
     margin-right: 8px;
 }
 
-.legendBody{
-    padding: 6px;
+.legendBody, .predictorLegend{
+    padding: 3px;
 }
 
-.legendBody label{
+.GIColourBody label{
     font-weight: normal;
 }
 
@@ -127,10 +127,10 @@ g.root.node circle{
     display: table-cell;
 }
 
-.islandpath{
+.islandpath, #islandpathLegend{
     fill: red;
 }
 
-.sigi{
+.sigi, #sigiLegend{
     fill: blue;
 }
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index fd0b075e..e4f0795d 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -518,6 +518,10 @@ function MultiVis(targetNode){
         if (this.isPrinterColors){
             this.setPrinterColors();
         }
+
+        // Colour GIs by specified method
+        this.predictorToggle();
+        this.GIColourToggle();
     };
 
     this.togglePrinterColors = function(){
@@ -532,6 +536,31 @@ function MultiVis(targetNode){
         $(".amrs").attr("class", "amrs print");
     };
 
+    this.predictorToggle = function() {
+        if ($(".GIColourBody :checkbox[name=sigi]").is(":checked")) {
+            $(".sigi").show();
+        } else {
+            $(".sigi").hide();
+        }
+        if ($(".GIColourBody :checkbox[name=islandpath]").is(":checked")) {
+            $(".islandpath").show();
+        } else {
+            $(".islandpath").hide();
+        }
+    };
+
+    this.GIColourToggle = function() {
+        if ($("input[name=GIColour]:checked").val() === "similarity") {
+            $(".GIColourBody :checkbox").attr("disabled", true);
+            $(".predictorLegend").hide();
+            $(".merged").show();
+        } else {
+            $(".GIColourBody :checkbox").removeAttr("disabled");
+            $(".predictorLegend").show();
+            $(".merged").hide();
+        }
+    };
+
     return this;
 }
 
diff --git a/web/pages/result.html b/web/pages/result.html
index d4512c77..dd7fa84d 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -26,22 +26,57 @@ <h1>
     <div class="box">
         <div class="box-header with-border">
             <div class="controlRow row">
-                <div class="col-xs-6">
+                <div class="col-xs-5">
                     <div class="clearfix">
                         <span id="#controlTitle">Controls: </span>
                     </div>
                     <div class="clearfix">
                         <button id="resetTree" class="btn btn-primary btn-flat">Reset Tree</button>
-                    <button id="resetRange" class="btn btn-primary btn-flat">Reset Zoom</button>
-                    <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
-                    <button id="printerColors" class="btn btn-primary btn-flat">Toggle Grayscale</button>
+                        <button id="resetRange" class="btn btn-primary btn-flat">Reset Zoom</button>
+                        <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
+                        <button id="printerColors" class="btn btn-primary btn-flat">Toggle Colour</button>
                     </div>
                 </div>
-                <div class="col-xs-6">
+                <div class="col-xs-2 GIColour">
+                    <div class="clearfix">
+                        <span id="GITitle">Colour GIs by: </span>
+                    </div>
+                    <div class="GIColourBody">
+                        <div class="clearfix">
+                            <label>
+                                <input type="radio" name="GIColour" value="similarity" checked/>Similarity
+                            </label>
+                            <label>
+                                <input type="radio" name="GIColour" value="predictor" />Predictor
+                            </label>
+                        </div>
+                        <div class="clearfix">
+                            <span id="sigiToggle">
+                                <label>
+                                    <input type="checkbox" name="sigi" value="sigi" id="sigi-box" checked disabled>Sigi-HMM
+                                </label>
+                            </span>
+                            <span id="islandpathToggle">
+                                <label>
+                                    <input type="checkbox" name="islandpath" value="islandpath" id="islandpath-box" checked disabled>IslandPath
+                                </label>
+                            </span>
+                        </div>
+                    </div>
+                </div>
+                <div class="col-xs-5">
                     <div class="clearfix">
                         <span id="legendTitle">Legend: </span>
                     </div>
                     <div class="legendBody clearfix">
+                        <svg id="homologousRegionLegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span class="legendText">Homologous Region</span>
+                        <svg id="amrLegend" height="10" width="10">
+                            <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Genes</title></polygon>
+                        </svg>
+                        <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
                         <span hidden class="genesLegend-toggle">
                             <svg id="genesLegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
@@ -56,29 +91,21 @@ <h1>
                             </svg>
                             <span class="legendText">rRNA</span>
                         </span>
-                        <svg id="homologosRegionLegend" height="10" width="10">
-                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
-                        </svg>
-                        <span class="legendText">Homologous Region</span>
-                        <svg id="genomeLegend" height="10" width="10">
-                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
-                        </svg>
-                        <span class="legendText">Genome</span>
-                        <svg id="amrLegend" height="10" width="10">
-                            <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Genes</title></polygon>
-                        </svg>
-                        <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
-
-                        <span id="sigiLegend">
-                            <label>
-                                <input type="checkbox" name="sigi" value="sigi" id="sigi-box" checked>Sigi-HMM
-                            </label>
+                    </div>
+                    <div class="predictorLegend clearfix" hidden>
+                        <span id="sigiLegend" class="legendText">
+                            <svg id="sigiLegend" height="10" width="10">
+                                <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                            </svg>
+                            Sigi-HMM Genomic Island
                         </span>
-                        <span id="islandpathLegend">
-                            <label>
-                                <input type="checkbox" name="islandpath" value="islandpath" id="islandpath-box" checked>IslandPath
-                            </label>
+                        <span id="islandpathLegend" class="legendText">
+                            <svg id="islandpathLegend" height="10" width="10">
+                                <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                            </svg>
+                            IslandPath Genomic Island
                         </span>
+
                     </div>
                 </div>
                 <!--
@@ -112,11 +139,16 @@ <h1>
                     var warning_message = "<h4>Warning:</h4>";
                     if ($.inArray("sigi",data["failed_components"]) !== -1) {
                         warning_message = warning_message.concat("<p>Sigi-HMM Failed</p>");
-                        $("#sigiLegend").hide();
+                        $("#sigiToggle").hide();
+                        $("#sigiLegend").remove();
                     }
                     if ($.inArray("islandpath",data["failed_components"]) !== -1) {
                         warning_message = warning_message.concat("<p>IslandPath Failed</p>");
-                        $("#islandpathLegend").hide();
+                        $("#islandpathToggle").hide();
+                        $("#islandpathLegend").remove();
+                    }
+                    if (data["failed_components"].length === 2) {
+                        $(".GIColour").hide();
                     }
                     $("#warning-box").html(warning_message)
                         .show();
@@ -219,13 +251,10 @@ <h1>
                 }
             });
 
-            $("#sigi-box").change(function() {
-                $(".sigi").toggle();
-            });
-
-            $("#islandpath-box").change(function() {
-                $(".islandpath").toggle();
-            });
+            // Changes to sigi/islandpath checkboxes will hide/show corresponding GIs
+            $(".GIColourBody :checkbox").change(container.predictorToggle);
+            // Changes to GI colour method will hide/show the merged GI set and enable/disable the predictor checkboxes
+            $("input[name=GIColour]").change(container.GIColourToggle);
         });
     }
 </script>
\ No newline at end of file

From 53fe72d1cf962d777b04f8134010273888b3ad8e Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 13 Jul 2017 13:09:13 -0700
Subject: [PATCH 24/76] Loading prompt while genes are asynchronously
 retrieved. Genes appear simultaneously

---
 web/dist/css/linearplot.css     | 12 ++++++++++++
 web/dist/js/genomecomparevis.js | 16 ++++++++++++++--
 web/pages/result.html           |  3 +++
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/web/dist/css/linearplot.css b/web/dist/css/linearplot.css
index 30b2cc5b..bbfc36a7 100644
--- a/web/dist/css/linearplot.css
+++ b/web/dist/css/linearplot.css
@@ -134,3 +134,15 @@ g.root.node circle{
 .sigi, #sigiLegend{
     fill: blue;
 }
+
+.GIColourBody label{
+    margin: 2px;
+}
+
+.loadingGenes{
+    margin: 5px;
+}
+
+.legendText{
+    margin-right: 5px;
+}
\ No newline at end of file
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index e4f0795d..e50f9e3f 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -429,15 +429,19 @@ function MultiVis(targetNode){
         //Add the genes to the plot
         var geneFilterValue = self.getGeneFilterValue();
         if((self.scale.domain()[1] - self.scale.domain()[0]) < geneFilterValue) {
-            $(".genesLegend-toggle").show();
             var geneContainer = sequenceHolder.append("g")
                 .attr("class", "genes")
                 .attr("transform", "translate(0," + (GENESIZE / 4 + GENESIZE/2) + ")");
+            var seqCount = 0;
             seq.each(function(d, i){
                 $.ajax({
                     url: genomesGenesUrl + d.sequenceId + "?start=" + Math.round(self.scale.domain()[0]) + "&end=" + Math.round(self.scale.domain()[1]),
-                    async: false,
+                    beforeSend: function() {
+                        $(".genesLegend-toggle").hide();
+                        $(".loadingGenes").show();
+                    },
                     success: function(data){
+                        seqCount++;
                         for (var geneIndex = 0; geneIndex < data['genes'].length; geneIndex++){
                             var gene = data['genes'][geneIndex];
                             if (gene['strand'] == 1) {
@@ -464,11 +468,19 @@ function MultiVis(targetNode){
                                 .attr("points", rectpoints)
                                 .attr("stroke-width", 1)
                                 .attr("class", gene['type'])
+                                .style("opacity", 0.0)
                                 .append("title")
                                     .text(function (d, i) {
                                         return geneName;
                                     });
                         }
+                        // Wait until all genes have been added so they can be made visible simultaneously
+                        if (seqCount === seqOrder.length) {
+                            $(".loadingGenes").hide();
+                            $(".genesLegend-toggle").show();
+                            geneContainer.selectAll("polygon")
+                                .style("opacity", 1);
+                        }
                     },
                     headers: {
                         "Authorization": 'Token' + ' ' + getAuthenticationCookie()
diff --git a/web/pages/result.html b/web/pages/result.html
index dd7fa84d..632a691c 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -77,6 +77,9 @@ <h1>
                             <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Genes</title></polygon>
                         </svg>
                         <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
+                        <span class="loadingGenes" hidden>
+                            <i class='fa fa-spinner fa-spin '></i> Loading Genes
+                        </span>
                         <span hidden class="genesLegend-toggle">
                             <svg id="genesLegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>

From b8ee241ceafee8567235accab012aa089a9646f8 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 13 Jul 2017 15:14:07 -0700
Subject: [PATCH 25/76] Add Gene model to genomes module. Genes are added to db
 on gbk upload

---
 IslandCompare/genomes/migrations/0002_gene.py | 28 +++++++++++++++++
 IslandCompare/genomes/models.py               |  9 ++++++
 IslandCompare/genomes/serializers.py          | 30 ++++++++++++++++++-
 IslandCompare/genomes/views.py                |  5 ++--
 4 files changed, 68 insertions(+), 4 deletions(-)
 create mode 100644 IslandCompare/genomes/migrations/0002_gene.py

diff --git a/IslandCompare/genomes/migrations/0002_gene.py b/IslandCompare/genomes/migrations/0002_gene.py
new file mode 100644
index 00000000..603e786c
--- /dev/null
+++ b/IslandCompare/genomes/migrations/0002_gene.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-07-13 21:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('genomes', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Gene',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=50)),
+                ('start', models.IntegerField()),
+                ('end', models.IntegerField()),
+                ('strand', models.SmallIntegerField()),
+                ('type', models.CharField(max_length=4)),
+                ('genome', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genomes.Genome')),
+            ],
+        ),
+    ]
diff --git a/IslandCompare/genomes/models.py b/IslandCompare/genomes/models.py
index 92ebf63f..8197684e 100644
--- a/IslandCompare/genomes/models.py
+++ b/IslandCompare/genomes/models.py
@@ -18,3 +18,12 @@ class Meta:
 def delete_gbk(sender, instance, **kwargs):
     if instance.gbk is not None:
         instance.gbk.delete(save=False)
+
+class Gene(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=50)
+    start = models.IntegerField()
+    end = models.IntegerField()
+    strand = models.SmallIntegerField()
+    type = models.CharField(max_length=4)
+    genome = models.ForeignKey(Genome)
\ No newline at end of file
diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index ca5ea78d..5c31f8ce 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -1,4 +1,4 @@
-from genomes.models import Genome
+from genomes.models import Genome, Gene
 from rest_framework import serializers
 from Bio import SeqIO
 from Bio.Seq import UnknownSeq
@@ -44,6 +44,33 @@ def validate_gbk(self, value):
                 raise serializers.ValidationError("Features not included in Genbank File")
         return value
 
+    def create_genes(self, file_path, genome):
+        with open(file_path) as genbank_handle:
+            record = SeqIO.read(genbank_handle, "genbank")
+            for feature in record.features:
+                if feature.type in ["gene", "tRNA", "rRNA"]:
+                    strand = feature.location.strand
+                    start = feature.location.start
+                    end = feature.location.end
+                    gene_type = feature.type
+                    try:
+                        name = feature.qualifiers['gene'][0]
+                    except KeyError:
+                        logging.info("No Name Found For This Gene")
+                        try:
+                            name = feature.qualifiers['locus_tag'][0]
+                        except KeyError:
+                            logging.info("No Locus Found For This Gene")
+                    gene = Gene(
+                        name=name,
+                        start=start,
+                        end=end,
+                        strand=strand,
+                        type=gene_type,
+                        genome=genome
+                    )
+                    gene.save()
+
     def create(self, validated_data):
         genome = Genome(
             name=validated_data['name'],
@@ -51,6 +78,7 @@ def create(self, validated_data):
             owner=self.context['request'].user
         )
         genome.save()
+        self.create_genes(genome.gbk.path, genome)
         return genome
 
     def update(self, instance, validated_data):
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index b597baa7..16e03c6c 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -57,9 +57,8 @@ def get_queryset(self):
 
     def delete(self, request, *args, **kwargs):
         genome = self.get_object()
-        for a in Analysis.objects.filter(genomes__id__contains=genome.id):
-            # print(a.name)
-            a.delete()
+        for analysis in Analysis.objects.filter(genomes__id__contains=genome.id):
+            analysis.delete()
         genome.delete()
         return response.Response(status=204)
 

From 1620a574feddce6c7d7eec10d3dfabddd9ee5673 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 13 Jul 2017 18:30:53 -0700
Subject: [PATCH 26/76] Genes will be retrieved from the db rather than from
 the gbk file

---
 IslandCompare/genomes/serializers.py |  9 ++++++++-
 IslandCompare/genomes/views.py       | 20 ++++++++++++--------
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index 5c31f8ce..c32d4886 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -165,4 +165,11 @@ def to_internal_value(self, data):
         return data
 
     def update(self, instance, validated_data):
-        return validated_data
\ No newline at end of file
+        return validated_data
+
+class GeneSerializer(serializers.Serializer):
+    name = serializers.CharField(max_length=50)
+    start = serializers.IntegerField()
+    end = serializers.IntegerField()
+    strand = serializers.IntegerField()
+    type = serializers.CharField(max_length=4)
\ No newline at end of file
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index 16e03c6c..53f38259 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -1,7 +1,7 @@
 from rest_framework import generics, response
 from rest_framework.permissions import IsAuthenticated
-from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GenomeGenesSerializer
-from genomes.models import Genome
+from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GeneSerializer
+from genomes.models import Genome, Gene
 from analysis.models import Analysis
 from rest_framework.parsers import MultiPartParser, FormParser
 from rest_framework.response import Response
@@ -68,7 +68,7 @@ class GenomeGeneRetrieveView(generics.RetrieveAPIView):
     Retrieve Gene Information for a Group of Genomes
     """
     permission_classes = [IsAuthenticated]
-    serializer_class = GenomeGenesSerializer
+    serializer_class = GeneSerializer
 
     def get_queryset(self):
         return Genome.objects.filter(owner=self.request.user)
@@ -77,10 +77,14 @@ def retrieve(self, request, *args, **kwargs):
         queryset = self.get_queryset()
         url_queries = self.request.query_params
 
-        serializer = GenomeGenesSerializer(queryset.get(id=kwargs['pk']))
+        genome = queryset.get(id=kwargs['pk'])
+        genes = Gene.objects.filter(genome__exact=genome)
 
-        if 'start' in url_queries and 'end' in url_queries:
-            serializer.start_cut_off = int(url_queries['start'])
-            serializer.end_cut_off = int(url_queries['end'])
+        if 'start' in url_queries:
+            genes = genes.filter(start__gte=url_queries['start'])
+        if 'end' in url_queries:
+            genes = genes.filter(end__lte=url_queries['end'])
 
-        return Response(serializer.data)
+        serializer = GeneSerializer(genes, many=True)
+
+        return Response({'genes': serializer.data})

From 8999e1ba84432fd0c81baeb543cc6cbba02bba74 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 14 Jul 2017 14:01:07 -0700
Subject: [PATCH 27/76] Add test case for new GeneSerializer

---
 IslandCompare/genomes/serializers.py |  8 +++--
 IslandCompare/genomes/tests.py       | 49 ++++++++++++++++++++++++++--
 2 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index c32d4886..4db9cd9c 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -44,8 +44,9 @@ def validate_gbk(self, value):
                 raise serializers.ValidationError("Features not included in Genbank File")
         return value
 
-    def create_genes(self, file_path, genome):
-        with open(file_path) as genbank_handle:
+    @staticmethod
+    def create_genes(genome):
+        with open(genome.gbk.path) as genbank_handle:
             record = SeqIO.read(genbank_handle, "genbank")
             for feature in record.features:
                 if feature.type in ["gene", "tRNA", "rRNA"]:
@@ -53,6 +54,7 @@ def create_genes(self, file_path, genome):
                     start = feature.location.start
                     end = feature.location.end
                     gene_type = feature.type
+                    name = []
                     try:
                         name = feature.qualifiers['gene'][0]
                     except KeyError:
@@ -78,7 +80,7 @@ def create(self, validated_data):
             owner=self.context['request'].user
         )
         genome.save()
-        self.create_genes(genome.gbk.path, genome)
+        self.create_genes(genome)
         return genome
 
     def update(self, instance, validated_data):
diff --git a/IslandCompare/genomes/tests.py b/IslandCompare/genomes/tests.py
index 8d6cacf2..8e7a9385 100644
--- a/IslandCompare/genomes/tests.py
+++ b/IslandCompare/genomes/tests.py
@@ -1,6 +1,6 @@
 from django.test import TestCase
 from rest_framework.test import APIRequestFactory, force_authenticate
-from genomes.models import Genome
+from genomes.models import Genome, Gene
 from genomes.views import GenomeListView, GenomeUploadView
 from django.contrib.auth.models import User
 from django.core.files.uploadedfile import SimpleUploadedFile
@@ -8,7 +8,7 @@
 from rest_framework.reverse import reverse
 from rest_framework.test import APIClient
 from django.core.files import File
-from genomes.serializers import GenomeGenesSerializer
+from genomes.serializers import GenomeSerializer, GenomeGenesSerializer, GeneSerializer
 
 # Create your tests here.
 
@@ -367,8 +367,51 @@ def test_gene_filter_serializer(self):
         serializer.start_cut_off = 4598500
         serializer.end_cut_off = 4744561
         serializer.is_valid()
-        self.assertEqual(2, len(serializer.data['genes']))
+        self.assertEqual(1, len(serializer.data['genes']))
 
     def tearDown(self):
         for genome in Genome.objects.all():
             genome.delete()
+
+
+class GeneSerializerTestCase(TestCase):
+    test_username = "username"
+    test_user = None
+
+    test_name = "test_genome"
+    test_gbk_path = '../TestFiles/AE009952.gbk'
+    test_genome = None
+
+    test_genes = None
+
+    def setUp(self):
+        self.test_user = User(username=self.test_username)
+        self.test_user.save()
+
+        test_gbk = File(open(self.test_gbk_path))
+
+        self.test_genome = Genome(name=self.test_name,
+                                  owner=self.test_user,
+                                  gbk=test_gbk)
+
+        self.test_genome.save()
+        test_gbk.close()
+        GenomeSerializer.create_genes(self.test_genome)
+        self.test_genes = Gene.objects.filter(genome__exact=self.test_genome)
+
+    def test_gene_serializer(self):
+        serializer = GeneSerializer(data=self.test_genes, many=True)
+        serializer.is_valid()
+        self.assertEqual(4241,  len(serializer.data))
+
+    def test_gene_filter_serializer(self):
+        filtered_genes = self.test_genes.filter(start__gte=4598500).filter(end__lte=4744561)
+        serializer = GeneSerializer(data=filtered_genes, many=True)
+        serializer.is_valid()
+        self.assertEqual(1, len(serializer.data))
+
+    def tearDown(self):
+        for genome in Genome.objects.all():
+            genome.delete()
+        for gene in Gene.objects.all():
+            gene.delete()
\ No newline at end of file

From db1160c23ce818a09740df730032566bd3873409 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 14 Jul 2017 17:24:01 -0700
Subject: [PATCH 28/76] Moved create_genes function to
 SetupGbkPipelineComponent

---
 IslandCompare/analysis/components.py       | 38 +++++++++++++++++++++-
 IslandCompare/analysis/tests_components.py | 11 +++----
 IslandCompare/genomes/serializers.py       | 37 +++------------------
 IslandCompare/genomes/tests.py             | 23 +++++++++----
 IslandCompare/genomes/views.py             |  7 ++--
 5 files changed, 65 insertions(+), 51 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 00110fde..6fabbe44 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -1,5 +1,6 @@
 from analysis.pipeline import PipelineComponent
 from analysis.models import Analysis, Genome
+from genomes.models import Gene
 from tempfile import mkdtemp, NamedTemporaryFile
 from shutil import rmtree
 from Bio import SeqIO, Phylo
@@ -41,11 +42,38 @@ def analysis(self, report):
 
 class SetupGbkPipelineComponent(PipelineComponent):
     """
-    Pipeline component that adds the gbk paths to the report.
+    Pipeline component that adds the gbk paths to the report and genes to the database.
     """
     name = "setup_gbk"
     result_types = ["gbk_paths"]
 
+    @staticmethod
+    def create_genes(genome):
+        with open(genome.gbk.path) as genbank_handle:
+            record = SeqIO.read(genbank_handle, "genbank")
+            for feature in record.features:
+                if feature.type in ["gene", "tRNA", "rRNA"]:
+                    strand = feature.location.strand
+                    start = feature.location.start
+                    end = feature.location.end
+                    gene_type = feature.type
+                    try:
+                        name = feature.qualifiers['gene'][0]
+                    except KeyError:
+                        try:
+                            name = feature.qualifiers['locus_tag'][0]
+                        except KeyError:
+                            name = ""
+                    gene = Gene(
+                        name=name,
+                        start=start,
+                        end=end,
+                        strand=strand,
+                        type=gene_type,
+                        genome=genome
+                    )
+                    gene.save()
+
     def analysis(self, report):
         analysis_entry = Analysis.objects.get(id=report['analysis'])
         genomes = analysis_entry.genomes
@@ -53,6 +81,11 @@ def analysis(self, report):
         report['gbk_paths'] = dict()
         for genome in genomes.all():
             report['gbk_paths'][str(genome.id)] = genome.gbk.path
+            if not genome.gene_set.exists():
+                self.logger.info("Creating gene set for genome {}".format(genome.name))
+                self.create_genes(genome)
+            else:
+                self.logger.info("{} gene set found".format(genome.name))
 
 
 class GbkMetadataComponent(PipelineComponent):
@@ -76,6 +109,9 @@ def analysis(self, report):
         report["gbk_metadata"] = output
 
 class RGIPipelineComponent(PipelineComponent):
+    """
+    Pipeline component that runs and adds RGI data to the report
+    """
     name = "rgi"
     dependencies = ["gbk_paths"]
     result_types = ["amr_genes"]
diff --git a/IslandCompare/analysis/tests_components.py b/IslandCompare/analysis/tests_components.py
index e1bff010..c441e19a 100644
--- a/IslandCompare/analysis/tests_components.py
+++ b/IslandCompare/analysis/tests_components.py
@@ -1,5 +1,4 @@
 from django.test import TestCase, mock
-from django.core.files.uploadedfile import SimpleUploadedFile
 from django.contrib.auth.models import User
 from genomes.models import Genome
 from analysis.components import SetupGbkPipelineComponent, ParsnpPipelineComponent, MauvePipelineComponent, \
@@ -16,15 +15,11 @@ class GbkComponentTestCase(TestCase):
 
     test_genome_1 = None
     test_genome_1_name = "genome_1"
-    test_genome_1_gbk_name = "test.gbk"
-    test_genome_1_gbk_contents = bytes("test", 'utf-8')
-    test_genome_1_gbk = SimpleUploadedFile(test_genome_1_gbk_name, test_genome_1_gbk_contents)
+    test_genome_1_gbk = File(open("../TestFiles/AE009952.gbk"))
 
     test_genome_2 = None
     test_genome_2_name = "genome_2"
-    test_genome_2_gbk_name = "test_2.gbk"
-    test_genome_2_gbk_contents = bytes("test2", 'utf-8')
-    test_genome_2_gbk = SimpleUploadedFile(test_genome_1_gbk_name, test_genome_1_gbk_contents)
+    test_genome_2_gbk = File(open("../TestFiles/BX936398.gbk"))
 
     serialized_pipeline = None
     component = SetupGbkPipelineComponent()
@@ -56,6 +51,8 @@ def test_run_gbk_component(self):
         self.assertEqual(self.test_genome_2.gbk.path, result['gbk_paths'][str(self.test_genome_2.id)])
         self.assertTrue('gbk_paths' in result['available_dependencies'])
         self.assertTrue('setup_gbk' in result['pipeline_components'])
+        for genome in Genome.objects.filter(owner=self.test_user):
+            self.assertTrue(genome.gene_set.exists())
 
     def tearDown(self):
         for genome in Genome.objects.all():
diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index 4db9cd9c..1fbc837d 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -31,7 +31,7 @@ def validate_gbk(self, value):
         """
         # 'value' is of type <class 'django.core.files.uploadedfile.TemporaryUploadedFile'>
         # SeqIO.parse cannot read this as its contents are of type bytes rather than str
-        # 'gbk' is created as a decded 'value', to be passed to SeqIO.parse
+        # 'gbk' is created as a decoded 'value', to be passed to SeqIO.parse
         with StringIO(value.read().decode("utf-8")) as gbk:
             value.seek(0)
             gbk_records = list(SeqIO.parse(gbk, 'genbank'))
@@ -44,35 +44,6 @@ def validate_gbk(self, value):
                 raise serializers.ValidationError("Features not included in Genbank File")
         return value
 
-    @staticmethod
-    def create_genes(genome):
-        with open(genome.gbk.path) as genbank_handle:
-            record = SeqIO.read(genbank_handle, "genbank")
-            for feature in record.features:
-                if feature.type in ["gene", "tRNA", "rRNA"]:
-                    strand = feature.location.strand
-                    start = feature.location.start
-                    end = feature.location.end
-                    gene_type = feature.type
-                    name = []
-                    try:
-                        name = feature.qualifiers['gene'][0]
-                    except KeyError:
-                        logging.info("No Name Found For This Gene")
-                        try:
-                            name = feature.qualifiers['locus_tag'][0]
-                        except KeyError:
-                            logging.info("No Locus Found For This Gene")
-                    gene = Gene(
-                        name=name,
-                        start=start,
-                        end=end,
-                        strand=strand,
-                        type=gene_type,
-                        genome=genome
-                    )
-                    gene.save()
-
     def create(self, validated_data):
         genome = Genome(
             name=validated_data['name'],
@@ -80,7 +51,6 @@ def create(self, validated_data):
             owner=self.context['request'].user
         )
         genome.save()
-        self.create_genes(genome)
         return genome
 
     def update(self, instance, validated_data):
@@ -111,7 +81,7 @@ def update(self, instance, validated_data):
 
 class GenomeGenesSerializer(serializers.Serializer):
     """
-    Serializer for returning genes for a genome
+    Serializer for returning genes for a genome by parsing the genbank file
     """
     logger = logging.getLogger(__name__)
     start_cut_off = None
@@ -170,6 +140,9 @@ def update(self, instance, validated_data):
         return validated_data
 
 class GeneSerializer(serializers.Serializer):
+    """
+    Serializer for Gene objects
+    """
     name = serializers.CharField(max_length=50)
     start = serializers.IntegerField()
     end = serializers.IntegerField()
diff --git a/IslandCompare/genomes/tests.py b/IslandCompare/genomes/tests.py
index 8e7a9385..36a57cef 100644
--- a/IslandCompare/genomes/tests.py
+++ b/IslandCompare/genomes/tests.py
@@ -396,22 +396,31 @@ def setUp(self):
 
         self.test_genome.save()
         test_gbk.close()
-        GenomeSerializer.create_genes(self.test_genome)
-        self.test_genes = Gene.objects.filter(genome__exact=self.test_genome)
+        Gene(name="test_gene_1",
+             start=0,
+             end=1000,
+             strand=1,
+             type="gene",
+             genome=self.test_genome).save()
+        Gene(name="test_gene_2",
+             start=5000,
+             end=10000,
+             strand=-1,
+             type="tRNA",
+             genome=self.test_genome).save()
+        self.test_genes = self.test_genome.gene_set.all()
 
     def test_gene_serializer(self):
         serializer = GeneSerializer(data=self.test_genes, many=True)
         serializer.is_valid()
-        self.assertEqual(4241,  len(serializer.data))
+        self.assertEqual(2,  len(serializer.data))
 
     def test_gene_filter_serializer(self):
-        filtered_genes = self.test_genes.filter(start__gte=4598500).filter(end__lte=4744561)
+        filtered_genes = self.test_genes.filter(start__gte=2500).filter(end__lte=1000000)
         serializer = GeneSerializer(data=filtered_genes, many=True)
         serializer.is_valid()
         self.assertEqual(1, len(serializer.data))
 
     def tearDown(self):
         for genome in Genome.objects.all():
-            genome.delete()
-        for gene in Gene.objects.all():
-            gene.delete()
\ No newline at end of file
+            genome.delete()
\ No newline at end of file
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index 53f38259..be6e44fa 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -1,7 +1,7 @@
 from rest_framework import generics, response
 from rest_framework.permissions import IsAuthenticated
 from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GeneSerializer
-from genomes.models import Genome, Gene
+from genomes.models import Genome
 from analysis.models import Analysis
 from rest_framework.parsers import MultiPartParser, FormParser
 from rest_framework.response import Response
@@ -65,7 +65,7 @@ def delete(self, request, *args, **kwargs):
 
 class GenomeGeneRetrieveView(generics.RetrieveAPIView):
     """
-    Retrieve Gene Information for a Group of Genomes
+    Retrieve Gene Information for a Genome
     """
     permission_classes = [IsAuthenticated]
     serializer_class = GeneSerializer
@@ -77,8 +77,7 @@ def retrieve(self, request, *args, **kwargs):
         queryset = self.get_queryset()
         url_queries = self.request.query_params
 
-        genome = queryset.get(id=kwargs['pk'])
-        genes = Gene.objects.filter(genome__exact=genome)
+        genes = queryset.get(id=kwargs['pk']).gene_set.all()
 
         if 'start' in url_queries:
             genes = genes.filter(start__gte=url_queries['start'])

From 19b3d29340d4b7ab8a2462f73d682743c755be3e Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 17 Jul 2017 11:43:21 -0700
Subject: [PATCH 29/76] Fixed alignment of genome name to sequence, reduced AMR
 gene height

---
 web/dist/js/genomecomparevis.js | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index e50f9e3f..f1326a8d 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -263,7 +263,7 @@ function MultiVis(targetNode){
         var sequenceHolder = visContainer.append("svg")
             .attr("width",this.visualizationWidth())
             .append("g")
-            .attr("transform","translate("+ 0 +","+GISIZE+")");
+            .attr("transform","translate("+ 0 +","+(GISIZE / 2)+")");
 
         //Draw Homologous Region Lines
         var lines = [];
@@ -381,9 +381,9 @@ function MultiVis(targetNode){
                         // min ensures the angles on the trapezoids are at most 45 degrees
                         polypoints += startPosition + Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
                         polypoints += endPosition - Math.min(+!plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i) + " ";
-                        polypoints += endPosition - Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE) + " ";
-                        polypoints += startPosition + Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE);
-                    // Else AMRs will be rectangles of reduced height to improve appearance from a wider view
+                        polypoints += endPosition - Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
+                        polypoints += startPosition + Math.min(+plus_strand * width / 3, GISIZE) + "," + (self.getSequenceModHeight() * i - GISIZE / 2);
+                    // Else AMRs will be rectangles to improve appearance from a wider view
                     } else {
                         polypoints += startPosition + "," + (self.getSequenceModHeight() * i) + " ";
                         polypoints += startPosition + 1 + "," + (self.getSequenceModHeight() * i) + " ";
@@ -393,12 +393,9 @@ function MultiVis(targetNode){
                     return polypoints;
                 })
                 .attr("transform", function(d) {
-                    var plus_strand = (d.strand === "+");
-                    // The rectangles are smaller so they require a smaller adjustment
-                    var adjust = GISIZE * 2 - (+!details * GISIZE / 2);
                     // Positive strand AMRs are in place, negative strand AMRs need to be moved to the other side
-                    if (!plus_strand) {
-                        return "translate(0," + adjust + ")";
+                    if (d.strand === "-") {
+                        return "translate(0," + (3 * GISIZE / 2) + ")";
                     }
                 });
         });

From 1cdc31d8a596812f20ce1158a168b588df40c0e8 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 17 Jul 2017 16:59:26 -0700
Subject: [PATCH 30/76] Each genbank file will be parsed only once in
 MashMclClusterPipelineComponent to reduce time

---
 IslandCompare/analysis/components.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index e2fe6b16..184fff69 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -636,13 +636,11 @@ def parse_mash_output(self, outputFile):
                                    'matchingHashes': row[4]})
         return outputList
 
-    def getSubsequence(self, genbankFile, startPosition, endPosition, islandNumber, description=None):
-        record_dict = SeqIO.index(genbankFile, "genbank")
-        sequenceName = list(record_dict.keys())[0]
+    def get_subsequence(self, record, startPosition, endPosition, islandNumber, description=None):
         if description is not None:
-            return SeqRecord(record_dict[sequenceName].seq[int(startPosition):int(endPosition)], id=sequenceName + "-" + str(islandNumber), description=description)
+            return SeqRecord(record.seq[int(startPosition):int(endPosition)], id=record.id + "-" + str(islandNumber), description=description)
         else:
-            return SeqRecord(record_dict[sequenceName].seq[int(startPosition):int(endPosition)], id=sequenceName + "-" + str(islandNumber))
+            return SeqRecord(record.seq[int(startPosition):int(endPosition)], id=record.id + "-" + str(islandNumber))
 
     def writeFastaFile(self, outputFileName, seqRecordList):
         with open(outputFileName, 'w') as outputFileHandle:
@@ -660,13 +658,14 @@ def create_gi_fasta_files(self, report):
         island_path_list = []
 
         for genome_id in genome_list.keys():
+            record = SeqIO.read(report['gbk_paths'][genome_id], "genbank")
             genome_fna_path = self.fna_dir_path + "/" + str(genome_id)
             os.mkdir(genome_fna_path)
 
             gi_counter = 0
             for gi in report['merge_gis'][genome_id]:
                 self.logger.info("Adding GI: " + str(genome_id) + "-" + str(gi_counter))
-                entrySequence = self.getSubsequence(report['gbk_paths'][genome_id], gi[0], gi[1], gi_counter)
+                entrySequence = self.get_subsequence(record, gi[0], gi[1], gi_counter)
                 self.writeFastaFile(self.fna_dir_path + "/" + str(genome_id) + "/" + str(gi_counter), entrySequence)
                 island_path_list.append(self.fna_dir_path + "/" + str(genome_id) + "/" + str(gi_counter))
                 gi_counter += 1

From 82f60ce8b064bc771b41d83eeefa4a42a03dcd22 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 18 Jul 2017 18:19:54 -0700
Subject: [PATCH 31/76] Fixed #113 - User uploaded gis and newick issue

---
 IslandCompare/analysis/serializers.py | 10 +++++++---
 IslandCompare/analysis/tasks.py       | 19 ++++++++++++-------
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 30cfe5d9..8cf64459 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -128,7 +128,7 @@ def to_representation(self, instance):
         output = dict()
 
         output["genomes"] = dict()
-        gi_types = {"sigi_gis", "islandpath_gis", "merge_gis", "user_gis"}.intersection(instance)
+        gi_types = {"sigi_gis", "islandpath_gis", "merge_gis"}.intersection(instance)
         if "cluster_gis" in instance:
             number_clusters = instance["cluster_gis"]["numberClusters"]
             color_index = self.get_spaced_colors(number_clusters)
@@ -138,8 +138,12 @@ def to_representation(self, instance):
             output["genomes"][genome.id]["length"] = instance["gbk_metadata"][str(genome.id)]['size']
             output["genomes"][genome.id]["amr_genes"] = [{'start': amr['orf_start'], 'end': amr['orf_end'], 'strand': amr['orf_strand']} for amr in instance["amr_genes"][str(genome.id)]]
             output["genomes"][genome.id]["genomic_islands"] = dict()
-            for gi_type in gi_types:
-                output["genomes"][genome.id]["genomic_islands"][gi_type[:-4].replace("merge", "merged")] = [{'start': island[0], 'end': island[1]} for island in instance[gi_type][str(genome.id)]]
+            if "user_gis" in instance:
+                output["genomes"][genome.id]["genomic_islands"]["user"] = instance["user_gis"][str(genome.id)]
+                print(output["genomes"][genome.id]["genomic_islands"]["user"])
+            else:
+                for gi_type in gi_types:
+                    output["genomes"][genome.id]["genomic_islands"][gi_type[:-4].replace("merge", "merged")] = [{'start': island[0], 'end': island[1]} for island in instance[gi_type][str(genome.id)]]
             if "cluster_gis" in instance:
                 for gi_index in range(len(instance["merge_gis"][str(genome.id)])):
                     clusters = instance['cluster_gis'][str(genome.id)]
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index 186ba781..0205f6a8 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -75,17 +75,22 @@ def run_pipeline(self, serialized_pipeline):
     pipeline.analysis.celery_task_id = self.request.id
     pipeline.analysis.save()
 
-    component_list = serialized_pipeline['pipeline_components']
     tasks = [run_pipeline_component.s(serialized_pipeline, pipeline.analysis.id, "start_pipeline")]
-    for comp in component_list[1:]:
-        if comp == "sigi":
-            tasks.append(run_pipeline_component.s(pipeline.analysis.id, comp)
+    for component in pipeline.pipeline_components[1:]:
+        if component.name == "sigi":
+            tasks.append(run_pipeline_component.s(pipeline.analysis.id,
+                                                  component.name,
+                                                  component.param)
                          .on_error(sigi_error_handler.s(pipeline.analysis.id)))
-        elif comp == "islandpath":
-            tasks.append(run_pipeline_component.s(pipeline.analysis.id, comp)
+        elif component.name == "islandpath":
+            tasks.append(run_pipeline_component.s(pipeline.analysis.id,
+                                                  component.name,
+                                                  component.param)
                          .on_error(ipath_error_handler.s(pipeline.analysis.id)))
         else:
-            tasks.append(run_pipeline_component.s(pipeline.analysis.id, comp))
+            tasks.append(run_pipeline_component.s(pipeline.analysis.id,
+                                                  component.name,
+                                                  component.param))
     task_chain = chain(*tasks)
 
     logger.info("End scheduling of pipeline components with celery task id: {}".format(self.request.id))

From 6b3b6c05a8305fe875b475549c29ec62e819d129 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 19 Jul 2017 17:14:08 -0700
Subject: [PATCH 32/76] Button to download csv of genes within GIs added to
 history page

---
 IslandCompare/analysis/serializers.py  | 32 ++++++++++++++++++++++
 IslandCompare/analysis/urls.py         |  1 +
 IslandCompare/analysis/views.py        | 28 ++++++++++++++++++-
 web/dist/js/islandcompare-rest-urls.js |  1 +
 web/pages/history.html                 | 38 ++++++++++++++++++++++++--
 5 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 8cf64459..2936f8fd 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -214,3 +214,35 @@ def create(self, validated_data):
 
     def update(self, instance, validated_data):
         return
+
+class ReportGeneCsvSerializer(serializers.BaseSerializer):
+    """
+    Serializer to return a csv of genes contained within GIs
+    """
+    def to_representation(self, instance):
+        output = StringIO()
+        fieldnames = ['genome', 'name', 'start', 'end', 'strand', 'gi_start', 'gi_end', 'gi_method']
+        writer = csv.DictWriter(output, fieldnames=fieldnames)
+        writer.writeheader()
+
+        method_keys = ["merge_gis", "islandpath_gis", "sigi_gis"]
+        methods = [method for method in method_keys if method in instance]
+
+        for key in instance["gbk_paths"]:
+            genome = Genome.objects.get(id=key)
+            for method in methods:
+                for island in instance[method][key]:
+                    for gi_gene in genome.gene_set.filter(start__gte=island[0]).filter(end__lte=island[1]):
+                        writer.writerow({'genome': genome.name,
+                                         'name': gi_gene.name,
+                                         'start': gi_gene.start,
+                                         'end': gi_gene.end,
+                                         'strand': gi_gene.strand,
+                                         'gi_start': island[0],
+                                         'gi_end': island[1],
+                                         'gi_method': method})
+
+        contents = output.getvalue()
+        output.close()
+
+        return contents
\ No newline at end of file
diff --git a/IslandCompare/analysis/urls.py b/IslandCompare/analysis/urls.py
index 628756bc..10213fc8 100644
--- a/IslandCompare/analysis/urls.py
+++ b/IslandCompare/analysis/urls.py
@@ -5,6 +5,7 @@
     url(r'^$', views.AnalysisListView.as_view(), name="analysis"),
     url(r'^details/(?P<pk>[0-9]+)$', views.AnalysisRetrieveUpdateView.as_view(), name="analysis_details"),
     url(r'^export/(?P<pk>[0-9]+)$', views.ExportAnalysisResultView.as_view(), name="analysis_export"),
+    url(r'^export-genes/(?P<pk>[0-9]+)$', views.ExportAnalysisGenesView.as_view(), name="analysis_genes_export"),
     url(r'^results/(?P<pk>[0-9]+)$', views.AnalysisResultsView.as_view(), name="analysis_results"),
     url(r'^run/', views.AnalysisRunView.as_view(), name="analysis_run"),
     url(r'^delete/(?P<pk>[0-9]+)$', views.AnalysisDestroyView.as_view(), name="analysis_destroy"),
diff --git a/IslandCompare/analysis/views.py b/IslandCompare/analysis/views.py
index 2f843347..64764736 100644
--- a/IslandCompare/analysis/views.py
+++ b/IslandCompare/analysis/views.py
@@ -2,7 +2,7 @@
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.views import APIView
 from analysis.serializers import AnalysisSerializer, RunAnalysisSerializer, ReportCsvSerializer, \
-    ReportVisualizationOverviewSerializer
+    ReportVisualizationOverviewSerializer, ReportGeneCsvSerializer
 from analysis.models import Analysis, AnalysisComponent
 from rest_framework.response import Response
 from analysis.pipeline import Pipeline
@@ -165,3 +165,29 @@ def retrieve(self, request, *args, **kwargs):
             response.status_code = 202
             response.data = {'content': "Job has not completed"}
             return response
+
+class ExportAnalysisGenesView(generics.RetrieveAPIView):
+    """
+    Retrieve a CSV of the Analysis GI genes
+    """
+    permission_classes = [IsAuthenticated]
+
+    def retrieve(self, request, *args, **kwargs):
+        analysis = Analysis.objects.get(id=kwargs['pk'])
+        task = AsyncResult(analysis.celery_task_id)
+
+        if task.status == 'SUCCESS':
+            end_pipeline = AnalysisComponent.objects.get(analysis=analysis,
+                                                         type__name="end_pipeline")
+            result = AsyncResult(end_pipeline.celery_task_id).get()
+            return Response(ReportGeneCsvSerializer(result).data)
+        elif task.status == 'FAILURE':
+            response = Response()
+            response.status_code = 500
+            response.data = {'content': "Job has errored"}
+            return response
+        else:
+            response = Response()
+            response.status_code = 202
+            response.data = {'content': "Job has not completed"}
+            return response
\ No newline at end of file
diff --git a/web/dist/js/islandcompare-rest-urls.js b/web/dist/js/islandcompare-rest-urls.js
index baa37b36..4a410f30 100644
--- a/web/dist/js/islandcompare-rest-urls.js
+++ b/web/dist/js/islandcompare-rest-urls.js
@@ -10,5 +10,6 @@ var analysisUrl = rootUrl + "analysis/";
 var analysisRunUrl = rootUrl + "analysis/run/";
 var analysisResultsUrl = rootUrl + "analysis/results/";
 var analysisExportUrl = rootUrl + "analysis/export/";
+var analysisGenesExportUrl = rootUrl + "analysis/export-genes/";
 var analysisDeleteUrl = rootUrl + "analysis/delete/";
 var analysisUpdateUrl = rootUrl + "analysis/details/";
\ No newline at end of file
diff --git a/web/pages/history.html b/web/pages/history.html
index 2c8d03d2..85a9c0d6 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -24,7 +24,7 @@ <h3 class="box-title">Job History</h3>
                     <table id="job-history" class="table table-bordered table-hover table-striped" width="100%">
                         <thead>
                         <tr>
-                            <th>Analysis ID</th>
+                            <th>ID</th>
                             <th>Analysis Name</th>
                             <th>RGI Status</th>
                             <th>Parsnp Status</th>
@@ -67,7 +67,7 @@ <h3 class="box-title">Job History</h3>
         );
     }
 
-    function export_job(jobid) {
+    function export_gis(jobid) {
         $.ajax({
             url: analysisExportUrl + jobid,
             async: false,
@@ -90,6 +90,29 @@ <h3 class="box-title">Job History</h3>
         });
     }
 
+    function export_genes(jobid) {
+        $.ajax({
+            url: analysisGenesExportUrl + jobid,
+            async: false,
+            beforeSend: function(request) {
+                request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
+            },
+            success: function(data) {
+                // Construct the <a> element
+                var link = document.createElement("a");
+                link.download = "IslandCompareGenes-" + jobid + ".csv";
+                // Construct the uri
+                var uri = 'data:text/csv;charset=utf-8;base64,' + btoa(data);
+                link.href = uri;
+                document.body.appendChild(link);
+                link.click();
+                // Cleanup the DOM
+                document.body.removeChild(link);
+                delete link;
+            }
+        });
+    }
+
     function rename_job(jobid) {
         //var new_name = prompt("Please enter new name");
         BootstrapDialog.confirm({
@@ -291,7 +314,16 @@ <h3 class="box-title">Job History</h3>
                             return '<button class="btn btn-block btn-flat disabled recent-job-item"><i class="fa fa-download" aria-hidden="true"></button>';
                         }
                         else if (end_status === "SUCCESS") {
-                            return '<a href="#" onclick="javascript:export_job(' + analysis_id + ');"><button class="btn btn-block btn-flat btn-success recent-job-item"><i class="fa fa-download" aria-hidden="true"></button></a>';
+                            return '<a href="#" onclick="javascript:export_gis(' + analysis_id + ');">' +
+                                       '<button class="btn btn-block btn-flat btn-success btn-xs recent-job-item">' +
+                                           'GIs' +
+                                       '</button>' +
+                                   '</a>' +
+                                   '<a href="#" onclick="javascript:export_genes(' + analysis_id + ');">' +
+                                       '<button class="btn btn-block btn-flat btn-success btn-xs recent-job-item">' +
+                                           'GI Genes' +
+                                       '</button>' +
+                                   '</a>';
                         }
                         else {
                             return '<button class="btn btn-block btn-flat disabled recent-job-item"><i class="fa fa-download" aria-hidden="true"></button>';

From 7c30209336fe1cbcbf7abd9b537b638196a61a6c Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 20 Jul 2017 12:27:10 -0700
Subject: [PATCH 33/76] Fixed failing tests

---
 IslandCompare/analysis/tests.py          | 1 +
 IslandCompare/analysis/tests_pipeline.py | 7 ++++++-
 IslandCompare/genomes/tests.py           | 8 ++++----
 IslandCompare/genomes/views.py           | 3 +--
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/IslandCompare/analysis/tests.py b/IslandCompare/analysis/tests.py
index 678fc503..3ff021b5 100644
--- a/IslandCompare/analysis/tests.py
+++ b/IslandCompare/analysis/tests.py
@@ -432,6 +432,7 @@ class RunAnalysisTestCase(APITestCase):
     expected_components = ["start_pipeline",
                            "setup_gbk",
                            "gbk_metadata",
+                           "rgi",
                            "parsnp",
                            "mauve",
                            "sigi",
diff --git a/IslandCompare/analysis/tests_pipeline.py b/IslandCompare/analysis/tests_pipeline.py
index cb05fa50..78000d99 100644
--- a/IslandCompare/analysis/tests_pipeline.py
+++ b/IslandCompare/analysis/tests_pipeline.py
@@ -161,6 +161,7 @@ class PipelineSerializerTestCase(TestCase):
     test_component = PipelineComponentStub()
     test_component.name = "component_1"
     test_pipeline_components = [test_component]
+    test_failed_components = ['failure_1']
 
     test_username = "test_username"
     test_user = None
@@ -179,6 +180,7 @@ def setUp(self):
         self.pipeline.available_dependencies = self.test_available_dependencies
         self.pipeline.pipeline_components = self.test_pipeline_components
         self.pipeline.analysis = self.test_analysis
+        self.pipeline.failed_components = self.test_failed_components
 
     def test_serialize_pipeline(self):
         json_dict = PipelineSerializer.serialize(self.pipeline)
@@ -186,13 +188,15 @@ def test_serialize_pipeline(self):
         self.assertEqual(self.test_available_dependencies, json_dict['available_dependencies'])
         self.assertTrue(self.test_component.name in json_dict['pipeline_components'])
         self.assertEqual(self.test_analysis.id, json_dict['analysis'])
+        self.assertEqual(self.test_failed_components, json_dict['failed_components'])
 
     def test_deserialize_pipeline(self):
         json_dict = {
             'available_dependencies': self.test_available_dependencies,
             'pipeline_components': [self.test_component.name],
             'analysis': self.test_analysis.id,
-            'component_param': {self.test_component.name: {}}
+            'component_param': {self.test_component.name: {}},
+            'failed_components': self.test_failed_components
         }
 
         mock_factory = mock.MagicMock()
@@ -204,3 +208,4 @@ def test_deserialize_pipeline(self):
         self.assertEqual(self.test_available_dependencies, deserialized_pipeline.available_dependencies)
         self.assertEqual(self.test_pipeline_components, deserialized_pipeline.pipeline_components)
         self.assertEqual(self.test_analysis, deserialized_pipeline.analysis)
+        self.assertEqual(self.test_failed_components, deserialized_pipeline.failed_components)
diff --git a/IslandCompare/genomes/tests.py b/IslandCompare/genomes/tests.py
index 36a57cef..54f21076 100644
--- a/IslandCompare/genomes/tests.py
+++ b/IslandCompare/genomes/tests.py
@@ -182,6 +182,7 @@ class UpdateGenomeTestCase(TestCase):
     test_name = "test_genome"
     test_gbk = None
     test_genome = None
+    test_gbk_path = '../TestFiles/AE009952.gbk'
 
     def setUp(self):
         self.factory = APIRequestFactory()
@@ -198,9 +199,7 @@ def test_authenticated_update_genome(self):
         url = reverse('genome_details', kwargs={'pk': self.test_genome.id})
 
         updated_name = "updated_genome"
-        updated_gbk_name = "test.gbk"
-        updated_gbk_content = bytes("test", 'utf-8')
-        updated_gbk = SimpleUploadedFile(updated_gbk_name, updated_gbk_content)
+        updated_gbk = File(open(self.test_gbk_path))
 
         client = APIClient()
         client.force_authenticate(user=self.test_user)
@@ -212,7 +211,8 @@ def test_authenticated_update_genome(self):
 
         self.assertEqual(200, response.status_code)
         self.assertEqual(updated_name, genome.name)
-        self.assertEqual(updated_gbk_content, genome.gbk.read())
+        updated_gbk.seek(0)
+        self.assertEqual(updated_gbk.read().encode('utf-8'), genome.gbk.read())
 
     def test_unauthenticated_update_genome(self):
         url = reverse('genome_details', kwargs={'pk': self.test_genome.id})
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index be6e44fa..f1ea302c 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -2,7 +2,6 @@
 from rest_framework.permissions import IsAuthenticated
 from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GeneSerializer
 from genomes.models import Genome
-from analysis.models import Analysis
 from rest_framework.parsers import MultiPartParser, FormParser
 from rest_framework.response import Response
 
@@ -57,7 +56,7 @@ def get_queryset(self):
 
     def delete(self, request, *args, **kwargs):
         genome = self.get_object()
-        for analysis in Analysis.objects.filter(genomes__id__contains=genome.id):
+        for analysis in genome.analysis_set.all():
             analysis.delete()
         genome.delete()
         return response.Response(status=204)

From d2c4a82e709c2c84f61c388a2eececb9c7997129 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 20 Jul 2017 16:59:22 -0700
Subject: [PATCH 34/76] locus_tag and product fields added to Gene model.
 Coinciding features are combined

---
 IslandCompare/analysis/components.py          | 49 +++++++++++++------
 .../migrations/0003_auto_20170720_2135.py     | 37 ++++++++++++++
 IslandCompare/genomes/models.py               |  8 +--
 IslandCompare/genomes/serializers.py          |  8 +--
 IslandCompare/genomes/views.py                |  4 +-
 5 files changed, 82 insertions(+), 24 deletions(-)
 create mode 100644 IslandCompare/genomes/migrations/0003_auto_20170720_2135.py

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 5ef61bc0..f55ca512 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -51,27 +51,44 @@ class SetupGbkPipelineComponent(PipelineComponent):
     def create_genes(genome):
         with open(genome.gbk.path) as genbank_handle:
             record = SeqIO.read(genbank_handle, "genbank")
+            last_gene = None
             for feature in record.features:
-                if feature.type in ["gene", "tRNA", "rRNA"]:
-                    strand = feature.location.strand
+                if feature.type in ["gene", "CDS", "tRNA", "rRNA"]:
                     start = feature.location.start
                     end = feature.location.end
-                    gene_type = feature.type
-                    try:
-                        name = feature.qualifiers['gene'][0]
-                    except KeyError:
+                    # Check if the current gene is in the same position as the last gene.
+                    if last_gene is not None and start == last_gene.start and end == last_gene.end:
+                        if feature.type is not "gene":
+                            # we don't want to replace the more specific types with "gene".
+                            last_gene.type = feature.type
+                        gene = last_gene
+                    else:
+                        gene = Gene(
+                            type=feature.type,
+                            gene="",
+                            locus_tag="",
+                            product="",
+                            start=start,
+                            end=end,
+                            strand=feature.location.strand,
+                            genome=genome
+                        )
+                        last_gene = gene
+                    if gene.gene == "":
+                        try:
+                            gene.gene = feature.qualifiers['gene'][0]
+                        except KeyError:
+                            pass
+                    if gene.locus_tag == "":
+                        try:
+                            gene.locus_tag = feature.qualifiers['locus_tag'][0]
+                        except KeyError:
+                            pass
+                    if gene.product == "":
                         try:
-                            name = feature.qualifiers['locus_tag'][0]
+                            gene.product = feature.qualifiers['product'][0]
                         except KeyError:
-                            name = ""
-                    gene = Gene(
-                        name=name,
-                        start=start,
-                        end=end,
-                        strand=strand,
-                        type=gene_type,
-                        genome=genome
-                    )
+                            pass
                     gene.save()
 
     def analysis(self, report):
diff --git a/IslandCompare/genomes/migrations/0003_auto_20170720_2135.py b/IslandCompare/genomes/migrations/0003_auto_20170720_2135.py
new file mode 100644
index 00000000..5465dbd6
--- /dev/null
+++ b/IslandCompare/genomes/migrations/0003_auto_20170720_2135.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-07-20 21:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('genomes', '0002_gene'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='gene',
+            name='name',
+        ),
+        migrations.AddField(
+            model_name='gene',
+            name='gene',
+            field=models.CharField(default='', max_length=12),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='gene',
+            name='locus_tag',
+            field=models.CharField(default='', max_length=12),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='gene',
+            name='product',
+            field=models.CharField(default='', max_length=200),
+            preserve_default=False,
+        ),
+    ]
diff --git a/IslandCompare/genomes/models.py b/IslandCompare/genomes/models.py
index 8197684e..cf5631b2 100644
--- a/IslandCompare/genomes/models.py
+++ b/IslandCompare/genomes/models.py
@@ -21,9 +21,11 @@ def delete_gbk(sender, instance, **kwargs):
 
 class Gene(models.Model):
     id = models.AutoField(primary_key=True)
-    name = models.CharField(max_length=50)
+    type = models.CharField(max_length=4)
+    gene = models.CharField(max_length=12)
+    locus_tag = models.CharField(max_length=12)
+    product = models.CharField(max_length=200)
     start = models.IntegerField()
     end = models.IntegerField()
     strand = models.SmallIntegerField()
-    type = models.CharField(max_length=4)
-    genome = models.ForeignKey(Genome)
\ No newline at end of file
+    genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
\ No newline at end of file
diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index 1fbc837d..bececd5f 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -143,8 +143,10 @@ class GeneSerializer(serializers.Serializer):
     """
     Serializer for Gene objects
     """
-    name = serializers.CharField(max_length=50)
+    type = serializers.CharField(max_length=4)
+    gene = serializers.CharField(max_length=12)
+    locus_tag = serializers.CharField(max_length=12)
+    product = serializers.CharField(max_length=200)
     start = serializers.IntegerField()
     end = serializers.IntegerField()
-    strand = serializers.IntegerField()
-    type = serializers.CharField(max_length=4)
\ No newline at end of file
+    strand = serializers.IntegerField()
\ No newline at end of file
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index f1ea302c..708ea718 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -79,9 +79,9 @@ def retrieve(self, request, *args, **kwargs):
         genes = queryset.get(id=kwargs['pk']).gene_set.all()
 
         if 'start' in url_queries:
-            genes = genes.filter(start__gte=url_queries['start'])
+            genes = genes.filter(end__gte=url_queries['start'])
         if 'end' in url_queries:
-            genes = genes.filter(end__lte=url_queries['end'])
+            genes = genes.filter(start__lte=url_queries['end'])
 
         serializer = GeneSerializer(genes, many=True)
 

From 9e7629f6061f4c5c73e4179b8eb8cbaf28062016 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 20 Jul 2017 17:05:36 -0700
Subject: [PATCH 35/76] Adjusted visualization to display gene's name, locus
 tag and product on mouse hover

---
 web/dist/js/genomecomparevis.js | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index f1326a8d..915c0a30 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -454,12 +454,6 @@ function MultiVis(targetNode){
                                 rectpoints += self.scale((gene['end'])) + "," + (self.getSequenceModHeight() * i + GENESIZE - 1) + " ";
                                 rectpoints += self.scale((gene['start'])) + "," + (self.getSequenceModHeight() * i + GENESIZE - 1) + " ";
                             }
-                            else {
-                                console.log("An error occurred while trying to draw: " + d.genes[geneIndex['name']]);
-                                continue;
-                            }
-
-                            var geneName = gene['name'];
 
                             geneContainer.append("polygon")
                                 .attr("points", rectpoints)
@@ -468,7 +462,17 @@ function MultiVis(targetNode){
                                 .style("opacity", 0.0)
                                 .append("title")
                                     .text(function (d, i) {
-                                        return geneName;
+                                        var hover = "";
+                                        if(gene['gene']) {
+                                            hover = hover.concat("Gene name: " + gene['gene'] + "\n");
+                                        }
+                                        if(gene['locus_tag']) {
+                                            hover = hover.concat("Locus tag: " + gene['locus_tag'] + "\n");
+                                        }
+                                        if(gene['product']) {
+                                            hover = hover.concat("Product: " + gene['product'])
+                                        }
+                                        return hover;
                                     });
                         }
                         // Wait until all genes have been added so they can be made visible simultaneously

From 3ad87f86b17a20606cc0c97bd84d194525a30aad Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 20 Jul 2017 17:22:22 -0700
Subject: [PATCH 36/76] Adjusted ReportGeneCsvSerializer to include gene names,
 locus tags and products

---
 IslandCompare/analysis/serializers.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 2936f8fd..113aebfd 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -221,7 +221,7 @@ class ReportGeneCsvSerializer(serializers.BaseSerializer):
     """
     def to_representation(self, instance):
         output = StringIO()
-        fieldnames = ['genome', 'name', 'start', 'end', 'strand', 'gi_start', 'gi_end', 'gi_method']
+        fieldnames = ['genome', 'gene_name', 'locus_tag', 'product', 'start', 'end', 'strand', 'gi_start', 'gi_end', 'gi_method']
         writer = csv.DictWriter(output, fieldnames=fieldnames)
         writer.writeheader()
 
@@ -234,7 +234,9 @@ def to_representation(self, instance):
                 for island in instance[method][key]:
                     for gi_gene in genome.gene_set.filter(start__gte=island[0]).filter(end__lte=island[1]):
                         writer.writerow({'genome': genome.name,
-                                         'name': gi_gene.name,
+                                         'gene_name': gi_gene.gene,
+                                         'locus_tag': gi_gene.locus_tag,
+                                         'product': gi_gene.product,
                                          'start': gi_gene.start,
                                          'end': gi_gene.end,
                                          'strand': gi_gene.strand,

From 051a6aab239abce41d1d803bfa2f1bb432aea2c8 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 21 Jul 2017 16:23:48 -0700
Subject: [PATCH 37/76] Result visualization now downloadable as svg or png

---
 web/dist/js/genomecomparevis.js | 55 +++++++++++++++++++++++++++++++++
 web/pages/result.html           | 26 +++++++++++++---
 2 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 915c0a30..eeea0352 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -574,6 +574,61 @@ function MultiVis(targetNode){
         }
     };
 
+    this.saveImage = function(mode) {
+        // https://stackoverflow.com/a/38085847
+        var svg = this.container.select("svg"),
+            img = new Image(),
+            serializer = new XMLSerializer(),
+            width = svg.node().getBoundingClientRect().width,
+            height = svg.node().getBoundingClientRect().height;
+
+        // get the css from the stylesheet
+        var style = "\nsvg {background-color: white;}\n";
+        for (var i=0; i<document.styleSheets.length; i++) {
+            var sheet = document.styleSheets[i];
+            if (sheet.href) {
+                var sheetName = sheet.href.split('/').pop();
+                if (sheetName === 'linearplot.css') {
+                    var rules = sheet.rules;
+                    if (rules) {
+                        for (var j = 0; j < rules.length; j++) {
+                            style += (rules[j].cssText + '\n');
+                        }
+                    }
+                }
+            }
+        }
+        // add the css to the svg
+        svg.insert("defs", ":first-child")
+            .append("style")
+            .attr('type', 'text/css')
+            .html(style);
+
+        var svgStr = serializer.serializeToString(svg.node());
+
+        if(mode === "png") {
+            var canvas = document.createElement("canvas");
+            canvas.width = width;
+            canvas.height = height;
+            var context = canvas.getContext("2d");
+            img.onload = function() {
+                context.drawImage(img, 0, 0);
+                var png = canvas.toDataURL("image/png");
+                var link = document.createElement("a");
+                link.setAttribute("href", png);
+                link.setAttribute("download", "result.png");
+                link.click();
+            };
+        }
+        img.src = 'data:image/svg+xml;base64,'+window.btoa(svgStr);
+        if(mode === "svg") {
+            var link = document.createElement("a");
+            link.setAttribute("href", img.src);
+            link.setAttribute("download", "result.svg");
+            link.click();
+        }
+    };
+
     return this;
 }
 
diff --git a/web/pages/result.html b/web/pages/result.html
index 632a691c..4f2adc01 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -30,11 +30,21 @@ <h1>
                     <div class="clearfix">
                         <span id="#controlTitle">Controls: </span>
                     </div>
-                    <div class="clearfix">
+                    <div id="btn-div" class="clearfix btn-group btn-group-sm">
+                        <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
                         <button id="resetTree" class="btn btn-primary btn-flat">Reset Tree</button>
+                        <div class="btn-group btn-group-sm">
+                            <button class="btn btn-primary btn-flat dropdown-toggle" data-toggle="dropdown">
+                                Save image <span class="caret"></span>
+                            </button>
+                            <ul class="dropdown-menu">
+                                <li><a class="dropdown-item" id="saveSvg" href="#">svg</a></li>
+                                <li><a class="dropdown-item" id="savePng" href="#">png</a></li>
+                            </ul>
+                        </div>
+                        <button id="printerColors" class="btn btn-primary btn-flat" style="clear: left; margin-left: 0px;">Toggle Colour</button>
                         <button id="resetRange" class="btn btn-primary btn-flat">Reset Zoom</button>
-                        <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
-                        <button id="printerColors" class="btn btn-primary btn-flat">Toggle Colour</button>
+
                     </div>
                 </div>
                 <div class="col-xs-2 GIColour">
@@ -253,11 +263,19 @@ <h1>
                     $("#amrLegend").attr("class", "#amrLegend");
                 }
             });
-
+            $("#saveSvg").on("click", this, function() {
+                container.saveImage("svg");
+            });
+            $("#savePng").on("click", this, function() {
+                container.saveImage("png");
+            });
             // Changes to sigi/islandpath checkboxes will hide/show corresponding GIs
             $(".GIColourBody :checkbox").change(container.predictorToggle);
             // Changes to GI colour method will hide/show the merged GI set and enable/disable the predictor checkboxes
             $("input[name=GIColour]").change(container.GIColourToggle);
+            // Set all buttons to have the same width
+            var buttons = $("#btn-div").children();
+            buttons.width(buttons.width());
         });
     }
 </script>
\ No newline at end of file

From 2f551305774633d0b49bf9cee6f75ee699b0002e Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 21 Jul 2017 17:58:11 -0700
Subject: [PATCH 38/76] Fixes #111 - resets visualization when the sidebar is
 toggled

---
 web/dist/js/genomecomparevis.js | 8 ++++++++
 web/pages/index.html            | 3 +++
 2 files changed, 11 insertions(+)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index eeea0352..342a4f5b 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -537,6 +537,14 @@ function MultiVis(targetNode){
         this.GIColourToggle();
     };
 
+    // Resets the scale when the sidebar is toggled
+    $(".sidebar-toggle").on("click", function() {
+        setTimeout(function() {
+            self.setScale(self.scale.domain()[0], self.scale.domain()[1]);
+            self.transition();
+        }, 500);
+    });
+
     this.togglePrinterColors = function(){
         this.isPrinterColors=!this.isPrinterColors;
     };
diff --git a/web/pages/index.html b/web/pages/index.html
index e17d5d67..dcb08223 100755
--- a/web/pages/index.html
+++ b/web/pages/index.html
@@ -151,10 +151,12 @@
     });
 
     $("#home-link").click(function(){
+        $(".sidebar-toggle").off("click");
         $("#content").load("home.html");
     });
 
     $("#jobs-link").click(function(){
+        $(".sidebar-toggle").off("click");
         $("#content").load("history.html");
     });
 
@@ -163,6 +165,7 @@
     });
 
     $("#about-link").click(function(){
+        $(".sidebar-toggle").off("click");
         $("#content").load("about.html");
     });
 </script>

From 6cd64ce7963cce4b1aa08b02a9a7b236ef3bc4fc Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 24 Jul 2017 15:16:36 -0700
Subject: [PATCH 39/76] Fixes #106 - multiple genomes were being submitted with
 the same id. Fixes colouring of genes with type CDS.

---
 IslandCompare/genomes/views.py |  2 +-
 web/dist/css/linearplot.css    |  2 +-
 web/pages/home.html            | 23 +++++------------------
 3 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index 708ea718..df6940d2 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -36,7 +36,7 @@ def create(self, request, *args, **kwargs):
         )
         if genome_serializer.is_valid():
             genome_serializer.save()
-            return response.Response(status=201)
+            return response.Response(data=genome_serializer.data['id'], status=201)
         else:
             return response.Response(genome_serializer.errors['gbk'], status=400)
 
diff --git a/web/dist/css/linearplot.css b/web/dist/css/linearplot.css
index bbfc36a7..183f589b 100644
--- a/web/dist/css/linearplot.css
+++ b/web/dist/css/linearplot.css
@@ -67,7 +67,7 @@ g.root.node circle{
                                  stroke:#ff9933;
                              }
 
-.genes polygon.gene, #genesLegend{
+.genes polygon.gene, .genes polygon.CDS, #genesLegend{
     fill: #6acbff;
     stroke: #6acbff;
 }
diff --git a/web/pages/home.html b/web/pages/home.html
index 311566a5..19420963 100644
--- a/web/pages/home.html
+++ b/web/pages/home.html
@@ -310,24 +310,11 @@ <h3 class="box-title">Run Analysis</h3>
             "Cache-Control": "",
             "Authorization": 'Token'+' '+getAuthenticationCookie()
         },
-        // If upload was successful, another GET call is made to find the id of the last genome - the one that was just
-        // uploaded. This id is needed to add a row for the new genome into the DataTable.
-        // There must be a better way to do this. While the data returned by the upload (up_data) contains the
-        // file name (up_data.name), it doesn't seem to contain the id.
-        success: function(up_data){
-            var new_row_id;
-            $.ajax({
-                type: "GET",
-                url: genomesUrl,
-                headers: {
-                    "Authorization": 'Token' + ' ' + getAuthenticationCookie()
-                },
-                success: function(data) {
-                    var new_row_id = data[data.length - 1].id; // TODO validate this is correct genome
-                    var genomesTable = $("#uploaded-genomes").DataTable();
-                    genomesTable.row.add({ name: up_data.name, id: new_row_id }).draw();
-                }
-            });
+        // If upload is successful add a row to the genome data table.
+        success: function(data){
+            var new_row_id = parseInt(data.xhr.response);
+            var genomesTable = $("#uploaded-genomes").DataTable();
+            genomesTable.row.add({ name: data.name, id: new_row_id }).draw();
         }
     });
 

From 99e2d5434a0695bd60c623afe7b5250c87afb307 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 25 Jul 2017 10:44:33 -0700
Subject: [PATCH 40/76] Long genome names on visualization will be shortened to
 not overlap with the sequences

---
 web/dist/js/genomecomparevis.js | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 342a4f5b..3818f255 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -521,6 +521,15 @@ function MultiVis(targetNode){
         var textLabels = text.attr("y", function(d,i){ return (i)*self.getSequenceModHeight()})
             .text(function(d){return d.shownName});
 
+        // Shorten long genbank file names that extend into the visContainer
+        text.each(function() {
+            if(this.getBoundingClientRect().width > LEFTPADDING - TREECONTAINERWIDTH - TEXTPADDING - 5) {
+                d3.select(this)
+                    .attr("textLength", LEFTPADDING - TREECONTAINERWIDTH - TEXTPADDING - 5)
+                    .attr("lengthAdjust", "spacingAndGlyphs");
+            }
+        });
+
         textContainer.attr("transform","translate("+(TREECONTAINERWIDTH+TEXTPADDING)+","+(TEXTTOPPADDING+TOPPADDING)+")");
 
         //Aligns the viscontainer to the right to make room for other containers

From 881a52e18e9ffdd2ef23f5d8077f5d8a2d7126b4 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 25 Jul 2017 13:24:40 -0700
Subject: [PATCH 41/76] Renaming or deleting jobs will not reset the job
 history table to page one

---
 web/pages/history.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/pages/history.html b/web/pages/history.html
index 85a9c0d6..8a066bf4 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -132,7 +132,7 @@ <h3 class="box-title">Job History</h3>
                             var jobtable = $("#job-history").DataTable();
                             var row_values = jobtable.row('#'+jobid).data();
                             row_values.name = new_name;
-                            jobtable.row('#'+jobid).data(row_values).draw();
+                            jobtable.row('#'+jobid).data(row_values).draw(paging=false);
                         },
                         error: function(message) {
                             BootstrapDialog.warning(message.responseText);
@@ -160,7 +160,7 @@ <h3 class="box-title">Job History</h3>
                         },
                         success: function(data){
                             var jobtable = $("#job-history").DataTable();
-                            jobtable.row('#' + jobid).remove().draw();
+                            jobtable.row('#' + jobid).remove().draw(paging=false);
                         },
                         error: function(message) {
                             BootstrapDialog.warning(message.responseText);

From 4eb3cbf66231b67eefed299ea975cff75f39c58f Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 26 Jul 2017 15:26:41 -0700
Subject: [PATCH 42/76] Add GenomicIsland model and migration

---
 .../genomes/migrations/0004_genomicisland.py  | 26 +++++++++++++++++++
 IslandCompare/genomes/models.py               |  7 +++++
 2 files changed, 33 insertions(+)
 create mode 100644 IslandCompare/genomes/migrations/0004_genomicisland.py

diff --git a/IslandCompare/genomes/migrations/0004_genomicisland.py b/IslandCompare/genomes/migrations/0004_genomicisland.py
new file mode 100644
index 00000000..f4739a15
--- /dev/null
+++ b/IslandCompare/genomes/migrations/0004_genomicisland.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-07-26 21:44
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('genomes', '0003_auto_20170720_2135'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GenomicIsland',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('method', models.CharField(max_length=10)),
+                ('start', models.IntegerField()),
+                ('end', models.IntegerField()),
+                ('genome', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genomes.Genome')),
+            ],
+        ),
+    ]
diff --git a/IslandCompare/genomes/models.py b/IslandCompare/genomes/models.py
index cf5631b2..303a99af 100644
--- a/IslandCompare/genomes/models.py
+++ b/IslandCompare/genomes/models.py
@@ -28,4 +28,11 @@ class Gene(models.Model):
     start = models.IntegerField()
     end = models.IntegerField()
     strand = models.SmallIntegerField()
+    genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
+
+class GenomicIsland(models.Model):
+    id = models.AutoField(primary_key=True)
+    method = models.CharField(max_length=10)
+    start = models.IntegerField()
+    end = models.IntegerField()
     genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
\ No newline at end of file

From 7804d44469eb52fe53f3ac4ba0536579baedda0a Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 26 Jul 2017 16:02:15 -0700
Subject: [PATCH 43/76] Predicted GIs will be added to database rather than
 report

---
 IslandCompare/analysis/components.py        | 152 +++++++++++---------
 IslandCompare/analysis/integration_tests.py |  43 ++++--
 IslandCompare/analysis/tests_components.py  | 124 +++++++++++++---
 3 files changed, 222 insertions(+), 97 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index f55ca512..3ecdd2c7 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -1,6 +1,6 @@
 from analysis.pipeline import PipelineComponent
 from analysis.models import Analysis, Genome
-from genomes.models import Gene
+from genomes.models import Gene, GenomicIsland
 from tempfile import mkdtemp, NamedTemporaryFile
 from shutil import rmtree
 from Bio import SeqIO, Phylo
@@ -454,7 +454,6 @@ class SigiHMMPipelineComponent(PipelineComponent):
     """
     name = "sigi"
     dependencies = ["gbk_paths"]
-    result_types = ["sigi_gis"]
     output_dir = settings.BIO_APP_TEMP_DIR + "sigi/"
     temp_dir_path = None
     embl_files = {}
@@ -510,25 +509,32 @@ def setup(self, report):
 
     def analysis(self, report):
         script_file = NamedTemporaryFile(delete=True)
-        report["sigi_gis"] = {}
 
         for embl_id in self.embl_files:
-            sigi_gff_output = os.path.abspath(self.temp_dir_path + "/" + str(embl_id) + ".gff")
-            sigi_output = os.path.abspath(self.temp_dir_path +"/" + str(embl_id) + ".embl")
-            with open(script_file.name, 'w') as script:
-                script.write("#!/bin/bash\n")
-                script.write("/usr/bin/java " + self.SIGIHMM_EXE + " input=" +
-                             self.embl_files[embl_id] + " output=" + sigi_output + " gff=" + sigi_gff_output)
-                script.close()
-
-            os.chmod(script_file.name, 0o755)
-            script_file.file.close()
-
-            with open(self.temp_dir_path + "/logs", 'w') as logs:
-                subprocess.check_call(script_file.name, stdout=logs, cwd=self.SIGIHMM_PATH)
-            script_file.close()
-
-            report["sigi_gis"][str(embl_id)] = self.parse_sigi_gff(sigi_gff_output)
+            genome = Genome.objects.get(id=embl_id)
+            if not genome.genomicisland_set.filter(method__exact="sigi").exists():
+                sigi_gff_output = os.path.abspath(self.temp_dir_path + "/" + str(embl_id) + ".gff")
+                sigi_output = os.path.abspath(self.temp_dir_path +"/" + str(embl_id) + ".embl")
+                with open(script_file.name, 'w') as script:
+                    script.write("#!/bin/bash\n")
+                    script.write("/usr/bin/java " + self.SIGIHMM_EXE + " input=" +
+                                 self.embl_files[embl_id] + " output=" + sigi_output + " gff=" + sigi_gff_output)
+                    script.close()
+
+                os.chmod(script_file.name, 0o755)
+                script_file.file.close()
+
+                with open(self.temp_dir_path + "/logs", 'w') as logs:
+                    subprocess.check_call(script_file.name, stdout=logs, cwd=self.SIGIHMM_PATH)
+                script_file.close()
+
+                for sigi_gi in self.parse_sigi_gff(sigi_gff_output):
+                    GenomicIsland(
+                        method="sigi",
+                        start=sigi_gi[0],
+                        end=sigi_gi[1],
+                        genome=genome
+                    ).save()
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
@@ -541,7 +547,6 @@ class IslandPathPipelineComponent(PipelineComponent):
     """
     name = "islandpath"
     dependencies = ["gbk_paths"]
-    result_types = ["islandpath_gis"]
     output_dir = settings.BIO_APP_TEMP_DIR + "islandpath/"
     ISLANDPATH_PATH = settings.ISLANDPATH_PATH
     log_path = None
@@ -563,28 +568,35 @@ def setup(self, report):
         os.mkdir(self.temp_dir_path, 0o777)
 
     def analysis(self, report):
-        report["islandpath_gis"] = {}
 
         for gbk_id in report["gbk_paths"]:
-            script_file = NamedTemporaryFile(delete=True)
-            temp_path = self.temp_dir_path + "/" + str(gbk_id)
-
-            with open(script_file.name, 'w') as script:
-                script.write("#!/bin/bash\n")
-                script.write(os.path.abspath(self.ISLANDPATH_PATH)
-                             + " " + os.path.abspath(report["gbk_paths"][gbk_id])
-                             + " " + os.path.abspath(temp_path))
-                script.close()
-
-            os.chmod(script_file.name, 0o755)
-            script_file.file.close()
-
-            self.log_path = self.temp_dir_path + "/logs_" + str(report["analysis"])
-            with open(self.log_path, 'w') as logs:
-                subprocess.check_call(script_file.name, stdout=logs, cwd=self.temp_dir_path)
-            script_file.close()
-
-            report["islandpath_gis"][gbk_id] = self.parse_islandpath(temp_path)
+            genome = Genome.objects.get(id=gbk_id)
+            if not genome.genomicisland_set.filter(method__exact="islandpath").exists():
+                script_file = NamedTemporaryFile(delete=True)
+                temp_path = self.temp_dir_path + "/" + str(gbk_id)
+
+                with open(script_file.name, 'w') as script:
+                    script.write("#!/bin/bash\n")
+                    script.write(os.path.abspath(self.ISLANDPATH_PATH)
+                                 + " " + os.path.abspath(report["gbk_paths"][gbk_id])
+                                 + " " + os.path.abspath(temp_path))
+                    script.close()
+
+                os.chmod(script_file.name, 0o755)
+                script_file.file.close()
+
+                self.log_path = self.temp_dir_path + "/logs_" + str(report["analysis"])
+                with open(self.log_path, 'w') as logs:
+                    subprocess.check_call(script_file.name, stdout=logs, cwd=self.temp_dir_path)
+                script_file.close()
+
+                for islandpath_gi in self.parse_islandpath(temp_path):
+                    GenomicIsland(
+                        method="islandpath",
+                        start=islandpath_gi[0],
+                        end=islandpath_gi[1],
+                        genome=genome
+                    ).save()
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
@@ -596,8 +608,6 @@ class MergeIslandsPipelineComponent(PipelineComponent):
     Pipeline component that merges islands and adds the merged islands to the report.
     """
     name = "merge_gis"
-    dependencies = ["islandpath_gis", "sigi_gis"]
-    result_types = ["merge_gis"]
     threshold = 500
 
     def merge_gi_list(self, first_list, second_list):
@@ -615,24 +625,33 @@ def merge_gi_list(self, first_list, second_list):
 
         return merged_gis
 
+    def merge_gi_queryset(self, gi_queryset):
+        ordered_gis = gi_queryset.order_by('start')
+
+        current_gi = ordered_gis[0]
+        current_gi.pk = None # Creates a copy of the gi object
+        for i in range(1, len(ordered_gis)):
+            next_gi = ordered_gis[i]
+            if current_gi.end + self.threshold >= next_gi.start:
+                if current_gi.end < next_gi.end:
+                    current_gi.end = next_gi.end
+            else:
+                current_gi.method = "merged"
+                current_gi.save() # Will create a new entry in the database because pk was set to None
+                current_gi = next_gi
+                current_gi.pk = None
+        current_gi.method = "merged"
+        current_gi.save()
+
     def set_threshold(self, threshold):
         self.threshold = threshold
 
     def analysis(self, report):
-        merged_gi_dict = dict()
-
-        if "sigi_gis" in report:
-            sigi_gis = report["sigi_gis"]
-        else:
-            sigi_gis = {genome_id: [] for genome_id in report["islandpath_gis"]}
-        if "islandpath_gis" in report:
-            islandpath_gis = report["islandpath_gis"]
-        else:
-            islandpath_gis = {genome_id: [] for genome_id in report["sigi_gis"]}
-
-        for genome_id in sigi_gis:
-            merged_gi_dict[genome_id] = self.merge_gi_list(islandpath_gis[str(genome_id)], sigi_gis[str(genome_id)])
-        report["merge_gis"] = merged_gi_dict
+        genomes = Genome.objects.filter(id__in=report['gbk_paths'].keys())
+        for genome in genomes:
+            gis = genome.genomicisland_set
+            if gis.exists():
+                self.merge_gi_queryset(gis.all())
 
 
 class MashMclClusterPipelineComponent(PipelineComponent):
@@ -640,7 +659,7 @@ class MashMclClusterPipelineComponent(PipelineComponent):
     Pipeline component that runs mash and clusters gis.
     """
     name = "mash_mcl"
-    dependencies = ["gbk_paths", "merge_gis"]
+    dependencies = ["gbk_paths"]
     result_types = ["cluster_gis"]
     output_dir = settings.BIO_APP_TEMP_DIR + "mash/"
     MASH_PATH = settings.MASH_PATH
@@ -707,18 +726,17 @@ def setup(self, report):
         os.mkdir(self.fna_dir_path)
 
     def create_gi_fasta_files(self, report):
-        genome_list = report["gbk_paths"]
         island_path_list = []
 
-        for genome_id in genome_list.keys():
+        for genome_id in report["gbk_paths"]:
             record = SeqIO.read(report['gbk_paths'][genome_id], "genbank")
             genome_fna_path = self.fna_dir_path + "/" + str(genome_id)
             os.mkdir(genome_fna_path)
 
             gi_counter = 0
-            for gi in report['merge_gis'][genome_id]:
+            for gi in Genome.objects.get(id=genome_id).genomicisland_set.filter(method="merged").all():
                 self.logger.info("Adding GI: " + str(genome_id) + "-" + str(gi_counter))
-                entrySequence = self.get_subsequence(record, gi[0], gi[1], gi_counter)
+                entrySequence = self.get_subsequence(record, gi.start, gi.end, gi_counter)
                 self.writeFastaFile(self.fna_dir_path + "/" + str(genome_id) + "/" + str(gi_counter), entrySequence)
                 island_path_list.append(self.fna_dir_path + "/" + str(genome_id) + "/" + str(gi_counter))
                 gi_counter += 1
@@ -751,10 +769,10 @@ def analysis(self, report):
         np.set_printoptions(threshold='nan')
 
         M, clusters = mcl(mclAdjacencyMatrix)
-        outputList = {}
+        output_dict = {}
 
-        for sequenceId in report["gbk_paths"].keys():
-            outputList[str(sequenceId)] = {}
+        for genome_id in report["gbk_paths"]:
+            output_dict[str(genome_id)] = {}
         islandIdList = list([i for i in range(len(island_path_list))])
 
         numberClusters = 0
@@ -770,12 +788,12 @@ def analysis(self, report):
                 islandId = splitIsland[1]
                 self.logger.info("Current Sequence Id: " + str(currentSequenceId))
                 self.logger.info("Current Island Id: " + str(islandId))
-                outputList[str(currentSequenceId)][str(islandId)] = numberClusters - 1
+                output_dict[str(currentSequenceId)][str(islandId)] = numberClusters - 1
             remainingIslands = filter(lambda x: x not in currentCluster, islandIdList)
             islandIdList = list(remainingIslands)
 
-        outputList['numberClusters'] = numberClusters - 1
-        report["cluster_gis"] = outputList
+        output_dict['numberClusters'] = numberClusters - 1
+        report["cluster_gis"] = output_dict
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
diff --git a/IslandCompare/analysis/integration_tests.py b/IslandCompare/analysis/integration_tests.py
index 1164e294..1953adf9 100644
--- a/IslandCompare/analysis/integration_tests.py
+++ b/IslandCompare/analysis/integration_tests.py
@@ -1,5 +1,5 @@
 from django.test import TestCase, override_settings
-from genomes.models import Genome
+from genomes.models import Genome, GenomicIsland
 from django.contrib.auth.models import User
 from django.core.files import File
 from analysis.components import ParsnpPipelineComponent, MauvePipelineComponent, \
@@ -155,10 +155,12 @@ def setUp(self):
         self.test_genome_1 = Genome.objects.create(name=self.test_genome_1_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_1_gbk)
+        self.test_genome_1.save()
 
         self.test_genome_2 = Genome.objects.create(name=self.test_genome_2_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_2_gbk)
+        self.test_genome_2.save()
 
     def test_sigihmm_analysis(self):
         report = {
@@ -175,8 +177,8 @@ def test_sigihmm_analysis(self):
         component.analysis(report)
         component.cleanup()
 
-        self.assertTrue("sigi_gis" in report)
-        self.assertEqual(2, len(report["sigi_gis"]))
+        self.assertTrue(self.test_genome_1.genomicisland_set.filter(method="sigi").exists())
+        self.assertTrue(self.test_genome_2.genomicisland_set.filter(method="sigi").exists())
 
     def tearDown(self):
         for genome in Genome.objects.all():
@@ -202,10 +204,12 @@ def setUp(self):
         self.test_genome_1 = Genome.objects.create(name=self.test_genome_1_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_1_gbk)
+        self.test_genome_1.save()
 
         self.test_genome_2 = Genome.objects.create(name=self.test_genome_2_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_2_gbk)
+        self.test_genome_2.save()
 
     def test_islandpath_analysis(self):
         report = {
@@ -222,10 +226,8 @@ def test_islandpath_analysis(self):
         component.analysis(report)
         component.cleanup()
 
-        self.assertTrue("islandpath_gis" in report)
-        self.assertEqual(2, len(report["islandpath_gis"]))
-        self.assertTrue(self.test_genome_1.id in report["islandpath_gis"])
-        self.assertTrue(self.test_genome_2.id in report["islandpath_gis"])
+        self.assertTrue(self.test_genome_1.genomicisland_set.filter(method="islandpath").exists())
+        self.assertTrue(self.test_genome_2.genomicisland_set.filter(method="islandpath").exists())
 
     def tearDown(self):
         for genome in Genome.objects.all():
@@ -292,6 +294,9 @@ class MashMCLTestCase(TestCase):
     test_genome_2_name = "genome_2"
     test_genome_2_gbk = File(open("../TestFiles/BX936398.gbk"))
 
+    test_gi_1 = None
+    test_gi_2 = None
+
     def setUp(self):
         self.test_user = User(username=self.test_username)
         self.test_user.save()
@@ -299,21 +304,34 @@ def setUp(self):
         self.test_genome_1 = Genome.objects.create(name=self.test_genome_1_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_1_gbk)
+        self.test_genome_1.save()
 
         self.test_genome_2 = Genome.objects.create(name=self.test_genome_2_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_2_gbk)
+        self.test_genome_2.save()
+
+        self.test_gi_1 = GenomicIsland(
+            method="merged",
+            start=0,
+            end=1000,
+            genome=self.test_genome_1
+        ).save()
+        self.test_gi_2 = GenomicIsland(
+            method="merged",
+            start=0,
+            end=1000,
+            genome=self.test_genome_2
+        ).save()
 
     def test_generate_gi_fna(self):
         component = MashMclClusterPipelineComponent()
 
         report = {
             "analysis": 1,
-            "available_dependencies": ["merge_gis", "gbk_paths"],
+            "available_dependencies": ["gbk_paths"],
             "gbk_paths": {self.test_genome_1.id: self.test_genome_1.gbk.path,
-                          self.test_genome_2.id: self.test_genome_2.gbk.path},
-            "merge_gis": {self.test_genome_1.id: [[0, 1000]],
-                          self.test_genome_2.id: [[0, 1000]]}
+                          self.test_genome_2.id: self.test_genome_2.gbk.path}
         }
 
         component.setup(report)
@@ -364,7 +382,6 @@ def setUp(self):
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_1_gbk)
 
-    @skip("Complete writing tests when component is defined and implemented")
     def test_rgi_component(self):
         report = {
             "analysis": 1,
@@ -372,7 +389,7 @@ def test_rgi_component(self):
             "gbk_paths": {self.test_genome_1.id: self.test_genome_1.gbk.path},
         }
 
-        component = RGIPipelineComponent()#mock.MagicMock(RGIPipelineComponent)
+        component = RGIPipelineComponent()
         component.setup(report)
         component.analysis(report)
         component.cleanup()
diff --git a/IslandCompare/analysis/tests_components.py b/IslandCompare/analysis/tests_components.py
index 63c63b6c..234233d3 100644
--- a/IslandCompare/analysis/tests_components.py
+++ b/IslandCompare/analysis/tests_components.py
@@ -1,10 +1,11 @@
 from django.test import TestCase, mock
 from django.contrib.auth.models import User
-from genomes.models import Genome
+from genomes.models import Genome, GenomicIsland
 from analysis.components import SetupGbkPipelineComponent, ParsnpPipelineComponent, MauvePipelineComponent, \
     SigiHMMPipelineComponent, GbkMetadataComponent, MergeIslandsPipelineComponent
 from analysis.pipeline import Pipeline, PipelineSerializer
 from django.core.files import File
+from django.core.files.uploadedfile import SimpleUploadedFile
 import filecmp
 import os
 
@@ -272,19 +273,49 @@ def tearDown(self):
 
 
 class MergeGITestCase(TestCase):
+    test_username = "test_user"
+    test_user = None
+
+    test_analysis_name = "test_analysis"
+
+    test_genome = None
+    test_genome_name = "genome_1"
+    test_genome_gbk_name = "test.gbk"
+    test_genome_gbk_contents = bytes("test", 'utf-8')
+    test_genome_gbk = SimpleUploadedFile(test_genome_gbk_name, test_genome_gbk_contents)
+
     report = None
 
     def setUp(self):
+        self.test_user = User(username=self.test_username)
+        self.test_user.save()
+
+        self.test_genome = Genome.objects.create(name=self.test_genome_name,
+                                                 owner=self.test_user,
+                                                 gbk=self.test_genome_gbk)
+        self.test_genome.save()
+
         self.report = {
             "analysis": 1,
             "available_dependencies": ["sigi_gis", "islandpath_gis"],
+            "gbk_paths": {self.test_genome.id: self.test_genome_gbk},
             "sigi_gis": {},
             "islandpath_gis": {}
         }
 
     def test_merge_single_gi_list(self):
-        self.report["sigi_gis"] = {"1": [["0", "100"], ["400", "600"]]}
-        self.report["islandpath_gis"] = {"1": []}
+        GenomicIsland(
+            method="sigi",
+            start=0,
+            end=100,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="sigi",
+            start=400,
+            end=600,
+            genome=self.test_genome
+        ).save()
 
         component = MergeIslandsPipelineComponent()
 
@@ -292,11 +323,28 @@ def test_merge_single_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertEqual(len(self.report["sigi_gis"]), len(self.report["merge_gis"]))
+        self.assertTrue(self.test_genome.genomicisland_set.filter(method="merged").exists())
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
 
     def test_no_merge_gi_list(self):
-        self.report["sigi_gis"] = {"1": [["0", "100"], ["400", "600"]]}
-        self.report["islandpath_gis"] = {"1": [["1000", "1200"]]}
+        GenomicIsland(
+            method="sigi",
+            start=0,
+            end=100,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="sigi",
+            start=400,
+            end=600,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="islandpath",
+            start=1000,
+            end=1200,
+            genome=self.test_genome
+        ).save()
 
         component = MergeIslandsPipelineComponent()
         component.set_threshold(100)
@@ -305,12 +353,21 @@ def test_no_merge_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertEqual(len(self.report["sigi_gis"]["1"]) + len(self.report["islandpath_gis"]["1"]),
-                         len(self.report["merge_gis"]["1"]))
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 3)
 
     def test_merge_gi_list(self):
-        self.report["sigi_gis"] = {"1": [["0", "100"]]}
-        self.report["islandpath_gis"] = {"1": [["199", "1200"]]}
+        GenomicIsland(
+            method="sigi",
+            start=0,
+            end=100,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="islandpath",
+            start=199,
+            end=1200,
+            genome=self.test_genome
+        ).save()
 
         component = MergeIslandsPipelineComponent()
 
@@ -318,11 +375,23 @@ def test_merge_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertListEqual([["0", "1200"]], self.report["merge_gis"]["1"])
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].start, 0)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].end, 1200)
 
     def test_merge_secondlist_gi_list(self):
-        self.report["sigi_gis"] = {"1": [["199", "1200"]]}
-        self.report["islandpath_gis"] = {"1": [["0", "100"]]}
+        GenomicIsland(
+            method="sigi",
+            start=199,
+            end=1200,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="islandpath",
+            start=0,
+            end=100,
+            genome=self.test_genome
+        ).save()
 
         component = MergeIslandsPipelineComponent()
 
@@ -330,11 +399,30 @@ def test_merge_secondlist_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertListEqual([["0", "1200"]], self.report["merge_gis"]["1"])
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].start, 0)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].end, 1200)
 
     def test_merge_encompassed_gi(self):
-        self.report["sigi_gis"] = {"1": [["1000", "2000"], ["3000", "4000"]]}
-        self.report["islandpath_gis"] = {"1": [["100", "5000"]]}
+        GenomicIsland(
+            method="sigi",
+            start=1000,
+            end=2000,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="sigi",
+            start=3000,
+            end=4000,
+            genome=self.test_genome
+        ).save()
+        GenomicIsland(
+            method="islandpath",
+            start=100,
+            end=5000,
+            genome=self.test_genome
+        ).save()
+
 
         component = MergeIslandsPipelineComponent()
 
@@ -342,4 +430,6 @@ def test_merge_encompassed_gi(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertListEqual([["100", "5000"]], self.report["merge_gis"]["1"])
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].start, 100)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].end, 5000)

From fd601e163f7455bb888dcee1223fd78ff6f2f6e6 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 26 Jul 2017 16:19:54 -0700
Subject: [PATCH 44/76] Add GenomicIslandCluster model and migration

---
 .../migrations/0002_auto_20170718_1718.py     | 25 ++++++++++++++++
 IslandCompare/analysis/models.py              |  2 --
 .../genomes/migrations/0005_gicluster.py      | 30 +++++++++++++++++++
 IslandCompare/genomes/models.py               | 12 ++++++--
 4 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 IslandCompare/analysis/migrations/0002_auto_20170718_1718.py
 create mode 100644 IslandCompare/genomes/migrations/0005_gicluster.py

diff --git a/IslandCompare/analysis/migrations/0002_auto_20170718_1718.py b/IslandCompare/analysis/migrations/0002_auto_20170718_1718.py
new file mode 100644
index 00000000..5e9e2a66
--- /dev/null
+++ b/IslandCompare/analysis/migrations/0002_auto_20170718_1718.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-07-18 17:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analysis', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='analysis',
+            name='celery_task_id',
+            field=models.CharField(max_length=765),
+        ),
+        migrations.AlterField(
+            model_name='analysiscomponent',
+            name='celery_task_id',
+            field=models.CharField(max_length=765),
+        ),
+    ]
diff --git a/IslandCompare/analysis/models.py b/IslandCompare/analysis/models.py
index 8fd71294..7326b9b5 100644
--- a/IslandCompare/analysis/models.py
+++ b/IslandCompare/analysis/models.py
@@ -3,8 +3,6 @@
 from genomes.models import Genome
 from django.utils import timezone
 
-# Create your models here.
-
 
 class Analysis(models.Model):
     id = models.AutoField(primary_key=True)
diff --git a/IslandCompare/genomes/migrations/0005_gicluster.py b/IslandCompare/genomes/migrations/0005_gicluster.py
new file mode 100644
index 00000000..fbe21f1b
--- /dev/null
+++ b/IslandCompare/genomes/migrations/0005_gicluster.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-07-26 23:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analysis', '0002_auto_20170718_1718'),
+        ('genomes', '0004_genomicisland'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GenomicIslandCluster',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('number', models.IntegerField()),
+                ('analysis', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analysis.Analysis')),
+                ('genomic_islands', models.ManyToManyField(to='genomes.GenomicIsland')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='genomicislandcluster',
+            unique_together=set([('number', 'analysis')]),
+        ),
+    ]
diff --git a/IslandCompare/genomes/models.py b/IslandCompare/genomes/models.py
index 303a99af..1eea2693 100644
--- a/IslandCompare/genomes/models.py
+++ b/IslandCompare/genomes/models.py
@@ -13,7 +13,6 @@ class Genome(models.Model):
     class Meta:
         unique_together = ('name', 'owner')
 
-
 @receiver(post_delete, sender=Genome)
 def delete_gbk(sender, instance, **kwargs):
     if instance.gbk is not None:
@@ -35,4 +34,13 @@ class GenomicIsland(models.Model):
     method = models.CharField(max_length=10)
     start = models.IntegerField()
     end = models.IntegerField()
-    genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
\ No newline at end of file
+    genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
+
+class GenomicIslandCluster(models.Model):
+    id = models.AutoField(primary_key=True)
+    number = models.IntegerField()
+    analysis = models.ForeignKey('analysis.Analysis')
+    genomic_islands = models.ManyToManyField(GenomicIsland)
+
+    class Meta:
+        unique_together = ('number', 'analysis')
\ No newline at end of file

From 50cac9629546e5078542f0e4d2721054fb819eef Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 26 Jul 2017 18:06:21 -0700
Subject: [PATCH 45/76] MashMclClusterPipelineComponent will add clusters to
 database rather than report

---
 IslandCompare/analysis/components.py | 52 +++++++++++++++-------------
 1 file changed, 27 insertions(+), 25 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 3ecdd2c7..85bf856b 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -610,21 +610,6 @@ class MergeIslandsPipelineComponent(PipelineComponent):
     name = "merge_gis"
     threshold = 500
 
-    def merge_gi_list(self, first_list, second_list):
-        merged_gis = first_list + second_list
-        merged_gis.sort(key=lambda x: int(x[0]))
-
-        i = 0
-        while i < len(merged_gis) - 1:
-            if int(merged_gis[i][1]) + self.threshold >= int(merged_gis[i + 1][0]):
-                if int(merged_gis[i][1]) < int(merged_gis[i + 1][1]):
-                    merged_gis[i][1] = merged_gis[i + 1][1]
-                del merged_gis[i + 1]
-            else:
-                i += 1
-
-        return merged_gis
-
     def merge_gi_queryset(self, gi_queryset):
         ordered_gis = gi_queryset.order_by('start')
 
@@ -650,7 +635,7 @@ def analysis(self, report):
         genomes = Genome.objects.filter(id__in=report['gbk_paths'].keys())
         for genome in genomes:
             gis = genome.genomicisland_set
-            if gis.exists():
+            if gis.filter(method__in=["sigi", "islandpath"]).exists() and not gis.filter(method="merged").exists():
                 self.merge_gi_queryset(gis.all())
 
 
@@ -718,13 +703,6 @@ def writeFastaFile(self, outputFileName, seqRecordList):
         with open(outputFileName, 'w') as outputFileHandle:
             SeqIO.write(seqRecordList, outputFileHandle, "fasta")
 
-    def setup(self, report):
-        self.temp_dir_path = self.output_dir + str(report["analysis"])
-        os.mkdir(self.temp_dir_path, 0o777)
-
-        self.fna_dir_path = self.temp_dir_path + "/fna"
-        os.mkdir(self.fna_dir_path)
-
     def create_gi_fasta_files(self, report):
         island_path_list = []
 
@@ -743,6 +721,30 @@ def create_gi_fasta_files(self, report):
 
         return island_path_list
 
+    def assign_clusters(self, report, cluster_dict):
+        self.logger.info("Saving GI clusters")
+        analysis = Analysis.objects.get(id=report['analysis'])
+        for genome_id in report['gbk_paths']:
+            genome = Genome.objects.get(id=genome_id)
+            merged_gis = genome.genomicisland_set.filter(method="merged")
+            for gi_index in range(len(merged_gis)):
+                cluster_index = cluster_dict[str(genome.id)][str(gi_index)]
+                cluster = analysis.genomicislandcluster_set.filter(number=cluster_index)
+                if cluster.exists():
+                    cluster[0].genomic_islands.add(merged_gis[gi_index])
+                else:
+                    merged_gis[gi_index].genomicislandcluster_set.create(
+                        number=cluster_index,
+                        analysis=analysis
+                    )
+
+    def setup(self, report):
+        self.temp_dir_path = self.output_dir + str(report["analysis"])
+        os.mkdir(self.temp_dir_path, 0o777)
+
+        self.fna_dir_path = self.temp_dir_path + "/fna"
+        os.mkdir(self.fna_dir_path)
+
     def analysis(self, report):
         island_path_list = self.create_gi_fasta_files(report)
         self.create_compound_sketch(island_path_list, self.temp_dir_path + "/compoundScratch")
@@ -792,8 +794,8 @@ def analysis(self, report):
             remainingIslands = filter(lambda x: x not in currentCluster, islandIdList)
             islandIdList = list(remainingIslands)
 
-        output_dict['numberClusters'] = numberClusters - 1
-        report["cluster_gis"] = output_dict
+        report['numberClusters'] = numberClusters - 1
+        self.assign_clusters(report, output_dict)
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):

From 33163a0df6b8e5f91565c7fcdfc6750a5541c2a9 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 27 Jul 2017 12:10:29 -0700
Subject: [PATCH 46/76] Refactoring for neatness and minor efficiency
 improvement

---
 IslandCompare/analysis/components.py        | 71 +++++++++++----------
 IslandCompare/analysis/integration_tests.py |  6 +-
 IslandCompare/analysis/tasks.py             |  2 -
 IslandCompare/analysis/tests_components.py  | 24 +++----
 web/dist/js/genomecomparevis.js             | 11 ----
 5 files changed, 54 insertions(+), 60 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 85bf856b..d74dc9a3 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -98,11 +98,11 @@ def analysis(self, report):
         report['gbk_paths'] = dict()
         for genome in genomes.all():
             report['gbk_paths'][str(genome.id)] = genome.gbk.path
-            if not genome.gene_set.exists():
+            if genome.gene_set.exists():
+                self.logger.info("{} gene set found".format(genome.name))
+            else:
                 self.logger.info("Creating gene set for genome {}".format(genome.name))
                 self.create_genes(genome)
-            else:
-                self.logger.info("{} gene set found".format(genome.name))
 
 
 class GbkMetadataComponent(PipelineComponent):
@@ -502,39 +502,42 @@ def setup(self, report):
         os.mkdir(self.temp_dir_path, 0o777)
 
         for gbk_id in report["gbk_paths"]:
-            basename = os.path.splitext(os.path.basename(report["gbk_paths"][gbk_id]))[0]
-            with open(self.output_dir + str(report["analysis"]) + "/" + basename + ".embl", 'w') as embl_output:
-                SeqIO.convert(report["gbk_paths"][gbk_id], "genbank", embl_output, "embl")
-                self.embl_files[gbk_id] = os.path.abspath(embl_output.name)
+            genome = Genome.objects.get(id=gbk_id)
+            if genome.genomicisland_set.filter(method="sigi").exists():
+                self.logger.info("{} Sigi-HMM genomic islands found".format(genome.name))
+            else:
+                basename = os.path.splitext(os.path.basename(report["gbk_paths"][gbk_id]))[0]
+                with open(self.output_dir + str(report["analysis"]) + "/" + basename + ".embl", 'w') as embl_output:
+                    SeqIO.convert(report["gbk_paths"][gbk_id], "genbank", embl_output, "embl")
+                    self.embl_files[gbk_id] = os.path.abspath(embl_output.name)
 
     def analysis(self, report):
         script_file = NamedTemporaryFile(delete=True)
 
         for embl_id in self.embl_files:
             genome = Genome.objects.get(id=embl_id)
-            if not genome.genomicisland_set.filter(method__exact="sigi").exists():
-                sigi_gff_output = os.path.abspath(self.temp_dir_path + "/" + str(embl_id) + ".gff")
-                sigi_output = os.path.abspath(self.temp_dir_path +"/" + str(embl_id) + ".embl")
-                with open(script_file.name, 'w') as script:
-                    script.write("#!/bin/bash\n")
-                    script.write("/usr/bin/java " + self.SIGIHMM_EXE + " input=" +
-                                 self.embl_files[embl_id] + " output=" + sigi_output + " gff=" + sigi_gff_output)
-                    script.close()
+            sigi_gff_output = os.path.abspath(self.temp_dir_path + "/" + str(embl_id) + ".gff")
+            sigi_output = os.path.abspath(self.temp_dir_path +"/" + str(embl_id) + ".embl")
+            with open(script_file.name, 'w') as script:
+                script.write("#!/bin/bash\n")
+                script.write("/usr/bin/java " + self.SIGIHMM_EXE + " input=" +
+                             self.embl_files[embl_id] + " output=" + sigi_output + " gff=" + sigi_gff_output)
+                script.close()
 
-                os.chmod(script_file.name, 0o755)
-                script_file.file.close()
+            os.chmod(script_file.name, 0o755)
+            script_file.file.close()
 
-                with open(self.temp_dir_path + "/logs", 'w') as logs:
-                    subprocess.check_call(script_file.name, stdout=logs, cwd=self.SIGIHMM_PATH)
-                script_file.close()
+            with open(self.temp_dir_path + "/logs", 'w') as logs:
+                subprocess.check_call(script_file.name, stdout=logs, cwd=self.SIGIHMM_PATH)
+            script_file.close()
 
-                for sigi_gi in self.parse_sigi_gff(sigi_gff_output):
-                    GenomicIsland(
-                        method="sigi",
-                        start=sigi_gi[0],
-                        end=sigi_gi[1],
-                        genome=genome
-                    ).save()
+            for sigi_gi in self.parse_sigi_gff(sigi_gff_output):
+                GenomicIsland(
+                    method="sigi",
+                    start=sigi_gi[0],
+                    end=sigi_gi[1],
+                    genome=genome
+                ).save()
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
@@ -571,7 +574,9 @@ def analysis(self, report):
 
         for gbk_id in report["gbk_paths"]:
             genome = Genome.objects.get(id=gbk_id)
-            if not genome.genomicisland_set.filter(method__exact="islandpath").exists():
+            if genome.genomicisland_set.filter(method__exact="islandpath").exists():
+                self.logger.info("{} IslandPath genomic islands found".format(genome.name))
+            else:
                 script_file = NamedTemporaryFile(delete=True)
                 temp_path = self.temp_dir_path + "/" + str(gbk_id)
 
@@ -621,11 +626,11 @@ def merge_gi_queryset(self, gi_queryset):
                 if current_gi.end < next_gi.end:
                     current_gi.end = next_gi.end
             else:
-                current_gi.method = "merged"
+                current_gi.method = "merge"
                 current_gi.save() # Will create a new entry in the database because pk was set to None
                 current_gi = next_gi
                 current_gi.pk = None
-        current_gi.method = "merged"
+        current_gi.method = "merge"
         current_gi.save()
 
     def set_threshold(self, threshold):
@@ -635,7 +640,7 @@ def analysis(self, report):
         genomes = Genome.objects.filter(id__in=report['gbk_paths'].keys())
         for genome in genomes:
             gis = genome.genomicisland_set
-            if gis.filter(method__in=["sigi", "islandpath"]).exists() and not gis.filter(method="merged").exists():
+            if gis.filter(method__in=["sigi", "islandpath"]).exists() and not gis.filter(method="merge").exists():
                 self.merge_gi_queryset(gis.all())
 
 
@@ -712,7 +717,7 @@ def create_gi_fasta_files(self, report):
             os.mkdir(genome_fna_path)
 
             gi_counter = 0
-            for gi in Genome.objects.get(id=genome_id).genomicisland_set.filter(method="merged").all():
+            for gi in Genome.objects.get(id=genome_id).genomicisland_set.filter(method="merge").all():
                 self.logger.info("Adding GI: " + str(genome_id) + "-" + str(gi_counter))
                 entrySequence = self.get_subsequence(record, gi.start, gi.end, gi_counter)
                 self.writeFastaFile(self.fna_dir_path + "/" + str(genome_id) + "/" + str(gi_counter), entrySequence)
@@ -726,7 +731,7 @@ def assign_clusters(self, report, cluster_dict):
         analysis = Analysis.objects.get(id=report['analysis'])
         for genome_id in report['gbk_paths']:
             genome = Genome.objects.get(id=genome_id)
-            merged_gis = genome.genomicisland_set.filter(method="merged")
+            merged_gis = genome.genomicisland_set.filter(method="merge")
             for gi_index in range(len(merged_gis)):
                 cluster_index = cluster_dict[str(genome.id)][str(gi_index)]
                 cluster = analysis.genomicislandcluster_set.filter(number=cluster_index)
diff --git a/IslandCompare/analysis/integration_tests.py b/IslandCompare/analysis/integration_tests.py
index 1953adf9..243b48a2 100644
--- a/IslandCompare/analysis/integration_tests.py
+++ b/IslandCompare/analysis/integration_tests.py
@@ -312,13 +312,13 @@ def setUp(self):
         self.test_genome_2.save()
 
         self.test_gi_1 = GenomicIsland(
-            method="merged",
+            method="merge",
             start=0,
             end=1000,
             genome=self.test_genome_1
         ).save()
         self.test_gi_2 = GenomicIsland(
-            method="merged",
+            method="merge",
             start=0,
             end=1000,
             genome=self.test_genome_2
@@ -342,6 +342,8 @@ def test_generate_gi_fna(self):
 
         component.cleanup()
 
+    # TODO - implement MashMclClusterPipelineComponent test
+    @skip("Analysis object needed for Mash component")
     def test_analysis(self):
         component = MashMclClusterPipelineComponent()
 
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index 0205f6a8..05c09ec4 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -127,7 +127,6 @@ def run_pipeline_component(self, report, analysis_id, pipeline_component_name, p
 @shared_task()
 def sigi_error_handler(context, exc, traceback, pipeline_id):
     logger.info("Sigi HMM Failed! analysis id: {}".format(pipeline_id))
-    del context.args[0]["sigi_gis"]
     context.args[0]["failed_components"].append("sigi")
     task_chain = chain(run_pipeline_component.s(context.args[0], pipeline_id, "islandpath")
                            .on_error(ipath_error_handler.s(pipeline_id)),
@@ -139,7 +138,6 @@ def sigi_error_handler(context, exc, traceback, pipeline_id):
 @shared_task()
 def ipath_error_handler(context, exc, traceback, pipeline_id):
     logger.info("Islandpath Failed! analysis id: {}".format(pipeline_id))
-    del context.args[0]["islandpath_gis"]
     context.args[0]["failed_components"].append("islandpath")
     if "sigi_gis" in context.args[0]:
         return chain(run_pipeline_component.s(context.args[0], pipeline_id, "merge_gis"),
diff --git a/IslandCompare/analysis/tests_components.py b/IslandCompare/analysis/tests_components.py
index 234233d3..fd794ce0 100644
--- a/IslandCompare/analysis/tests_components.py
+++ b/IslandCompare/analysis/tests_components.py
@@ -323,8 +323,8 @@ def test_merge_single_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertTrue(self.test_genome.genomicisland_set.filter(method="merged").exists())
-        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
+        self.assertTrue(self.test_genome.genomicisland_set.filter(method="merge").exists())
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merge")), 1)
 
     def test_no_merge_gi_list(self):
         GenomicIsland(
@@ -353,7 +353,7 @@ def test_no_merge_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 3)
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merge")), 3)
 
     def test_merge_gi_list(self):
         GenomicIsland(
@@ -375,9 +375,9 @@ def test_merge_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
-        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].start, 0)
-        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].end, 1200)
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merge")), 1)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merge")[0].start, 0)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merge")[0].end, 1200)
 
     def test_merge_secondlist_gi_list(self):
         GenomicIsland(
@@ -399,9 +399,9 @@ def test_merge_secondlist_gi_list(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
-        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].start, 0)
-        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].end, 1200)
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merge")), 1)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merge")[0].start, 0)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merge")[0].end, 1200)
 
     def test_merge_encompassed_gi(self):
         GenomicIsland(
@@ -430,6 +430,6 @@ def test_merge_encompassed_gi(self):
         component.analysis(self.report)
         component.cleanup()
 
-        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merged")), 1)
-        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].start, 100)
-        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merged")[0].end, 5000)
+        self.assertEqual(len(self.test_genome.genomicisland_set.filter(method="merge")), 1)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merge")[0].start, 100)
+        self.assertEqual(self.test_genome.genomicisland_set.filter(method="merge")[0].end, 5000)
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 3818f255..166eaf92 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -665,17 +665,6 @@ function Backbone() {
         return -1;
     };
 
-    // Remove if unused
-    this.getIndexFromId = function (id) {
-        id = id.toString();
-        for (var index = 0; index < this.sequences.length; index++) {
-            if (this.sequences[index]['sequenceId'] == id) {
-                return index;
-            }
-        }
-        return -1;
-    };
-
     this.getIndicesFromIds = function(ids) {
         var adjuster = 0;
         var indices = [];

From a88fd0c9d007386190ed9e7b1b279822e499bc37 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 27 Jul 2017 15:59:59 -0700
Subject: [PATCH 47/76] Modified serializers to work with new GI and GI cluster
 database entries. Added AnalysisGenomicIslandRetrieveView

---
 IslandCompare/analysis/serializers.py | 96 ++++++++++++++-------------
 IslandCompare/analysis/urls.py        |  1 +
 IslandCompare/analysis/views.py       | 35 +++++++---
 web/pages/result.html                 |  4 +-
 4 files changed, 76 insertions(+), 60 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 113aebfd..b8222345 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -1,6 +1,7 @@
 from rest_framework import serializers
 from analysis.models import Analysis, AnalysisComponent, AnalysisType
 from genomes.models import Genome
+from genomes.serializers import GenomeSerializer
 from io import StringIO
 import csv
 from celery.result import AsyncResult
@@ -98,6 +99,7 @@ def validate(self, data):
                                                           name__exact=leaf.name)
                 if not selected_genome.exists():
                     raise serializers.ValidationError("Genome with Name: {} Does not Exist".format(leaf))
+        # if 'gi' in data.keys(): ... TODO - verification for user supplied GI file
         return data
 
     def create(self, validated_data):
@@ -123,16 +125,18 @@ def get_spaced_colors(n):
 
     def to_representation(self, instance):
         analysis = Analysis.objects.get(id=instance["analysis"])
-        genomes = analysis.genomes
 
         output = dict()
 
         output["genomes"] = dict()
-        gi_types = {"sigi_gis", "islandpath_gis", "merge_gis"}.intersection(instance)
-        if "cluster_gis" in instance:
-            number_clusters = instance["cluster_gis"]["numberClusters"]
-            color_index = self.get_spaced_colors(number_clusters)
-        for genome in genomes.all():
+        gi_types = ["sigi", "islandpath", "merge"]
+        if "numberClusters" in instance:
+            color_list = self.get_spaced_colors(instance["numberClusters"])
+        cluster_dict = {}
+        for cluster in analysis.genomicislandcluster_set.all():
+            for gi in cluster.genomic_islands.all():
+                cluster_dict[gi.id] = cluster.number
+        for genome in analysis.genomes.all():
             output["genomes"][genome.id] = dict()
             output["genomes"][genome.id]["name"] = genome.name
             output["genomes"][genome.id]["length"] = instance["gbk_metadata"][str(genome.id)]['size']
@@ -143,12 +147,12 @@ def to_representation(self, instance):
                 print(output["genomes"][genome.id]["genomic_islands"]["user"])
             else:
                 for gi_type in gi_types:
-                    output["genomes"][genome.id]["genomic_islands"][gi_type[:-4].replace("merge", "merged")] = [{'start': island[0], 'end': island[1]} for island in instance[gi_type][str(genome.id)]]
-            if "cluster_gis" in instance:
-                for gi_index in range(len(instance["merge_gis"][str(genome.id)])):
-                    clusters = instance['cluster_gis'][str(genome.id)]
-                    cluster_index = int(clusters[str(gi_index)])
-                    output["genomes"][genome.id]["genomic_islands"]["merged"][gi_index]['color'] = color_index[cluster_index]
+                    gis = genome.genomicisland_set.filter(method=gi_type)
+                    if gi_type == "merge":
+                        gi_dicts = [{'start': gi.start, 'end': gi.end, 'color': color_list[cluster_dict[gi.id]]} for gi in gis]
+                    else:
+                        gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
+                    output["genomes"][genome.id]["genomic_islands"][gi_type] = gi_dicts
 
         output["newick"] = instance["newick"]
         output["alignment"] = instance["alignment"]
@@ -170,36 +174,27 @@ class ReportCsvSerializer(serializers.BaseSerializer):
     """
     Serializer needed to return a csv file to the user
     """
-    def to_representation(self, instance):
+    def to_representation(self, analysis):
         output = StringIO()
         fieldnames = ['name', 'start', 'end', 'method', 'cluster_id']
         writer = csv.DictWriter(output, fieldnames=fieldnames)
         writer.writeheader()
 
-        method_keys = ["islandpath_gis", "sigi_gis"]
-
-        if "merge_gis" in instance:
-            for key in instance["gbk_paths"]:
-                counter = 0
-                for island in instance["merge_gis"][key]:
-                    genome = Genome.objects.get(id=key)
-                    writer.writerow({'name': genome.name,
-                                     'start': island[0],
-                                     'end': island[1],
-                                     'method': 'merge_gis',
-                                     'cluster_id': instance["cluster_gis"][str(genome.id)][str(counter)]
-                                     })
-                    counter += 1
+        method_keys = ["merge", "islandpath", "sigi"]
 
         for method in method_keys:
-            if method in instance:
-                for key in instance["gbk_paths"]:
-                    for island in instance[method][key]:
-                        genome = Genome.objects.get(id=key)
-                        writer.writerow({'name': genome.name,
-                                         'start': island[0],
-                                         'end': island[1],
-                                         'method': method})
+            for genome in analysis.genomes.all():
+                for island in genome.genomicisland_set.filter(method=method):
+                    row = {
+                        'name': genome.name,
+                        'start': island.start,
+                        'end': island.end,
+                        'method': method + "_gis"
+                    }
+                    if method == "merge":
+                        cluster = island.genomicislandcluster_set.get(analysis=analysis)
+                        row['cluster_id'] = cluster.number
+                    writer.writerow(row)
 
         contents = output.getvalue()
         output.close()
@@ -219,20 +214,18 @@ class ReportGeneCsvSerializer(serializers.BaseSerializer):
     """
     Serializer to return a csv of genes contained within GIs
     """
-    def to_representation(self, instance):
+    def to_representation(self, analysis):
         output = StringIO()
         fieldnames = ['genome', 'gene_name', 'locus_tag', 'product', 'start', 'end', 'strand', 'gi_start', 'gi_end', 'gi_method']
         writer = csv.DictWriter(output, fieldnames=fieldnames)
         writer.writeheader()
 
-        method_keys = ["merge_gis", "islandpath_gis", "sigi_gis"]
-        methods = [method for method in method_keys if method in instance]
+        method_keys = ["merge", "islandpath", "sigi"]
 
-        for key in instance["gbk_paths"]:
-            genome = Genome.objects.get(id=key)
-            for method in methods:
-                for island in instance[method][key]:
-                    for gi_gene in genome.gene_set.filter(start__gte=island[0]).filter(end__lte=island[1]):
+        for genome in analysis.genomes.all():
+            for method in method_keys:
+                for island in genome.genomicisland_set.filter(method=method):
+                    for gi_gene in genome.gene_set.filter(start__gte=island.start).filter(end__lte=island.end):
                         writer.writerow({'genome': genome.name,
                                          'gene_name': gi_gene.gene,
                                          'locus_tag': gi_gene.locus_tag,
@@ -240,11 +233,20 @@ def to_representation(self, instance):
                                          'start': gi_gene.start,
                                          'end': gi_gene.end,
                                          'strand': gi_gene.strand,
-                                         'gi_start': island[0],
-                                         'gi_end': island[1],
-                                         'gi_method': method})
+                                         'gi_start': island.start,
+                                         'gi_end': island.end,
+                                         'gi_method': method + "_gis"})
 
         contents = output.getvalue()
         output.close()
 
-        return contents
\ No newline at end of file
+        return contents
+
+class AnalysisGenomicIslandSerializer(serializers.Serializer):
+    """
+    Serializer for GenomicIsland objects
+    """
+    method = serializers.CharField(max_length=10)
+    start = serializers.IntegerField()
+    end = serializers.IntegerField()
+    genome = GenomeSerializer()
\ No newline at end of file
diff --git a/IslandCompare/analysis/urls.py b/IslandCompare/analysis/urls.py
index 10213fc8..66fe43d4 100644
--- a/IslandCompare/analysis/urls.py
+++ b/IslandCompare/analysis/urls.py
@@ -9,4 +9,5 @@
     url(r'^results/(?P<pk>[0-9]+)$', views.AnalysisResultsView.as_view(), name="analysis_results"),
     url(r'^run/', views.AnalysisRunView.as_view(), name="analysis_run"),
     url(r'^delete/(?P<pk>[0-9]+)$', views.AnalysisDestroyView.as_view(), name="analysis_destroy"),
+    url(r'^islands/(?P<pk>[0-9]+)$', views.AnalysisGenomicIslandRetrieveView.as_view(), name="analysis_islands"),
 ]
diff --git a/IslandCompare/analysis/views.py b/IslandCompare/analysis/views.py
index 64764736..607579af 100644
--- a/IslandCompare/analysis/views.py
+++ b/IslandCompare/analysis/views.py
@@ -2,7 +2,7 @@
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.views import APIView
 from analysis.serializers import AnalysisSerializer, RunAnalysisSerializer, ReportCsvSerializer, \
-    ReportVisualizationOverviewSerializer, ReportGeneCsvSerializer
+    ReportVisualizationOverviewSerializer, ReportGeneCsvSerializer, AnalysisGenomicIslandSerializer
 from analysis.models import Analysis, AnalysisComponent
 from rest_framework.response import Response
 from analysis.pipeline import Pipeline
@@ -10,6 +10,7 @@
 from analysis.tasks import run_pipeline_wrapper
 from celery.result import AsyncResult
 from rest_framework.parsers import FormParser, MultiPartParser
+from genomes.models import GenomicIsland
 
 
 
@@ -138,7 +139,7 @@ def retrieve(self, request, *args, **kwargs):
 
 class ExportAnalysisResultView(generics.RetrieveAPIView):
     """
-    Retrieve a CSV of the Analysis
+    Retrieve a CSV of the Analysis GIs
     """
     permission_classes = [IsAuthenticated]
     serializer_class = AnalysisSerializer
@@ -151,10 +152,7 @@ def retrieve(self, request, *args, **kwargs):
         task = AsyncResult(analysis.celery_task_id)
 
         if task.status == 'SUCCESS':
-            end_pipeline = AnalysisComponent.objects.get(analysis=analysis,
-                                                         type__name="end_pipeline")
-            result = AsyncResult(end_pipeline.celery_task_id).get()
-            return Response(ReportCsvSerializer(result).data)
+            return Response(ReportCsvSerializer(analysis).data)
         elif task.status == 'FAILURE':
             response = Response()
             response.status_code = 500
@@ -177,10 +175,7 @@ def retrieve(self, request, *args, **kwargs):
         task = AsyncResult(analysis.celery_task_id)
 
         if task.status == 'SUCCESS':
-            end_pipeline = AnalysisComponent.objects.get(analysis=analysis,
-                                                         type__name="end_pipeline")
-            result = AsyncResult(end_pipeline.celery_task_id).get()
-            return Response(ReportGeneCsvSerializer(result).data)
+            return Response(ReportGeneCsvSerializer(analysis).data)
         elif task.status == 'FAILURE':
             response = Response()
             response.status_code = 500
@@ -190,4 +185,22 @@ def retrieve(self, request, *args, **kwargs):
             response = Response()
             response.status_code = 202
             response.data = {'content': "Job has not completed"}
-            return response
\ No newline at end of file
+            return response
+
+class AnalysisGenomicIslandRetrieveView(generics.RetrieveAPIView):
+    """
+    Retrieve GIs of the analysis
+    """
+    permission_classes = [IsAuthenticated]
+    serializer_class = AnalysisGenomicIslandSerializer
+
+    def get_queryset(self):
+        return Analysis.objects.filter(owner=self.request.user)
+
+    def retrieve(self, request, *args, **kwargs):
+        analysis = Analysis.objects.get(id=kwargs['pk'])
+        gis = GenomicIsland.objects.filter(genome__in=analysis.genomes.all())
+
+        serializer = self.serializer_class(gis, many=True)
+
+        return Response(serializer.data)
\ No newline at end of file
diff --git a/web/pages/result.html b/web/pages/result.html
index 4f2adc01..5fd2db15 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -196,8 +196,8 @@ <h1>
                         if ("sigi" in data["genomes"][genomeId]["genomic_islands"]) {
                             currentSeq.addGI("sigi", data["genomes"][genomeId]["genomic_islands"]["sigi"]);
                         }
-                        if ("merged" in data["genomes"][genomeId]["genomic_islands"]) {
-                            currentSeq.addGI("merged", data["genomes"][genomeId]["genomic_islands"]["merged"]);
+                        if ("merge" in data["genomes"][genomeId]["genomic_islands"]) {
+                            currentSeq.addGI("merged", data["genomes"][genomeId]["genomic_islands"]["merge"]);
                         }
                     }
 

From e9e4b412417c5ee1634591b080324115a8a24e05 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 28 Jul 2017 15:10:25 -0700
Subject: [PATCH 48/76] Added genomes GenomicIslandRetrieveView

---
 IslandCompare/genomes/serializers.py | 14 +++++++++++++-
 IslandCompare/genomes/urls.py        |  1 +
 IslandCompare/genomes/views.py       | 26 +++++++++++++++++++++++++-
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index bececd5f..1f5594b0 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -10,6 +10,10 @@ class GenomeSerializer(serializers.ModelSerializer):
     Serializer for genomes submitted by the user.
     Ensures that genome file names are unique and that gbk files are valid
     """
+    id = serializers.IntegerField()
+    name = serializers.CharField(max_length=100)
+    gbk = serializers.FileField()
+
     class Meta:
         model = Genome
         fields = ('id', 'name', 'gbk')
@@ -149,4 +153,12 @@ class GeneSerializer(serializers.Serializer):
     product = serializers.CharField(max_length=200)
     start = serializers.IntegerField()
     end = serializers.IntegerField()
-    strand = serializers.IntegerField()
\ No newline at end of file
+    strand = serializers.IntegerField()
+
+class GenomicIslandSerializer(serializers.Serializer):
+    """
+    Serializer for GenomicIsland objects
+    """
+    method = serializers.CharField(max_length=10)
+    start = serializers.IntegerField()
+    end = serializers.IntegerField()
\ No newline at end of file
diff --git a/IslandCompare/genomes/urls.py b/IslandCompare/genomes/urls.py
index cb6ff191..531d7cc9 100644
--- a/IslandCompare/genomes/urls.py
+++ b/IslandCompare/genomes/urls.py
@@ -6,4 +6,5 @@
     url(r'^genes/(?P<pk>[0-9]+)$', views.GenomeGeneRetrieveView.as_view(), name="genome_genes"),
     url(r'^upload/$', views.GenomeUploadView.as_view(), name="genome_upload"),
     url(r'^details/(?P<pk>[0-9]+)$', views.GenomeRetrieveUpdateDestroyView.as_view(), name="genome_details"),
+    url(r'^islands/(?P<pk>[0-9]+)$', views.GenomicIslandRetrieveView.as_view(), name="genomic_islands")
 ]
diff --git a/IslandCompare/genomes/views.py b/IslandCompare/genomes/views.py
index df6940d2..d18d331b 100644
--- a/IslandCompare/genomes/views.py
+++ b/IslandCompare/genomes/views.py
@@ -1,6 +1,6 @@
 from rest_framework import generics, response
 from rest_framework.permissions import IsAuthenticated
-from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GeneSerializer
+from genomes.serializers import GenomeSerializer, GenomeUploadSerializer, GeneSerializer, GenomicIslandSerializer
 from genomes.models import Genome
 from rest_framework.parsers import MultiPartParser, FormParser
 from rest_framework.response import Response
@@ -86,3 +86,27 @@ def retrieve(self, request, *args, **kwargs):
         serializer = GeneSerializer(genes, many=True)
 
         return Response({'genes': serializer.data})
+
+class GenomicIslandRetrieveView(generics.RetrieveAPIView):
+    """
+    Retrieve Genomic Islands for a Genome
+    """
+    permission_classes = [IsAuthenticated]
+    serializer_class = GenomicIslandSerializer
+
+    def get_queryset(self):
+        return Genome.objects.filter(owner=self.request.user)
+
+    def retrieve(self, request, *args, **kwargs):
+        url_queries = self.request.query_params
+        genome = Genome.objects.get(id=kwargs['pk'])
+        gis = genome.genomicisland_set.all()
+
+        if 'start' in url_queries:
+            gis = gis.filter(end__gte=url_queries['start'])
+        if 'end' in url_queries:
+            gis = gis.filter(start__lte=url_queries['end'])
+
+        serializer = self.serializer_class(gis, many=True)
+
+        return Response(serializer.data)
\ No newline at end of file

From 58fa4ec6c5db3216bc38a6ce33ab6756ba43996a Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 31 Jul 2017 11:11:09 -0700
Subject: [PATCH 49/76] Clusters moved to a single TextField under Analysis to
 reduce number of queries on result visualization

---
 IslandCompare/analysis/components.py          | 21 ++-----------
 .../migrations/0003_analysis_clusters.py      | 20 +++++++++++++
 IslandCompare/analysis/models.py              |  1 +
 IslandCompare/analysis/serializers.py         | 16 +++++-----
 .../genomes/migrations/0005_gicluster.py      | 30 -------------------
 IslandCompare/genomes/models.py               |  9 ------
 6 files changed, 31 insertions(+), 66 deletions(-)
 create mode 100644 IslandCompare/analysis/migrations/0003_analysis_clusters.py
 delete mode 100644 IslandCompare/genomes/migrations/0005_gicluster.py

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index d74dc9a3..aaf95342 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -726,23 +726,6 @@ def create_gi_fasta_files(self, report):
 
         return island_path_list
 
-    def assign_clusters(self, report, cluster_dict):
-        self.logger.info("Saving GI clusters")
-        analysis = Analysis.objects.get(id=report['analysis'])
-        for genome_id in report['gbk_paths']:
-            genome = Genome.objects.get(id=genome_id)
-            merged_gis = genome.genomicisland_set.filter(method="merge")
-            for gi_index in range(len(merged_gis)):
-                cluster_index = cluster_dict[str(genome.id)][str(gi_index)]
-                cluster = analysis.genomicislandcluster_set.filter(number=cluster_index)
-                if cluster.exists():
-                    cluster[0].genomic_islands.add(merged_gis[gi_index])
-                else:
-                    merged_gis[gi_index].genomicislandcluster_set.create(
-                        number=cluster_index,
-                        analysis=analysis
-                    )
-
     def setup(self, report):
         self.temp_dir_path = self.output_dir + str(report["analysis"])
         os.mkdir(self.temp_dir_path, 0o777)
@@ -800,7 +783,9 @@ def analysis(self, report):
             islandIdList = list(remainingIslands)
 
         report['numberClusters'] = numberClusters - 1
-        self.assign_clusters(report, output_dict)
+        analysis = Analysis.objects.get(id=report['analysis'])
+        analysis.clusters = str(output_dict)
+        analysis.save()
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
diff --git a/IslandCompare/analysis/migrations/0003_analysis_clusters.py b/IslandCompare/analysis/migrations/0003_analysis_clusters.py
new file mode 100644
index 00000000..87425b55
--- /dev/null
+++ b/IslandCompare/analysis/migrations/0003_analysis_clusters.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-07-31 17:44
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analysis', '0002_auto_20170718_1718'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='analysis',
+            name='clusters',
+            field=models.TextField(blank=True),
+        ),
+    ]
diff --git a/IslandCompare/analysis/models.py b/IslandCompare/analysis/models.py
index 7326b9b5..1e92ab61 100644
--- a/IslandCompare/analysis/models.py
+++ b/IslandCompare/analysis/models.py
@@ -13,6 +13,7 @@ class Analysis(models.Model):
     start_time = models.DateTimeField(null=True)
     complete_time = models.DateTimeField(null=True)
     owner = models.ForeignKey(User)
+    clusters = models.TextField(blank=True)
 
     class Meta:
         unique_together = ('name', 'owner')
diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index b8222345..c9ef53dd 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -3,7 +3,7 @@
 from genomes.models import Genome
 from genomes.serializers import GenomeSerializer
 from io import StringIO
-import csv
+import csv, ast
 from celery.result import AsyncResult
 from Bio import Phylo
 
@@ -132,10 +132,7 @@ def to_representation(self, instance):
         gi_types = ["sigi", "islandpath", "merge"]
         if "numberClusters" in instance:
             color_list = self.get_spaced_colors(instance["numberClusters"])
-        cluster_dict = {}
-        for cluster in analysis.genomicislandcluster_set.all():
-            for gi in cluster.genomic_islands.all():
-                cluster_dict[gi.id] = cluster.number
+            clusters = ast.literal_eval(analysis.clusters)
         for genome in analysis.genomes.all():
             output["genomes"][genome.id] = dict()
             output["genomes"][genome.id]["name"] = genome.name
@@ -148,11 +145,12 @@ def to_representation(self, instance):
             else:
                 for gi_type in gi_types:
                     gis = genome.genomicisland_set.filter(method=gi_type)
-                    if gi_type == "merge":
-                        gi_dicts = [{'start': gi.start, 'end': gi.end, 'color': color_list[cluster_dict[gi.id]]} for gi in gis]
-                    else:
-                        gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
+                    gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
                     output["genomes"][genome.id]["genomic_islands"][gi_type] = gi_dicts
+                if "merge" in output["genomes"][genome.id]["genomic_islands"]:
+                    for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"]["merge"])):
+                        cluster_index = int(clusters[str(genome.id)][str(gi_index)])
+                        output["genomes"][genome.id]["genomic_islands"]["merge"][gi_index]["color"] = color_list[cluster_index]
 
         output["newick"] = instance["newick"]
         output["alignment"] = instance["alignment"]
diff --git a/IslandCompare/genomes/migrations/0005_gicluster.py b/IslandCompare/genomes/migrations/0005_gicluster.py
deleted file mode 100644
index fbe21f1b..00000000
--- a/IslandCompare/genomes/migrations/0005_gicluster.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.10.5 on 2017-07-26 23:18
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analysis', '0002_auto_20170718_1718'),
-        ('genomes', '0004_genomicisland'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='GenomicIslandCluster',
-            fields=[
-                ('id', models.AutoField(primary_key=True, serialize=False)),
-                ('number', models.IntegerField()),
-                ('analysis', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analysis.Analysis')),
-                ('genomic_islands', models.ManyToManyField(to='genomes.GenomicIsland')),
-            ],
-        ),
-        migrations.AlterUniqueTogether(
-            name='genomicislandcluster',
-            unique_together=set([('number', 'analysis')]),
-        ),
-    ]
diff --git a/IslandCompare/genomes/models.py b/IslandCompare/genomes/models.py
index 1eea2693..0c119b0f 100644
--- a/IslandCompare/genomes/models.py
+++ b/IslandCompare/genomes/models.py
@@ -35,12 +35,3 @@ class GenomicIsland(models.Model):
     start = models.IntegerField()
     end = models.IntegerField()
     genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
-
-class GenomicIslandCluster(models.Model):
-    id = models.AutoField(primary_key=True)
-    number = models.IntegerField()
-    analysis = models.ForeignKey('analysis.Analysis')
-    genomic_islands = models.ManyToManyField(GenomicIsland)
-
-    class Meta:
-        unique_together = ('number', 'analysis')
\ No newline at end of file

From 1f69e8923bf8601361e90badd6ddec0563ff3200 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 1 Aug 2017 11:24:45 -0700
Subject: [PATCH 50/76] Improved error catching and handling

---
 IslandCompare/analysis/components.py     | 181 +++++++++++++----------
 IslandCompare/analysis/pipeline.py       |   2 +-
 IslandCompare/analysis/serializers.py    |   5 +-
 IslandCompare/analysis/tasks.py          |  13 +-
 IslandCompare/analysis/tests_pipeline.py |   2 +-
 IslandCompare/genomes/serializers.py     |   1 -
 web/pages/result.html                    |  55 +++++--
 7 files changed, 152 insertions(+), 107 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index aaf95342..428fc848 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -10,7 +10,6 @@
 from io import StringIO
 import csv
 from datetime import datetime
-import copy
 from Bio.SeqRecord import SeqRecord
 import numpy as np
 from analysis.lib.mcl_clustering import mcl
@@ -450,7 +449,7 @@ def cleanup(self):
 
 class SigiHMMPipelineComponent(PipelineComponent):
     """
-    Pipeline component that runs and adds sigi-hmm data for each genome in the report to the report.
+    Pipeline component that runs and adds sigi-hmm data for each genome in the report to the database.
     """
     name = "sigi"
     dependencies = ["gbk_paths"]
@@ -459,6 +458,7 @@ class SigiHMMPipelineComponent(PipelineComponent):
     embl_files = {}
     SIGIHMM_PATH = settings.SIGIHMM_PATH
     SIGIHMM_EXE = settings.SIGIHMM_EXE
+    exception = None
 
     @staticmethod
     def parse_sigi_gff(gff_file):
@@ -477,12 +477,15 @@ def parse_sigi_gff(gff_file):
                     cleaned_line = ' '.join(line.split())
                     gene_dict = cleaned_line.split(' ')
                     # at the start of a genomic island, set start and end of possible genomic island
-                    if gene_dict[2] == 'PUTAL' and not island_flag:
+                    # some results contain many islands that start at one and extend very far into the genome
+                    # this is presumed to be an error, so these results will be skipped
+                    if gene_dict[2] == 'PUTAL' and not island_flag and gene_dict[3] != '1':
                         start = gene_dict[3]
                         end = gene_dict[4]
                         island_flag = True
                     # continuation of current genomic island, change end and continue
-                    elif gene_dict[2] == 'PUTAL' and island_flag:
+                    # some results have an end position of 0. These results will be skipped
+                    elif gene_dict[2] == 'PUTAL' and island_flag and int(gene_dict[4]) > int(start):
                         end = gene_dict[4]
                     # end of genomic island, append current start and end to list
                     elif island_flag:
@@ -527,26 +530,35 @@ def analysis(self, report):
             os.chmod(script_file.name, 0o755)
             script_file.file.close()
 
-            with open(self.temp_dir_path + "/logs", 'w') as logs:
-                subprocess.check_call(script_file.name, stdout=logs, cwd=self.SIGIHMM_PATH)
-            script_file.close()
+            try:
+                with open(self.temp_dir_path + "/logs", 'w') as logs:
+                    subprocess.check_call(script_file.name, stdout=logs, cwd=self.SIGIHMM_PATH)
+                script_file.close()
 
-            for sigi_gi in self.parse_sigi_gff(sigi_gff_output):
-                GenomicIsland(
-                    method="sigi",
-                    start=sigi_gi[0],
-                    end=sigi_gi[1],
-                    genome=genome
-                ).save()
+                for sigi_gi in self.parse_sigi_gff(sigi_gff_output):
+                    GenomicIsland(method="sigi",
+                                  start=sigi_gi[0],
+                                  end=sigi_gi[1],
+                                  genome=genome
+                                  ).save()
+            except subprocess.CalledProcessError as err:
+                self.logger.info("Sigi-HMM Failed! analysis {}, genome {}".format(report["analysis"], embl_id))
+                self.exception = err
+                if "sigi" not in report["failed_components"]:
+                    report["failed_components"]["sigi"] = [genome.name]
+                else:
+                    report["failed_components"]["sigi"].append(genome.name)
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
             rmtree(self.temp_dir_path)
+        if self.exception:
+            raise self.exception
 
 
 class IslandPathPipelineComponent(PipelineComponent):
     """
-    Pipeline component that runs and adds islandpath data to the report.
+    Pipeline component that runs and adds islandpath data to the database.
     """
     name = "islandpath"
     dependencies = ["gbk_paths"]
@@ -554,6 +566,7 @@ class IslandPathPipelineComponent(PipelineComponent):
     ISLANDPATH_PATH = settings.ISLANDPATH_PATH
     log_path = None
     temp_dir_path = None
+    exception = None
 
     @staticmethod
     def parse_islandpath(islandpath_file):
@@ -574,7 +587,7 @@ def analysis(self, report):
 
         for gbk_id in report["gbk_paths"]:
             genome = Genome.objects.get(id=gbk_id)
-            if genome.genomicisland_set.filter(method__exact="islandpath").exists():
+            if genome.genomicisland_set.filter(method="islandpath").exists():
                 self.logger.info("{} IslandPath genomic islands found".format(genome.name))
             else:
                 script_file = NamedTemporaryFile(delete=True)
@@ -591,26 +604,35 @@ def analysis(self, report):
                 script_file.file.close()
 
                 self.log_path = self.temp_dir_path + "/logs_" + str(report["analysis"])
-                with open(self.log_path, 'w') as logs:
-                    subprocess.check_call(script_file.name, stdout=logs, cwd=self.temp_dir_path)
-                script_file.close()
-
-                for islandpath_gi in self.parse_islandpath(temp_path):
-                    GenomicIsland(
-                        method="islandpath",
-                        start=islandpath_gi[0],
-                        end=islandpath_gi[1],
-                        genome=genome
-                    ).save()
+                try:
+                    with open(self.log_path, 'w') as logs:
+                        subprocess.check_call(script_file.name, stdout=logs, cwd=self.temp_dir_path)
+                    script_file.close()
+
+                    for islandpath_gi in self.parse_islandpath(temp_path):
+                        GenomicIsland(method="islandpath",
+                                      start=islandpath_gi[0],
+                                      end=islandpath_gi[1],
+                                      genome=genome
+                                      ).save()
+                except subprocess.CalledProcessError as err:
+                    self.logger.info("IslandPath Failed! analysis {}, genome {}".format(report["analysis"], gbk_id))
+                    self.exception = err
+                    if "islandpath" not in report["failed_components"]:
+                        report["failed_components"]["islandpath"] = [genome.name]
+                    else:
+                        report["failed_components"]["islandpath"].append(genome.name)
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
             rmtree(self.temp_dir_path)
+        if self.exception:
+            raise self.exception
 
 
 class MergeIslandsPipelineComponent(PipelineComponent):
     """
-    Pipeline component that merges islands and adds the merged islands to the report.
+    Pipeline component that merges islands and adds the merged islands to the database.
     """
     name = "merge_gis"
     threshold = 500
@@ -735,57 +757,58 @@ def setup(self, report):
 
     def analysis(self, report):
         island_path_list = self.create_gi_fasta_files(report)
-        self.create_compound_sketch(island_path_list, self.temp_dir_path + "/compoundScratch")
-
-        distance_matrix = []
-
-        for island in island_path_list:
-            self.logger.info("Processing island: " + island)
-            if os.path.isfile(self.temp_dir_path + "/output"):
-                os.remove(self.temp_dir_path + "/output")
-            self.calculate_mash_distance(self.temp_dir_path + "/compoundScratch.msh", island, self.temp_dir_path + "/output")
-            distance_matrix.append([float(i['mashDistance']) for i in self.parse_mash_output(self.temp_dir_path + "/output")])
-
-        baseMatrix = []
-        for i in range(len(island_path_list)):
-            nextRow = []
-            for j in range(len(island_path_list)):
-                nextRow.append(1)
-            baseMatrix.append(nextRow)
-        numpyBaseMatrix = np.array(baseMatrix)
-        numpyDistanceMatrix = np.array(distance_matrix)
-
-        mclAdjacencyMatrix = np.subtract(numpyBaseMatrix, numpyDistanceMatrix)
-        np.set_printoptions(threshold='nan')
-
-        M, clusters = mcl(mclAdjacencyMatrix)
-        output_dict = {}
-
-        for genome_id in report["gbk_paths"]:
-            output_dict[str(genome_id)] = {}
-        islandIdList = list([i for i in range(len(island_path_list))])
-
-        numberClusters = 0
-
-        while len(islandIdList) > 0:
-            numberClusters += 1
-            currentCluster = clusters[islandIdList[0]]
-            for i in currentCluster:
-                self.logger.info("Assigning clusters for cluster {}".format(numberClusters))
-                island = island_path_list[i]
-                splitIsland = island.split('/')[-2:]
-                currentSequenceId = splitIsland[0]
-                islandId = splitIsland[1]
-                self.logger.info("Current Sequence Id: " + str(currentSequenceId))
-                self.logger.info("Current Island Id: " + str(islandId))
-                output_dict[str(currentSequenceId)][str(islandId)] = numberClusters - 1
-            remainingIslands = filter(lambda x: x not in currentCluster, islandIdList)
-            islandIdList = list(remainingIslands)
-
-        report['numberClusters'] = numberClusters - 1
-        analysis = Analysis.objects.get(id=report['analysis'])
-        analysis.clusters = str(output_dict)
-        analysis.save()
+        if len(island_path_list) > 0:
+            self.create_compound_sketch(island_path_list, self.temp_dir_path + "/compoundScratch")
+
+            distance_matrix = []
+
+            for island in island_path_list:
+                self.logger.info("Processing island: " + island)
+                if os.path.isfile(self.temp_dir_path + "/output"):
+                    os.remove(self.temp_dir_path + "/output")
+                self.calculate_mash_distance(self.temp_dir_path + "/compoundScratch.msh", island, self.temp_dir_path + "/output")
+                distance_matrix.append([float(i['mashDistance']) for i in self.parse_mash_output(self.temp_dir_path + "/output")])
+
+            baseMatrix = []
+            for i in range(len(island_path_list)):
+                nextRow = []
+                for j in range(len(island_path_list)):
+                    nextRow.append(1)
+                baseMatrix.append(nextRow)
+            numpyBaseMatrix = np.array(baseMatrix)
+            numpyDistanceMatrix = np.array(distance_matrix)
+
+            mclAdjacencyMatrix = np.subtract(numpyBaseMatrix, numpyDistanceMatrix)
+            np.set_printoptions(threshold='nan')
+
+            M, clusters = mcl(mclAdjacencyMatrix)
+            output_dict = {}
+
+            for genome_id in report["gbk_paths"]:
+                output_dict[str(genome_id)] = {}
+            islandIdList = list([i for i in range(len(island_path_list))])
+
+            numberClusters = 0
+
+            while len(islandIdList) > 0:
+                numberClusters += 1
+                currentCluster = clusters[islandIdList[0]]
+                for i in currentCluster:
+                    self.logger.info("Assigning clusters for cluster {}".format(numberClusters))
+                    island = island_path_list[i]
+                    splitIsland = island.split('/')[-2:]
+                    currentSequenceId = splitIsland[0]
+                    islandId = splitIsland[1]
+                    self.logger.info("Current Sequence Id: " + str(currentSequenceId))
+                    self.logger.info("Current Island Id: " + str(islandId))
+                    output_dict[str(currentSequenceId)][str(islandId)] = numberClusters - 1
+                remainingIslands = filter(lambda x: x not in currentCluster, islandIdList)
+                islandIdList = list(remainingIslands)
+
+            report['numberClusters'] = numberClusters
+            analysis = Analysis.objects.get(id=report['analysis'])
+            analysis.clusters = str(output_dict)
+            analysis.save()
 
     def cleanup(self):
         if self.temp_dir_path is not None and os.path.exists(self.temp_dir_path):
diff --git a/IslandCompare/analysis/pipeline.py b/IslandCompare/analysis/pipeline.py
index f3fb8c53..8d1d0f8f 100644
--- a/IslandCompare/analysis/pipeline.py
+++ b/IslandCompare/analysis/pipeline.py
@@ -143,7 +143,7 @@ def __init__(self):
         self.pipeline_components = []
         self.available_dependencies = []
         self.analysis = None
-        self.failed_components = []
+        self.failed_components = {}
 
     def create_database_entry(self, name, genomes, owner):
         """
diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index c9ef53dd..6f1a9eee 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -145,8 +145,9 @@ def to_representation(self, instance):
             else:
                 for gi_type in gi_types:
                     gis = genome.genomicisland_set.filter(method=gi_type)
-                    gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
-                    output["genomes"][genome.id]["genomic_islands"][gi_type] = gi_dicts
+                    if gis.exists():
+                        gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
+                        output["genomes"][genome.id]["genomic_islands"][gi_type] = gi_dicts
                 if "merge" in output["genomes"][genome.id]["genomic_islands"]:
                     for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"]["merge"])):
                         cluster_index = int(clusters[str(genome.id)][str(gi_index)])
diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index 05c09ec4..ccbf87a8 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -124,10 +124,9 @@ def run_pipeline_component(self, report, analysis_id, pipeline_component_name, p
         else:
             raise
 
+# Error handlers are used here because if exceptions are handled in the component, status will be 'SUCCESS'
 @shared_task()
 def sigi_error_handler(context, exc, traceback, pipeline_id):
-    logger.info("Sigi HMM Failed! analysis id: {}".format(pipeline_id))
-    context.args[0]["failed_components"].append("sigi")
     task_chain = chain(run_pipeline_component.s(context.args[0], pipeline_id, "islandpath")
                            .on_error(ipath_error_handler.s(pipeline_id)),
                        run_pipeline_component.s(pipeline_id, "merge_gis"),
@@ -137,10 +136,6 @@ def sigi_error_handler(context, exc, traceback, pipeline_id):
 
 @shared_task()
 def ipath_error_handler(context, exc, traceback, pipeline_id):
-    logger.info("Islandpath Failed! analysis id: {}".format(pipeline_id))
-    context.args[0]["failed_components"].append("islandpath")
-    if "sigi_gis" in context.args[0]:
-        return chain(run_pipeline_component.s(context.args[0], pipeline_id, "merge_gis"),
-                     run_pipeline_component.s(pipeline_id, "mash_mcl"),
-                     run_pipeline_component.s(pipeline_id, "end_pipeline")).apply_async()
-    return run_pipeline_component.s(context.args[0], pipeline_id, "end_pipeline").apply_async()
+    return chain(run_pipeline_component.s(context.args[0], pipeline_id, "merge_gis"),
+                 run_pipeline_component.s(pipeline_id, "mash_mcl"),
+                 run_pipeline_component.s(pipeline_id, "end_pipeline")).apply_async()
diff --git a/IslandCompare/analysis/tests_pipeline.py b/IslandCompare/analysis/tests_pipeline.py
index 78000d99..f85cda26 100644
--- a/IslandCompare/analysis/tests_pipeline.py
+++ b/IslandCompare/analysis/tests_pipeline.py
@@ -161,7 +161,7 @@ class PipelineSerializerTestCase(TestCase):
     test_component = PipelineComponentStub()
     test_component.name = "component_1"
     test_pipeline_components = [test_component]
-    test_failed_components = ['failure_1']
+    test_failed_components = {'failure_1': 1}
 
     test_username = "test_username"
     test_user = None
diff --git a/IslandCompare/genomes/serializers.py b/IslandCompare/genomes/serializers.py
index 1f5594b0..99e374fb 100644
--- a/IslandCompare/genomes/serializers.py
+++ b/IslandCompare/genomes/serializers.py
@@ -10,7 +10,6 @@ class GenomeSerializer(serializers.ModelSerializer):
     Serializer for genomes submitted by the user.
     Ensures that genome file names are unique and that gbk files are valid
     """
-    id = serializers.IntegerField()
     name = serializers.CharField(max_length=100)
     gbk = serializers.FileField()
 
diff --git a/web/pages/result.html b/web/pages/result.html
index 5fd2db15..53eec9df 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -107,13 +107,13 @@ <h1>
                     </div>
                     <div class="predictorLegend clearfix" hidden>
                         <span id="sigiLegend" class="legendText">
-                            <svg id="sigiLegend" height="10" width="10">
+                            <svg height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
                             Sigi-HMM Genomic Island
                         </span>
                         <span id="islandpathLegend" class="legendText">
-                            <svg id="islandpathLegend" height="10" width="10">
+                            <svg height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
                             IslandPath Genomic Island
@@ -148,24 +148,38 @@ <h1>
                 request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
             },
             success: function(data) {
-                if (data["failed_components"].length !== 0) {
+                var sigiFails = 0;
+                var islandPathFails = 0;
+                if (Object.keys(data["failed_components"]).length) {
                     var warning_message = "<h4>Warning:</h4>";
-                    if ($.inArray("sigi",data["failed_components"]) !== -1) {
-                        warning_message = warning_message.concat("<p>Sigi-HMM Failed</p>");
-                        $("#sigiToggle").hide();
-                        $("#sigiLegend").remove();
-                    }
-                    if ($.inArray("islandpath",data["failed_components"]) !== -1) {
-                        warning_message = warning_message.concat("<p>IslandPath Failed</p>");
-                        $("#islandpathToggle").hide();
-                        $("#islandpathLegend").remove();
+                    if ("sigi" in data["failed_components"]) {
+                        warning_message = warning_message.concat("<p>Sigi-HMM Failed ");
+                        sigiFails = data["failed_components"]["sigi"].length;
+                        if (sigiFails === 1) {
+                            warning_message = warning_message.concat("for genome " + data["failed_components"]["sigi"][0]);
+                        } else {
+                            warning_message = warning_message.concat("for genomes ");
+                            var failed_genomes = data["failed_components"]["sigi"];
+                            warning_message = warning_message.concat(failed_genomes.join(", "));
+                        }
+                        warning_message = warning_message.concat("</p>");
                     }
-                    if (data["failed_components"].length === 2) {
-                        $(".GIColour").hide();
+                    if ("islandpath" in data["failed_components"]) {
+                        warning_message = warning_message.concat("<p>IslandPath Failed ");
+                        islandPathFails = data["failed_components"]["islandpath"].length;
+                        if (islandPathFails === 1) {
+                            warning_message = warning_message.concat("for genome " + data["failed_components"]["islandpath"][0]);
+                        } else {
+                            warning_message = warning_message.concat("for genomes ");
+                            var failed_genomes = data["failed_components"]["islandpath"];
+                            warning_message = warning_message.concat(failed_genomes.join(", "));
+                        }
+                        warning_message = warning_message.concat("</p>");
                     }
                     $("#warning-box").html(warning_message)
                         .show();
                 }
+
                 // Add the newick data to generate a tree
                 container.newickData = Newick.parse(data["newick"]);
                 // Set the root of the tree
@@ -174,6 +188,19 @@ <h1>
                 // Traverse tree to find sequence id order
                 var treeOrder = container.traverseTreeForOrder(container.newickRoot);
 
+                // Hide irrelevant legends and controls
+                if (islandPathFails === treeOrder.length) {
+                    $("#islandpathToggle").hide();
+                    $("#islandpathLegend").remove();
+                }
+                if (sigiFails === treeOrder.length) {
+                    $("#sigiToggle").hide();
+                    $("#sigiLegend").remove();
+                }
+                if (islandPathFails === treeOrder.length && sigiFails === treeOrder.length) {
+                    $(".GIColour").hide();
+                }
+
                 // Add Genomes to Visualization
                 for (var i = 0; i < treeOrder.length; i++) {
                     var genomeId = treeOrder[i];

From d0fbd2c7637974923cdb592b2a87f8f93118c886 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 1 Aug 2017 14:22:07 -0700
Subject: [PATCH 51/76] Added validation for user provided GI files

---
 IslandCompare/analysis/serializers.py | 22 ++++++++++++++++++++--
 IslandCompare/analysis/views.py       |  5 +----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 8253ec02..23309db9 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -99,8 +99,26 @@ def validate(self, data):
                 selected_genome = selected_genomes.filter(owner__exact=self.context['request'].user,
                                                           name__exact=leaf.name)
                 if not selected_genome.exists():
-                    raise serializers.ValidationError("Genome with Name: {} Does not Exist".format(leaf))
-        # if 'gi' in data.keys(): ... TODO - verification for user supplied GI file
+                    raise serializers.ValidationError("Genome: {} not included in this analysis".format(leaf))
+        if 'gi' in data.keys():
+            selected_genomes = [genome.name for genome in data['genomes']]
+            formatted_lines = []
+
+            # Replace carriage returns to avoid decode errors
+            user_gis = re.sub(b'\r\n?', b'\n', data["gi"].read())
+            # Replace problematic file extensions
+            user_gis = re.sub(b'.genbank|.gbff', b'.gbk', user_gis)
+            for line in user_gis.decode("utf-8").split("\n"):
+                if line is not "":
+                    formatted_lines.append(line)
+                    columns = line.split("\t")
+                    if not 3 <= len(columns) <= 4:
+                        raise serializers.ValidationError("Improperly formatted genomic islands file")
+                    if columns[0] not in selected_genomes:
+                        raise serializers.ValidationError("Genome: {} not included in this analysis".format(columns[0]))
+
+            # Replace user supplied gi data with properly formatted version
+            data["gi"] = "\n".join(formatted_lines)
         return data
 
     def create(self, validated_data):
diff --git a/IslandCompare/analysis/views.py b/IslandCompare/analysis/views.py
index 0e4cb360..41935d95 100644
--- a/IslandCompare/analysis/views.py
+++ b/IslandCompare/analysis/views.py
@@ -83,11 +83,8 @@ def post(self, request):
         pipeline.append_component(components.MauvePipelineComponent())
 
         if 'gi' in serializer.validated_data:
-            serializer.validated_data['gi'].seek(0)
             user_gi_component = components.UserGIPipelineComponent()
-            user_gi_file_contents = serializer.validated_data['gi'].read().decode('utf-8')
-            user_gi_file_contents = re.sub(r'(\.genbank|\.gbff)', ".gbk", user_gi_file_contents)
-            user_gi_component.set_gi(user_gi_file_contents)
+            user_gi_component.set_gi(serializer.validated_data['gi'])
             pipeline.append_component(user_gi_component)
         else:
             pipeline.append_component(components.SigiHMMPipelineComponent())

From ac6d4667a672e36506f280ef733e2612add31f7d Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 1 Aug 2017 15:29:12 -0700
Subject: [PATCH 52/76] MashMclClusterPipelineComponent will run if user
 provides GI file with no colours

---
 IslandCompare/analysis/serializers.py | 7 ++++++-
 IslandCompare/analysis/views.py       | 1 +
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 23309db9..cb2f916e 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -88,6 +88,7 @@ def validate_genomes(self, value):
         return value
 
     def validate(self, data):
+        cluster_flag = True
         if 'newick' in data.keys():
             selected_genomes = Genome.objects.filter(id__in=[genome.id for genome in data['genomes']])
 
@@ -112,13 +113,17 @@ def validate(self, data):
                 if line is not "":
                     formatted_lines.append(line)
                     columns = line.split("\t")
-                    if not 3 <= len(columns) <= 4:
+                    if len(columns) == 4:
+                        cluster_flag = False
+                    elif len(columns) != 3:
                         raise serializers.ValidationError("Improperly formatted genomic islands file")
                     if columns[0] not in selected_genomes:
                         raise serializers.ValidationError("Genome: {} not included in this analysis".format(columns[0]))
 
             # Replace user supplied gi data with properly formatted version
             data["gi"] = "\n".join(formatted_lines)
+        # Tells AnalysisRunView whether to add MashMclClusterPipelineComponent
+        data["cluster_gis"] = cluster_flag
         return data
 
     def create(self, validated_data):
diff --git a/IslandCompare/analysis/views.py b/IslandCompare/analysis/views.py
index 41935d95..1b08e6d9 100644
--- a/IslandCompare/analysis/views.py
+++ b/IslandCompare/analysis/views.py
@@ -90,6 +90,7 @@ def post(self, request):
             pipeline.append_component(components.SigiHMMPipelineComponent())
             pipeline.append_component(components.IslandPathPipelineComponent())
             pipeline.append_component(components.MergeIslandsPipelineComponent())
+        if serializer.validated_data["cluster_gis"]:
             pipeline.append_component(components.MashMclClusterPipelineComponent())
 
         pipeline.append_component(components.EndPipelineComponent())

From a4ce62366c312c698fcad9d974a3ed0e29436b3d Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 1 Aug 2017 15:43:02 -0700
Subject: [PATCH 53/76] Added UserGenomicIsland model and migration

---
 .../migrations/0005_usergenomicisland.py      | 26 +++++++++++++++++++
 IslandCompare/genomes/models.py               |  4 +++
 2 files changed, 30 insertions(+)
 create mode 100644 IslandCompare/genomes/migrations/0005_usergenomicisland.py

diff --git a/IslandCompare/genomes/migrations/0005_usergenomicisland.py b/IslandCompare/genomes/migrations/0005_usergenomicisland.py
new file mode 100644
index 00000000..da2ff347
--- /dev/null
+++ b/IslandCompare/genomes/migrations/0005_usergenomicisland.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-08-01 22:41
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analysis', '0003_analysis_clusters'),
+        ('genomes', '0004_genomicisland'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UserGenomicIsland',
+            fields=[
+                ('genomicisland_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='genomes.GenomicIsland')),
+                ('color', models.CharField(max_length=30)),
+                ('analysis', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analysis.Analysis')),
+            ],
+            bases=('genomes.genomicisland',),
+        ),
+    ]
diff --git a/IslandCompare/genomes/models.py b/IslandCompare/genomes/models.py
index 0c119b0f..f97fb361 100644
--- a/IslandCompare/genomes/models.py
+++ b/IslandCompare/genomes/models.py
@@ -35,3 +35,7 @@ class GenomicIsland(models.Model):
     start = models.IntegerField()
     end = models.IntegerField()
     genome = models.ForeignKey(Genome, on_delete=models.CASCADE)
+
+class UserGenomicIsland(GenomicIsland):
+    analysis = models.ForeignKey("analysis.Analysis")
+    color = models.CharField(max_length=30)

From 712dd9e6d053646585c57744b08f7f41c5bb4941 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 1 Aug 2017 16:48:49 -0700
Subject: [PATCH 54/76] Pipeline will add user provided GIs to database

---
 IslandCompare/analysis/components.py | 37 +++++++++++++++++++---------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 428fc848..8dd940dd 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -1,6 +1,6 @@
 from analysis.pipeline import PipelineComponent
 from analysis.models import Analysis, Genome
-from genomes.models import Gene, GenomicIsland
+from genomes.models import Gene, GenomicIsland, UserGenomicIsland
 from tempfile import mkdtemp, NamedTemporaryFile
 from shutil import rmtree
 from Bio import SeqIO, Phylo
@@ -286,7 +286,7 @@ def set_newick(self, user_newick):
         self.param['user_file_contents'] = user_newick
 
     def analysis(self, report):
-        selected_genomes = Genome.objects.filter(id__in=[int(_) for _ in report["gbk_paths"].keys()])
+        selected_genomes = Genome.objects.filter(id__in=report["gbk_paths"].keys())
 
         tree = Phylo.read(StringIO(self.param['user_file_contents']), 'newick')
         terminals = tree.get_terminals()
@@ -311,7 +311,6 @@ class UserGIPipelineComponent(PipelineComponent):
     """
     name = "user_gi"
     dependencies = ["gbk_paths"]
-    result_types = ["user_gis"]
 
     def set_gi(self, user_gi):
         self.logger.info("Set User GI as:\n{}".format(user_gi))
@@ -338,17 +337,24 @@ def parse_gi_file(gifile):
         return genomeDict
 
     def analysis(self, report):
-        selected_genomes = Genome.objects.filter(id__in=[int(_) for _ in report["gbk_paths"].keys()])
-        output_dict = dict()
+        analysis = Analysis.objects.get(id=report["analysis"])
+        selected_genomes = Genome.objects.filter(id__in=report["gbk_paths"].keys())
 
         gi_dict = self.parse_gi_file(StringIO(self.param['user_file_contents']))
 
         for key in gi_dict.keys():
-            selected_genome = selected_genomes.filter(name__exact=key)
-            key_id = selected_genome.get().id
-            output_dict[str(key_id)] = gi_dict[key]
-
-        report["user_gis"] = output_dict
+            selected_genome = selected_genomes.get(name=key)
+            for island in gi_dict[key]:
+                if 'color' in island:
+                    color = island['color']
+                else:
+                    color = ""
+                UserGenomicIsland(method="user",
+                                  start=island['start'],
+                                  end=island['end'],
+                                  genome=selected_genome,
+                                  analysis=analysis,
+                                  color=color).save()
 
 
 class MauvePipelineComponent(PipelineComponent):
@@ -672,7 +678,7 @@ class MashMclClusterPipelineComponent(PipelineComponent):
     """
     name = "mash_mcl"
     dependencies = ["gbk_paths"]
-    result_types = ["cluster_gis"]
+    result_types = ["numberClusters"]
     output_dir = settings.BIO_APP_TEMP_DIR + "mash/"
     MASH_PATH = settings.MASH_PATH
     temp_dir_path = None
@@ -732,6 +738,10 @@ def writeFastaFile(self, outputFileName, seqRecordList):
 
     def create_gi_fasta_files(self, report):
         island_path_list = []
+        if "user_gi" in report["pipeline_components"]:
+            method = "user"
+        else:
+            method = "merge"
 
         for genome_id in report["gbk_paths"]:
             record = SeqIO.read(report['gbk_paths'][genome_id], "genbank")
@@ -739,7 +749,10 @@ def create_gi_fasta_files(self, report):
             os.mkdir(genome_fna_path)
 
             gi_counter = 0
-            for gi in Genome.objects.get(id=genome_id).genomicisland_set.filter(method="merge").all():
+            gis = Genome.objects.get(id=genome_id).genomicisland_set.filter(method=method)
+            if method == "user":
+                gis = gis.filter(usergenomicisland__analysis__id=report['analysis'])
+            for gi in gis.all():
                 self.logger.info("Adding GI: " + str(genome_id) + "-" + str(gi_counter))
                 entrySequence = self.get_subsequence(record, gi.start, gi.end, gi_counter)
                 self.writeFastaFile(self.fna_dir_path + "/" + str(genome_id) + "/" + str(gi_counter), entrySequence)

From 6769d0fbe2504b8ba7c052f5b5565585c4027673 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 1 Aug 2017 17:48:49 -0700
Subject: [PATCH 55/76] ReportVisualizationOverviewSerializer will get user gis
 from db and assign colors if clustering occurred

---
 IslandCompare/analysis/serializers.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index cb2f916e..ded583e6 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -154,6 +154,7 @@ def to_representation(self, instance):
 
         output["genomes"] = dict()
         gi_types = ["sigi", "islandpath", "merge"]
+        # numberClusters signifies that clustering of genomic islands was run
         if "numberClusters" in instance:
             color_list = self.get_spaced_colors(instance["numberClusters"])
             clusters = ast.literal_eval(analysis.clusters)
@@ -163,19 +164,26 @@ def to_representation(self, instance):
             output["genomes"][genome.id]["length"] = instance["gbk_metadata"][str(genome.id)]['size']
             output["genomes"][genome.id]["amr_genes"] = [{'start': amr['orf_start'], 'end': amr['orf_end'], 'strand': amr['orf_strand']} for amr in instance["amr_genes"][str(genome.id)]]
             output["genomes"][genome.id]["genomic_islands"] = dict()
-            if "user_gis" in instance:
-                output["genomes"][genome.id]["genomic_islands"]["user"] = instance["user_gis"][str(genome.id)]
-                print(output["genomes"][genome.id]["genomic_islands"]["user"])
+            if "user_gi" in instance["pipeline_components"]:
+                gis = genome.genomicisland_set.filter(method="user").filter(usergenomicisland__analysis=analysis)
+                gi_dicts = [{'start': gi.start, 'end': gi.end, 'color': gi.usergenomicisland.color} for gi in gis]
+                output["genomes"][genome.id]["genomic_islands"]["user"] = gi_dicts
             else:
                 for gi_type in gi_types:
                     gis = genome.genomicisland_set.filter(method=gi_type)
                     if gis.exists():
                         gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
                         output["genomes"][genome.id]["genomic_islands"][gi_type] = gi_dicts
-                if "merge" in output["genomes"][genome.id]["genomic_islands"]:
-                    for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"]["merge"])):
+            if "numberClusters" in instance:
+                method = None
+                if "user" in output["genomes"][genome.id]["genomic_islands"]:
+                    method = "user"
+                elif "merge" in output["genomes"][genome.id]["genomic_islands"]:
+                    method = "merge"
+                if method:
+                    for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"][method])):
                         cluster_index = int(clusters[str(genome.id)][str(gi_index)])
-                        output["genomes"][genome.id]["genomic_islands"]["merge"][gi_index]["color"] = color_list[cluster_index]
+                        output["genomes"][genome.id]["genomic_islands"][method][gi_index]["color"] = color_list[cluster_index]
 
         output["newick"] = instance["newick"]
         output["alignment"] = instance["alignment"]

From a6ab86599490bf7842f2453e602e140231ca6fac Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 2 Aug 2017 11:47:49 -0700
Subject: [PATCH 56/76] Visualization legend and controls altered if user
 provides GIs

---
 IslandCompare/analysis/serializers.py | 27 +++++++++++++-------------
 web/pages/result.html                 | 28 ++++++++++++++++++++-------
 2 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index ded583e6..8544451a 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -153,41 +153,42 @@ def to_representation(self, instance):
         output = dict()
 
         output["genomes"] = dict()
-        gi_types = ["sigi", "islandpath", "merge"]
+        if "user_gi" in instance["pipeline_components"]:
+            gi_method = "user"
+        else:
+            gi_method = "merge"
         # numberClusters signifies that clustering of genomic islands was run
         if "numberClusters" in instance:
+            clustering = True
             color_list = self.get_spaced_colors(instance["numberClusters"])
             clusters = ast.literal_eval(analysis.clusters)
+        else:
+            clustering = False
         for genome in analysis.genomes.all():
             output["genomes"][genome.id] = dict()
             output["genomes"][genome.id]["name"] = genome.name
             output["genomes"][genome.id]["length"] = instance["gbk_metadata"][str(genome.id)]['size']
             output["genomes"][genome.id]["amr_genes"] = [{'start': amr['orf_start'], 'end': amr['orf_end'], 'strand': amr['orf_strand']} for amr in instance["amr_genes"][str(genome.id)]]
             output["genomes"][genome.id]["genomic_islands"] = dict()
-            if "user_gi" in instance["pipeline_components"]:
+            if gi_method == "user":
                 gis = genome.genomicisland_set.filter(method="user").filter(usergenomicisland__analysis=analysis)
                 gi_dicts = [{'start': gi.start, 'end': gi.end, 'color': gi.usergenomicisland.color} for gi in gis]
                 output["genomes"][genome.id]["genomic_islands"]["user"] = gi_dicts
             else:
-                for gi_type in gi_types:
+                for gi_type in ["sigi", "islandpath", "merge"]:
                     gis = genome.genomicisland_set.filter(method=gi_type)
                     if gis.exists():
                         gi_dicts = [{'start': gi.start, 'end': gi.end} for gi in gis]
                         output["genomes"][genome.id]["genomic_islands"][gi_type] = gi_dicts
-            if "numberClusters" in instance:
-                method = None
-                if "user" in output["genomes"][genome.id]["genomic_islands"]:
-                    method = "user"
-                elif "merge" in output["genomes"][genome.id]["genomic_islands"]:
-                    method = "merge"
-                if method:
-                    for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"][method])):
-                        cluster_index = int(clusters[str(genome.id)][str(gi_index)])
-                        output["genomes"][genome.id]["genomic_islands"][method][gi_index]["color"] = color_list[cluster_index]
+            if clustering and gi_method in output["genomes"][genome.id]["genomic_islands"]:
+                for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"][gi_method])):
+                    cluster_index = int(clusters[str(genome.id)][str(gi_index)])
+                    output["genomes"][genome.id]["genomic_islands"][gi_method][gi_index]["color"] = color_list[cluster_index]
 
         output["newick"] = instance["newick"]
         output["alignment"] = instance["alignment"]
         output["failed_components"] = instance["failed_components"]
+        output["cluster_gis"] = clustering and gi_method # 'user', 'merge', or False
 
         return output
 
diff --git a/web/pages/result.html b/web/pages/result.html
index 53eec9df..f49fd2b8 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -56,7 +56,7 @@ <h1>
                             <label>
                                 <input type="radio" name="GIColour" value="similarity" checked/>Similarity
                             </label>
-                            <label>
+                            <label id="predictorButton">
                                 <input type="radio" name="GIColour" value="predictor" />Predictor
                             </label>
                         </div>
@@ -189,16 +189,30 @@ <h1>
                 var treeOrder = container.traverseTreeForOrder(container.newickRoot);
 
                 // Hide irrelevant legends and controls
-                if (islandPathFails === treeOrder.length) {
-                    $("#islandpathToggle").hide();
-                    $("#islandpathLegend").remove();
+                if (!data["cluster_gis"]) {
+                    // No clustering occurred - predictors failed for every genome or GIs were provided with colors
+                    // GI legend in unnecessary
+                    $(".GIColour").hide();
                 }
-                if (sigiFails === treeOrder.length) {
+                else if (data["cluster_gis"] === "user") {
+                    // GIs were provided so coloring by predictor does not apply
+                    $("#islandpathToggle").hide();
                     $("#sigiToggle").hide();
+                    $("#islandpathLegend").remove();
                     $("#sigiLegend").remove();
+                    $("#predictorButton").hide();
                 }
-                if (islandPathFails === treeOrder.length && sigiFails === treeOrder.length) {
-                    $(".GIColour").hide();
+                else {
+                    if (islandPathFails === treeOrder.length) {
+                        // IslandPath failed for all genomes so legend and toggle are unnecessary
+                        $("#islandpathToggle").hide();
+                        $("#islandpathLegend").remove();
+                    }
+                    if (sigiFails === treeOrder.length) {
+                        // Sigi-HMM failed for all genomes so legend and toggle are unnecessary
+                        $("#sigiToggle").hide();
+                        $("#sigiLegend").remove();
+                    }
                 }
 
                 // Add Genomes to Visualization

From 56594c4712da2f18fabe0fe8b3e79eee96bd7af0 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 2 Aug 2017 17:13:32 -0700
Subject: [PATCH 57/76] Clusters are highlighted on mouse over

---
 IslandCompare/analysis/serializers.py |  1 +
 web/dist/js/genomecomparevis.js       | 94 ++++++++++++++++++---------
 web/pages/result.html                 | 14 ++++
 3 files changed, 80 insertions(+), 29 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 8544451a..f92337a6 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -183,6 +183,7 @@ def to_representation(self, instance):
             if clustering and gi_method in output["genomes"][genome.id]["genomic_islands"]:
                 for gi_index in range(len(output["genomes"][genome.id]["genomic_islands"][gi_method])):
                     cluster_index = int(clusters[str(genome.id)][str(gi_index)])
+                    output["genomes"][genome.id]["genomic_islands"][gi_method][gi_index]["cluster"] = cluster_index
                     output["genomes"][genome.id]["genomic_islands"][gi_method][gi_index]["color"] = color_list[cluster_index]
 
         output["newick"] = instance["newick"]
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 166eaf92..41fd9b25 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -294,6 +294,30 @@ function MultiVis(targetNode){
                         "["+homologousRegions[j].start2+","+homologousRegions[j].end2+"]");
             }
         }
+
+        //Add the brush for zooming and focusing
+        var brush = d3.svg.brush()
+            .x(self.scale)
+            .on("brush", brushmove)
+            .on("brushend", brushend);
+
+        sequenceHolder.append("g")
+            .attr("class", "brush")
+            .call(brush)
+            .selectAll('rect')
+            .attr('height', this.containerHeight())
+            .attr("transform","translate(0,"+(-0.7)*this.getSequenceModHeight()+")");
+
+        function brushmove() {
+            var extent = brush.extent();
+        }
+
+        function brushend() {
+            var extent = brush.extent();
+            self.setScale(extent[0],extent[1]);
+            self.transition();
+        }
+
         //Create the sequence list from the ordered sequence list
         var sequenceData = [];
         for (var index=0;index<seqOrder.length;index++){
@@ -340,13 +364,11 @@ function MultiVis(targetNode){
                     .attr("points", function(d) {
                         var startPosition = self.scale(parseInt(d.start));
                         var endPosition = self.scale(parseInt(d.end));
-                        return startPosition + "," + (self.getSequenceModHeight() * i + GISIZE / 2) + " " +
-                                 endPosition + "," + (self.getSequenceModHeight() * i + GISIZE / 2) + " " +
-                                 endPosition + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " " +
-                               startPosition + "," + (self.getSequenceModHeight() * i - GISIZE / 2) + " ";
+                        return startPosition + "," + (self.getSequenceModHeight() * i + GISIZE) + " " +
+                                 endPosition + "," + (self.getSequenceModHeight() * i + GISIZE) + " " +
+                                 endPosition + "," + (self.getSequenceModHeight() * i) + " " +
+                               startPosition + "," + (self.getSequenceModHeight() * i) + " ";
                     })
-                    .attr("stroke-width", 1)
-                    .attr("transform", "translate(0," + (GISIZE / 2) + ")")
                     .attr("fill", function(d) {
                         if (d.color != null) {
                             return d.color;
@@ -356,6 +378,21 @@ function MultiVis(targetNode){
                         if (d.color != null) {
                             return d.color;
                         }
+                    })
+                    .attr("class", function(d) {
+                        if (d.cluster) {
+                            return "cluster-" + d.cluster
+                        }
+                    })
+                    .on("mouseover", function(d) {
+                        if(d.cluster) {
+                            self.highlightCluster(".cluster-" + d.cluster);
+                        }
+                    })
+                    .on("mouseout", function(d) {
+                        if (d.cluster) {
+                            self.unhighlightCluster(".cluster-" + d.cluster);
+                        }
                     });
             });
         });
@@ -400,29 +437,6 @@ function MultiVis(targetNode){
                 });
         });
 
-        //Add the brush for zooming and focusing
-        var brush = d3.svg.brush()
-            .x(self.scale)
-            .on("brush", brushmove)
-            .on("brushend", brushend);
-
-        sequenceHolder.append("g")
-            .attr("class", "brush")
-            .call(brush)
-            .selectAll('rect')
-            .attr('height', this.containerHeight())
-            .attr("transform","translate(0,"+(-0.7)*this.getSequenceModHeight()+")");
-
-        function brushmove() {
-            var extent = brush.extent();
-        }
-
-        function brushend() {
-            var extent = brush.extent();
-            self.setScale(extent[0],extent[1]);
-            self.transition();
-        }
-
         //Add the genes to the plot
         var geneFilterValue = self.getGeneFilterValue();
         if((self.scale.domain()[1] - self.scale.domain()[0]) < geneFilterValue) {
@@ -646,6 +660,28 @@ function MultiVis(targetNode){
         }
     };
 
+    this.highlightCluster = function(className) {
+        $(className).attr("filter", "url(#shadow)");
+        $(".sigi, .islandpath").hide();
+        var otherIslands = $("svg .genomicIslands polygon").not(className);
+        otherIslands.attr("fill-opacity", "0.4");
+        otherIslands.attr("stroke-opacity", "0");
+        var genes = $(".CDS, .tRNA, .rRNA, .gene");
+        genes.attr("fill-opacity", "0.4");
+        genes.attr("stroke-opacity", "0");
+    };
+
+    this.unhighlightCluster = function(className) {
+        $(className).removeAttr("filter");
+        $(".sigi, .islandpath").show();
+        var otherIslands = $("svg .genomicIslands polygon").not(className);
+        otherIslands.removeAttr("fill-opacity");
+        otherIslands.removeAttr("stroke-opacity");
+        var genes = $(".CDS, .tRNA, .rRNA, .gene");
+        genes.removeAttr("fill-opacity");
+        genes.removeAttr("stroke-opacity");
+    };
+
     return this;
 }
 
diff --git a/web/pages/result.html b/web/pages/result.html
index f49fd2b8..17af8e99 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -132,6 +132,20 @@ <h1>
         <!-- /.box-header -->
         <div class="box-body">
             <div id="visualization-body"></div>
+            <svg>
+                <filter id="shadow" x="-5" y="-5" width="200" height="200">
+                    <feGaussianBlur in="SourceAlpha" stdDeviation="5" />
+                    <feColorMatrix result="matrixOut" in="offOut" type="matrix"
+                        values="0   0   0   0   0
+                                0   0   0   0   0
+                                0   0   0   0   0
+                                0   0   0   2   0" />
+                    <feMerge>
+                        <feMergeNode/>
+                        <feMergeNode in="SourceGraphic"/>
+                    </feMerge>
+                </filter>
+            </svg>
         </div>
         <!-- /.box-body -->
     </div>

From 73fb6ad6620fc51c3452b2badec87e6c8a193ef7 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 8 Aug 2017 16:53:46 -0700
Subject: [PATCH 58/76] Clicking a GI will toggle cluster view and enable
 cluster zoom

---
 web/dist/css/linearplot.css     |   2 +-
 web/dist/js/genomecomparevis.js | 114 +++++++++++++++++++++++---------
 web/pages/result.html           |  25 +++++--
 3 files changed, 103 insertions(+), 38 deletions(-)

diff --git a/web/dist/css/linearplot.css b/web/dist/css/linearplot.css
index 183f589b..175392c0 100644
--- a/web/dist/css/linearplot.css
+++ b/web/dist/css/linearplot.css
@@ -115,7 +115,7 @@ g.root.node circle{
     margin-right: 8px;
 }
 
-.legendBody, .predictorLegend{
+.legendBody, .predictorLegend, #clusterLegend{
     padding: 3px;
 }
 
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 41fd9b25..d4343fc9 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -30,6 +30,7 @@ function MultiVis(targetNode){
     this.newickData = null;
     this.newickRoot = null;
     this.trueBranchLengths = false;
+    this.cluster = null;
 
     this.toggleTrueBranchLengths = function(){
         this.trueBranchLengths = !this.trueBranchLengths;
@@ -329,7 +330,10 @@ function MultiVis(targetNode){
             .data(sequenceData)
             .enter()
             .append("g")
-            .attr("class", "sequences");
+            .attr("class", "sequences")
+            .attr("sequence", function(sequence) {
+                return sequence.sequenceId;
+            });
 
         //Add the sequences to the SVG
         seq.append("rect")
@@ -345,57 +349,68 @@ function MultiVis(targetNode){
             });
 
         //Add GIs to each sequence
-        seq.append("g").attr("class","genomicIslands").each(function(d, i){
+        seq.append("g").attr("class","genomicIslands").each(function(seqData, i){
             var methods = d3.select(this).selectAll("g")
-                .data(Object.keys(d.gi))
+                .data(Object.keys(seqData.gi))
                 .enter().append("g")
-                .attr("class", function(d) {
-                    return d;
+                .attr("class", function(method) {
+                    return method;
                 });
             methods.each(function(method) {
                 d3.select(this).selectAll("polygon")
-                    .data(d.gi[method].filter(function(d) {
+                    .data(seqData.gi[method].filter(function(gi) {
                         // Filter out the GIs that aren't within the current scale or are too small
-                        var withinScale = d.start < self.scale.domain()[1] && d.end > self.scale.domain()[0];
-                        var withinLength = d.end - d.start > self.getGIFilterValue();
+                        var withinScale = gi.start < self.scale.domain()[1] && gi.end > self.scale.domain()[0];
+                        var withinLength = gi.end - gi.start > self.getGIFilterValue();
                         return withinScale && withinLength;
                     }))
                     .enter().append("polygon")
-                    .attr("points", function(d) {
-                        var startPosition = self.scale(parseInt(d.start));
-                        var endPosition = self.scale(parseInt(d.end));
+                    .attr("points", function(gi) {
+                        var startPosition = self.scale(parseInt(gi.start));
+                        var endPosition = self.scale(parseInt(gi.end));
                         return startPosition + "," + (self.getSequenceModHeight() * i + GISIZE) + " " +
                                  endPosition + "," + (self.getSequenceModHeight() * i + GISIZE) + " " +
                                  endPosition + "," + (self.getSequenceModHeight() * i) + " " +
                                startPosition + "," + (self.getSequenceModHeight() * i) + " ";
                     })
-                    .attr("fill", function(d) {
-                        if (d.color != null) {
-                            return d.color;
+                    .attr("fill", function(gi) {
+                        if (gi.color != null) {
+                            return gi.color;
+                        }
+                    })
+                    .attr("stroke", function(gi) {
+                        if (gi.color != null) {
+                            return gi.color;
                         }
                     })
-                    .attr("stroke", function(d) {
-                        if (d.color != null) {
-                            return d.color;
+                    .attr("class", function(gi) {
+                        if (gi.cluster) {
+                            return "cluster-" + gi.cluster
                         }
                     })
-                    .attr("class", function(d) {
-                        if (d.cluster) {
-                            return "cluster-" + d.cluster
+                    .attr("start", function(gi) { return gi.start; }).attr("end",function(gi) { return gi.end; })
+                    .on("mouseover", function(gi) {
+                        if(gi.cluster) {
+                            self.highlightCluster(".cluster-" + gi.cluster);
                         }
                     })
-                    .on("mouseover", function(d) {
-                        if(d.cluster) {
-                            self.highlightCluster(".cluster-" + d.cluster);
+                    .on("mouseout", function(gi) {
+                        if (gi.cluster) {
+                            self.unhighlightCluster(".cluster-" + gi.cluster);
                         }
                     })
-                    .on("mouseout", function(d) {
-                        if (d.cluster) {
-                            self.unhighlightCluster(".cluster-" + d.cluster);
+                    .on("click", function(gi) {
+                        if (gi.cluster) {
+                            self.toggleClusterView(gi.cluster, gi.color);
                         }
-                    });
+                    })
+                    .append("title").text(function(gi) { return "Click to toggle GI cluster " + gi.cluster + " view"});
             });
         });
+        // Hide GIs if a cluster is selected
+        if (self.cluster) {
+            $("svg .genomicIslands polygon").not(".cluster-" + self.cluster).hide();
+        }
 
         //Add AMR genes to each sequence
         seq.append("g").attr("class", "amrs").each(function(d, i) {
@@ -660,10 +675,10 @@ function MultiVis(targetNode){
         }
     };
 
-    this.highlightCluster = function(className) {
-        $(className).attr("filter", "url(#shadow)");
+    this.highlightCluster = function(clusterClass) {
+        $(clusterClass).attr("filter", "url(#shadow)");
         $(".sigi, .islandpath").hide();
-        var otherIslands = $("svg .genomicIslands polygon").not(className);
+        var otherIslands = $("svg .genomicIslands polygon").not(clusterClass);
         otherIslands.attr("fill-opacity", "0.4");
         otherIslands.attr("stroke-opacity", "0");
         var genes = $(".CDS, .tRNA, .rRNA, .gene");
@@ -671,10 +686,10 @@ function MultiVis(targetNode){
         genes.attr("stroke-opacity", "0");
     };
 
-    this.unhighlightCluster = function(className) {
-        $(className).removeAttr("filter");
+    this.unhighlightCluster = function(clusterClass) {
+        $(clusterClass).removeAttr("filter");
         $(".sigi, .islandpath").show();
-        var otherIslands = $("svg .genomicIslands polygon").not(className);
+        var otherIslands = $("svg .genomicIslands polygon").not(clusterClass);
         otherIslands.removeAttr("fill-opacity");
         otherIslands.removeAttr("stroke-opacity");
         var genes = $(".CDS, .tRNA, .rRNA, .gene");
@@ -682,6 +697,39 @@ function MultiVis(targetNode){
         genes.removeAttr("stroke-opacity");
     };
 
+    this.toggleClusterView = function(cluster, color = null) {
+        $("svg .genomicIslands polygon").not(".cluster-" + cluster).toggle();
+        $(".GIColour").toggle();
+        $(".clusterButton").toggle();
+        $("#clusterLegend").toggle();
+        if (self.cluster == null) {
+            self.cluster = cluster;
+            $("#clusterLegend circle").attr("fill", color);
+            $("#clusterLegendSpan").text("Cluster " + cluster + " Genomic Island");
+        } else {
+            self.cluster = null;
+        }
+    };
+
+    this.zoomCluster = function() {
+        var clusterClass = ".cluster-" + self.cluster;
+        var gis = $(clusterClass);
+            var start_array = [];
+            var end_array = [];
+            var seq_set = new Set();
+            gis.each(function() {
+                start_array.push(parseInt($(this).attr("start")));
+                end_array.push(parseInt($(this).attr("end")));
+                seq_set = seq_set.add($(this).parents(".sequences").attr("sequence"));
+            });
+            var start = Math.min.apply(null, start_array);
+            var end = Math.max.apply(null, end_array);
+            var buffer = parseFloat(end - start) / 80;
+            self.setScale(start-buffer, end+buffer);
+            self.transition();
+        self.unhighlightCluster(clusterClass);
+    };
+
     return this;
 }
 
diff --git a/web/pages/result.html b/web/pages/result.html
index 17af8e99..7c43cebe 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -33,6 +33,9 @@ <h1>
                     <div id="btn-div" class="clearfix btn-group btn-group-sm">
                         <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
                         <button id="resetTree" class="btn btn-primary btn-flat">Reset Tree</button>
+                        <button id="resetGIs" class="btn btn-primary btn-flat clusterButton" style="display: none;">
+                            Restore GIs
+                        </button>
                         <div class="btn-group btn-group-sm">
                             <button class="btn btn-primary btn-flat dropdown-toggle" data-toggle="dropdown">
                                 Save image <span class="caret"></span>
@@ -44,7 +47,9 @@ <h1>
                         </div>
                         <button id="printerColors" class="btn btn-primary btn-flat" style="clear: left; margin-left: 0px;">Toggle Colour</button>
                         <button id="resetRange" class="btn btn-primary btn-flat">Reset Zoom</button>
-
+                        <button id="zoomCluster" class="btn btn-primary btn-flat clusterButton" style="display: none;">
+                            Zoom to Cluster
+                        </button>
                     </div>
                 </div>
                 <div class="col-xs-2 GIColour">
@@ -84,9 +89,9 @@ <h1>
                         </svg>
                         <span class="legendText">Homologous Region</span>
                         <svg id="amrLegend" height="10" width="10">
-                            <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Genes</title></polygon>
+                            <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Gene</title></polygon>
                         </svg>
-                        <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Genes</span>
+                        <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Gene</span>
                         <span class="loadingGenes" hidden>
                             <i class='fa fa-spinner fa-spin '></i> Loading Genes
                         </span>
@@ -94,7 +99,7 @@ <h1>
                             <svg id="genesLegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
-                            <span class="legendText">Genes</span>
+                            <span class="legendText">Gene</span>
                             <svg id="tRNALegend" height="10" width="10">
                                 <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
                             </svg>
@@ -120,6 +125,12 @@ <h1>
                         </span>
 
                     </div>
+                    <div id="clusterLegend" class="clearfix" hidden>
+                        <svg height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span id="clusterLegendSpan" class="legendText"></span>
+                    </div>
                 </div>
                 <!--
                 --Currently Not used
@@ -324,6 +335,12 @@ <h1>
             $("#savePng").on("click", this, function() {
                 container.saveImage("png");
             });
+            $("#resetGIs").on("click", this, function() {
+                container.toggleClusterView(container.cluster);
+            });
+            $("#zoomCluster").on("click", this, function() {
+                container.zoomCluster();
+            });
             // Changes to sigi/islandpath checkboxes will hide/show corresponding GIs
             $(".GIColourBody :checkbox").change(container.predictorToggle);
             // Changes to GI colour method will hide/show the merged GI set and enable/disable the predictor checkboxes

From e340a18d1f50fdebcd8daa623f43f78318a0a236 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 8 Aug 2017 17:04:38 -0700
Subject: [PATCH 59/76] Fixed ReportCsvSerializer

---
 IslandCompare/analysis/serializers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 8253ec02..e5366f96 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -181,10 +181,11 @@ def to_representation(self, analysis):
         writer.writeheader()
 
         method_keys = ["merge", "islandpath", "sigi"]
+        clusters = ast.literal_eval(analysis.clusters)
 
         for method in method_keys:
             for genome in analysis.genomes.all():
-                for island in genome.genomicisland_set.filter(method=method):
+                for i, island in enumerate(genome.genomicisland_set.filter(method=method)):
                     row = {
                         'name': genome.name,
                         'start': island.start,
@@ -192,8 +193,7 @@ def to_representation(self, analysis):
                         'method': method + "_gis"
                     }
                     if method == "merge":
-                        cluster = island.genomicislandcluster_set.get(analysis=analysis)
-                        row['cluster_id'] = cluster.number
+                        row['cluster_id'] = clusters[str(genome.id)][str(i)]
                     writer.writerow(row)
 
         contents = output.getvalue()

From 3d513e10fae553aa1355688900acb6caed918ecf Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 9 Aug 2017 15:48:04 -0700
Subject: [PATCH 60/76] Refactoring of parse_sigi_gff function

---
 IslandCompare/analysis/components.py | 35 +++++++++++++---------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 428fc848..41004dda 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -476,28 +476,25 @@ def parse_sigi_gff(gff_file):
                 else:
                     cleaned_line = ' '.join(line.split())
                     gene_dict = cleaned_line.split(' ')
-                    # at the start of a genomic island, set start and end of possible genomic island
-                    # some results contain many islands that start at one and extend very far into the genome
-                    # this is presumed to be an error, so these results will be skipped
-                    if gene_dict[2] == 'PUTAL' and not island_flag and gene_dict[3] != '1':
-                        start = gene_dict[3]
-                        end = gene_dict[4]
-                        island_flag = True
-                    # continuation of current genomic island, change end and continue
-                    # some results have an end position of 0. These results will be skipped
-                    elif gene_dict[2] == 'PUTAL' and island_flag and int(gene_dict[4]) > int(start):
-                        end = gene_dict[4]
-                    # end of genomic island, append current start and end to list
+                    # Some results contain lines where start is 1 / end is 0. Skip to avoid erroneous GIs
+                    if gene_dict[3] == '1' or gene_dict[4] == '0':
+                        continue
+                    if gene_dict[2] == 'PUTAL':
+                        # At the start of a genomic island, set start and end of possible genomic island
+                        if not island_flag:
+                            start = gene_dict[3]
+                            end = gene_dict[4]
+                            island_flag = True
+                        # Continuation of current genomic island, change end and continue
+                        else:
+                            end = gene_dict[4]
+                    # End of genomic island, append current start and end to list
                     elif island_flag:
                         gi_list.append([int(start), int(end)])
                         island_flag = False
-                    # not currently in a genomic island, continue to next line
-                    elif not island_flag:
-                        continue
-                    # condition not included in above reached, throw an error
-                    else:
-                        raise Exception("Error occurred in sigi, unexpected condition reached")
-
+            # For cases where the last line is part of a genomic island
+            if island_flag:
+                gi_list.append([int(start), int(end)])
         return gi_list
 
     def setup(self, report):

From b568af38613f8dfc6124715491451c758ca7a5fd Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 8 Aug 2017 17:22:57 -0700
Subject: [PATCH 61/76] Minor visual improvement to result box-header. Refactor
 genomecomparevis.js

---
 web/dist/js/genomecomparevis.js | 60 +++++++++++++--------------------
 web/pages/result.html           |  6 ++--
 2 files changed, 27 insertions(+), 39 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index d4343fc9..5f6fbfce 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -373,38 +373,29 @@ function MultiVis(targetNode){
                                  endPosition + "," + (self.getSequenceModHeight() * i) + " " +
                                startPosition + "," + (self.getSequenceModHeight() * i) + " ";
                     })
+                    .attr("start", function(gi) { return gi.start; })
+                    .attr("end",function(gi) { return gi.end; })
                     .attr("fill", function(gi) {
-                        if (gi.color != null) {
-                            return gi.color;
-                        }
+                        if (gi.color != null) { return gi.color; }
                     })
                     .attr("stroke", function(gi) {
-                        if (gi.color != null) {
-                            return gi.color;
-                        }
+                        if (gi.color != null) { return gi.color; }
                     })
                     .attr("class", function(gi) {
-                        if (gi.cluster) {
-                            return "cluster-" + gi.cluster
-                        }
+                        if (gi.cluster != null) { return "cluster-" + gi.cluster; }
                     })
-                    .attr("start", function(gi) { return gi.start; }).attr("end",function(gi) { return gi.end; })
                     .on("mouseover", function(gi) {
-                        if(gi.cluster) {
-                            self.highlightCluster(".cluster-" + gi.cluster);
-                        }
+                        if(gi.cluster != null) { self.highlightCluster(".cluster-" + gi.cluster); }
                     })
                     .on("mouseout", function(gi) {
-                        if (gi.cluster) {
-                            self.unhighlightCluster(".cluster-" + gi.cluster);
-                        }
+                        if (gi.cluster != null) { self.unhighlightCluster(".cluster-" + gi.cluster); }
                     })
                     .on("click", function(gi) {
-                        if (gi.cluster) {
-                            self.toggleClusterView(gi.cluster, gi.color);
-                        }
+                        if (gi.cluster != null) { self.toggleClusterView(gi.cluster, gi.color); }
                     })
-                    .append("title").text(function(gi) { return "Click to toggle GI cluster " + gi.cluster + " view"});
+                    .append("title").text(function(gi) {
+                        if(gi.cluster != null) { return "Click to toggle GI cluster " + gi.cluster + " view"; }
+                    });
             });
         });
         // Hide GIs if a cluster is selected
@@ -697,9 +688,9 @@ function MultiVis(targetNode){
         genes.removeAttr("stroke-opacity");
     };
 
-    this.toggleClusterView = function(cluster, color = null) {
+    this.toggleClusterView = function(cluster, color) {
         $("svg .genomicIslands polygon").not(".cluster-" + cluster).toggle();
-        $(".GIColour").toggle();
+        $(".GIColour").children().toggle();
         $(".clusterButton").toggle();
         $("#clusterLegend").toggle();
         if (self.cluster == null) {
@@ -713,20 +704,17 @@ function MultiVis(targetNode){
 
     this.zoomCluster = function() {
         var clusterClass = ".cluster-" + self.cluster;
-        var gis = $(clusterClass);
-            var start_array = [];
-            var end_array = [];
-            var seq_set = new Set();
-            gis.each(function() {
-                start_array.push(parseInt($(this).attr("start")));
-                end_array.push(parseInt($(this).attr("end")));
-                seq_set = seq_set.add($(this).parents(".sequences").attr("sequence"));
-            });
-            var start = Math.min.apply(null, start_array);
-            var end = Math.max.apply(null, end_array);
-            var buffer = parseFloat(end - start) / 80;
-            self.setScale(start-buffer, end+buffer);
-            self.transition();
+        var start_array = [];
+        var end_array = [];
+        $(clusterClass).each(function() {
+            start_array.push(parseInt($(this).attr("start")));
+            end_array.push(parseInt($(this).attr("end")));
+        });
+        var start = Math.min.apply(null, start_array);
+        var end = Math.max.apply(null, end_array);
+        var buffer = parseFloat(end - start) / 80;
+        self.setScale(start-buffer, end+buffer);
+        self.transition();
         self.unhighlightCluster(clusterClass);
     };
 
diff --git a/web/pages/result.html b/web/pages/result.html
index 7c43cebe..a7db2377 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -34,7 +34,7 @@ <h1>
                         <button id="toggleBranchLength" class="btn btn-primary btn-flat">Toggle Branches</button>
                         <button id="resetTree" class="btn btn-primary btn-flat">Reset Tree</button>
                         <button id="resetGIs" class="btn btn-primary btn-flat clusterButton" style="display: none;">
-                            Restore GIs
+                            Reset GIs
                         </button>
                         <div class="btn-group btn-group-sm">
                             <button class="btn btn-primary btn-flat dropdown-toggle" data-toggle="dropdown">
@@ -217,7 +217,7 @@ <h1>
                 if (!data["cluster_gis"]) {
                     // No clustering occurred - predictors failed for every genome or GIs were provided with colors
                     // GI legend in unnecessary
-                    $(".GIColour").hide();
+                    $(".GIColour").children().hide();
                 }
                 else if (data["cluster_gis"] === "user") {
                     // GIs were provided so coloring by predictor does not apply
@@ -336,7 +336,7 @@ <h1>
                 container.saveImage("png");
             });
             $("#resetGIs").on("click", this, function() {
-                container.toggleClusterView(container.cluster);
+                container.toggleClusterView(container.cluster, null);
             });
             $("#zoomCluster").on("click", this, function() {
                 container.zoomCluster();

From ea055d3c276222bbe711d27ad7479de839174d18 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 9 Aug 2017 12:56:22 -0700
Subject: [PATCH 62/76] Improved RunAnalysisSerializer validation of newick,
 resolving #87

---
 IslandCompare/analysis/serializers.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index f92337a6..95c9a50f 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -90,17 +90,26 @@ def validate_genomes(self, value):
     def validate(self, data):
         cluster_flag = True
         if 'newick' in data.keys():
-            selected_genomes = Genome.objects.filter(id__in=[genome.id for genome in data['genomes']])
+            selected_genome_names = [genome.name for genome in data['genomes']]
 
             tree = Phylo.read(StringIO(data['newick'].read().decode('utf-8')), 'newick')
             terminals = tree.get_terminals()
 
             for leaf in terminals:
                 leaf.name = re.sub(r'(\.genbank|\.gbff)$', ".gbk", leaf.name)
-                selected_genome = selected_genomes.filter(owner__exact=self.context['request'].user,
+                selected_genome = Genome.objects.filter(owner__exact=self.context['request'].user,
                                                           name__exact=leaf.name)
                 if not selected_genome.exists():
-                    raise serializers.ValidationError("Genome: {} not included in this analysis".format(leaf))
+                    raise serializers.ValidationError("Genome {} does not exist".format(leaf.name))
+                if leaf.name not in selected_genome_names:
+                    raise serializers.ValidationError("Genome {} in newick file not included in this analysis".format(leaf))
+            if len(terminals) != len(selected_genome_names):
+                if len(selected_genome_names) - len(terminals) > 1:
+                    error_message = "Selected genomes missing from newick file"
+                else:
+                    error_message = "Selected genome missing from newick file"
+                raise serializers.ValidationError(error_message)
+
         if 'gi' in data.keys():
             selected_genomes = [genome.name for genome in data['genomes']]
             formatted_lines = []
@@ -118,7 +127,7 @@ def validate(self, data):
                     elif len(columns) != 3:
                         raise serializers.ValidationError("Improperly formatted genomic islands file")
                     if columns[0] not in selected_genomes:
-                        raise serializers.ValidationError("Genome: {} not included in this analysis".format(columns[0]))
+                        raise serializers.ValidationError("Genome {} in GI file not included in this analysis".format(columns[0]))
 
             # Replace user supplied gi data with properly formatted version
             data["gi"] = "\n".join(formatted_lines)

From 20718526a1f04a195e3a864d4f4c097d675c0d2c Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Wed, 9 Aug 2017 17:43:15 -0700
Subject: [PATCH 63/76] Updated MashMCLTestCase for changes to
 MashMclClusterPipelineComponent

---
 IslandCompare/analysis/integration_tests.py | 54 ++++++++++++---------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/IslandCompare/analysis/integration_tests.py b/IslandCompare/analysis/integration_tests.py
index 243b48a2..d55e8d5a 100644
--- a/IslandCompare/analysis/integration_tests.py
+++ b/IslandCompare/analysis/integration_tests.py
@@ -1,5 +1,6 @@
 from django.test import TestCase, override_settings
 from genomes.models import Genome, GenomicIsland
+from analysis.models import Analysis
 from django.contrib.auth.models import User
 from django.core.files import File
 from analysis.components import ParsnpPipelineComponent, MauvePipelineComponent, \
@@ -286,6 +287,8 @@ class MashMCLTestCase(TestCase):
     test_username = "username"
     test_user = None
 
+    test_cluster_analysis = None
+
     test_genome_1 = None
     test_genome_1_name = "genome_1"
     test_genome_1_gbk = File(open("../TestFiles/AE009952.gbk"))
@@ -297,10 +300,19 @@ class MashMCLTestCase(TestCase):
     test_gi_1 = None
     test_gi_2 = None
 
+    report = None
+
     def setUp(self):
         self.test_user = User(username=self.test_username)
         self.test_user.save()
 
+        self.test_cluster_analysis = Analysis(
+            celery_task_id="1",
+            name="test_cluster_analysis",
+            owner=self.test_user
+        )
+        self.test_cluster_analysis.save()
+
         self.test_genome_1 = Genome.objects.create(name=self.test_genome_1_name,
                                                    owner=self.test_user,
                                                    gbk=self.test_genome_1_gbk)
@@ -311,6 +323,10 @@ def setUp(self):
                                                    gbk=self.test_genome_2_gbk)
         self.test_genome_2.save()
 
+        self.test_cluster_analysis.genomes.add(self.test_genome_1)
+        self.test_cluster_analysis.genomes.add(self.test_genome_2)
+        self.test_cluster_analysis.save()
+
         self.test_gi_1 = GenomicIsland(
             method="merge",
             start=0,
@@ -324,48 +340,40 @@ def setUp(self):
             genome=self.test_genome_2
         ).save()
 
-    def test_generate_gi_fna(self):
-        component = MashMclClusterPipelineComponent()
-
-        report = {
-            "analysis": 1,
+        self.report = {
+            "analysis": self.test_cluster_analysis.id,
             "available_dependencies": ["gbk_paths"],
             "gbk_paths": {self.test_genome_1.id: self.test_genome_1.gbk.path,
-                          self.test_genome_2.id: self.test_genome_2.gbk.path}
+                          self.test_genome_2.id: self.test_genome_2.gbk.path},
+            "pipeline_components": "merge"
         }
 
-        component.setup(report)
-        component.create_gi_fasta_files(report)
+    def test_generate_gi_fna(self):
+        component = MashMclClusterPipelineComponent()
+
+        component.setup(self.report)
+        component.create_gi_fasta_files(self.report)
 
         self.assertTrue(os.path.exists(component.temp_dir_path + "/fna/1/0"))
         self.assertTrue(os.path.exists(component.temp_dir_path + "/fna/2/0"))
 
         component.cleanup()
 
-    # TODO - implement MashMclClusterPipelineComponent test
-    @skip("Analysis object needed for Mash component")
     def test_analysis(self):
         component = MashMclClusterPipelineComponent()
 
-        report = {
-            "analysis": 1,
-            "available_dependencies": ["merge_gis", "gbk_paths"],
-            "gbk_paths": {self.test_genome_1.id: self.test_genome_1.gbk.path,
-                          self.test_genome_2.id: self.test_genome_2.gbk.path},
-            "merge_gis": {self.test_genome_1.id: [[0, 1000]],
-                          self.test_genome_2.id: [[0, 1000]]}
-        }
-
-        component.setup(report)
-        component.analysis(report)
+        component.setup(self.report)
+        component.analysis(self.report)
         component.cleanup()
 
-        self.assertTrue("cluster_gis" in report)
-        self.assertEqual(0, report["cluster_gis"]['numberClusters'])
+        self.assertTrue("numberClusters" in self.report)
+        self.assertTrue(Analysis.objects.get(id=self.report["analysis"]).clusters != "")
 
     def tearDown(self):
         for genome in Genome.objects.all():
             genome.delete()
+        for analysis in Analysis.objects.all():
+            analysis.delete()
 
 
 class RGIComponentIntegrationTestCase(TestCase):

From 79dbfb2d2eaaba7c507fb29e03447b676ff25856 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 10 Aug 2017 10:23:13 -0700
Subject: [PATCH 64/76] Resolved negative width error when zooming into an area
 which does not include all sequences

---
 web/dist/js/genomecomparevis.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 5f6fbfce..346c4de4 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -345,7 +345,10 @@ function MultiVis(targetNode){
             .attr("rx",4)
             .attr("ry",4)
             .attr("width", function (d){
-                return self.scale(d.scale(d.getSequenceSize()));
+                var width = self.scale(d.scale(d.getSequenceSize()));
+                if (width >= 0) {
+                    return width;
+                }
             });
 
         //Add GIs to each sequence

From e779dc2445547907b88467722971d7267b13462d Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 10 Aug 2017 15:44:23 -0700
Subject: [PATCH 65/76] Move RGI to end of pipeline

---
 IslandCompare/analysis/tasks.py | 2 ++
 IslandCompare/analysis/views.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/IslandCompare/analysis/tasks.py b/IslandCompare/analysis/tasks.py
index ccbf87a8..e472df49 100644
--- a/IslandCompare/analysis/tasks.py
+++ b/IslandCompare/analysis/tasks.py
@@ -131,6 +131,7 @@ def sigi_error_handler(context, exc, traceback, pipeline_id):
                            .on_error(ipath_error_handler.s(pipeline_id)),
                        run_pipeline_component.s(pipeline_id, "merge_gis"),
                        run_pipeline_component.s(pipeline_id, "mash_mcl"),
+                       run_pipeline_component.s(pipeline_id, "rgi"),
                        run_pipeline_component.s(pipeline_id, "end_pipeline"))
     return task_chain.apply_async()
 
@@ -138,4 +139,5 @@ def sigi_error_handler(context, exc, traceback, pipeline_id):
 def ipath_error_handler(context, exc, traceback, pipeline_id):
     return chain(run_pipeline_component.s(context.args[0], pipeline_id, "merge_gis"),
                  run_pipeline_component.s(pipeline_id, "mash_mcl"),
+                 run_pipeline_component.s(pipeline_id, "rgi"),
                  run_pipeline_component.s(pipeline_id, "end_pipeline")).apply_async()
diff --git a/IslandCompare/analysis/views.py b/IslandCompare/analysis/views.py
index 0e4cb360..30917bc4 100644
--- a/IslandCompare/analysis/views.py
+++ b/IslandCompare/analysis/views.py
@@ -68,7 +68,6 @@ def post(self, request):
         pipeline.append_component(components.StartPipelineComponent())
         pipeline.append_component(components.SetupGbkPipelineComponent())
         pipeline.append_component(components.GbkMetadataComponent())
-        pipeline.append_component(components.RGIPipelineComponent())
 
         if 'newick' in serializer.validated_data:
             serializer.validated_data['newick'].seek(0)
@@ -95,6 +94,7 @@ def post(self, request):
             pipeline.append_component(components.MergeIslandsPipelineComponent())
             pipeline.append_component(components.MashMclClusterPipelineComponent())
 
+        pipeline.append_component(components.RGIPipelineComponent())
         pipeline.append_component(components.EndPipelineComponent())
         pipeline.create_database_entry(name=serializer.validated_data['name'],
                                        genomes=serializer.validated_data['genomes'],

From 24c865efd32186dcf9051827efe69b2d2a5dd135 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 14 Aug 2017 17:04:01 -0700
Subject: [PATCH 66/76] Simple cluster visualization opened in new page on GI
 click

---
 web/dist/js/genomecomparevis.js |  45 ++++-
 web/pages/cluster.html          | 283 ++++++++++++++++++++++++++++++++
 2 files changed, 327 insertions(+), 1 deletion(-)
 create mode 100644 web/pages/cluster.html

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 346c4de4..990204ab 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -394,7 +394,7 @@ function MultiVis(targetNode){
                         if (gi.cluster != null) { self.unhighlightCluster(".cluster-" + gi.cluster); }
                     })
                     .on("click", function(gi) {
-                        if (gi.cluster != null) { self.toggleClusterView(gi.cluster, gi.color); }
+                        if (gi.cluster != null) { self.openClusterPage(self, gi.cluster, gi.color); }
                     })
                     .append("title").text(function(gi) {
                         if(gi.cluster != null) { return "Click to toggle GI cluster " + gi.cluster + " view"; }
@@ -721,6 +721,49 @@ function MultiVis(targetNode){
         self.unhighlightCluster(clusterClass);
     };
 
+    this.openClusterPage = function(multiVis, cluster, color) {
+        var clusterDict = {};
+        clusterDict["cluster"] = cluster;
+        clusterDict["color"] = color;
+        var sequences = {};
+        // Get all GIs in cluster
+        var cluster = $(".cluster-" + cluster);
+        // For each cluster record seq, start, end
+        cluster.each(function() {
+            cluster = $(this);
+            var seq = cluster.parents(".sequences").attr("sequence");
+            var start = cluster.attr("start");
+            var end = cluster.attr("end");
+            if (sequences[seq] === undefined) {
+                sequences[seq] = {"id": seq, "islands": []};
+            }
+            sequences[seq]["islands"].push([start, end]);
+        });
+        for (var i = 0; i < multiVis.sequences.length; i++) {
+            var currentSeq = multiVis.sequences[i];
+            if (sequences[currentSeq.sequenceId] !== undefined) {
+                sequences[currentSeq.sequenceId]["name"] = currentSeq.sequenceName;
+            }
+        }
+        clusterDict["sequences"] = [];
+        for (var i in sequences) {
+            clusterDict["sequences"].push(sequences[i]);
+        }
+        // For each seq, record homologous regions
+
+
+        var clusterPage = open("cluster.html");
+        clusterPage.clusterDict = clusterDict;
+
+        // To load cluster page on current page
+        // $("#content").load("cluster.html",
+        //     null,
+        //     function(){
+        //         createClusterVisualization(clusterDict, multiVis);
+        //     }
+        // );
+    };
+
     return this;
 }
 
diff --git a/web/pages/cluster.html b/web/pages/cluster.html
new file mode 100644
index 00000000..996f914f
--- /dev/null
+++ b/web/pages/cluster.html
@@ -0,0 +1,283 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <title>IslandCompare Cluster</title>
+  <!-- Tell the browser to be responsive to screen width -->
+  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">
+  <!-- Bootstrap 3.3.6 -->
+  <link rel="stylesheet" href="../bootstrap/css/bootstrap.min.css">
+  <!-- Font Awesome -->
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.5.0/css/font-awesome.min.css">
+  <!-- Ionicons -->
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/ionicons/2.0.1/css/ionicons.min.css">
+  <!-- Theme style -->
+  <link rel="stylesheet" href="../dist/css/AdminLTE.min.css">
+  <!-- AdminLTE Skins -->
+  <link rel="stylesheet" href="../dist/css/skins/skin-blue-light.min.css">
+  <!-- Bootstrap Diag -->
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap3-dialog/1.34.7/css/bootstrap-dialog.min.css">
+
+
+    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
+  <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+  <!--[if lt IE 9]>
+  <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+  <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+  <![endif]-->
+</head>
+
+<body class="hold-transition skin-blue-light fixed">
+<div class="wrapper">
+
+  <!-- Main Header -->
+  <header class="main-header">
+
+    <!-- Logo -->
+    <a href="#" class="logo">
+        <span class="logo-mini"><b>IC</b></span>
+        <span class="logo-lg"><b>IslandCompare</b></span>
+    </a>
+
+    <!-- Header Navbar -->
+    <nav class="navbar navbar-static-top" role="navigation">
+    <!-- Sidebar toggle button-->
+      <a href="#" class="sidebar-toggle" data-toggle="offcanvas" role="button">
+        <span class="sr-only">Toggle navigation</span>
+      </a>
+      <!-- Navbar Right Menu -->
+      <div class="navbar-custom-menu">
+        <ul class="nav navbar-nav">
+          <!-- User Account Menu -->
+          <li class="dropdown user user-menu">
+            <!-- Menu Toggle Button -->
+            <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+              <!-- The user image in the navbar-->
+              <i class="fa fa-user"></i>
+              <!-- hidden-xs hides the username on small devices so only the image appears. -->
+              <span class="hidden-xs" id="profile-header">Profile</span>
+            </a>
+            <ul class="dropdown-menu">
+              <!-- Menu Body-->
+                <div class="btn-group-vertical btn-block">
+                    <button type="button" class="btn btn-default btn-block">Settings</button>
+                    <button type="button" class="btn btn-default btn-block" id="signout-button">Sign out</button>
+                </div>
+            </ul>
+          </li>
+        </ul>
+      </div>
+    </nav>
+  </header>
+  <!-- Left side column. contains the logo and sidebar -->
+  <aside class="main-sidebar">
+
+    <!-- sidebar: style can be found in sidebar.less -->
+    <section class="sidebar">
+
+      <!-- Sidebar Menu -->
+      <ul class="sidebar-menu">
+        <li class="header">Navigation</li>
+        <!-- Optionally, you can add icons to the links -->
+        <li>
+            <a href="#" id="home-link"><i class="fa fa-home"></i> <span>Home</span>
+            <span class="pull-right-container">
+              <i class="fa fa-angle-left pull-right"></i>
+            </span></a>
+        </li>
+        <li><a href="#" id="jobs-link"><i class="fa fa-tasks"></i> <span>Job History</span>
+              <span class="pull-right-container">
+              <i class="fa fa-angle-left pull-right"></i>
+            </span></a>
+        </li>
+        <li><a href="#" id="api-link"><i class="fa fa-file"></i> <span>API</span>
+            <span class="pull-right-container">
+              <i class="fa fa-angle-left pull-right"></i>
+            </span></a>
+        </li>
+        <li><a href="#" id="about-link"><i class="fa fa-info-circle"></i> <span>About</span>
+            <span class="pull-right-container">
+              <i class="fa fa-angle-left pull-right"></i>
+            </span></a>
+        </li>
+      </ul>
+      <!-- /.sidebar-menu -->
+    </section>
+    <!-- /.sidebar -->
+  </aside>
+
+  <!-- Content Wrapper. Contains page content -->
+  <div class="content-wrapper" id="content">
+      <!-- Content Header (Page header) -->
+        <section class="content-header">
+            <h1 id="header"></h1>
+            <ol class="breadcrumb">
+                <li class="active"><a href="#"><i class="fa fa-dashboard"></i> Cluster</a></li>
+            </ol>
+        </section>
+
+        <section class="content">
+            <div class="box">
+                <div class="box-header with-border">
+                    <!--<button id="backButton" class="btn btn-primary btn-flat"><i class="fa fa-chevron-left"></i> Back</button>-->
+                </div>
+                <div class="box-body">
+                    <div id="visualizationBody"></div>
+                </div>
+            </div>
+        </section>
+  </div>
+  <!-- /.content-wrapper -->
+</div>
+
+<!-- REQUIRED JS SCRIPTS -->
+
+<!-- jQuery 2.2.3 -->
+<script src="../plugins/jQuery/jquery-2.2.3.min.js"></script>
+<!-- Bootstrap 3.3.6 -->
+<script src="../bootstrap/js/bootstrap.min.js"></script>
+<!-- slimscroll -->
+<script src="../plugins/slimScroll/jquery.slimscroll.min.js"></script>
+<!-- FastClick -->
+<script src="../plugins/fastclick/fastclick.js"></script>
+<!-- AdminLTE App -->
+<script src="../dist/js/app.min.js"></script>
+<!-- Load REST urls -->
+<script src="../dist/js/islandcompare-rest-urls.js"></script>
+<!-- Load auth -->
+<script src="../dist/js/auth.js"></script>
+<!-- Bootstrap Diag -->
+<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap3-dialog/1.34.7/js/bootstrap-dialog.min.js"></script>
+<!-- D3 -->
+<script src="../dist/js/d3.min.js"></script>
+
+<script>
+    $(document).ready(function(){
+        requireLogin();
+    });
+
+    $("#signout-button").click(function(){
+        deleteAuthenticationCookie();
+        window.location.href="login.html"
+    });
+
+    $("#home-link").click(function(){
+        $(".sidebar-toggle").off("click");
+        $("#content").load("home.html");
+    });
+
+    $("#jobs-link").click(function(){
+        $(".sidebar-toggle").off("click");
+        $("#content").load("history.html");
+    });
+
+    $("#api-link").click(function(){
+        window.open(docUrl);
+    });
+
+    $("#about-link").click(function(){
+        $(".sidebar-toggle").off("click");
+        $("#content").load("about.html");
+    });
+
+    // Re-calls the visualization when the sidebar is toggled to adjust widths
+    $(".sidebar-toggle").on("click", function() {
+        setTimeout(function() {
+            createClusterVisualization();
+        }, 500);
+    });
+
+    var createClusterVisualization = function() {
+        var clusterDict = window.clusterDict;
+        if (clusterDict == undefined) {
+            clusterDict = JSON.parse(sessionStorage.getItem("clusterDict"));
+        } else {
+            // Store the clusterDict so it can be retrieved if the page is refreshed
+            sessionStorage.setItem("clusterDict", JSON.stringify(clusterDict));
+        }
+
+        var cluster = clusterDict["cluster"];
+        $("#header").html("Cluster " + cluster);
+        var color = clusterDict["color"];
+        var visWidth = $("#visualizationBody").width();
+        var scale = getScale(clusterDict["sequences"], visWidth - 50);
+
+        // D3.js //
+        var seq = d3.select("#visualizationBody")
+            .selectAll("div")
+            .data(clusterDict["sequences"]);
+
+        // Update widths
+        seq.select("svg").attr("width", visWidth - 30);
+        seq.select("svg").selectAll("rect")
+            .attr("width", function(island) {
+                return scale(Math.abs(island[1] - island[0]));
+            });
+
+        // Enter
+        var div = seq.enter().append("div");
+        div.attr("class", function(seq) {
+            return seq.name;
+        });
+
+        div.append("p")
+            .html(function(seq) {
+                return seq.name;
+            });
+
+        var svg = div.append("svg")
+            .attr("height", function(seq) {
+                return seq.islands.length * 25;
+            })
+            .attr("width", visWidth - 30);
+
+        var rects = svg.selectAll("rect")
+            .data(function(seq) {
+                return seq.islands;
+            });
+
+        rects.enter().append("rect")
+            .attr("x", 10)
+            .attr("y", function(d, i) {
+                return i * 25;
+            })
+            .attr("width", function(island) {
+                return scale(Math.abs(island[1] - island[0]));
+            })
+            .attr("height", 10)
+            .attr("rx", 5)
+            .attr("ry", 5)
+            .attr("fill", color);
+    };
+
+    function getScale(sequences, containerWidth) {
+        var lengths = [];
+        for (var seqIndex = 0; seqIndex < sequences.length; seqIndex++) {
+            var islands = sequences[seqIndex]["islands"];
+            for (var islandIndex = 0; islandIndex < islands.length; islandIndex++) {
+                lengths.push(Math.abs(islands[islandIndex][1] - islands[islandIndex][0]));
+            }
+        }
+        var max = Math.max.apply(null, lengths);
+        return d3.scale.linear()
+            .domain([0,max])
+            .range([0,containerWidth])
+            .clamp(true);
+    }
+
+    $(document).ready(createClusterVisualization);
+
+//    // To load the cluster view on the same page
+//    function createClusterVisualization(clusterDict, multiVis) {
+//        $("#backButton").on("click", this, function() {
+//            $("#content").load("result.html",
+//                null,
+//                function(){
+//                    createVisualization(url);
+//                }
+//            );
+//        });
+//    }
+</script>
+</body>

From 1b34a3b3fac216c4b31ef2990e62da5883ebf83f Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 14 Aug 2017 17:55:48 -0700
Subject: [PATCH 67/76] Add start and end text to cluster visualization

---
 web/pages/cluster.html | 51 ++++++++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/web/pages/cluster.html b/web/pages/cluster.html
index 996f914f..82df07a6 100644
--- a/web/pages/cluster.html
+++ b/web/pages/cluster.html
@@ -189,6 +189,12 @@ <h1 id="header"></h1>
     });
 
     var createClusterVisualization = function() {
+        var ENDPADDING = 50;
+        var SEQPADDING = 40;
+        var SVGPADDING = 30;
+        var TEXTOFFSET = 15;
+        var RECTOFFSET = 20;
+
         var clusterDict = window.clusterDict;
         if (clusterDict == undefined) {
             clusterDict = JSON.parse(sessionStorage.getItem("clusterDict"));
@@ -201,7 +207,7 @@ <h1 id="header"></h1>
         $("#header").html("Cluster " + cluster);
         var color = clusterDict["color"];
         var visWidth = $("#visualizationBody").width();
-        var scale = getScale(clusterDict["sequences"], visWidth - 50);
+        var scale = getScale(clusterDict["sequences"], visWidth - SVGPADDING);
 
         // D3.js //
         var seq = d3.select("#visualizationBody")
@@ -209,11 +215,15 @@ <h1 id="header"></h1>
             .data(clusterDict["sequences"]);
 
         // Update widths
-        seq.select("svg").attr("width", visWidth - 30);
+        seq.select("svg").attr("width", visWidth);
         seq.select("svg").selectAll("rect")
             .attr("width", function(island) {
                 return scale(Math.abs(island[1] - island[0]));
             });
+        seq.select("svg").selectAll("g").select("text:nth-child(2)")
+            .attr("x", function(d) {
+                return scale(Math.abs(d[1] - d[0])) - ENDPADDING;
+            });
 
         // Enter
         var div = seq.enter().append("div");
@@ -221,31 +231,48 @@ <h1 id="header"></h1>
             return seq.name;
         });
 
-        div.append("p")
+        div.append("p").append("b")
             .html(function(seq) {
                 return seq.name;
             });
 
         var svg = div.append("svg")
             .attr("height", function(seq) {
-                return seq.islands.length * 25;
+                return seq.islands.length * SEQPADDING;
             })
-            .attr("width", visWidth - 30);
+            .attr("width", visWidth - SVGPADDING);
 
-        var rects = svg.selectAll("rect")
+        var rectGroup = svg.selectAll("g")
             .data(function(seq) {
                 return seq.islands;
             });
 
-        rects.enter().append("rect")
-            .attr("x", 10)
+        rectGroup.enter().append("g");
+        rectGroup.append("text")
+            .text(function(d) {
+                return d[0];
+            })
             .attr("y", function(d, i) {
-                return i * 25;
+                return (i * SEQPADDING) + TEXTOFFSET;
+            });
+        rectGroup.append("text")
+            .text(function(d) {
+                return d[1];
             })
-            .attr("width", function(island) {
-                return scale(Math.abs(island[1] - island[0]));
+            .attr("x", function(d) {
+                return scale(Math.abs(d[1] - d[0])) - ENDPADDING;
+            })
+            .attr("y", function(d, i) {
+                return (i * SEQPADDING) + TEXTOFFSET;
+            });
+        rectGroup.append("rect")
+            .attr("y", function(d, i) {
+                return (i * SEQPADDING) + RECTOFFSET;
+            })
+            .attr("height", "10")
+            .attr("width", function(d) {
+                return scale(Math.abs(d[1] - d[0]));
             })
-            .attr("height", 10)
             .attr("rx", 5)
             .attr("ry", 5)
             .attr("fill", color);

From 3006dfd0a68cb5b55d1ab467c7159cc3f71c3480 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 15 Aug 2017 13:35:35 -0700
Subject: [PATCH 68/76] Improved padding and fixed sequence ordering on cluster
 visualization

---
 web/dist/js/genomecomparevis.js | 38 +++++++++++++++++++--------------
 web/pages/cluster.html          | 24 +++++++++++++--------
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 990204ab..b08da20a 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -394,7 +394,13 @@ function MultiVis(targetNode){
                         if (gi.cluster != null) { self.unhighlightCluster(".cluster-" + gi.cluster); }
                     })
                     .on("click", function(gi) {
-                        if (gi.cluster != null) { self.openClusterPage(self, gi.cluster, gi.color); }
+                        if (gi.cluster != null) {
+                            try {
+                                self.openClusterPage(self, gi.cluster, gi.color);
+                            } catch(err) {
+                                alert("Problem encountered when constructing cluster view: \n\n" + err);
+                            }
+                        }
                     })
                     .append("title").text(function(gi) {
                         if(gi.cluster != null) { return "Click to toggle GI cluster " + gi.cluster + " view"; }
@@ -723,34 +729,34 @@ function MultiVis(targetNode){
 
     this.openClusterPage = function(multiVis, cluster, color) {
         var clusterDict = {};
-        clusterDict["cluster"] = cluster;
-        clusterDict["color"] = color;
-        var sequences = {};
+        clusterDict.cluster = cluster;
+        clusterDict.color = color;
+        clusterDict.sequences = [];
         // Get all GIs in cluster
         var cluster = $(".cluster-" + cluster);
-        // For each cluster record seq, start, end
+        // For each cluster record ID, start, end
         cluster.each(function() {
             cluster = $(this);
-            var seq = cluster.parents(".sequences").attr("sequence");
+            var seqID = cluster.parents(".sequences").attr("sequence");
             var start = cluster.attr("start");
             var end = cluster.attr("end");
-            if (sequences[seq] === undefined) {
-                sequences[seq] = {"id": seq, "islands": []};
+            var index = clusterDict.sequences.findIndex(i => i.id === seqID);
+            if (index === -1) {
+                clusterDict.sequences.push({"id": seqID, "islands": []});
+                index = clusterDict.sequences.length - 1;
             }
-            sequences[seq]["islands"].push([start, end]);
+            clusterDict.sequences[index].islands.push([start, end]);
         });
+        // Assign sequence names
         for (var i = 0; i < multiVis.sequences.length; i++) {
             var currentSeq = multiVis.sequences[i];
-            if (sequences[currentSeq.sequenceId] !== undefined) {
-                sequences[currentSeq.sequenceId]["name"] = currentSeq.sequenceName;
+            var index = clusterDict.sequences.findIndex(i => i.id === currentSeq.sequenceId);
+            if (index !== -1) {
+                clusterDict.sequences[index].name =  currentSeq.sequenceName;
             }
         }
-        clusterDict["sequences"] = [];
-        for (var i in sequences) {
-            clusterDict["sequences"].push(sequences[i]);
-        }
-        // For each seq, record homologous regions
 
+        clusterDict.alignment = multiVis.backbone.backbone;
 
         var clusterPage = open("cluster.html");
         clusterPage.clusterDict = clusterDict;
diff --git a/web/pages/cluster.html b/web/pages/cluster.html
index 82df07a6..a209e959 100644
--- a/web/pages/cluster.html
+++ b/web/pages/cluster.html
@@ -185,11 +185,11 @@ <h1 id="header"></h1>
     $(".sidebar-toggle").on("click", function() {
         setTimeout(function() {
             createClusterVisualization();
-        }, 500);
+        }, 300);
     });
 
     var createClusterVisualization = function() {
-        var ENDPADDING = 50;
+        var ENDPADDING = 7;
         var SEQPADDING = 40;
         var SVGPADDING = 30;
         var TEXTOFFSET = 15;
@@ -203,16 +203,16 @@ <h1 id="header"></h1>
             sessionStorage.setItem("clusterDict", JSON.stringify(clusterDict));
         }
 
-        var cluster = clusterDict["cluster"];
+        var cluster = clusterDict.cluster;
         $("#header").html("Cluster " + cluster);
-        var color = clusterDict["color"];
+        var color = clusterDict.color;
         var visWidth = $("#visualizationBody").width();
-        var scale = getScale(clusterDict["sequences"], visWidth - SVGPADDING);
+        var scale = getScale(clusterDict.sequences, visWidth - SVGPADDING);
 
         // D3.js //
         var seq = d3.select("#visualizationBody")
             .selectAll("div")
-            .data(clusterDict["sequences"]);
+            .data(clusterDict.sequences);
 
         // Update widths
         seq.select("svg").attr("width", visWidth);
@@ -222,7 +222,10 @@ <h1 id="header"></h1>
             });
         seq.select("svg").selectAll("g").select("text:nth-child(2)")
             .attr("x", function(d) {
-                return scale(Math.abs(d[1] - d[0])) - ENDPADDING;
+                return Math.max(
+                    scale(Math.abs(d[1] - d[0])) - (d[1].length * ENDPADDING),
+                    (d[1].length + 1) * ENDPADDING
+                );
             });
 
         // Enter
@@ -260,7 +263,10 @@ <h1 id="header"></h1>
                 return d[1];
             })
             .attr("x", function(d) {
-                return scale(Math.abs(d[1] - d[0])) - ENDPADDING;
+                return Math.max(
+                    scale(Math.abs(d[1] - d[0])) - (d[1].length * ENDPADDING),
+                    (d[1].length + 1) * ENDPADDING
+                );
             })
             .attr("y", function(d, i) {
                 return (i * SEQPADDING) + TEXTOFFSET;
@@ -281,7 +287,7 @@ <h1 id="header"></h1>
     function getScale(sequences, containerWidth) {
         var lengths = [];
         for (var seqIndex = 0; seqIndex < sequences.length; seqIndex++) {
-            var islands = sequences[seqIndex]["islands"];
+            var islands = sequences[seqIndex].islands;
             for (var islandIndex = 0; islandIndex < islands.length; islandIndex++) {
                 lengths.push(Math.abs(islands[islandIndex][1] - islands[islandIndex][0]));
             }

From 71ea3450fd38379c8c56b8f0a8a64252dc5c5dc9 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Fri, 18 Aug 2017 17:57:01 -0700
Subject: [PATCH 69/76] MashMclClusterPipelineComponent will properly assign
 cluster numbers to genomic islands

---
 IslandCompare/analysis/components.py | 77 ++++++++++------------------
 1 file changed, 28 insertions(+), 49 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 8dd940dd..ecf09fd7 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -714,17 +714,11 @@ def calculate_mash_distance(self, referenceFile, queryFastaFile, outputFile):
         subprocess.check_call(scriptFile.name)
         scriptFile.close()
 
-    def parse_mash_output(self, outputFile):
-        outputList = []
-        with open(outputFile, 'r') as output:
+    def parse_mash_output(self, output_file):
+        with open(output_file, 'r') as output:
             reader = csv.reader(output, delimiter='\t')
-            for row in reader:
-                outputList.append({'referenceId': row[0],
-                                   'queryId': row[1],
-                                   'mashDistance': row[2],
-                                   'pValue': row[3],
-                                   'matchingHashes': row[4]})
-        return outputList
+            # The third column contains the mash distance
+            return [row[2] for row in reader]
 
     def get_subsequence(self, record, startPosition, endPosition, islandNumber, description=None):
         if description is not None:
@@ -773,54 +767,39 @@ def analysis(self, report):
         if len(island_path_list) > 0:
             self.create_compound_sketch(island_path_list, self.temp_dir_path + "/compoundScratch")
 
+            # Calculates the distance [0,1] from each genomic island to each genomic island
             distance_matrix = []
-
             for island in island_path_list:
                 self.logger.info("Processing island: " + island)
                 if os.path.isfile(self.temp_dir_path + "/output"):
                     os.remove(self.temp_dir_path + "/output")
                 self.calculate_mash_distance(self.temp_dir_path + "/compoundScratch.msh", island, self.temp_dir_path + "/output")
-                distance_matrix.append([float(i['mashDistance']) for i in self.parse_mash_output(self.temp_dir_path + "/output")])
-
-            baseMatrix = []
-            for i in range(len(island_path_list)):
-                nextRow = []
-                for j in range(len(island_path_list)):
-                    nextRow.append(1)
-                baseMatrix.append(nextRow)
-            numpyBaseMatrix = np.array(baseMatrix)
-            numpyDistanceMatrix = np.array(distance_matrix)
-
-            mclAdjacencyMatrix = np.subtract(numpyBaseMatrix, numpyDistanceMatrix)
+                distance_matrix.append([float(i) for i in self.parse_mash_output(self.temp_dir_path + "/output")])
+
+            numpy_distance_matrix = np.array(distance_matrix)
+            # Convert to adjacency matrix by setting each value as 1 - value
+            mcl_adjacency_matrix = np.vectorize(lambda i: 1 - i)(numpy_distance_matrix)
             np.set_printoptions(threshold='nan')
 
-            M, clusters = mcl(mclAdjacencyMatrix)
-            output_dict = {}
-
-            for genome_id in report["gbk_paths"]:
-                output_dict[str(genome_id)] = {}
-            islandIdList = list([i for i in range(len(island_path_list))])
-
-            numberClusters = 0
-
-            while len(islandIdList) > 0:
-                numberClusters += 1
-                currentCluster = clusters[islandIdList[0]]
-                for i in currentCluster:
-                    self.logger.info("Assigning clusters for cluster {}".format(numberClusters))
-                    island = island_path_list[i]
-                    splitIsland = island.split('/')[-2:]
-                    currentSequenceId = splitIsland[0]
-                    islandId = splitIsland[1]
-                    self.logger.info("Current Sequence Id: " + str(currentSequenceId))
-                    self.logger.info("Current Island Id: " + str(islandId))
-                    output_dict[str(currentSequenceId)][str(islandId)] = numberClusters - 1
-                remainingIslands = filter(lambda x: x not in currentCluster, islandIdList)
-                islandIdList = list(remainingIslands)
-
-            report['numberClusters'] = numberClusters
+            # mcl computes genomic island clusters
+            M, raw_clusters = mcl(mcl_adjacency_matrix)
+
+            processed_clusters = {str(genome_id): {} for genome_id in report["gbk_paths"]}
+            cluster_number = 0
+            for island in raw_clusters:
+                advance_flag = 0 # Only advance the cluster_number if it is used
+                for cluster_mate in raw_clusters[island]:
+                    # island_path_list to get the genome ID and genome-specific GI number from the unspecific GI number
+                    genome_id, island_num = island_path_list[cluster_mate].split("/")[-2:]
+                    # Only add GIs to the same cluster if they both list each other in their cluster
+                    if island in raw_clusters[cluster_mate] and island_num not in processed_clusters[genome_id]:
+                        processed_clusters[genome_id][island_num] = cluster_number
+                        advance_flag = 1
+                cluster_number += advance_flag
+
+            report['numberClusters'] = cluster_number
             analysis = Analysis.objects.get(id=report['analysis'])
-            analysis.clusters = str(output_dict)
+            analysis.clusters = str(processed_clusters)
             analysis.save()
 
     def cleanup(self):

From f52313e0dc31591b5f471b89ce793a0f2b85b5b8 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 22 Aug 2017 13:33:48 -0700
Subject: [PATCH 70/76] Applied cutoff to cluster adjacency matrix to improve
 cluster homology

---
 IslandCompare/analysis/components.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index ecf09fd7..b9821dff 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -755,6 +755,14 @@ def create_gi_fasta_files(self, report):
 
         return island_path_list
 
+    def apply_cutoff(self, matrix, cutoff=0.95):
+        n = len(matrix)
+        for i in range(n - 1):
+            for j in range(i + 1, n):
+                if matrix[i][j] < cutoff:
+                    matrix[i][j] = matrix[j][i] = 0
+        return matrix
+
     def setup(self, report):
         self.temp_dir_path = self.output_dir + str(report["analysis"])
         os.mkdir(self.temp_dir_path, 0o777)
@@ -781,6 +789,9 @@ def analysis(self, report):
             mcl_adjacency_matrix = np.vectorize(lambda i: 1 - i)(numpy_distance_matrix)
             np.set_printoptions(threshold='nan')
 
+            # The cutoff ensures only very closely matches islands will be placed in a cluster together
+            mcl_adjacency_matrix = self.apply_cutoff(mcl_adjacency_matrix)
+
             # mcl computes genomic island clusters
             M, raw_clusters = mcl(mcl_adjacency_matrix)
 

From 7c904e01ce1f4c13d944e0379819e4a4f2258d54 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 24 Aug 2017 11:13:54 -0700
Subject: [PATCH 71/76] Update ReportCsvSerializer and history page for user
 uploaded GIs

---
 IslandCompare/analysis/serializers.py | 14 +++++++++++---
 web/pages/history.html                | 26 ++++++++++++++++++++------
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/IslandCompare/analysis/serializers.py b/IslandCompare/analysis/serializers.py
index 3bc3107d..6490a51b 100644
--- a/IslandCompare/analysis/serializers.py
+++ b/IslandCompare/analysis/serializers.py
@@ -199,6 +199,7 @@ def to_representation(self, instance):
         output["alignment"] = instance["alignment"]
         output["failed_components"] = instance["failed_components"]
         output["cluster_gis"] = clustering and gi_method # 'user', 'merge', or False
+        output["analysis_name"] = analysis.name
 
         return output
 
@@ -222,19 +223,26 @@ def to_representation(self, analysis):
         writer = csv.DictWriter(output, fieldnames=fieldnames)
         writer.writeheader()
 
-        method_keys = ["merge", "islandpath", "sigi"]
+        if analysis.usergenomicisland_set.exists():
+            method_keys = ["user"]
+        else:
+            method_keys = ["merge", "islandpath", "sigi"]
         clusters = ast.literal_eval(analysis.clusters)
 
         for method in method_keys:
             for genome in analysis.genomes.all():
-                for i, island in enumerate(genome.genomicisland_set.filter(method=method)):
+                if method == "user":
+                    gis = analysis.usergenomicisland_set.filter(genome=genome)
+                else:
+                    gis = genome.genomicisland_set.filter(method=method)
+                for i, island in enumerate(gis):
                     row = {
                         'name': genome.name,
                         'start': island.start,
                         'end': island.end,
                         'method': method + "_gis"
                     }
-                    if method == "merge":
+                    if method in ["merge", "user"]:
                         row['cluster_id'] = clusters[str(genome.id)][str(i)]
                     writer.writerow(row)
 
diff --git a/web/pages/history.html b/web/pages/history.html
index 8a066bf4..fccbc695 100644
--- a/web/pages/history.html
+++ b/web/pages/history.html
@@ -211,7 +211,7 @@ <h3 class="box-title">Job History</h3>
                                 return full['analysiscomponent_set']['mash_mcl']['status']
                             }
                         } else {
-                            return "UNKNOWN"
+                            return "USER DEFINED CLUSTERS"
                         }
                     }
                 },
@@ -291,6 +291,7 @@ <h3 class="box-title">Job History</h3>
                         var analysis_id = full['id'];
                         var sigi_status = null;
                         var islandpath_status = null;
+                        var mash_mcl_status = true;
                         var end_status = null;
                         if( typeof full['analysiscomponent_set']['sigi'] !== 'undefined' ) {
                             sigi_status = full['analysiscomponent_set']['sigi']['status'];
@@ -304,6 +305,9 @@ <h3 class="box-title">Job History</h3>
                         else{
                             islandpath_status = "UNKNOWN"
                         }
+                        if ( typeof full['analysiscomponent_set']['mash_mcl'] === 'undefined') {
+                            mash_mcl_status = false;
+                        }
                         if( typeof full['analysiscomponent_set']['end_pipeline'] !== 'undefined' ) {
                             end_status = full['analysiscomponent_set']['end_pipeline']['status'];
                         }
@@ -314,11 +318,21 @@ <h3 class="box-title">Job History</h3>
                             return '<button class="btn btn-block btn-flat disabled recent-job-item"><i class="fa fa-download" aria-hidden="true"></button>';
                         }
                         else if (end_status === "SUCCESS") {
-                            return '<a href="#" onclick="javascript:export_gis(' + analysis_id + ');">' +
-                                       '<button class="btn btn-block btn-flat btn-success btn-xs recent-job-item">' +
-                                           'GIs' +
-                                       '</button>' +
-                                   '</a>' +
+                            var gis_button = null;
+                            if (mash_mcl_status) {
+                                gis_button = '<a href="#" onclick="javascript:export_gis(' + analysis_id + ');">' +
+                                                 '<button class="btn btn-block btn-flat btn-success btn-xs recent-job-item">' +
+                                                     'GIs' +
+                                                 '</button>' +
+                                             '</a>'
+                            } else {
+                                gis_button = '<a href="#">' +
+                                                 '<button class="btn btn-block btn-flat disabled btn-xs">' +
+                                                     'GIs' +
+                                                 '</button>' +
+                                             '</a>'
+                            }
+                            return gis_button +
                                    '<a href="#" onclick="javascript:export_genes(' + analysis_id + ');">' +
                                        '<button class="btn btn-block btn-flat btn-success btn-xs recent-job-item">' +
                                            'GI Genes' +

From 47dcdc8172680c84d4ff994c3c9971bda2e4a69d Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 24 Aug 2017 11:15:56 -0700
Subject: [PATCH 72/76] Add job title and loading message to result page

---
 web/pages/result.html | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/web/pages/result.html b/web/pages/result.html
index a7db2377..d15771bf 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -13,7 +13,7 @@
         <div hidden id="warning-box" class="alert alert-warning alert-dismissable col-xs-6 text-center"></div>
         <div class="col-xs-3"></div>
     </div>
-    <h1>
+    <h1 id="resultHeader">
         Results
     </h1>
     <ol class="breadcrumb">
@@ -142,7 +142,9 @@ <h1>
         </div>
         <!-- /.box-header -->
         <div class="box-body">
-            <div id="visualization-body"></div>
+            <div id="visualization-body">
+                <h1 id="loading" class="text-center"><i class='fa fa-spinner fa-spin'></i> Loading results</h1>
+            </div>
             <svg>
                 <filter id="shadow" x="-5" y="-5" width="200" height="200">
                     <feGaussianBlur in="SourceAlpha" stdDeviation="5" />
@@ -173,6 +175,8 @@ <h1>
                 request.setRequestHeader("Authorization", 'Token'+' '+getAuthenticationCookie());
             },
             success: function(data) {
+                $("#resultHeader").html(data["analysis_name"]);
+                $("#loading").hide();
                 var sigiFails = 0;
                 var islandPathFails = 0;
                 if (Object.keys(data["failed_components"]).length) {

From a7b683b469bbb17bad062e1a59fb5f37edb7fe39 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Thu, 24 Aug 2017 17:27:57 -0700
Subject: [PATCH 73/76] Added genes to cluster visualization

---
 IslandCompare/analysis/components.py |   2 +-
 web/dist/css/linearplot.css          |   6 +-
 web/dist/js/genomecomparevis.js      |  25 +-
 web/pages/cluster.html               | 326 +++++++++++++--------------
 web/pages/result.html                |   4 +-
 5 files changed, 169 insertions(+), 194 deletions(-)

diff --git a/IslandCompare/analysis/components.py b/IslandCompare/analysis/components.py
index 6511a9ba..d691ae27 100644
--- a/IslandCompare/analysis/components.py
+++ b/IslandCompare/analysis/components.py
@@ -752,7 +752,7 @@ def create_gi_fasta_files(self, report):
 
         return island_path_list
 
-    def apply_cutoff(self, matrix, cutoff=0.95):
+    def apply_cutoff(self, matrix, cutoff=0.96):
         n = len(matrix)
         for i in range(n - 1):
             for j in range(i + 1, n):
diff --git a/web/dist/css/linearplot.css b/web/dist/css/linearplot.css
index 175392c0..b2eba5d8 100644
--- a/web/dist/css/linearplot.css
+++ b/web/dist/css/linearplot.css
@@ -69,17 +69,17 @@ g.root.node circle{
 
 .genes polygon.gene, .genes polygon.CDS, #genesLegend{
     fill: #6acbff;
-    stroke: #6acbff;
+    /*stroke: #6acbff;*/
 }
 
 .genes polygon.tRNA, #tRNALegend{
     fill: #c26dff;
-    stroke: #c26dff;
+    /*stroke: #c26dff;*/
 }
 
 .genes polygon.rRNA, #rRNALegend{
     fill: #7ceeb2;
-    stroke: #7ceeb2;
+    /*stroke: #7ceeb2;*/
 }
 
 .genes.print polygon, #genesLegend.print, #tRNALegend.print, #rRNALegend.print{
diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index b08da20a..8e550958 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -403,7 +403,7 @@ function MultiVis(targetNode){
                         }
                     })
                     .append("title").text(function(gi) {
-                        if(gi.cluster != null) { return "Click to toggle GI cluster " + gi.cluster + " view"; }
+                        if(gi.cluster != null) { return "Click to open GI cluster " + gi.cluster + " view"; }
                     });
             });
         });
@@ -727,25 +727,24 @@ function MultiVis(targetNode){
         self.unhighlightCluster(clusterClass);
     };
 
-    this.openClusterPage = function(multiVis, cluster, color) {
+    this.openClusterPage = function(multiVis, clusterNum, color) {
         var clusterDict = {};
-        clusterDict.cluster = cluster;
+        clusterDict.cluster = clusterNum;
         clusterDict.color = color;
         clusterDict.sequences = [];
         // Get all GIs in cluster
-        var cluster = $(".cluster-" + cluster);
+        var cluster = $(".cluster-" + clusterNum);
         // For each cluster record ID, start, end
         cluster.each(function() {
-            cluster = $(this);
-            var seqID = cluster.parents(".sequences").attr("sequence");
-            var start = cluster.attr("start");
-            var end = cluster.attr("end");
+            var seqID = $(this).parents(".sequences").attr("sequence");
+            var start = $(this).attr("start");
+            var end = $(this).attr("end");
             var index = clusterDict.sequences.findIndex(i => i.id === seqID);
             if (index === -1) {
                 clusterDict.sequences.push({"id": seqID, "islands": []});
                 index = clusterDict.sequences.length - 1;
             }
-            clusterDict.sequences[index].islands.push([start, end]);
+            clusterDict.sequences[index].islands.push([start, end, seqID]);
         });
         // Assign sequence names
         for (var i = 0; i < multiVis.sequences.length; i++) {
@@ -760,14 +759,6 @@ function MultiVis(targetNode){
 
         var clusterPage = open("cluster.html");
         clusterPage.clusterDict = clusterDict;
-
-        // To load cluster page on current page
-        // $("#content").load("cluster.html",
-        //     null,
-        //     function(){
-        //         createClusterVisualization(clusterDict, multiVis);
-        //     }
-        // );
     };
 
     return this;
diff --git a/web/pages/cluster.html b/web/pages/cluster.html
index a209e959..32351e7a 100644
--- a/web/pages/cluster.html
+++ b/web/pages/cluster.html
@@ -1,23 +1,25 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-  <meta charset="utf-8">
-  <meta http-equiv="X-UA-Compatible" content="IE=edge">
-  <title>IslandCompare Cluster</title>
-  <!-- Tell the browser to be responsive to screen width -->
-  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">
-  <!-- Bootstrap 3.3.6 -->
-  <link rel="stylesheet" href="../bootstrap/css/bootstrap.min.css">
-  <!-- Font Awesome -->
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.5.0/css/font-awesome.min.css">
-  <!-- Ionicons -->
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/ionicons/2.0.1/css/ionicons.min.css">
-  <!-- Theme style -->
-  <link rel="stylesheet" href="../dist/css/AdminLTE.min.css">
-  <!-- AdminLTE Skins -->
-  <link rel="stylesheet" href="../dist/css/skins/skin-blue-light.min.css">
-  <!-- Bootstrap Diag -->
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap3-dialog/1.34.7/css/bootstrap-dialog.min.css">
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <title>IslandCompare Cluster</title>
+    <!-- Tell the browser to be responsive to screen width -->
+    <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">
+    <!-- Bootstrap 3.3.6 -->
+    <link rel="stylesheet" href="../bootstrap/css/bootstrap.min.css">
+    <!-- Font Awesome -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.5.0/css/font-awesome.min.css">
+    <!-- Ionicons -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/ionicons/2.0.1/css/ionicons.min.css">
+    <!-- Theme style -->
+    <link rel="stylesheet" href="../dist/css/AdminLTE.min.css">
+    <!-- AdminLTE Skins -->
+    <link rel="stylesheet" href="../dist/css/skins/skin-blue-light.min.css">
+    <!-- Bootstrap Diag -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap3-dialog/1.34.7/css/bootstrap-dialog.min.css">
+
+    <link rel="stylesheet" href="../dist/css/linearplot.css">
 
 
     <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
@@ -28,107 +30,89 @@
   <![endif]-->
 </head>
 
-<body class="hold-transition skin-blue-light fixed">
+<body class="skin-blue-light">
 <div class="wrapper">
-
-  <!-- Main Header -->
-  <header class="main-header">
-
+    <!-- Main Header -->
+    <header class="main-header">
     <!-- Logo -->
     <a href="#" class="logo">
         <span class="logo-mini"><b>IC</b></span>
         <span class="logo-lg"><b>IslandCompare</b></span>
     </a>
-
     <!-- Header Navbar -->
     <nav class="navbar navbar-static-top" role="navigation">
-    <!-- Sidebar toggle button-->
-      <a href="#" class="sidebar-toggle" data-toggle="offcanvas" role="button">
-        <span class="sr-only">Toggle navigation</span>
-      </a>
-      <!-- Navbar Right Menu -->
-      <div class="navbar-custom-menu">
-        <ul class="nav navbar-nav">
-          <!-- User Account Menu -->
-          <li class="dropdown user user-menu">
-            <!-- Menu Toggle Button -->
-            <a href="#" class="dropdown-toggle" data-toggle="dropdown">
-              <!-- The user image in the navbar-->
-              <i class="fa fa-user"></i>
-              <!-- hidden-xs hides the username on small devices so only the image appears. -->
-              <span class="hidden-xs" id="profile-header">Profile</span>
-            </a>
-            <ul class="dropdown-menu">
-              <!-- Menu Body-->
-                <div class="btn-group-vertical btn-block">
-                    <button type="button" class="btn btn-default btn-block">Settings</button>
-                    <button type="button" class="btn btn-default btn-block" id="signout-button">Sign out</button>
-                </div>
+        <!-- Navbar Right Menu -->
+        <div class="navbar-custom-menu">
+            <ul class="nav navbar-nav">
+                <!-- User Account Menu -->
+                <li class="dropdown user user-menu">
+                    <!-- Menu Toggle Button -->
+                    <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+                        <!-- The user image in the navbar-->
+                        <i class="fa fa-user"></i>
+                        <!-- hidden-xs hides the username on small devices so only the image appears. -->
+                        <span class="hidden-xs" id="profile-header">Profile</span>
+                    </a>
+                    <ul class="dropdown-menu">
+                        <!-- Menu Body-->
+                        <div class="btn-group-vertical btn-block">
+                            <button type="button" class="btn btn-default btn-block">Settings</button>
+                            <button type="button" class="btn btn-default btn-block" id="signout-button">
+                                Sign out
+                            </button>
+                        </div>
+                    </ul>
+                </li>
             </ul>
-          </li>
-        </ul>
-      </div>
+        </div>
     </nav>
-  </header>
-  <!-- Left side column. contains the logo and sidebar -->
-  <aside class="main-sidebar">
-
-    <!-- sidebar: style can be found in sidebar.less -->
-    <section class="sidebar">
+    </header>
 
-      <!-- Sidebar Menu -->
-      <ul class="sidebar-menu">
-        <li class="header">Navigation</li>
-        <!-- Optionally, you can add icons to the links -->
-        <li>
-            <a href="#" id="home-link"><i class="fa fa-home"></i> <span>Home</span>
-            <span class="pull-right-container">
-              <i class="fa fa-angle-left pull-right"></i>
-            </span></a>
-        </li>
-        <li><a href="#" id="jobs-link"><i class="fa fa-tasks"></i> <span>Job History</span>
-              <span class="pull-right-container">
-              <i class="fa fa-angle-left pull-right"></i>
-            </span></a>
-        </li>
-        <li><a href="#" id="api-link"><i class="fa fa-file"></i> <span>API</span>
-            <span class="pull-right-container">
-              <i class="fa fa-angle-left pull-right"></i>
-            </span></a>
-        </li>
-        <li><a href="#" id="about-link"><i class="fa fa-info-circle"></i> <span>About</span>
-            <span class="pull-right-container">
-              <i class="fa fa-angle-left pull-right"></i>
-            </span></a>
-        </li>
-      </ul>
-      <!-- /.sidebar-menu -->
-    </section>
-    <!-- /.sidebar -->
-  </aside>
-
-  <!-- Content Wrapper. Contains page content -->
-  <div class="content-wrapper" id="content">
-      <!-- Content Header (Page header) -->
+    <div id="content">
+        <!-- Content Header (Page header) -->
         <section class="content-header">
             <h1 id="header"></h1>
             <ol class="breadcrumb">
                 <li class="active"><a href="#"><i class="fa fa-dashboard"></i> Cluster</a></li>
             </ol>
         </section>
-
         <section class="content">
             <div class="box">
                 <div class="box-header with-border">
-                    <!--<button id="backButton" class="btn btn-primary btn-flat"><i class="fa fa-chevron-left"></i> Back</button>-->
+                    <div class="clearfix">
+                        <span id="legendTitle">Legend: </span>
+                    </div>
+                    <div class="legendBody clearfix">
+                        <!--<svg id="homologousRegionLegend" height="10" width="10">-->
+                            <!--<circle cx="5" cy="5" r="4" stroke-width="1"></circle>-->
+                        <!--</svg>-->
+                        <!--<span class="legendText">Homologous Region</span>-->
+                        <svg id="amrLegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1">
+                                <title>AntiMicrobial Resistance Gene</title>
+                            </circle>
+                        </svg>
+                        <span title="AntiMicrobial Resistance Gene" class="legendText">AMR Gene</span>
+                        <svg id="genesLegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span class="legendText">Gene</span>
+                        <svg id="tRNALegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span class="legendText">tRNA</span>
+                        <svg id="rRNALegend" height="10" width="10">
+                            <circle cx="5" cy="5" r="4" stroke-width="1"></circle>
+                        </svg>
+                        <span class="legendText">rRNA</span>
+                    </div>
                 </div>
                 <div class="box-body">
                     <div id="visualizationBody"></div>
                 </div>
             </div>
         </section>
-  </div>
-  <!-- /.content-wrapper -->
+    </div>
 </div>
 
 <!-- REQUIRED JS SCRIPTS -->
@@ -162,41 +146,15 @@ <h1 id="header"></h1>
         window.location.href="login.html"
     });
 
-    $("#home-link").click(function(){
-        $(".sidebar-toggle").off("click");
-        $("#content").load("home.html");
-    });
-
-    $("#jobs-link").click(function(){
-        $(".sidebar-toggle").off("click");
-        $("#content").load("history.html");
-    });
-
-    $("#api-link").click(function(){
-        window.open(docUrl);
-    });
-
-    $("#about-link").click(function(){
-        $(".sidebar-toggle").off("click");
-        $("#content").load("about.html");
-    });
-
-    // Re-calls the visualization when the sidebar is toggled to adjust widths
-    $(".sidebar-toggle").on("click", function() {
-        setTimeout(function() {
-            createClusterVisualization();
-        }, 300);
-    });
-
-    var createClusterVisualization = function() {
-        var ENDPADDING = 7;
-        var SEQPADDING = 40;
+    function createClusterVisualization() {
+        var SEQSTART = 115;
         var SVGPADDING = 30;
+        var SEQPADDING = 25;
         var TEXTOFFSET = 15;
-        var RECTOFFSET = 20;
+        var RECTHEIGHT = 10;
 
         var clusterDict = window.clusterDict;
-        if (clusterDict == undefined) {
+        if (clusterDict === undefined) {
             clusterDict = JSON.parse(sessionStorage.getItem("clusterDict"));
         } else {
             // Store the clusterDict so it can be retrieved if the page is refreshed
@@ -207,33 +165,18 @@ <h1 id="header"></h1>
         $("#header").html("Cluster " + cluster);
         var color = clusterDict.color;
         var visWidth = $("#visualizationBody").width();
-        var scale = getScale(clusterDict.sequences, visWidth - SVGPADDING);
+        var scale = getScale(clusterDict.sequences, visWidth - SVGPADDING - SEQSTART);
 
         // D3.js //
         var seq = d3.select("#visualizationBody")
             .selectAll("div")
             .data(clusterDict.sequences);
 
-        // Update widths
-        seq.select("svg").attr("width", visWidth);
-        seq.select("svg").selectAll("rect")
-            .attr("width", function(island) {
-                return scale(Math.abs(island[1] - island[0]));
-            });
-        seq.select("svg").selectAll("g").select("text:nth-child(2)")
-            .attr("x", function(d) {
-                return Math.max(
-                    scale(Math.abs(d[1] - d[0])) - (d[1].length * ENDPADDING),
-                    (d[1].length + 1) * ENDPADDING
-                );
-            });
-
-        // Enter
         var div = seq.enter().append("div");
         div.attr("class", function(seq) {
             return seq.name;
         });
-
+        // Genome name
         div.append("p").append("b")
             .html(function(seq) {
                 return seq.name;
@@ -251,38 +194,85 @@ <h1 id="header"></h1>
             });
 
         rectGroup.enter().append("g");
+        // Start and Stop text
         rectGroup.append("text")
             .text(function(d) {
-                return d[0];
-            })
-            .attr("y", function(d, i) {
-                return (i * SEQPADDING) + TEXTOFFSET;
-            });
-        rectGroup.append("text")
-            .text(function(d) {
-                return d[1];
-            })
-            .attr("x", function(d) {
-                return Math.max(
-                    scale(Math.abs(d[1] - d[0])) - (d[1].length * ENDPADDING),
-                    (d[1].length + 1) * ENDPADDING
-                );
+                return d[0] + " - " + d[1];
             })
             .attr("y", function(d, i) {
                 return (i * SEQPADDING) + TEXTOFFSET;
             });
+        // The rect representing the genomic island
         rectGroup.append("rect")
             .attr("y", function(d, i) {
-                return (i * SEQPADDING) + RECTOFFSET;
+                return (i * SEQPADDING) + RECTHEIGHT / 2;
             })
-            .attr("height", "10")
+            .attr("height", RECTHEIGHT)
             .attr("width", function(d) {
                 return scale(Math.abs(d[1] - d[0]));
             })
             .attr("rx", 5)
             .attr("ry", 5)
-            .attr("fill", color);
-    };
+            .attr("fill", color)
+            .attr("transform", "translate(" + SEQSTART +",0)");
+        // The genes
+        rectGroup.append("g")
+            .attr("class", "genes")
+            .attr("transform", function(d, i) {
+                return "translate(" + SEQSTART + "," + i * SEQPADDING + ")"
+            })
+            .selectAll("polygon")
+            .data(function(island, i) {
+                var genes;
+                $.ajax({
+                    url: genomesGenesUrl + island[2] + "?start=" + island[0] + "&end=" + island[1],
+                    async: false,
+                    headers: {
+                        "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+                    },
+                    success: function(data) {
+                        // Change gene start and end positions to be start and end relative to GI start
+                        for (var gene_i = 0; gene_i < data.genes.length; gene_i++) {
+                            data.genes[gene_i].start -= island[0];
+                            // If gene extends beyond the end of the GI it is cut short
+                            data.genes[gene_i].end = Math.min(data.genes[gene_i].end - island[0], island[1] - island[0]);
+                        }
+                        genes = data.genes;
+                    }
+                });
+                return genes;
+            })
+            .enter().append("polygon")
+            .attr("points", function(gene) {
+                return scale(gene.start) + "," + RECTHEIGHT/2 + "," +
+                    scale(gene.end) + "," + RECTHEIGHT/2 + "," +
+                    scale(gene.end) + "," + RECTHEIGHT + "," +
+                    scale(gene.start) + "," + RECTHEIGHT;
+            })
+            .attr("transform", function(gene) {
+                // Move -1 strand genes down
+                if (gene.strand === -1) {
+                    return "translate(0," + RECTHEIGHT / 2 + ")"
+                }
+            })
+            .attr("class", function(gene) {
+                return gene.type;
+            })
+            .append("title")
+                .text(function (gene) {
+                    var hover = "";
+                    if(gene.gene) {
+                        hover = hover.concat("Gene name: " + gene.gene + "\n");
+                    }
+                    if(gene.locus_tag) {
+                        hover = hover.concat("Locus tag: " + gene.locus_tag + "\n");
+                    }
+                    if(gene.product) {
+                        hover = hover.concat("Product: " + gene.product)
+                    }
+                    return hover;
+                });
+    }
 
     function getScale(sequences, containerWidth) {
         var lengths = [];
@@ -299,18 +289,12 @@ <h1 id="header"></h1>
             .clamp(true);
     }
 
-    $(document).ready(createClusterVisualization);
-
-//    // To load the cluster view on the same page
-//    function createClusterVisualization(clusterDict, multiVis) {
-//        $("#backButton").on("click", this, function() {
-//            $("#content").load("result.html",
-//                null,
-//                function(){
-//                    createVisualization(url);
-//                }
-//            );
-//        });
-//    }
+    $(window).on("load", createClusterVisualization);
 </script>
+<!--Force content to fill screen-->
+<style>
+    html, body, .wrapper {
+        height: 100% !important;
+    }
+</style>
 </body>
diff --git a/web/pages/result.html b/web/pages/result.html
index d15771bf..c2c53d7d 100644
--- a/web/pages/result.html
+++ b/web/pages/result.html
@@ -89,9 +89,9 @@ <h1 id="resultHeader">
                         </svg>
                         <span class="legendText">Homologous Region</span>
                         <svg id="amrLegend" height="10" width="10">
-                            <polygon points="3,0,7,0,10,10,0,10"><title>AntiMicrobial Resistance Gene</title></polygon>
+                            <circle cx="5" cy="5" r="4" stroke-width="1"><title>AntiMicrobial Resistance Gene</title></circle>
                         </svg>
-                        <span title="AntiMicrobial Resistance Genes" class="legendText">AMR Gene</span>
+                        <span title="AntiMicrobial Resistance Gene" class="legendText">AMR Gene</span>
                         <span class="loadingGenes" hidden>
                             <i class='fa fa-spinner fa-spin '></i> Loading Genes
                         </span>

From ed98448c972808fd80f2971177b725d178c7e4c9 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Mon, 28 Aug 2017 17:18:33 -0700
Subject: [PATCH 74/76] Added AMR genes to cluster visualization page

---
 web/dist/js/genomecomparevis.js |  1 +
 web/pages/cluster.html          | 61 +++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/web/dist/js/genomecomparevis.js b/web/dist/js/genomecomparevis.js
index 8e550958..41799b70 100644
--- a/web/dist/js/genomecomparevis.js
+++ b/web/dist/js/genomecomparevis.js
@@ -752,6 +752,7 @@ function MultiVis(targetNode){
             var index = clusterDict.sequences.findIndex(i => i.id === currentSeq.sequenceId);
             if (index !== -1) {
                 clusterDict.sequences[index].name =  currentSeq.sequenceName;
+                clusterDict.sequences[index].amr = currentSeq.amr;
             }
         }
 
diff --git a/web/pages/cluster.html b/web/pages/cluster.html
index 32351e7a..04443ce4 100644
--- a/web/pages/cluster.html
+++ b/web/pages/cluster.html
@@ -272,6 +272,67 @@ <h1 id="header"></h1>
                     }
                     return hover;
                 });
+        // The AMR genes
+        rectGroup.append("g").attr("class", "amrs")
+            .attr("transform", function(d, i) {
+                return "translate(" + SEQSTART + "," + i * SEQPADDING + ")"
+            })
+            .each(function(island, i) {
+                var start = island[0];
+                var end = island[1];
+                var seq = island[2];
+                var index = clusterDict.sequences.findIndex(i => i.id === seq);
+                var amrs = clusterDict.sequences[index].amr;
+                for (var i = 0; i < amrs.length; i++) {
+                    if (amrs[i].start < end && amrs[i].end > start) {
+                        var amr_start = scale(amrs[i].start - start);
+                        var amr_end = scale(amrs[i].end - start);
+                        var neg_strand = (amrs[i].strand === "-");
+                        d3.select(this).append("polygon")
+                            .attr("points", function() {
+                                // Returns a trapezoid shape.
+                                // Shape depends on whether it is on negative strand and
+                                // whether it is within the boundaries of the GI
+                                var points = "";
+                                if (neg_strand) {
+                                    if (amrs[i].start < start) {
+                                        points += "0,0,0," + RECTHEIGHT / 2 + ",";
+                                    } else {
+                                        points += amr_start + ",0," +
+                                            (amr_start + RECTHEIGHT / 2) + "," + RECTHEIGHT / 2 + ",";
+                                    }
+                                    if (amrs[i].end > end) {
+                                        points += scale(end - start) + "," + RECTHEIGHT / 2 +
+                                            "," + scale(end - start) + ",0";
+                                    } else {
+                                        points += (amr_end - RECTHEIGHT / 2) + "," + RECTHEIGHT / 2 + "," +
+                                                amr_end + ",0";
+                                    }
+                                } else {
+                                    if (amrs[i].start < start) {
+                                        points += "0,0,0," + RECTHEIGHT / 2 + ",";
+                                    } else {
+                                        points += (amr_start + RECTHEIGHT / 2) + ",0," + amr_start + "," + RECTHEIGHT / 2 + ",";
+                                    }
+                                    if (amrs[i].end > end) {
+                                        points += scale(end - start) + "," + RECTHEIGHT / 2 +
+                                            "," + scale(end - start) + ",0";
+                                    } else {
+                                        points += amr_end + "," + RECTHEIGHT / 2 + "," +
+                                            (amr_end - RECTHEIGHT / 2) + ",0";
+                                    }
+                                }
+                                return points;
+                            })
+                            .attr("transform", function() {
+                                // Move negative strand AMR genes to the underside of the GI
+                                if (neg_strand) {
+                                    return "translate(0, "+ 3 / 2 * RECTHEIGHT +")"
+                                }
+                            });
+                    }
+                }
+        });
     }
 
     function getScale(sequences, containerWidth) {

From 7ae7b228a02de21227338c6b35820a043a776785 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 29 Aug 2017 10:47:45 -0700
Subject: [PATCH 75/76] Genes are fetched asynchronously on cluster vis page to
 improve responsiveness

---
 web/pages/cluster.html | 107 ++++++++++++++++++++---------------------
 1 file changed, 52 insertions(+), 55 deletions(-)

diff --git a/web/pages/cluster.html b/web/pages/cluster.html
index 04443ce4..6c4ee499 100644
--- a/web/pages/cluster.html
+++ b/web/pages/cluster.html
@@ -147,11 +147,11 @@ <h1 id="header"></h1>
     });
 
     function createClusterVisualization() {
-        var SEQSTART = 115;
-        var SVGPADDING = 30;
-        var SEQPADDING = 25;
-        var TEXTOFFSET = 15;
-        var RECTHEIGHT = 10;
+        var SEQSTART = 115; // Islands will begin at this horizontal distance from the left - makes room for position.
+        var SVGPADDING = 30; // Accounts for padding of each div
+        var SEQPADDING = 25; // Vertical distance between each island in a genome
+        var TEXTOFFSET = 15; // Adjustment to align position text with island
+        var RECTHEIGHT = 10; // Height of the island rect elements
 
         var clusterDict = window.clusterDict;
         if (clusterDict === undefined) {
@@ -176,7 +176,7 @@ <h1 id="header"></h1>
         div.attr("class", function(seq) {
             return seq.name;
         });
-        // Genome name
+        // Add genome name
         div.append("p").append("b")
             .html(function(seq) {
                 return seq.name;
@@ -194,7 +194,7 @@ <h1 id="header"></h1>
             });
 
         rectGroup.enter().append("g");
-        // Start and Stop text
+        // Add start and Stop text
         rectGroup.append("text")
             .text(function(d) {
                 return d[0] + " - " + d[1];
@@ -202,7 +202,7 @@ <h1 id="header"></h1>
             .attr("y", function(d, i) {
                 return (i * SEQPADDING) + TEXTOFFSET;
             });
-        // The rect representing the genomic island
+        // Add the rect representing the genomic island
         rectGroup.append("rect")
             .attr("y", function(d, i) {
                 return (i * SEQPADDING) + RECTHEIGHT / 2;
@@ -215,64 +215,61 @@ <h1 id="header"></h1>
             .attr("ry", 5)
             .attr("fill", color)
             .attr("transform", "translate(" + SEQSTART +",0)");
-        // The genes
+        // Add the genes
         rectGroup.append("g")
             .attr("class", "genes")
             .attr("transform", function(d, i) {
-                return "translate(" + SEQSTART + "," + i * SEQPADDING + ")"
-            })
-            .selectAll("polygon")
-            .data(function(island, i) {
-                var genes;
+                return "translate(" + SEQSTART + "," + i * SEQPADDING + ")";
+            }).each(function(island) {
+                var geneGroup = d3.select(this);
+                // For each island, an asynchronous call is made to fetch the genes within its boundaries.
+                // This prevents the page from freezing up while it gets all the genes.
                 $.ajax({
                     url: genomesGenesUrl + island[2] + "?start=" + island[0] + "&end=" + island[1],
-                    async: false,
                     headers: {
                         "Authorization": 'Token' + ' ' + getAuthenticationCookie()
                     },
                     success: function(data) {
-                        // Change gene start and end positions to be start and end relative to GI start
-                        for (var gene_i = 0; gene_i < data.genes.length; gene_i++) {
-                            data.genes[gene_i].start -= island[0];
-                            // If gene extends beyond the end of the GI it is cut short
-                            data.genes[gene_i].end = Math.min(data.genes[gene_i].end - island[0], island[1] - island[0]);
-                        }
-                        genes = data.genes;
-                    }
-                });
-                return genes;
-            })
-            .enter().append("polygon")
-            .attr("points", function(gene) {
-                return scale(gene.start) + "," + RECTHEIGHT/2 + "," +
-                    scale(gene.end) + "," + RECTHEIGHT/2 + "," +
-                    scale(gene.end) + "," + RECTHEIGHT + "," +
-                    scale(gene.start) + "," + RECTHEIGHT;
-            })
-            .attr("transform", function(gene) {
-                // Move -1 strand genes down
-                if (gene.strand === -1) {
-                    return "translate(0," + RECTHEIGHT / 2 + ")"
-                }
-            })
-            .attr("class", function(gene) {
-                return gene.type;
-            })
-            .append("title")
-                .text(function (gene) {
-                    var hover = "";
-                    if(gene.gene) {
-                        hover = hover.concat("Gene name: " + gene.gene + "\n");
-                    }
-                    if(gene.locus_tag) {
-                        hover = hover.concat("Locus tag: " + gene.locus_tag + "\n");
-                    }
-                    if(gene.product) {
-                        hover = hover.concat("Product: " + gene.product)
+                        // The genes are added to the island once they have been retreived from the database.
+                        geneGroup.selectAll("polygon")
+                            .data(data.genes)
+                            .enter().append("polygon")
+                            .attr("points", function(gene) {
+                                var geneStart = gene.start - island[0];
+                                // Cut off genes that extend beyond the island boundary
+                                var geneEnd = Math.min(gene.end - island[0], island[1] - island[0]);
+                                return scale(geneStart) + "," + RECTHEIGHT / 2 + "," +
+                                        scale(geneEnd) + "," + RECTHEIGHT / 2 + "," +
+                                        scale(geneEnd) + "," + RECTHEIGHT + "," +
+                                        scale(geneStart) + "," + RECTHEIGHT;
+                            })
+                            .attr("class", function(gene) {
+                                return gene.type; // gene, rRNA, or tRNA
+                            })
+                            .attr("transform", function(gene) {
+                                // Move negative strand genes to the bottom half of the island
+                                if (gene.strand === -1) {
+                                    return "translate(0," + RECTHEIGHT / 2 +")";
+                                }
+                            })
+                            .append("title")
+                                .text(function(gene) {
+                                    var hover = "";
+                                    if(gene.gene) {
+                                        hover = hover.concat("Gene name: " + gene.gene + "\n");
+                                    }
+                                    if(gene.locus_tag) {
+                                        hover = hover.concat("Locus tag: " + gene.locus_tag + "\n");
+                                    }
+                                    if(gene.product) {
+                                        hover = hover.concat("Product: " + gene.product)
+                                    }
+                                    return hover;
+                                });
                     }
-                    return hover;
                 });
-        // The AMR genes
+            });
+        // Add the AMR genes
         rectGroup.append("g").attr("class", "amrs")
             .attr("transform", function(d, i) {
                 return "translate(" + SEQSTART + "," + i * SEQPADDING + ")"

From 10381c43ac29e38eae17df2bf39dfb0502fca059 Mon Sep 17 00:00:00 2001
From: BeefTron <beeftilley@gmail.com>
Date: Tue, 29 Aug 2017 13:03:18 -0700
Subject: [PATCH 76/76] Moved cluster visualization JavaScript from
 cluster.html to clustervis.js

---
 web/dist/js/clustervis.js | 200 +++++++++++++++++++++++++++++++++++++
 web/pages/cluster.html    | 203 +-------------------------------------
 2 files changed, 202 insertions(+), 201 deletions(-)
 create mode 100644 web/dist/js/clustervis.js

diff --git a/web/dist/js/clustervis.js b/web/dist/js/clustervis.js
new file mode 100644
index 00000000..7724cf95
--- /dev/null
+++ b/web/dist/js/clustervis.js
@@ -0,0 +1,200 @@
+function createClusterVisualization() {
+    var SEQSTART = 115; // Islands will begin at this horizontal distance from the left - makes room for position.
+    var SVGPADDING = 30; // Accounts for padding of each div
+    var SEQPADDING = 25; // Vertical distance between each island in a genome
+    var TEXTOFFSET = 15; // Adjustment to align position text with island
+    var RECTHEIGHT = 10; // Height of the island rect elements
+
+    var clusterDict = window.clusterDict;
+    if (clusterDict === undefined) {
+        clusterDict = JSON.parse(sessionStorage.getItem("clusterDict"));
+    } else {
+        // Store the clusterDict so it can be retrieved if the page is refreshed
+        sessionStorage.setItem("clusterDict", JSON.stringify(clusterDict));
+    }
+
+    var cluster = clusterDict.cluster;
+    $("#header").html("Cluster " + cluster);
+    var color = clusterDict.color;
+    var visWidth = $("#visualizationBody").width();
+    var scale = getScale(clusterDict.sequences, visWidth - SVGPADDING - SEQSTART);
+
+    // D3.js //
+    var seq = d3.select("#visualizationBody")
+        .selectAll("div")
+        .data(clusterDict.sequences);
+
+    var div = seq.enter().append("div");
+    div.attr("class", function(seq) {
+        return seq.name;
+    });
+    // Add genome name
+    div.append("p").append("b")
+        .html(function(seq) {
+            return seq.name;
+        });
+
+    var svg = div.append("svg")
+        .attr("height", function(seq) {
+            return seq.islands.length * SEQPADDING;
+        })
+        .attr("width", visWidth - SVGPADDING);
+
+    var rectGroup = svg.selectAll("g")
+        .data(function(seq) {
+            return seq.islands;
+        });
+
+    rectGroup.enter().append("g");
+    // Add start and Stop text
+    rectGroup.append("text")
+        .text(function(d) {
+            return d[0] + " - " + d[1];
+        })
+        .attr("y", function(d, i) {
+            return (i * SEQPADDING) + TEXTOFFSET;
+        });
+    // Add the rect representing the genomic island
+    rectGroup.append("rect")
+        .attr("y", function(d, i) {
+            return (i * SEQPADDING) + RECTHEIGHT / 2;
+        })
+        .attr("height", RECTHEIGHT)
+        .attr("width", function(d) {
+            return scale(Math.abs(d[1] - d[0]));
+        })
+        .attr("rx", 5)
+        .attr("ry", 5)
+        .attr("fill", color)
+        .attr("transform", "translate(" + SEQSTART +",0)");
+    // Add the genes
+    rectGroup.append("g")
+        .attr("class", "genes")
+        .attr("transform", function(d, i) {
+            return "translate(" + SEQSTART + "," + i * SEQPADDING + ")";
+        }).each(function(island) {
+            var geneGroup = d3.select(this);
+            // For each island, an asynchronous call is made to fetch the genes within its boundaries.
+            // This prevents the page from freezing up while it gets all the genes.
+            $.ajax({
+                url: genomesGenesUrl + island[2] + "?start=" + island[0] + "&end=" + island[1],
+                headers: {
+                    "Authorization": 'Token' + ' ' + getAuthenticationCookie()
+                },
+                success: function(data) {
+                    // The genes are added to the island once they have been retreived from the database.
+                    geneGroup.selectAll("polygon")
+                        .data(data.genes)
+                        .enter().append("polygon")
+                        .attr("points", function(gene) {
+                            var geneStart = gene.start - island[0];
+                            // Cut off genes that extend beyond the island boundary
+                            var geneEnd = Math.min(gene.end - island[0], island[1] - island[0]);
+                            return scale(geneStart) + "," + RECTHEIGHT / 2 + "," +
+                                    scale(geneEnd) + "," + RECTHEIGHT / 2 + "," +
+                                    scale(geneEnd) + "," + RECTHEIGHT + "," +
+                                    scale(geneStart) + "," + RECTHEIGHT;
+                        })
+                        .attr("class", function(gene) {
+                            return gene.type; // gene, rRNA, or tRNA
+                        })
+                        .attr("transform", function(gene) {
+                            // Move negative strand genes to the bottom half of the island
+                            if (gene.strand === -1) {
+                                return "translate(0," + RECTHEIGHT / 2 +")";
+                            }
+                        })
+                        .append("title")
+                            .text(function(gene) {
+                                var hover = "";
+                                if(gene.gene) {
+                                    hover = hover.concat("Gene name: " + gene.gene + "\n");
+                                }
+                                if(gene.locus_tag) {
+                                    hover = hover.concat("Locus tag: " + gene.locus_tag + "\n");
+                                }
+                                if(gene.product) {
+                                    hover = hover.concat("Product: " + gene.product)
+                                }
+                                return hover;
+                            });
+                }
+            });
+        });
+    // Add the AMR genes
+    rectGroup.append("g").attr("class", "amrs")
+        .attr("transform", function(d, i) {
+            return "translate(" + SEQSTART + "," + i * SEQPADDING + ")"
+        })
+        .each(function(island, i) {
+            var start = island[0];
+            var end = island[1];
+            var seq = island[2];
+            var index = clusterDict.sequences.findIndex(i => i.id === seq);
+            var amrs = clusterDict.sequences[index].amr;
+            for (var i = 0; i < amrs.length; i++) {
+                if (amrs[i].start < end && amrs[i].end > start) {
+                    var amr_start = scale(amrs[i].start - start);
+                    var amr_end = scale(amrs[i].end - start);
+                    var neg_strand = (amrs[i].strand === "-");
+                    d3.select(this).append("polygon")
+                        .attr("points", function() {
+                            // Returns a trapezoid shape.
+                            // Shape depends on whether it is on negative strand and
+                            // whether it is within the boundaries of the GI
+                            var points = "";
+                            if (neg_strand) {
+                                if (amrs[i].start < start) {
+                                    points += "0,0,0," + RECTHEIGHT / 2 + ",";
+                                } else {
+                                    points += amr_start + ",0," +
+                                        (amr_start + RECTHEIGHT / 2) + "," + RECTHEIGHT / 2 + ",";
+                                }
+                                if (amrs[i].end > end) {
+                                    points += scale(end - start) + "," + RECTHEIGHT / 2 +
+                                        "," + scale(end - start) + ",0";
+                                } else {
+                                    points += (amr_end - RECTHEIGHT / 2) + "," + RECTHEIGHT / 2 + "," +
+                                            amr_end + ",0";
+                                }
+                            } else {
+                                if (amrs[i].start < start) {
+                                    points += "0,0,0," + RECTHEIGHT / 2 + ",";
+                                } else {
+                                    points += (amr_start + RECTHEIGHT / 2) + ",0," + amr_start + "," + RECTHEIGHT / 2 + ",";
+                                }
+                                if (amrs[i].end > end) {
+                                    points += scale(end - start) + "," + RECTHEIGHT / 2 +
+                                        "," + scale(end - start) + ",0";
+                                } else {
+                                    points += amr_end + "," + RECTHEIGHT / 2 + "," +
+                                        (amr_end - RECTHEIGHT / 2) + ",0";
+                                }
+                            }
+                            return points;
+                        })
+                        .attr("transform", function() {
+                            // Move negative strand AMR genes to the underside of the GI
+                            if (neg_strand) {
+                                return "translate(0, "+ 3 / 2 * RECTHEIGHT +")"
+                            }
+                        });
+                }
+            }
+    });
+}
+
+function getScale(sequences, containerWidth) {
+    var lengths = [];
+    for (var seqIndex = 0; seqIndex < sequences.length; seqIndex++) {
+        var islands = sequences[seqIndex].islands;
+        for (var islandIndex = 0; islandIndex < islands.length; islandIndex++) {
+            lengths.push(Math.abs(islands[islandIndex][1] - islands[islandIndex][0]));
+        }
+    }
+    var max = Math.max.apply(null, lengths);
+    return d3.scale.linear()
+        .domain([0,max])
+        .range([0,containerWidth])
+        .clamp(true);
+}
\ No newline at end of file
diff --git a/web/pages/cluster.html b/web/pages/cluster.html
index 6c4ee499..b4edf5f1 100644
--- a/web/pages/cluster.html
+++ b/web/pages/cluster.html
@@ -135,6 +135,8 @@ <h1 id="header"></h1>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap3-dialog/1.34.7/js/bootstrap-dialog.min.js"></script>
 <!-- D3 -->
 <script src="../dist/js/d3.min.js"></script>
+<!-- Cluster Visualization JavaScript -->
+<script src="../dist/js/clustervis.js"></script>
 
 <script>
     $(document).ready(function(){
@@ -146,207 +148,6 @@ <h1 id="header"></h1>
         window.location.href="login.html"
     });
 
-    function createClusterVisualization() {
-        var SEQSTART = 115; // Islands will begin at this horizontal distance from the left - makes room for position.
-        var SVGPADDING = 30; // Accounts for padding of each div
-        var SEQPADDING = 25; // Vertical distance between each island in a genome
-        var TEXTOFFSET = 15; // Adjustment to align position text with island
-        var RECTHEIGHT = 10; // Height of the island rect elements
-
-        var clusterDict = window.clusterDict;
-        if (clusterDict === undefined) {
-            clusterDict = JSON.parse(sessionStorage.getItem("clusterDict"));
-        } else {
-            // Store the clusterDict so it can be retrieved if the page is refreshed
-            sessionStorage.setItem("clusterDict", JSON.stringify(clusterDict));
-        }
-
-        var cluster = clusterDict.cluster;
-        $("#header").html("Cluster " + cluster);
-        var color = clusterDict.color;
-        var visWidth = $("#visualizationBody").width();
-        var scale = getScale(clusterDict.sequences, visWidth - SVGPADDING - SEQSTART);
-
-        // D3.js //
-        var seq = d3.select("#visualizationBody")
-            .selectAll("div")
-            .data(clusterDict.sequences);
-
-        var div = seq.enter().append("div");
-        div.attr("class", function(seq) {
-            return seq.name;
-        });
-        // Add genome name
-        div.append("p").append("b")
-            .html(function(seq) {
-                return seq.name;
-            });
-
-        var svg = div.append("svg")
-            .attr("height", function(seq) {
-                return seq.islands.length * SEQPADDING;
-            })
-            .attr("width", visWidth - SVGPADDING);
-
-        var rectGroup = svg.selectAll("g")
-            .data(function(seq) {
-                return seq.islands;
-            });
-
-        rectGroup.enter().append("g");
-        // Add start and Stop text
-        rectGroup.append("text")
-            .text(function(d) {
-                return d[0] + " - " + d[1];
-            })
-            .attr("y", function(d, i) {
-                return (i * SEQPADDING) + TEXTOFFSET;
-            });
-        // Add the rect representing the genomic island
-        rectGroup.append("rect")
-            .attr("y", function(d, i) {
-                return (i * SEQPADDING) + RECTHEIGHT / 2;
-            })
-            .attr("height", RECTHEIGHT)
-            .attr("width", function(d) {
-                return scale(Math.abs(d[1] - d[0]));
-            })
-            .attr("rx", 5)
-            .attr("ry", 5)
-            .attr("fill", color)
-            .attr("transform", "translate(" + SEQSTART +",0)");
-        // Add the genes
-        rectGroup.append("g")
-            .attr("class", "genes")
-            .attr("transform", function(d, i) {
-                return "translate(" + SEQSTART + "," + i * SEQPADDING + ")";
-            }).each(function(island) {
-                var geneGroup = d3.select(this);
-                // For each island, an asynchronous call is made to fetch the genes within its boundaries.
-                // This prevents the page from freezing up while it gets all the genes.
-                $.ajax({
-                    url: genomesGenesUrl + island[2] + "?start=" + island[0] + "&end=" + island[1],
-                    headers: {
-                        "Authorization": 'Token' + ' ' + getAuthenticationCookie()
-                    },
-                    success: function(data) {
-                        // The genes are added to the island once they have been retreived from the database.
-                        geneGroup.selectAll("polygon")
-                            .data(data.genes)
-                            .enter().append("polygon")
-                            .attr("points", function(gene) {
-                                var geneStart = gene.start - island[0];
-                                // Cut off genes that extend beyond the island boundary
-                                var geneEnd = Math.min(gene.end - island[0], island[1] - island[0]);
-                                return scale(geneStart) + "," + RECTHEIGHT / 2 + "," +
-                                        scale(geneEnd) + "," + RECTHEIGHT / 2 + "," +
-                                        scale(geneEnd) + "," + RECTHEIGHT + "," +
-                                        scale(geneStart) + "," + RECTHEIGHT;
-                            })
-                            .attr("class", function(gene) {
-                                return gene.type; // gene, rRNA, or tRNA
-                            })
-                            .attr("transform", function(gene) {
-                                // Move negative strand genes to the bottom half of the island
-                                if (gene.strand === -1) {
-                                    return "translate(0," + RECTHEIGHT / 2 +")";
-                                }
-                            })
-                            .append("title")
-                                .text(function(gene) {
-                                    var hover = "";
-                                    if(gene.gene) {
-                                        hover = hover.concat("Gene name: " + gene.gene + "\n");
-                                    }
-                                    if(gene.locus_tag) {
-                                        hover = hover.concat("Locus tag: " + gene.locus_tag + "\n");
-                                    }
-                                    if(gene.product) {
-                                        hover = hover.concat("Product: " + gene.product)
-                                    }
-                                    return hover;
-                                });
-                    }
-                });
-            });
-        // Add the AMR genes
-        rectGroup.append("g").attr("class", "amrs")
-            .attr("transform", function(d, i) {
-                return "translate(" + SEQSTART + "," + i * SEQPADDING + ")"
-            })
-            .each(function(island, i) {
-                var start = island[0];
-                var end = island[1];
-                var seq = island[2];
-                var index = clusterDict.sequences.findIndex(i => i.id === seq);
-                var amrs = clusterDict.sequences[index].amr;
-                for (var i = 0; i < amrs.length; i++) {
-                    if (amrs[i].start < end && amrs[i].end > start) {
-                        var amr_start = scale(amrs[i].start - start);
-                        var amr_end = scale(amrs[i].end - start);
-                        var neg_strand = (amrs[i].strand === "-");
-                        d3.select(this).append("polygon")
-                            .attr("points", function() {
-                                // Returns a trapezoid shape.
-                                // Shape depends on whether it is on negative strand and
-                                // whether it is within the boundaries of the GI
-                                var points = "";
-                                if (neg_strand) {
-                                    if (amrs[i].start < start) {
-                                        points += "0,0,0," + RECTHEIGHT / 2 + ",";
-                                    } else {
-                                        points += amr_start + ",0," +
-                                            (amr_start + RECTHEIGHT / 2) + "," + RECTHEIGHT / 2 + ",";
-                                    }
-                                    if (amrs[i].end > end) {
-                                        points += scale(end - start) + "," + RECTHEIGHT / 2 +
-                                            "," + scale(end - start) + ",0";
-                                    } else {
-                                        points += (amr_end - RECTHEIGHT / 2) + "," + RECTHEIGHT / 2 + "," +
-                                                amr_end + ",0";
-                                    }
-                                } else {
-                                    if (amrs[i].start < start) {
-                                        points += "0,0,0," + RECTHEIGHT / 2 + ",";
-                                    } else {
-                                        points += (amr_start + RECTHEIGHT / 2) + ",0," + amr_start + "," + RECTHEIGHT / 2 + ",";
-                                    }
-                                    if (amrs[i].end > end) {
-                                        points += scale(end - start) + "," + RECTHEIGHT / 2 +
-                                            "," + scale(end - start) + ",0";
-                                    } else {
-                                        points += amr_end + "," + RECTHEIGHT / 2 + "," +
-                                            (amr_end - RECTHEIGHT / 2) + ",0";
-                                    }
-                                }
-                                return points;
-                            })
-                            .attr("transform", function() {
-                                // Move negative strand AMR genes to the underside of the GI
-                                if (neg_strand) {
-                                    return "translate(0, "+ 3 / 2 * RECTHEIGHT +")"
-                                }
-                            });
-                    }
-                }
-        });
-    }
-
-    function getScale(sequences, containerWidth) {
-        var lengths = [];
-        for (var seqIndex = 0; seqIndex < sequences.length; seqIndex++) {
-            var islands = sequences[seqIndex].islands;
-            for (var islandIndex = 0; islandIndex < islands.length; islandIndex++) {
-                lengths.push(Math.abs(islands[islandIndex][1] - islands[islandIndex][0]));
-            }
-        }
-        var max = Math.max.apply(null, lengths);
-        return d3.scale.linear()
-            .domain([0,max])
-            .range([0,containerWidth])
-            .clamp(true);
-    }
-
     $(window).on("load", createClusterVisualization);
 </script>
 <!--Force content to fill screen-->