Add Tall Table Writer and Implement Opinionated Data Retrieval

Have a writer function which allows for tall tables, and make data retrieval from reduced data be through an opinionated method.

view-simulation-results/src/main/java/org/vcell/N5/reduction/DTO/RangeOfImage.java

@@ -17,6 +17,18 @@ public RangeOfImage(int timeStart, int timeEnd, int zStart, int zEnd, int channe
         this.channelEnd = channelEnd;
     }
 
+    public int getNChannels(){
+        return this.channelEnd - this.channelStart + 1;
+    }
+
+    public int getNFrames(){
+        return this.timeEnd - this.timeStart + 1;
+    }
+
+    public int getNSlices(){
+        return this.zEnd - this.zStart + 1;
+    }
+
     /**
      * Only regards time for range, and is used for normalization.
      * @param timeStart

view-simulation-results/src/main/java/org/vcell/N5/reduction/DTO/ReducedData.java

@@ -0,0 +1,47 @@
+package org.vcell.N5.reduction.DTO;
+
+import org.vcell.N5.reduction.SelectMeasurements;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+// Per Image, contain all ROI data
+// Shape: [time * z][channel * roi]
+public class ReducedData {
+    private final int nChannels;
+    private final int nSlices;
+    private final HashMap<SelectMeasurements.AvailableMeasurements, double[][]> dataMap = new HashMap<>();
+
+    public final ArrayList<SelectMeasurements.AvailableMeasurements> measurements;
+    public final RangeOfImage rangeOfImage;
+    public final int nROIs;
+    public final HashMap<Integer, String> roiNames = new HashMap<>();
+    public final HashMap<Integer, String> channelNames = new HashMap<>();
+    public final String imageName;
+    public ReducedData(String imageName, RangeOfImage rangeOfImage, int nROIs, ArrayList<SelectMeasurements.AvailableMeasurements> measurements){
+        int nFrames = rangeOfImage.timeEnd - rangeOfImage.timeStart + 1;
+        nSlices = rangeOfImage.zEnd - rangeOfImage.zStart + 1;
+        this.measurements = measurements;
+        this.nChannels = rangeOfImage.getNChannels();
+        this.nROIs = nROIs;
+        this.rangeOfImage = rangeOfImage;
+        this.imageName = imageName;
+
+        for (SelectMeasurements.AvailableMeasurements measurement: measurements){
+            dataMap.put(measurement, new double[nFrames * nSlices][nChannels * nROIs]);
+        }
+    }
+
+    public void putDataPoint(double data,int time, int z, int channel, int roi, SelectMeasurements.AvailableMeasurements measurementType){
+        dataMap.get(measurementType)[(time * nSlices) + z][(roi * nChannels) + channel] = data;
+    }
+
+    public double getDataPoint(int time, int z, int channel, int roi, SelectMeasurements.AvailableMeasurements measurementType){
+        return dataMap.get(measurementType)[(time * nSlices) + z][(roi * nChannels) + channel];
+    }
+
+    public String getWideTableHeader(int r, int c){
+        return imageName + ":" + roiNames.get(r) + ":" + channelNames.get(c);
+    }
+
+}

view-simulation-results/src/main/java/org/vcell/N5/reduction/DataReductionManager.java

@@ -8,6 +8,7 @@
 import org.vcell.N5.UI.ControlButtonsPanel;
 import org.vcell.N5.UI.MainPanel;
 import org.vcell.N5.reduction.DTO.RangeOfImage;
+import org.vcell.N5.reduction.DTO.ReducedData;
 import org.vcell.N5.retrieving.SimLoadingListener;
 import org.vcell.N5.retrieving.SimResultsLoader;
 
@@ -21,7 +22,7 @@ public class DataReductionManager implements SimLoadingListener {
     private final ArrayList<Roi> arrayOfSimRois;
     private final Object csvMatrixLock = new Object();
 
-    private int numOfCalculationsTimesNumImages;
+    private int numOfImagesToOpen;
     private final ReductionCalculations calculations;
 
     public final DataReductionGUI.DataReductionSubmission submission;
@@ -31,31 +32,12 @@ public class DataReductionManager implements SimLoadingListener {
 
     private DataReductionWriter dataReductionWriter;
 
-
-    // Per Image
-    public static class ReducedData{
-        public final double[][] data;
-        public final ArrayList<String> columnHeaders;
-        public final SelectMeasurements.AvailableMeasurements measurementType;
-        public final RangeOfImage rangeOfImage;
-        public final int colLen;
-        public ReducedData(RangeOfImage rangeOfImage, int colLen, SelectMeasurements.AvailableMeasurements measurementType){
-            int nFrames = rangeOfImage.timeEnd - rangeOfImage.timeStart + 1;
-            int nSlices = rangeOfImage.zEnd - rangeOfImage.zStart + 1;
-            data = new double[nFrames * nSlices][colLen]; // row - col
-            columnHeaders = new ArrayList<>();
-            this.measurementType = measurementType;
-            this.colLen = colLen;
-            this.rangeOfImage = rangeOfImage;
-        }
-    }
-
     public DataReductionManager(DataReductionGUI.DataReductionSubmission submission){
         N5ImageHandler.loadingManager.addSimLoadingListener(this);
         this.submission = submission;
 
         this.arrayOfSimRois = submission.arrayOfSimRois;
-        this.numOfCalculationsTimesNumImages = (submission.numOfSimImages + 1) * submission.selectedMeasurements.size(); // Plus one for the lab image
+        this.numOfImagesToOpen = submission.numOfSimImages + 1; // Plus one for the lab image
         this.calculations = new ReductionCalculations(submission.normalizeMeasurementsBool);
 
         Thread processLabResults = new Thread(() -> {
@@ -84,27 +66,25 @@ private void calculateAndAddResults(ImagePlus imagePlus, RangeOfImage normRange,
             normValue = calculations.calculateNormalValue(imagePlus, normRange, rois, imageRange);
         }
 
-        int nChannels = imageRange.channelEnd - imageRange.channelStart + 1;
+        ReducedData reducedData = new ReducedData(imagePlus.getTitle(), imageRange, arrayOfSimRois.size(), submission.selectedMeasurements);
+        calculations.addAppropriateHeaders(rois, imageRange, reducedData, channelInfo);
 
-        ArrayList<ReducedData> reducedDataArrayList = new ArrayList<>();
-        for (SelectMeasurements.AvailableMeasurements measurement : submission.selectedMeasurements){
-            ReducedData reducedData = new ReducedData(imageRange, nChannels * arrayOfSimRois.size(), measurement);
-            reducedDataArrayList.add(reducedData);
-            calculations.addAppropriateHeaders(imagePlus, rois, imageRange, reducedData, channelInfo);
-        }
         AtomicBoolean continueOperation = threadPool.get(threadName).continueOperation;
-        calculations.calculateStatistics(imagePlus, rois, normValue, reducedDataArrayList, imageRange, continueOperation);
-        for (ReducedData reducedData: reducedDataArrayList){
-            if (continueOperation.get()){
-                synchronized (csvMatrixLock){
-                    dataReductionWriter.addValuesToWideCSVMatrix(reducedData);
-                    numOfCalculationsTimesNumImages -= 1;
+        calculations.calculateStatistics(imagePlus, rois, normValue, reducedData, imageRange, continueOperation);
+        if (continueOperation.get()){
+            synchronized (csvMatrixLock){
+                try {
+                    dataReductionWriter.consumeNewData(reducedData);
+                } catch (IOException e) {
+                    stopAllThreads();
+                    throw new RuntimeException(e);
                 }
+                numOfImagesToOpen -= 1;
             }
         }
-        if (numOfCalculationsTimesNumImages == 0){
+        if (numOfImagesToOpen == 0 && continueOperation.get()){
             try{
-                dataReductionWriter.writeCSVMatrix();
+                dataReductionWriter.close();
             } catch (IOException ioException){
                 throw new RuntimeException(ioException);
             } finally {

view-simulation-results/src/main/java/org/vcell/N5/reduction/DataReductionWriter.java

@@ -5,13 +5,14 @@
 import org.vcell.N5.UI.MainPanel;
 import org.vcell.N5.UI.N5ExportTable;
 import org.vcell.N5.reduction.DTO.RangeOfImage;
-import org.vcell.N5.reduction.DataReductionManager.ReducedData;
+import org.vcell.N5.reduction.DTO.ReducedData;
 import org.vcell.N5.retrieving.SimResultsLoader;
 
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 
 public class DataReductionWriter{
@@ -39,6 +40,8 @@ public class DataReductionWriter{
     private final int maxZ;
     private final int maxT;
 
+    private final boolean wideTable = true;
+
     ///////////////////////////////////////
     // Initialize Sheet and Lab results //
     /////////////////////////////////////
@@ -51,12 +54,23 @@ public DataReductionWriter(DataReductionGUI.DataReductionSubmission submission,
         this.maxT = maxT;
     }
 
+    public void consumeNewData(ReducedData reducedData) throws IOException {
+        if (wideTable){
+            addValuesToWideCSVMatrix(reducedData);
+        } else {
+            appendAndWriteTallTable(reducedData);
+        }
+    }
+
+    public void close() throws IOException {
+        if (wideTable){
+            writeWideTableCSVMatrix();
+        }
+    }
+
     public void initializeDataSheets(){
         ArrayList<String> headers = new ArrayList<String>(){{add("Time Frame");}};
         boolean is3D = maxZ > 1;
-        if (is3D){
-            headers.add("Z Index");
-        }
 
         // Add Time and Z-Index Columns
         for (SelectMeasurements.AvailableMeasurements measurement : selectedMeasurements){
@@ -77,54 +91,40 @@ public void initializeDataSheets(){
         metaDataSheet.get(0).add("Simulation Name");
         metaDataSheet.get(0).add("N5 URL");
 
-        // Fill in Time and Z-Index Columns with selected range
-
-        for (SelectMeasurements.AvailableMeasurements measurement : selectedMeasurements){
-            ArrayList<ArrayList<String>> dataSheet = sheetsAvailable.get(measurement);
-            for (int t = 1; t <= maxT; t++){
-                for (int z = 1; z <= maxZ; z++){
-                    ArrayList<String> pointRow = new ArrayList<>();
-                    pointRow.add(0, String.valueOf(t));
-                    if (is3D){
-                        pointRow.add(1, String.valueOf(z));
-                    }
-                    dataSheet.add(pointRow);
-                }
+        if (wideTable){
+            // Fill in Time and Z-Index Columns with selected range
+            if (is3D){
+                headers.add("Z Index");
             }
-        }
-    }
-
-    ////////////////////////
-    // General Functions //
-    //////////////////////
-
-    public void addValuesToWideCSVMatrix(ReducedData reducedData){
-        ArrayList<ArrayList<String>> dataSheet = sheetsAvailable.get(reducedData.measurementType);
-        int colIndex = columnsForSheets.get(reducedData.measurementType);
-        fillWithEmptySpace(dataSheet.get(0), colIndex);
-        for (int c = 0; c < reducedData.columnHeaders.size(); c++){
-            dataSheet.get(0).add(colIndex, reducedData.columnHeaders.get(c));
-        }
-        RangeOfImage rangeOfImage = reducedData.rangeOfImage;
-        int tzCounter = 1;
-        for (int t = 1; t <= maxT; t++){
-            for (int z = 1; z <= maxZ; z++){
-                boolean inBetweenTime = t <= rangeOfImage.timeEnd && rangeOfImage.timeStart <= t;
-                boolean inBetweenZ = z <= rangeOfImage.zEnd && rangeOfImage.zStart <= z;
-                ArrayList<String> row = dataSheet.get(tzCounter);
-                fillWithEmptySpace(row, colIndex);
-                for (int c = 0; c < reducedData.colLen; c++){
-                    if (inBetweenTime && inBetweenZ){
-                        int dataRow = ((t - rangeOfImage.timeStart) * (z - rangeOfImage.zStart)) + (z - rangeOfImage.zStart);
-                        double mean = reducedData.data[dataRow][c];
-                        row.add(String.valueOf(mean));
+            for (SelectMeasurements.AvailableMeasurements measurement : selectedMeasurements){
+                ArrayList<ArrayList<String>> dataSheet = sheetsAvailable.get(measurement);
+                for (int t = 1; t <= maxT; t++){
+                    for (int z = 1; z <= maxZ; z++){
+                        ArrayList<String> pointRow = new ArrayList<>();
+                        pointRow.add(0, String.valueOf(t));
+                        if (is3D){
+                            pointRow.add(1, String.valueOf(z));
+                        }
+                        dataSheet.add(pointRow);
                     }
                 }
-                tzCounter += 1;
+            }
+        } else {
+            headers.add(1, "Z Index");
+            headers.add("Image Name");
+            headers.add("ROI Name");
+            headers.add("Channel Name");
+            for (SelectMeasurements.AvailableMeasurements measurement : selectedMeasurements){
+                headers.add(measurement.publicName);
+            }
+            File file = new File(this.file.getAbsolutePath() + ".csv");
+            try(FileWriter fileWriter = new FileWriter(file)) {
+                CSVWriter csvWriter = new CSVWriter(fileWriter);
+                csvWriter.writeNext(headers.toArray(new String[0]));
+            } catch (IOException ioException){
+                throw new RuntimeException("Can't write to CSV file.", ioException);
             }
         }
-        colIndex += 1 + reducedData.data[0].length;
-        columnsForSheets.replace(reducedData.measurementType, colIndex);
     }
 
     // If parameter is not in list of parameters, add new column. If simulation does not have parameter say "not-present"
@@ -158,15 +158,75 @@ public void addMetaData(SimResultsLoader loadedResults){
         }
     }
 
+    ////////////////////////
+    // Private Functions //
+    //////////////////////
+
+    private void addValuesToWideCSVMatrix(ReducedData reducedData){
+        for (SelectMeasurements.AvailableMeasurements measurement:  reducedData.measurements){
+            ArrayList<ArrayList<String>> dataSheet = sheetsAvailable.get(measurement);
+            int colIndex = columnsForSheets.get(measurement);
+            fillWithEmptySpace(dataSheet.get(0), colIndex);
+            RangeOfImage rangeOfImage = reducedData.rangeOfImage;
+            for (int c = 0; c < rangeOfImage.getNChannels(); c++){
+                for (int r = 0; r < reducedData.nROIs; r++){
+                    dataSheet.get(0).add(colIndex, reducedData.getWideTableHeader(r, c));
+                    int tzCounter = 1;
+                    for (int t = 1; t <= maxT; t++){
+                        for (int z = 1; z <= maxZ; z++){
+                            boolean inBetweenTime = t <= rangeOfImage.timeEnd && rangeOfImage.timeStart <= t;
+                            boolean inBetweenZ = z <= rangeOfImage.zEnd && rangeOfImage.zStart <= z;
+                            ArrayList<String> row = dataSheet.get(tzCounter);
+                            fillWithEmptySpace(row, colIndex);
+                            if (inBetweenTime && inBetweenZ){
+                                int nt = t - rangeOfImage.timeStart;
+                                int nz = z - rangeOfImage.zStart;
+                                row.add(String.valueOf(reducedData.getDataPoint(nt, nz, c, r, measurement)));
+                            }
+                            tzCounter += 1;
+                        }
+                    }
+                    colIndex += 1;
+                }
+            }
+            colIndex += 1;
+            columnsForSheets.replace(measurement, colIndex);
+        }
+    }
+
     // If specific entry to be added isn't in array list length, add empty space until it is
     private void fillWithEmptySpace(ArrayList<String> arrayList, int col){
         while (arrayList.size() < col){
             arrayList.add("");
         }
     }
 
+    // Each reduced data is a different measurement type
+    private void appendAndWriteTallTable(ReducedData reducedData) throws IOException {
+        RangeOfImage rangeOfImage = reducedData.rangeOfImage;
+        for (int t = rangeOfImage.timeStart; t <= rangeOfImage.timeEnd; t++) {
+            for (int z = rangeOfImage.zStart; z <= rangeOfImage.zEnd; z++) {
+                for (int channel = 0; channel < rangeOfImage.getNChannels(); channel++){
+                    for(int roi= 0; roi < reducedData.nROIs; roi++){
+                        File file = new File(this.file.getAbsolutePath() + ".csv");
+                        ArrayList<String> newRow = new ArrayList<>(Arrays.asList(String.valueOf(t), String.valueOf(z),
+                                reducedData.imageName, reducedData.roiNames.get(roi), reducedData.channelNames.get(channel)));
+                        for (SelectMeasurements.AvailableMeasurements measurement : reducedData.measurements){
+                            int nt = t - rangeOfImage.timeStart;
+                            int nz = z - rangeOfImage.zStart;
+                            newRow.add(String.valueOf(reducedData.getDataPoint(nt, nz, channel, roi, measurement)));
+                        }
+                        try (FileWriter fileWriter = new FileWriter(file, true)) {
+                            CSVWriter csvWriter = new CSVWriter(fileWriter);
+                            csvWriter.writeNext(newRow.toArray(new String[0]));
+                        }
+                    }
+                }
+            }
+        }
+    }
 
-    public void writeCSVMatrix() throws IOException {
+    private void writeWideTableCSVMatrix() throws IOException {
         for (SelectMeasurements.AvailableMeasurements measurements : sheetsAvailable.keySet()){
             if (!sheetsAvailable.get(measurements).isEmpty()){
                 File currentFile = new File(file.getAbsolutePath() + "-" + measurements.publicName + ".csv");