Support for ROIs with different size or resolution

src/silverlabnwb/header.py

@@ -9,6 +9,12 @@
 class LabViewVersions(Enum):
     pre2018 = "pre-2018 (original)"
     v231 = "2.3.1"
+    v300 = "3.0.0"
+
+    @property
+    def is_legacy(self):
+        """Return whether this version should trigger legacy behaviour."""
+        return self is self.pre2018
 
 
 class LabViewHeader(metaclass=abc.ABCMeta):
@@ -63,6 +69,8 @@ def from_file(cls, filename):
         else:
             if version == '2.3.1':
                 return LabViewHeader231(fields, parsed_fields)
+            elif version == '3.0.0':
+                return LabViewHeader300(fields, parsed_fields)
             else:
                 raise ValueError('Unsupported LabView version {}.'.format(version))
 
@@ -140,6 +148,11 @@ def get_raw_fields(self):
         """
         return self._raw_fields
 
+    @property
+    def allows_variable_rois(self):
+        """Check whether ROIs with variable shape or resolution are allowed."""
+        return False  # by default (in older versions), they are not
+
 
 class LabViewHeaderPre2018(LabViewHeader):
 
@@ -170,7 +183,7 @@ def _imaging_section(self):
         return self["GLOBAL PARAMETERS"]
 
 
-class LabViewHeader231(LabViewHeader):
+class LabViewHeaderPost2018(LabViewHeader):
 
     property_names = {
         "frame_size": "Frame Size",
@@ -182,18 +195,14 @@ class LabViewHeader231(LabViewHeader):
         "gain_green": "pmt 2",
     }
 
-    # In this version of LabView, the trial times are stored in their own
+    # In these versions of LabView, the trial times are stored in their own
     # (misleadingly titled) section of the header.
     trial_times_section = 'Intertrial FIFO Times'
 
     def __init__(self, fields, processed_fields):
         super().__init__(fields, processed_fields)
         self._parse_trial_times()
 
-    @property
-    def version(self):
-        return LabViewVersions.v231
-
     def _determine_imaging_mode(self):
         volume_imaging = self['IMAGING MODES']['Volume Imaging']
         functional_imaging = self['IMAGING MODES']['Functional Imaging']
@@ -216,8 +225,8 @@ def _determine_imaging_mode(self):
                              ' or "Functional Imaging" must be true.')
 
     def _imaging_section(self):
-        # In LabView version 2.3.1, imaging parameters are stored under the
-        # relevant imaging mode section.
+        # In LabView version 2.3.1 and newer, imaging parameters are stored
+        # under the relevant imaging mode section.
         imaging_section_name = ("VOLUME IMAGING"
                                 if self.imaging_mode is Modes.volume
                                 else "FUNCTIONAL IMAGING")
@@ -253,3 +262,20 @@ def determine_trial_times(self):
                 end = None  # determine final 'end' later from speed data
             trial_times.append((start, end))
         return trial_times
+
+
+class LabViewHeader231(LabViewHeaderPost2018):
+    @property
+    def version(self):
+        return LabViewVersions.v231
+
+
+class LabViewHeader300(LabViewHeaderPost2018):
+    @property
+    def version(self):
+        return LabViewVersions.v300
+
+    @property
+    def allows_variable_rois(self):
+        return (self._imaging_section()['Variable Length'] == 'TRUE'
+                or self._imaging_section()['Variable Resolution'] == 'TRUE')

src/silverlabnwb/nwb_file.py

@@ -19,6 +19,7 @@
 from . import metadata
 from .header import LabViewHeader, LabViewVersions
 from .imaging import Modes
+from .rois import RoiReader
 from .timings import LabViewTimings231, LabViewTimingsPre2018
 
 try:
@@ -184,6 +185,8 @@ def rel(file_name):
         self.read_user_config()
         header_fields = self.parse_experiment_header_ini(rel('Experiment Header.ini'))
         speed_data, expt_start_time = self.read_speed_data(rel('Speed_Data/Speed data 001.txt'))
+        if expt_start_time is None:
+            expt_start_time = pd.Timestamp.now()  # FIXME get experiment start time from header if not present in speed data
         localized_start_time = expt_start_time.tz_localize(timezone('Europe/London'))
         # Create the NWB file
         extensions = ["e-labview.py", "e-pixeltimes.py"]
@@ -247,7 +250,8 @@ def rel(file_name):
             """Return the path of a file name relative to the Labview folder."""
             return os.path.join(folder_path, file_name)
 
-        self.add_speed_data(speed_data, expt_start_time)
+        if speed_data is not None:
+            self.add_speed_data(speed_data, expt_start_time)
         self.determine_trial_times()
         self.add_stimulus()
         self.read_cycle_relative_times(folder_path)
@@ -356,10 +360,12 @@ def parse_experiment_header_ini(self, filename):
         self.imaging_info = header.get_imaging_information()
         # TODO this should probably go into a determine_trial_times function
         #  that hides the handling of the different LabView versions.
-        if self.labview_version is LabViewVersions.v231:
+        if not self.labview_version.is_legacy:
             self.determine_trial_times_from_header(header)
         # Use the user specified in the header to select default session etc. metadata
         self.record_metadata(header['LOGIN']['User'])
+        # Determine how to read the ROIs based on the header information
+        self.roi_reader = RoiReader.get_reader(header)
         return header.get_raw_fields()
 
     def record_metadata(self, user):
@@ -446,17 +452,20 @@ def read_speed_data(self, file_name):
         Pandas data table, and initial_time is the experiment start time, which sets the
         session_start_time for the NWB file.
         """
-        self.log('Loading speed data from {}', file_name)
-        assert os.path.isfile(file_name)
-        speed_data = pd.read_csv(file_name, sep='\t', header=None, usecols=[0, 1, 2, 3], index_col=0,
-                                 names=('Date', 'Time', 'Trial time', 'Speed'),
-                                 dtype={'Trial time': np.int32, 'Speed': np.float32},
-                                 parse_dates=[[0, 1]],  # Combine first two cols
-                                 dayfirst=True, infer_datetime_format=True,
-                                 memory_map=True)
-        initial_offset = pd.Timedelta(microseconds=speed_data['Trial time'][0])
-        initial_time = speed_data.index[0] - initial_offset
-        return speed_data, initial_time
+        if os.path.isfile(file_name):
+            self.log('Loading speed data from {}', file_name)
+            speed_data = pd.read_csv(file_name, sep='\t', header=None, usecols=[0, 1, 2, 3], index_col=0,
+                                     names=('Date', 'Time', 'Trial time', 'Speed'),
+                                     dtype={'Trial time': np.int32, 'Speed': np.float32},
+                                     parse_dates=[[0, 1]],  # Combine first two cols
+                                     dayfirst=True, infer_datetime_format=True,
+                                     memory_map=True)
+            initial_offset = pd.Timedelta(microseconds=speed_data['Trial time'][0])
+            initial_time = speed_data.index[0] - initial_offset
+            return speed_data, initial_time
+        else:
+            self.log('No speed data found.')
+            return None, None
 
     def add_speed_data(self, speed_data, initial_time):
         """Add acquired speed data the the NWB file.
@@ -509,26 +518,32 @@ def determine_trial_times(self):
         There is a short interval between trials which still has speed data recorded, so it's
         the second reset which marks the start of the next trial.
         """
-        speed_data_ts = self.nwb_file.get_acquisition('speed_data')
-        if self.labview_version is LabViewVersions.pre2018:
-            self.log('Calculating trial times from speed data')
-            trial_times_ts = self.nwb_file.get_acquisition('trial_times')
-            trial_times = np.array(trial_times_ts.data)
-            # Prepend -1 so we pick up the first trial start
-            # Append -1 in case there isn't a reset recorded at the end of the last trial
-            deltas = np.ediff1d(trial_times, to_begin=-1, to_end=-1)
-            # Find resets and pair these up to mark start & end points
-            reset_idxs = (deltas < 0).nonzero()[0].copy()
-            assert reset_idxs.ndim == 1
-            num_trials = reset_idxs.size // 2  # Drop the extra reset added at the end if
-            reset_idxs = np.resize(reset_idxs, (num_trials, 2))  # it's not needed
-            reset_idxs[:, 1] -= 1  # Select end of previous segment, not start of next
-            # Index the timestamps to find the actual start & end times of each trial. The start
-            # time is calculated using the offset value in the first reading within the trial.
-            rel_times = self.get_times(trial_times_ts)
-            epoch_times = rel_times[reset_idxs]
-            epoch_times[:, 0] -= trial_times[reset_idxs[:, 0]] * 1e-6
-        elif self.labview_version is LabViewVersions.v231:
+        try:  # see if there was speed data in LabView, in which case it's already in our file.
+            speed_data_ts = self.nwb_file.get_acquisition('speed_data')
+        except KeyError:  # otherwise, there was no speed data in the LabView folder.
+            speed_data_ts = None
+        if self.labview_version.is_legacy:
+            if speed_data_ts is not None:
+                self.log('Calculating trial times from speed data')
+                trial_times_ts = self.nwb_file.get_acquisition('trial_times')
+                trial_times = np.array(trial_times_ts.data)
+                # Prepend -1 so we pick up the first trial start
+                # Append -1 in case there isn't a reset recorded at the end of the last trial
+                deltas = np.ediff1d(trial_times, to_begin=-1, to_end=-1)
+                # Find resets and pair these up to mark start & end points
+                reset_idxs = (deltas < 0).nonzero()[0].copy()
+                assert reset_idxs.ndim == 1
+                num_trials = reset_idxs.size // 2  # Drop the extra reset added at the end if
+                reset_idxs = np.resize(reset_idxs, (num_trials, 2))  # it's not needed
+                reset_idxs[:, 1] -= 1  # Select end of previous segment, not start of next
+                # Index the timestamps to find the actual start & end times of each trial. The start
+                # time is calculated using the offset value in the first reading within the trial.
+                rel_times = self.get_times(trial_times_ts)
+                epoch_times = rel_times[reset_idxs]
+                epoch_times[:, 0] -= trial_times[reset_idxs[:, 0]] * 1e-6
+            else:
+                raise ValueError("Legacy code relies on having speed data to compute trial times.")
+        else:
             epoch_times = self.trial_times
         # Create the epochs in the NWB file
         # Note that we cannot pass the actual start time to nwb_file.add_epoch since it
@@ -539,18 +554,20 @@ def determine_trial_times(self):
         # maybe better thought of as the time of the last junk speed reading.
         # We also massage the end time since otherwise data points at exactly that time are
         # omitted.
-        self.nwb_file.add_epoch_column('epoch_name', 'the name of the epoch')
+        if speed_data_ts is not None:
+            self.nwb_file.add_epoch_column('epoch_name', 'the name of the epoch')
         for i, (start_time, stop_time) in enumerate(epoch_times):
             assert stop_time > start_time >= 0
-            trial = 'trial_{:04d}'.format(i + 1)
-            self.nwb_file.add_epoch(
-                epoch_name=trial,
-                start_time=start_time if i == 0 else start_time + 1e-9,
-                stop_time=stop_time + 1e-9,
-                timeseries=[speed_data_ts])
+            if speed_data_ts is not None:
+                trial = 'trial_{:04d}'.format(i + 1)
+                self.nwb_file.add_epoch(
+                    epoch_name=trial,
+                    start_time=start_time if i == 0 else start_time + 1e-9,
+                    stop_time=stop_time + 1e-9,
+                    timeseries=[speed_data_ts])
             # We also record exact start & end times in the trial table, since our epochs
             # correspond to trials.
-            self.nwb_file.add_trial(start_time=start_time, stop_time=stop_time)
+            self.nwb_file.add_trial(start_time=start_time if i == 0 else start_time + 1e-9, stop_time=stop_time + 1e-9)
         self._write()
 
     def add_stimulus(self):
@@ -576,11 +593,11 @@ def add_stimulus(self):
             # TODO We can maybe build puffs and times a bit more efficiently or
             # in fewer steps, although it probably won't make a huge difference
             # (eg we can now get all the times with a single indexing expression)
-            num_epochs = len(self.nwb_file.epochs)
-            puffs = [u'puff'] * num_epochs
+            num_trials = len(self.nwb_file.trials)
+            puffs = [u'puff'] * num_trials
             times = np.zeros((len(puffs),), dtype=np.float64)
-            for i in range(num_epochs):
-                times[i] = self.nwb_file.epochs[i, 'start_time'] + stim['trial_time_offset']
+            for i in range(num_trials):
+                times[i] = self.nwb_file.trials[i, 'start_time'] + stim['trial_time_offset']
             attrs['timestamps'] = times
             attrs['data'] = puffs
             self.nwb_file.add_stimulus(TimeSeries(**attrs))
@@ -602,13 +619,13 @@ def rel(file_name):
 
         roi_path = rel('ROI.dat')
         assert os.path.isfile(roi_path)
-        if self.labview_version is LabViewVersions.pre2018:
+        if self.labview_version.is_legacy:
             file_path = rel('Single cycle relative times.txt')
             assert os.path.isfile(file_path)
             timings = LabViewTimingsPre2018(relative_times_path=file_path,
                                             roi_path=roi_path,
                                             dwell_time=self.imaging_info.dwell_time / 1e6)
-        elif self.labview_version is LabViewVersions.v231:
+        elif self.labview_version is LabViewVersions.v231 or self.labview_version is LabViewVersions.v300:
             file_path = rel('Single cycle relative times_HW.txt')
             assert os.path.isfile(file_path)
             timings = LabViewTimings231(relative_times_path=file_path,
@@ -657,15 +674,14 @@ def read_functional_data(self, folder_path):
         self.log('Loading functional data from {}', folder_path)
         assert os.path.isdir(folder_path)
         # Figure out timestamps, measured in seconds
-        epoch_names = self.nwb_file.epochs[:, 'epoch_name']
-        trials = [int(s[6:]) for s in epoch_names]  # names start with 'trial_'
+        trials = [s+1 for s in self.nwb_file.trials.id.data]
         cycles_per_trial = self.imaging_info.cycles_per_trial
-        num_times = cycles_per_trial * len(epoch_names)
+        num_times = cycles_per_trial * len(trials)
         single_trial_times = np.arange(cycles_per_trial) * self.cycle_time
         times = np.zeros((num_times,), dtype=float)
         # TODO Perhaps this loop can be vectorised
-        for i in range(len(epoch_names)):
-            trial_start = self.nwb_file.epochs[i, 'start_time']
+        for i in range(len(trials)):
+            trial_start = self.nwb_file.trials[i, 'start_time']
             times[i * cycles_per_trial:
                   (i + 1) * cycles_per_trial] = single_trial_times + trial_start
         self.custom_silverlab_dict['cycle_time'] = self.cycle_time
@@ -969,28 +985,10 @@ def add_rois(self, roi_path):
         organised by ROI number and channel name, so we can iterate there. Issue #16.
         """
         self.log('Loading ROI locations from {}', roi_path)
-        assert os.path.isfile(roi_path)
-        roi_data = pd.read_csv(
-            roi_path, sep='\t', header=0, index_col=False, dtype=np.float16,
-            converters={'Z start': np.float64, 'Z stop': np.float64}, memory_map=True)
-        # Rename the columns so that we can use them as identifiers later on
-        column_mapping = {
-            'ROI index': 'roi_index', 'Pixels in ROI': 'num_pixels',
-            'X start': 'x_start', 'Y start': 'y_start', 'Z start': 'z_start',
-            'X stop': 'x_stop', 'Y stop': 'y_stop', 'Z stop': 'z_stop',
-            'Laser Power (%)': 'laser_power', 'ROI Time (ns)': 'roi_time_ns',
-            'Angle (deg)': 'angle_deg', 'Composite ID': 'composite_id',
-            'Number of lines': 'num_lines', 'Frame Size': 'frame_size',
-            'Zoom': 'zoom', 'ROI group ID': 'roi_group_id'
-        }
-        roi_data.rename(columns=column_mapping, inplace=True)
+        roi_data = self.roi_reader.read_roi_table(roi_path)
         module = self.nwb_file.create_processing_module(
             'Acquired_ROIs',
             'ROI locations and acquired fluorescence readings made directly by the AOL microscope.')
-        # Convert some columns to int
-        roi_data = roi_data.astype(
-            {'x_start': np.uint16, 'x_stop': np.uint16, 'y_start': np.uint16, 'y_stop': np.uint16,
-             'num_pixels': int})
         seg_iface = ImageSegmentation()
         module.add(seg_iface)
         self._write()
@@ -1015,9 +1013,8 @@ def add_rois(self, roi_path):
             )
             # Specify the non-standard data we will be storing for each ROI,
             # which includes all the raw data fields from the original file
-            plane.add_column('dimensions', 'Dimensions of the ROI')
-            for old_name, new_name in column_mapping.items():
-                plane.add_column(new_name, old_name)
+            for column_name, column_description in self.roi_reader.columns.items():
+                plane.add_column(column_name, column_description)
             index = 0  # index of the row as it will be stored in the ROI table
             self.roi_mapping[plane_name] = {}
             for row in roi_group.itertuples():
@@ -1044,7 +1041,7 @@ def add_rois(self, roi_path):
                     pixels[i, 2] = 1  # weight for this pixel
                 plane.add_roi(id=roi_id, pixel_mask=[tuple(r) for r in pixels.tolist()],
                               dimensions=dimensions,
-                              **{field: getattr(row, field) for field in column_mapping.values()})
+                              **self.roi_reader.get_row_attributes(row))
                 self.roi_mapping[plane_name][roi_id] = index
                 index += 1
         self._write()