[MRG] Adds Sources for scans.tsv files

mne_bids/copyfiles.py

@@ -347,7 +347,7 @@ def copyfile_brainvision(vhdr_src, vhdr_dest, anonymize=None, verbose=None):
 
     if anonymize is not None:
         raw = read_raw_brainvision(vhdr_src, preload=False, verbose=0)
-        daysback, keep_his = _check_anonymize(anonymize, raw, '.vhdr')
+        daysback, keep_his, _ = _check_anonymize(anonymize, raw, '.vhdr')
         raw.info = anonymize_info(raw.info, daysback=daysback,
                                   keep_his=keep_his)
         _anonymize_brainvision(fname_dest + '.vhdr',
@@ -453,7 +453,7 @@ def copyfile_edf(src, dest, anonymize=None):
         start_date, admin_code, tech, equip = rec_info.split(' ')[1:5]
 
         # Try to anonymize the recording date
-        daysback, keep_his = _check_anonymize(anonymize, raw, '.edf')
+        daysback, keep_his, _ = _check_anonymize(anonymize, raw, '.edf')
         anonymize_info(raw.info, daysback=daysback, keep_his=keep_his)
         start_date = '01-JAN-1985'
         meas_date = '01.01.85'

mne_bids/tests/test_write.py

@@ -2573,7 +2573,7 @@ def test_coordsystem_json_compliance(
     warning_str['edf_warning'],
     warning_str['brainvision_unit']
 )
-def test_anonymize(subject, dir_name, fname, reader, tmp_path):
+def test_anonymize(subject, dir_name, fname, reader, tmp_path, _bids_validate):
     """Test writing anonymized EDF data."""
     data_path = testing.data_path()
 
@@ -2600,6 +2600,7 @@ def test_anonymize(subject, dir_name, fname, reader, tmp_path):
         bids_path.update(suffix='eeg', datatype='eeg')
     daysback_min, daysback_max = get_anonymization_daysback(raw)
     anonymize = dict(daysback=daysback_min + 1)
+    orig_bids_path = bids_path.copy()
     bids_path = \
         write_raw_bids(raw, bids_path, overwrite=True,
                        anonymize=anonymize, verbose=False)
@@ -2619,6 +2620,26 @@ def test_anonymize(subject, dir_name, fname, reader, tmp_path):
         assert _raw.info['meas_date'].day == 1
     assert raw2.info['meas_date'].year < 1925
 
+    # write without sources
+    scans_fname = BIDSPath(subject=bids_path.subject, session=bids_path.session,
+                            suffix='scans', extension='.tsv',
+                            root=bids_path.root)
+    anonymize['keep_source'] = False
+    bids_path = \
+        write_raw_bids(raw, orig_bids_path, overwrite=True,
+                       anonymize=anonymize, verbose=False)
+    scans_tsv = _from_tsv(scans_fname)
+    assert 'sources' not in scans_tsv.keys()
+
+    # Write with sources this time get the scans tsv
+    with pytest.warns(RuntimeWarning, match='`daysback` is too small'):
+        bids_path = write_raw_bids(
+            raw, orig_bids_path, overwrite=True,
+            anonymize=dict(daysback=0, keep_source=True), verbose=False)
+    scans_tsv = _from_tsv(scans_fname)
+    assert scans_tsv['sources'] == [op.basename(fname) for fname in raw.filenames]
+    _bids_validate(bids_path.root)
+
 
 @pytest.mark.filterwarnings(warning_str['channel_unit_changed'])
 def test_sidecar_encoding(_bids_validate, tmp_path):

mne_bids/utils.py

@@ -331,7 +331,8 @@ def _check_anonymize(anonymize, raw, ext):
                              'is able to store in FIF format, must '
                              f'be less than {daysback_max}')
     keep_his = anonymize['keep_his'] if 'keep_his' in anonymize else False
-    return daysback, keep_his
+    keep_src = anonymize['keep_source'] if 'keep_source' in anonymize else False
+    return daysback, keep_his, keep_src
 
 
 def _get_anonymization_daysback(raw):

mne_bids/write.py

@@ -425,7 +425,7 @@ def _participants_json(fname, overwrite=False):
     _write_json(fname, cols, overwrite)
 
 
-def _scans_tsv(raw, raw_fname, fname, overwrite=False):
+def _scans_tsv(raw, raw_fname, fname, keep_src, overwrite=False):
     """Create a scans.tsv file and save it.
 
     Parameters
@@ -436,6 +436,9 @@ def _scans_tsv(raw, raw_fname, fname, overwrite=False):
         Relative path to the raw data file.
     fname : str
         Filename to save the scans.tsv to.
+    keep_src : bool
+        If ``True`` (default), the filename of the raw source will
+        be stored in the ``sources`` column.
     overwrite : bool
         Defaults to False.
         Whether to overwrite the existing data in the file.
@@ -447,10 +450,6 @@ def _scans_tsv(raw, raw_fname, fname, overwrite=False):
     meas_date = raw.info['meas_date']
     if meas_date is None:
         acq_time = 'n/a'
-    # The "Z" indicates UTC time
-    elif isinstance(meas_date, (tuple, list, np.ndarray)):  # pragma: no cover
-        # for MNE < v0.20
-        acq_time = _stamp_to_dt(meas_date).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
     elif isinstance(meas_date, datetime):
         # for MNE >= v0.20
         acq_time = meas_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
@@ -477,6 +476,24 @@ def _scans_tsv(raw, raw_fname, fname, overwrite=False):
         [('filename', ['{:s}'.format(raw_f.replace(os.sep, '/'))
           for raw_f in raw_fnames]),
             ('acq_time', [acq_time] * len(raw_fnames))])
+
+    # add source filename if desired
+    if keep_src:
+        data['sources'] = [op.basename(fname) for fname in raw.filenames]
+
+        # write out a sidecar JSON if not exists
+        sidecar_path = str(fname).replace('.tsv', '.json')
+        sidecar_json = {'sources': 'Original source filename.'}
+        if not op.exists(sidecar_path):
+            with open(sidecar_path, 'w') as fout:
+                json.dump(sidecar_json, fout)
+        else:
+            with open(sidecar_path, 'r') as fin:
+                curr_sidecar_json = json.load(fin)
+            if 'sources' not in curr_sidecar_json:
+                sidecar_json.update(curr_sidecar_json)
+                with open(sidecar_path, 'w') as fout:
+                    json.dump(sidecar_json, fout)
 
     if os.path.exists(fname):
         orig_data = _from_tsv(fname)
@@ -485,6 +502,14 @@ def _scans_tsv(raw, raw_fname, fname, overwrite=False):
             raise FileExistsError(f'"{raw_fname}" already exists in '
                                   f'the scans list. Please set '
                                   f'overwrite to True.')
+
+        for key in data.keys():
+            if key in orig_data:
+                continue
+
+            # add 'n/a' if any missing columns
+            orig_data[key] = ['n/a'] * len(next(iter(data.values())))
+
         # otherwise add the new data
         data = _combine_rows(orig_data, data, 'filename')
 
@@ -1157,6 +1182,10 @@ def write_raw_bids(raw, bids_path, events_data=None, event_id=None,
             recording date will be overwritten as well. If True, keep subject
             information apart from the recording date.
 
+        ``keep_source`` : bool
+            If ``True`` (default), the filename of the raw source fill will
+            be stored in the sidecar JSON ``Sources`` key.
+
     format : 'auto' | 'BrainVision' | 'EDF' | 'FIF'
         Controls the file format of the data after BIDS conversion. If
         ``'auto'``, MNE-BIDS will attempt to convert the input data to BIDS
@@ -1454,8 +1483,9 @@ def write_raw_bids(raw, bids_path, events_data=None, event_id=None,
         suffix='channels', extension='.tsv')
 
     # Anonymize
+    keep_src = False
     if anonymize is not None:
-        daysback, keep_his = _check_anonymize(anonymize, raw, ext)
+        daysback, keep_his, keep_src = _check_anonymize(anonymize, raw, ext)
         raw.anonymize(daysback=daysback, keep_his=keep_his)
 
         if bids_path.datatype == 'meg' and ext != '.fif':
@@ -1637,7 +1667,7 @@ def write_raw_bids(raw, bids_path, events_data=None, event_id=None,
 
     # write to the scans.tsv file the output file written
     scan_relative_fpath = op.join(bids_path.datatype, bids_path.fpath.name)
-    _scans_tsv(raw, scan_relative_fpath, scans_path.fpath, overwrite)
+    _scans_tsv(raw, scan_relative_fpath, scans_path.fpath, keep_src, overwrite)
     logger.info(f'Wrote {scans_path.fpath} entry with '
                 f'{scan_relative_fpath}.')
 
@@ -2568,7 +2598,8 @@ def anonymize_dataset(bids_root_in, bids_root_out, daysback='auto',
                 bids_path=bp_out,
                 anonymize={
                     'daysback': daysback,
-                    'keep_his': False
+                    'keep_his': False,
+                    'keep_src': False,
                 },
                 empty_room=bp_er_out,
                 verbose='error'