From 008a01c02333a9fc317591267224d0e4562b6bb2 Mon Sep 17 00:00:00 2001 From: eedyyidi Date: Thu, 29 Aug 2024 15:02:30 +0800 Subject: [PATCH 1/8] Add resampling functionality in diff.py --- src/mintpy/diff.py | 72 +++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index 088c45c4b..c7e9abf2f 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -55,23 +55,31 @@ def check_reference(atr1, atr2): return ref_date, ref_y, ref_x +def resample_file(file, target_length, target_width): + """Resample the entire file to match the target length and width.""" + data = readfile.read(file)[0] # Read the whole file + return resize(data, (data.shape[0], target_length, target_width), order=1, preserve_range=True) + def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8): """Calculate the difference between two time-series files. - Parameters: file1 - str, path of file1 - file2 - str, path of file2 - out_file - str, path of output file - force_diff - bool, overwrite existing output file - max_num_pixel - float, maximum number of pixels for each block - Returns: out_file - str, path of output file + Parameters: + file1 - str, path of file1 + file2 - str, path of file2 + out_file - str, path of output file + force_diff - bool, overwrite existing output file + max_num_pixel - float, maximum number of pixels for each block + Returns: + out_file - str, path of output file """ - # basic info + # Basic info atr1 = readfile.read_attribute(file1) atr2 = readfile.read_attribute(file2) k1 = atr1['FILE_TYPE'] k2 = atr2['FILE_TYPE'] date_list1 = timeseries(file1).get_date_list() + if k2 == 'timeseries': date_list2 = timeseries(file2).get_date_list() unit_fac = 1. @@ -79,10 +87,10 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) date_list2 = giantTimeseries(file2).get_date_list() unit_fac = 0.001 - # check reference point + # Check reference point ref_date, ref_y, ref_x = check_reference(atr1, atr2) - # check dates shared by two timeseries files + # Check dates shared by two time-series files date_list_shared = [i for i in date_list1 if i in date_list2] date_flag_shared = np.ones((len(date_list1)), dtype=np.bool_) if date_list_shared != date_list1: @@ -101,14 +109,27 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) else: ref_val = None - # instantiate the output file + # Instantiate the output file writefile.layout_hdf5(out_file, ref_file=file1) - # block-by-block IO - length, width = int(atr1['LENGTH']), int(atr1['WIDTH']) - num_box = int(np.ceil(len(date_list1) * length * width / max_num_pixel)) + # Resample file2 if necessary + length1, width1 = int(atr1['LENGTH']), int(atr1['WIDTH']) + length2, width2 = int(atr2['LENGTH']), int(atr2['WIDTH']) + + if not force_diff and (length1 != length2 or width1 != width2): + raise Exception('Length and Width of the files do not match. Use --force option to proceed.') + + # Resample the whole file2 to match the dimensions of file1 + if force_diff and (length1 != length2 or width1 != width2): + print(f'Resampling {file2} to match {file1} dimensions.') + data2_resampled = resample_file(file2, length1, width1) * unit_fac + else: + data2_resampled = readfile.read(file2)[0] * unit_fac + + # Split both files into corresponding blocks + num_box = int(np.ceil(len(date_list1) * length1 * width1 / max_num_pixel)) box_list, num_box = cluster.split_box2sub_boxes( - box=(0, 0, width, length), + box=(0, 0, width1, length1), num_split=num_box, dimension='y', print_msg=True, @@ -119,9 +140,8 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) print(f'\n------- processing patch {i+1} out of {num_box} --------------') print(f'box: {box}') - # read data2 (consider different reference_date/pixel) - print(f'read from file: {file2}') - data2 = readfile.read(file2, datasetName=date_list_shared, box=box)[0] * unit_fac + # Extract corresponding block from resampled data2 + data2 = data2_resampled[:, box[1]:box[3], box[0]:box[2]] if ref_val is not None: print(f'* referencing data from {os.path.basename(file2)} to y/x: {ref_y}/{ref_x}') @@ -132,19 +152,19 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) ref_ind = date_list_shared.index(ref_date) data2 -= np.tile(data2[ref_ind, :, :], (data2.shape[0], 1, 1)) - # read data1 + # Read the corresponding block from file1 print(f'read from file: {file1}') - data = readfile.read(file1, box=box)[0] + data1 = readfile.read(file1, box=box)[0] - # apply differencing - mask = data == 0. - data[date_flag_shared] -= data2 - data[mask] = 0. # Do not change zero phase value + # Apply differencing + mask = data1 == 0. + data1[date_flag_shared] -= data2 + data1[mask] = 0. # Do not change zero phase value del data2 - # write the block - block = [0, data.shape[0], box[1], box[3], box[0], box[2]] - writefile.write_hdf5_block(out_file, data=data, datasetName=k1, block=block) + # Write the block to the output file + block = [0, data1.shape[0], box[1], box[3], box[0], box[2]] + writefile.write_hdf5_block(out_file, data=data1, datasetName=k1, block=block) return out_file From 70ed9e8410e695c92b3e3d9d93bce2bb8b79c626 Mon Sep 17 00:00:00 2001 From: eedyyidi Date: Mon, 9 Sep 2024 20:52:40 +0800 Subject: [PATCH 2/8] Add diff resampling functionality --- src/mintpy/diff.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index c7e9abf2f..7b09e8ac7 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -63,23 +63,20 @@ def resample_file(file, target_length, target_width): def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8): """Calculate the difference between two time-series files. - Parameters: - file1 - str, path of file1 - file2 - str, path of file2 - out_file - str, path of output file - force_diff - bool, overwrite existing output file - max_num_pixel - float, maximum number of pixels for each block - Returns: - out_file - str, path of output file + Parameters: file1 - str, path of file1 + file2 - str, path of file2 + out_file - str, path of output file + force_diff - bool, overwrite existing output file + max_num_pixel - float, maximum number of pixels for each block + Returns: out_file - str, path of output file """ - # Basic info + # basic info atr1 = readfile.read_attribute(file1) atr2 = readfile.read_attribute(file2) k1 = atr1['FILE_TYPE'] k2 = atr2['FILE_TYPE'] date_list1 = timeseries(file1).get_date_list() - if k2 == 'timeseries': date_list2 = timeseries(file2).get_date_list() unit_fac = 1. @@ -87,10 +84,10 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) date_list2 = giantTimeseries(file2).get_date_list() unit_fac = 0.001 - # Check reference point + # check reference point ref_date, ref_y, ref_x = check_reference(atr1, atr2) - # Check dates shared by two time-series files + # check dates shared by two timeseries files date_list_shared = [i for i in date_list1 if i in date_list2] date_flag_shared = np.ones((len(date_list1)), dtype=np.bool_) if date_list_shared != date_list1: @@ -109,7 +106,7 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) else: ref_val = None - # Instantiate the output file + # instantiate the output file writefile.layout_hdf5(out_file, ref_file=file1) # Resample file2 if necessary @@ -140,8 +137,10 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) print(f'\n------- processing patch {i+1} out of {num_box} --------------') print(f'box: {box}') - # Extract corresponding block from resampled data2 - data2 = data2_resampled[:, box[1]:box[3], box[0]:box[2]] + # read data2 (consider different reference_date/pixel) + print(f'read from file: {file2}') + data2 = data2_resampled[:, box[1]:box[3], box[0]:box[2]]* unit_fac + if ref_val is not None: print(f'* referencing data from {os.path.basename(file2)} to y/x: {ref_y}/{ref_x}') @@ -152,23 +151,22 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) ref_ind = date_list_shared.index(ref_date) data2 -= np.tile(data2[ref_ind, :, :], (data2.shape[0], 1, 1)) - # Read the corresponding block from file1 + # read data1 print(f'read from file: {file1}') data1 = readfile.read(file1, box=box)[0] - # Apply differencing + # apply differencing mask = data1 == 0. data1[date_flag_shared] -= data2 - data1[mask] = 0. # Do not change zero phase value + data1[mask] = 0. # Do not change zero phase value del data2 - # Write the block to the output file + # write the block block = [0, data1.shape[0], box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(out_file, data=data1, datasetName=k1, block=block) return out_file - def diff_timeseries_and_velocity(file1, file2, out_file, max_num_pixel=2e8): """Calculate the difference between a time-series file and a velocity file. From b1248d6bd5b2ecf04d29438ec0f0ebee225261d4 Mon Sep 17 00:00:00 2001 From: eedyyidi Date: Mon, 9 Sep 2024 21:26:52 +0800 Subject: [PATCH 3/8] Add diff resampling functionality --- src/mintpy/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index 7b09e8ac7..cf7029901 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -102,7 +102,7 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) if ref_y and ref_x: ref_box = (ref_x, ref_y, ref_x + 1, ref_y + 1) - ref_val = readfile.read(file2, datasetName=date_list_shared, box=ref_box)[0] * unit_fac + ref_val = readfile.read(file1, datasetName=date_list_shared, box=ref_box)[0] * unit_fac else: ref_val = None From a264f9676ba24d6026df210d0791a0789be061ce Mon Sep 17 00:00:00 2001 From: Zhang Yunjun Date: Fri, 20 Sep 2024 16:52:23 +0800 Subject: [PATCH 4/8] `utils.readfile.read()`: add `resize2shape` arg + utils.readfile.read(): add `resize2shape` argument to support resizing directly, to facilitate the differencing between two files with different size/resolution. --- src/mintpy/utils/readfile.py | 41 +++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/mintpy/utils/readfile.py b/src/mintpy/utils/readfile.py index 24c1c029c..776467a7b 100644 --- a/src/mintpy/utils/readfile.py +++ b/src/mintpy/utils/readfile.py @@ -20,6 +20,7 @@ import h5py import numpy as np from numpy.typing import DTypeLike +from skimage.transform import resize from mintpy.objects import ( DSET_UNIT_DICT, @@ -317,14 +318,17 @@ def gdal_to_numpy_dtype(gdal_dtype: Union[str, int]) -> np.dtype: ######################################################################### def read(fname, box=None, datasetName=None, print_msg=True, xstep=1, ystep=1, data_type=None, - no_data_values=None): + resize2shape=None, no_data_values=None): """Read one dataset and its attributes from input file. Parameters: fname - str, path of file to read datasetName - str or list of str, slice names box - 4-tuple of int area to read, defined in (x0, y0, x1, y1) in pixel coordinate + as defined in the (resized) shape. x/ystep - int, number of pixels to pick/multilook for each output pixel data_type - numpy data type, e.g. np.float32, np.bool_, etc. Change the output data type + resize2shape - tuple of 2 int, resize the native matrix to the given shape, + set to None for not resizing no_data_values - list of 2 numbers, change the no-data-value in the output Returns: data - 2/3/4D matrix in numpy.array format, return None if failed atr - dictionary, attributes of data, return None if failed @@ -340,6 +344,7 @@ def read(fname, box=None, datasetName=None, print_msg=True, xstep=1, ystep=1, da data, atr = readfile.read('geometryRadar.h5', datasetName='bperp') data, atr = readfile.read('100120-110214.unw', box=(100,1100, 500, 2500)) """ + vprint = print if print_msg else lambda *args, **kwargs: None fname = os.fspath(fname) # Convert from possible pathlib.Path # metadata dsname4atr = None #used to determine UNIT @@ -349,15 +354,19 @@ def read(fname, box=None, datasetName=None, print_msg=True, xstep=1, ystep=1, da dsname4atr = datasetName.split('-')[0] atr = read_attribute(fname, datasetName=dsname4atr) - # box + # check: box & resize2shape length, width = int(atr['LENGTH']), int(atr['WIDTH']) + # ignore resize arg if it is the same as the original shape + if resize2shape and resize2shape == (length, width): + resize2shape = None if not box: box = (0, 0, width, length) + box2read = (0, 0, width, length) if resize2shape else box # read data kwargs = dict( datasetName=datasetName, - box=box, + box=box2read, xstep=xstep, ystep=ystep, ) @@ -365,20 +374,36 @@ def read(fname, box=None, datasetName=None, print_msg=True, xstep=1, ystep=1, da fext = os.path.splitext(os.path.basename(fname))[1].lower() if fext in ['.h5', '.he5']: data = read_hdf5_file(fname, print_msg=print_msg, **kwargs) - else: data, atr = read_binary_file(fname, **kwargs) + # resize + if resize2shape: + # link: https://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.resize + data = resize( + data, + output_shape=resize2shape, + order=1, + mode='constant', + anti_aliasing=True, + preserve_range=True, + ) + + # subset by box + if tuple(box) != (0, 0, width, length): + data = data[ + box[1]:box[3], + box[0]:box[2], + ] + # customized output data type if data_type is not None and data_type != data.dtype: - if print_msg: - print(f'convert numpy array from {data.dtype} to {data_type}') + vprint(f'convert numpy array from {data.dtype} to {data_type}') data = np.array(data, dtype=data_type) # convert no-data-value if isinstance(no_data_values, list): - if print_msg: - print(f'convert no-data-value from {no_data_values[0]} to {no_data_values[1]}') + vprint(f'convert no-data-value from {no_data_values[0]} to {no_data_values[1]}') data[data == no_data_values[0]] = no_data_values[1] return data, atr From ed1797fc4a00594f82807c5a3b8a0a5d0a673929 Mon Sep 17 00:00:00 2001 From: Zhang Yunjun Date: Fri, 20 Sep 2024 16:54:26 +0800 Subject: [PATCH 5/8] diff: preliminary diff_timeseries_v2() --- src/mintpy/cli/diff.py | 4 ++ src/mintpy/diff.py | 117 ++++++++++++++++++++++++++++++----------- 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/src/mintpy/cli/diff.py b/src/mintpy/cli/diff.py index dd6a0c8ab..3e96e4bbc 100755 --- a/src/mintpy/cli/diff.py +++ b/src/mintpy/cli/diff.py @@ -20,6 +20,10 @@ diff.py timeseries_ERA5_ramp_demErr.h5 ../GIANT/Stack/LS-PARAMS.h5 -o mintpy_giant.h5 diff.py reconUnwrapIfgram.h5 ./inputs/ifgramStack.h5 -o diffUnwrapIfgram.h5 + # different resolutions + diff.py timeseries.h5 inputs/SET.h5 -o timeseries_SET.h5 + diff.py timeseries_SET.h5 ion.h5 -o timeseries_SET_ion.h5 + # different file types diff.py filt_20220905_20230220.unw ./inputs/ERA5.h5 -o filt_20220905_20230220_ERA5.unw diff.py timeseries.h5 ./inputs/ITRF14.h5 -o timeseries_ITRF14.h5 diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index cf7029901..4bd0bc054 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -55,10 +55,70 @@ def check_reference(atr1, atr2): return ref_date, ref_y, ref_x -def resample_file(file, target_length, target_width): - """Resample the entire file to match the target length and width.""" - data = readfile.read(file)[0] # Read the whole file - return resize(data, (data.shape[0], target_length, target_width), order=1, preserve_range=True) +def diff_timeseries_v2(file1, file2, out_file, force_diff=False): + """Calculate the difference between two time-series files, + assuming the two files have the same spatial coverage. + """ + + # basic info + atr1 = readfile.read_attribute(file1) + atr2 = readfile.read_attribute(file2) + k1 = atr1['FILE_TYPE'] + k2 = atr2['FILE_TYPE'] + length1, width1 = int(atr1['LENGTH']), int(atr1['WIDTH']) + length2, width2 = int(atr2['LENGTH']), int(atr2['WIDTH']) + date_list1 = timeseries(file1).get_date_list() + date_list2 = timeseries(file2).get_date_list() + + # check file size + different_size = False + if length1 != length2 or width1 != width2: + different_size = True + kwargs = dict( + output_shape=(length1, width1), + order=1, + mode='constant', + anti_aliasing=True, + preserve_range=True, + ) + print('WARNING: file 1/2 have different sizes:') + print(f' file 1: ({atr1["LENGTH"]}, {atr1["WIDTH"]})') + print(f' file 2: ({atr2["LENGTH"]}, {atr2["WIDTH"]})') + if different_size and not force_diff: + raise Exception('To enforce the differencing anyway, use --force option.') + + # check reference date / point + ref_date, ref_y, ref_x = check_reference(atr1, atr2) + if ref_date: + ref_data = readfile.read(file2, datasetName=ref_date, resize2shape=(length1, width1))[0] + if different_size: + ref_data = resize(ref_data, **kwargs) + + # check dates shared by two timeseries files + date_list_shared = [i for i in date_list1 if i in date_list2] + if date_list_shared != date_list1: + print(f'WARNING: {file2} does not contain all dates in {file1}') + if force_diff: + date_list_ex = list(set(date_list1) - set(date_list_shared)) + print('Continue and enforce the differencing for their shared dates only.') + print(f'\twith following dates are ignored for differencing:\n{date_list_ex}') + else: + raise Exception('To enforce the differencing anyway, use --force option.') + + # get reference matrix + if ref_date: + ref_val = readfile.read(file2, datasetName=ref_date, resize2shape=(length1, width1))[0] + else: + ref_val = None + + #for date_str in date_list1: + # if date_str in date_list2: + # # read + # # substract + # else: + # # do nothing + # #write + def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8): """Calculate the difference between two time-series files. @@ -84,6 +144,16 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) date_list2 = giantTimeseries(file2).get_date_list() unit_fac = 0.001 + # check file size and resolution + different_size = False + if any(int(atr1[x]) != int(atr2[x]) for x in ['LENGTH', 'WIDTH']): + different_size = True + print('WARNING: file 1/2 have different sizes:') + print(f' file 1: ({atr1["LENGTH"]}, {atr1["WIDTH"]})') + print(f' file 2: ({atr2["LENGTH"]}, {atr2["WIDTH"]})') + if different_size and not force_diff: + raise Exception('Could NOT run differencing on files with different sizes! Use --force option to overwrite.') + # check reference point ref_date, ref_y, ref_x = check_reference(atr1, atr2) @@ -102,31 +172,18 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) if ref_y and ref_x: ref_box = (ref_x, ref_y, ref_x + 1, ref_y + 1) - ref_val = readfile.read(file1, datasetName=date_list_shared, box=ref_box)[0] * unit_fac + ref_val = readfile.read(file2, datasetName=date_list_shared, box=ref_box)[0] * unit_fac else: ref_val = None # instantiate the output file writefile.layout_hdf5(out_file, ref_file=file1) - # Resample file2 if necessary - length1, width1 = int(atr1['LENGTH']), int(atr1['WIDTH']) - length2, width2 = int(atr2['LENGTH']), int(atr2['WIDTH']) - - if not force_diff and (length1 != length2 or width1 != width2): - raise Exception('Length and Width of the files do not match. Use --force option to proceed.') - - # Resample the whole file2 to match the dimensions of file1 - if force_diff and (length1 != length2 or width1 != width2): - print(f'Resampling {file2} to match {file1} dimensions.') - data2_resampled = resample_file(file2, length1, width1) * unit_fac - else: - data2_resampled = readfile.read(file2)[0] * unit_fac - - # Split both files into corresponding blocks - num_box = int(np.ceil(len(date_list1) * length1 * width1 / max_num_pixel)) + # block-by-block IO + length, width = int(atr1['LENGTH']), int(atr1['WIDTH']) + num_box = int(np.ceil(len(date_list1) * length * width / max_num_pixel)) box_list, num_box = cluster.split_box2sub_boxes( - box=(0, 0, width1, length1), + box=(0, 0, width, length), num_split=num_box, dimension='y', print_msg=True, @@ -139,8 +196,7 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) # read data2 (consider different reference_date/pixel) print(f'read from file: {file2}') - data2 = data2_resampled[:, box[1]:box[3], box[0]:box[2]]* unit_fac - + data2 = readfile.read(file2, datasetName=date_list_shared, box=box)[0] * unit_fac if ref_val is not None: print(f'* referencing data from {os.path.basename(file2)} to y/x: {ref_y}/{ref_x}') @@ -153,20 +209,21 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) # read data1 print(f'read from file: {file1}') - data1 = readfile.read(file1, box=box)[0] + data = readfile.read(file1, box=box)[0] # apply differencing - mask = data1 == 0. - data1[date_flag_shared] -= data2 - data1[mask] = 0. # Do not change zero phase value + mask = data == 0. + data[date_flag_shared] -= data2 + data[mask] = 0. # Do not change zero phase value del data2 # write the block - block = [0, data1.shape[0], box[1], box[3], box[0], box[2]] - writefile.write_hdf5_block(out_file, data=data1, datasetName=k1, block=block) + block = [0, data.shape[0], box[1], box[3], box[0], box[2]] + writefile.write_hdf5_block(out_file, data=data, datasetName=k1, block=block) return out_file + def diff_timeseries_and_velocity(file1, file2, out_file, max_num_pixel=2e8): """Calculate the difference between a time-series file and a velocity file. From 219288da9016ab41f2fc9607fe9254535754611e Mon Sep 17 00:00:00 2001 From: Yidi Wang <101039873+eedyyidi@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:38:21 +0800 Subject: [PATCH 6/8] =?UTF-8?q?Update=20diff.py=EF=BC=8C=20update=20diff?= =?UTF-8?q?=5Ftimeseries=20to=20support=20different=20sizes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mintpy/diff.py | 100 ++++++++++++++------------------------------- 1 file changed, 31 insertions(+), 69 deletions(-) diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index 4bd0bc054..a0bc4fe15 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -19,6 +19,7 @@ timeseries, ) from mintpy.utils import ptime, readfile, time_func, writefile +from skimage.transform import resize ##################################################################################### @@ -55,9 +56,16 @@ def check_reference(atr1, atr2): return ref_date, ref_y, ref_x -def diff_timeseries_v2(file1, file2, out_file, force_diff=False): - """Calculate the difference between two time-series files, - assuming the two files have the same spatial coverage. +def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8): + """Calculate the difference between two time-series files. + file1.shape and file1.shape are different. + + Parameters: file1 - str, path of file1 + file2 - str, path of file2 + out_file - str, path of output file + force_diff - bool, overwrite existing output file + max_num_pixel - float, maximum number of pixels for each block + Returns: out_file - str, path of output file """ # basic info @@ -68,7 +76,12 @@ def diff_timeseries_v2(file1, file2, out_file, force_diff=False): length1, width1 = int(atr1['LENGTH']), int(atr1['WIDTH']) length2, width2 = int(atr2['LENGTH']), int(atr2['WIDTH']) date_list1 = timeseries(file1).get_date_list() - date_list2 = timeseries(file2).get_date_list() + if k2 == 'timeseries': + date_list2 = timeseries(file2).get_date_list() + unit_fac = 1. + elif k2 == 'giantTimeseries': + date_list2 = giantTimeseries(file2).get_date_list() + unit_fac = 0.001 # check file size different_size = False @@ -94,69 +107,6 @@ def diff_timeseries_v2(file1, file2, out_file, force_diff=False): if different_size: ref_data = resize(ref_data, **kwargs) - # check dates shared by two timeseries files - date_list_shared = [i for i in date_list1 if i in date_list2] - if date_list_shared != date_list1: - print(f'WARNING: {file2} does not contain all dates in {file1}') - if force_diff: - date_list_ex = list(set(date_list1) - set(date_list_shared)) - print('Continue and enforce the differencing for their shared dates only.') - print(f'\twith following dates are ignored for differencing:\n{date_list_ex}') - else: - raise Exception('To enforce the differencing anyway, use --force option.') - - # get reference matrix - if ref_date: - ref_val = readfile.read(file2, datasetName=ref_date, resize2shape=(length1, width1))[0] - else: - ref_val = None - - #for date_str in date_list1: - # if date_str in date_list2: - # # read - # # substract - # else: - # # do nothing - # #write - - -def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8): - """Calculate the difference between two time-series files. - - Parameters: file1 - str, path of file1 - file2 - str, path of file2 - out_file - str, path of output file - force_diff - bool, overwrite existing output file - max_num_pixel - float, maximum number of pixels for each block - Returns: out_file - str, path of output file - """ - - # basic info - atr1 = readfile.read_attribute(file1) - atr2 = readfile.read_attribute(file2) - k1 = atr1['FILE_TYPE'] - k2 = atr2['FILE_TYPE'] - date_list1 = timeseries(file1).get_date_list() - if k2 == 'timeseries': - date_list2 = timeseries(file2).get_date_list() - unit_fac = 1. - elif k2 == 'giantTimeseries': - date_list2 = giantTimeseries(file2).get_date_list() - unit_fac = 0.001 - - # check file size and resolution - different_size = False - if any(int(atr1[x]) != int(atr2[x]) for x in ['LENGTH', 'WIDTH']): - different_size = True - print('WARNING: file 1/2 have different sizes:') - print(f' file 1: ({atr1["LENGTH"]}, {atr1["WIDTH"]})') - print(f' file 2: ({atr2["LENGTH"]}, {atr2["WIDTH"]})') - if different_size and not force_diff: - raise Exception('Could NOT run differencing on files with different sizes! Use --force option to overwrite.') - - # check reference point - ref_date, ref_y, ref_x = check_reference(atr1, atr2) - # check dates shared by two timeseries files date_list_shared = [i for i in date_list1 if i in date_list2] date_flag_shared = np.ones((len(date_list1)), dtype=np.bool_) @@ -170,12 +120,22 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) else: raise Exception('To enforce the differencing anyway, use --force option.') + # get reference matrix if ref_y and ref_x: ref_box = (ref_x, ref_y, ref_x + 1, ref_y + 1) - ref_val = readfile.read(file2, datasetName=date_list_shared, box=ref_box)[0] * unit_fac + ref_val = [] + for date in date_list_shared: + ref_val.append(readfile.read(file2, datasetName=date, box=ref_box,resize2shape=(length1, width1))[0]) + ref_val = np.array(ref_val)* unit_fac else: ref_val = None + # resample data2 + data2_resample = [] + for date in date_list_shared: + data2_resample.append(readfile.read(file2, datasetName=date, resize2shape=(length1, width1))[0]) + data2_resample = np.array(data2_resample)* unit_fac + # instantiate the output file writefile.layout_hdf5(out_file, ref_file=file1) @@ -196,10 +156,12 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) # read data2 (consider different reference_date/pixel) print(f'read from file: {file2}') - data2 = readfile.read(file2, datasetName=date_list_shared, box=box)[0] * unit_fac + data2 = data2_resample[:,box[1]:box[3],box[0]:box[2]] if ref_val is not None: print(f'* referencing data from {os.path.basename(file2)} to y/x: {ref_y}/{ref_x}') + print(np.tile(ref_val.reshape(-1, 1, 1), (1, data2.shape[1], data2.shape[2])).shape) + print(ref_val.shape) data2 -= np.tile(ref_val.reshape(-1, 1, 1), (1, data2.shape[1], data2.shape[2])) if ref_date: From dbcf401402bcf4e8d621418efc13707aae2a16ed Mon Sep 17 00:00:00 2001 From: Yidi Wang <101039873+eedyyidi@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:54:14 +0800 Subject: [PATCH 7/8] Update diff.py --- src/mintpy/diff.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index a0bc4fe15..b35e654ee 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -160,8 +160,6 @@ def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8) if ref_val is not None: print(f'* referencing data from {os.path.basename(file2)} to y/x: {ref_y}/{ref_x}') - print(np.tile(ref_val.reshape(-1, 1, 1), (1, data2.shape[1], data2.shape[2])).shape) - print(ref_val.shape) data2 -= np.tile(ref_val.reshape(-1, 1, 1), (1, data2.shape[1], data2.shape[2])) if ref_date: From 704e5cce392d49da9a5d8336b30b63758099abfa Mon Sep 17 00:00:00 2001 From: eedyyidi <1372190696@qq.com> Date: Mon, 25 Nov 2024 09:43:15 +0800 Subject: [PATCH 8/8] update diff_timeseries to support different sizes --- src/mintpy/diff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mintpy/diff.py b/src/mintpy/diff.py index b35e654ee..a28bae0c5 100644 --- a/src/mintpy/diff.py +++ b/src/mintpy/diff.py @@ -10,6 +10,7 @@ import time import numpy as np +from skimage.transform import resize from mintpy.objects import ( IFGRAM_DSET_NAMES, @@ -19,7 +20,6 @@ timeseries, ) from mintpy.utils import ptime, readfile, time_func, writefile -from skimage.transform import resize ##################################################################################### @@ -57,8 +57,8 @@ def check_reference(atr1, atr2): def diff_timeseries(file1, file2, out_file, force_diff=False, max_num_pixel=2e8): - """Calculate the difference between two time-series files. - file1.shape and file1.shape are different. + """Calculate the difference between two time-series files. + file1.shape and file2.shape are different. Parameters: file1 - str, path of file1 file2 - str, path of file2