Merge pull request #236 from texadactyl/master

Fix problems in various regression tests

VERSION-HISTORY.md

@@ -3,6 +3,7 @@ This file is a version history of blimpy amendments, beginning with version 2.0.
 <br>
 |    Date    | Version | Contents |
 | :--: | :--: | :-- |
+| 2021-08-17 | 2.0.27 | Fix problems in various regression tests.  |
 | 2021-08-17 | 2.0.26 | Implement requirements_test.txt (Issue #234).  |
 | 2021-08-16 | 2.0.25 | Try to remove HDF5 files that cannot be fully written due to an exception (Issue #232).  |
 | 2021-08-13 | 2.0.24 | Check the data B-tree in issue #226.  |

blimpy/waterfall.py

@@ -385,7 +385,8 @@ def cmd_tool(args=None):
 
     parser.add_argument('filename', type=str,
                         help='Name of file to read')
-    parser.add_argument('-p', action='store',  default='a', dest='what_to_plot', type=str,
+    parser.add_argument('-p', action='store', dest='what_to_plot', type=str,
+                        choices=['w', 's', 't', 'k', 'mm', 'a', 'ank'],
                         help='Show: "w" waterfall (freq vs. time) plot; "s" integrated spectrum plot; \
                         "t" for time series; "mm" for spectrum including min max; "k" for kurtosis; \
                         "a" for all available plots and information; and "ank" for all but kurtosis.')
@@ -445,10 +446,7 @@ def cmd_tool(args=None):
             n_coarse_chan = fil.calc_n_coarse_chan()
             fil.blank_dc(n_coarse_chan)
 
-        if parse_args.what_to_plot == "w":
-            plt.figure("waterfall", figsize=(8, 6))
-            fil.plot_waterfall(f_start=parse_args.f_start, f_stop=parse_args.f_stop)
-        elif parse_args.what_to_plot == "s":
+        if parse_args.what_to_plot == "s":
             plt.figure("Spectrum", figsize=(8, 6))
             fil.plot_spectrum(logged=True, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t='all')
         elif parse_args.what_to_plot == "mm":
@@ -466,6 +464,9 @@ def cmd_tool(args=None):
         elif parse_args.what_to_plot == "ank":
             plt.figure("Multiple diagnostic plots", figsize=(12, 9),facecolor='white')
             fil.plot_all(logged=True, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t='all',kutosis=False)
+        else: # parse_args.what_to_plot = "w"
+            plt.figure("waterfall", figsize=(8, 6))
+            fil.plot_waterfall(f_start=parse_args.f_start, f_stop=parse_args.f_stop)
 
         if parse_args.plt_filename != '':
             plt.savefig(parse_args.plt_filename)

setup.py

@@ -3,7 +3,7 @@
 """
 from setuptools import setup, find_packages
 
-__version__ = '2.0.26'
+__version__ = '2.0.27'
 
 with open("README.md", "r") as fh:
     long_description = fh.read()

tests/run_tests.sh

@@ -1,9 +1,4 @@
 #!/usr/bin/env bash
-echo "------ Running Coverage Tests! ------"
-python3 setup.py install; cd tests
-pip3 install coverage codecov pyyaml
-apt-get install git
-cd ..
 coverage run --source=blimpy -m pytest
 EXITCODE=$?
 if [ $EXITCODE -ne 0 ]; then
@@ -12,12 +7,12 @@ if [ $EXITCODE -ne 0 ]; then
     echo
     exit $EXITCODE
 fi
-coverage report
+coverage html -d coverage_html
 EXITCODE=$?
 if [ $EXITCODE -ne 0 ]; then
     echo
     echo '*** Oops, coverage report failed, exit code = '$EXITCODE' ***'
     echo
     exit $EXITCODE
 fi
-codecov
+

tests/test_compare_voyager.py

@@ -4,16 +4,15 @@
 from tests.data import voyager_fil, voyager_h5
 from blimpy.plotting.config import plt
 
-def compare_waterfall_fil_to_h5():
+
+def test_compare_waterfall_fil_to_h5():
     """ Load Voyager dataset and test that both fil and hdf5 readers return same headers and data """
 
     print("Loading FIL and HDF5 data with Waterfall()..."),
     a = bl.Waterfall(voyager_h5)
     b = bl.Waterfall(voyager_fil)
-    print("OK")
 
     print("Reading headers..")
-    print("\nHDF5 file header:")
     pprint(a.header)
     print("\nFIL file header:")
     pprint(b.header)
@@ -22,18 +21,16 @@ def compare_waterfall_fil_to_h5():
     print("\nChecking header values match..."),
     for key in b.header.keys():
         assert b.header[key] == a.header[key]
-    print("OK")
 
     print("Checking datatype matches..."),
     assert a.data.dtype == b.data.dtype
-    print("OK")
 
     print("Checking data matches..."),
     assert np.allclose(a.data, b.data)
     assert a.data.dtype == b.data.dtype
-    print("OK")
 
-def compare_waterfall_fil_to_h5_methods_and_attributes():
+
+def test_waterfall_fil_to_h5_methods_and_attributes():
     """ Compare attributes and check methods """
     a = bl.Waterfall(voyager_h5)
     b = bl.Waterfall(voyager_fil)
@@ -42,7 +39,6 @@ def compare_waterfall_fil_to_h5_methods_and_attributes():
     assert a.beam_axis == b.beam_axis
     assert a.freq_axis == b.freq_axis
     assert a.time_axis == b.time_axis
-    #assert a.stokes_axis == b.stokes_axis  # Fil shouldn't have stokes axis ...
 
     assert a.calc_n_coarse_chan() == b.calc_n_coarse_chan()
     assert a.file_shape == b.file_shape
@@ -65,48 +61,19 @@ def compare_waterfall_fil_to_h5_methods_and_attributes():
     b.blank_dc(1)
     b.calibrate_band_pass_N1()
 
-    # TODO: 'compute_lsrk' -- need PYSLALIB
-    # TODO: 'compute_lst' -- need PYSLALIB
-
-    # Expected to differ between: 'ext', 'file_size_bytes', 'filename', 'container'
-    # Unused? Inherited from Filterbank.py 'gen_from_header', 'generate_freqs', 'read_filterbank', 'read_hdf5'
-
-    # TODO: 'grab_data',
-    # TODO: 'read_data',
-    # TODO: 'write_to_fil',
-    # TODO: 'write_to_filterbank',
-    # TODO: 'write_to_hdf5']
-
     dir_a = dir(a)
     dir_b = dir(b)
 
     print("Attr/methods in HDF5 but not in FIL:")
     for item in dir_a:
         if item not in dir_b:
-            print(item)
+            raise ValueError("HDF5 item is not in FIL:" + str(item))
 
     print("Attr/methods in FIL but not in HDF5:")
     for item in dir_b:
         if item not in dir_a:
-            print(item)
-
-def compare_waterfall_fil_h5_conatiners():
-    """ Compare the two containers for fil.container and h5.container """
-    a = bl.Waterfall(voyager_h5)
-    b = bl.Waterfall(voyager_fil)
-
-    dir_a = dir(a.container)
-    dir_b = dir(b.container)
+            raise ValueError("FIL item is not in HDF5:" + str(item))
 
-    print("Attr/methods in HDF5 container but not in FIL:")
-    for item in dir_a:
-        if item not in dir_b:
-            print(item)
-
-    print("Attr/methods in FIL container but not in HDF5:")
-    for item in dir_b:
-        if item not in dir_a:
-            print(item)
 
 def test_plotting_doesnt_cause_exceptions():
     """ Try running the plotting routines. They should not raise expections even without X windows """
@@ -139,8 +106,3 @@ def test_plotting_doesnt_cause_exceptions():
     b.plot_time_series()
     plt.clf()
 
-if __name__ == "__main__":
-    compare_waterfall_fil_to_h5()
-    compare_waterfall_fil_to_h5_methods_and_attributes()
-    compare_waterfall_fil_h5_conatiners()
-    test_plotting_doesnt_cause_exceptions()

tests/test_filterbank_voyager.py

@@ -6,13 +6,12 @@
 from blimpy.plotting.config import plt
 
 
-def compare_filterbank_fil_to_h5():
+def test_comparison_filterbank_fil_to_h5():
     """ Load Voyager dataset and test that both fil and hdf5 readers return same headers and data """
 
     print("Loading FIL and HDF5 data with Waterfall()..."),
     a = bl.Waterfall(voyager_h5)
     b = bl.Waterfall(voyager_fil)
-    print("OK")
 
     print("Reading headers..")
     print("\nHDF5 file header:")
@@ -24,16 +23,13 @@ def compare_filterbank_fil_to_h5():
     print("\nChecking header values match..."),
     for key in b.header.keys():
         assert b.header[key] == a.header[key]
-    print("OK")
-
+
     print("Checking datatype matches..."),
     assert a.data.dtype == b.data.dtype
-    print("OK")
 
     print("Checking data matches..."),
     assert np.allclose(a.data, b.data)
     assert a.data.dtype == b.data.dtype
-    print("OK")
 
 
 def test_plotting_doesnt_cause_exceptions():
@@ -62,9 +58,3 @@ def test_cmdtool():
     with pytest.raises(SystemExit):
         bl.waterfall.cmd_tool(args=[])
 
-
-if __name__ == "__main__":
-    compare_filterbank_fil_to_h5()
-    test_plotting_doesnt_cause_exceptions()
-
-    test_cmdtool()

tests/test_match_fils.py

@@ -1,14 +1,7 @@
+import pytest
 import blimpy as bl
 from tests.data import voyager_fil
 
-""" This test is broken for some reason.
-It works on my machine but fails the Docker Travis CI,
-so I will keep working on it.
-
-def test_size():
-    bl.match_fils.find_header_size(voyager_fil)
-"""
-
 def test_reset_works():
     """
     The test is trivial, but coverage must be
@@ -24,3 +17,12 @@ def test_batcher():
         can be patched up.
     """
     bl.match_fils.make_batch_script()
+
+def test_find_header_size():
+    assert bl.match_fils.find_header_size(voyager_fil) > 0
+
+def test_cmdline():
+    args = [voyager_fil, voyager_fil]
+    with pytest.raises(OSError):
+        bl.match_fils.cmd_tool(args)
+

tests/test_waterfall.py

@@ -2,6 +2,7 @@
 import numpy as np
 from tests.data import voyager_h5, voyager_fil
 import blimpy as bl
+from blimpy.waterfall import cmd_tool
 
 import pytest
 
@@ -22,6 +23,7 @@ def test_info():
     # plenty of missed if suites
     a._get_blob_dimensions((300, 300, 300, 300))
     a._update_header()
+    del a
 
 def test_get_freqs():
     wf = bl.Waterfall(voyager_h5)
@@ -38,10 +40,43 @@ def test_get_freqs():
     assert sum1 == sum2
 
 def test_cmdline():
-    from blimpy.waterfall import cmd_tool
+    # Plots
+
+    args = [voyager_h5, '-S', '-p', 'w', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 's', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 'mm', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 'k', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 't', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 'a', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 'ank', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    args = [voyager_h5, '-S', '-p', 'ank', '-s', OUTDIR + 'test.png']
+    cmd_tool(args)
+
+    # Blank DC to .h5
+
+    args = [voyager_h5, '-D', '-H', '-o', OUTDIR + 'test.h5']
+    cmd_tool(args)
+
+    # Blank DC to .fil
 
-    args = [voyager_h5, '-S', '-s', OUTDIR + 'test.png']
+    args = [voyager_h5, '-D', '-F', '-o', OUTDIR + 'test.fil']
     cmd_tool(args)
+
+    # info with foff negative
 
     args = [voyager_h5, '-i']
     cmd_tool(args)
@@ -61,13 +96,51 @@ def test_cmdline():
         os.remove(OUTDIR + 'test.fil')
 
 def test_cmd_arguments():
-    from blimpy.waterfall import cmd_tool
-
     args = [voyager_h5, '-H', '-F', '-o', OUTDIR + 'test.fil']
     with pytest.raises(ValueError):
         cmd_tool(args)
 
-if __name__ == "__main__":
-    test_info()
-    test_cmdline()
-    test_cmd_arguments()
+def test_neg_blank_dc():
+    wf = bl.Waterfall(voyager_h5)
+    wf.blank_dc(0)
+    wf.blank_dc(1.1)
+    del wf
+
+def test_get_chunk_dimensions():
+    wf = bl.Waterfall(voyager_h5)
+
+    wf.header['foff'] = 0.99e-5
+    assert wf._get_chunk_dimensions() == (1, 1, 1048576)
+    wf.header['foff'] = 1.1e-5
+
+    wf.header['tsamp'] = 0.99e-3
+    assert wf._get_chunk_dimensions() == (2048, 1, 512)
+    wf.header['tsamp'] = 1.1e-3
+
+    wf.header['foff'] = 0.99e-2
+    assert wf._get_chunk_dimensions() == (10, 1, 65536)
+
+    wf.header['foff'] = 1e-1
+    assert wf._get_chunk_dimensions() == (1, 1, 512)
+
+    del wf
+
+def test_neg_info_foff():
+    wf = bl.Waterfall(voyager_h5)
+    wf.header['foff'] = -1
+    wf.info()
+    del wf
+
+def test_no_filename():
+    header = { "banana": "is a fruit" }
+    data = np.arange(1, 11)
+    class Puddle(bl.Waterfall):
+        def _setup_freqs(self):
+            print("Hello from function _setup_freqs")
+    puddle = Puddle(header_dict=header, data_array=data)
+    puddle = Puddle(header_dict=header)
+    del puddle
+
+test_cmdline()
+
+