Merge pull request #19 from CSymes/develop

Release v1.0.1

README.md

@@ -1,6 +1,6 @@
 # quickDDM [![Build Status](https://dev.azure.com/s3550167/quickDDM/_apis/build/status/CSymes.quickDDM?branchName=develop)](https://dev.azure.com/s3550167/quickDDM/_build/latest?definitionId=1&branchName=develop)
 Efficient processing of Differential Dynamic Microscopy (DDM) allowing both
-traditional CPU-based and GPU-accelerated processing, with a GUI for analysis.
+traditional CPU-based, and GPU-accelerated processing, with a GUI for analysis.
 
 ## Contributors
 Cary Symes ([email protected])  
@@ -21,26 +21,40 @@ matplotlib==3.0.3
 numpy==1.16.2
 opencv-python==4.0.0.21
 Pillow==6.0.0
+scipy==1.3.0
+
+[Optionally for OpenCL utilisation]
 pyopencl==2018.2.5
 reikna==0.7.2
-scipy==1.3.0
 ```
 
 These may be installed easily by running `pip install -r requirements.txt`
 from in the source directory, assuming pip is already installed.
+It is recommended that a virtual environment is used to separate this from the
+system environment.
 
 ## Running
 
-To launch the UI from the command line, navigate to the 'quickDDM' directory
-within the source directory and execute `python ui_tk.py`. Alternativly,
-download and run a binary build from [Releases](https://github.com/CSymes/quickDDM/releases)
+To launch the UI from the command line, navigate to the project directory
+and execute `python launcher.py`.  
+Alternatively, download and run a binary build from
+[Releases](https://github.com/CSymes/quickDDM/releases)
 or a build artifact from [Azure](dev.azure.com/s3550167/quickDDM/_build).
 
+## Testing
+
+There are a series of tests in the `tests` directory.
+They can be run by calling
+`python -m unittest [-v]`.  
+Individual tests may be run as such:
+(e.g.) `python -m unittest tests.testFourierTransforms`
+
 ## Building
 
 If you need a binary build, run `publish.py` from the project root.
-Creates a portable executable `ui_tk.exe` in a `dist` folder, with all
-necessary libraries bundled.
+Creates a portable build of the program in a `dist` folder, with all
+necessary libraries bundled. It can be run by executing the `quickDDM.exe`
+file inside it.  
 Requires the `PyInstaller` packaged to be installed.
 
 ## Credits
@@ -55,6 +69,6 @@ The flow of "sequentialChunkerMain" in "processingCore.py" and
 "sequentialGPUChunker" in "gpuCore.py" in paricular are
 closely based on the GPU memory management technique detailed therein.
 
-Our understanding of the core process was heaviy informed by:  
+Our understanding of the core process was heavily informed by:  
 L. G. Wilson et al., "Differential Dynamic Microscopy of Bacterial Motility,"
 Physical Review Letters, vol. 106, no. 1, p. 4, Jan 2011.

azure-pipelines.yml

@@ -107,18 +107,18 @@ steps:
 
 # Creates a .exe build of the whole program
 # Only runs on the Windows build
-
+# Also creates a .bat shortcut into the build folder, since it's a bit messy
 - script: |
     pip install pypiwin32 pyinstaller
     python publish.py
-    mv ./dist/ui_tk.exe ./quickDDM.exe
+    echo '@start "" "quickDDM/quickDDM.exe"' > dist/quickDDM.bat
   displayName: 'Create Windows Build'
   condition: eq( variables.platform, 'windows' )
 
 - task: CopyFiles@2
   inputs:
-    sourceFolder: '.'
-    contents: '*.exe'
+    sourceFolder: 'dist'
+    contents: '**/*'
     targetFolder: $(Build.ArtifactStagingDirectory)
   displayName: 'Stage artifact'
   condition: eq( variables.platform, 'windows' )
@@ -127,6 +127,6 @@ steps:
 - task: PublishBuildArtifacts@1
   inputs:
     pathtoPublish: $(Build.ArtifactStagingDirectory)
-    artifactName: WindowsArtifacts
+    artifactName: WindowsBuild
   displayName: 'Publish artifact'
   condition: eq( variables.platform, 'windows' )

launcher.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#quickDDM.py
+'''
+Program launcher
+@created: 2019-06-22
+@author: Cary
+'''
+
+if __name__ == '__main__':
+	from quickDDM.ui_tk import launch
+
+	print('Launching TK-based UI')
+	launch()

publish.py

@@ -1,4 +1,7 @@
-import subprocess
+import PyInstaller.__main__
+import win32com.client
+from os.path import abspath
+
 import reikna
 import cv2
 import os
@@ -13,12 +16,15 @@
     cv_p = cv2.__path__[0]
     r_p = reikna.__path__[0]
 
-    command = (r'pyinstaller quickDDM/ui_tk.py -y '
-               r'-p "./quickDDM" '
-               r'--onefile '
-               r'--noconsole '
-               r'--exclude-module libopenblas '
-              fr'--add-data "{r_p};reikna" '
-              fr'--add-binary "{cv_p}/opencv_ffmpeg400_64.dll;."')
+    params = [r'launcher.py',
+              r'-y',
+              r'--paths=quickDDM',
+              r'--name=quickDDM',
+              r'--onedir',
+              r'--noconsole',
+              r'--exclude-module=libopenblas',
+             fr'--add-data={r_p};reikna',
+             fr'--add-binary={cv_p}\opencv_ffmpeg400_64.dll;.'
+             ]
 
-    subprocess.call(command)
+    PyInstaller.__main__.run(params)

quickDDM/gpuCore.py

@@ -18,9 +18,9 @@
 
 from collections import deque
 
-from readVideo import readVideo, readFramerate
-from calculateQCurves import calculateWithCalls
-from calculateCorrelation import calculateCorrelation
+from quickDDM.readVideo import readVideo, readFramerate
+from quickDDM.calculateQCurves import calculateWithCalls
+from quickDDM.calculateCorrelation import calculateCorrelation
 
 
 
@@ -59,14 +59,20 @@ def createNormalisationKernel(thread, shape):
     thread: Reikna CLUDA thread object
     kernel: compiled GPU kernel - assumed to have compiled signature (out, in)
     frame: data to operate on
+    outType: a numpy data type to store the resulting data as
 Returns:
     On-GPU buffer containing the results (same dimensions as `frame`)
 """
-def runKernelOperation(thread, kernel, frame):
-    frame = numpy.ascontiguousarray(frame).astype(numpy.complex128)
-
-    devFr = thread.to_device(frame) # Send frame to device
-    fBuffer = thread.array(frame.shape, dtype=numpy.complex128)
+def runKernelOperation(thread, kernel, frame, outType=numpy.complex128):
+    if type(frame) is reikna.cluda.ocl.Array:
+        # frame is already on the GPU
+        devFr = frame
+    else:
+        # It's in main memory - need to cast and send to VRAM
+        frame = numpy.ascontiguousarray(frame).astype(numpy.complex128)
+        devFr = thread.to_device(frame) # Send frame to device
+
+    fBuffer = thread.array(frame.shape, dtype=outType)
 
     kernel(fBuffer, devFr)
     return fBuffer

quickDDM/processingCore.py

@@ -10,10 +10,10 @@
 
 import sys
 import numpy as np
-import readVideo as rV
-import twoDFourier as tDF
-import calculateQCurves as cQC
-import calculateCorrelation as cC
+import quickDDM.readVideo as rV
+import quickDDM.twoDFourier as tDF
+import quickDDM.calculateQCurves as cQC
+import quickDDM.calculateCorrelation as cC
 from collections import deque
 
 """

quickDDM/ui_tk.py

@@ -9,11 +9,11 @@
 
 HAS_BACKEND_GPU = False
 
-from curveFitting import fitCorrelationsToFunction, generateFittedCurves
-from curveFitting import FITTING_FUNCTIONS
-from processingCore import sequentialChunkerMain
+from quickDDM.curveFitting import fitCorrelationsToFunction, generateFittedCurves
+from quickDDM.curveFitting import FITTING_FUNCTIONS
+from quickDDM.processingCore import sequentialChunkerMain
 try: # Attempt to load GPU backend, and check if hardware/drivers are present
-    from gpuCore import sequentialGPUChunker
+    from quickDDM.gpuCore import sequentialGPUChunker
 
     try:
         from pyopencl._cl import LogicError
@@ -924,7 +924,7 @@ def center(win):
 
     win.geometry(f'+{x}+{y}') # Set position
 
-if __name__ == '__main__':
+def launch():
     # Create a new Tk framework instance / window
     window = Tk()
 
@@ -970,3 +970,6 @@ def onQuit():
 
 # TODO Think about splitting this into multiple files
 #      Maybe have a UI subpackage instead of a UI module, aye
+
+if __name__ == '__main__':
+    launch()

tests/testGPUProcessing.py

@@ -13,6 +13,8 @@
 
 from quickDDM.readVideo import readVideo
 from quickDDM.twoDFourier import twoDFourierUnnormalized, castToReal
+from quickDDM.gpuCore import (createComplexFFTKernel,
+    createNormalisationKernel, runKernelOperation)
 
 import numpy, numpy.testing
 import unittest
@@ -25,54 +27,36 @@ def setUpClass(self):
         self.frames = readVideo('tests/data/small.avi').astype(numpy.int16)
         self.firstDiff = self.frames[1] - self.frames[0]
 
-        api = reikna.cluda.ocl_api()
-        self.thread = api.Thread.create()
+        self.thread = reikna.cluda.ocl_api().Thread.create()
 
-        # One array style for the complex data (FFT out) and floats for post-normalisation
-        footprint = self.thread.array(self.frames[0].shape, dtype=numpy.complex)
-        footprint_out = self.thread.array(self.frames[0].shape, dtype=numpy.float)
-
-        self.fft = FFT(footprint).compile(self.thread) # FFT Computation object
-
-        fftshift = FFTShift(footprint)
-        div = div_const(footprint, numpy.sqrt(numpy.prod(self.frames[0].shape))) # divide by frame size
-        norm = norm_const(footprint, 2) # abs (reduce to real mag.) and square
-
-        # attach transformations to fftshift computation
-        fftshift.parameter.output.connect(div, div.input, output_prime=div.output)
-        fftshift.parameter.output_prime.connect(norm, norm.input, output_prime_2=norm.output)
-
-        self.normalise = fftshift.compile(self.thread) # Compile FFTShift with normalisation Transformations
+        # Get the OpenCL kernels from the 2DF module
+        self.fft = createComplexFFTKernel(self.thread, self.frames[0].shape)
+        self.normalise = createNormalisationKernel(self.thread, self.frames[0].shape)
 
     def testSimpleFourierMatchesCPU(self):
-        devFr = self.thread.to_device(self.firstDiff.astype(numpy.complex))
-        self.fft(devFr, devFr)
+        local = runKernelOperation(self.thread, self.fft, self.firstDiff).get()
 
-        local = devFr.get()
-        cpu = numpy.fft.fft2(self.firstDiff)
+        # Need to un-shift since twoDFourierUnnormalized does it already, but
+        # the OCL kernel doesn't
+        ftframe = numpy.asarray([self.firstDiff])
+        cpu = numpy.fft.fftshift(twoDFourierUnnormalized(ftframe)[0])
 
         numpy.testing.assert_allclose(local, cpu)
 
     def testFourierWithNormalisationMatchesCPU(self):
-        res = self.thread.array(self.frames[0].shape, dtype=numpy.float64)
-        devFr = self.thread.to_device(self.firstDiff.astype(numpy.complex))
-        self.fft(devFr, devFr)
-        self.normalise(res, devFr)
-
-        local = res.get()
+        local = runKernelOperation(self.thread, self.fft, self.firstDiff)
+        local = runKernelOperation(self.thread, self.normalise, local,
+            outType=numpy.float64).get()
 
         ftframe = numpy.asarray([self.firstDiff])
         cpu = castToReal(twoDFourierUnnormalized(ftframe)[0])
 
         numpy.testing.assert_allclose(local, cpu)
 
     def testFourierWithNormalisationMatchesMatlab(self):
-        res = self.thread.array(self.frames[0].shape, dtype=numpy.float64)
-        devFr = self.thread.to_device(self.firstDiff.astype(numpy.complex))
-        self.fft(devFr, devFr)
-        self.normalise(res, devFr)
-
-        local = res.get()
+        local = runKernelOperation(self.thread, self.fft, self.firstDiff)
+        local = runKernelOperation(self.thread, self.normalise, local,
+            outType=numpy.float64).get()
 
         with open('tests/data/fft_matlab_f2-f1.csv', 'rb') as mf:
             m = numpy.loadtxt(mf, delimiter=',')

tests/testReadVideo.py

@@ -51,7 +51,7 @@ def testBadPermissionsReadFails(self):
 
     def testEmptyVideoFails(self):
         with self.assertRaises(OSError):
-            print('\nExpecting ioctl error... ', end='')
+            print('\nExpecting IO error... ', end='')
             frames = readVideo('tests/data/empty.avi')
 
     def testReadFramerate(self):