Merge pull request #67 from mantidproject/add_config_and_flexible_cac…

…hing

Add flexible caching and CLI

EVSVesuvio/analysis_runner.py

@@ -0,0 +1,51 @@
+import time
+from pathlib import Path
+import importlib
+import sys
+from os import path
+#from EVSVesuvio.vesuvio_analysis.core_functions.bootstrap import runBootstrap
+#from EVSVesuvio.vesuvio_analysis.core_functions.bootstrap_analysis import runAnalysisOfStoredBootstrap
+from EVSVesuvio.vesuvio_analysis.core_functions.run_script import runScript
+from mantid.api import AnalysisDataService
+from EVSVesuvio.scripts import handle_config
+
+
+def run():
+    scriptName =  handle_config.read_config_var('caching.experiment')
+    experimentsPath = Path(handle_config.read_config_var('caching.location')) / "experiments" / scriptName # Path to the repository
+    inputs_path = experimentsPath / "analysis_inputs.py"
+    ai = import_from_path(inputs_path, "analysis_inputs")
+
+    ipFilesPath = Path(path.dirname(path.dirname(handle_config.__file__))) / "vesuvio_analysis" / "ip_files"
+
+    start_time = time.time()
+
+    wsBackIC = ai.LoadVesuvioBackParameters(ipFilesPath)
+    wsFrontIC = ai.LoadVesuvioFrontParameters(ipFilesPath)
+    bckwdIC = ai.BackwardInitialConditions(ipFilesPath)
+    fwdIC = ai.ForwardInitialConditions
+    yFitIC = ai.YSpaceFitInitialConditions
+    bootIC = ai.BootstrapInitialConditions
+    userCtr = ai.UserScriptControls
+
+    runScript(userCtr, scriptName, wsBackIC, wsFrontIC, bckwdIC, fwdIC, yFitIC, bootIC)
+
+    AnalysisDataService.clear()
+    userCtr.procedure = "FORWARD"
+
+    runScript(userCtr, scriptName, wsBackIC, wsFrontIC, bckwdIC, fwdIC, yFitIC, bootIC)
+
+    end_time = time.time()
+    print("\nRunning time: ", end_time-start_time, " seconds")
+
+
+def import_from_path(path, name):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+if __name__ == '__main__':
+    run()

EVSVesuvio/config/Mantid.user.properties

@@ -0,0 +1,3 @@
+default.facility=ISIS
+default.instrument=Vesuvio
+datasearch.searcharchive=On

EVSVesuvio/original_inputs.py → EVSVesuvio/config/analysis_inputs.py

@@ -1,51 +1,46 @@
-import time
 import numpy as np
-from pathlib import Path
-from vesuvio_analysis.core_functions.bootstrap import runBootstrap
-from vesuvio_analysis.core_functions.bootstrap_analysis import runAnalysisOfStoredBootstrap
-from vesuvio_analysis.core_functions.run_script import runScript
-from mantid.api import AnalysisDataService
-
-scriptName =  Path(__file__).name.split(".")[0]  # Take out .py
-experimentPath = Path(__file__).absolute().parent / "experiments" / scriptName  # Path to the repository
-ipFilesPath = Path(__file__).absolute().parent / "vesuvio_analysis" / "ip_files"
 
 
 class LoadVesuvioBackParameters:
+    def __init__(self, ipFilesPath):
+        self.ipfile=ipFilesPath / "ip2019.par"
+
     runs="43066-43076"         # 77K         # The numbers of the runs to be analysed
     empty_runs="41876-41923"   # 77K         # The numbers of the empty runs to be subtracted
     spectra='3-134'                          # Spectra to be analysed
     mode='DoubleDifference'
-    ipfile=ipFilesPath / "ip2019.par"   
 
     subEmptyFromRaw = True         # Flag to control wether empty ws gets subtracted from raw
-    scaleEmpty = 1       # None or scaling factor 
+    scaleEmpty = 1       # None or scaling factor
     scaleRaw = 1
 
 
 class LoadVesuvioFrontParameters:
+    def __init__(self, ipFilesPath):
+        self.ipfile=ipFilesPath / "ip2018_3.par"
+
     runs='43066-43076'         # 100K        # The numbers of the runs to be analysed
     empty_runs='43868-43911'   # 100K        # The numbers of the empty runs to be subtracted
     spectra='144-182'                        # Spectra to be analysed
     mode='SingleDifference'
-    ipfile=ipFilesPath / "ip2018_3.par" 
 
     subEmptyFromRaw = False         # Flag to control wether empty ws gets subtracted from raw
-    scaleEmpty = 1       # None or scaling factor 
+    scaleEmpty = 1       # None or scaling factor
     scaleRaw = 1
 
 
 class GeneralInitialConditions:
     """Used to define initial conditions shared by both Back and Forward scattering"""
-    
+
     transmission_guess =  0.8537        # Experimental value from VesuvioTransmission
     multiple_scattering_order, number_of_events = 2, 1.e5
     # Sample slab parameters
     vertical_width, horizontal_width, thickness = 0.1, 0.1, 0.001  # Expressed in meters
 
 
 class BackwardInitialConditions(GeneralInitialConditions):
-    InstrParsPath = ipFilesPath / "ip2018_3.par" 
+    def __init__(self, ipFilesPath):
+        self.InstrParsPath=ipFilesPath / "ip2018_3.par"
 
     HToMassIdxRatio = 19.0620008206  # Set to zero or None when H is not present
     massIdx = 0
@@ -54,11 +49,11 @@ class BackwardInitialConditions(GeneralInitialConditions):
     masses = np.array([12, 16, 27])
     # noOfMasses = len(masses)
 
-    initPars = np.array([ 
-    # Intensities, NCP widths, NCP centers   
-            1, 12, 0.,    
-            1, 12, 0.,   
-            1, 12.5, 0.    
+    # Intensities, NCP widths, NCP centers
+    initPars = np.array([
+            1, 12, 0.,
+            1, 12, 0.,
+            1, 12.5, 0.
         ])
     bounds = np.array([
             [0, np.nan], [8, 16], [-3, 1],
@@ -80,22 +75,22 @@ class BackwardInitialConditions(GeneralInitialConditions):
     # # Parameters of workspaces in input_ws
     tofBinning='275.,1.,420'                    # Binning of ToF spectra
     maskTOFRange = None
-    
+
     # Original data uses histogram data instead of point data
     runHistData = True
     normVoigt = False
 
 
 class ForwardInitialConditions(GeneralInitialConditions):
 
-    masses = np.array([1.0079, 12, 16, 27]) 
+    masses = np.array([1.0079, 12, 16, 27])
 
-    initPars = np.array([ 
-    # Intensities, NCP widths, NCP centers  
-        1, 4.7, 0, 
-        1, 12.71, 0.,    
-        1, 8.76, 0.,   
-        1, 13.897, 0.    
+    # Intensities, NCP widths, NCP centers
+    initPars = np.array([
+        1, 4.7, 0,
+        1, 12.71, 0.,
+        1, 8.76, 0.,
+        1, 13.897, 0.
     ])
     bounds = np.array([
         [0, np.nan], [3, 6], [-3, 1],
@@ -117,7 +112,7 @@ class ForwardInitialConditions(GeneralInitialConditions):
 
     tofBinning="110,1,430"                 # Binning of ToF spectra
     maskTOFRange = None
-  
+
     # Original data uses histogram data instead of point data
     runHistData = True
     normVoigt = False
@@ -129,7 +124,7 @@ class YSpaceFitInitialConditions:
 
 class UserScriptControls:
     runRoutine = True
-    
+
     # Choose main procedure to run
     procedure = "BACKWARD"   # Options: None, "BACKWARD", "FORWARD", "JOINT"
 
@@ -139,25 +134,3 @@ class UserScriptControls:
 
 class BootstrapInitialConditions:
     runBootstrap = False
-
-
-start_time = time.time()
-
-wsBackIC = LoadVesuvioBackParameters
-wsFrontIC = LoadVesuvioFrontParameters  
-bckwdIC = BackwardInitialConditions
-fwdIC = ForwardInitialConditions
-yFitIC = YSpaceFitInitialConditions
-bootIC = BootstrapInitialConditions
-userCtr = UserScriptControls
-
-runScript(userCtr, scriptName, wsBackIC, wsFrontIC, bckwdIC, fwdIC, yFitIC, bootIC)
-
-AnalysisDataService.clear()
-userCtr.procedure = "FORWARD"
-
-runScript(userCtr, scriptName, wsBackIC, wsFrontIC, bckwdIC, fwdIC, yFitIC, bootIC)
-
-
-end_time = time.time()
-print("\nRunning time: ", end_time-start_time, " seconds")

EVSVesuvio/config/vesuvio.user.properties

@@ -0,0 +1,3 @@
+# properties that suit your particular installation.
+caching.location =
+caching.experiment =

EVSVesuvio/scripts/__init__.py

@@ -1,9 +1,53 @@
-"""Package defining top-level application
-and entry points.
+"""Package defining entry points.
 """
+import argparse
+from os import environ, path
+from EVSVesuvio.scripts import handle_config
 
 
 def main():
-    """Placeholder.
-    """
-    print("TEST_VESUVIO_LAUNCH")
+    parser = __set_up_parser()
+    args = parser.parse_args()
+    if args.command == "config":
+        __setup_config(args)
+
+    if args.command == "run":
+        if not handle_config.config_set():
+            __setup_config(None)
+        __run_analysis()
+
+
+def __set_up_parser():
+    parser = argparse.ArgumentParser(description="Package to analyse Vesuvio instrument data")
+    subparsers = parser.add_subparsers(dest='command', required=True)
+    config_parser = subparsers.add_parser("config", help="set mvesuvio configuration")
+    config_parser.add_argument("--set-cache", "-c", help="set the cache directory", default="", type=str)
+    config_parser.add_argument("--set-experiment", "-e", help="set the current experiment", default="", type=str)
+
+    config_parser = subparsers.add_parser("run", help="run mvesuvio analysis")
+    return parser
+
+
+def __setup_config(args):
+    config_dir = handle_config.VESUVIO_CONFIG_PATH
+    handle_config.setup_config_dir(config_dir)
+
+    if handle_config.config_set():
+        cache_dir = handle_config.read_config_var('caching.location') if not args or not args.set_cache else args.set_cache
+        experiment = handle_config.read_config_var('caching.experiment') if not args or not args.set_experiment else args.set_experiment
+    else:
+        cache_dir = config_dir if not args or not args.set_cache else args.set_cache
+        experiment = "default" if not args or not args.set_experiment else args.set_experiment
+    handle_config.set_config_vars({'caching.location': cache_dir,
+                                   'caching.experiment': experiment})
+    handle_config.setup_expr_dir(cache_dir, experiment)
+
+
+def __run_analysis():
+    environ['MANTIDPROPERTIES'] = path.join(handle_config.VESUVIO_CONFIG_PATH, "Mantid.user.properties")
+    from EVSVesuvio import analysis_runner
+    analysis_runner.run()
+
+
+if __name__ == '__main__':
+    main()

EVSVesuvio/scripts/handle_config.py

@@ -0,0 +1,90 @@
+import os
+from shutil import copyfile
+
+VESUVIO_CONFIG_PATH = os.path.join(os.path.expanduser("~"), '.mvesuvio')
+VESUVIO_CONFIG_FILE = "vesuvio.user.properties"
+VESUVIO_INPUTS_FILE = "analysis_inputs.py"
+VESUVIO_PACKAGE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+MANTID_CONFIG_FILE = "Mantid.user.properties"
+
+
+def __read_config(config_file_path, throw_on_not_found=True):
+    lines = ""
+    try:
+        with open(config_file_path, 'r') as file:
+            lines = file.readlines()
+    except IOError:
+        if throw_on_not_found:
+            raise RuntimeError(f"Could not read from vesuvio config file: {config_file_path}")
+    return lines
+
+
+def set_config_vars(var_dict):
+    file_path = os.path.join(VESUVIO_CONFIG_PATH, VESUVIO_CONFIG_FILE)
+    lines = __read_config(file_path)
+
+    updated_lines = []
+    for line in lines:
+        match = False
+        for var in var_dict:
+            if line.startswith(var):
+                new_line = f'{var}={var_dict[var]}'
+                updated_lines.append(f'{new_line}\n')
+                match = True
+                print(f'Setting: {new_line}')
+                break
+
+        if not match:
+            updated_lines.append(line)
+
+    with open(file_path, 'w') as file:
+        file.writelines(updated_lines)
+
+
+def read_config_var(var, throw_on_not_found=True):
+    file_path = f'{VESUVIO_CONFIG_PATH}/{VESUVIO_CONFIG_FILE}'
+    lines = __read_config(file_path, throw_on_not_found)
+
+    result = ""
+    for line in lines:
+        if line.startswith(var):
+            result = line.split("=", 2)[1].strip('\n')
+            break
+    if not result and throw_on_not_found:
+        raise ValueError(f'{var} was not found in the vesuvio config')
+    return result
+
+
+def setup_config_dir(config_dir):
+    success = __mk_dir('config', config_dir)
+    if success:
+        copyfile(os.path.join(VESUVIO_PACKAGE_PATH, "config", VESUVIO_CONFIG_FILE), os.path.join(config_dir, VESUVIO_CONFIG_FILE))
+        copyfile(os.path.join(VESUVIO_PACKAGE_PATH, "config", MANTID_CONFIG_FILE), os.path.join(config_dir, MANTID_CONFIG_FILE))
+
+
+def setup_expr_dir(cache_dir, experiment):
+    expr_path = os.path.join(cache_dir, "experiments", experiment)
+    __mk_dir('experiment', expr_path)
+    copyfile(os.path.join(VESUVIO_PACKAGE_PATH, "config", VESUVIO_INPUTS_FILE), input_file_path(cache_dir, experiment))
+
+
+def __mk_dir(type, path):
+    success = False
+    if not os.path.isdir(path):
+        try:
+            os.makedirs(path)
+            success = True
+        except:
+            print(f'Unable to make {type} directory at location: {path}')
+    return success
+
+
+def config_set():
+    if(read_config_var('caching.location', False)):
+        return True
+    else:
+        return False
+
+
+def input_file_path(cache_dir, experiment):
+    return os.path.join(cache_dir, "experiments", experiment, VESUVIO_INPUTS_FILE)

EVSVesuvio/vesuvio_analysis/core_functions/ICHelpers.py

@@ -1,8 +1,8 @@
 from mantid.simpleapi import LoadVesuvio, SaveNexus
 from pathlib import Path
+from EVSVesuvio.scripts import handle_config
 
-currentPath = Path(__file__).absolute().parent
-experimentsPath = currentPath / ".."/ ".." / "experiments"
+experimentsPath = Path(handle_config.read_config_var('caching.location')) / "experiments"
 
 
 def completeICFromInputs(IC, scriptName, wsIC):
@@ -163,7 +163,6 @@ def setBootstrapDirs(bckwdIC, fwdIC, bootIC, yFitIC):
 
     # Select script name and experiments path
     sampleName = bckwdIC.scriptName   # Name of sample currently running
-    experimentsPath = currentPath/".."/".."/"experiments"
 
     # Used to store running times required to estimate Bootstrap total run time.
     bootIC.runTimesPath = experimentsPath / sampleName / "running_times.txt"

EVSVesuvio/vesuvio_analysis/core_functions/bootstrap_analysis.py

@@ -4,13 +4,8 @@
 from EVSVesuvio.vesuvio_analysis.core_functions.fit_in_yspace import selectModelAndPars
 import numpy as np
 import matplotlib .pyplot as plt
-from pathlib import Path
 from scipy import stats
 
-currentPath = Path(__file__).parent.absolute()
-experimentsPath = currentPath / ".." / ".. " / "experiments"
-IPFilesPath = currentPath / ".." / "ip_files"
-
 
 def runAnalysisOfStoredBootstrap(bckwdIC, fwdIC, yFitIC, bootIC, analysisIC, userCtr):