From 87f29531c81091dd796bacd8533997b2826561fc Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Thu, 24 Jan 2013 11:29:41 -0600 Subject: [PATCH] BUG: Need to propogate FS failures from sub-scripts. --- AutoWorkup/ShortWorkupT1T2.py | 1 + AutoWorkup/WorkupT1T2.py | 21 +++--- AutoWorkup/WorkupT1T2FreeSurfer_custom.py | 6 +- AutoWorkup/baw_exp.py | 19 ++--- AutoWorkup/fsscript.py | 84 ++++++++--------------- 5 files changed, 56 insertions(+), 75 deletions(-) diff --git a/AutoWorkup/ShortWorkupT1T2.py b/AutoWorkup/ShortWorkupT1T2.py index c77e9028..77a5046a 100644 --- a/AutoWorkup/ShortWorkupT1T2.py +++ b/AutoWorkup/ShortWorkupT1T2.py @@ -290,6 +290,7 @@ def AccumulateLikeTissuePosteriors(posteriorImages): def ShortWorkupT1T2(subjectid, mountPrefix, ExperimentBaseDirectoryCache, ExperimentBaseDirectoryResults, ExperimentDatabase, atlas_fname_wpath, BCD_model_path, + GLOBAL_DATA_SINK_REWRITE, InterpolationMode="Linear", Mode=10, DwiList=[], WORKFLOW_COMPONENTS=[], CLUSTER_QUEUE='', CLUSTER_QUEUE_LONG=''): """ Run autoworkup on all subjects data defined in the ExperimentDatabase diff --git a/AutoWorkup/WorkupT1T2.py b/AutoWorkup/WorkupT1T2.py index 8d475579..4f787deb 100644 --- a/AutoWorkup/WorkupT1T2.py +++ b/AutoWorkup/WorkupT1T2.py @@ -300,7 +300,7 @@ def MakeNewAtlasTemplate(t1_image, deformed_list, ########################################################################### ########################################################################### def WorkupT1T2(subjectid, mountPrefix, ExperimentBaseDirectoryCache, ExperimentBaseDirectoryResults, ExperimentDatabase, atlas_fname_wpath, BCD_model_path, - InterpolationMode="Linear", Mode=10, DwiList=[], WORKFLOW_COMPONENTS=[], CLUSTER_QUEUE='', CLUSTER_QUEUE_LONG='', SGE_JOB_SCRIPT='#!/bin/bash'): + GLOBAL_DATA_SINK_REWRITE, InterpolationMode="Linear", Mode=10, DwiList=[], WORKFLOW_COMPONENTS=[], CLUSTER_QUEUE='', CLUSTER_QUEUE_LONG='', SGE_JOB_SCRIPT='#!/bin/bash'): """ Run autoworkup on all subjects data defined in the ExperimentDatabase @@ -1079,9 +1079,10 @@ def WorkupT1T2(subjectid, mountPrefix, ExperimentBaseDirectoryCache, ExperimentB FS_DS[sessionid].inputs.regexp_substitutions = [ ('/_uid_(?P[^/]*)', r'/\g') ] - baw200.connect(myLocalFSWF[sessionid], 'outputspec.FreeSurferOutputDirectory', FS_DS[sessionid], 'FREESURFER_SUBJ.@FreeSurferOutputDirectory') + baw200.connect(myLocalFSWF[sessionid], 'outputspec.FreeSurferOutputDirectory', FS_DS[sessionid], 'FREESURFER52_SUBJECTS.@FreeSurferOutputDirectory') ### Now define where the final organized outputs should go. - FSPREP_DataSink[sessionid] = pe.Node(nio.DataSink(), name="FREESURFER_PREP_" + str(subjectid) + "_" + str(sessionid)) + FSPREP_DataSink[sessionid] = pe.Node(nio.DataSink(), name="FREESURFER_PREP_DS_" + str(subjectid) + "_" + str(sessionid)) + FSPREP_DataSink[sessionid].overwrite = GLOBAL_DATA_SINK_REWRITE FSPREP_DataSink[sessionid].inputs.base_directory = ExperimentBaseDirectoryResults FREESURFER_PREP_PATTERNS = GenerateOutputPattern(projectid, subjectid, sessionid, 'FREESURFER_PREP') FSPREP_DataSink[sessionid].inputs.regexp_substitutions = FREESURFER_PREP_PATTERNS @@ -1095,7 +1096,8 @@ def WorkupT1T2(subjectid, mountPrefix, ExperimentBaseDirectoryCache, ExperimentB baw200.connect(myLocalFSWF[sessionid], 'outputspec.cnr_optimal_image', FSPREP_DataSink[sessionid], 'FREESURFER_PREP.@cnr_optimal_image') #} end of "for sessionid in allSessions:" - #{ Do longitudinal + #{ Do template building + """ ##HACK : Move later FS_TEMPLATE_oneSubjWorkflow = CreateFreeSurferSubjectTemplate(projectid, subjectid, @@ -1115,12 +1117,12 @@ def WorkupT1T2(subjectid, mountPrefix, ExperimentBaseDirectoryCache, ExperimentB baw200.connect(FreeSurferSessionID_MergeNode[subjectid],'out', FS_TEMPLATE_oneSubjWorkflow, 'inputspec.FreeSurferSession_IDs') FSTEMP_DataSink[subjectid] = pe.Node(nio.DataSink(), name='FREESURFER_TEMP_' + str(subjectid)) + FSTEMP_DataSink[subjectid].overwrite = GLOBAL_DATA_SINK_REWRITE FREESURFER_TEMP_PATTERNS = GenerateOutputPattern(projectid, subjectid, 'FREESURFER_TEMP', '') FSTEMP_DataSink[subjectid].inputs.regexp_substitutions = FREESURFER_TEMP_PATTERNS - FSTEMP_DataSink[subjectid].overwrite = GLOBAL_DATA_SINK_REWRITE - baw200.connect(FS_TEMPLATE_oneSubjWorkflow, 'outputspec.FreeSurferTemplateDir', FSTEMP_DataSink[subjectid], 'FREESURFER_SUBJ.@FreeSurferTemplateDir') + baw200.connect(FS_TEMPLATE_oneSubjWorkflow, 'outputspec.FreeSurferTemplateDir', FSTEMP_DataSink[subjectid], 'FREESURFER52_SUBJECTS.@FreeSurferTemplateDir') #} - #{ + #{ Do longitudinal analysis FSLONG_DataSink = dict() FS_LONG_oneSubjWorkflow = dict() for sessionid in allSessions: @@ -1136,14 +1138,15 @@ def WorkupT1T2(subjectid, mountPrefix, ExperimentBaseDirectoryCache, ExperimentB baw200.connect(FS_TEMPLATE_oneSubjWorkflow, 'outputspec.FreeSurferTemplateDir', FS_LONG_oneSubjWorkflow[sessionid], 'inputspec.SingleSubject_ID') # baw200.connect(FREESURFER_SUBJ_ID, 'subjectTemplate_id', FS_LONG_oneSubjWorkflow[sessionid], 'inputspec.SingleSubject_ID') FSLONG_DataSink[sessionid] = pe.Node(nio.DataSink(), name='_'.join(['FREESURFER_LONG', str(subjectid), str(sessionid)])) + FSLONG_DataSink[sessionid].overwrite = GLOBAL_DATA_SINK_REWRITE FSLONG_DataSink[sessionid].inputs.base_directory = ExperimentBaseDirectoryResults FREESURFER_LONG_PATTERNS = GenerateOutputPattern(projectid, subjectid, sessionid, 'FREESURFER_LONG') FSLONG_DataSink[sessionid].inputs.regexp_substitutions = FREESURFER_LONG_PATTERNS - FSLONG_DataSink[sessionid].overwrite = GLOBAL_DATA_SINK_REWRITE - baw200.connect(FS_LONG_oneSubjWorkflow[sessionid], 'outputspec.FreeSurferLongitudinalDir', FSLONG_DataSink[sessionid], 'FREESURFER_SUBJ.@longitudinalDirs') + baw200.connect(FS_LONG_oneSubjWorkflow[sessionid], 'outputspec.FreeSurferLongitudinalDir', FSLONG_DataSink[sessionid], 'FREESURFER52_SUBJECTS.@longitudinalDirs') #} end of "for sessionid in allSessions:" + """ else: print "Skipping freesurfer" return baw200 diff --git a/AutoWorkup/WorkupT1T2FreeSurfer_custom.py b/AutoWorkup/WorkupT1T2FreeSurfer_custom.py index 1d40a16d..ef5b0ce0 100644 --- a/AutoWorkup/WorkupT1T2FreeSurfer_custom.py +++ b/AutoWorkup/WorkupT1T2FreeSurfer_custom.py @@ -68,7 +68,7 @@ def CreateFreeSurferWorkflow_custom(projectid, subjectid, sessionid, WFname, CLU freesurfer_sge_options_dictionary = {'qsub_args': '-S /bin/bash -pe smp1 1 -l h_vmem=18G,mem_free=8G -o /dev/null -e /dev/null ' + CLUSTER_QUEUE, 'overwrite': True} if RunAllFSComponents == True: print("""Run FreeSurfer ReconAll at""") - fs_reconall = pe.Node(interface=fswrap.FSScript(), name="FS52_custom") + fs_reconall = pe.Node(interface=fswrap.FSScript(), name="FS52_cross_"+str(sessionid)) fs_reconall.plugin_args = freesurfer_sge_options_dictionary fs_reconall.inputs.subcommand = 'autorecon' # fs_reconall.inputs.directive = 'all' @@ -108,7 +108,7 @@ def CreateFreeSurferSubjectTemplate(projectid, subjectid, session_ids, WFname, C os.environ['SUBJECTS_DIR'] = constructed_FS_SUBJECTS_DIR inputsSpec.inputs.subjects_dir = constructed_FS_SUBJECTS_DIR # HACK print("""Run FreeSurfer Within Subject Template at""") - fs_template = pe.Node(interface=fswrap.FSScript(), name="FS55_template_build_"+str(subjectid)) + fs_template = pe.Node(interface=fswrap.FSScript(), name="FS52_base_"+str(subjectid)) fs_template.plugin_args = freesurfer_sge_options_dictionary fs_template.inputs.session_ids = session_ids fs_template.inputs.subcommand = 'template' @@ -133,7 +133,7 @@ def CreateFreeSurferLongitudinalWorkflow(projectid, sessionid, subjectid, WFname os.environ['SUBJECTS_DIR'] = constructed_FS_SUBJECTS_DIR inputsSpec.inputs.subjects_dir = constructed_FS_SUBJECTS_DIR # HACK - fs_longitudinal = pe.Node(interface=fswrap.FSScript(), name="FS55_longitudinal_"+str(sessionid)) + fs_longitudinal = pe.Node(interface=fswrap.FSScript(), name="FS52_long_"+str(sessionid)) fs_longitudinal.plugin_args = freesurfer_sge_options_dictionary fs_longitudinal.inputs.subcommand = 'longitudinal' fs_longitudinal.inputs.session_id = sessionid diff --git a/AutoWorkup/baw_exp.py b/AutoWorkup/baw_exp.py index 02c990e5..ef609899 100644 --- a/AutoWorkup/baw_exp.py +++ b/AutoWorkup/baw_exp.py @@ -15,8 +15,6 @@ import sys ############################################################################## -GLOBAL_DATA_SINK_REWRITE = True - def get_global_sge_script(pythonPathsList, binPathsList, customEnvironment={}): """This is a wrapper script for running commands on an SGE cluster @@ -96,17 +94,18 @@ def setGlobalDatasinkRewrite(cli, cfg): :param cfg: configuration file value :type cfg: bool - Sets the global variable `GLOBAL_DATA_SINK_REWRITE` constant flag used in :mod:`WorkupT1T2()` + Sets the variable `GLOBAL_DATA_SINK_REWRITE` constant flag used in :mod:`WorkupT1T2()` """ assert isinstance(cli, bool) and isinstance(cfg, bool), \ "Inputs are not boolean: {0}, {1}".format(cli, cfg) - global GLOBAL_DATA_SINK_REWRITE + GLOBAL_DATA_SINK_REWRITE=False if cli or cfg: print "*** Ignoring datasinks for pipeline rewriting ***" # TODO: Use logging GLOBAL_DATA_SINK_REWRITE = False else: GLOBAL_DATA_SINK_REWRITE = True + return GLOBAL_DATA_SINK_REWRITE def main(argv=None): @@ -140,8 +139,8 @@ def main(argv=None): expConfig.read(input_arguments.ExperimentConfig) # Pipeline-specific information - ignore_datasinks = expConfig.getboolean('PIPELINE', 'IGNORE_DATASINKS') - setGlobalDatasinkRewrite(input_arguments.ignore_datasinks, ignore_datasinks) + ignore_datasinks = expConfig.getboolean('PIPELINE', 'GLOBAL_DATA_SINK_REWRITE') + GLOBAL_DATA_SINK_REWRITE=setGlobalDatasinkRewrite(input_arguments.ignore_datasinks, ignore_datasinks) # Experiment specific information subject_data_file = expConfig.get('EXPERIMENT_DATA', 'SESSION_DB') @@ -303,14 +302,18 @@ def main(argv=None): ExperimentBaseDirectoryResults, ExperimentDatabase, CACHE_ATLASPATH, - CACHE_BCDMODELPATH, WORKFLOW_COMPONENTS=WORKFLOW_COMPONENTS, CLUSTER_QUEUE=CLUSTER_QUEUE, CLUSTER_QUEUE_LONG=CLUSTER_QUEUE_LONG) + CACHE_BCDMODELPATH, + GLOBAL_DATA_SINK_REWRITE, + WORKFLOW_COMPONENTS=WORKFLOW_COMPONENTS, CLUSTER_QUEUE=CLUSTER_QUEUE, CLUSTER_QUEUE_LONG=CLUSTER_QUEUE_LONG) else: baw200 = WorkupT1T2.WorkupT1T2(subjectid, mountPrefix, os.path.join(ExperimentBaseDirectoryCache, str(subjectid)), ExperimentBaseDirectoryResults, ExperimentDatabase, CACHE_ATLASPATH, - CACHE_BCDMODELPATH, WORKFLOW_COMPONENTS=WORKFLOW_COMPONENTS, CLUSTER_QUEUE=CLUSTER_QUEUE, CLUSTER_QUEUE_LONG=CLUSTER_QUEUE_LONG, SGE_JOB_SCRIPT=JOB_SCRIPT) + CACHE_BCDMODELPATH, + GLOBAL_DATA_SINK_REWRITE, + WORKFLOW_COMPONENTS=WORKFLOW_COMPONENTS, CLUSTER_QUEUE=CLUSTER_QUEUE, CLUSTER_QUEUE_LONG=CLUSTER_QUEUE_LONG, SGE_JOB_SCRIPT=JOB_SCRIPT) print "Start Processing" SGEFlavor = 'SGE' diff --git a/AutoWorkup/fsscript.py b/AutoWorkup/fsscript.py index f9f7117a..f2f04df2 100755 --- a/AutoWorkup/fsscript.py +++ b/AutoWorkup/fsscript.py @@ -44,7 +44,10 @@ def run_mri_convert_script(niftiVol, mgzVol, subjects_dir, FREESURFER_HOME, FS_S export FREESURFER_HOME={FSHOME} export SUBJECTS_DIR={FSSUBJDIR} source {SOURCE_SCRIPT} -{FSHOME}/bin/mri_convert --conform --out_data_type uchar {invol} {outvol}""".format(SOURCE_SCRIPT=FS_SCRIPT_FN, +{FSHOME}/bin/mri_convert --conform --out_data_type uchar {invol} {outvol} +status=$? +exit $status +""".format(SOURCE_SCRIPT=FS_SCRIPT_FN, FSHOME=FREESURFER_HOME, FSSUBJDIR=subjects_dir, invol=niftiVol, @@ -57,7 +60,9 @@ def run_mri_convert_script(niftiVol, mgzVol, subjects_dir, FREESURFER_HOME, FS_S script_name_stdout = mgzVol + '_convert.out' script_name_stdout_fid = open(script_name_stdout, 'w') print "Starting mri_convert" - subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + scriptStatus = subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + if scriptStatus != 0: + sys.exit(scriptStatus) print "Ending mri_convert" script_name_stdout_fid.close() return @@ -71,6 +76,8 @@ def run_mri_mask_script(output_brainmask_fn_mgz, output_custom_brainmask_fn_mgz, source {SOURCE_SCRIPT} {FSHOME}/bin/mri_add_xform_to_header -c {CURRENT}/transforms/talairach.xfm {maskvol} {maskvol} {FSHOME}/bin/mri_mask {invol} {maskvol} {outvol} +status=$? +exit $status """.format(SOURCE_SCRIPT=FS_SCRIPT_FN, FSHOME=FREESURFER_HOME, FSSUBJDIR=subjects_dir, @@ -87,59 +94,13 @@ def run_mri_mask_script(output_brainmask_fn_mgz, output_custom_brainmask_fn_mgz, script_name_stdout = output_brainmask_fn_mgz + '_convert.out' script_name_stdout_fid = open(script_name_stdout, 'w') print "Starting mri_mask" - subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + scriptStatus = subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + if scriptStatus != 0: + sys.exit(scriptStatus) print "Ending mri_mask" script_name_stdout_fid.close() return -""" -def baw_Recon1(t1_fn, wm_fn, brainmask, subjects_dir, FREESURFER_HOME, FS_SCRIPT, subject_id): - base_subj_dir = os.path.join(subjects_dir, subject_id, 'mri') - output_brainmask_fn = os.path.join(base_subj_dir, 'brainmask.nii.gz') - output_nu_fn = os.path.join(base_subj_dir, 'nu.nii.gz') - output_brainmask_fn_mgz = os.path.join(base_subj_dir, 'brainmask.mgz') - output_nu_fn_mgz = os.path.join(base_subj_dir, 'nu.mgz') - if IsFirstNewerThanSecond(t1_fn, output_brainmask_fn_mgz): - print "PREPARING ALTERNATE recon-auto1 stage" - mkdir_p(base_subj_dir) - t1 = sitk.ReadImage(t1_fn) - wm = sitk.ReadImage(wm_fn) - t1_new = sitk.Cast(normalizeWM(t1, wm), sitk.sitkUInt8) - - orig_001_mgz_fn=os.path.join(base_subj_dir, 'orig/001.mgz') - sitk.WriteImage(t1_new,orig_001_mgz_fn) - rawavg_mgz_fn=os.path.join(base_subj_dir, 'rawavg.mgz') - sitk.WriteImage(t1_new,rawavg_mgz_fn) - orig_mgz_fn=os.path.join(base_subj_dir, 'orig.mgz') - sitk.WriteImage(t1_new,orig_mgz_fn) - make_dummy_file( os.path.join(base_subj_dir, 'transforms/talairach.auto.xfm')) - make_dummy_file( os.path.join(base_subj_dir, 'transforms/talairach.xfm')) - - sitk.WriteImage(t1_new, output_nu_fn) - - t1_mgz=os.path.join(base_subj_dir, 'T1.mgz') - sitk.WriteImage(t1_new, t1_mgz) - run_mri_convert_script(output_nu_fn, output_nu_fn_mgz, subjects_dir, FREESURFER_HOME, FS_SCRIPT) - - make_dummy_file(os.path.join( base_subj_dir, 'transforms/talairach_with_skull.lta')) - make_dummy_file(os.path.join(base_subj_dir, 'brainmask.auto.mgz')) - brain = sitk.ReadImage(brain_fn) - blood = sitk.BinaryThreshold(brain, 5, 5) - not_blood = 1 - blood - clipping = sitk.BinaryThreshold(brain, 1, 1000000) - blood - fill_size = 2 - ## HACK: Unfortunately we need to hole fill because of a WM bug in BABC where - ## some white matter is being classified as background when it it being avoid due - ## to too strict of multi-modal thresholding. - hole_filled = sitk.ErodeObjectMorphology(sitk.DilateObjectMorphology(clipping, fill_size), fill_size) - clipped = sitk.Cast(t1_new * hole_filled * not_blood, sitk.sitkUInt8) - sitk.WriteImage(clipped, output_brainmask_fn) # brain_matter image with values normalized 0-110, no skull or surface blood - run_mri_convert_script(output_brainmask_fn, output_brainmask_fn_mgz, subjects_dir, FREESURFER_HOME, FS_SCRIPT) - else: - print "NOTHING TO BE DONE, SO SKIPPING." - return # Nothing to be done, files are already up-to-date. -""" - def baw_FixBrainMask(brainmask, subjects_dir, FREESURFER_HOME, FS_SCRIPT, subject_id): base_subj_dir = os.path.join(subjects_dir, subject_id, 'mri') mkdir_p(base_subj_dir) @@ -191,6 +152,8 @@ def runAutoReconStage(subject_id, StageToRun, t1_fn, subjects_dir, FREESURFER_HO export SUBJECTS_DIR={FSSUBJDIR} source {SOURCE_SCRIPT} {FSHOME}/bin/recon-all -debug -subjid {SUBJID} -make autorecon{AUTORECONSTAGE} +status=$? +exit $status """.format(SOURCE_SCRIPT=FS_SCRIPT_FN, FSHOME=FREESURFER_HOME, FSSUBJDIR=subjects_dir, @@ -206,7 +169,9 @@ def runAutoReconStage(subject_id, StageToRun, t1_fn, subjects_dir, FREESURFER_HO script_name_stdout = script_name + '_out' script_name_stdout_fid = open(script_name_stdout, 'w') print "Starting auto_recon Stage: {0} for SubjectSession {1}".format(StageToRun, subject_id) - subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + scriptStatus = subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + if scriptStatus != 0: + sys.exit(scriptStatus) print "Ending auto_recon Stage: {0} for SubjectSession {1}".format(StageToRun, subject_id) script_name_stdout_fid.close() return @@ -229,7 +194,10 @@ def runSubjectTemplate(args, FREESURFER_HOME, FS_SCRIPT): export FREESURFER_HOME={FSHOME} export SUBJECTS_DIR={FSSUBJDIR} source {SOURCE_SCRIPT} -{FSHOME}/bin/recon-all -debug -base {TEMPLATEID} """.format(SOURCE_SCRIPT=FS_SCRIPT_FN, +{FSHOME}/bin/recon-all -debug -base {TEMPLATEID} +status=$? +exit $status +""".format(SOURCE_SCRIPT=FS_SCRIPT_FN, FSHOME=FREESURFER_HOME, FSSUBJDIR=subjects_dir, TEMPLATEID=subjectTemplate_id) @@ -246,7 +214,9 @@ def runSubjectTemplate(args, FREESURFER_HOME, FS_SCRIPT): script_name_stdout = script_name + '_out' script_name_stdout_fid = open(script_name_stdout, 'w') print "Starting auto_recon Stage: {0} for SubjectSession {1}".format(StageToRun, subjectTemplate_id) - subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + scriptStatus = subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + if scriptStatus != 0: + sys.exit(scriptStatus) print "Ending auto_recon Stage: {0} for SubjectSession {1}".format(StageToRun, subjectTemplate_id) script_name_stdout_fid.close() return @@ -265,7 +235,9 @@ def runLongitudinal(args, FREESURFER_HOME, FS_SCRIPT): export SUBJECTS_DIR={FSSUBJDIR} source {SOURCE_SCRIPT} {FSHOME}/bin/recon-all -debug -long {TIMEPOINT} {TEMPLATEID} -all +status=$? mv -n {FSSUBJDIR}/{TIMEPOINT}.long.{TEMPLATEID} {FSSUBJDIR}/{TEMPLATEID}_{TIMEPOINT}.long +exit $status """.format(SOURCE_SCRIPT=FS_SCRIPT_FN, FSHOME=FREESURFER_HOME, FSSUBJDIR=subjects_dir, @@ -281,7 +253,9 @@ def runLongitudinal(args, FREESURFER_HOME, FS_SCRIPT): script_name_stdout = script_name + '_out' script_name_stdout_fid = open(script_name_stdout, 'w') print "Starting auto_recon Stage: {0} for SubjectSession {1}".format(StageToRun, template_id) - subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + scriptStatus = subprocess.check_call([script_name], stdout=script_name_stdout_fid, stderr=subprocess.STDOUT, shell='/bin/bash') + if scriptStatus != 0: + sys.exit(scriptStatus) print "Ending auto_recon Stage: {0} for SubjectSession {1}".format(StageToRun, template_id) script_name_stdout_fid.close() return