From 68c261fbc34714f925182d73c6959bb1345c774c Mon Sep 17 00:00:00 2001
From: sebastian <sebmuell@phys.ethz.ch>
Date: Fri, 20 Oct 2017 20:31:26 +0200
Subject: [PATCH] reconstruct timeseries and compare feature extraction.

---
 ...ss4_compare_time_series_reconstruction.xml | 200 ++++++++++++++++++
 1 file changed, 200 insertions(+)
 create mode 100644 photon_stream/production/resources/observations_pass4_compare_time_series_reconstruction.xml

diff --git a/photon_stream/production/resources/observations_pass4_compare_time_series_reconstruction.xml b/photon_stream/production/resources/observations_pass4_compare_time_series_reconstruction.xml
new file mode 100644
index 0000000..453ac5b
--- /dev/null
+++ b/photon_stream/production/resources/observations_pass4_compare_time_series_reconstruction.xml
@@ -0,0 +1,200 @@
+<container>
+    <properties url="classpath:/default/settings.properties" />
+    <property name="infile" value="file:src/main/resources/testDataFile.fits.gz" />
+    <property name="drsfile" value="file:src/main/resources/testDrsFile.drs.fits.gz" />
+    <property name="integralGainFile" value="classpath:/default/gain_sorted_20131127.csv" />
+    <property name="pixelDelayFile" value="classpath:/default/delays_lightpulser_20150217.csv" />
+    <property name="out_path_basename" value="output" />
+    <property name="aux_dir" value="file:src/main/resources/aux/" />
+    <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${aux_dir}" />
+    <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+    <stream id="fact" class="fact.io.hdureader.FITSStream" url="${infile}"/>
+
+    <process id="2" input="fact">
+
+        <fact.utils.PreviousEventInfo
+            startCellKey="StartCellData"
+            outputKey="prevEvents"
+            limitEvents="${prevEvents_limitEvents}"
+        />
+
+        <stream.flow.Skip 
+            condition="%{data.TriggerType}!=1 AND %{data.TriggerType}!=2 AND %{data.TriggerType}!=4 AND %{data.TriggerType}!=1024"
+        /> 
+        <!--    FACT trigger types:
+                Design and commissioning of the trigger
+                electronics for a novel Geiger-mode avalanche
+                photodiode based camera for Imaging
+                Atmospheric Cherenkov Telescopes 
+
+                Dissertation Patrick Vogler, page 93
+
+                1: external 1 GPS (like pedestal)
+                2: external 2 GPS (like pedestal)
+                1024: pedestal
+                4: physics (self triggered)
+        -->
+
+        <include url="classpath:/default/data/calibration.xml" />
+        <include url="classpath:/default/data/extraction.xml" />
+
+        <fact.extraction.AboveThreshold 
+            dataKey="Data" 
+            threshold="2040" 
+            outputKey="Saturation"
+        />
+
+        <stream.flow.If condition="%{data.SaturationPixelCount} &lt; 100">
+            <fact.photonstream.SinglePulseExtraction
+                dataKey="DataCalibrated"
+                outputKey="PhotonArrivals"
+                maxIterations="500"
+            />
+        </stream.flow.If>
+        <stream.flow.If condition="%{data.SaturationPixelCount} &gt; 99">
+            <fact.photonstream.SinglePulseExtraction
+                dataKey="DataCalibrated"
+                outputKey="PhotonArrivals"
+                maxIterations="0"
+            />
+        </stream.flow.If>
+
+        <fact.features.source.SourcePosition
+            outputKey="sourcePosition"
+            auxService="auxService"
+        />
+
+        <fact.utils.ElementwiseMultiplyDoubleArray
+            inputKey="PhotonArrivalsBaseLine"
+            outputKey="CentiBaseLines"
+            factor="100.0"
+        />
+
+        <fact.utils.CastDoubleArrayToIntArray
+            inputKey="CentiBaseLines"
+            outputKey="CentiBaseLines"
+        />
+
+
+        <!-- From raw time-series -->
+        <!-- =============================================================== -->
+        <fact.cleaning.TwoLevelTimeNeighbor
+            calibService="calibService"
+            photonChargeKey="photoncharge"
+            outputKey="shower"
+            arrivalTimeKey="arrivalTime"
+            corePixelThreshold="${twoLevelTimeNeighbor_coreThreshold}"
+            neighborPixelThreshold="${twoLevelTimeNeighbor_neighborThreshold}"
+            timeLimit="${twoLevelTimeNeighbor_timeLimit}"
+            minNumberOfPixel="${twoLevelTimeNeighbor_minNumberOfPixel}"
+        />
+        <include url="classpath:/default/mc/pedestalParameters_mc.xml" />
+        <include url="classpath:/default/data/parameterCalc.xml" />
+        <include url="classpath:/default/data/sourceParameter.xml" />
+        <fact.io.JSONWriter keys="${keysForOutput}" 
+            url="${out_path_basename}_std.jsonl.gz" 
+            writeListOfItems="True" 
+            specialDoubleValuesAsString="True"
+            pixelSetsAsInt="True"
+            gzip="true"
+            jsonl="true"
+        />
+
+
+        <!-- From reconstructed time-series without baseline -->
+        <!-- =============================================================== -->
+        <fact.photonstream.ConvertSinglePulses2Timeseries
+                singlePulsesKey="PhotonArrivals"
+                timeSeriesKey="DataCalibrated"
+        />
+        <include url="classpath:/default/data/extraction.xml" />
+        <fact.cleaning.TwoLevelTimeNeighbor
+            calibService="calibService"
+            photonChargeKey="photoncharge"
+            outputKey="shower"
+            arrivalTimeKey="arrivalTime"
+            corePixelThreshold="${twoLevelTimeNeighbor_coreThreshold}"
+            neighborPixelThreshold="${twoLevelTimeNeighbor_neighborThreshold}"
+            timeLimit="${twoLevelTimeNeighbor_timeLimit}"
+            minNumberOfPixel="${twoLevelTimeNeighbor_minNumberOfPixel}"
+        />
+        <include url="classpath:/default/mc/pedestalParameters_mc.xml" />
+        <include url="classpath:/default/data/parameterCalc.xml" />
+        <include url="classpath:/default/data/sourceParameter.xml" />
+        <fact.io.JSONWriter keys="${keysForOutput}" 
+            url="${out_path_basename}_reco_no_baseline.jsonl.gz"
+            writeListOfItems="True" 
+            specialDoubleValuesAsString="True"
+            pixelSetsAsInt="True"
+            gzip="true"
+            jsonl="true"
+        />
+        <!-- tested in viewer, baseline of DataCalibrated is 0.0 -->
+
+
+        <!-- From reconstructed time-series with baseline -->
+        <!-- =============================================================== -->
+        <fact.photonstream.ConvertSinglePulses2Timeseries
+                singlePulsesKey="PhotonArrivals"
+                baseLineKey="PhotonArrivalsBaseLine"
+                timeSeriesKey="DataCalibrated"
+        />
+        <include url="classpath:/default/data/extraction.xml" />
+        <fact.cleaning.TwoLevelTimeNeighbor
+            calibService="calibService"
+            photonChargeKey="photoncharge"
+            outputKey="shower"
+            arrivalTimeKey="arrivalTime"
+            corePixelThreshold="${twoLevelTimeNeighbor_coreThreshold}"
+            neighborPixelThreshold="${twoLevelTimeNeighbor_neighborThreshold}"
+            timeLimit="${twoLevelTimeNeighbor_timeLimit}"
+            minNumberOfPixel="${twoLevelTimeNeighbor_minNumberOfPixel}"
+        />
+        <include url="classpath:/default/mc/pedestalParameters_mc.xml" />
+        <include url="classpath:/default/data/parameterCalc.xml" />
+        <include url="classpath:/default/data/sourceParameter.xml" />
+        <fact.io.JSONWriter keys="${keysForOutput}" 
+            url="${out_path_basename}_reco_with_baseline.jsonl.gz"
+            writeListOfItems="True" 
+            specialDoubleValuesAsString="True"
+            pixelSetsAsInt="True"
+            gzip="true"
+            jsonl="true"
+        />
+        <!-- tested in viewer, baseline of DataCalibrated is not 0.0 -->
+
+        <!-- output photon_stream -->
+        <!-- =============================================================== -->
+        <stream.data.RenameKey from="NIGHT" to="Night"/>
+        <stream.data.RenameKey from="RUNID" to="Run"/>
+        <stream.data.RenameKey from="EventNum" to="Event"/>
+        <stream.data.RenameKey from="UnixTimeUTC" to="UnixTime_s_us"/>
+        <stream.data.RenameKey from="TriggerType" to="Trigger"/>
+        <stream.data.RenameKey from="AzPointing" to="Az_deg"/>
+        <stream.data.RenameKey from="ZdPointing" to="Zd_deg"/>
+        <stream.data.RenameKey from="PhotonArrivals" to="PhotonArrivals_500ps"/>
+        <stream.data.RenameKey from="Saturation" to="SaturatedPixels"/>
+
+        <fact.io.JSONWriter 
+            keys="Night,Run,Event,UnixTime_s_us,Trigger,Az_deg,Zd_deg,PhotonArrivals_500ps,SaturatedPixels"
+            url="${out_path_basename}.phs.jsonl.gz" 
+            writeListOfItems="true"
+            pixelSetsAsInt="true"
+            gzip="true"
+            jsonl="true"
+         />
+
+        <fact.io.JSONWriter 
+            keys="Night,Run,Event,CentiBaseLines"
+            url="${out_path_basename}.bsl.jsonl.gz" 
+            writeListOfItems="true"
+            pixelSetsAsInt="true"
+            gzip="true"
+            jsonl="true"
+        />
+
+        <!-- view events -->
+        <!-- <fact.ShowViewer key="DataCalibrated" /> -->
+        <!-- stop the viewer using killall -9 java -->
+    </process>
+</container>