Merge pull request #142 from EA31337/dev

Fixes issues with EA/script running

.travis.yml

@@ -15,6 +15,7 @@ env:
     DOCKER_REPO=ea31337/$DOCKER_IMAGE:$DOCKER_TAG
     PATH=${PATH//:\.\/node_modules\/\.bin/} # Workaround for Travis issue #4862
     CWD="$TRAVIS_BUILD_DIR"
+    DOCKER_ARG="-e OPT_TRACE=$OPT_TRACE"
     DOCKER_TAG=$([[ "$TRAVIS_BRANCH" == "master" ]] && echo "latest" || echo ${TRAVIS_BRANCH////-})
     WINETRICKS_DOWNLOADER_RETRIES=20
   - secure: "waZbxfpNTIp/etY6lwR8GB+aSWDIw73tu6HxFdwVYdDJ/25SlY+gkJMCPqlpkjA1Aenv1I3TDDd/3dQIMYUeNbgIKOx6AtWkddnfTykqYYFJ4j6MlWPxZKaxDLjehLvjj0ZqkfXcswgXDaIoNnunKbLKiVohprLhO9Yg6I/j1LfzmPPGmFez2Mg2K0jFmri1i8prxpL4cctH0cR9NEZ+Lw13nDjY4b7AMoY6mzt6TFe9/yEt1OzYVGAh36hWgI1AL8h86cyDlUYHMRrzZjMKUGZFyOkkz3BRA6wvyOtjRhNlh2SIY+Y+HiwQaLONeQGpZ9YjNCUb+f5GIxXyEry9RV1oaMNIUnXNSQstUvgzy3QO2muD468KhfvPafpPj0gDHmfGEH+2IMcrz/WYNobsdN9JOjvPWutnxgYAzd27jwDIQ4D2WewZ3COgrRXxCAgwsOQKvlzDaFAOvPObOnCmIVsbOpONE3nSRGnrQpxJl0aaspoHLnVyKAUeCTAMhtSZirhWO+Iob9mhicB+9xEMRpxFnRD/FBUcSY/SyVlZjXxK9zIqn9t5KBK0iVZ10l/O5nV6sM7YfpAz8d4MqJDAGT05tKn/o2uDJtVvnl6szCIP5iQwsMUsQqQCuNPbmfm1V/q2ipjxLwiwOx2NRbSjFx3/5NnyXiIqehmvlfnfUjQ="
@@ -23,21 +24,21 @@ env:
   - CMD="docker-compose run RunHelp; scripts/eval.sh help; scripts/run_backtest.sh -?"
   - CMD="sudo scripts/provision.sh"
   - CMD="make docker-ci DOCKER_TAG=$DOCKER_TAG; [[ '$TRAVIS_PULL_REQUEST' = 'false' ]] && make docker-push DOCKER_TAG=$DOCKER_TAG || true"
-  - CMD="docker-compose run TestSyntaxVagrantfile; docker-compose run TestSyntaxBash; docker-compose run TestSyntaxShellcheck"
-  - CMD="docker-compose run RunCompileMql"
-  - CMD="docker-compose run ScriptPrintPaths"
-  - CMD="docker-compose run TestHSTHeader; docker-compose run TestFXTHeader"
-  - CMD="docker-compose run TestLotstep4"
+  - CMD="docker-compose run $DOCKER_ARG TestSyntaxVagrantfile; docker-compose run $DOCKER_ARG TestSyntaxBash; docker-compose run $DOCKER_ARG TestSyntaxShellcheck"
+  - CMD="docker-compose run $DOCKER_ARG RunCompileMql"
+  - CMD="docker-compose run $DOCKER_ARG ScriptPrintPaths"
+  - CMD="docker-compose run $DOCKER_ARG TestHSTHeader; docker-compose run TestFXTHeader"
+  - CMD="docker-compose run $DOCKER_ARG TestLotstep4"
 # @fixme: Error: Expected lot step for 5 digits: 0.01, found: 0.1.
-# - CMD="docker-compose run TestLotstep5"
-  - CMD="docker-compose run TestTimeframes"
-  - CMD="docker-compose run TestBands"
-  - CMD="docker-compose run TestEnvelopes"
-  - CMD="docker-compose run TestModellingQuality"
-  - CMD="docker-compose run RunInstallMt4"
-  - CMD="docker-compose run RunInstallMt4x"
-  - CMD="docker-compose run RunInstallMt5"
-  - CMD="docker-compose run DockerLargeFiles"
+# - CMD="docker-compose run $DOCKER_ARG TestLotstep5"
+  - CMD="docker-compose run $DOCKER_ARG TestTimeframes"
+  - CMD="docker-compose run $DOCKER_ARG TestBands"
+  - CMD="docker-compose run $DOCKER_ARG TestEnvelopes"
+  - CMD="docker-compose run $DOCKER_ARG TestModellingQuality"
+  - CMD="docker-compose run $DOCKER_ARG RunInstallMt4"
+  - CMD="docker-compose run $DOCKER_ARG RunInstallMt4x"
+  - CMD="docker-compose run $DOCKER_ARG RunInstallMt5"
+  - CMD="docker-compose run $DOCKER_ARG DockerLargeFiles"
 
 # Tests which suppose to be working in older versions only.
 # - TEST_ARGS="-v -t -d 1000 -p EURUSD -m 1-12 -y 2015 -S 10 -b DS -D5 -e TestFXTHeader -M4.0.0.971 -R"
@@ -80,21 +81,3 @@ cache:
   directories:
   - $HOME/.cache
   - $HOME/.docker
-#before_deploy:
-#- MT_VER=$("$CWD"/scripts/eval.sh filever terminal.exe)
-#- bash "$CWD"/scripts/eval.sh '[ "$TERMINAL_DIR" ] && clean_bt && clean_files && clean_ea' || exit 0
-#- cd "$HOME/.wine/drive_c/Program Files (x86)"/
-#- zip -vr "${HOME}/mt-${MT_VER}.zip" Meta*
-#- cd ~
-#deploy:
-#  provider: releases
-#  api_key:
-#    secure: JsrCjD9M6FPnvKbDvlY62uvBStGsiSwgC90EuJr01288GNQPsp8eHOjZn6isBg+LCp8eqAvQOa7hQJbXG2RSZAw66Mvf1xjz7CEjzLRig1GC8do/t6qzRSaUg4YxDLn9lOY3OZ7GUdNQcDsgg5sVpVecx7B9ve3ZleH0l0kT1OHDN33v3a/oBHWsm/e06cXRw5M4L49ovE3V6fNCJDgaqtUn+0/pT002bMw+nobJaYE1KVZstfFohY/Sy4rGiXFT9dChwXzSp+hZf2Imcm15t4t2EFxObozWubZin5njXmRDr66xj1skkvhaxRxYGs9c8K75ySB+nK8pMk8dGoCquzvHuwoIxSYHTfJCwug3Es2LbTwQXNnfnjvXiqQU5886NvdzZiU17aYSg+fTEoAaFoCsMK12xx+Y3DxvCuunJSIz+Ke2CnEJm2yH7UAyaTWPbkZPfTcauFwegModuV2CW5yemXIMddmDunoY1EMIp6qURjYZS/HqYGX9fsKHYyyzm+dBG9dF5beMMpuxJ7C/e+GZa4z/BQ6bcRwUP8PIaqZXrm5UZ4l4PvJlLtetbV1FVLNl8StrWffsWRcZtLtMgAHYswmL93RafjlMI6HhWzQxmY+U3TS94/bgRuy3bIb/+GnCBtXi8y749zu1S+r5UApuCyyhOfwonhpfbie1m8M=
-#  file_glob: true
-#  skip_cleanup: true
-#  file: "mt-*.zip"
-#  on:
-#    repo: EA31337/EA-Tester
-#    tags: true
-#after_deploy:
-#- echo "done."

Dockerfile

@@ -58,6 +58,7 @@ RUN find /var/lib/apt/lists -type f -delete
 RUN find /tmp -mindepth 1 '(' -type d -o -type f ')' -delete
 USER ubuntu
 RUN eval.sh clean_bt
+RUN eval.sh clean_ea
 RUN eval.sh clean_files
 
 # Expose SSH and VNC when installed.

docker-compose.yml

@@ -25,7 +25,7 @@ services:
     - ./scripts:/opt/scripts
   RunInstallMt4:
     command: install_mt 4
-    image: ea31337/ea-tester
+    image: ea31337/ea-tester:dev
     volumes:
     - ./scripts:/opt/scripts
   RunInstallMt4x:
@@ -35,7 +35,7 @@ services:
     - ./scripts:/opt/scripts
   RunInstallMt5:
     command: install_mt 5
-    image: ea31337/ea-tester
+    image: ea31337/ea-tester:dev
     volumes:
     - ./scripts:/opt/scripts
   RunCompileMql:
@@ -66,6 +66,7 @@ services:
     environment:
       BT_DAYS: 10-12
       BT_MONTHS: 1
+      RUN_ON_START: clean_ea
   TestEnvelopes:
     command: for tf in M15 M30; do run_backtest -e TestEnvelopes -T $$tf; done
     image: ea31337/ea-tester:EURUSD-2018-DS
@@ -74,6 +75,7 @@ services:
     environment:
       BT_DAYS: 10-12
       BT_MONTHS: 1
+      RUN_ON_START: clean_ea
   TestFXTHeader:
     command: for tf in M1 M30; do run_backtest -e TestFXTHeader -T $$tf; done
     image: ea31337/ea-tester:EURUSD-2018-DS
@@ -82,6 +84,7 @@ services:
     environment:
       BT_DAYS: 10-12
       BT_MONTHS: 1
+      RUN_ON_START: clean_ea
   TestHSTHeader:
     command: for tf in M1 M30; do run_backtest -e TestHSTHeader -T $$tf; done
     image: ea31337/ea-tester:EURUSD-2018-DS
@@ -90,6 +93,7 @@ services:
     environment:
       BT_DAYS: 10-12
       BT_MONTHS: 1
+      RUN_ON_START: clean_ea
   TestLotstep4:
     command: run_backtest -e TestLotstep -D 4
     image: ea31337/ea-tester:EURUSD-2018-DS
@@ -122,6 +126,7 @@ services:
     environment:
       BT_DAYS: 10-12
       BT_MONTHS: 1
+      RUN_ON_START: clean_ea
   TestSpread:
     command: run_backtest -e TestSpread
     image: ea31337/ea-tester:EURUSD-2018-DS
@@ -170,10 +175,11 @@ services:
     environment:
       BT_DAYS: 10-12
       BT_MONTHS: 1
+      RUN_ON_START: clean_ea
 
 # MQL Scripts.
   ScriptPrintPaths:
     command: run_backtest -s PrintPaths
-    image: ea31337/ea-tester
+    image: ea31337/ea-tester:dev
     volumes:
     - ./scripts:/opt/scripts

scripts/.funcs.cmds.inc.sh

@@ -297,7 +297,7 @@ compile() {
   elif [ -s "$rel_path" ] && [ -d "$(dirname "$rel_path")" ]; then
     # If path, enter the folder containing the file.
     cd "$(dirname "$rel_path")"
-    target=.
+    target=$(basename "$rel_path")
     log_file=${log_file:-mql.log}
   elif [ ! -s "$rel_path" ]; then
     # If file does not exist, find in the current folder.

scripts/.funcs.inc.sh

@@ -93,9 +93,8 @@ get_time() {
 }
 
 # Check logs for errors.
-# Usage: check_log_errors [filter] [args]
+# Usage: check_log_errors
 check_log_errors() {
-  local log_file="$(find "$MQLOG_DIR" -type f -name "$(date +%Y%m%d)*.log" -print -quit)"
   local errors=()
   errors+=("cannot open")
   errors+=("not initialized")
@@ -115,8 +114,9 @@ check_log_errors() {
   errors+=("Configuration issue .\+")
   errors+=("Assert fail on .\+")
   errors+=("Testing pass stopped .\+")
+  cd "$TERMINAL_DIR"
   ! check_logs ".\+ no history data" || { ini_del "bt_data" "$CUSTOM_INI"; }
-  ! eval grep --color -iw -C2 "$(printf -- '-e "%s" ' "${errors[@]}")" \"$log_file\"
+  ! eval grep --color -iw -C2 "$(printf -- '-e "%s" ' "${errors[@]}")" */*.log */*/*.log
 }
 
 # Save time (in hours) and store in rule file if exists.

scripts/.funcs.sets.inc.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Define shell functions for dealing with SET file.
+#
+
+## Initialize.
+[ -n "$OPT_VERBOSE" ] && echo "Loading ${BASH_SOURCE[0]}... " >&2
+CWD="${CWD:-$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)}"
+
+#
+## Define SET functions. ##
+#
+
+# Set optimization params
+# Usage: set_opt_params [param] [start] [end] (step)
+set_opt_params() {
+  # Optimization settings (F-On, 1-Min, 2-Step, 3-Max).
+  input_set ^$1,F 1       # On.
+  input_set ^$1,1 $2      # Min.
+  input_set ^$1,2 ${4:-1} # Step.
+  input_set ^$1,3 $3      # Max.
+}

scripts/install_mt4.verb

@@ -21,7 +21,7 @@ load_install_mt4()
     # No documented silent install option, unfortunately.
     w_try_cd "$W_CACHE/$W_PACKAGE"
     w_ahk_do "
-        Run, "$(find . -name "*.exe" -print -quit)"
+        Run, "$(echo *.exe)"
         SetTitleMatchMode, RegEx
         WinWait, 4 Setup
         Send, !n

scripts/install_mt5.verb

@@ -19,7 +19,7 @@ load_install_mt5()
 
     w_try_cd "$W_CACHE/$W_PACKAGE"
     w_ahk_do "
-        Run, "$(find . -name "*.exe" -print -quit)"
+        Run, "$(echo *.exe)"
         SetTitleMatchMode, RegEx
         WinWait, MetaTrader 5 Setup
         Send, {Enter}

scripts/provision.sh

@@ -124,7 +124,8 @@ case "$(uname -s)" in
     apt-get install -qq winehq-devel wine-gecko --install-recommends              # Install Wine.
     apt-get install -qq xvfb xdotool x11-utils xterm                              # Virtual frame buffer and X11 utils.
 
-    # Setup display.
+    # Install Winetricks.
+    curl -sL https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks | install /dev/stdin /usr/local/bin/winetricks
 
     # Install AHK.
     if [ -n "$PROVISION_AHK" ]; then

scripts/run_backtest.sh

@@ -22,10 +22,17 @@ usage() {
 
 # Invoke on test success.
 on_success() {
+
+  # Fail on error in the logs.
   echo "Checking logs..." >&2
-  show_logs
   check_log_errors
+  if [ $? -ne 0 ]; then
+    echo "ERROR: RUN failed." >&2
+    exit 1
+  fi
+
   echo "RUN succeeded." >&2
+  show_logs
   parse_results $@
   on_finish
   local OPTIND
@@ -50,9 +57,10 @@ on_success() {
 on_failure() {
   echo "FAIL?!" >&2
   # Sometimes MT4 fails on success, therefore double checking.
-  TEST_REPORT_BASE="$(basename "$(ini_get TestReport)")"
+  TEST_REPORT_BASE="$(basename "$(ini_get TestReport)" .htm)"
+
   if [ -n "$TEST_REPORT_BASE" ]; then
-    TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*" -print -quit)
+    TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*.htm" -print -quit)
     test -f "$TEST_REPORT_HTM" && { on_success $@; return; }
   elif [ -z "$TEST_EXPERT" -a -n "$SCRIPT" ]; then
     # Report success when script was run and platform killed.
@@ -67,7 +75,7 @@ on_failure() {
   fi
   echo "Printing logs..." >&2
   show_logs
-  echo "RUN failed." >&2
+  echo "ERROR: RUN failed." >&2
   on_finish
 }
 
@@ -80,20 +88,20 @@ on_finish() {
 
 # Parse report files.
 parse_results() {
-  TEST_REPORT_BASE="$(basename "$(ini_get TestReport)")"
+  TEST_REPORT_BASE="$(basename "$(ini_get TestReport)" .htm)"
+
+  echo "Checking the total time elapsed..." >&2
+  save_time
 
   # Ignore if no test results or test expert name is set (e.g. when running the script).
   [ -z "$TEST_REPORT_BASE" -o -z "$TEST_EXPERT" ] && return
 
   # Locate the report file.
-  TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*" -print -quit)
+  TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*.htm" -print -quit)
   TEST_REPORT_DIR="$(dirname "$TEST_REPORT_HTM")"
   test -d "$TEST_REPORT_DIR" || exit 1
   test -f "$TEST_REPORT_HTM" || exit 1
 
-  echo "Checking the total time elapsed..." >&2
-  save_time
-
   if [ -n "$OPT_FORMAT_JSON" ]; then
     # Convert test report file into JSON format.
     echo "Converting HTML report ($TEST_REPORT_DIR) into JSON file..." >&2
@@ -197,13 +205,13 @@ done
 
 # Invoke includes.
 . "$CWD"/.aliases.inc.sh
-. "$CWD"/.funcs.cmds.inc.sh
 . "$CWD"/.funcs.inc.sh
+. "$CWD"/.funcs.cmds.inc.sh
+. "$CWD"/.funcs.sets.inc.sh
 . "$CWD"/.vars.inc.sh
 
 # Initialize.
 initialize
-set_display
 
 [ -n "$NOERR" ] || set -e
 [ -n "$OPT_TRACE" ] && set -x
@@ -246,7 +254,6 @@ while getopts $ARGS arg; do
       ;;
 
     C) # Clear previous backtest data files.
-      clean_files
       clean_bt
       ;;
 
@@ -424,7 +431,7 @@ if [ -n "$SETFILE" -a ! -s "$SETFILE" ]; then
   cp -f $VFLAG "$TESTER_DIR/$exported_setfile" "$SETFILE"
 fi
 if [ -s "$SETFILE" -a ! -f "$TESTER_DIR/$EA_SETFILE" ]; then
-  echo "EA's SET file does not exist ($EA_SETFILE), copying from $SETFILE..." >&2
+  echo "EA's SET file is missing ($EA_SETFILE), copying from $SETFILE..." >&2
   cp -f $VFLAG "$SETFILE" "$TESTER_DIR/$EA_SETFILE"
 fi
 
@@ -739,6 +746,9 @@ if [ -n "$OPT_DRY_RUN" ]; then
   exit $?
 fi
 
+# Start X virtual framebuffer.
+set_display
+
 # Clean files before run.
 clean_files