diff --git a/.travis.yml b/.travis.yml index ec242d02..2022ea1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ env: DOCKER_REPO=ea31337/$DOCKER_IMAGE:$DOCKER_TAG PATH=${PATH//:\.\/node_modules\/\.bin/} # Workaround for Travis issue #4862 CWD="$TRAVIS_BUILD_DIR" + DOCKER_ARG="-e OPT_TRACE=$OPT_TRACE" DOCKER_TAG=$([[ "$TRAVIS_BRANCH" == "master" ]] && echo "latest" || echo ${TRAVIS_BRANCH////-}) WINETRICKS_DOWNLOADER_RETRIES=20 - secure: "waZbxfpNTIp/etY6lwR8GB+aSWDIw73tu6HxFdwVYdDJ/25SlY+gkJMCPqlpkjA1Aenv1I3TDDd/3dQIMYUeNbgIKOx6AtWkddnfTykqYYFJ4j6MlWPxZKaxDLjehLvjj0ZqkfXcswgXDaIoNnunKbLKiVohprLhO9Yg6I/j1LfzmPPGmFez2Mg2K0jFmri1i8prxpL4cctH0cR9NEZ+Lw13nDjY4b7AMoY6mzt6TFe9/yEt1OzYVGAh36hWgI1AL8h86cyDlUYHMRrzZjMKUGZFyOkkz3BRA6wvyOtjRhNlh2SIY+Y+HiwQaLONeQGpZ9YjNCUb+f5GIxXyEry9RV1oaMNIUnXNSQstUvgzy3QO2muD468KhfvPafpPj0gDHmfGEH+2IMcrz/WYNobsdN9JOjvPWutnxgYAzd27jwDIQ4D2WewZ3COgrRXxCAgwsOQKvlzDaFAOvPObOnCmIVsbOpONE3nSRGnrQpxJl0aaspoHLnVyKAUeCTAMhtSZirhWO+Iob9mhicB+9xEMRpxFnRD/FBUcSY/SyVlZjXxK9zIqn9t5KBK0iVZ10l/O5nV6sM7YfpAz8d4MqJDAGT05tKn/o2uDJtVvnl6szCIP5iQwsMUsQqQCuNPbmfm1V/q2ipjxLwiwOx2NRbSjFx3/5NnyXiIqehmvlfnfUjQ=" @@ -23,21 +24,21 @@ env: - CMD="docker-compose run RunHelp; scripts/eval.sh help; scripts/run_backtest.sh -?" - CMD="sudo scripts/provision.sh" - CMD="make docker-ci DOCKER_TAG=$DOCKER_TAG; [[ '$TRAVIS_PULL_REQUEST' = 'false' ]] && make docker-push DOCKER_TAG=$DOCKER_TAG || true" - - CMD="docker-compose run TestSyntaxVagrantfile; docker-compose run TestSyntaxBash; docker-compose run TestSyntaxShellcheck" - - CMD="docker-compose run RunCompileMql" - - CMD="docker-compose run ScriptPrintPaths" - - CMD="docker-compose run TestHSTHeader; docker-compose run TestFXTHeader" - - CMD="docker-compose run TestLotstep4" + - CMD="docker-compose run $DOCKER_ARG TestSyntaxVagrantfile; docker-compose run $DOCKER_ARG TestSyntaxBash; docker-compose run $DOCKER_ARG TestSyntaxShellcheck" + - CMD="docker-compose run $DOCKER_ARG RunCompileMql" + - CMD="docker-compose run $DOCKER_ARG ScriptPrintPaths" + - CMD="docker-compose run $DOCKER_ARG TestHSTHeader; docker-compose run TestFXTHeader" + - CMD="docker-compose run $DOCKER_ARG TestLotstep4" # @fixme: Error: Expected lot step for 5 digits: 0.01, found: 0.1. -# - CMD="docker-compose run TestLotstep5" - - CMD="docker-compose run TestTimeframes" - - CMD="docker-compose run TestBands" - - CMD="docker-compose run TestEnvelopes" - - CMD="docker-compose run TestModellingQuality" - - CMD="docker-compose run RunInstallMt4" - - CMD="docker-compose run RunInstallMt4x" - - CMD="docker-compose run RunInstallMt5" - - CMD="docker-compose run DockerLargeFiles" +# - CMD="docker-compose run $DOCKER_ARG TestLotstep5" + - CMD="docker-compose run $DOCKER_ARG TestTimeframes" + - CMD="docker-compose run $DOCKER_ARG TestBands" + - CMD="docker-compose run $DOCKER_ARG TestEnvelopes" + - CMD="docker-compose run $DOCKER_ARG TestModellingQuality" + - CMD="docker-compose run $DOCKER_ARG RunInstallMt4" + - CMD="docker-compose run $DOCKER_ARG RunInstallMt4x" + - CMD="docker-compose run $DOCKER_ARG RunInstallMt5" + - CMD="docker-compose run $DOCKER_ARG DockerLargeFiles" # Tests which suppose to be working in older versions only. # - TEST_ARGS="-v -t -d 1000 -p EURUSD -m 1-12 -y 2015 -S 10 -b DS -D5 -e TestFXTHeader -M4.0.0.971 -R" @@ -80,21 +81,3 @@ cache: directories: - $HOME/.cache - $HOME/.docker -#before_deploy: -#- MT_VER=$("$CWD"/scripts/eval.sh filever terminal.exe) -#- bash "$CWD"/scripts/eval.sh '[ "$TERMINAL_DIR" ] && clean_bt && clean_files && clean_ea' || exit 0 -#- cd "$HOME/.wine/drive_c/Program Files (x86)"/ -#- zip -vr "${HOME}/mt-${MT_VER}.zip" Meta* -#- cd ~ -#deploy: -# provider: releases -# api_key: -# secure: JsrCjD9M6FPnvKbDvlY62uvBStGsiSwgC90EuJr01288GNQPsp8eHOjZn6isBg+LCp8eqAvQOa7hQJbXG2RSZAw66Mvf1xjz7CEjzLRig1GC8do/t6qzRSaUg4YxDLn9lOY3OZ7GUdNQcDsgg5sVpVecx7B9ve3ZleH0l0kT1OHDN33v3a/oBHWsm/e06cXRw5M4L49ovE3V6fNCJDgaqtUn+0/pT002bMw+nobJaYE1KVZstfFohY/Sy4rGiXFT9dChwXzSp+hZf2Imcm15t4t2EFxObozWubZin5njXmRDr66xj1skkvhaxRxYGs9c8K75ySB+nK8pMk8dGoCquzvHuwoIxSYHTfJCwug3Es2LbTwQXNnfnjvXiqQU5886NvdzZiU17aYSg+fTEoAaFoCsMK12xx+Y3DxvCuunJSIz+Ke2CnEJm2yH7UAyaTWPbkZPfTcauFwegModuV2CW5yemXIMddmDunoY1EMIp6qURjYZS/HqYGX9fsKHYyyzm+dBG9dF5beMMpuxJ7C/e+GZa4z/BQ6bcRwUP8PIaqZXrm5UZ4l4PvJlLtetbV1FVLNl8StrWffsWRcZtLtMgAHYswmL93RafjlMI6HhWzQxmY+U3TS94/bgRuy3bIb/+GnCBtXi8y749zu1S+r5UApuCyyhOfwonhpfbie1m8M= -# file_glob: true -# skip_cleanup: true -# file: "mt-*.zip" -# on: -# repo: EA31337/EA-Tester -# tags: true -#after_deploy: -#- echo "done." diff --git a/Dockerfile b/Dockerfile index 70f6841b..fc081a60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,7 @@ RUN find /var/lib/apt/lists -type f -delete RUN find /tmp -mindepth 1 '(' -type d -o -type f ')' -delete USER ubuntu RUN eval.sh clean_bt +RUN eval.sh clean_ea RUN eval.sh clean_files # Expose SSH and VNC when installed. diff --git a/docker-compose.yml b/docker-compose.yml index fcb317f0..917ab732 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: - ./scripts:/opt/scripts RunInstallMt4: command: install_mt 4 - image: ea31337/ea-tester + image: ea31337/ea-tester:dev volumes: - ./scripts:/opt/scripts RunInstallMt4x: @@ -35,7 +35,7 @@ services: - ./scripts:/opt/scripts RunInstallMt5: command: install_mt 5 - image: ea31337/ea-tester + image: ea31337/ea-tester:dev volumes: - ./scripts:/opt/scripts RunCompileMql: @@ -66,6 +66,7 @@ services: environment: BT_DAYS: 10-12 BT_MONTHS: 1 + RUN_ON_START: clean_ea TestEnvelopes: command: for tf in M15 M30; do run_backtest -e TestEnvelopes -T $$tf; done image: ea31337/ea-tester:EURUSD-2018-DS @@ -74,6 +75,7 @@ services: environment: BT_DAYS: 10-12 BT_MONTHS: 1 + RUN_ON_START: clean_ea TestFXTHeader: command: for tf in M1 M30; do run_backtest -e TestFXTHeader -T $$tf; done image: ea31337/ea-tester:EURUSD-2018-DS @@ -82,6 +84,7 @@ services: environment: BT_DAYS: 10-12 BT_MONTHS: 1 + RUN_ON_START: clean_ea TestHSTHeader: command: for tf in M1 M30; do run_backtest -e TestHSTHeader -T $$tf; done image: ea31337/ea-tester:EURUSD-2018-DS @@ -90,6 +93,7 @@ services: environment: BT_DAYS: 10-12 BT_MONTHS: 1 + RUN_ON_START: clean_ea TestLotstep4: command: run_backtest -e TestLotstep -D 4 image: ea31337/ea-tester:EURUSD-2018-DS @@ -122,6 +126,7 @@ services: environment: BT_DAYS: 10-12 BT_MONTHS: 1 + RUN_ON_START: clean_ea TestSpread: command: run_backtest -e TestSpread image: ea31337/ea-tester:EURUSD-2018-DS @@ -170,10 +175,11 @@ services: environment: BT_DAYS: 10-12 BT_MONTHS: 1 + RUN_ON_START: clean_ea # MQL Scripts. ScriptPrintPaths: command: run_backtest -s PrintPaths - image: ea31337/ea-tester + image: ea31337/ea-tester:dev volumes: - ./scripts:/opt/scripts diff --git a/scripts/.funcs.cmds.inc.sh b/scripts/.funcs.cmds.inc.sh index 0178accd..6a7ca0a9 100644 --- a/scripts/.funcs.cmds.inc.sh +++ b/scripts/.funcs.cmds.inc.sh @@ -297,7 +297,7 @@ compile() { elif [ -s "$rel_path" ] && [ -d "$(dirname "$rel_path")" ]; then # If path, enter the folder containing the file. cd "$(dirname "$rel_path")" - target=. + target=$(basename "$rel_path") log_file=${log_file:-mql.log} elif [ ! -s "$rel_path" ]; then # If file does not exist, find in the current folder. diff --git a/scripts/.funcs.inc.sh b/scripts/.funcs.inc.sh index e403beb2..32486053 100644 --- a/scripts/.funcs.inc.sh +++ b/scripts/.funcs.inc.sh @@ -93,9 +93,8 @@ get_time() { } # Check logs for errors. -# Usage: check_log_errors [filter] [args] +# Usage: check_log_errors check_log_errors() { - local log_file="$(find "$MQLOG_DIR" -type f -name "$(date +%Y%m%d)*.log" -print -quit)" local errors=() errors+=("cannot open") errors+=("not initialized") @@ -115,8 +114,9 @@ check_log_errors() { errors+=("Configuration issue .\+") errors+=("Assert fail on .\+") errors+=("Testing pass stopped .\+") + cd "$TERMINAL_DIR" ! check_logs ".\+ no history data" || { ini_del "bt_data" "$CUSTOM_INI"; } - ! eval grep --color -iw -C2 "$(printf -- '-e "%s" ' "${errors[@]}")" \"$log_file\" + ! eval grep --color -iw -C2 "$(printf -- '-e "%s" ' "${errors[@]}")" */*.log */*/*.log } # Save time (in hours) and store in rule file if exists. diff --git a/scripts/.funcs.sets.inc.sh b/scripts/.funcs.sets.inc.sh new file mode 100644 index 00000000..707e72f9 --- /dev/null +++ b/scripts/.funcs.sets.inc.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Define shell functions for dealing with SET file. +# + +## Initialize. +[ -n "$OPT_VERBOSE" ] && echo "Loading ${BASH_SOURCE[0]}... " >&2 +CWD="${CWD:-$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)}" + +# +## Define SET functions. ## +# + +# Set optimization params +# Usage: set_opt_params [param] [start] [end] (step) +set_opt_params() { + # Optimization settings (F-On, 1-Min, 2-Step, 3-Max). + input_set ^$1,F 1 # On. + input_set ^$1,1 $2 # Min. + input_set ^$1,2 ${4:-1} # Step. + input_set ^$1,3 $3 # Max. +} diff --git a/scripts/install_mt4.verb b/scripts/install_mt4.verb index 33aad1f5..0689d4d0 100644 --- a/scripts/install_mt4.verb +++ b/scripts/install_mt4.verb @@ -21,7 +21,7 @@ load_install_mt4() # No documented silent install option, unfortunately. w_try_cd "$W_CACHE/$W_PACKAGE" w_ahk_do " - Run, "$(find . -name "*.exe" -print -quit)" + Run, "$(echo *.exe)" SetTitleMatchMode, RegEx WinWait, 4 Setup Send, !n diff --git a/scripts/install_mt5.verb b/scripts/install_mt5.verb index 85f64c3d..dc179cee 100644 --- a/scripts/install_mt5.verb +++ b/scripts/install_mt5.verb @@ -19,7 +19,7 @@ load_install_mt5() w_try_cd "$W_CACHE/$W_PACKAGE" w_ahk_do " - Run, "$(find . -name "*.exe" -print -quit)" + Run, "$(echo *.exe)" SetTitleMatchMode, RegEx WinWait, MetaTrader 5 Setup Send, {Enter} diff --git a/scripts/provision.sh b/scripts/provision.sh index da9614ef..1f17bda9 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -124,7 +124,8 @@ case "$(uname -s)" in apt-get install -qq winehq-devel wine-gecko --install-recommends # Install Wine. apt-get install -qq xvfb xdotool x11-utils xterm # Virtual frame buffer and X11 utils. - # Setup display. + # Install Winetricks. + curl -sL https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks | install /dev/stdin /usr/local/bin/winetricks # Install AHK. if [ -n "$PROVISION_AHK" ]; then diff --git a/scripts/run_backtest.sh b/scripts/run_backtest.sh index 17f37df2..7bf5560d 100755 --- a/scripts/run_backtest.sh +++ b/scripts/run_backtest.sh @@ -22,10 +22,17 @@ usage() { # Invoke on test success. on_success() { + + # Fail on error in the logs. echo "Checking logs..." >&2 - show_logs check_log_errors + if [ $? -ne 0 ]; then + echo "ERROR: RUN failed." >&2 + exit 1 + fi + echo "RUN succeeded." >&2 + show_logs parse_results $@ on_finish local OPTIND @@ -50,9 +57,10 @@ on_success() { on_failure() { echo "FAIL?!" >&2 # Sometimes MT4 fails on success, therefore double checking. - TEST_REPORT_BASE="$(basename "$(ini_get TestReport)")" + TEST_REPORT_BASE="$(basename "$(ini_get TestReport)" .htm)" + if [ -n "$TEST_REPORT_BASE" ]; then - TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*" -print -quit) + TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*.htm" -print -quit) test -f "$TEST_REPORT_HTM" && { on_success $@; return; } elif [ -z "$TEST_EXPERT" -a -n "$SCRIPT" ]; then # Report success when script was run and platform killed. @@ -67,7 +75,7 @@ on_failure() { fi echo "Printing logs..." >&2 show_logs - echo "RUN failed." >&2 + echo "ERROR: RUN failed." >&2 on_finish } @@ -80,20 +88,20 @@ on_finish() { # Parse report files. parse_results() { - TEST_REPORT_BASE="$(basename "$(ini_get TestReport)")" + TEST_REPORT_BASE="$(basename "$(ini_get TestReport)" .htm)" + + echo "Checking the total time elapsed..." >&2 + save_time # Ignore if no test results or test expert name is set (e.g. when running the script). [ -z "$TEST_REPORT_BASE" -o -z "$TEST_EXPERT" ] && return # Locate the report file. - TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*" -print -quit) + TEST_REPORT_HTM=$(find "$TESTER_DIR" "$TERMINAL_DIR" -maxdepth 2 -name "${TEST_REPORT_BASE//[][]/?}*.htm" -print -quit) TEST_REPORT_DIR="$(dirname "$TEST_REPORT_HTM")" test -d "$TEST_REPORT_DIR" || exit 1 test -f "$TEST_REPORT_HTM" || exit 1 - echo "Checking the total time elapsed..." >&2 - save_time - if [ -n "$OPT_FORMAT_JSON" ]; then # Convert test report file into JSON format. echo "Converting HTML report ($TEST_REPORT_DIR) into JSON file..." >&2 @@ -197,13 +205,13 @@ done # Invoke includes. . "$CWD"/.aliases.inc.sh -. "$CWD"/.funcs.cmds.inc.sh . "$CWD"/.funcs.inc.sh +. "$CWD"/.funcs.cmds.inc.sh +. "$CWD"/.funcs.sets.inc.sh . "$CWD"/.vars.inc.sh # Initialize. initialize -set_display [ -n "$NOERR" ] || set -e [ -n "$OPT_TRACE" ] && set -x @@ -246,7 +254,6 @@ while getopts $ARGS arg; do ;; C) # Clear previous backtest data files. - clean_files clean_bt ;; @@ -424,7 +431,7 @@ if [ -n "$SETFILE" -a ! -s "$SETFILE" ]; then cp -f $VFLAG "$TESTER_DIR/$exported_setfile" "$SETFILE" fi if [ -s "$SETFILE" -a ! -f "$TESTER_DIR/$EA_SETFILE" ]; then - echo "EA's SET file does not exist ($EA_SETFILE), copying from $SETFILE..." >&2 + echo "EA's SET file is missing ($EA_SETFILE), copying from $SETFILE..." >&2 cp -f $VFLAG "$SETFILE" "$TESTER_DIR/$EA_SETFILE" fi @@ -739,6 +746,9 @@ if [ -n "$OPT_DRY_RUN" ]; then exit $? fi +# Start X virtual framebuffer. +set_display + # Clean files before run. clean_files