Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

【benchmark】add timeout for test_tipc/benchmark_train.sh #472

Open
wants to merge 8 commits into
base: develop
Choose a base branch
from
Open

【benchmark】add timeout for test_tipc/benchmark_train.sh #472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fa14245
add max_mem_reserved for benchmark
mmglove Nov 21, 2023
c9af6a1
add max_mem_reserved for benchmark
mmglove Nov 21, 2023
b75c03b
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle3D in…
mmglove Dec 5, 2023
dce223f
fix
mmglove Dec 5, 2023
22d0503
Merge branch 'PaddlePaddle:develop' into develop
mmglove Aug 1, 2024
b7cfd52
Update benchmark_train.sh
mmglove Aug 1, 2024
74edca6
Update test_train_inference_python.sh
mmglove Aug 1, 2024
106ced7
Update benchmark_train.sh
mmglove Aug 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions test_tipc/benchmark_train.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ repo_name=$(get_repo_name )
SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log
mkdir -p "${SAVE_LOG}/benchmark_log/"
status_log="${SAVE_LOG}/benchmark_log/results.log"
# get benchmark profiling params : PROFILING_TIMER_ONLY=no|True|False
PROFILING_TIMER_ONLY=${PROFILING_TIMER_ONLY:-"no"}

# The number of lines in which train params can be replaced.
line_python=3
Expand Down Expand Up @@ -175,28 +177,35 @@ for batch_size in ${batch_size_list[*]}; do
gpu_id=$(set_gpu_id $device_num)

if [ ${#gpu_id} -le 1 ];then
log_path="$SAVE_LOG/profiling_log"
mkdir -p $log_path
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_${to_static}profiling"
func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id
# set profile_option params
tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`
if [[ ${PROFILING_TIMER_ONLY} != "no" ]];then
echo "run profile"
if [[ ${PROFILING_TIMER_ONLY} = "False" ]];then
profile_option="${profile_option};timer_only=False"
fi
log_path="$SAVE_LOG/profiling_log"
mkdir -p $log_path
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_${to_static}profiling"
func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id
# set profile_option params
tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`

# run test_train_inference_python.sh
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
echo $cmd
eval $cmd
eval "cat ${log_path}/${log_name}"
# run test_train_inference_python.sh
cmd="timeout 2m bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
echo $cmd
eval $cmd
eval "cat ${log_path}/${log_name}"
fi

# without profile
echo "run without profile"
log_path="$SAVE_LOG/train_log"
speed_log_path="$SAVE_LOG/index"
mkdir -p $log_path
mkdir -p $speed_log_path
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_${to_static}log"
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_${to_static}speed"
func_sed_params "$FILENAME" "${line_profile}" "null" # sed profile_id as null
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
cmd="timeout 5m bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
echo $cmd
job_bt=`date '+%Y%m%d%H%M%S'`
eval $cmd
Expand Down
6 changes: 3 additions & 3 deletions test_tipc/test_train_inference_python.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ function func_inference(){
}

if [ ${MODE} = "whole_infer" ]; then
GPUID=$3
GPUID=$CUDA_VISIBLE_DEVICES
if [ ${#GPUID} -le 0 ];then
env=" "
else
Expand Down Expand Up @@ -197,12 +197,12 @@ else
if [ ${gpu} = "-1" ];then
env=""
elif [ ${#gpu} -le 1 ];then
env="export CUDA_VISIBLE_DEVICES=${gpu}"
env="export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}"
eval ${env}
elif [ ${#gpu} -le 15 ];then
IFS=","
array=(${gpu})
env="export CUDA_VISIBLE_DEVICES=${gpu}"
env="export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}"
IFS="|"
else
IFS=";"
Expand Down