function distribute_test() {

echo "Start gpups tests"

parallel_test_base_gpups

echo "End gpups tests"

echo "Download ..."

cd ${work_dir}

git clone --depth=1 https://github.com/PaddlePaddle/PaddleNLP.git -b stable/paddle-ci

cd PaddleNLP

sed -i '/lac/d' scripts/regression/requirements_ci.txt

pip install -r requirements.txt

pip install -r scripts/regression/requirements_ci.txt

pip install -r ./csrc/requirements.txt

python setup.py install

python -m pip install pytest-timeout

cd csrc && python setup_cuda.py install

cd ${work_dir}

wget -q --no-proxy https://paddle-qa.bj.bcebos.com/paddlenlp/Bos.zip --no-check-certificate

unzip -P'41maLgwWnCLaFTCehlwQ6n4l3oZpS/r5gPq4K4VLj5M1024' Bos.zip

mkdir paddlenlp && mv Bos/* ./paddlenlp/

rm -rf ./paddlenlp/upload/*

rm -rf ./paddlenlp/models/bigscience/*

sed -i '35c # gpt_auto_recompute_bs16_fp16_o2_DP4-MP2-Sharding4_stage1' PaddleNLP/scripts/distribute/ci_case_auto.sh

sed -i '37c # gpt_auto_recompute_bs16_fp16_o2_DP4-MP2-Sharding4_stage3' PaddleNLP/scripts/distribute/ci_case_auto.sh

sed -i '38c # gpt_auto_recompute_bs16_fp16_o2_DP2-MP1-PP4_Sharding2_stage1' PaddleNLP/scripts/distribute/ci_case_auto.sh

sed -i '40c # gpt_auto_recompute_bs16_fp16_o2_DP2-MP1-PP4_Sharding2_stage3' PaddleNLP/scripts/distribute/ci_case_auto.sh

sed -i '41c # gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage1' PaddleNLP/scripts/distribute/ci_case_auto.sh

sed -i '43c # gpt_auto_recompute_bs16_fp16_o2_DP2-MP2-PP2_Sharding2_stage3' PaddleNLP/scripts/distribute/ci_case_auto.sh

sed -i -e 's/case_list=(\$(awk/case_list=(auto_unit_test) # /g' ./tools/auto_parallel/ci_auto_parallel.sh

export FLAGS_dynamic_static_unified_comm=True

echo "Start LLM Test"

# Disable Test: test_gradio

cd ${work_dir}/PaddleNLP

pids=()

env CUDA_VISIBLE_DEVICES=0,1 python -m pytest -s -v tests/llm/test_finetune.py &

pids+=($!)

env CUDA_VISIBLE_DEVICES=2,3 python -m pytest -s -v tests/llm/test_lora.py tests/llm/test_predictor.py &

pids+=($!)

env CUDA_VISIBLE_DEVICES=4,5 python -m pytest -s -v tests/llm/test_prefix_tuning.py tests/llm/test_pretrain.py &

pids+=($!)

env CUDA_VISIBLE_DEVICES=6,7 python -m pytest -s -v tests/llm/test_ptq.py tests/llm/testing_utils.py &

pids+=($!)

for pid in "${pids[@]}"; do

wait $pid

done

echo "End LLM Test"

echo "Start auto_parallel Test"

cd ${work_dir}

timeout 50m bash tools/auto_parallel/ci_auto_parallel.sh

EXIT_CODE=$?

echo "End auto_parallel Test"

if [[ "$EXIT_CODE" != "0" ]]; then

exit 8;

fi

}

distribute_test)

distribute_test

;;

if(WITH_CUDNN_FRONTEND)

set_tests_properties(test_dist_fuse_resunit_pass PROPERTIES TIMEOUT 350)

endif()

if [ -n "$paddle" ];then

python -m pip install --user ${paddle} --force-reinstall --no-dependencies;

fi

function get_quickly_disable_ut() {

python -m pip install httpx

if disable_ut_quickly=$(python ${PADDLE_ROOT}/tools/get_quick_disable_lt.py); then

echo "========================================="

echo "The following unittests have been disabled:"

echo ${disable_ut_quickly}

echo "========================================="

else

exit 102

disable_ut_quickly='disable_ut'

fi

}

get_quickly_disable_ut||disable_ut_quickly='disable_ut'

NUM_PROC=4

EXIT_CODE=0

pids=()

for (( i = 0; i < $NUM_PROC; i++ )); do

cuda_list="$((i*2)),$((i*2+1))"

(env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC --output-on-failure -R "($parallel_list)" -E "($disable_ut_quickly)" --timeout 120 -j4 | tee -a $tmpfile; test ${PIPESTATUS[0]} -eq 0)&

pids+=($!)

for pid in "${pids[@]}"; do

wait $pid

status=$?

if [ $status -ne 0 ]; then

EXIT_CODE=8

fi

pids=()

for (( i = 0; i < $NUM_PROC; i++ )); do

cuda_list="$((i*2)),$((i*2+1))"

(env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC --output-on-failure -R "($serial_list)" -E "($disable_ut_quickly)" --timeout 180 -j1 | tee -a $tmpfile; test ${PIPESTATUS[0]} -eq 0)&

pids+=($!)

for pid in "${pids[@]}"; do

wait $pid

status=$?

if [ $status -ne 0 ]; then

EXIT_CODE=8

fi

if [ "${EXIT_CODE}" != "0" ];then