Update framework versions for CI (#1462)

chensuyue · web-flow · commit 6c78dfefccf0 · 2023-12-22T16:24:00.000+08:00
Signed-off-by: chensuyue &lt;suyue.chen@intel.com&gt;
Signed-off-by: Xin He &lt;xin3.he@intel.com&gt;
diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml
@@ -155,9 +155,9 @@ stages:
               patterns: "**/*_tuning_info.log"
               path: $(OUT_SCRIPT_PATH)
           - task: UsePythonVersion@0
-            displayName: "Use Python 3.8"
+            displayName: "Use Python 3.10"
             inputs:
-              versionSpec: "3.8"
+              versionSpec: "3.10"
           - script: |
               cd ${OUT_SCRIPT_PATH}
               mkdir generated
diff --git a/.azure-pipelines/scripts/fwk_version.sh b/.azure-pipelines/scripts/fwk_version.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+echo "export FWs version..."
+test_mode=$1
+
+if [ "$test_mode" == "coverage" ] || [ "$test_mode" == "latest" ]; then
+    export tensorflow_version='2.14.0'
+    export pytorch_version='2.1.0+cpu'
+    export torchvision_version='0.16.0+cpu'
+    export ipex_version='2.1.0+cpu'
+    export onnx_version='1.14.1'
+    export onnxruntime_version='1.16.3'
+    export mxnet_version='1.9.1'
+else
+    export tensorflow_version='2.13.0'
+    export pytorch_version='2.0.1+cpu'
+    export torchvision_version='0.15.2+cpu'
+    export ipex_version='2.0.1+cpu'
+    export onnx_version='1.14.1'
+    export onnxruntime_version='1.15.1'
+    export mxnet_version='1.9.1'
+fi
+
+
+
+
+
diff --git a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh
@@ -21,8 +21,10 @@ do
     esac
 done
 
+echo "specify FWs version..."
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
 FRAMEWORK="mxnet"
-FRAMEWORK_VERSION="1.9.1"
+FRAMEWORK_VERSION=${mxnet_version}
 
 inc_new_api=false
 # ======== set up config for mxnet models ========
diff --git a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh
@@ -21,8 +21,10 @@ do
     esac
 done
 
+echo "specify FWs version..."
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
 FRAMEWORK="onnxrt"
-FRAMEWORK_VERSION="1.15.1"
+FRAMEWORK_VERSION=${onnxruntime_version}
 
 inc_new_api=false
 # ======== set up config for onnxrt models ========
diff --git a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh
@@ -21,9 +21,11 @@ do
     esac
 done
 
+echo "specify FWs version..."
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
 FRAMEWORK="pytorch"
-FRAMEWORK_VERSION="2.0.1+cpu"
-TORCH_VISION_VERSION="0.15.2+cpu"
+FRAMEWORK_VERSION=${pytorch_version}
+TORCH_VISION_VERSION=${torchvision_version}
 
 inc_new_api=false
 # ======== set up config for pytorch models ========
diff --git a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh
@@ -21,8 +21,10 @@ do
     esac
 done
 
+echo "specify FWs version..."
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh 'latest'
 FRAMEWORK="tensorflow"
-FRAMEWORK_VERSION="2.13.0"
+FRAMEWORK_VERSION=${tensorflow_version}
 
 inc_new_api=false
 # ======== set up config for tensorflow models ========
diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh
@@ -48,19 +48,15 @@ if [[ "${torchvision_version}" != "" ]]; then
     pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html
 fi
 
-if [[ "${ipex_version}" == "1.13.0+cpu" ]]; then
-    ipex_whl="https://github.com/intel/intel-extension-for-pytorch/releases/download/v1.13.0%2Bcpu/intel_extension_for_pytorch-1.13.0-cp310-cp310-manylinux2014_x86_64.whl"
-    pip install $ipex_whl
-elif [[ "${ipex_version}" == "2.0.0+cpu" ]]; then
+if [[ "${ipex_version}" == "2.0.0+cpu" ]]; then
     ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.0%2Bcpu-cp310-cp310-linux_x86_64.whl"
     pip install $ipex_whl
 elif [[ "${ipex_version}" == "2.0.1+cpu" ]]; then
     ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.100%2Bcpu-cp310-cp310-linux_x86_64.whl"
     pip install $ipex_whl
-elif [[ "${ipex_version}" == "2.1.0" ]]; then
-    pip install /tf_dataset/pt_binary/ww32/torch-*.whl
-    pip install /tf_dataset/pt_binary/ww32/torchvision-*.whl
-    pip install /tf_dataset/pt_binary/ww32/intel_extension_for_pytorch-*.whl
+elif [[ "${ipex_version}" == "2.1.0+cpu" ]]; then
+    ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.1.0%2Bcpu-cp310-cp310-linux_x86_64.whl"
+    pip install $ipex_whl
 fi
 
 if [[ "${onnx_version}" != "" ]]; then
@@ -88,7 +84,6 @@ fi
 # install special test env requirements
 # common deps
 pip install cmake
-pip install horovod
 pip install transformers
 
 if [[ $(echo "${test_case}" | grep -c "others") != 0 ]];then
@@ -97,6 +92,10 @@ elif [[ $(echo "${test_case}" | grep -c "nas") != 0 ]]; then
     pip install dynast==1.6.0rc1
 elif [[ $(echo "${test_case}" | grep -c "tf pruning") != 0 ]]; then
     pip install tensorflow-addons
+    # Workaround
+    # horovod can't be install in the env with TF and PT together
+    # so test distribute cases in the env with single fw installed
+    pip install horovod
 fi
 # test deps
 pip install coverage
@@ -106,3 +105,12 @@ pip list
 echo "[DEBUG] list pipdeptree..."
 pip install pipdeptree
 pipdeptree
+
+# import torch before import tensorflow
+if [[ $(echo "${test_case}" | grep -c "run basic api") != 0 ]] || [[ $(echo "${test_case}" | grep -c "run basic others") != 0 ]] || [[ $(echo "${test_case}" | grep -c "run basic adaptor") != 0 ]]; then
+    cd /neural-compressor/test || exit 1
+    find . -name "test*.py" | xargs sed -i 's/import tensorflow as tf/import torch; import tensorflow as tf/g'
+    find . -name "test*.py" | xargs sed -i 's/import tensorflow.compat.v1 as tf/import torch; import tensorflow.compat.v1 as tf/g'
+    find . -name "test*.py" | xargs sed -i 's/from tensorflow import keras/import torch; from tensorflow import keras/g'
+fi
+
diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh
@@ -4,7 +4,7 @@ test_case="run basic adaptor"
 echo "${test_case}"
 
 echo "specify fwk version..."
-source /neural-compressor/.azure-pipelines/scripts/ut/ut_fwk_version.sh $1
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh $1
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"
diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 python -c "import neural_compressor as nc;print(nc.version.__version__)"
-test_case="run basic adaptor tfnewapi"
+test_case="run basic tfnewapi"
 echo "${test_case}"
 
 echo "specify fwk version..."
diff --git a/.azure-pipelines/scripts/ut/run_basic_api.sh b/.azure-pipelines/scripts/ut/run_basic_api.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 python -c "import neural_compressor as nc;print(nc.version.__version__)"
-test_case="run basic quantization/benchmark/export/mixed_precision/distillation/scheduler/nas"
+test_case="run basic api quantization/benchmark/export/mixed_precision/distillation/scheduler/nas"
 echo "${test_case}"
 
 echo "specify fwk version..."
-source /neural-compressor/.azure-pipelines/scripts/ut/ut_fwk_version.sh $1
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh $1
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"
diff --git a/.azure-pipelines/scripts/ut/run_basic_itex.sh b/.azure-pipelines/scripts/ut/run_basic_itex.sh
@@ -4,10 +4,10 @@ test_case="run basic itex"
 echo "${test_case}"
 
 echo "specify fwk version..."
-export itex_version='1.1.0'
-export tensorflow_version='2.11.0-official'
-export onnx_version='1.13.0'
-export onnxruntime_version='1.13.1'
+export itex_version='2.14.0.1'
+export tensorflow_version='2.14.0-official'
+export onnx_version='1.14.1'
+export onnxruntime_version='1.16.3'
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"
diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh
@@ -4,7 +4,7 @@ test_case="run basic others"
 echo "${test_case}"
 
 echo "specify fwk version..."
-source /neural-compressor/.azure-pipelines/scripts/ut/ut_fwk_version.sh $1
+source /neural-compressor/.azure-pipelines/scripts/fwk_version.sh $1
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"
@@ -26,6 +26,7 @@ sed -i '/ distillation\//d' run.sh
 sed -i '/ scheduler\//d' run.sh
 sed -i '/ nas\//d' run.sh
 sed -i '/ 3x\//d' run.sh
+sed -i '/ distributed\//d' run.sh
 
 echo "copy model for dynas..."
 mkdir -p .torch/ofa_nets || true
diff --git a/.azure-pipelines/scripts/ut/run_basic_pt_pruning.sh b/.azure-pipelines/scripts/ut/run_basic_pt_pruning.sh
@@ -4,16 +4,17 @@ test_case="run basic pt pruning"
 echo "${test_case}"
 
 echo "specify fwk version..."
-export pytorch_version='2.0.0+cpu'
-export torchvision_version='0.15.1+cpu'
-export ipex_version='2.0.0+cpu'
+export pytorch_version='2.1.0+cpu'
+export torchvision_version='0.16.0+cpu'
+export ipex_version='2.1.0+cpu'
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"
 export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.file
 lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))')
 cd /neural-compressor/test || exit 1
 find ./pruning_with_pt -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
+# find ./distributed -name "test_distributed_pt_train.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'>> run.sh
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}
diff --git a/.azure-pipelines/scripts/ut/run_basic_tf_pruning.sh b/.azure-pipelines/scripts/ut/run_basic_tf_pruning.sh
@@ -4,14 +4,15 @@ test_case="run basic tf pruning"
 echo "${test_case}"
 
 echo "specify fwk version..."
-export tensorflow_version='2.12.0'
+export tensorflow_version='2.14.0'
 
 echo "set up UT env..."
 bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh "${test_case}"
 export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.file
 lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))')
 cd /neural-compressor/test || exit 1
 find ./pruning_with_tf -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
+find ./distributed -name "test_distributed_tf_dataloader.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'>> run.sh
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}
diff --git a/.azure-pipelines/scripts/ut/run_basic_v2.1_ipex.sh b/.azure-pipelines/scripts/ut/run_basic_v2.1_ipex.sh
diff --git a/.azure-pipelines/scripts/ut/run_itrex.sh b/.azure-pipelines/scripts/ut/run_itrex.sh
@@ -11,6 +11,8 @@ bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
 bash /intel-extension-for-transformers/.github/workflows/script/install_binary.sh
 
 # prepare test env
+# tmp install transformers for incompatible issue
+pip install transformers==4.34.1
 pip install -r /intel-extension-for-transformers/tests/requirements.txt
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}
diff --git a/.azure-pipelines/scripts/ut/ut_fwk_version.sh b/.azure-pipelines/scripts/ut/ut_fwk_version.sh
diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml
@@ -7,7 +7,7 @@ parameters:
     default: "neural-compressor"
   - name: repoTag
     type: string
-    default: "py38"
+    default: "py310"
   - name: dockerFileName
     type: string
     default: "Dockerfile"
diff --git a/.azure-pipelines/template/model-template.yml b/.azure-pipelines/template/model-template.yml
@@ -15,7 +15,7 @@ steps:
     parameters:
       dockerConfigName: "commonDockerConfig"
       repoName: "neural-compressor"
-      repoTag: "py38"
+      repoTag: "py310"
       dockerFileName: "Dockerfile"
       containerName: ${{ parameters.modelContainerName }}
 
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
@@ -23,7 +23,7 @@ steps:
     parameters:
       dockerConfigName: ${{ parameters.dockerConfigName }}
       repoName: "neural-compressor"
-      repoTag: "py38"
+      repoTag: "py310"
       dockerFileName: "Dockerfile"
       containerName: ${{ parameters.utContainerName }}
       repo: ${{ parameters.repo }}
diff --git a/.azure-pipelines/ut-3x-pt.yml b/.azure-pipelines/ut-3x-pt.yml
@@ -18,7 +18,7 @@ pool: ICX-16C
 
 variables:
   IMAGE_NAME: "neural-compressor"
-  IMAGE_TAG: "py38"
+  IMAGE_TAG: "py310"
   UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   ARTIFACT_NAME: "UT_coverage_report_3x_pt"
diff --git a/.azure-pipelines/ut-3x-tf.yml b/.azure-pipelines/ut-3x-tf.yml
@@ -18,7 +18,7 @@ pool: ICX-16C
 
 variables:
   IMAGE_NAME: "neural-compressor"
-  IMAGE_TAG: "py38"
+  IMAGE_TAG: "py310"
   UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   ARTIFACT_NAME: "UT_coverage_report_3x_tf"
diff --git a/.azure-pipelines/ut-basic-no-cover.yml b/.azure-pipelines/ut-basic-no-cover.yml
diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml
diff --git a/neural_compressor/utils/pytorch.py b/neural_compressor/utils/pytorch.py
diff --git a/test/data/test_dataloader.py b/test/data/test_dataloader.py
diff --git a/test/distributed/test_distributed_metrics.py b/test/distributed/test_distributed_metrics.py
diff --git a/test/distributed/test_distributed_pt_train.py b/test/distributed/test_distributed_pt_train.py
diff --git a/test/model/test_model_pytorch.py b/test/model/test_model_pytorch.py
