Make benchmarks compiling in clang-cl (#5533)

AlexGuteniev · StephanTLavavej · web-flow · commit 7381f2fef61e · 2025-06-14T00:03:16.000-07:00
Co-authored-by: Stephan T. Lavavej &lt;stl@microsoft.com&gt;
diff --git a/azure-devops/build-and-test.yml b/azure-devops/build-and-test.yml
@@ -58,8 +58,21 @@ jobs:
       targetPlatform: ${{ parameters.targetPlatform }}
       analyzeBuild: ${{ parameters.analyzeBuild }}
       asanBuild: ${{ parameters.asanBuild }}
-      buildBenchmarks: ${{ parameters.buildBenchmarks }}
       testsBuildOnly: ${{ parameters.testsBuildOnly }}
+  - template: build-benchmarks.yml
+    parameters:
+      hostArch: ${{ parameters.hostArch }}
+      targetArch: ${{ parameters.targetArch }}
+      targetPlatform: ${{ parameters.targetPlatform }}
+      buildBenchmarks: ${{ parameters.buildBenchmarks }}
+      compiler: cl
+  - template: build-benchmarks.yml
+    parameters:
+      hostArch: ${{ parameters.hostArch }}
+      targetArch: ${{ parameters.targetArch }}
+      targetPlatform: ${{ parameters.targetPlatform }}
+      buildBenchmarks: ${{ parameters.buildBenchmarks }}
+      compiler: clang-cl
   - template: run-tests.yml
     parameters:
       hostArch: ${{ parameters.hostArch }}
diff --git a/azure-devops/build-benchmarks.yml b/azure-devops/build-benchmarks.yml
@@ -0,0 +1,46 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+parameters:
+- name: hostArch
+  type: string
+- name: targetArch
+  type: string
+- name: targetPlatform
+  type: string
+- name: buildBenchmarks
+  type: boolean
+- name: compiler
+  type: string
+
+steps:
+- script: |
+    if exist "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" (
+      rmdir /S /Q "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}"
+    )
+    call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
+    -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
+    cmake -G Ninja ^
+    -DCMAKE_CXX_COMPILER=${{ parameters.compiler }} ^
+    -DCMAKE_BUILD_TYPE=Release ^
+    -DSTL_BINARY_DIR="$(buildOutputLocation)" ^
+    -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ^
+    -S $(Build.SourcesDirectory)/benchmarks -B "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}"
+  displayName: 'Configure the benchmarks for ${{ parameters.compiler }}'
+  timeoutInMinutes: 2
+  env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
+  # TRANSITION, we currently don't build the benchmarks with Clang for ARM64 or ARM64EC
+  condition: >
+    and(succeeded(), ${{ parameters.buildBenchmarks }},
+      not(and(eq('${{ parameters.compiler }}', 'clang-cl'), startsWith('${{ parameters.targetPlatform }}', 'arm64'))))
+- script: |
+    call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
+    -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
+    cmake --build "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}"
+  displayName: 'Build the benchmarks for ${{ parameters.compiler }}'
+  timeoutInMinutes: 2
+  env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
+  # TRANSITION, we currently don't build the benchmarks with Clang for ARM64 or ARM64EC
+  condition: >
+    and(succeeded(), ${{ parameters.buildBenchmarks }},
+      not(and(eq('${{ parameters.compiler }}', 'clang-cl'), startsWith('${{ parameters.targetPlatform }}', 'arm64'))))
diff --git a/azure-devops/cmake-configure-build.yml b/azure-devops/cmake-configure-build.yml
@@ -12,8 +12,6 @@ parameters:
   type: boolean
 - name: asanBuild
   type: boolean
-- name: buildBenchmarks
-  type: boolean
 - name: testsBuildOnly
   type: boolean
 - name: litFlags
@@ -50,27 +48,3 @@ steps:
   displayName: 'Build the STL'
   timeoutInMinutes: 5
   env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
-- script: |
-    if exist "$(benchmarkBuildOutputLocation)" (
-      rmdir /S /Q "$(benchmarkBuildOutputLocation)"
-    )
-    call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
-    -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
-    cmake -G Ninja ^
-    -DCMAKE_CXX_COMPILER=cl ^
-    -DCMAKE_BUILD_TYPE=Release ^
-    -DSTL_BINARY_DIR="$(buildOutputLocation)" ^
-    -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ^
-    -S $(Build.SourcesDirectory)/benchmarks -B "$(benchmarkBuildOutputLocation)"
-  displayName: 'Configure the benchmarks'
-  timeoutInMinutes: 2
-  env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
-  condition: and(succeeded(), ${{ parameters.buildBenchmarks }})
-- script: |
-    call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^
-    -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo
-    cmake --build "$(benchmarkBuildOutputLocation)"
-  displayName: 'Build the benchmarks'
-  timeoutInMinutes: 2
-  env: { TMP: $(tmpDir), TEMP: $(tmpDir) }
-  condition: and(succeeded(), ${{ parameters.buildBenchmarks }})
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
@@ -57,7 +57,8 @@ set(CMAKE_MSVC_RUNTIME_LIBRARY "${STL_BENCHMARK_MSVC_RUNTIME_LIBRARY}")
 set(CMAKE_BUILD_TYPE Release)
 
 # /utf-8 affects <format>.
-add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:/Zi;/nologo;/diagnostics:caret;/W4;/WX;/w14265;/w15038;/w15262;/utf-8;/Zc:preprocessor>")
+add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:/Zi;/nologo;/diagnostics:caret;/W4;/WX;/w14265;/w15038;/w15262;/utf-8>")
+add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,MSVC>:/Zc:preprocessor>") # TRANSITION, LLVM-48220 clang-cl: ignore /Zc:preprocessor
 
 add_link_options("/DEBUG")
 
diff --git a/benchmarks/src/bitset_from_string.cpp b/benchmarks/src/bitset_from_string.cpp
@@ -42,7 +42,7 @@ const auto random_digits = random_digits_init<N, charT, 2048>();
 
 template <length_type Length, size_t N, class charT>
 void bitset_from_string(benchmark::State& state) {
-    const auto& digit_array = random_digits<N, charT>;
+    auto digit_array = random_digits<N, charT>;
     for (auto _ : state) {
         benchmark::DoNotOptimize(digit_array);
         const auto arr_data = digit_array.data();
diff --git a/benchmarks/src/bitset_to_string.cpp b/benchmarks/src/bitset_to_string.cpp
@@ -30,10 +30,11 @@ void BM_bitset_to_string(benchmark::State& state) {
     static_assert(N <= 64);
 
     for (auto _ : state) {
-        for (const auto& bits : random_bits<>) {
+        // make a copy, so that it can be potentially modified by DoNotOptimize
+        for (auto bits : random_bits<>) {
             benchmark::DoNotOptimize(bits);
             bitset<N> bs{bits};
-            benchmark::DoNotOptimize(bs.to_string<charT>());
+            benchmark::DoNotOptimize(bs.template to_string<charT>());
         }
     }
 }
@@ -43,10 +44,10 @@ void BM_bitset_to_string_large_single(benchmark::State& state) {
     static_assert(N % 64 == 0 && N >= 64);
     const auto& bitset_data = random_bits<N / 64>;
 
-    const auto large_bitset = bit_cast<bitset<N>>(bitset_data);
+    auto large_bitset = bit_cast<bitset<N>>(bitset_data);
     for (auto _ : state) {
         benchmark::DoNotOptimize(large_bitset);
-        benchmark::DoNotOptimize(large_bitset.to_string<charT>());
+        benchmark::DoNotOptimize(large_bitset.template to_string<charT>());
     }
 }
 
diff --git a/benchmarks/src/filesystem.cpp b/benchmarks/src/filesystem.cpp
@@ -6,12 +6,12 @@
 #include <system_error>
 
 void symlink_status(benchmark::State& state) {
-    const auto path = std::filesystem::temp_directory_path();
+    auto path = std::filesystem::temp_directory_path();
 
     for (auto _ : state) {
         std::error_code ec;
         benchmark::DoNotOptimize(path);
-        const auto status = std::filesystem::symlink_status(path, ec);
+        auto status = std::filesystem::symlink_status(path, ec);
         benchmark::DoNotOptimize(status);
         benchmark::DoNotOptimize(ec);
     }
diff --git a/benchmarks/src/has_single_bit.cpp b/benchmarks/src/has_single_bit.cpp
@@ -11,7 +11,7 @@ using namespace std;
 
 template <typename T>
 void bm_has_single_bit_if(benchmark::State& state) {
-    const auto random_v = random_vector<T>(8);
+    auto random_v = random_vector<T>(8);
     for (auto _ : state) {
         benchmark::DoNotOptimize(random_v);
         unsigned int count_true  = 0;
@@ -28,7 +28,7 @@ void bm_has_single_bit_if(benchmark::State& state) {
 
 template <typename T>
 void bm_has_single_bit(benchmark::State& state) {
-    const auto random_v = random_vector<T>(8);
+    auto random_v = random_vector<T>(8);
     for (auto _ : state) {
         benchmark::DoNotOptimize(random_v);
         unsigned int r = 0;
diff --git a/benchmarks/src/locale_classic.cpp b/benchmarks/src/locale_classic.cpp
@@ -8,7 +8,8 @@ using namespace std;
 // GH-3048 <locale>: Double-checked locking for locale::classic
 void BM_locale_classic(benchmark::State& state) {
     for (auto _ : state) {
-        benchmark::DoNotOptimize(locale::classic());
+        auto v = locale::classic();
+        benchmark::DoNotOptimize(v);
     }
 }
 BENCHMARK(BM_locale_classic);
diff --git a/benchmarks/src/move_only_function.cpp b/benchmarks/src/move_only_function.cpp
@@ -5,6 +5,10 @@
 #include <functional>
 #include <utility>
 
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wunqualified-std-cast-call"
+#endif // defined(__clang__)
+
 using namespace std;
 
 void mof_none(benchmark::State& state) {
diff --git a/benchmarks/src/priority_queue_push_range.cpp b/benchmarks/src/priority_queue_push_range.cpp
@@ -5,7 +5,6 @@
 #include <benchmark/benchmark.h>
 #include <cstddef>
 #include <cstdint>
-#include <cstdio>
 #include <queue>
 #include <random>
 #include <span>
@@ -57,31 +56,20 @@ void BM_push_range(benchmark::State& state) {
     }
 }
 
-template <size_t L>
-void putln(const benchmark::State&) {
-    static bool b = [] {
-        puts("");
-        return true;
-    }();
+void common_args(auto bm) {
+    bm->RangeMultiplier(100)->Range(1, vec_size)->Arg(vec_size / 2 + 1);
 }
 
-#define TEST_PUSH_RANGE(T, source)      \
-    BENCHMARK(BM_push_range<T, source>) \
-        ->Setup(putln<__LINE__>)        \
-        ->RangeMultiplier(100)          \
-        ->Range(1, vec_size)            \
-        ->Arg(vec_size / 2 + 1);
+BENCHMARK(BM_push_range<uint8_t, vec_u8>)->Apply(common_args);
+BENCHMARK(BM_push_range<uint16_t, vec_u16>)->Apply(common_args);
+BENCHMARK(BM_push_range<uint32_t, vec_u32>)->Apply(common_args);
+BENCHMARK(BM_push_range<uint64_t, vec_u64>)->Apply(common_args);
+BENCHMARK(BM_push_range<float, vec_float>)->Apply(common_args);
+BENCHMARK(BM_push_range<double, vec_double>)->Apply(common_args);
 
-TEST_PUSH_RANGE(uint8_t, vec_u8);
-TEST_PUSH_RANGE(uint16_t, vec_u16);
-TEST_PUSH_RANGE(uint32_t, vec_u32);
-TEST_PUSH_RANGE(uint64_t, vec_u64);
-TEST_PUSH_RANGE(float, vec_float);
-TEST_PUSH_RANGE(double, vec_double);
-
-TEST_PUSH_RANGE(string_view, vec_str);
-TEST_PUSH_RANGE(string, vec_str);
-TEST_PUSH_RANGE(wstring_view, vec_wstr);
-TEST_PUSH_RANGE(wstring, vec_wstr);
+BENCHMARK(BM_push_range<string_view, vec_str>)->Apply(common_args);
+BENCHMARK(BM_push_range<string, vec_str>)->Apply(common_args);
+BENCHMARK(BM_push_range<wstring_view, vec_wstr>)->Apply(common_args);
+BENCHMARK(BM_push_range<wstring, vec_wstr>)->Apply(common_args);
 
 BENCHMARK_MAIN();
diff --git a/benchmarks/src/search.cpp b/benchmarks/src/search.cpp
@@ -32,7 +32,8 @@ template <size_t Size, bool Last_is_different>
 constexpr std::array fill_pattern_array = make_fill_pattern_array<Size, Last_is_different>();
 
 template <size_t Size, bool Last_is_different>
-constexpr std::string_view fill_pattern_view = fill_pattern_array<Size, Last_is_different>;
+constexpr std::string_view fill_pattern_view{
+    fill_pattern_array<Size, Last_is_different>.data(), fill_pattern_array<Size, Last_is_different>.size()};
 
 struct data_and_pattern {
     std::string_view data;
@@ -60,8 +61,8 @@ void c_strstr(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const not_highly_aligned_string haystack(src_haystack);
-    const not_highly_aligned_string needle(src_needle);
+    not_highly_aligned_string haystack(src_haystack);
+    not_highly_aligned_string needle(src_needle);
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -76,8 +77,8 @@ void classic_search(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
-    const std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
+    std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
+    std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -92,8 +93,8 @@ void ranges_search(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
-    const std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
+    std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
+    std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -108,8 +109,8 @@ void search_default_searcher(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
-    const std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
+    std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
+    std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -124,8 +125,8 @@ void member_find(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const T haystack(src_haystack.begin(), src_haystack.end());
-    const T needle(src_needle.begin(), src_needle.end());
+    T haystack(src_haystack.begin(), src_haystack.end());
+    T needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -140,8 +141,8 @@ void classic_find_end(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
-    const std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
+    std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
+    std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -156,8 +157,8 @@ void ranges_find_end(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
-    const std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
+    std::vector<T, not_highly_aligned_allocator<T>> haystack(src_haystack.begin(), src_haystack.end());
+    std::vector<T, not_highly_aligned_allocator<T>> needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
@@ -172,8 +173,8 @@ void member_rfind(benchmark::State& state) {
     const auto& src_haystack = patterns[static_cast<size_t>(state.range())].data;
     const auto& src_needle   = patterns[static_cast<size_t>(state.range())].pattern;
 
-    const T haystack(src_haystack.begin(), src_haystack.end());
-    const T needle(src_needle.begin(), src_needle.end());
+    T haystack(src_haystack.begin(), src_haystack.end());
+    T needle(src_needle.begin(), src_needle.end());
 
     for (auto _ : state) {
         benchmark::DoNotOptimize(haystack);
diff --git a/benchmarks/src/sv_equal.cpp b/benchmarks/src/sv_equal.cpp
@@ -31,7 +31,7 @@ constexpr std::array<std::string_view, Count> make_svs() {
 
 template <std::size_t Length>
 void sv_equal(benchmark::State& state) {
-    constexpr auto arr = make_svs<Length>();
+    auto arr = make_svs<Length>();
     benchmark::DoNotOptimize(arr);
 
     for (auto _ : state) {

Original file line number	Diff line number	Diff line change
`@@ -30,10 +30,11 @@ void BM_bitset_to_string(benchmark::State& state) {`
`30`	`30`	`static_assert(N <= 64);`
`31`	`31`
`32`	`32`	`for (auto _ : state) {`
`33`		`- for (const auto& bits : random_bits<>) {`
	`33`	`+ // make a copy, so that it can be potentially modified by DoNotOptimize`
	`34`	`+ for (auto bits : random_bits<>) {`
`34`	`35`	`benchmark::DoNotOptimize(bits);`
`35`	`36`	`bitset<N> bs{bits};`
`36`		`- benchmark::DoNotOptimize(bs.to_string<charT>());`
	`37`	`+ benchmark::DoNotOptimize(bs.template to_string<charT>());`
`37`	`38`	`}`
`38`	`39`	`}`
`39`	`40`	`}`
`@@ -43,10 +44,10 @@ void BM_bitset_to_string_large_single(benchmark::State& state) {`
`43`	`44`	`static_assert(N % 64 == 0 && N >= 64);`
`44`	`45`	`const auto& bitset_data = random_bits<N / 64>;`
`45`	`46`
`46`		`- const auto large_bitset = bit_cast<bitset<N>>(bitset_data);`
	`47`	`+ auto large_bitset = bit_cast<bitset<N>>(bitset_data);`
`47`	`48`	`for (auto _ : state) {`
`48`	`49`	`benchmark::DoNotOptimize(large_bitset);`
`49`		`- benchmark::DoNotOptimize(large_bitset.to_string<charT>());`
	`50`	`+ benchmark::DoNotOptimize(large_bitset.template to_string<charT>());`
`50`	`51`	`}`
`51`	`52`	`}`
`52`	`53`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,8 @@ using namespace std;`
`8`	`8`	`// GH-3048 <locale>: Double-checked locking for locale::classic`
`9`	`9`	`void BM_locale_classic(benchmark::State& state) {`
`10`	`10`	`for (auto _ : state) {`
`11`		`- benchmark::DoNotOptimize(locale::classic());`
	`11`	`+ auto v = locale::classic();`
	`12`	`+ benchmark::DoNotOptimize(v);`
`12`	`13`	`}`
`13`	`14`	`}`
`14`	`15`	`BENCHMARK(BM_locale_classic);`