Skip to content

Commit 0e5305a

Browse files
mingxwamingxwa
committed
Attach environment information to benchmarking report
1 parent bc7221f commit 0e5305a

10 files changed

+151
-46
lines changed

‎.github/workflows/bvt-appleclang.yml

+6-4
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,16 @@ jobs:
2323
cmake -B build -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release
2424
cmake --build build -j
2525
ctest --test-dir build -j
26+
mkdir build/drop
27+
chmod +x tools/dump_build_env.sh
28+
./tools/dump_build_env.sh clang++ build/drop/env-info.json
2629
2730
- name: run benchmarks
2831
run: |
29-
cd build/benchmarks
30-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
32+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3133
3234
- name: archive benchmarking results
3335
uses: actions/upload-artifact@v4
3436
with:
35-
name: benchmarking-results-appleclang
36-
path: build/benchmarks/benchmarking-results.json
37+
name: drop-appleclang
38+
path: build/drop/

‎.github/workflows/bvt-clang.yml

+6-4
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,16 @@ jobs:
2626
cmake -B build -DCMAKE_C_COMPILER=clang-19 -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release
2727
cmake --build build -j
2828
ctest --test-dir build -j
29+
mkdir build/drop
30+
chmod +x tools/dump_build_env.sh
31+
./tools/dump_build_env.sh clang++-19 build/drop/env-info.json
2932
3033
- name: run benchmarks
3134
run: |
32-
cd build/benchmarks
33-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
35+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3436
3537
- name: archive benchmarking results
3638
uses: actions/upload-artifact@v4
3739
with:
38-
name: benchmarking-results-clang
39-
path: build/benchmarks/benchmarking-results.json
40+
name: drop-clang
41+
path: build/drop/

‎.github/workflows/bvt-gcc.yml

+6-4
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,16 @@ jobs:
2626
cmake -B build -DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release
2727
cmake --build build -j
2828
ctest --test-dir build -j
29+
mkdir build/drop
30+
chmod +x tools/dump_build_env.sh
31+
./tools/dump_build_env.sh g++-14 build/drop/env-info.json
2932
3033
- name: run benchmarks
3134
run: |
32-
cd build/benchmarks
33-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
35+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3436
3537
- name: archive benchmarking results
3638
uses: actions/upload-artifact@v4
3739
with:
38-
name: benchmarking-results-gcc
39-
path: build/benchmarks/benchmarking-results.json
40+
name: drop-gcc
41+
path: build/drop/

‎.github/workflows/bvt-msvc.yml

+8-4
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,23 @@ jobs:
1313
with:
1414
ref: ${{ inputs.branch }}
1515

16+
- name: add cl.exe to PATH
17+
uses: ilammy/msvc-dev-cmd@v1
18+
1619
- name: build and run test with MSVC
1720
run: |
1821
cmake -B build -DCMAKE_CXX_STANDARD=23
1922
cmake --build build --config Release -j
2023
ctest --test-dir build -j
24+
mkdir build\drop > $null
25+
.\tools\dump_build_env_msvc.ps1 -OutputPath build\drop\env-info.json
2126
2227
- name: run benchmarks
2328
run: |
24-
cd build\benchmarks
25-
.\Release\msft_proxy_benchmarks.exe --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
29+
build\benchmarks\Release\msft_proxy_benchmarks.exe --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build\drop\benchmarking-results.json
2630
2731
- name: archive benchmarking results
2832
uses: actions/upload-artifact@v4
2933
with:
30-
name: benchmarking-results-msvc
31-
path: build/benchmarks/benchmarking-results.json
34+
name: drop-msvc
35+
path: build/drop/

‎.github/workflows/bvt-nvhpc.yml

+6-4
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,16 @@ jobs:
2626
cmake -B build -DCMAKE_C_COMPILER=nvc -DCMAKE_CXX_COMPILER=nvc++ -DCMAKE_BUILD_TYPE=Release
2727
cmake --build build -j
2828
ctest --test-dir build -j
29+
mkdir build/drop
30+
chmod +x tools/dump_build_env.sh
31+
./tools/dump_build_env.sh nvc++ build/drop/env-info.json
2932
3033
- name: run benchmarks
3134
run: |
32-
cd build/benchmarks
33-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
35+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3436
3537
- name: archive benchmarking results
3638
uses: actions/upload-artifact@v4
3739
with:
38-
name: benchmarking-results-nvhpc
39-
path: build/benchmarks/benchmarking-results.json
40+
name: drop-nvhpc
41+
path: build/drop/

‎CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
cmake_minimum_required(VERSION 3.5)
1+
cmake_minimum_required(VERSION 3.10)
22

33
project(msft_proxy VERSION 3.2.1 LANGUAGES CXX)
44
add_library(msft_proxy INTERFACE)

‎tools/dump_build_env.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
if [ $# -ne 2 ]; then
4+
echo "Usage: $0 <compiler> <output path>"
5+
exit 1
6+
fi
7+
8+
COMPILER_INFO="$("$1" --version 2>/dev/null | awk 'NF {print; exit}')"
9+
if [ -z "$COMPILER_INFO" ]; then
10+
echo "Unable to retrieve compiler info for '$1'."
11+
exit 1
12+
fi
13+
14+
OS_NAME="$(uname -s)"
15+
if [ "$OS_NAME" = "Linux" ]; then
16+
if [ -f /etc/os-release ]; then
17+
OS_NAME="$(grep ^PRETTY_NAME= /etc/os-release | cut -d= -f2 | tr -d '"')"
18+
fi
19+
elif [ "$OS_NAME" = "Darwin" ]; then
20+
OS_NAME="$(sw_vers -productName) $(sw_vers -productVersion)"
21+
fi
22+
23+
cat <<EOF > $2
24+
{
25+
"OS": "$OS_NAME",
26+
"KernelVersion": "$(uname -r)",
27+
"Architecture": "$(uname -m)",
28+
"Compiler": "$COMPILER_INFO"
29+
}
30+
EOF

‎tools/dump_build_env_msvc.ps1

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[CmdletBinding()]
2+
param(
3+
[Parameter(Mandatory=$true)]
4+
[string]$OutputPath
5+
)
6+
7+
$osInfo = Get-CimInstance Win32_OperatingSystem
8+
cl /? | Out-String
9+
$compiler = ((cl /? 2>&1 | Out-String) -split '\r')[0] -replace '^cl\s:\s',''
10+
$jsonObject = [ordered]@{
11+
OS = $osInfo.Caption
12+
KernelVersion = $osInfo.Version
13+
Architecture = $env:PROCESSOR_ARCHITECTURE
14+
Compiler = $compiler
15+
} | ConvertTo-Json
16+
Set-Content -Path $OutputPath -Value $jsonObject

‎tools/report_generator/main.cpp

+57-15
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,63 @@
1313

1414
#include <nlohmann/json.hpp>
1515

16-
struct EnvironmentInfo {
16+
struct Environment {
1717
std::string Description;
18-
std::string Path;
18+
std::string InfoPath;
19+
std::string BenchmarkingResultsPath;
1920

20-
friend void to_json(nlohmann::json& j, const EnvironmentInfo& e) {
21+
friend void to_json(nlohmann::json& j, const Environment& e) {
2122
j = nlohmann::json{
2223
{"Description", e.Description},
23-
{"Path", e.Path}
24+
{"InfoPath", e.InfoPath},
25+
{"BenchmarkingResultsPath", e.BenchmarkingResultsPath},
2426
};
2527
}
2628

27-
friend void from_json(const nlohmann::json& j, EnvironmentInfo& e) {
29+
friend void from_json(const nlohmann::json& j, Environment& e) {
2830
j.at("Description").get_to(e.Description);
29-
j.at("Path").get_to(e.Path);
31+
j.at("InfoPath").get_to(e.InfoPath);
32+
j.at("BenchmarkingResultsPath").get_to(e.BenchmarkingResultsPath);
3033
}
3134
};
3235

33-
struct MetricInfo {
36+
struct EnvironmentInfo {
37+
std::string OS;
38+
std::string KernelVersion;
39+
std::string Architecture;
40+
std::string Compiler;
41+
42+
friend void to_json(nlohmann::json& j, const EnvironmentInfo& e) {
43+
j = nlohmann::json{
44+
{"OS", e.OS},
45+
{"KernelVersion", e.KernelVersion},
46+
{"Architecture", e.Architecture},
47+
{"Compiler", e.Compiler},
48+
};
49+
}
50+
51+
friend void from_json(const nlohmann::json& j, EnvironmentInfo& e) {
52+
j.at("OS").get_to(e.OS);
53+
j.at("KernelVersion").get_to(e.KernelVersion);
54+
j.at("Architecture").get_to(e.Architecture);
55+
j.at("Compiler").get_to(e.Compiler);
56+
}
57+
};
58+
59+
struct Metric {
3460
std::string Name;
3561
std::string TargetBenchmarkName;
3662
std::string BaselineBenchmarkName;
3763

38-
friend void to_json(nlohmann::json& j, const MetricInfo& m) {
64+
friend void to_json(nlohmann::json& j, const Metric& m) {
3965
j = nlohmann::json{
4066
{"Name", m.Name},
4167
{"TargetBenchmarkName", m.TargetBenchmarkName},
42-
{"BaselineBenchmarkName", m.BaselineBenchmarkName}
68+
{"BaselineBenchmarkName", m.BaselineBenchmarkName},
4369
};
4470
}
4571

46-
friend void from_json(const nlohmann::json& j, MetricInfo& m) {
72+
friend void from_json(const nlohmann::json& j, Metric& m) {
4773
j.at("Name").get_to(m.Name);
4874
j.at("TargetBenchmarkName").get_to(m.TargetBenchmarkName);
4975
j.at("BaselineBenchmarkName").get_to(m.BaselineBenchmarkName);
@@ -54,16 +80,16 @@ struct ReportConfig {
5480
std::string TargetName;
5581
double YellowIndicatorThreshold;
5682
std::string OutputPath;
57-
std::vector<EnvironmentInfo> Environments;
58-
std::vector<MetricInfo> Metrics;
83+
std::vector<Environment> Environments;
84+
std::vector<Metric> Metrics;
5985

6086
friend void to_json(nlohmann::json& j, const ReportConfig& rc) {
6187
j = nlohmann::json{
6288
{"TargetName", rc.TargetName},
6389
{"YellowIndicatorThreshold", rc.YellowIndicatorThreshold},
6490
{"OutputPath", rc.OutputPath},
6591
{"Environments", rc.Environments},
66-
{"Metrics", rc.Metrics}
92+
{"Metrics", rc.Metrics},
6793
};
6894
}
6995

@@ -78,7 +104,16 @@ struct ReportConfig {
78104

79105
const std::string_view MedianSuffix = "_median";
80106

81-
std::unordered_map<std::string, double> Parse(const std::filesystem::path& file) {
107+
EnvironmentInfo ParseEnvironmentInfo(const std::filesystem::path& file) {
108+
nlohmann::json obj;
109+
std::ifstream in;
110+
in.exceptions(std::ios_base::failbit | std::ios_base::badbit);
111+
in.open(file, std::ios_base::in | std::ios_base::binary);
112+
in >> obj;
113+
return obj.get<EnvironmentInfo>();
114+
}
115+
116+
std::unordered_map<std::string, double> ParseBenchmarkingResults(const std::filesystem::path& file) {
82117
nlohmann::json obj;
83118
{
84119
std::ifstream in;
@@ -111,7 +146,7 @@ void GenerateReport(const std::filesystem::path& config_path) {
111146
std::vector<std::unordered_map<std::string, double>> benchmarks;
112147
benchmarks.reserve(config.Environments.size());
113148
for (auto& environment : config.Environments) {
114-
benchmarks.push_back(Parse(environment.Path));
149+
benchmarks.push_back(ParseBenchmarkingResults(environment.BenchmarkingResultsPath));
115150
}
116151
std::ofstream out;
117152
out.exceptions(std::ios_base::failbit | std::ios_base::badbit);
@@ -155,6 +190,13 @@ void GenerateReport(const std::filesystem::path& config_path) {
155190
}
156191
out << "\n";
157192
}
193+
out << "\n## Environments\n\n";
194+
out << "| | Operating System | Kernel Version | Architecture | Compiler |\n";
195+
out << "| - | - | - | - | - |\n";
196+
for (auto& environment : config.Environments) {
197+
EnvironmentInfo env_info = ParseEnvironmentInfo(environment.InfoPath);
198+
out << "| **" << environment.Description << "** | " << env_info.OS << " | " << env_info.KernelVersion << " | " << env_info.Architecture << " | " << env_info.Compiler << " |\n";
199+
}
158200
}
159201

160202
int main(int argc, char** argv) {

‎tools/report_generator/report-config.json

+15-10
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,29 @@
44
"OutputPath": "benchmarking-report.md",
55
"Environments": [
66
{
7-
"Description": "MSVC on Windows Server 2022 (x64)",
8-
"Path": "artifacts/benchmarking-results-msvc/benchmarking-results.json"
7+
"Description": "MSVC on Windows",
8+
"InfoPath": "artifacts/drop-msvc/env-info.json",
9+
"BenchmarkingResultsPath": "artifacts/drop-msvc/benchmarking-results.json"
910
},
1011
{
11-
"Description": "GCC on Ubuntu 24.04 (x64)",
12-
"Path": "artifacts/benchmarking-results-gcc/benchmarking-results.json"
12+
"Description": "GCC on Ubuntu",
13+
"InfoPath": "artifacts/drop-gcc/env-info.json",
14+
"BenchmarkingResultsPath": "artifacts/drop-gcc/benchmarking-results.json"
1315
},
1416
{
15-
"Description": "Clang on Ubuntu 24.04 (x64)",
16-
"Path": "artifacts/benchmarking-results-clang/benchmarking-results.json"
17+
"Description": "Clang on Ubuntu",
18+
"InfoPath": "artifacts/drop-clang/env-info.json",
19+
"BenchmarkingResultsPath": "artifacts/drop-clang/benchmarking-results.json"
1720
},
1821
{
19-
"Description": "Apple Clang on macOS 15 (ARM64)",
20-
"Path": "artifacts/benchmarking-results-appleclang/benchmarking-results.json"
22+
"Description": "Apple Clang on macOS",
23+
"InfoPath": "artifacts/drop-appleclang/env-info.json",
24+
"BenchmarkingResultsPath": "artifacts/drop-appleclang/benchmarking-results.json"
2125
},
2226
{
23-
"Description": "NVIDIA HPC on Ubuntu 24.04 (x64)",
24-
"Path": "artifacts/benchmarking-results-nvhpc/benchmarking-results.json"
27+
"Description": "NVIDIA HPC on Ubuntu",
28+
"InfoPath": "artifacts/drop-nvhpc/env-info.json",
29+
"BenchmarkingResultsPath": "artifacts/drop-nvhpc/benchmarking-results.json"
2530
}
2631
],
2732
"Metrics": [

0 commit comments

Comments
 (0)