Skip to content

Commit 5875bb2

Browse files
nbelakovskiwpbonelli
nbelakovski
and
wpbonelli
authored
Update testing strategy (#51)
* separate testing and reporting mode via dynamic test matrix * test known supported combinations on push and pull request * test all combos and report compatibility on schedule and manual dispatch * add long_compat.csv to version control (easier dynamic matrix generation) * update compatibility files and readme to reflect recently added nvidia support * move test programs and compatibility files and scripts to subdirectories * introduce requirements.txt file for python dependencies * run scheduled testing weekly not monthly * update .gitignore Co-authored-by: w-bonelli <wbonelli@ucar.edu>
1 parent a01b254 commit 5875bb2

17 files changed

+419
-201
lines changed

‎.github/actions/test-cc/action.yml

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ runs:
3030
[[ "$ccv" == ${{ inputs.version }}* ]] && (echo "found ${{ env.CC }} version: $ccv") || (echo "unexpected ${{ env.CC }} version: $ccv"; exit 1)
3131
3232
- name: Test compile (bash)
33+
working-directory: test
3334
shell: bash
3435
run: |
3536
${{ env.CC }} -o hw hw.c

‎.github/actions/test-cxx/action.yml

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ runs:
3030
[[ "$cxxv" == ${{ matrix.toolchain.version }}* ]] && (echo "found ${{ env.CXX }} version: $cxxv") || (echo "unexpected ${{ env.CXX }} version: $cxxv"; exit 1)
3131
3232
- name: Test compile (bash)
33+
working-directory: test
3334
shell: bash
3435
run: |
3536
${{ env.CXX }} -o hw hw.cpp

‎.github/actions/test-fc/action.yml

+7-2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ runs:
3030
[[ "$fcv" == ${{ inputs.version }}* ]] && (echo "found ${{ env.FC }} version: $fcv") || (echo "unexpected ${{ env.FC }} version: $fcv"; exit 1)
3131
3232
- name: Test compile (bash)
33+
working-directory: test
3334
shell: bash
3435
run: |
3536
# macos-13/gfortran 7-9 compatibility workaround
@@ -42,13 +43,15 @@ runs:
4243
args="-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"
4344
fi
4445
fi
45-
46+
47+
# hello world program
4648
${{ env.FC }} $args -o hw hw.f90
4749
output=$(./hw '2>&1')
48-
[[ "$output" == *"hello world"* ]] && echo "$output" || (echo "Unexpected Fortran program output: $output"; exit 1)
50+
[[ "$output" == *"hello world"* ]] && echo "$output" || (echo "Unexpected Fortran program 'hw' output: $output"; exit 1)
4951
rm hw
5052
5153
- name: Test compile Fortran (pwsh)
54+
working-directory: test
5255
if: ${{ (success() || failure()) && runner.os == 'Windows' }}
5356
shell: pwsh
5457
run: |
@@ -63,6 +66,7 @@ runs:
6366
rm hw.exe
6467
6568
- name: Test compile Fortran (powershell)
69+
working-directory: test
6670
if: ${{ (success() || failure()) && runner.os == 'Windows' }}
6771
shell: powershell
6872
run: |
@@ -77,6 +81,7 @@ runs:
7781
rm hw.exe
7882
7983
- name: Test compile Fortran (cmd)
84+
working-directory: test
8085
if: ${{ (success() || failure()) && runner.os == 'Windows' }}
8186
shell: cmd
8287
run: |

‎.github/compat/compat.csv

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
compiler,gcc,gcc,gcc,gcc,gcc,gcc,gcc,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel-classic,intel,intel,intel,intel,intel,intel,intel,intel,intel,intel,intel,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc,nvidia-hpc
2+
version,10,11,12,13,7,8,9,2021.1,2021.10,2021.2,2021.3,2021.4,2021.5,2021.6,2021.7.1,2021.7,2021.8,2021.9,2021.1.2,2021.1.2,2021.1,2021.2,2021.4,2022.0,2022.1,2022.2.1,2022.2,2023.0,2023.1,2023.2,20.11,21.11,22.1,22.11,23.11,23.3,23.5,23.7,23.9
3+
runner,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
4+
macos-11,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,,,,,,,,,,,,,,,,,,,,,
5+
macos-12,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,,,,,,,,,,,,,,,,,,,,,
6+
macos-13,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,,,,,,,,,,,,,,,,,,,,,
7+
ubuntu-20.04,&check;,&check;,,&check;,&check;,&check;,&check;,&check;,&check;,&check;,,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;
8+
ubuntu-22.04,&check;,&check;,&check;,&check;,,,&check;,&check;,&check;,&check;,,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;,&check;
9+
windows-2019,&check;,&check;,&check;,&check;,,&check;,&check;,,&check;,,,,,&check;,,&check;,&check;,&check;,,,,,,,&check;,,&check;,&check;,&check;,&check;,,,,,,,,,
10+
windows-2022,&check;,&check;,&check;,&check;,,&check;,&check;,,&check;,,,,,&check;,,&check;,&check;,&check;,,,,,,,&check;,,&check;,&check;,&check;,&check;,,,,,,,,,

‎.github/compat/long_compat.csv

+209
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
runner,compiler,version,support
2+
macos-11,gcc,10,&check;
3+
macos-11,gcc,11,&check;
4+
macos-11,gcc,12,&check;
5+
macos-11,gcc,13,&check;
6+
macos-11,gcc,6,
7+
macos-11,gcc,7,&check;
8+
macos-11,gcc,8,&check;
9+
macos-11,gcc,9,&check;
10+
macos-11,intel-classic,2021.1.2,
11+
macos-11,intel-classic,2021.1,&check;
12+
macos-11,intel-classic,2021.10,&check;
13+
macos-11,intel-classic,2021.2,&check;
14+
macos-11,intel-classic,2021.3,&check;
15+
macos-11,intel-classic,2021.4,&check;
16+
macos-11,intel-classic,2021.5,&check;
17+
macos-11,intel-classic,2021.6,&check;
18+
macos-11,intel-classic,2021.7.1,&check;
19+
macos-11,intel-classic,2021.7,&check;
20+
macos-11,intel-classic,2021.8,&check;
21+
macos-11,intel-classic,2021.9,&check;
22+
macos-12,gcc,10,&check;
23+
macos-12,gcc,11,&check;
24+
macos-12,gcc,12,&check;
25+
macos-12,gcc,13,&check;
26+
macos-12,gcc,6,
27+
macos-12,gcc,7,&check;
28+
macos-12,gcc,8,&check;
29+
macos-12,gcc,9,&check;
30+
macos-12,intel-classic,2021.1.2,
31+
macos-12,intel-classic,2021.1,&check;
32+
macos-12,intel-classic,2021.10,&check;
33+
macos-12,intel-classic,2021.2,&check;
34+
macos-12,intel-classic,2021.3,&check;
35+
macos-12,intel-classic,2021.4,&check;
36+
macos-12,intel-classic,2021.5,&check;
37+
macos-12,intel-classic,2021.6,&check;
38+
macos-12,intel-classic,2021.7.1,&check;
39+
macos-12,intel-classic,2021.7,&check;
40+
macos-12,intel-classic,2021.8,&check;
41+
macos-12,intel-classic,2021.9,&check;
42+
macos-13,gcc,10,&check;
43+
macos-13,gcc,11,&check;
44+
macos-13,gcc,12,&check;
45+
macos-13,gcc,13,&check;
46+
macos-13,gcc,6,
47+
macos-13,gcc,7,&check;
48+
macos-13,gcc,8,&check;
49+
macos-13,gcc,9,&check;
50+
macos-13,intel-classic,2021.1.2,
51+
macos-13,intel-classic,2021.1,&check;
52+
macos-13,intel-classic,2021.10,&check;
53+
macos-13,intel-classic,2021.2,&check;
54+
macos-13,intel-classic,2021.3,&check;
55+
macos-13,intel-classic,2021.4,&check;
56+
macos-13,intel-classic,2021.5,&check;
57+
macos-13,intel-classic,2021.6,&check;
58+
macos-13,intel-classic,2021.7.1,&check;
59+
macos-13,intel-classic,2021.7,&check;
60+
macos-13,intel-classic,2021.8,&check;
61+
macos-13,intel-classic,2021.9,&check;
62+
ubuntu-20.04,gcc,10,&check;
63+
ubuntu-20.04,gcc,11,&check;
64+
ubuntu-20.04,gcc,12,
65+
ubuntu-20.04,gcc,13,&check;
66+
ubuntu-20.04,gcc,6,
67+
ubuntu-20.04,gcc,7,&check;
68+
ubuntu-20.04,gcc,8,&check;
69+
ubuntu-20.04,gcc,9,&check;
70+
ubuntu-20.04,intel-classic,2021.1.2,&check;
71+
ubuntu-20.04,intel-classic,2021.1,&check;
72+
ubuntu-20.04,intel-classic,2021.10,&check;
73+
ubuntu-20.04,intel-classic,2021.2,&check;
74+
ubuntu-20.04,intel-classic,2021.3,
75+
ubuntu-20.04,intel-classic,2021.4,&check;
76+
ubuntu-20.04,intel-classic,2021.5,&check;
77+
ubuntu-20.04,intel-classic,2021.6,&check;
78+
ubuntu-20.04,intel-classic,2021.7.1,&check;
79+
ubuntu-20.04,intel-classic,2021.7,&check;
80+
ubuntu-20.04,intel-classic,2021.8,&check;
81+
ubuntu-20.04,intel-classic,2021.9,&check;
82+
ubuntu-20.04,intel,2021.1.2,&check;
83+
ubuntu-20.04,intel,2021.1,&check;
84+
ubuntu-20.04,intel,2021.2,&check;
85+
ubuntu-20.04,intel,2021.4,&check;
86+
ubuntu-20.04,intel,2022.0,&check;
87+
ubuntu-20.04,intel,2022.1,&check;
88+
ubuntu-20.04,intel,2022.2.1,&check;
89+
ubuntu-20.04,intel,2022.2,&check;
90+
ubuntu-20.04,intel,2023.0,&check;
91+
ubuntu-20.04,intel,2023.1,&check;
92+
ubuntu-20.04,intel,2023.2,&check;
93+
ubuntu-20.04,nvidia-hpc,20.11,&check;
94+
ubuntu-20.04,nvidia-hpc,20.7,
95+
ubuntu-20.04,nvidia-hpc,20.9,
96+
ubuntu-20.04,nvidia-hpc,21.1,
97+
ubuntu-20.04,nvidia-hpc,21.11,&check;
98+
ubuntu-20.04,nvidia-hpc,22.1,&check;
99+
ubuntu-20.04,nvidia-hpc,22.11,&check;
100+
ubuntu-20.04,nvidia-hpc,23.11,&check;
101+
ubuntu-20.04,nvidia-hpc,23.3,&check;
102+
ubuntu-20.04,nvidia-hpc,23.5,&check;
103+
ubuntu-20.04,nvidia-hpc,23.7,&check;
104+
ubuntu-20.04,nvidia-hpc,23.9,&check;
105+
ubuntu-22.04,gcc,10,&check;
106+
ubuntu-22.04,gcc,11,&check;
107+
ubuntu-22.04,gcc,12,&check;
108+
ubuntu-22.04,gcc,13,&check;
109+
ubuntu-22.04,gcc,6,
110+
ubuntu-22.04,gcc,7,
111+
ubuntu-22.04,gcc,8,
112+
ubuntu-22.04,gcc,9,&check;
113+
ubuntu-22.04,intel-classic,2021.1.2,&check;
114+
ubuntu-22.04,intel-classic,2021.1,&check;
115+
ubuntu-22.04,intel-classic,2021.10,&check;
116+
ubuntu-22.04,intel-classic,2021.2,&check;
117+
ubuntu-22.04,intel-classic,2021.3,
118+
ubuntu-22.04,intel-classic,2021.4,&check;
119+
ubuntu-22.04,intel-classic,2021.5,&check;
120+
ubuntu-22.04,intel-classic,2021.6,&check;
121+
ubuntu-22.04,intel-classic,2021.7.1,&check;
122+
ubuntu-22.04,intel-classic,2021.7,&check;
123+
ubuntu-22.04,intel-classic,2021.8,&check;
124+
ubuntu-22.04,intel-classic,2021.9,&check;
125+
ubuntu-22.04,intel,2021.1.2,&check;
126+
ubuntu-22.04,intel,2021.1,&check;
127+
ubuntu-22.04,intel,2021.2,&check;
128+
ubuntu-22.04,intel,2021.4,&check;
129+
ubuntu-22.04,intel,2022.0,&check;
130+
ubuntu-22.04,intel,2022.1,&check;
131+
ubuntu-22.04,intel,2022.2.1,&check;
132+
ubuntu-22.04,intel,2022.2,&check;
133+
ubuntu-22.04,intel,2023.0,&check;
134+
ubuntu-22.04,intel,2023.1,&check;
135+
ubuntu-22.04,intel,2023.2,&check;
136+
ubuntu-22.04,nvidia-hpc,20.11,&check;
137+
ubuntu-22.04,nvidia-hpc,20.7,
138+
ubuntu-22.04,nvidia-hpc,20.9,
139+
ubuntu-22.04,nvidia-hpc,21.1,
140+
ubuntu-22.04,nvidia-hpc,21.11,&check;
141+
ubuntu-22.04,nvidia-hpc,22.1,&check;
142+
ubuntu-22.04,nvidia-hpc,22.11,&check;
143+
ubuntu-22.04,nvidia-hpc,23.11,&check;
144+
ubuntu-22.04,nvidia-hpc,23.3,&check;
145+
ubuntu-22.04,nvidia-hpc,23.5,&check;
146+
ubuntu-22.04,nvidia-hpc,23.7,&check;
147+
ubuntu-22.04,nvidia-hpc,23.9,&check;
148+
windows-2019,gcc,10,&check;
149+
windows-2019,gcc,11,&check;
150+
windows-2019,gcc,12,&check;
151+
windows-2019,gcc,13,&check;
152+
windows-2019,gcc,6,
153+
windows-2019,gcc,7,
154+
windows-2019,gcc,8,&check;
155+
windows-2019,gcc,9,&check;
156+
windows-2019,intel-classic,2021.1.2,
157+
windows-2019,intel-classic,2021.1,
158+
windows-2019,intel-classic,2021.10,&check;
159+
windows-2019,intel-classic,2021.2,
160+
windows-2019,intel-classic,2021.3,
161+
windows-2019,intel-classic,2021.4,
162+
windows-2019,intel-classic,2021.5,
163+
windows-2019,intel-classic,2021.6,&check;
164+
windows-2019,intel-classic,2021.7.1,
165+
windows-2019,intel-classic,2021.7,&check;
166+
windows-2019,intel-classic,2021.8,&check;
167+
windows-2019,intel-classic,2021.9,&check;
168+
windows-2019,intel,2021.1.2,
169+
windows-2019,intel,2021.1,
170+
windows-2019,intel,2021.2,
171+
windows-2019,intel,2021.4,
172+
windows-2019,intel,2022.0,
173+
windows-2019,intel,2022.1,&check;
174+
windows-2019,intel,2022.2.1,
175+
windows-2019,intel,2022.2,&check;
176+
windows-2019,intel,2023.0,&check;
177+
windows-2019,intel,2023.1,&check;
178+
windows-2019,intel,2023.2,&check;
179+
windows-2022,gcc,10,&check;
180+
windows-2022,gcc,11,&check;
181+
windows-2022,gcc,12,&check;
182+
windows-2022,gcc,13,&check;
183+
windows-2022,gcc,6,
184+
windows-2022,gcc,7,
185+
windows-2022,gcc,8,&check;
186+
windows-2022,gcc,9,&check;
187+
windows-2022,intel-classic,2021.1.2,
188+
windows-2022,intel-classic,2021.1,
189+
windows-2022,intel-classic,2021.10,&check;
190+
windows-2022,intel-classic,2021.2,
191+
windows-2022,intel-classic,2021.3,
192+
windows-2022,intel-classic,2021.4,
193+
windows-2022,intel-classic,2021.5,
194+
windows-2022,intel-classic,2021.6,&check;
195+
windows-2022,intel-classic,2021.7.1,
196+
windows-2022,intel-classic,2021.7,&check;
197+
windows-2022,intel-classic,2021.8,&check;
198+
windows-2022,intel-classic,2021.9,&check;
199+
windows-2022,intel,2021.1.2,
200+
windows-2022,intel,2021.1,
201+
windows-2022,intel,2021.2,
202+
windows-2022,intel,2021.4,
203+
windows-2022,intel,2022.0,
204+
windows-2022,intel,2022.1,&check;
205+
windows-2022,intel,2022.2.1,
206+
windows-2022,intel,2022.2,&check;
207+
windows-2022,intel,2023.0,&check;
208+
windows-2022,intel,2023.1,&check;
209+
windows-2022,intel,2023.2,&check;

‎.github/compat/matrix.yml

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
os:
2+
- ubuntu-22.04
3+
- ubuntu-20.04
4+
- macos-13
5+
- macos-12
6+
- macos-11
7+
- windows-2022
8+
- windows-2019
9+
toolchain:
10+
- {compiler: gcc, version: 13}
11+
- {compiler: gcc, version: 12}
12+
- {compiler: gcc, version: 11}
13+
- {compiler: gcc, version: 10}
14+
- {compiler: gcc, version: 9}
15+
- {compiler: gcc, version: 8}
16+
- {compiler: gcc, version: 7}
17+
- {compiler: gcc, version: 6}
18+
- {compiler: intel, version: '2023.2'}
19+
- {compiler: intel, version: '2023.1'}
20+
- {compiler: intel, version: '2023.0'}
21+
- {compiler: intel, version: '2022.2.1'}
22+
- {compiler: intel, version: '2022.2'}
23+
- {compiler: intel, version: '2022.1'}
24+
- {compiler: intel, version: '2022.0'}
25+
- {compiler: intel, version: '2021.4'}
26+
- {compiler: intel, version: '2021.2'}
27+
- {compiler: intel, version: '2021.1.2'}
28+
- {compiler: intel, version: '2021.1'}
29+
- {compiler: intel-classic, version: '2021.10'}
30+
- {compiler: intel-classic, version: '2021.9'}
31+
- {compiler: intel-classic, version: '2021.8'}
32+
- {compiler: intel-classic, version: '2021.7.1'}
33+
- {compiler: intel-classic, version: '2021.7'}
34+
- {compiler: intel-classic, version: '2021.6'}
35+
- {compiler: intel-classic, version: '2021.5'}
36+
- {compiler: intel-classic, version: '2021.4'}
37+
- {compiler: intel-classic, version: '2021.3'}
38+
- {compiler: intel-classic, version: '2021.2'}
39+
- {compiler: intel-classic, version: '2021.1.2'}
40+
- {compiler: intel-classic, version: '2021.1'}
41+
- {compiler: nvidia-hpc, version: '23.11'}
42+
- {compiler: nvidia-hpc, version: '23.9'}
43+
- {compiler: nvidia-hpc, version: '23.7'}
44+
- {compiler: nvidia-hpc, version: '23.5'}
45+
- {compiler: nvidia-hpc, version: '23.3'}
46+
- {compiler: nvidia-hpc, version: '23.1'}
47+
- {compiler: nvidia-hpc, version: '22.11'}
48+
- {compiler: nvidia-hpc, version: '22.9'}
49+
- {compiler: nvidia-hpc, version: '22.7'}
50+
- {compiler: nvidia-hpc, version: '22.5'}
51+
- {compiler: nvidia-hpc, version: '22.3'}
52+
- {compiler: nvidia-hpc, version: '22.2'}
53+
- {compiler: nvidia-hpc, version: '22.1'}
54+
- {compiler: nvidia-hpc, version: '21.11'}
55+
- {compiler: nvidia-hpc, version: '21.9'}
56+
- {compiler: nvidia-hpc, version: '21.7'}
57+
- {compiler: nvidia-hpc, version: '21.5'}
58+
- {compiler: nvidia-hpc, version: '21.3'}
59+
- {compiler: nvidia-hpc, version: '21.2'}
60+
- {compiler: nvidia-hpc, version: '21.1'}
61+
- {compiler: nvidia-hpc, version: '20.11'}
62+
- {compiler: nvidia-hpc, version: '20.9'}
63+
- {compiler: nvidia-hpc, version: '20.7'}
64+
exclude:
65+
# ifx not available for mac
66+
- os: macos-13
67+
toolchain: {compiler: intel}
68+
- os: macos-12
69+
toolchain: {compiler: intel}
70+
- os: macos-11
71+
toolchain: {compiler: intel}
72+
# nvidia-hpc not available for mac
73+
- os: macos-13
74+
toolchain: {compiler: nvidia-hpc}
75+
- os: macos-12
76+
toolchain: {compiler: nvidia-hpc}
77+
- os: macos-11
78+
toolchain: {compiler: nvidia-hpc}
79+
# nvidia-hpc not available for windows
80+
- os: windows-2022
81+
toolchain: {compiler: nvidia-hpc}
82+
- os: windows-2019
83+
toolchain: {compiler: nvidia-hpc}

‎.github/compat/matrix_json_from_csv.py

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import csv
2+
import json
3+
import sys
4+
from pathlib import Path
5+
6+
csv_path = Path(sys.argv[1]) # path to CSV file
7+
jsn_path = Path(sys.argv[2]) # path to JSON file
8+
9+
include = []
10+
11+
with open(csv_path, "r") as csv_file:
12+
reader = csv.DictReader(csv_file)
13+
for row in reader:
14+
if not any(row["support"].strip()):
15+
continue
16+
include.append(
17+
{
18+
"os": row["runner"],
19+
"toolchain": {"compiler": row["compiler"], "version": row["version"]},
20+
}
21+
)
22+
23+
with open(jsn_path, "w") as jsn_file:
24+
json.dump({"include": include}, jsn_file)

‎update_compat_table.py ‎.github/compat/update_compat_table.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@
1919
r"<!\-\- compat starts \-\->.*<!\-\- compat ends \-\->",
2020
re.DOTALL,
2121
)
22-
ct = '<!-- compat starts -->{}<!-- compat ends -->'.format('\n{}\n'.format(table))
22+
ct = "<!-- compat starts -->{}<!-- compat ends -->".format("\n{}\n".format(table))
2323
readme = update_path.open().read()
2424
update_path.open("w").write(r.sub(ct, readme))

0 commit comments

Comments
 (0)