Skip to content

Commit 7c189da

Browse files
lukaszstolarczuklukaszstolarczuk
authored
Merge pull request #5558 from tszczyp/CI_GHA_PKG_python_scripts
common: import python scripts for PKG tests
2 parents b74832a + a7d7f40 commit 7c189da

File tree

4 files changed

+766
-0
lines changed

4 files changed

+766
-0
lines changed

‎utils/gha-runners/scripts/cleanup-packages.py

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!usr/bin/env python3
2+
#
3+
# SPDX-License-Identifier: BSD-3-Clause
4+
# Copyright 2020-2023, Intel Corporation
5+
6+
"""
7+
This module includes functions which clean up environment system after
8+
installation of rpm packages from PMDK library.
9+
"""
10+
11+
from os import listdir, path, linesep
12+
import distro
13+
from subprocess import check_output
14+
from argparse import ArgumentParser
15+
import re
16+
import json
17+
from pathlib import Path
18+
19+
PMDK_VERSION = ''
20+
SYSTEM_ARCHITECTURE = ''
21+
22+
23+
def get_package_version_and_system_architecture(pmdk_path):
24+
"""
25+
Returns packages version and system architecture from names of directories
26+
from rpm directory.
27+
"""
28+
version = ''
29+
architecture = ''
30+
os_distro=distro.id()
31+
32+
# first check if there is a json file with version and architecture
33+
pkg_version_file_path = Path(pmdk_path).joinpath("pkgVersion.json")
34+
if pkg_version_file_path.exists() and pkg_version_file_path.is_file():
35+
with open(pkg_version_file_path) as pkg_version_file:
36+
version_from_file = {}
37+
try:
38+
version_from_file = json.load(pkg_version_file)
39+
version = version_from_file.get('PMDK_VERSION', '')
40+
architecture = version_from_file.get('SYSTEM_ARCHITECTURE', '')
41+
except:
42+
pass
43+
44+
# if cannot read values from json file, read them from rpms:
45+
if version == '' or architecture == '':
46+
if os_distro == 'fedora' or os_distro == "rhel":
47+
pkg_directory = path.join(pmdk_path, 'rpm')
48+
for elem in listdir(pkg_directory):
49+
if '.src.rpm' in elem:
50+
version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1)
51+
else:
52+
architecture = elem
53+
54+
elif os_distro == 'ubuntu':
55+
pkg_directory = path.join(pmdk_path, 'dpkg')
56+
for elem in listdir(pkg_directory):
57+
if '.changes' in elem:
58+
# looks for the version number of dpkg package in dpkg package name
59+
version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1)
60+
architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2)
61+
62+
return version, architecture
63+
64+
65+
def remove_install_rpm_packages(pmdk_path):
66+
"""
67+
Removes binaries installed from packages from PMDK library.
68+
"""
69+
try:
70+
rpm_to_uninstall = check_output(
71+
'rpm -qa | grep ' + PMDK_VERSION, cwd=pmdk_path, shell=True)
72+
pkg_to_uninstall = rpm_to_uninstall.decode(
73+
'UTF-8').replace(linesep, ' ')
74+
except:
75+
pkg_to_uninstall = ''
76+
77+
if pkg_to_uninstall:
78+
check_output('rpm -e ' + pkg_to_uninstall, cwd=pmdk_path, shell=True)
79+
80+
81+
def remove_install_dpkg_packages(pmdk_path):
82+
"""
83+
Removes binaries installed from packages from PMDK library.
84+
"""
85+
try:
86+
dpkg_to_uninstall = check_output(
87+
'dpkg-query --show | grep ' + PMDK_VERSION + ''' | awk '{print $1}' ''', cwd=pmdk_path, shell=True)
88+
pkg_to_uninstall = dpkg_to_uninstall.decode(
89+
'UTF-8').replace(linesep, ' ')
90+
except:
91+
pkg_to_uninstall = ''
92+
93+
if pkg_to_uninstall:
94+
check_output('dpkg -r ' + pkg_to_uninstall, cwd=pmdk_path, shell=True)
95+
96+
97+
if __name__ == '__main__':
98+
parser = ArgumentParser(
99+
description='''Clean up system environment after installation rpm
100+
packages from PMDK library''')
101+
parser.add_argument("-r", "--pmdk-path", required=True,
102+
help="the PMDK library root path.")
103+
104+
args = parser.parse_args()
105+
PMDK_VERSION, SYSTEM_ARCHITECTURE =\
106+
get_package_version_and_system_architecture(args.pmdk_path)
107+
108+
os_distro=distro.id()
109+
if os_distro == 'fedora' or os_distro == "rhel":
110+
remove_install_rpm_packages(args.pmdk_path)
111+
elif os_distro == 'ubuntu':
112+
remove_install_dpkg_packages(args.pmdk_path)

‎utils/gha-runners/scripts/install-packages.py

+189
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
#!usr/bin/env python3
2+
#
3+
# SPDX-License-Identifier: BSD-3-Clause
4+
# Copyright 2020-2023, Intel Corporation
5+
6+
7+
"""
8+
This module includes functions which install packages from PMDK library.
9+
"""
10+
11+
from os import listdir, path, linesep
12+
from subprocess import check_output
13+
from argparse import ArgumentParser
14+
import distro
15+
import re
16+
import json
17+
18+
PMDK_VERSION = ''
19+
SYSTEM_ARCHITECTURE = ''
20+
21+
def get_package_version_and_system_architecture(pmdk_path):
22+
"""
23+
Returns packages version and system architecture from names of directories
24+
from packages directory.
25+
"""
26+
os_distro=distro.id()
27+
if os_distro == 'fedora' or os_distro == "rhel":
28+
pkg_directory = path.join(pmdk_path, 'rpm')
29+
elif os_distro == 'ubuntu':
30+
pkg_directory = path.join(pmdk_path, 'dpkg')
31+
32+
version = ''
33+
architecture = ''
34+
for elem in listdir(pkg_directory):
35+
if os_distro == 'fedora' or os_distro == "rhel":
36+
if '.src.rpm' in elem:
37+
# looks for the version number of package in package name
38+
version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1)
39+
else:
40+
architecture = elem
41+
42+
elif os_distro == 'ubuntu':
43+
if '.changes' in elem:
44+
# looks for the version number of packages in package name
45+
version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1)
46+
architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2)
47+
return version, architecture
48+
49+
50+
def save_pkg_version(pkg_version_file_path):
51+
values_to_save = {
52+
'PMDK_VERSION': PMDK_VERSION,
53+
'SYSTEM_ARCHITECTURE': SYSTEM_ARCHITECTURE
54+
}
55+
with open(pkg_version_file_path, 'w') as file:
56+
json.dump(values_to_save, file, indent=2)
57+
58+
59+
def get_built_rpm_packages(pmdk_path):
60+
"""
61+
Returns built pkg packages from pkg directory.
62+
"""
63+
path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE)
64+
packages = listdir(path_to_rpm_files)
65+
return packages
66+
67+
68+
def get_built_dpkg_packages(pmdk_path):
69+
"""
70+
Returns built pkg packages from pkg directory.
71+
"""
72+
path_to_dpkg_files = path.join(pmdk_path, 'dpkg')
73+
packages =''
74+
for elem in listdir(path_to_dpkg_files):
75+
if '.deb' in elem:
76+
packages += elem + ' '
77+
78+
return packages
79+
80+
81+
def install_rpm_packages(pmdk_path):
82+
"""
83+
Install packages from PMDK library.
84+
"""
85+
packages = ' '.join(get_built_rpm_packages(pmdk_path))
86+
path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE)
87+
check_output('rpm -i --nodeps ' + packages,
88+
cwd=path_to_rpm_files, shell=True)
89+
90+
91+
def install_dpkg_packages(pmdk_path):
92+
"""
93+
Install packages from PMDK library.
94+
"""
95+
packages = get_built_dpkg_packages(pmdk_path)
96+
path_to_dpkg_files = path.join(pmdk_path, 'dpkg')
97+
check_output('dpkg -i ' + packages,
98+
cwd=path_to_dpkg_files, shell=True)
99+
100+
101+
def get_names_of_rpm_content():
102+
"""
103+
Returns names of elements, for which are installed from packages from PMDK
104+
library.
105+
"""
106+
packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE)
107+
installed_packages = check_output(
108+
'ls | grep ' + PMDK_VERSION, cwd=packages_path, shell=True)
109+
delimiter = '-'
110+
111+
installed_packages = installed_packages.decode(
112+
'UTF-8').split(linesep)
113+
114+
libraries_names = [item.split(
115+
'-')[0] for item in installed_packages if item.split('-')[0]]
116+
return set(libraries_names)
117+
118+
119+
def get_names_of_dpkg_content():
120+
"""
121+
Returns names of elements, for which are installed from packages from PMDK
122+
library.
123+
"""
124+
packages_path = path.join(args.pmdk_path, 'dpkg')
125+
installed_packages = check_output(
126+
'ls | grep ' + PMDK_VERSION + ' | grep .deb', cwd=packages_path, shell=True)
127+
128+
installed_packages = installed_packages.decode(
129+
'UTF-8').split(linesep)
130+
131+
libraries_names = [item.split(
132+
'_')[0] for item in installed_packages if item.split('-')[0]]
133+
return set(libraries_names)
134+
135+
136+
def get_installed_packages(so_path, split_param, packages_path):
137+
"""
138+
Returns names of packages from PMDK library, which are installed.
139+
"""
140+
libraries = get_names_of_pkg_content_func()
141+
installed_packages = []
142+
143+
for library in libraries:
144+
if library == "pmempool" and check_output(
145+
'find /usr/bin/ -name ' + library, cwd=args.pmdk_path,
146+
shell=True):
147+
installed_packages.append(library)
148+
elif library == "libpmemobj++" and check_output(
149+
'find /usr/include/' + library + ' -name *.hpp',
150+
cwd=args.pmdk_path, shell=True):
151+
installed_packages.append(library)
152+
elif library == "pmdk":
153+
pass
154+
elif check_output('find ' + so_path + ' -name ' + library + '.so',
155+
cwd=args.pmdk_path, shell=True):
156+
installed_packages.append(library)
157+
return installed_packages
158+
159+
160+
if __name__ == '__main__':
161+
parser = ArgumentParser(
162+
description='Install packages from PMDK library')
163+
parser.add_argument("-r", "--pmdk-path", required=True,
164+
help="the PMDK library root path.")
165+
166+
args = parser.parse_args()
167+
os_distro=distro.id()
168+
PMDK_VERSION, SYSTEM_ARCHITECTURE =\
169+
get_package_version_and_system_architecture(args.pmdk_path)
170+
save_pkg_version(args.pmdk_path + "/pkgVersion.json")
171+
if os_distro == 'fedora' or os_distro == "rhel":
172+
so_path = '/usr/lib64/'
173+
split_param = '-'
174+
packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE)
175+
install_cmd = 'rpm -i --nodeps '
176+
install_func = install_rpm_packages
177+
get_names_of_pkg_content_func = get_names_of_rpm_content
178+
elif os_distro == 'ubuntu':
179+
so_path = '/lib/x86_64-linux-gnu/'
180+
split_param = '_'
181+
packages_path = path.join(args.pmdk_path, 'dpkg')
182+
install_cmd = 'dpkg -i '
183+
install_func = install_dpkg_packages
184+
get_names_of_pkg_content_func = get_names_of_dpkg_content
185+
186+
if not get_installed_packages(so_path, packages_path, get_names_of_pkg_content_func):
187+
install_func(args.pmdk_path)
188+
else:
189+
print("PMDK library is still installed")

0 commit comments

Comments
 (0)