Skip to content

Commit 56d9ae9

Browse files
joke325Daniel Wyatt
joke325
authored and
Daniel Wyatt
committed
Migration to Python 3
issue rnpgp#812
1 parent a031f85 commit 56d9ae9

9 files changed

+259
-145
lines changed

‎Brewfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ brew "libtool"
77
brew "pkg-config"
88
brew "gnupg"
99
brew "wget"
10-
brew "python@2"
10+
brew "python"
1111

‎CMakeLists.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
# 3.10+ for CPackFreeBSD (FreeBSD packaging)
2828
# 3.10+ for gtest_discover_tests (parallel rnp_tests)
2929
# 3.12+ for NAMELINK_COMPONENT (for better RPM packaging)
30-
cmake_minimum_required(VERSION 3.10)
30+
# 3.12+ for Python3 find module
31+
cmake_minimum_required(VERSION 3.12)
3132

3233
# contact email, other info
3334
include(cmake/info.cmake)

‎ci/before_install.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ msys_install() {
8282
mingw64/mingw-w64-x86_64-cmake
8383
mingw64/mingw-w64-x86_64-gcc
8484
mingw64/mingw-w64-x86_64-json-c
85-
mingw64/mingw-w64-x86_64-python2
85+
mingw64/mingw-w64-x86_64-python3
8686
"
8787
pacman --noconfirm -S --needed ${packages}
8888
pacman --noconfirm -U \

‎docs/installation.adoc

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ pacman -Syu --noconfirm --needed
9393
# Then most likely you'll need to close msys console and run it agian:
9494
pacman -Syu --noconfirm --needed
9595
# Install packages
96-
pacman --noconfirm -S --needed tar zlib-devel libbz2-devel git automake autoconf libtool automake-wrapper gnupg2 make pkgconfig mingw64/mingw-w64-x86_64-cmake mingw64/mingw-w64-x86_64-gcc mingw64/mingw-w64-x86_64-json-c mingw64/mingw-w64-x86_64-libbotan mingw64/mingw-w64-x86_64-python2
96+
pacman --noconfirm -S --needed tar zlib-devel libbz2-devel git automake autoconf libtool automake-wrapper gnupg2 make pkgconfig mingw64/mingw-w64-x86_64-cmake mingw64/mingw-w64-x86_64-gcc mingw64/mingw-w64-x86_64-json-c mingw64/mingw-w64-x86_64-libbotan mingw64/mingw-w64-x86_64-python3
9797
----
9898

9999
Then clone the repository, say to rnp folder, and:

‎src/tests/CMakeLists.txt

+3-3
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ find_package(JSON-C 0.11 REQUIRED)
2828
# Note that we do this call early because Google Test will also do
2929
# this but with less strict version requirements, which will cause
3030
# problems for us.
31-
find_package(PythonInterp 2.7 EXACT)
31+
find_package(Python3 COMPONENTS Interpreter)
3232
find_package(GnuPG 2.2 COMPONENTS gpg gpgconf)
3333

3434
include(GoogleTest)
@@ -135,7 +135,7 @@ function(add_cli_test suite)
135135
set(_test_name cli_tests-${suite})
136136
add_test(
137137
NAME ${_test_name}
138-
COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -v -d "${suite}"
138+
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -v -d "${suite}"
139139
)
140140
set(_env)
141141
list(APPEND _env
@@ -152,7 +152,7 @@ function(add_cli_test suite)
152152
endfunction()
153153
# get a list of test suites
154154
execute_process(
155-
COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -ls
155+
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -ls
156156
RESULT_VARIABLE _ec
157157
OUTPUT_VARIABLE suitelist
158158
OUTPUT_STRIP_TRAILING_WHITESPACE

‎src/tests/cli_common.py

+34-11
Original file line numberDiff line numberDiff line change
@@ -34,24 +34,22 @@ def raise_err(msg, log = None):
3434
raise CLIError(msg, log)
3535

3636
def size_to_readable(num, suffix = 'B'):
37-
for unit in ['','K','M','G','T','P','E','Z']:
37+
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
3838
if abs(num) < 1024.0:
3939
return "%3.1f%s%s" % (num, unit, suffix)
4040
num /= 1024.0
4141
return "%.1f%s%s" % (num, 'Yi', suffix)
4242

4343
def list_upto(lst, count):
44-
res = lst[:]
45-
while len(res) < count:
46-
res = res + lst[:]
47-
return res[:count]
44+
return (list(lst)*(count//len(lst)+1))[:count]
4845

4946
def pswd_pipe(password):
5047
pr, pw = os.pipe()
5148
with os.fdopen(pw, 'w') as fw:
5249
fw.write(password)
5350
fw.write('\n')
5451
fw.write(password)
52+
os.set_inheritable(pr, True)
5553

5654
if not is_windows():
5755
return pr
@@ -63,15 +61,16 @@ def pswd_pipe(password):
6361
def random_text(path, size):
6462
# Generate random text, with 50% probability good-compressible
6563
if random.randint(0, 10) < 5:
66-
st = ''.join(random.choice(string.ascii_letters + string.digits + " \t\n-,.") for _ in range(size))
64+
st = ''.join(random.choice(string.ascii_letters + string.digits + " \t\n-,.")
65+
for _ in range(size))
6766
else:
6867
st = ''.join(random.choice("abcdef0123456789 \t\n-,.") for _ in range(size))
6968
with open(path, 'w+') as f:
7069
f.write(st)
7170

7271
def file_text(path):
73-
with open(path, 'r') as f:
74-
return f.read()
72+
with open(path, 'rb') as f:
73+
return f.read().decode().replace('\r\r', '\r')
7574

7675
def find_utility(name, exitifnone = True):
7776
path = distutils.spawn.find_executable(name)
@@ -100,17 +99,26 @@ def run_proc_windows(proc, params, stdin=None):
10099

101100
exe = os.path.basename(proc)
102101
# We need to escape empty parameters/ones with spaces with quotes
103-
params = map(lambda st: st if (st and not any(x in st for x in [' ','\r','\t'])) else '"%s"' % st, [exe] + params)
102+
params = tuple(map(lambda st: st if (st and not any(x in st for x in [' ','\r','\t'])) else '"%s"' % st, [exe] + params))
104103
sys.stdout.flush()
105104

106105
stdin_path = os.path.join(WORKDIR, 'stdin.txt')
107106
stdout_path = os.path.join(WORKDIR, 'stdout.txt')
108107
stderr_path = os.path.join(WORKDIR, 'stderr.txt')
108+
pass_path = os.path.join(WORKDIR, 'pass.txt')
109+
passfd = 0
110+
passfo = None
111+
try:
112+
idx = params.index('--pass-fd')
113+
if idx < len(params):
114+
passfd = int(params[idx+1])
115+
passfo = os.fdopen(passfd, 'r', closefd=False)
116+
except (ValueError, OSError): pass
109117
# We may use pipes here (ensuring we use dup to inherit handles), but those have limited buffer
110118
# so we'll need to poll process
111119
if stdin:
112120
with open(stdin_path, "wb+") as stdinf:
113-
stdinf.write(stdin)
121+
stdinf.write(stdin.encode() if isinstance(stdin, str) else stdin)
114122
stdin_fl = os.open(stdin_path, os.O_RDONLY | os.O_BINARY)
115123
stdin_no = sys.stdin.fileno()
116124
stdin_cp = os.dup(stdin_no)
@@ -120,6 +128,11 @@ def run_proc_windows(proc, params, stdin=None):
120128
stderr_fl = os.open(stderr_path, os.O_CREAT | os.O_RDWR | os.O_BINARY)
121129
stderr_no = sys.stderr.fileno()
122130
stderr_cp = os.dup(stderr_no)
131+
if passfo:
132+
with open(pass_path, "w+") as passf:
133+
passf.write(passfo.read())
134+
pass_fl = os.open(pass_path, os.O_RDONLY | os.O_BINARY)
135+
pass_cp = os.dup(passfd)
123136

124137
try:
125138
os.dup2(stdout_fl, stdout_no)
@@ -129,6 +142,9 @@ def run_proc_windows(proc, params, stdin=None):
129142
if stdin:
130143
os.dup2(stdin_fl, stdin_no)
131144
os.close(stdin_fl)
145+
if passfo:
146+
os.dup2(pass_fl, passfd)
147+
os.close(pass_fl)
132148
retcode = os.spawnv(os.P_WAIT, proc, params)
133149
finally:
134150
os.dup2(stdout_cp, stdout_no)
@@ -138,12 +154,18 @@ def run_proc_windows(proc, params, stdin=None):
138154
if stdin:
139155
os.dup2(stdin_cp, stdin_no)
140156
os.close(stdin_cp)
157+
if passfo:
158+
os.dup2(pass_cp, passfd)
159+
os.close(pass_cp)
160+
passfo.close()
141161
out = file_text(stdout_path).replace('\r\n', '\n')
142162
err = file_text(stderr_path).replace('\r\n', '\n')
143163
os.unlink(stdout_path)
144164
os.unlink(stderr_path)
145165
if stdin:
146166
os.unlink(stdin_path)
167+
if passfo:
168+
os.unlink(pass_path)
147169
logging.debug(err.strip())
148170
logging.debug(out.strip())
149171
return (retcode, out, err)
@@ -154,7 +176,8 @@ def run_proc(proc, params, stdin=None):
154176
return run_proc_windows(proc, params, stdin)
155177

156178
logging.debug((proc + ' ' + ' '.join(params)).strip())
157-
process = Popen([proc] + params, stdout=PIPE, stderr=PIPE, stdin=PIPE if stdin else None)
179+
process = Popen([proc] + params, stdout=PIPE, stderr=PIPE,
180+
stdin=PIPE if stdin else None, close_fds=False, universal_newlines=True)
158181
output, errout = process.communicate(stdin)
159182
retcode = process.poll()
160183
logging.debug(errout.strip())

‎src/tests/cli_perf.py

+51-27
Original file line numberDiff line numberDiff line change
@@ -55,31 +55,34 @@ def setup(workdir):
5555
# Creating working directory and populating it with test files
5656
RNPDIR = path.join(WORKDIR, '.rnp')
5757
GPGDIR = path.join(WORKDIR, '.gpg')
58-
os.mkdir(RNPDIR, 0700)
59-
os.mkdir(GPGDIR, 0700)
58+
os.mkdir(RNPDIR, 0o700)
59+
os.mkdir(GPGDIR, 0o700)
6060

6161
# Generating key
6262
pipe = pswd_pipe(PASSWORD)
63-
params = ['--homedir', RNPDIR, '--pass-fd', str(pipe), '--userid', 'performance@rnp', '--generate-key']
63+
params = ['--homedir', RNPDIR, '--pass-fd', str(pipe), '--userid', 'performance@rnp',
64+
'--generate-key']
6465
# Run key generation
6566
ret, out, err = run_proc(RNPK, params)
6667
os.close(pipe)
6768

6869
# Importing keys to GnuPG so it can build trustdb and so on
69-
ret, out, err = run_proc(GPG, ['--batch', '--passphrase', '', '--homedir', GPGDIR, '--import', path.join(RNPDIR, 'pubring.gpg'), path.join(RNPDIR, 'secring.gpg')])
70+
ret, out, err = run_proc(GPG, ['--batch', '--passphrase', '', '--homedir', GPGDIR,
71+
'--import', path.join(RNPDIR, 'pubring.gpg'),
72+
path.join(RNPDIR, 'secring.gpg')])
7073

7174
# Generating small file for tests
72-
SMALLSIZE = 3312;
73-
st = 'lorem ipsum dol ' * (SMALLSIZE/16)
75+
SMALLSIZE = 3312
76+
st = 'lorem ipsum dol ' * (SMALLSIZE//16+1)
7477
with open(path.join(WORKDIR, SMALLFILE), 'w+') as small_file:
7578
small_file.write(st)
7679

7780
# Generating large file for tests
78-
print 'Generating large file of size {}'.format(size_to_readable(LARGESIZE))
81+
print('Generating large file of size {}'.format(size_to_readable(LARGESIZE)))
7982

80-
st = '0123456789ABCDEF' * (1024/16)
83+
st = '0123456789ABCDEF' * (1024//16)
8184
with open(path.join(WORKDIR, LARGEFILE), 'w') as fd:
82-
for i in range(0, LARGESIZE / 1024 - 1):
85+
for i in range(0, LARGESIZE // 1024):
8386
fd.write(st)
8487

8588
def run_iterated(iterations, func, src, dst, *args):
@@ -96,28 +99,34 @@ def run_iterated(iterations, func, src, dst, *args):
9699
return res
97100

98101
def rnp_symencrypt_file(src, dst, cipher, zlevel = 6, zalgo = 'zip', armor = False):
99-
params = ['--homedir', RNPDIR, '--password', PASSWORD, '--cipher', cipher, '-z', str(zlevel), '--' + zalgo, '-c', src, '--output', dst]
102+
params = ['--homedir', RNPDIR, '--password', PASSWORD, '--cipher', cipher,
103+
'-z', str(zlevel), '--' + zalgo, '-c', src, '--output', dst]
100104
if armor:
101105
params += ['--armor']
102106
ret = run_proc_fast(RNP, params)
103107
if ret != 0:
104108
raise_err('rnp symmetric encryption failed')
105109

106110
def rnp_decrypt_file(src, dst):
107-
ret = run_proc_fast(RNP, ['--homedir', RNPDIR, '--password', PASSWORD, '--decrypt', src, '--output', dst])
111+
ret = run_proc_fast(RNP, ['--homedir', RNPDIR, '--password', PASSWORD, '--decrypt', src,
112+
'--output', dst])
108113
if ret != 0:
109114
raise_err('rnp decryption failed')
110115

111116
def gpg_symencrypt_file(src, dst, cipher = 'AES', zlevel = 6, zalgo = 1, armor = False):
112-
params = ['--homedir', GPGDIR, '-c', '-z', str(zlevel), '--s2k-count', '524288', '--compress-algo', str(zalgo), '--batch', '--passphrase', PASSWORD, '--cipher-algo', cipher, '--output', dst, src]
117+
params = ['--homedir', GPGDIR, '-c', '-z', str(zlevel), '--s2k-count', '524288',
118+
'--compress-algo', str(zalgo), '--batch', '--passphrase', PASSWORD,
119+
'--cipher-algo', cipher, '--output', dst, src]
113120
if armor:
114121
params.insert(2, '--armor')
115122
ret = run_proc_fast(GPG, params)
116123
if ret != 0:
117124
raise_err('gpg symmetric encryption failed for cipher ' + cipher)
118125

119126
def gpg_decrypt_file(src, dst, keypass):
120-
ret = run_proc_fast(GPG, ['--homedir', GPGDIR, '--pinentry-mode=loopback', '--batch', '--yes', '--passphrase', keypass, '--trust-model', 'always', '-o', dst, '-d', src])
127+
ret = run_proc_fast(GPG, ['--homedir', GPGDIR, '--pinentry-mode=loopback', '--batch',
128+
'--yes', '--passphrase', keypass, '--trust-model', 'always',
129+
'-o', dst, '-d', src])
121130
if ret != 0:
122131
raise_err('gpg decryption failed')
123132

@@ -132,27 +141,33 @@ def print_test_results(fsize, rnptime, gpgtime, operation):
132141

133142
if rnpruns >= gpgruns:
134143
percents = (rnpruns - gpgruns) / gpgruns * 100
135-
logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(operation, percents, runstr))
144+
logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(
145+
operation, percents, runstr))
136146
else:
137147
percents = (gpgruns - rnpruns) / gpgruns * 100
138-
logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(operation, percents, runstr))
148+
logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(
149+
operation, percents, runstr))
139150
else:
140151
rnpspeed = fsize / 1024.0 / 1024.0 / rnptime
141152
gpgspeed = fsize / 1024.0 / 1024.0 / gpgtime
142153
spdstr = '{:.2f} MB/sec vs {:.2f} MB/sec'.format(rnpspeed, gpgspeed)
143154

144155
if rnpspeed >= gpgspeed:
145156
percents = (rnpspeed - gpgspeed) / gpgspeed * 100
146-
logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(operation, percents, spdstr))
157+
logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(
158+
operation, percents, spdstr))
147159
else:
148160
percents = (gpgspeed - rnpspeed) / gpgspeed * 100
149-
logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(operation, percents, spdstr))
161+
logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(
162+
operation, percents, spdstr))
150163

151164
def get_file_params(filetype):
152165
if filetype == 'small':
153-
infile, outfile, iterations, fsize = (SMALLFILE, SMALLFILE + '.gpg', SMALL_ITERATIONS, SMALLSIZE)
166+
infile, outfile, iterations, fsize = (SMALLFILE, SMALLFILE + '.gpg',
167+
SMALL_ITERATIONS, SMALLSIZE)
154168
else:
155-
infile, outfile, iterations, fsize = (LARGEFILE, LARGEFILE + '.gpg', LARGE_ITERATIONS, LARGESIZE)
169+
infile, outfile, iterations, fsize = (LARGEFILE, LARGEFILE + '.gpg',
170+
LARGE_ITERATIONS, LARGESIZE)
156171

157172
infile = path.join(WORKDIR, infile)
158173
rnpout = path.join(WORKDIR, outfile + '.rnp')
@@ -172,8 +187,10 @@ def small_file_symmetric_encryption(self):
172187
'''
173188
infile, rnpout, gpgout, iterations, fsize = get_file_params('small')
174189
for armor in [False, True]:
175-
tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout, 'AES128', 0, 'zip', armor)
176-
tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout, 'AES128', 0, 1, armor)
190+
tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout,
191+
'AES128', 0, 'zip', armor)
192+
tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout,
193+
'AES128', 0, 1, armor)
177194
testname = 'ENCRYPT-SMALL-{}'.format('ARMOR' if armor else 'BINARY')
178195
print_test_results(fsize, tmrnp, tmgpg, testname)
179196

@@ -183,8 +200,10 @@ def large_file_symmetric_encryption(self):
183200
'''
184201
infile, rnpout, gpgout, iterations, fsize = get_file_params('large')
185202
for cipher in ['AES128', 'AES192', 'AES256', 'TWOFISH', 'BLOWFISH', 'CAST5', 'CAMELLIA128', 'CAMELLIA192', 'CAMELLIA256']:
186-
tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout, cipher, 0, 'zip', False)
187-
tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout, cipher, 0, 1, False)
203+
tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout,
204+
cipher, 0, 'zip', False)
205+
tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout,
206+
cipher, 0, 1, False)
188207
testname = 'ENCRYPT-{}-BINARY'.format(cipher)
189208
print_test_results(fsize, tmrnp, tmgpg, testname)
190209

@@ -193,7 +212,8 @@ def large_file_armored_encryption(self):
193212
Large file armored encryption
194213
'''
195214
infile, rnpout, gpgout, iterations, fsize = get_file_params('large')
196-
tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout, 'AES128', 0, 'zip', True)
215+
tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout,
216+
'AES128', 0, 'zip', True)
197217
tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout, 'AES128', 0, 1, True)
198218
print_test_results(fsize, tmrnp, tmgpg, 'ENCRYPT-LARGE-ARMOR')
199219

@@ -217,7 +237,8 @@ def large_file_symmetric_decryption(self):
217237
'''
218238
infile, rnpout, gpgout, iterations, fsize = get_file_params('large')
219239
inenc = infile + '.enc'
220-
for cipher in ['AES128', 'AES192', 'AES256', 'TWOFISH', 'BLOWFISH', 'CAST5', 'CAMELLIA128', 'CAMELLIA192', 'CAMELLIA256']:
240+
for cipher in ['AES128', 'AES192', 'AES256', 'TWOFISH', 'BLOWFISH', 'CAST5',
241+
'CAMELLIA128', 'CAMELLIA192', 'CAMELLIA256']:
221242
gpg_symencrypt_file(infile, inenc, cipher, 0, 1, False)
222243
tmrnp = run_iterated(iterations, rnp_decrypt_file, inenc, rnpout)
223244
tmgpg = run_iterated(iterations, gpg_decrypt_file, inenc, gpgout, PASSWORD)
@@ -266,11 +287,14 @@ def large_file_armored_decryption(self):
266287
help="Name of the comma-separated benchmarks to run", metavar="benchmarks")
267288
parser.add_argument("-w", "--workdir", dest="workdir",
268289
help="Working directory to use", metavar="workdir")
269-
parser.add_argument("-l", "--list", help="Print list of available benchmarks and exit", action="store_true")
290+
parser.add_argument("-l", "--list", help="Print list of available benchmarks and exit",
291+
action="store_true")
270292
args = parser.parse_args()
271293

272294
# get list of benchamrks to run
273-
bench_methods = [ x[0] for x in inspect.getmembers(Benchmark,predicate=inspect.ismethod) if not x[0].startswith("_") ]
295+
bench_methods = [ x[0] for x in inspect.getmembers(Benchmark,
296+
predicate=lambda x: inspect.ismethod(x) or inspect.isfunction(x))]
297+
print(bench_methods)
274298

275299
if args.list:
276300
for name in bench_methods:

0 commit comments

Comments
 (0)