From ab1df7147669e3ae05e23406bb6fe6d22fc051e7 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Thu, 3 Oct 2019 14:36:31 +0200 Subject: [PATCH 1/3] Make random generator API fluent Signed-off-by: Robert Baldyga --- tests/functional/tests/utils/random.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/tests/utils/random.py b/tests/functional/tests/utils/random.py index 5c608855..924e0186 100644 --- a/tests/functional/tests/utils/random.py +++ b/tests/functional/tests/utils/random.py @@ -43,6 +43,7 @@ def __init__(self, base_range=DefaultRanges.INT, count=1000): def exclude_range(self, excl_range): self.exclude.append(excl_range) + return self def __iter__(self): return self From b9b39fdb4643bea43de47b784f67ff317366fdbc Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Thu, 3 Oct 2019 14:38:34 +0200 Subject: [PATCH 2/3] Add more fine-grained fixtures for fuzzy tests Signed-off-by: Robert Baldyga --- tests/functional/tests/security/conftest.py | 52 +++++++++++- .../security/test_management_start_fuzzy.py | 81 ++++++++----------- 2 files changed, 85 insertions(+), 48 deletions(-) diff --git a/tests/functional/tests/security/conftest.py b/tests/functional/tests/security/conftest.py index c5860280..7d9ca3bb 100644 --- a/tests/functional/tests/security/conftest.py +++ b/tests/functional/tests/security/conftest.py @@ -11,13 +11,28 @@ c_uint16, c_int ) -from tests.utils.random import RandomStringGenerator, RandomGenerator, DefaultRanges +from tests.utils.random import RandomStringGenerator, RandomGenerator, DefaultRanges, Range + +from pyocf.types.cache import CacheMode, EvictionPolicy, MetadataLayout, PromotionPolicy +from pyocf.types.shared import CacheLineSize import pytest sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir)) +def enum_min(enum): + return list(enum)[0].value + + +def enum_max(enum): + return list(enum)[-1].value + + +def enum_range(enum): + return Range(enum_min(enum), enum_max(enum)) + + @pytest.fixture(params=RandomGenerator(DefaultRanges.UINT16)) def c_uint16_randomize(request): return request.param @@ -46,3 +61,38 @@ def c_int_sector_randomize(request): @pytest.fixture(params=RandomStringGenerator()) def string_randomize(request): return request.param + + +@pytest.fixture( + params=RandomGenerator(DefaultRanges.UINT32).exclude_range(enum_range(CacheMode)) +) +def not_cache_mode_randomize(request): + return request.param + + +@pytest.fixture( + params=RandomGenerator(DefaultRanges.UINT32).exclude_range(enum_range(CacheLineSize)) +) +def not_cache_line_size_randomize(request): + return request.param + + +@pytest.fixture( + params=RandomGenerator(DefaultRanges.UINT32).exclude_range(enum_range(EvictionPolicy)) +) +def not_eviction_policy_randomize(request): + return request.param + + +@pytest.fixture( + params=RandomGenerator(DefaultRanges.UINT32).exclude_range(enum_range(PromotionPolicy)) +) +def not_promotion_policy_randomize(request): + return request.param + + +@pytest.fixture( + params=RandomGenerator(DefaultRanges.UINT32).exclude_range(enum_range(MetadataLayout)) +) +def not_metadata_layout_randomize(request): + return request.param diff --git a/tests/functional/tests/security/test_management_start_fuzzy.py b/tests/functional/tests/security/test_management_start_fuzzy.py index ff934829..20f1ea8e 100644 --- a/tests/functional/tests/security/test_management_start_fuzzy.py +++ b/tests/functional/tests/security/test_management_start_fuzzy.py @@ -11,7 +11,7 @@ from pyocf.types.shared import OcfError, CacheLineSize from pyocf.types.volume import Volume from pyocf.utils import Size -from tests.utils.random import RandomGenerator, DefaultRanges +from tests.utils.random import RandomGenerator, DefaultRanges, Range logger = logging.getLogger(__name__) @@ -21,38 +21,30 @@ def try_start_cache(**config): cache = Cache.start_on_device(cache_device, **config) cache.stop() - @pytest.mark.security @pytest.mark.parametrize("cls", CacheLineSize) -def test_fuzzy_start_cache_mode(pyocf_ctx, cls, c_uint32_randomize): +def test_fuzzy_start_cache_mode(pyocf_ctx, cls, not_cache_mode_randomize): """ Test whether it is impossible to start cache with invalid cache mode value. :param pyocf_ctx: basic pyocf context fixture :param cls: cache line size value to start cache with :param c_uint32_randomize: cache mode enum value to start cache with """ - if c_uint32_randomize not in [item.value for item in CacheMode]: - with pytest.raises(OcfError, match="OCF_ERR_INVALID_CACHE_MODE"): - try_start_cache(cache_mode=c_uint32_randomize, cache_line_size=cls) - else: - logger.warning(f"Test skipped for valid cache mode enum value: '{c_uint32_randomize}'. ") + with pytest.raises(OcfError, match="OCF_ERR_INVALID_CACHE_MODE"): + try_start_cache(cache_mode=not_cache_mode_randomize, cache_line_size=cls) @pytest.mark.security @pytest.mark.parametrize("cm", CacheMode) -def test_fuzzy_start_cache_line_size(pyocf_ctx, c_uint64_randomize, cm): +def test_fuzzy_start_cache_line_size(pyocf_ctx, not_cache_line_size_randomize, cm): """ Test whether it is impossible to start cache with invalid cache line size value. :param pyocf_ctx: basic pyocf context fixture :param c_uint64_randomize: cache line size enum value to start cache with :param cm: cache mode value to start cache with """ - if c_uint64_randomize not in [item.value for item in CacheLineSize]: - with pytest.raises(OcfError, match="OCF_ERR_INVALID_CACHE_LINE_SIZE"): - try_start_cache(cache_mode=cm, cache_line_size=c_uint64_randomize) - else: - logger.warning( - f"Test skipped for valid cache line size enum value: '{c_uint64_randomize}'. ") + with pytest.raises(OcfError, match="OCF_ERR_INVALID_CACHE_LINE_SIZE"): + try_start_cache(cache_mode=cm, cache_line_size=not_cache_line_size_randomize) @pytest.mark.security @@ -84,7 +76,7 @@ def test_fuzzy_start_name(pyocf_ctx, string_randomize, cm, cls): @pytest.mark.security @pytest.mark.parametrize("cm", CacheMode) @pytest.mark.parametrize("cls", CacheLineSize) -def test_fuzzy_start_eviction_policy(pyocf_ctx, c_uint32_randomize, cm, cls): +def test_fuzzy_start_eviction_policy(pyocf_ctx, not_eviction_policy_randomize, cm, cls): """ Test whether it is impossible to start cache with invalid eviction policy value. :param pyocf_ctx: basic pyocf context fixture @@ -92,18 +84,18 @@ def test_fuzzy_start_eviction_policy(pyocf_ctx, c_uint32_randomize, cm, cls): :param cm: cache mode value to start cache with :param cls: cache line size value to start cache with """ - if c_uint32_randomize not in [item.value for item in EvictionPolicy]: - with pytest.raises(OcfError, match="OCF_ERR_INVAL"): - try_start_cache(eviction_policy=c_uint32_randomize, cache_mode=cm, cache_line_size=cls) - else: - logger.warning( - f"Test skipped for valid eviction policy enum value: '{c_uint32_randomize}'. ") + with pytest.raises(OcfError, match="OCF_ERR_INVAL"): + try_start_cache( + eviction_policy=not_eviction_policy_randomize, + cache_mode=cm, + cache_line_size=cls + ) @pytest.mark.security @pytest.mark.parametrize("cm", CacheMode) @pytest.mark.parametrize("cls", CacheLineSize) -def test_fuzzy_start_metadata_layout(pyocf_ctx, c_uint32_randomize, cm, cls): +def test_fuzzy_start_metadata_layout(pyocf_ctx, not_metadata_layout_randomize, cm, cls): """ Test whether it is impossible to start cache with invalid metadata layout value. :param pyocf_ctx: basic pyocf context fixture @@ -111,12 +103,12 @@ def test_fuzzy_start_metadata_layout(pyocf_ctx, c_uint32_randomize, cm, cls): :param cm: cache mode value to start cache with :param cls: cache line size value to start cache with """ - if c_uint32_randomize not in [item.value for item in MetadataLayout]: - with pytest.raises(OcfError, match="OCF_ERR_INVAL"): - try_start_cache(metadata_layout=c_uint32_randomize, cache_mode=cm, cache_line_size=cls) - else: - logger.warning( - f"Test skipped for valid metadata layout enum value: '{c_uint32_randomize}'. ") + with pytest.raises(OcfError, match="OCF_ERR_INVAL"): + try_start_cache( + metadata_layout=not_metadata_layout_randomize, + cache_mode=cm, + cache_line_size=cls + ) @pytest.mark.security @@ -131,23 +123,18 @@ def test_fuzzy_start_max_queue_size(pyocf_ctx, max_wb_queue_size, c_uint32_rando :param c_uint32_randomize: queue unblock size value to start cache with :param cls: cache line size value to start cache with """ - if c_uint32_randomize >= max_wb_queue_size: - with pytest.raises(OcfError, match="OCF_ERR_INVAL"): - try_start_cache( - max_queue_size=max_wb_queue_size, - queue_unblock_size=c_uint32_randomize, - cache_mode=CacheMode.WB, - cache_line_size=cls) - else: - logger.warning(f"Test skipped for valid values: " - f"'max_queue_size={max_wb_queue_size}, " - f"queue_unblock_size={c_uint32_randomize}'.") + with pytest.raises(OcfError, match="OCF_ERR_INVAL"): + try_start_cache( + max_queue_size=max_wb_queue_size, + queue_unblock_size=max_wb_queue_size + c_uint32_randomize, + cache_mode=CacheMode.WB, + cache_line_size=cls) @pytest.mark.security @pytest.mark.parametrize("cm", CacheMode) @pytest.mark.parametrize("cls", CacheLineSize) -def test_fuzzy_start_promotion_policy(pyocf_ctx, c_uint32_randomize, cm, cls): +def test_fuzzy_start_promotion_policy(pyocf_ctx, not_promotion_policy_randomize, cm, cls): """ Test whether it is impossible to start cache with invalid promotion policy :param pyocf_ctx: basic pyocf context fixture @@ -155,9 +142,9 @@ def test_fuzzy_start_promotion_policy(pyocf_ctx, c_uint32_randomize, cm, cls): :param cm: cache mode value to start cache with :param cls: cache line size to start cache with """ - if c_uint32_randomize not in [item.value for item in PromotionPolicy]: - with pytest.raises(OcfError, match="OCF_ERR_INVAL"): - try_start_cache(cache_mode=cm, cache_line_size=cls, promotion_policy=c_uint32_randomize) - else: - logger.warning( - f"Test skipped for valid promotion policy: '{c_uint32_randomize}'. ") + with pytest.raises(OcfError, match="OCF_ERR_INVAL"): + try_start_cache( + cache_mode=cm, + cache_line_size=cls, + promotion_policy=not_promotion_policy_randomize + ) From 1525e85805e22f3434b7f9e9e87e15c1e8acbac0 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Thu, 3 Oct 2019 14:43:21 +0200 Subject: [PATCH 3/3] Add global random seed config This is needed to make fuzzy tests work with xdist as each xdist gateway expects to receive the same set of parameter values, which for random generators may be achieved only by providing globally shared seed. Signed-off-by: Robert Baldyga --- tests/functional/Makefile | 7 +++++-- tests/functional/config/random.cfg | 2 ++ tests/functional/tests/utils/random.py | 8 ++++++-- tests/functional/utils/configure_random.py | 13 +++++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 tests/functional/config/random.cfg create mode 100755 tests/functional/utils/configure_random.py diff --git a/tests/functional/Makefile b/tests/functional/Makefile index b8963b27..71a77dc1 100755 --- a/tests/functional/Makefile +++ b/tests/functional/Makefile @@ -18,7 +18,7 @@ SRC=$(shell find $(SRCDIR) $(WRAPDIR) -name \*.c) OBJS=$(patsubst %.c, %.o, $(SRC)) OCFLIB=$(ADAPTERDIR)/libocf.so -all: | sync +all: | sync config_random $(MAKE) $(OCFLIB) $(OCFLIB): $(OBJS) @@ -36,6 +36,9 @@ sync: @$(MAKE) -C $(OCFDIR) src O=$(ADAPTERDIR)/ocf @$(MAKE) -C $(OCFDIR) env O=$(ADAPTERDIR)/ocf OCF_ENV=posix +config_random: + @python3 utils/configure_random.py + clean: @rm -rf $(OCFLIB) $(OBJS) @echo " CLEAN " @@ -46,4 +49,4 @@ distclean: clean @rm -rf $(INCDIR)/ocf @echo " DISTCLEAN " -.PHONY: all clean sync distclean +.PHONY: all clean sync config_random distclean diff --git a/tests/functional/config/random.cfg b/tests/functional/config/random.cfg new file mode 100644 index 00000000..f7ab2125 --- /dev/null +++ b/tests/functional/config/random.cfg @@ -0,0 +1,2 @@ +# This file content will be generated by utils/configure_random.py +# triggered from the Makefile diff --git a/tests/functional/tests/utils/random.py b/tests/functional/tests/utils/random.py index 924e0186..27735700 100644 --- a/tests/functional/tests/utils/random.py +++ b/tests/functional/tests/utils/random.py @@ -36,6 +36,8 @@ class DefaultRanges(Range, enum.Enum): class RandomGenerator: def __init__(self, base_range=DefaultRanges.INT, count=1000): + with open("config/random.cfg") as f: + self.random = random.Random(int(f.read())) self.exclude = [] self.range = base_range self.count = count @@ -53,7 +55,7 @@ def __next__(self): raise StopIteration() self.n += 1 while True: - val = random.randint(self.range.min, self.range.max) + val = self.random.randint(self.range.min, self.range.max) if self.exclude: excl_map = map(lambda e: e.is_within(val), self.exclude) is_excluded = reduce(lambda a, b: a or b, excl_map) @@ -64,6 +66,8 @@ def __next__(self): class RandomStringGenerator: def __init__(self, len_range=Range(0, 20), count=700): + with open("config/random.cfg") as f: + self.random = random.Random(int(f.read())) self.generator = self.__string_generator(len_range) self.count = count self.n = 0 @@ -78,7 +82,7 @@ def __string_generator(self, len_range): string.punctuation, string.hexdigits]: yield ''.join(random.choice(t) for _ in range( - random.randint(len_range.min, len_range.max) + self.random.randint(len_range.min, len_range.max) )) def __iter__(self): diff --git a/tests/functional/utils/configure_random.py b/tests/functional/utils/configure_random.py new file mode 100755 index 00000000..71a04401 --- /dev/null +++ b/tests/functional/utils/configure_random.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +# +# Copyright(c) 2012-2018 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import sys +import random + + +with open("config/random.cfg", "w") as f: + f.write(str(random.randint(0, sys.maxsize)))