From bd103b8ebd3e8224be0de8cc7d1a4201724c42a2 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:16:02 -0600 Subject: [PATCH 01/36] refactoring ResolvableProperty --- sceptre/resolvers/__init__.py | 264 +++++++++++++++++++++++++++------- 1 file changed, 214 insertions(+), 50 deletions(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 846ee2076..0f0a9eda6 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -3,30 +3,32 @@ import logging from contextlib import contextmanager from threading import RLock +from typing import Any, TYPE_CHECKING, Type, Union, TypeVar -import six from sceptre.helpers import _call_func_on_values +if TYPE_CHECKING: + from sceptre import stack -class RecursiveGet(Exception): +T_Container = TypeVar('T_Container', bound=Union[dict, list]) + +logger = logging.getLogger(__name__) + + +class RecursiveResolve(Exception): pass -@six.add_metaclass(abc.ABCMeta) -class Resolver: +class Resolver(abc.ABC): """ Resolver is an abstract base class that should be inherited by all Resolvers. :param argument: The argument of the resolver. - :type argument: str :param stack: The associated stack of the resolver. - :type stack: sceptre.stack.Stack """ - __metaclass__ = abc.ABCMeta - - def __init__(self, argument=None, stack=None): + def __init__(self, argument: Any = None, stack: 'stack.Stack' = None): self.logger = logging.getLogger(__name__) self.argument = argument self.stack = stack @@ -49,62 +51,234 @@ def resolve(self): """ pass # pragma: no cover + def clone(self, stack: 'stack.Stack') -> 'Resolver': + """ + Produces a "fresh" copy of the Resolver, with the specified stack. + + :param stack: The stack to set on the cloned resolver + """ + return type(self)(self.argument, stack) + -class ResolvableProperty(object): +class ResolvableProperty(abc.ABC): """ - This is a descriptor class used to store an attribute that may contain - Resolver objects. When retrieving the dictionary or list, any Resolver - objects contains are a value or within a list are resolved to a primitive - type. Supports nested dictionary and lists. + This is an abstract base class for a descriptor used to store an attribute that have values + associated with Resolver objects. :param name: Attribute suffix used to store the property in the instance. - :type name: str + :param placeholder_override: If specified, this is the value that will be used as the placeholder + rather than the resolver's returned placeholder value, but only when placeholders are allowed + via the use_resolver_placeholders_on_error context manager. """ def __init__(self, name): self.name = "_" + name self.logger = logging.getLogger(__name__) - self._get_in_progress = False + self._lock = RLock() - def __get__(self, instance, type): + def __get__(self, stack: 'stack.Stack', stack_class: Type['stack.Stack']) -> Any: """ - Attribute getter which resolves any Resolver object contained in the - complex data structure. + Attribute getter which resolves the resolver(s). + :param stack: The Stack instance the property is being retrieved for + :param stack_class: The class of the stack that the property is being retrieved for. :return: The attribute stored with the suffix ``name`` in the instance. - :rtype: dict or list + :rtype: The obtained value, as resolved by the """ - with self._lock, self._no_recursive_get(): - def resolve(attr, key, value): - try: - attr[key] = value.resolve() - except RecursiveGet: - attr[key] = self.ResolveLater(instance, self.name, key, - lambda: value.resolve()) - - if hasattr(instance, self.name): - retval = _call_func_on_values( - resolve, getattr(instance, self.name), Resolver - ) - return retval + with self._lock, self._no_recursive_get(stack): + if hasattr(stack, self.name): + return self.get_resolved_value(stack, stack_class) - def __set__(self, instance, value): + def __set__(self, stack: 'stack.Stack', value: Any): """ Attribute setter which adds a stack reference to any resolvers in the data structure `value` and calls the setup method. + :param stack: The Stack instance the value is being set onto + :param value: The value being set on the property + """ + with self._lock: + self.assign_value_to_stack(stack, value) + + @contextmanager + def _no_recursive_get(self, stack: 'stack.Stack'): + # We don't care about recursive gets on the same property but different Stack instances, + # only recursive gets on the same stack. Some Resolvers access the same property on OTHER + # stacks and that actually shouldn't be a problem. Remember, these descriptor instances are + # set on the CLASS and so instance variables on them are shared across all classes that + # access them. Thus, we set this "get_in_progress" attribute on the stack instance rather + # than the descriptor instance. + get_status_name = f'_{self.name}_get_in_progress' + if getattr(stack, get_status_name, False): + raise RecursiveResolve(f"Resolving Stack.{self.name[1:]} required resolving itself") + setattr(stack, get_status_name, True) + try: + yield + finally: + setattr(stack, get_status_name, False) + + def get_setup_resolver_for_stack(self, stack: 'stack.Stack', resolver: Resolver) -> Resolver: + """Obtains a clone of the resolver with the stack set on it and the setup method having + been called on it. + + :param stack: The stack to set on the Resolver + :param resolver: The Resolver to clone and set up + :return: The cloned resolver. """ - def setup(attr, key, value): - value.stack = instance - value.setup() + # We clone the resolver when we assign the value so that every stack gets its own resolver + # rather than potentially having one resolver instance shared in memory across multiple + # stacks. + clone = resolver.clone(stack) + clone.setup() + return clone + + @abc.abstractmethod + def get_resolved_value(self, stack: 'stack.Stack', stack_class: Type['stack.Stack']) -> Any: + """Implement this method to return the value of the resolvable_property.""" + pass + + @abc.abstractmethod + def assign_value_to_stack(self, stack: 'stack.Stack', value: Any): + """Implement this method to assign the value to the resolvable property.""" + pass + + def resolve_resolver_value(self, resolver: 'Resolver') -> Any: + """Returns the resolved parameter value. + + :param resolver: The resolver to resolve. + :return: The resolved value (or placeholder, in certain circumstances) + """ + return resolver.resolve() + + def __repr__(self) -> str: + return f'<{self.__class__.__name__}({self.name[1:]})>' + + +class ResolvableContainerProperty(ResolvableProperty): + """ + This is a descriptor class used to store an attribute that may CONTAIN + Resolver objects. Resolvers will be resolved upon access of this property. + When resolvers are resolved, they will be replaced in the container with their + resolved value, in order to avoid redundant resolutions. + + Supports nested dictionary and lists. + + :param name: Attribute suffix used to store the property in the instance. + :type name: str + """ + + def __get__(self, stack: 'stack.Stack', stack_class: Type['stack.Stack']) -> T_Container: + container = super().__get__(stack, stack_class) with self._lock: - _call_func_on_values(setup, value, Resolver) - setattr(instance, self.name, value) + # Resolve any deferred resolvers, now that the recursive get lock has been released. + self._resolve_deferred_resolvers(stack, container) + + return container + + def get_resolved_value(self, stack: 'stack.Stack', stack_class: Type['stack.Stack']) -> T_Container: + """Obtains the resolved value for this property. + + :param stack: The Stack instance to obtain the value for + :param stack_class: The class of the Stack instance. + :return: The fully resolved container. + """ + + def resolve(attr: Union[dict, list], key: Union[int, str], value: Resolver): + # Update the container key's value with the resolved value, if possible... + try: + result = self.resolve_resolver_value(value) + attr[key] = result + except RecursiveResolve: + # It's possible that resolving the resolver might attempt to access another + # resolvable property's value in this same container. In this case, we'll delay + # resolution and instead return a ResolveLater so the value can be resolved outside + # this recursion. + attr[key] = self.ResolveLater( + stack, + self.name, + key, + lambda: value.resolve(), + ) + + container = getattr(stack, self.name) + _call_func_on_values( + resolve, container, Resolver + ) + + return container - class ResolveLater(object): + def assign_value_to_stack(self, stack: 'stack.Stack', value: Union[dict, list]): + """Assigns a COPY of the specified value to the stack instance. This method copies the value + rather than directly assigns it to avoid bugs related to shared objects in memory. + + :param stack: The stack to assign the value to + :param value: The value to assign + """ + cloned = self._clone_container_with_resolvers(value, stack) + setattr(stack, self.name, cloned) + + def _clone_container_with_resolvers( + self, + container: T_Container, + stack: 'stack.Stack' + ) -> T_Container: + """Recurses into the container, cloning and setting up resolvers and creating a copy of all + nested containers. + + :param container: The container being recursed into and cloned + :param stack: The stack the container is being copied for + :return: The fully copied container with resolvers fully set up. + """ + def recurse(obj): + if isinstance(obj, Resolver): + return self.get_setup_resolver_for_stack(stack, obj) + if isinstance(obj, list): + return [ + recurse(item) + for item in obj + ] + elif isinstance(obj, dict): + return { + key: recurse(val) + for key, val in obj.items() + } + return obj + + return recurse(container) + + def _resolve_deferred_resolvers(self, stack: 'stack.Stack', container: T_Container): + def raise_if_not_resolved(attr, key, value): + # If this function has been hit, it means that after attempting to resolve all the + # ResolveLaters, there STILL are ResolveLaters left in the container. Rather than + # continuing to try to resolve (possibly infinitely), we'll raise a RecursiveGet to + # break that infinite loop. This situation would happen if a resolver accesses a resolver + # in the same container, which then accesses another resolver (possibly the same one) in + # the same container. + raise RecursiveResolve(f"Resolving Stack.{self.name[1:]} required resolving itself") + + has_been_resolved_attr_name = f'{self.name}_is_resolved' + if not getattr(stack, has_been_resolved_attr_name, False): + # We set it first rather than after to avoid entering this block again on this property + # for this stack. + setattr(stack, has_been_resolved_attr_name, True) + _call_func_on_values( + lambda attr, key, value: value(), + container, + self.ResolveLater + ) + # Search the container to see if there are any ResolveLaters left; + # Raise a RecursiveResolve if there are. + _call_func_on_values( + raise_if_not_resolved, + container, + self.ResolveLater + ) + + class ResolveLater: """Represents a value that could not yet be resolved but can be resolved in the future.""" + def __init__(self, instance, name, key, resolution_function): self._instance = instance self._name = name @@ -115,13 +289,3 @@ def __call__(self): """Resolve the value.""" attr = getattr(self._instance, self._name) attr[self._key] = self._resolution_function() - - @contextmanager - def _no_recursive_get(self): - if self._get_in_progress: - raise RecursiveGet() - self._get_in_progress = True - try: - yield - finally: - self._get_in_progress = False From 5325395e9374d372bd532e5dd28bad3955a186e7 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:19:14 -0600 Subject: [PATCH 02/36] creating stack tags resolvable property; removing unnecessary user data --- sceptre/stack.py | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/sceptre/stack.py b/sceptre/stack.py index 5f956408f..7c8ae7bee 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -8,13 +8,12 @@ """ import logging -from typing import Mapping, Sequence from sceptre.connection_manager import ConnectionManager from sceptre.exceptions import InvalidConfigFileError from sceptre.helpers import get_external_stack_name, sceptreise_path from sceptre.hooks import HookProperty -from sceptre.resolvers import ResolvableProperty +from sceptre.resolvers import ResolvableContainerProperty from sceptre.template import Template @@ -109,14 +108,21 @@ class Stack(object): will result in no timeout. Supports only positive integer value. :type stack_timeout: int + :param is_project_dependency: Indicates whether or not the the stack is a\ + special dependency of the stack. If True, disables all dependencies\ + on the the current stack. + :type is_project_dependency: bool + :param stack_group_config: The StackGroup config for the Stack :type stack_group_config: dict """ - parameters = ResolvableProperty("parameters") - _sceptre_user_data = ResolvableProperty("_sceptre_user_data") - notifications = ResolvableProperty("notifications") + parameters = ResolvableContainerProperty("parameters") + sceptre_user_data = ResolvableContainerProperty("sceptre_user_data") + notifications = ResolvableContainerProperty("notifications") + tags = ResolvableContainerProperty('tags') + hooks = HookProperty("hooks") def __init__( @@ -152,15 +158,16 @@ def __init__( self.role_arn = role_arn self.on_failure = on_failure self.dependencies = dependencies or [] - self.tags = tags or {} self.stack_timeout = stack_timeout - self.iam_role = iam_role self.profile = profile + + self.iam_role = iam_role + self.tags = tags or {} self.hooks = hooks or {} self.parameters = parameters or {} - self._sceptre_user_data = sceptre_user_data or {} - self._sceptre_user_data_is_resolved = False + self.sceptre_user_data = sceptre_user_data or {} self.notifications = notifications or [] + self.stack_group_config = stack_group_config or {} def __repr__(self): @@ -263,17 +270,6 @@ def connection_manager(self): return self._connection_manager - @property - def sceptre_user_data(self): - """Returns sceptre_user_data after ensuring that it is fully resolved. - - :rtype: dict or list or None - """ - if not self._sceptre_user_data_is_resolved: - self._sceptre_user_data_is_resolved = True - self._resolve_sceptre_user_data() - return self._sceptre_user_data - @property def template(self): """ @@ -300,15 +296,3 @@ def template(self): connection_manager=self.connection_manager ) return self._template - - def _resolve_sceptre_user_data(self): - data = self._sceptre_user_data - if isinstance(data, Mapping): - iterator = data.values() - elif isinstance(data, Sequence): - iterator = data - else: - return - for value in iterator: - if isinstance(value, ResolvableProperty.ResolveLater): - value() From fedb989b2ef307158c5c0797aaf2a617f639649c Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:19:47 -0600 Subject: [PATCH 03/36] removing reference to is_project_dependency --- sceptre/stack.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sceptre/stack.py b/sceptre/stack.py index 7c8ae7bee..352528bde 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -108,11 +108,6 @@ class Stack(object): will result in no timeout. Supports only positive integer value. :type stack_timeout: int - :param is_project_dependency: Indicates whether or not the the stack is a\ - special dependency of the stack. If True, disables all dependencies\ - on the the current stack. - :type is_project_dependency: bool - :param stack_group_config: The StackGroup config for the Stack :type stack_group_config: dict From 3065e6c94f0c64c4b852e02cff00704e36bba981 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:25:21 -0600 Subject: [PATCH 04/36] updating tests for fixes --- tests/test_resolvers/test_resolver.py | 85 ++++++++++++++++++++++----- tests/test_stack.py | 2 +- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/tests/test_resolvers/test_resolver.py b/tests/test_resolvers/test_resolver.py index 7e0e1bf8b..2ad6a41a7 100644 --- a/tests/test_resolvers/test_resolver.py +++ b/tests/test_resolvers/test_resolver.py @@ -1,8 +1,14 @@ # -*- coding: utf-8 -*- +from unittest.mock import call, Mock +import pytest from mock import sentinel, MagicMock -from sceptre.resolvers import Resolver, ResolvableProperty +from sceptre.resolvers import ( + Resolver, + ResolvableContainerProperty, + RecursiveResolve +) class MockResolver(Resolver): @@ -17,7 +23,7 @@ def resolve(self): class MockClass(object): - resolvable_property = ResolvableProperty("resolvable_property") + resolvable_container_property = ResolvableContainerProperty("resolvable_container_property") config = MagicMock() @@ -34,14 +40,14 @@ def test_init(self): assert self.mock_resolver.argument == sentinel.argument -class TestResolvablePropertyDescriptor(object): +class TestResolvableContainerPropertyDescriptor(object): def setup_method(self, test_method): self.mock_object = MockClass() def test_setting_resolvable_property_with_none(self): - self.mock_object.resolvable_property = None - assert self.mock_object._resolvable_property is None + self.mock_object.resolvable_container_property = None + assert self.mock_object._resolvable_container_property is None def test_setting_resolvable_property_with_nested_lists(self): mock_resolver = MagicMock(spec=MockResolver) @@ -64,13 +70,35 @@ def test_setting_resolvable_property_with_nested_lists(self): ] ] - self.mock_object.resolvable_property = complex_data_structure - assert self.mock_object._resolvable_property == complex_data_structure - assert mock_resolver.stack == self.mock_object + cloned_data_structure = [ + "String", + mock_resolver.clone.return_value, + [ + mock_resolver.clone.return_value, + "String", + [ + [ + mock_resolver.clone.return_value, + "String", + None + ], + mock_resolver.clone.return_value, + "String" + ] + ] + ] + + self.mock_object.resolvable_container_property = complex_data_structure + assert self.mock_object._resolvable_container_property == cloned_data_structure + expected_calls = [ + call(self.mock_object), + call().setup() + ] * 4 + mock_resolver.clone.assert_has_calls(expected_calls) def test_getting_resolvable_property_with_none(self): - self.mock_object._resolvable_property = None - assert self.mock_object.resolvable_property is None + self.mock_object._resolvable_container_property = None + assert self.mock_object.resolvable_container_property is None def test_getting_resolvable_property_with_nested_lists(self): mock_resolver = MagicMock(spec=MockResolver) @@ -116,8 +144,8 @@ def test_getting_resolvable_property_with_nested_lists(self): None ] - self.mock_object._resolvable_property = complex_data_structure - prop = self.mock_object.resolvable_property + self.mock_object._resolvable_container_property = complex_data_structure + prop = self.mock_object.resolvable_container_property assert prop == resolved_complex_data_structure def test_getting_resolvable_property_with_nested_dictionaries_and_lists( @@ -212,8 +240,8 @@ def test_getting_resolvable_property_with_nested_dictionaries_and_lists( } } - self.mock_object._resolvable_property = complex_data_structure - prop = self.mock_object.resolvable_property + self.mock_object._resolvable_container_property = complex_data_structure + prop = self.mock_object.resolvable_container_property assert prop == resolved_complex_data_structure def test_getting_resolvable_property_with_nested_dictionaries(self): @@ -254,6 +282,31 @@ def test_getting_resolvable_property_with_nested_dictionaries(self): } } - self.mock_object._resolvable_property = complex_data_structure - prop = self.mock_object.resolvable_property + self.mock_object._resolvable_container_property = complex_data_structure + prop = self.mock_object.resolvable_container_property assert prop == resolved_complex_data_structure + + def test_get__resolver_references_same_property_for_other_value__resolves_it(self): + class MyResolver(Resolver): + def resolve(self): + return self.stack.resolvable_container_property['other_value'] + + resolver = MyResolver() + self.mock_object.resolvable_container_property = { + 'other_value': 'abc', + 'resolver': resolver + } + + assert self.mock_object.resolvable_container_property['resolver'] == 'abc' + + def test_get__resolver_references_itself__raises_recursive_resolve(self): + class RecursiveResolver(Resolver): + def resolve(self): + return self.stack.resolvable_container_property['resolver'] + + resolver = RecursiveResolver() + self.mock_object.resolvable_container_property = { + 'resolver': resolver + } + with pytest.raises(RecursiveResolve): + self.mock_object.resolvable_container_property diff --git a/tests/test_stack.py b/tests/test_stack.py index 50cadc51f..7263aaae1 100644 --- a/tests/test_stack.py +++ b/tests/test_stack.py @@ -206,7 +206,7 @@ def resolve(self): return self.stack.sceptre_user_data['primitive'] stack = stack_factory() - stack._sceptre_user_data = { + stack.sceptre_user_data = { 'primitive': sentinel.primitive_value, 'resolved': TestResolver(stack=stack), } From a2a5dc808dff68234eadd6863c5897d774837291 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:30:30 -0600 Subject: [PATCH 05/36] adding test for same_property access --- tests/test_resolvers/test_resolver.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_resolvers/test_resolver.py b/tests/test_resolvers/test_resolver.py index 2ad6a41a7..f6dca1f89 100644 --- a/tests/test_resolvers/test_resolver.py +++ b/tests/test_resolvers/test_resolver.py @@ -310,3 +310,22 @@ def resolve(self): } with pytest.raises(RecursiveResolve): self.mock_object.resolvable_container_property + + def test_get__resolvable_container_property_references_same_property_of_other_stack__resolves(self): + stack1 = MockClass() + stack1.resolvable_container_property = { + 'testing': 'stack1' + } + + class OtherStackResolver(Resolver): + def resolve(self): + return stack1.resolvable_container_property['testing'] + + stack2 = MockClass() + stack2.resolvable_container_property = { + 'resolver': OtherStackResolver() + } + + assert stack2.resolvable_container_property == { + 'resolver': 'stack1' + } From cce17a718577ef98b556000ca302bdd69651128e Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:47:16 -0600 Subject: [PATCH 06/36] deepcopying in strategies --- sceptre/config/strategies.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sceptre/config/strategies.py b/sceptre/config/strategies.py index 125673de2..f04955d3f 100644 --- a/sceptre/config/strategies.py +++ b/sceptre/config/strategies.py @@ -6,6 +6,7 @@ This module contains the implementations of the strategies used to merge config attributes. """ +from copy import deepcopy def list_join(a, b): @@ -21,16 +22,17 @@ def list_join(a, b): """ if a and not isinstance(a, list): raise TypeError('{} is not a list'.format(a)) + if b and not isinstance(b, list): raise TypeError('{} is not a list'.format(b)) if a is None: - return b + return deepcopy(b) if b is not None: - return a + b + return deepcopy(a + b) - return a + return deepcopy(a) def dict_merge(a, b): @@ -50,13 +52,12 @@ def dict_merge(a, b): raise TypeError('{} is not a dict'.format(b)) if a is None: - return b + return deepcopy(b) if b is not None: - a.update(b) - return a + return deepcopy({**a, **b}) - return a + return deepcopy(a) def child_wins(a, b): From b5dd62e63577e111450a4d6e0d854bbba9d30560 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 13:54:12 -0600 Subject: [PATCH 07/36] removing unused import --- tests/test_resolvers/test_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resolvers/test_resolver.py b/tests/test_resolvers/test_resolver.py index f6dca1f89..85a803162 100644 --- a/tests/test_resolvers/test_resolver.py +++ b/tests/test_resolvers/test_resolver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from unittest.mock import call, Mock +from unittest.mock import call import pytest from mock import sentinel, MagicMock From 0eab02a881a42ab547c8af0c90159bc63544875e Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:09:58 -0600 Subject: [PATCH 08/36] creating resolvable value property --- sceptre/resolvers/__init__.py | 48 ++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 0f0a9eda6..7a7c6c881 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -60,15 +60,15 @@ def clone(self, stack: 'stack.Stack') -> 'Resolver': return type(self)(self.argument, stack) +NO_OVERRIDE = 'NO_OVERRIDE' + + class ResolvableProperty(abc.ABC): """ This is an abstract base class for a descriptor used to store an attribute that have values associated with Resolver objects. :param name: Attribute suffix used to store the property in the instance. - :param placeholder_override: If specified, this is the value that will be used as the placeholder - rather than the resolver's returned placeholder value, but only when placeholders are allowed - via the use_resolver_placeholders_on_error context manager. """ def __init__(self, name): @@ -289,3 +289,45 @@ def __call__(self): """Resolve the value.""" attr = getattr(self._instance, self._name) attr[self._key] = self._resolution_function() + + +class ResolvableValueProperty(ResolvableProperty): + """ + This is a descriptor class used to store an attribute that may BE a single + Resolver object. If it is a resolver, it will be resolved upon access of this property. + When resolved, the resolved value will replace the resolver on the stack in order to avoid + redundant resolutions. + + :param name: Attribute suffix used to store the property in the instance. + :type name: str + """ + + def get_resolved_value(self, stack: 'stack.Stack', stack_class: Type['stack.Stack']) -> Any: + """Gets the fully-resolved value from the property. Resolvers will be replaced on the stack + instance with their resolved value to avoid redundant resolutions. + + :param stack: The Stack instance to obtain the value from + :param stack_class: The class of the Stack instance + :return: The fully resolved value + """ + raw_value = getattr(stack, self.name) + if isinstance(raw_value, Resolver): + value = self.resolve_resolver_value(raw_value) + # Overwrite the stored resolver value with the resolved value to avoid resolving the + # same value multiple times. + setattr(stack, self.name, value) + else: + value = raw_value + + return value + + def assign_value_to_stack(self, stack: 'stack.Stack', value: Any): + """Assigns the value to the Stack instance passed, setting up and cloning the value if it + is a Resolver. + + :param stack: The Stack instance to set the value on + :param value: The value to set + """ + if isinstance(value, Resolver): + value = self.get_setup_resolver_for_stack(stack, value) + setattr(stack, self.name, value) From 637095711c893c741619643b7407837ee25fa2bb Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:10:26 -0600 Subject: [PATCH 09/36] removing docs for property that doesn't exist --- sceptre/resolvers/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 0f0a9eda6..edae12dbf 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -66,9 +66,6 @@ class ResolvableProperty(abc.ABC): associated with Resolver objects. :param name: Attribute suffix used to store the property in the instance. - :param placeholder_override: If specified, this is the value that will be used as the placeholder - rather than the resolver's returned placeholder value, but only when placeholders are allowed - via the use_resolver_placeholders_on_error context manager. """ def __init__(self, name): From 9122ae0ad0e8b338dc58e8e45373af92270e1af4 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:11:33 -0600 Subject: [PATCH 10/36] removing references to placeholders --- sceptre/resolvers/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 7a7c6c881..36338a2df 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -60,9 +60,6 @@ def clone(self, stack: 'stack.Stack') -> 'Resolver': return type(self)(self.argument, stack) -NO_OVERRIDE = 'NO_OVERRIDE' - - class ResolvableProperty(abc.ABC): """ This is an abstract base class for a descriptor used to store an attribute that have values @@ -147,7 +144,7 @@ def resolve_resolver_value(self, resolver: 'Resolver') -> Any: """Returns the resolved parameter value. :param resolver: The resolver to resolve. - :return: The resolved value (or placeholder, in certain circumstances) + :return: The resolved value """ return resolver.resolve() From f9fb2f09d5302e9d61bdcd83a53de440d7304ebd Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:12:04 -0600 Subject: [PATCH 11/36] removing reference to placeholder in docstring --- sceptre/resolvers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index edae12dbf..927782001 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -144,7 +144,7 @@ def resolve_resolver_value(self, resolver: 'Resolver') -> Any: """Returns the resolved parameter value. :param resolver: The resolver to resolve. - :return: The resolved value (or placeholder, in certain circumstances) + :return: The resolved value """ return resolver.resolve() From 10a883f0c067a32f39440bb9d0c61430196f0e3e Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:17:50 -0600 Subject: [PATCH 12/36] testing resolvable value propety --- tests/test_resolvers/test_resolver.py | 74 +++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/tests/test_resolvers/test_resolver.py b/tests/test_resolvers/test_resolver.py index 85a803162..0167bae30 100644 --- a/tests/test_resolvers/test_resolver.py +++ b/tests/test_resolvers/test_resolver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from unittest.mock import call +from unittest.mock import call, Mock import pytest from mock import sentinel, MagicMock @@ -7,6 +7,7 @@ from sceptre.resolvers import ( Resolver, ResolvableContainerProperty, + ResolvableValueProperty, RecursiveResolve ) @@ -24,6 +25,7 @@ def resolve(self): class MockClass(object): resolvable_container_property = ResolvableContainerProperty("resolvable_container_property") + resolvable_value_property = ResolvableValueProperty('resolvable_value_property') config = MagicMock() @@ -39,8 +41,7 @@ def test_init(self): assert self.mock_resolver.stack == sentinel.stack assert self.mock_resolver.argument == sentinel.argument - -class TestResolvableContainerPropertyDescriptor(object): +class TestResolvableContainerPropertyDescriptor: def setup_method(self, test_method): self.mock_object = MockClass() @@ -329,3 +330,70 @@ def resolve(self): assert stack2.resolvable_container_property == { 'resolver': 'stack1' } + + +class TestResolvableValueProperty: + def setup_method(self, test_method): + self.mock_object = MockClass() + + @pytest.mark.parametrize( + 'value', + ['string', True, 123, 1.23, None] + ) + def test_set__non_resolver__sets_private_variable_as_value(self, value): + self.mock_object.resolvable_value_property = value + assert self.mock_object._resolvable_value_property == value + + def test_set__resolver__sets_private_variable_with_clone_of_resolver_with_instance(self): + resolver = Mock(spec=MockResolver) + self.mock_object.resolvable_value_property = resolver + assert self.mock_object._resolvable_value_property == resolver.clone.return_value + + def test_set__resolver__sets_up_cloned_resolver(self): + resolver = Mock(spec=MockResolver) + self.mock_object.resolvable_value_property = resolver + resolver.clone.return_value.setup.assert_any_call() + + @pytest.mark.parametrize( + 'value', + ['string', True, 123, 1.23, None] + ) + def test_get__non_resolver__returns_value(self, value): + self.mock_object._resolvable_value_property = value + assert self.mock_object.resolvable_value_property == value + + def test_get__resolver__returns_resolved_value(self): + resolver = Mock(spec=MockResolver) + self.mock_object._resolvable_value_property = resolver + assert self.mock_object.resolvable_value_property == resolver.resolve.return_value + + def test_get__resolver__updates_set_value_with_resolved_value(self): + resolver = Mock(spec=MockResolver) + self.mock_object._resolvable_value_property = resolver + self.mock_object.resolvable_value_property + assert self.mock_object._resolvable_value_property == resolver.resolve.return_value + + def test_get__resolver__resolver_attempts_to_access_resolver__raises_recursive_resolve(self): + class RecursiveResolver(Resolver): + def resolve(self): + # This should blow up! + self.stack.resolvable_value_property + + resolver = RecursiveResolver() + self.mock_object.resolvable_value_property = resolver + + with pytest.raises(RecursiveResolve): + self.mock_object.resolvable_value_property + + def test_get__resolvable_value_property_references_same_property_of_other_stack__resolves(self): + stack1 = MockClass() + stack1.resolvable_value_property = 'stack1' + + class OtherStackResolver(Resolver): + def resolve(self): + return stack1.resolvable_value_property + + stack2 = MockClass() + stack2.resolvable_value_property = OtherStackResolver() + + assert stack2.resolvable_value_property == 'stack1' From 2a043fb19edebe5078785b063d6782aa87efeb49 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:22:30 -0600 Subject: [PATCH 13/36] making template_bucket_name and template_key_prefix resolvable properties --- sceptre/stack.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/sceptre/stack.py b/sceptre/stack.py index 352528bde..40cbc0007 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -13,7 +13,7 @@ from sceptre.exceptions import InvalidConfigFileError from sceptre.helpers import get_external_stack_name, sceptreise_path from sceptre.hooks import HookProperty -from sceptre.resolvers import ResolvableContainerProperty +from sceptre.resolvers import ResolvableContainerProperty, ResolvableValueProperty from sceptre.template import Template @@ -116,7 +116,13 @@ class Stack(object): parameters = ResolvableContainerProperty("parameters") sceptre_user_data = ResolvableContainerProperty("sceptre_user_data") notifications = ResolvableContainerProperty("notifications") - tags = ResolvableContainerProperty('tags') + tags = ResolvableContainerProperty("tags") + + s3_details = ResolvableContainerProperty("s3_details") + template_bucket_name = ResolvableValueProperty("template_bucket_name") + template_key_prefix = ResolvableValueProperty("template_key_prefix") + + role_arn = ResolvableValueProperty("role_arn") hooks = HookProperty("hooks") @@ -139,8 +145,6 @@ def __init__( self.name = sceptreise_path(name) self.project_code = project_code self.region = region - self.template_bucket_name = template_bucket_name - self.template_key_prefix = template_key_prefix self.required_version = required_version self.external_name = external_name or get_external_stack_name(self.project_code, self.name) self.template_path = template_path @@ -150,7 +154,6 @@ def __init__( self._connection_manager = None self.protected = protected - self.role_arn = role_arn self.on_failure = on_failure self.dependencies = dependencies or [] self.stack_timeout = stack_timeout @@ -159,6 +162,10 @@ def __init__( self.iam_role = iam_role self.tags = tags or {} self.hooks = hooks or {} + self.role_arn = role_arn + self.s3_details = s3_details + self.template_key_prefix = template_key_prefix + self.template_bucket_name = template_bucket_name self.parameters = parameters or {} self.sceptre_user_data = sceptre_user_data or {} self.notifications = notifications or [] From b502bb1c3c0e06fd121c7a02e5a8202d111916d5 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:22:53 -0600 Subject: [PATCH 14/36] removing unnecessary line --- sceptre/stack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sceptre/stack.py b/sceptre/stack.py index 40cbc0007..40d25db29 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -121,7 +121,6 @@ class Stack(object): s3_details = ResolvableContainerProperty("s3_details") template_bucket_name = ResolvableValueProperty("template_bucket_name") template_key_prefix = ResolvableValueProperty("template_key_prefix") - role_arn = ResolvableValueProperty("role_arn") hooks = HookProperty("hooks") From c3207c0b470971b965454432ee0a3620157c5d2f Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:23:17 -0600 Subject: [PATCH 15/36] adding extra line --- tests/test_resolvers/test_resolver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_resolvers/test_resolver.py b/tests/test_resolvers/test_resolver.py index 0167bae30..777745353 100644 --- a/tests/test_resolvers/test_resolver.py +++ b/tests/test_resolvers/test_resolver.py @@ -41,6 +41,7 @@ def test_init(self): assert self.mock_resolver.stack == sentinel.stack assert self.mock_resolver.argument == sentinel.argument + class TestResolvableContainerPropertyDescriptor: def setup_method(self, test_method): From 391370bb59f00b4b62b94dcdd5b5b59850b66e19 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:28:50 -0600 Subject: [PATCH 16/36] handling case of template_bucket_name set to None --- sceptre/template.py | 3 ++- tests/test_template.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sceptre/template.py b/sceptre/template.py index e7a1eb0af..e3f08c682 100644 --- a/sceptre/template.py +++ b/sceptre/template.py @@ -215,7 +215,8 @@ def get_boto_call_parameter(self): :returns: The boto call parameter for the template. :rtype: dict """ - if self.s3_details: + # If bucket_name is set to None, it should be ignored and not uploaded. + if self.s3_details and self.s3_details.get("bucket_name") is not None: url = self.upload_to_s3() return {"TemplateURL": url} else: diff --git a/tests/test_template.py b/tests/test_template.py index f98dda34d..3143e389c 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -200,6 +200,17 @@ def test_get_boto_call_parameter_with_s3_details(self, mock_upload_to_s3): assert boto_parameter == {"TemplateURL": sentinel.template_url} + def test_get_boto_call_parameter__has_s3_details_but_bucket_name_is_none__gets_template_body_dict(self): + self.template._body = sentinel.body + self.template.s3_details = { + "bucket_name": None, + "bucket_key": sentinel.bucket_key + } + + boto_parameter = self.template.get_boto_call_parameter() + + assert boto_parameter == {"TemplateBody": sentinel.body} + def test_get_template_details_without_upload(self): self.template.s3_details = None self.template._body = sentinel.body From 7f95f8bf7659e28efeda91f6a8eb30c676feff68 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:47:35 -0600 Subject: [PATCH 17/36] fixing reader --- sceptre/config/reader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sceptre/config/reader.py b/sceptre/config/reader.py index 779cfa625..51bc31aec 100644 --- a/sceptre/config/reader.py +++ b/sceptre/config/reader.py @@ -475,7 +475,9 @@ def _collect_s3_details(stack_name, config): :rtype: dict """ s3_details = None - if "template_bucket_name" in config: + # If the config explicitly sets the template_bucket_name to None, we don't want to enter + # this conditional block. + if config.get("template_bucket_name"): template_key = "/".join([ sceptreise_path(stack_name), "{time_stamp}.json".format( time_stamp=datetime.datetime.utcnow().strftime( From 393687776e412994f89082446156db707e365603 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 7 Nov 2021 14:48:45 -0600 Subject: [PATCH 18/36] making conditional more explicit --- sceptre/config/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/config/reader.py b/sceptre/config/reader.py index 51bc31aec..a85e2c504 100644 --- a/sceptre/config/reader.py +++ b/sceptre/config/reader.py @@ -477,7 +477,7 @@ def _collect_s3_details(stack_name, config): s3_details = None # If the config explicitly sets the template_bucket_name to None, we don't want to enter # this conditional block. - if config.get("template_bucket_name"): + if config.get("template_bucket_name") is not None: template_key = "/".join([ sceptreise_path(stack_name), "{time_stamp}.json".format( time_stamp=datetime.datetime.utcnow().strftime( From d97bd15c0507cf5b1c9bb559378bd44b207f2f5b Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Tue, 9 Nov 2021 09:16:28 -0600 Subject: [PATCH 19/36] backing off better --- integration-tests/steps/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/steps/helpers.py b/integration-tests/steps/helpers.py index 792be1ff2..1995b8cf2 100644 --- a/integration-tests/steps/helpers.py +++ b/integration-tests/steps/helpers.py @@ -73,7 +73,7 @@ def get_cloudformation_stack_name(context, stack_name): def retry_boto_call(func, *args, **kwargs): - delay = 2 + delay = 5 max_retries = 150 attempts = 0 while attempts < max_retries: From a05b16ab96581da1d086ad6c825cc5d9add7c6dd Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 24 Nov 2021 15:45:54 -0600 Subject: [PATCH 20/36] removing module logger because it wasn't used --- sceptre/resolvers/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 927782001..096336e2e 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -12,8 +12,6 @@ T_Container = TypeVar('T_Container', bound=Union[dict, list]) -logger = logging.getLogger(__name__) - class RecursiveResolve(Exception): pass From 805b9b7321c9d4e9cf22349b3839c04f5de3641c Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 24 Nov 2021 15:47:10 -0600 Subject: [PATCH 21/36] fixing docstring --- sceptre/resolvers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/resolvers/__init__.py b/sceptre/resolvers/__init__.py index 096336e2e..e84cd6edc 100644 --- a/sceptre/resolvers/__init__.py +++ b/sceptre/resolvers/__init__.py @@ -79,7 +79,7 @@ def __get__(self, stack: 'stack.Stack', stack_class: Type['stack.Stack']) -> Any :param stack: The Stack instance the property is being retrieved for :param stack_class: The class of the stack that the property is being retrieved for. :return: The attribute stored with the suffix ``name`` in the instance. - :rtype: The obtained value, as resolved by the + :rtype: The obtained value, as resolved by the property """ with self._lock, self._no_recursive_get(stack): if hasattr(stack, self.name): From 2aeb0c52f9fc91f91f23c7488012aa76b33d1549 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 09:17:46 -0600 Subject: [PATCH 22/36] getting docs to understand type hints --- docs/_source/conf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/_source/conf.py b/docs/_source/conf.py index 454b94e12..874842185 100644 --- a/docs/_source/conf.py +++ b/docs/_source/conf.py @@ -42,6 +42,7 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.coverage', @@ -204,5 +205,11 @@ ('py:class', 'sceptre.config.reader.Attributes'), ('py:class', 'sceptre.diffing.stack_differ.DiffType'), ('py:obj', 'sceptre.diffing.stack_differ.DiffType'), - ('py:class', 'TextIO') + ('py:class', 'DiffType'), + ('py:class', 'TextIO'), + ('py:class', '_io.StringIO'), + ('py:class', 'yaml.loader.SafeLoader'), + ('py:class', 'yaml.dumper.Dumper') ] + +set_type_checking_flag = True From a3964eb0a55335a4535f63fc48f87648df14dfb8 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 09:22:31 -0600 Subject: [PATCH 23/36] improving resolver docs --- docs/_source/docs/stack_config.rst | 58 +++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/_source/docs/stack_config.rst b/docs/_source/docs/stack_config.rst index cbe9e8c78..9b9487cda 100644 --- a/docs/_source/docs/stack_config.rst +++ b/docs/_source/docs/stack_config.rst @@ -1,8 +1,9 @@ Stack Config ============ -Stack config stores config related to a particular Stack, such as the path to -that Stack’s Template, and any parameters that Stack may require. +A Stack config stores configurations related to a particular Stack, such as the path to +that Stack’s Template, and any parameters that Stack may require. Many of these configuration keys +support resolvers and can be inherited from parent StackGroup configs. .. _stack_config-structure: @@ -31,6 +32,8 @@ you will receive an error when deploying the stack. template_path ~~~~~~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: No The path to the CloudFormation, Jinja2 or Python template to build the Stack from. The path can either be absolute or relative to the Sceptre Directory. @@ -44,6 +47,8 @@ from the Stack config filename. template ~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: No Configuration for a template handler. Template handlers can take in parameters and resolve that to a CloudFormation template. This enables you to not only @@ -66,6 +71,9 @@ developing your own in the :doc:`template_handlers` section. dependencies ~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Appended to parent's dependencies A list of other Stacks in the environment that this Stack depends on. Note that if a Stack fetches an output value from another Stack using the @@ -74,12 +82,18 @@ and that Stack need not be added as an explicit dependency. hooks ~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set A list of arbitrary shell or Python commands or scripts to run. Find out more in the :doc:`hooks` section. notifications ~~~~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set List of SNS topic ARNs to publish Stack related events to. A maximum of 5 ARNs can be specified per Stack. This configuration will be used by the ``create``, @@ -89,6 +103,9 @@ documentation`_. on_failure ~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set This parameter describes the action taken by CloudFormation when a Stack fails to create. For more information and valid values see the `AWS Documentation`_. @@ -104,6 +121,9 @@ Examples include: parameters ~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set .. warning:: @@ -166,6 +186,9 @@ Example: protected ~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set Stack protection against execution of the following commands: @@ -180,12 +203,18 @@ throw an error. role_arn ~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set The ARN of a `CloudFormation Service Role`_ that is assumed by CloudFormation to create, update or delete resources. iam_role ~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set This is the IAM Role ARN that **Sceptre** should *assume* using AWS STS when executing any actions on the Stack. @@ -208,6 +237,9 @@ permits the user to assume that role. sceptre_user_data ~~~~~~~~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set Represents data to be passed to the ``sceptre_handler(sceptre_user_data)`` function in Python templates or accessible under ``sceptre_user_data`` variable @@ -215,6 +247,8 @@ key within Jinja2 templates. stack_name ~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: No A custom name to use instead of the Sceptre default. @@ -248,11 +282,17 @@ referring to is in a different AWS account or region. stack_tags ~~~~~~~~~~ +* Resolvable: Yes +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set A dictionary of `CloudFormation Tags`_ to be applied to the Stack. stack_timeout ~~~~~~~~~~~~~ +* Resolvable: No +* Can be inherited from StackGroup: Yes +* Inheritance strategy: Overrides parent if set A timeout in minutes before considering the Stack deployment as failed. After the specified timeout, the Stack will be rolled back. Specifiyng zero, as well @@ -360,19 +400,19 @@ Examples tag_1: value_1 tag_2: value_2 -.. _template_path: #template_path +.. _template_path: #template-path .. _template: #template .. _dependencies: #dependencies .. _hooks: #hooks .. _notifications: #notifications -.. _on_failure: #on_failure +.. _on_failure: #on-failure .. _parameters: #parameters .. _protected: #protected -.. _role_arn: #role_arn -.. _sceptre_user_data: #sceptre_user_data -.. _stack_name: #stack_name -.. _stack_tags: #stack_tags -.. _stack_timeout: #stack_timeout +.. _role_arn: #role-arn +.. _sceptre_user_data: #sceptre-user-data +.. _stack_name: #stack-name +.. _stack_tags: #stack-tags +.. _stack_timeout: #stack-timeout .. _AWS CloudFormation API documentation: http://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html .. _AWS Documentation: http://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html .. _CloudFormation Service Role: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-iam-servicerole.html From b877276a7829e96b466843ce9ced53926fbc6270 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 09:25:39 -0600 Subject: [PATCH 24/36] documenting stack group configs --- docs/_source/docs/stack_group_config.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/_source/docs/stack_group_config.rst b/docs/_source/docs/stack_group_config.rst index 0c4592d65..ffb35f434 100644 --- a/docs/_source/docs/stack_group_config.rst +++ b/docs/_source/docs/stack_group_config.rst @@ -28,6 +28,8 @@ made available via ``stack_group_confg`` attribute on ``Stack()``. profile ~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child The name of the profile as defined in ``~/.aws/config`` and ``~/.aws/credentials``. Use the `aws configure --profile ` command @@ -37,17 +39,23 @@ Reference: `AWS_CLI_Configure`_ project_code ~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child A string which is prepended to the Stack names of all Stacks built by Sceptre. region ~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child The AWS region to build Stacks in. Sceptre should work in any `region which supports CloudFormation`_. template_bucket_name ~~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child The name of an S3 bucket to upload CloudFormation Templates to. Note that S3 bucket names must be globally unique. If the bucket does not exist, Sceptre @@ -60,6 +68,8 @@ supplied in this way have a lower maximum length, so using the template_key_prefix ~~~~~~~~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Overrides parent if set by child A string which is prefixed onto the key used to store templates uploaded to S3. Templates are stored using the key: @@ -77,7 +87,9 @@ Note that if ``template_bucket_name`` is not supplied, this parameter is ignored. j2_environment -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~ +* Resolvable: No +* Inheritance strategy: Child configs will be merged with parent configs A dictionary that is combined with the default jinja2 environment. It's converted to keyword arguments then passed to [jinja2.Environment](https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.Environment). @@ -134,7 +146,8 @@ General configurations should be defined at a high level, and more specific configurations should be defined at a lower directory level. YAML files that define configuration settings with conflicting keys, the child -configuration file will take precedence. +configuration file will usually take precedence (see the specific config keys as documented +for the inheritance strategy employed). In the above directory structure, ``config/config.yaml`` will be read in first, followed by ``config/account-1/config.yaml``, followed by From 8443892a15fc90d0ec3c6c981b87d517005d9f10 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 09:30:52 -0600 Subject: [PATCH 25/36] adding typehints autodoc requirement --- requirements/dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index b8832c610..75d83f057 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,6 +15,7 @@ setuptools>=40.6.2,<41.0.0 Sphinx>=1.6.5,<4.3 sphinx-click>=2.0.1,<4.0.0 sphinx-rtd-theme==0.5.2 +sphinx-autodoc-typehints==1.12.0 docutils<0.17 # temporary fix for sphinx-rtd-theme==0.5.2, it depends on docutils<0.17 tox>=3.23.0,<4.0.0 twine>=1.12.1,<2.0.0 From 31198ff453837f61c55465e0d5a91a80c768bfb3 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 10:09:04 -0600 Subject: [PATCH 26/36] documenting new resolvable properties --- docs/_source/docs/stack_config.rst | 4 +-- docs/_source/docs/stack_group_config.rst | 43 ++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docs/_source/docs/stack_config.rst b/docs/_source/docs/stack_config.rst index 9b9487cda..aebb521db 100644 --- a/docs/_source/docs/stack_config.rst +++ b/docs/_source/docs/stack_config.rst @@ -203,11 +203,11 @@ throw an error. role_arn ~~~~~~~~ -* Resolvable: No +* Resolvable: Yes * Can be inherited from StackGroup: Yes * Inheritance strategy: Overrides parent if set -The ARN of a `CloudFormation Service Role`_ that is assumed by CloudFormation +The ARN of a `CloudFormation Service Role`_ that is assumed by *CloudFormation* (not Sceptre) to create, update or delete resources. iam_role diff --git a/docs/_source/docs/stack_group_config.rst b/docs/_source/docs/stack_group_config.rst index ffb35f434..016663351 100644 --- a/docs/_source/docs/stack_group_config.rst +++ b/docs/_source/docs/stack_group_config.rst @@ -54,7 +54,7 @@ supports CloudFormation`_. template_bucket_name ~~~~~~~~~~~~~~~~~~~~ -* Resolvable: No +* Resolvable: Yes * Inheritance strategy: Overrides parent if set by child The name of an S3 bucket to upload CloudFormation Templates to. Note that S3 @@ -66,9 +66,16 @@ supplies the template to Boto3 via the ``TemplateBody`` argument. Templates supplied in this way have a lower maximum length, so using the ``template_bucket_name`` parameter is recommended. +.. warning:: + + If you resolve ``template_bucket_name`` using the ``!stack_output`` + resolver on a StackGroup, the stack that outputs that bucket name *cannot* be + defined in that StackGroup. Otherwise, a circular dependency will exist and Sceptre + will raise an error when attempting any Stack action. + template_key_prefix ~~~~~~~~~~~~~~~~~~~ -* Resolvable: No +* Resolvable: Yes * Inheritance strategy: Overrides parent if set by child A string which is prefixed onto the key used to store templates uploaded to S3. @@ -157,6 +164,38 @@ For example, if you wanted the ``dev`` StackGroup to build to a different region, this setting could be specified in the ``config/dev/config.yaml`` file, and would only be applied to builds in the ``dev`` StackGroup. +.. _setting_dependencies_for_stack_groups: + +Setting Dependencies for StackGroups +------------------------------------ +There are a few pieces of AWS infrastructure that Sceptre can (optionally) use to support the needs +and concerns of the project. These include: + +* The S3 bucket where templates are uploaded to and then referenced from for stack actions (i.e. the + ``template_bucket_name`` config key). +* The CloudFormation service role added to the stack(s) that CloudFormation uses to execute stack + actions (i.e. the ``role_arn`` config key). +* The role that Sceptre will assume to execute stack actions (i.e. the ``iam_role`` config key). +* SNS topics that cloudformation will notify with the results of stack actions (i.e. the + ``notifications`` config key). + +These sorts of dependencies CAN be defined in Sceptre and added at the StackGroup level, referenced +using ``!stack_output``. Doing so will make it so that every stack in the StackGroup will have those +dependencies and get those values from Sceptre-managed stacks. + +Beyond the above mentioned config keys, it is possible to set the ``dependencies`` config key in a +StackGroup config to be inherited by all Stack configs in that group. All dependencies in child +stacks will be added to their inherited StackGroup dependencies, so be careful how you structure +dependencies. + +.. warning:: + + You might have already considered that this might cause a circular dependency for those + dependency stacks, the ones that output the template bucket name, role arn, iam_role, or topic arns. + In order to avoid the circular dependency issue, it is important that you define these items in a + Stack that is *outside* the StackGroup you reference them in. + + .. _stack_group_config_templating: Templating From db847935673d5d74fb75dcee0be8f6223d46613e Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 10:51:49 -0600 Subject: [PATCH 27/36] integration testing stack group dependencies --- .../features/dependency-resolution.feature | 12 +-- .../features/project-dependencies.feature | 19 ++++ .../features/validate-template.feature | 2 +- .../project-deps/dependencies/bucket.yaml | 3 + .../project-deps/dependencies/topic.yaml | 3 + .../project-deps/main-project/config.yaml | 3 + .../project-deps/main-project/resource.yaml | 2 + .../templates/jinja/valid_template.j2 | 2 +- .../project-dependencies/bucket.yaml | 10 ++ .../templates/project-dependencies/topic.yaml | 10 ++ .../templates/valid_template.yaml | 2 +- .../steps/project_dependencies.py | 97 +++++++++++++++++++ integration-tests/steps/stack_groups.py | 31 +++++- integration-tests/steps/stacks.py | 2 +- sceptre/stack.py | 18 ++-- 15 files changed, 195 insertions(+), 21 deletions(-) create mode 100644 integration-tests/features/project-dependencies.feature create mode 100644 integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml create mode 100644 integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml create mode 100644 integration-tests/sceptre-project/config/project-deps/main-project/config.yaml create mode 100644 integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml create mode 100644 integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml create mode 100644 integration-tests/sceptre-project/templates/project-dependencies/topic.yaml create mode 100644 integration-tests/steps/project_dependencies.py diff --git a/integration-tests/features/dependency-resolution.feature b/integration-tests/features/dependency-resolution.feature index a10a67e17..861e3e97d 100644 --- a/integration-tests/features/dependency-resolution.feature +++ b/integration-tests/features/dependency-resolution.feature @@ -2,17 +2,17 @@ Feature: Dependency resolution Scenario: launch a stack_group with dependencies that is partially complete Given stack "3/A" exists in "CREATE_COMPLETE" state - and stack "3/B" exists in "CREATE_COMPLETE" state - and stack "3/C" does not exist + And stack "3/B" exists in "CREATE_COMPLETE" state + And stack "3/C" does not exist When the user launches stack_group "3" Then all the stacks in stack_group "3" are in "CREATE_COMPLETE" - and that stack "3/A" was created before "3/B" - and that stack "3/B" was created before "3/C" + And that stack "3/A" was created before "3/B" + And that stack "3/B" was created before "3/C" Scenario: delete a stack_group with dependencies that is partially complete Given stack "3/A" exists in "CREATE_COMPLETE" state - and stack "3/B" exists in "CREATE_COMPLETE" state - and stack "3/C" does not exist + And stack "3/B" exists in "CREATE_COMPLETE" state + And stack "3/C" does not exist When the user deletes stack_group "3" Then all the stacks in stack_group "3" do not exist diff --git a/integration-tests/features/project-dependencies.feature b/integration-tests/features/project-dependencies.feature new file mode 100644 index 000000000..8229f1dbf --- /dev/null +++ b/integration-tests/features/project-dependencies.feature @@ -0,0 +1,19 @@ +Feature: StackGroup Dependencies managed within Sceptre + + Background: + Given stack_group "project-deps" does not exist + + Scenario: launch stack group with dependencies + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then all the stacks in stack_group "project-deps" are in "CREATE_COMPLETE" + + Scenario: template_bucket_name is managed in stack group + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then the template for stack "project-deps/main-project/resource" has been uploaded + + Scenario: notifications are managed in stack group + Given all files in template bucket for stack "project-deps/main-project/resource" are deleted at cleanup + When the user launches stack_group "project-deps" + Then the stack "project-deps/main-project/resource" has a notification defined by stack "project-deps/dependencies/topic" diff --git a/integration-tests/features/validate-template.feature b/integration-tests/features/validate-template.feature index 2cc3411a2..88b4a672b 100644 --- a/integration-tests/features/validate-template.feature +++ b/integration-tests/features/validate-template.feature @@ -11,7 +11,7 @@ Feature: Validate template Then a "ClientError" is raised and the user is told "the template is malformed" - Scenario: validate a vaild template with ignore dependencies + Scenario: validate a valid template with ignore dependencies Given the template for stack "1/A" is "valid_template.json" When the user validates the template for stack "1/A" with ignore dependencies Then the user is told "the template is valid" diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml new file mode 100644 index 000000000..5918db469 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml @@ -0,0 +1,3 @@ +is_project_dependency: True +template: + path: project-dependencies/bucket.yaml diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml new file mode 100644 index 000000000..07c3e0e79 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml @@ -0,0 +1,3 @@ +is_project_dependency: True +template: + path: project-dependencies/topic.yaml diff --git a/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml b/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml new file mode 100644 index 000000000..83ff95822 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml @@ -0,0 +1,3 @@ +template_bucket_name: !stack_output project-deps/dependencies/bucket.yaml::BucketName +notifications: + - !stack_output project-deps/dependencies/topic.yaml::TopicArn diff --git a/integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml b/integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml new file mode 100644 index 000000000..339227ae6 --- /dev/null +++ b/integration-tests/sceptre-project/config/project-deps/main-project/resource.yaml @@ -0,0 +1,2 @@ +template: + path: "valid_template.yaml" diff --git a/integration-tests/sceptre-project/templates/jinja/valid_template.j2 b/integration-tests/sceptre-project/templates/jinja/valid_template.j2 index bb7f35bb4..09d89d2c0 100644 --- a/integration-tests/sceptre-project/templates/jinja/valid_template.j2 +++ b/integration-tests/sceptre-project/templates/jinja/valid_template.j2 @@ -1,4 +1,4 @@ Resources: WaitConditionHandle: Type: "{{ sceptre_user_data.type }}" - Properties: + Properties: {} diff --git a/integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml b/integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml new file mode 100644 index 000000000..69b5c76e3 --- /dev/null +++ b/integration-tests/sceptre-project/templates/project-dependencies/bucket.yaml @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: { } + +Outputs: + BucketName: + Value: !Ref Bucket diff --git a/integration-tests/sceptre-project/templates/project-dependencies/topic.yaml b/integration-tests/sceptre-project/templates/project-dependencies/topic.yaml new file mode 100644 index 000000000..806145ea9 --- /dev/null +++ b/integration-tests/sceptre-project/templates/project-dependencies/topic.yaml @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Resources: + Topic: + Type: AWS::SNS::Topic + Properties: {} + +Outputs: + TopicArn: + Value: !Ref Topic diff --git a/integration-tests/sceptre-project/templates/valid_template.yaml b/integration-tests/sceptre-project/templates/valid_template.yaml index 3d435e9ca..9843e4fb2 100644 --- a/integration-tests/sceptre-project/templates/valid_template.yaml +++ b/integration-tests/sceptre-project/templates/valid_template.yaml @@ -1,4 +1,4 @@ Resources: WaitConditionHandle: Type: "AWS::CloudFormation::WaitConditionHandle" - Properties: + Properties: {} diff --git a/integration-tests/steps/project_dependencies.py b/integration-tests/steps/project_dependencies.py new file mode 100644 index 000000000..156c4020a --- /dev/null +++ b/integration-tests/steps/project_dependencies.py @@ -0,0 +1,97 @@ +from itertools import chain +from typing import ContextManager + +import boto3 +from behave import given, then, when +from behave.runner import Context + +from helpers import get_cloudformation_stack_name, retry_boto_call +from sceptre.context import SceptreContext +from sceptre.plan.plan import SceptrePlan + + +@given('all files in template bucket for stack "{stack_name}" are deleted at cleanup') +def step_impl(context: Context, stack_name): + """Add this as a given to ensure that the template bucket is cleaned up before we attempt to + delete it; Otherwise, it will fail since you can't delete a bucket with objects in it. + """ + context.add_cleanup( + cleanup_template_files_in_bucket, + context.sceptre_dir, + stack_name + ) + + +@then('the template for stack "{stack_name}" has been uploaded') +def step_impl(context: Context, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + '.yaml', + project_path=context.sceptre_dir + ) + plan = SceptrePlan(sceptre_context) + buckets = get_template_buckets(plan) + assert len(buckets) > 0 + filtered_objects = list(chain.from_iterable( + bucket.objects.filter( + Prefix=stack_name + ) + for bucket in buckets + )) + + assert len(filtered_objects) == len(plan.command_stacks) + for stack in plan.command_stacks: + for obj in filtered_objects: + if obj.key.startswith(stack.name): + s3_template = obj.get()['Body'].read().decode('utf-8') + expected = stack.template.body + assert s3_template == expected + break + else: + assert False, "Could not found uploaded template" + + +@then('the stack "{resource_stack_name}" has a notification defined by stack "{topic_stack_name}"') +def step_impl(context, resource_stack_name, topic_stack_name): + topic_stack_resources = get_stack_resources(context, topic_stack_name) + topic = topic_stack_resources[0]['PhysicalResourceId'] + resource_stack = describe_stack(context, resource_stack_name) + notification_arns = resource_stack['NotificationARNs'] + assert topic in notification_arns + + +def cleanup_template_files_in_bucket(sceptre_dir, stack_name): + sceptre_context = SceptreContext( + command_path=stack_name + '.yaml', + project_path=sceptre_dir + ) + plan = SceptrePlan(sceptre_context) + buckets = get_template_buckets(plan) + for bucket in buckets: + bucket.objects.delete() + + +def get_template_buckets(plan: SceptrePlan): + s3_resource = boto3.resource('s3') + return [ + s3_resource.Bucket(stack.template_bucket_name) + for stack in plan.command_stacks + if stack.template_bucket_name is not None + ] + + +def get_stack_resources(context, stack_name): + cf_stack_name = get_cloudformation_stack_name(context, stack_name) + resources = retry_boto_call( + context.client.describe_stack_resources, + StackName=cf_stack_name + ) + return resources['StackResources'] + + +def describe_stack(context, stack_name): + cf_stack_name = get_cloudformation_stack_name(context, stack_name) + response = retry_boto_call( + context.client.describe_stacks, + StackName=cf_stack_name + ) + return response['Stacks'][0] diff --git a/integration-tests/steps/stack_groups.py b/integration-tests/steps/stack_groups.py index 5d22c4ed6..0109df928 100644 --- a/integration-tests/steps/stack_groups.py +++ b/integration-tests/steps/stack_groups.py @@ -1,11 +1,15 @@ -from behave import * import os import time -from sceptre.plan.plan import SceptrePlan -from sceptre.context import SceptreContext + +from behave import * from botocore.exceptions import ClientError + from helpers import read_template_file, get_cloudformation_stack_name from helpers import retry_boto_call +from sceptre.context import SceptreContext +from sceptre.diffing.diff_writer import DeepDiffWriter +from sceptre.diffing.stack_differ import DeepDiffStackDiffer, DifflibStackDiffer +from sceptre.plan.plan import SceptrePlan from stacks import wait_for_final_state from templates import set_template_path @@ -266,6 +270,27 @@ def step_impl(context, first_stack, second_stack): assert creation_times[stacks[0]] < creation_times[stacks[1]] +@when('the user diffs stack group "{group_name}" with "{diff_type}"') +def step_impl(context, group_name, diff_type): + sceptre_context = SceptreContext( + command_path=group_name, + project_path=context.sceptre_dir + ) + sceptre_plan = SceptrePlan(sceptre_context) + differ_classes = { + 'deepdiff': DeepDiffStackDiffer, + 'difflib': DifflibStackDiffer + } + writer_class = { + 'deepdiff': DeepDiffWriter, + 'difflib': DeepDiffWriter + } + + differ = differ_classes[diff_type]() + context.writer_class = writer_class[diff_type] + context.output = list(sceptre_plan.diff(differ).values()) + + def get_stack_creation_times(context, stacks): creation_times = {} response = retry_boto_call(context.client.describe_stacks) diff --git a/integration-tests/steps/stacks.py b/integration-tests/steps/stacks.py index 21b97477c..eddcd810c 100644 --- a/integration-tests/steps/stacks.py +++ b/integration-tests/steps/stacks.py @@ -505,7 +505,7 @@ def delete_stack(context, stack_name): retry_boto_call(stack.delete) waiter = context.client.get_waiter('stack_delete_complete') - waiter.config.delay = 4 + waiter.config.delay = 5 waiter.config.max_attempts = 240 waiter.wait(StackName=stack_name) diff --git a/sceptre/stack.py b/sceptre/stack.py index ca4bf4010..bfa9cf435 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -148,28 +148,30 @@ def __init__( self.external_name = external_name or get_external_stack_name(self.project_code, self.name) self.template_path = template_path self.template_handler_config = template_handler_config - self.s3_details = s3_details - self._template = None - self._connection_manager = None - + self.dependencies = dependencies or [] self.protected = protected self.on_failure = on_failure - self.dependencies = dependencies or [] + self.stack_group_config = stack_group_config or {} self.stack_timeout = stack_timeout self.profile = profile + self._template = None + self._connection_manager = None + + # Resolvers and hooks need to be assigned last + self.s3_details = s3_details self.iam_role = iam_role self.tags = tags or {} - self.hooks = hooks or {} self.role_arn = role_arn - self.s3_details = s3_details self.template_key_prefix = template_key_prefix self.template_bucket_name = template_bucket_name + + self.s3_details = s3_details self.parameters = parameters or {} self.sceptre_user_data = sceptre_user_data or {} self.notifications = notifications or [] - self.stack_group_config = stack_group_config or {} + self.hooks = hooks or {} def __repr__(self): return ( From 66b597e06cb69987d04549636e2b8fa621847751 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 10:53:43 -0600 Subject: [PATCH 28/36] removing unsupported config property --- .../sceptre-project/config/project-deps/dependencies/bucket.yaml | 1 - .../sceptre-project/config/project-deps/dependencies/topic.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml index 5918db469..8a67bf664 100644 --- a/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/bucket.yaml @@ -1,3 +1,2 @@ -is_project_dependency: True template: path: project-dependencies/bucket.yaml diff --git a/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml b/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml index 07c3e0e79..8593392f3 100644 --- a/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml +++ b/integration-tests/sceptre-project/config/project-deps/dependencies/topic.yaml @@ -1,3 +1,2 @@ -is_project_dependency: True template: path: project-dependencies/topic.yaml From 3eff1fb4dbd45a945e97b6bb3e9c0b3884b5a880 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 10:59:22 -0600 Subject: [PATCH 29/36] template key prefix cannot be resolvable --- docs/_source/docs/stack_group_config.rst | 2 +- integration-tests/sceptre-project/config/9/B.yaml | 2 +- .../config/project-deps/main-project/config.yaml | 2 ++ sceptre/stack.py | 3 +-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/_source/docs/stack_group_config.rst b/docs/_source/docs/stack_group_config.rst index 016663351..83880ed54 100644 --- a/docs/_source/docs/stack_group_config.rst +++ b/docs/_source/docs/stack_group_config.rst @@ -75,7 +75,7 @@ supplied in this way have a lower maximum length, so using the template_key_prefix ~~~~~~~~~~~~~~~~~~~ -* Resolvable: Yes +* Resolvable: No * Inheritance strategy: Overrides parent if set by child A string which is prefixed onto the key used to store templates uploaded to S3. diff --git a/integration-tests/sceptre-project/config/9/B.yaml b/integration-tests/sceptre-project/config/9/B.yaml index ad179dacc..ae0df2008 100644 --- a/integration-tests/sceptre-project/config/9/B.yaml +++ b/integration-tests/sceptre-project/config/9/B.yaml @@ -1,4 +1,4 @@ template_path: dependencies/dependent_template_local_export.json region: eu-west-1 parameters: - DependentStackName: !stack_output_external "{project_code}-9-A::StackName default::eu-west-1" + DependentStackName: !stack_output_external "sceptre-integration-tests-15388f6e5aa311ecbeaeacde48001122-9-A::StackName default::eu-west-1" diff --git a/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml b/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml index 83ff95822..2218a1800 100644 --- a/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml +++ b/integration-tests/sceptre-project/config/project-deps/main-project/config.yaml @@ -1,3 +1,5 @@ template_bucket_name: !stack_output project-deps/dependencies/bucket.yaml::BucketName notifications: - !stack_output project-deps/dependencies/topic.yaml::TopicArn +stack_tags: + greeting: !rcmd echo "hello" diff --git a/sceptre/stack.py b/sceptre/stack.py index bfa9cf435..f7817bd42 100644 --- a/sceptre/stack.py +++ b/sceptre/stack.py @@ -120,7 +120,6 @@ class Stack(object): s3_details = ResolvableContainerProperty("s3_details") template_bucket_name = ResolvableValueProperty("template_bucket_name") - template_key_prefix = ResolvableValueProperty("template_key_prefix") role_arn = ResolvableValueProperty("role_arn") hooks = HookProperty("hooks") @@ -154,6 +153,7 @@ def __init__( self.stack_group_config = stack_group_config or {} self.stack_timeout = stack_timeout self.profile = profile + self.template_key_prefix = template_key_prefix self._template = None self._connection_manager = None @@ -163,7 +163,6 @@ def __init__( self.iam_role = iam_role self.tags = tags or {} self.role_arn = role_arn - self.template_key_prefix = template_key_prefix self.template_bucket_name = template_bucket_name self.s3_details = s3_details From 4003b6f573dce2a08e0eba0ce697cc7ee40f5af4 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 11:03:33 -0600 Subject: [PATCH 30/36] undoing accidental commitment --- integration-tests/sceptre-project/config/9/B.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/sceptre-project/config/9/B.yaml b/integration-tests/sceptre-project/config/9/B.yaml index ae0df2008..ad179dacc 100644 --- a/integration-tests/sceptre-project/config/9/B.yaml +++ b/integration-tests/sceptre-project/config/9/B.yaml @@ -1,4 +1,4 @@ template_path: dependencies/dependent_template_local_export.json region: eu-west-1 parameters: - DependentStackName: !stack_output_external "sceptre-integration-tests-15388f6e5aa311ecbeaeacde48001122-9-A::StackName default::eu-west-1" + DependentStackName: !stack_output_external "{project_code}-9-A::StackName default::eu-west-1" From b9727560ff1f17a871e863b4619fea08d6927d92 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 11:05:08 -0600 Subject: [PATCH 31/36] removing unused step --- integration-tests/steps/stack_groups.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/integration-tests/steps/stack_groups.py b/integration-tests/steps/stack_groups.py index 0109df928..f0a7c0d17 100644 --- a/integration-tests/steps/stack_groups.py +++ b/integration-tests/steps/stack_groups.py @@ -270,27 +270,6 @@ def step_impl(context, first_stack, second_stack): assert creation_times[stacks[0]] < creation_times[stacks[1]] -@when('the user diffs stack group "{group_name}" with "{diff_type}"') -def step_impl(context, group_name, diff_type): - sceptre_context = SceptreContext( - command_path=group_name, - project_path=context.sceptre_dir - ) - sceptre_plan = SceptrePlan(sceptre_context) - differ_classes = { - 'deepdiff': DeepDiffStackDiffer, - 'difflib': DifflibStackDiffer - } - writer_class = { - 'deepdiff': DeepDiffWriter, - 'difflib': DeepDiffWriter - } - - differ = differ_classes[diff_type]() - context.writer_class = writer_class[diff_type] - context.output = list(sceptre_plan.diff(differ).values()) - - def get_stack_creation_times(context, stacks): creation_times = {} response = retry_boto_call(context.client.describe_stacks) From 15ac0da4002f5635462a6c95cdb67d75261b8ee8 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sat, 11 Dec 2021 11:08:01 -0600 Subject: [PATCH 32/36] checking for falsey value --- sceptre/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sceptre/template.py b/sceptre/template.py index e3f08c682..1bf9656c2 100644 --- a/sceptre/template.py +++ b/sceptre/template.py @@ -216,7 +216,7 @@ def get_boto_call_parameter(self): :rtype: dict """ # If bucket_name is set to None, it should be ignored and not uploaded. - if self.s3_details and self.s3_details.get("bucket_name") is not None: + if self.s3_details and self.s3_details.get("bucket_name"): url = self.upload_to_s3() return {"TemplateURL": url} else: From 516de1028c2f4f96251735cd9a87ad5251cf3ae6 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 12 Dec 2021 11:10:30 -0600 Subject: [PATCH 33/36] removing unused imports --- integration-tests/steps/project_dependencies.py | 3 +-- integration-tests/steps/stack_groups.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/integration-tests/steps/project_dependencies.py b/integration-tests/steps/project_dependencies.py index 156c4020a..5596f109f 100644 --- a/integration-tests/steps/project_dependencies.py +++ b/integration-tests/steps/project_dependencies.py @@ -1,8 +1,7 @@ from itertools import chain -from typing import ContextManager import boto3 -from behave import given, then, when +from behave import given, then from behave.runner import Context from helpers import get_cloudformation_stack_name, retry_boto_call diff --git a/integration-tests/steps/stack_groups.py b/integration-tests/steps/stack_groups.py index f0a7c0d17..1c51160b7 100644 --- a/integration-tests/steps/stack_groups.py +++ b/integration-tests/steps/stack_groups.py @@ -4,11 +4,8 @@ from behave import * from botocore.exceptions import ClientError -from helpers import read_template_file, get_cloudformation_stack_name -from helpers import retry_boto_call +from helpers import read_template_file, get_cloudformation_stack_name, retry_boto_call from sceptre.context import SceptreContext -from sceptre.diffing.diff_writer import DeepDiffWriter -from sceptre.diffing.stack_differ import DeepDiffStackDiffer, DifflibStackDiffer from sceptre.plan.plan import SceptrePlan from stacks import wait_for_final_state from templates import set_template_path @@ -205,10 +202,12 @@ def step_impl(context): @then('stack "{stack_name}" is described as "{status}"') def step_impl(context, stack_name, status): - response = next(( - stack for stack in context.response - if stack_name in stack - ), {stack_name: 'PENDING'}) + response = next( + ( + stack for stack in context.response + if stack_name in stack + ), {stack_name: 'PENDING'} + ) assert response[stack_name] == status @@ -311,7 +310,7 @@ def create_stacks(context, stack_names): ) except ClientError as e: if e.response['Error']['Code'] == 'AlreadyExistsException' \ - and e.response['Error']['Message'].endswith("already exists"): + and e.response['Error']['Message'].endswith("already exists"): pass else: raise e From 41accd671910d02eefe469ca51a2b60cb08fe8b1 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Sun, 12 Dec 2021 11:43:49 -0600 Subject: [PATCH 34/36] fixing linting error --- integration-tests/steps/stack_groups.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration-tests/steps/stack_groups.py b/integration-tests/steps/stack_groups.py index 1c51160b7..60cbb9810 100644 --- a/integration-tests/steps/stack_groups.py +++ b/integration-tests/steps/stack_groups.py @@ -309,8 +309,10 @@ def create_stacks(context, stack_names): TemplateBody=body ) except ClientError as e: - if e.response['Error']['Code'] == 'AlreadyExistsException' \ - and e.response['Error']['Message'].endswith("already exists"): + if ( + e.response['Error']['Code'] == 'AlreadyExistsException' + and e.response['Error']['Message'].endswith("already exists") + ): pass else: raise e From 1158b88e22182dae9f7a20854b8cbb617da5b1f6 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Thu, 23 Dec 2021 08:47:42 -0600 Subject: [PATCH 35/36] giving example project structure with template bucket --- docs/_source/docs/stack_group_config.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/_source/docs/stack_group_config.rst b/docs/_source/docs/stack_group_config.rst index 83880ed54..d467c3df2 100644 --- a/docs/_source/docs/stack_group_config.rst +++ b/docs/_source/docs/stack_group_config.rst @@ -71,7 +71,22 @@ supplied in this way have a lower maximum length, so using the If you resolve ``template_bucket_name`` using the ``!stack_output`` resolver on a StackGroup, the stack that outputs that bucket name *cannot* be defined in that StackGroup. Otherwise, a circular dependency will exist and Sceptre - will raise an error when attempting any Stack action. + will raise an error when attempting any Stack action. The proper way to do this would + be to define all your project stacks inside a StackGroup and then your template bucket + stack *outside* that StackGroup. Here's an example project structure for something like + this: + + .. code-block:: yaml + + config/ + - config.yaml # This is the StackGroup Config for your whole project. + - template-bucket.yaml # The template for this stack outputs the bucket name + - project/ # You can put all your other stacks in this StackGroup + - config.yaml # In this StackGroup Config is... + # template_bucket_name: !stack_output template-bucket.yaml::BucketName + - vpc.yaml # Put all your other project stacks inside project/ + - other-stack.yaml + template_key_prefix ~~~~~~~~~~~~~~~~~~~ From 7542570d6ab8591e8b75a90fdb2e4a357e079a81 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Thu, 23 Dec 2021 08:54:33 -0600 Subject: [PATCH 36/36] giving example of project dependencies structure --- docs/_source/docs/stack_group_config.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/_source/docs/stack_group_config.rst b/docs/_source/docs/stack_group_config.rst index d467c3df2..ac8a260e2 100644 --- a/docs/_source/docs/stack_group_config.rst +++ b/docs/_source/docs/stack_group_config.rst @@ -208,7 +208,19 @@ dependencies. You might have already considered that this might cause a circular dependency for those dependency stacks, the ones that output the template bucket name, role arn, iam_role, or topic arns. In order to avoid the circular dependency issue, it is important that you define these items in a - Stack that is *outside* the StackGroup you reference them in. + Stack that is *outside* the StackGroup you reference them in. Here's an example project structure + that would support doing this: + + .. code-block:: yaml + + config/ + - config.yaml # This is the StackGroup Config for your whole project. + - sceptre-dependencies.yaml # This stack defines your template bucket, iam role, topics, etc... + - project/ # You can put all your other stacks in this StackGroup + - config.yaml # In this StackGroup Config you can use !stack_output to + # reference outputs from sceptre-dependencies.yaml. + - vpc.yaml # Put all your other project stacks inside project/ + - other-stack.yaml .. _stack_group_config_templating: