Skip to content
This repository was archived by the owner on Jun 22, 2022. It is now read-only.

Commit 99275ea

Browse files
author
Kamil A. Kaczmarek
authored
Revert "removed suffixes (#110)" (#111)
This reverts commit 520a17a.
1 parent 520a17a commit 99275ea

File tree

3 files changed

+51
-37
lines changed

3 files changed

+51
-37
lines changed

docs/conf.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
# The short X.Y version
2727
version = '0.1'
2828
# The full version, including alpha/beta/rc tags
29-
release = '0.1.12'
29+
release = '0.1.11'
3030

3131

3232
# -- General configuration ---------------------------------------------------

setup.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@
1313

1414
setup(name='steppy',
1515
packages=['steppy'],
16-
version='0.1.12',
16+
version='0.1.11',
1717
description='A lightweight, open-source, Python library for fast and reproducible experimentation',
1818
long_description=long_description,
1919
url='https://github.com/minerva-ml/steppy',
20-
download_url='https://github.com/minerva-ml/steppy/archive/0.1.12.tar.gz',
20+
download_url='https://github.com/minerva-ml/steppy/archive/0.1.11.tar.gz',
2121
author='Kamil A. Kaczmarek, Jakub Czakon',
2222
author_email='kamil.kaczmarek@neptune.ml, jakub.czakon@neptune.ml',
2323
keywords=['machine-learning', 'reproducibility', 'pipeline', 'data-science'],

steppy/base.py

+48-34
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
'load_persisted_output': False
1919
}
2020

21+
_ALL_STEPS_NAMES = list()
22+
2123

2224
class Step:
2325
"""Step is a building block of steppy pipelines.
@@ -178,42 +180,41 @@ def __init__(self,
178180
cache_output=False,
179181
load_persisted_output=False):
180182

181-
self.name = self._format_step_name(name, transformer)
183+
name = self._format_step_name(name, transformer)
182184

183185
if experiment_directory is not None:
184186
assert isinstance(experiment_directory, str),\
185187
'Step {} error, experiment_directory must ' \
186-
'be str, got {} instead.'.format(self.name, type(experiment_directory))
188+
'be str, got {} instead.'.format(name, type(experiment_directory))
187189
else:
188190
experiment_directory = os.path.join(os.path.expanduser("~"), '.steppy')
189191
logger.info('Using default experiment directory: {}'.format(experiment_directory))
190192

191193
if output_directory is not None:
192194
assert isinstance(output_directory, str),\
193-
'Step {}, output_directory must be str, got {} instead'.format(self.name, type(output_directory))
195+
'Step {}, output_directory must be str, got {} instead'.format(name, type(output_directory))
194196

195197
if input_data is not None:
196198
assert isinstance(input_data, list), 'Step {} error, input_data must be list, ' \
197-
'got {} instead.'.format(self.name, type(input_data))
199+
'got {} instead.'.format(name, type(input_data))
198200
if input_steps is not None:
199201
assert isinstance(input_steps, list), 'Step {} error, input_steps must be list, ' \
200-
'got {} instead.'.format(self.name, type(input_steps))
202+
'got {} instead.'.format(name, type(input_steps))
201203
if adapter is not None:
202204
assert isinstance(adapter, Adapter), 'Step {} error, adapter must be an instance ' \
203-
'of {}'.format(self.name, str(Adapter))
205+
'of {}'.format(name, str(Adapter))
204206

205207
assert isinstance(cache_output, bool), 'Step {} error, cache_output must be bool, ' \
206-
'got {} instead.'.format(self.name, type(cache_output))
208+
'got {} instead.'.format(name, type(cache_output))
207209
assert isinstance(persist_output, bool), 'Step {} error, persist_output must be bool, ' \
208-
'got {} instead.'.format(self.name, type(persist_output))
210+
'got {} instead.'.format(name, type(persist_output))
209211
assert isinstance(load_persisted_output, bool),\
210212
'Step {} error, load_persisted_output ' \
211-
'must be bool, got {} instead.'.format(self.name, type(load_persisted_output))
213+
'must be bool, got {} instead.'.format(name, type(load_persisted_output))
212214
assert isinstance(force_fitting, bool), 'Step {} error, force_fitting must be bool, ' \
213-
'got {} instead.'.format(self.name, type(force_fitting))
215+
'got {} instead.'.format(name, type(force_fitting))
214216

215-
self._validate_upstream_names()
216-
logger.info('Initializing Step {}'.format(self.name))
217+
logger.info('Initializing Step {}'.format(name))
217218

218219
self.transformer = transformer
219220
self.output_directory = output_directory
@@ -227,7 +228,11 @@ def __init__(self,
227228
self.force_fitting = force_fitting
228229

229230
self.output = None
231+
self.name = self._apply_suffix(name)
232+
_ALL_STEPS_NAMES.append(self.name)
233+
230234
self.experiment_directory = os.path.join(experiment_directory)
235+
231236
self._prepare_experiment_directories()
232237
self._mode = 'train'
233238

@@ -487,7 +492,7 @@ def get_step_by_name(self, name):
487492
return self.all_upstream_steps[name]
488493
except KeyError as e:
489494
msg = 'No Step with name "{}" found. ' \
490-
'You have following Steps: {}'.format(name, list(self.all_upstream_steps.keys()))
495+
'You have following Steps: {}'.format(name, _ALL_STEPS_NAMES)
491496
raise StepError(msg) from e
492497

493498
def persist_upstream_structure(self):
@@ -520,8 +525,9 @@ def _fit_transform_operation(self, step_inputs):
520525
try:
521526
step_output_data = self.transformer.transform(**step_inputs)
522527
except Exception as e:
523-
msg = 'Step {}, Transformer "{}" error ' \
524-
'during "transform()" operation.'.format(self.name, self.transformer.__class__.__name__)
528+
msg = 'Step {}, Transformer "{}" error during "transform()" operation. ' \
529+
'Check "Step.transformer" implementation"'.format(self.name,
530+
self.transformer.__class__.__name__)
525531
raise StepError(msg) from e
526532

527533
logger.info('Step {}, transforming completed'.format(self.name))
@@ -531,8 +537,9 @@ def _fit_transform_operation(self, step_inputs):
531537
try:
532538
step_output_data = self.transformer.fit_transform(**step_inputs)
533539
except Exception as e:
534-
msg = 'Step {}, Transformer "{}" error ' \
535-
'during "fit_transform()" operation.'.format(self.name, self.transformer.__class__.__name__)
540+
msg = 'Step {}, Transformer "{}" error during "fit_transform()" operation. ' \
541+
'Check "Step.transformer" implementation"'.format(self.name,
542+
self.transformer.__class__.__name__)
536543
raise StepError(msg) from e
537544

538545
logger.info('Step {}, fitting and transforming completed'.format(self.name))
@@ -545,8 +552,10 @@ def _fit_transform_operation(self, step_inputs):
545552
try:
546553
step_output_data = self.transformer.transform(**step_inputs)
547554
except Exception as e:
548-
msg = 'Step {}, Transformer "{}" error ' \
549-
'during "transform()" operation.'.format(self.name, self.transformer.__class__.__name__)
555+
msg = 'Step {}, Transformer "{}" error during "transform()" operation. ' \
556+
'This Transformer is not fittable. ' \
557+
'Check "Step.transformer" implementation"'.format(self.name,
558+
self.transformer.__class__.__name__)
550559
raise StepError(msg) from e
551560

552561
logger.info('Step {}, transforming completed'.format(self.name))
@@ -570,8 +579,9 @@ def _transform_operation(self, step_inputs):
570579
try:
571580
step_output_data = self.transformer.transform(**step_inputs)
572581
except Exception as e:
573-
msg = 'Step {}, Transformer "{}" error ' \
574-
'during "transform()" operation.'.format(self.name, self.transformer.__class__.__name__)
582+
msg = 'Step {}, Transformer "{}" error during "transform()" operation. ' \
583+
'Check "Step.transformer" implementation"'.format(self.name,
584+
self.transformer.__class__.__name__)
575585
raise StepError(msg) from e
576586

577587
logger.info('Step {}, transforming completed'.format(self.name))
@@ -585,8 +595,10 @@ def _transform_operation(self, step_inputs):
585595
try:
586596
step_output_data = self.transformer.transform(**step_inputs)
587597
except Exception as e:
588-
msg = 'Step {}, Transformer "{}" error ' \
589-
'during "transform()" operation.'.format(self.name, self.transformer.__class__.__name__)
598+
msg = 'Step {}, Transformer "{}" error during "transform()" operation. ' \
599+
'This Transformer is not fittable. ' \
600+
'Check "Step.transformer" implementation"'.format(self.name,
601+
self.transformer.__class__.__name__)
590602
raise StepError(msg) from e
591603

592604
logger.info('Step {}, transforming completed'.format(self.name))
@@ -640,7 +652,6 @@ def _prepare_experiment_directories(self):
640652
os.makedirs(os.path.join(self.experiment_directory, dir_name), exist_ok=True)
641653

642654
def _get_steps(self, all_steps):
643-
self._check_name_uniqueness(all_steps=all_steps)
644655
for input_step in self.input_steps:
645656
all_steps = input_step._get_steps(all_steps)
646657
all_steps[self.name] = self
@@ -659,16 +670,19 @@ def _validate_step_name(self, name):
659670
assert isinstance(name, str) or isinstance(name, float) or isinstance(name, int),\
660671
'Step name must be str, float or int. Got {} instead.'.format(type(name))
661672

662-
def _check_name_uniqueness(self, all_steps):
663-
if self.name in all_steps.keys():
664-
raise ValueError('Step with name "{}", already exist. Assign unique Step name.'.format(self.name))
665-
666-
def _validate_upstream_names(self):
667-
try:
668-
_ = self.all_upstream_steps.keys()
669-
except ValueError as e:
670-
msg = 'Incorrect Step names'
671-
raise StepError(msg) from e
673+
def _apply_suffix(self, name):
674+
"""returns suffix '_k'
675+
Where 'k' is int that denotes highest increment of step with the same name.
676+
"""
677+
highest_id = 0
678+
for x in _ALL_STEPS_NAMES:
679+
if not x == name:
680+
key_id = x.split('_')[-1]
681+
key_stripped = x[:-len(key_id) - 1]
682+
if key_stripped == name:
683+
if int(key_id) >= highest_id:
684+
highest_id += 1
685+
return '{}_{}'.format(name, highest_id)
672686

673687
def _build_structure_dict(self, structure_dict):
674688
for input_step in self.input_steps:

0 commit comments

Comments
 (0)