Skip to content

Commit c22e03b

Browse files
theofidrytheofidry
authored
Introduce a custom logger (#80)
A significant part of the code is about logging, which in the current state is both hard to test (even manually) and very unstable (it is the constant source of test failure). This PR introduces a dedicated logger which we can later manage to inject and will allow to have better control of the parallel processing output during the tests as well as testing the output itself. I also think the logger itself will need rework, but this is for later. This PR focuses on giving the exact same output.
1 parent a853d8e commit c22e03b

8 files changed

+475
-52
lines changed

‎src/Logger/DebugProgressBarFactory.php

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Webmozarts Console Parallelization package.
5+
*
6+
* (c) Webmozarts GmbH <office@webmozarts.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
declare(strict_types=1);
13+
14+
namespace Webmozarts\Console\Parallelization\Logger;
15+
16+
use Symfony\Component\Console\Helper\ProgressBar;
17+
use Symfony\Component\Console\Output\OutputInterface;
18+
19+
final class DebugProgressBarFactory implements ProgressBarFactory
20+
{
21+
public function create(
22+
OutputInterface $output,
23+
int $numberOfItems
24+
): ProgressBar {
25+
$progressBar = new ProgressBar($output, $numberOfItems);
26+
$progressBar->setFormat(ProgressBar::FORMAT_DEBUG);
27+
$progressBar->start();
28+
29+
return $progressBar;
30+
}
31+
}

‎src/Logger/Logger.php

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Webmozarts Console Parallelization package.
5+
*
6+
* (c) Webmozarts GmbH <office@webmozarts.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
declare(strict_types=1);
13+
14+
namespace Webmozarts\Console\Parallelization\Logger;
15+
16+
interface Logger
17+
{
18+
public function logConfiguration(
19+
int $segmentSize,
20+
int $batchSize,
21+
int $numberOfItems,
22+
int $numberOfSegments,
23+
int $numberOfBatches,
24+
int $numberOfProcesses,
25+
string $itemName
26+
): void;
27+
28+
public function startProgress(int $numberOfItems): void;
29+
30+
public function advance(int $steps = 1): void;
31+
32+
public function finish(string $itemName): void;
33+
34+
public function logUnexpectedOutput(string $buffer): void;
35+
36+
public function logCommandStarted(string $string): void;
37+
38+
public function logCommandFinished(string $string): void;
39+
}

‎src/Logger/ProgressBarFactory.php

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Webmozarts Console Parallelization package.
5+
*
6+
* (c) Webmozarts GmbH <office@webmozarts.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
declare(strict_types=1);
13+
14+
namespace Webmozarts\Console\Parallelization\Logger;
15+
16+
use Symfony\Component\Console\Helper\ProgressBar;
17+
use Symfony\Component\Console\Output\OutputInterface;
18+
19+
interface ProgressBarFactory
20+
{
21+
/**
22+
* @param 0|positive-int $numberOfItems
23+
*/
24+
public function create(
25+
OutputInterface $output,
26+
int $numberOfItems
27+
): ProgressBar;
28+
}

‎src/Logger/StandardLogger.php

+142
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Webmozarts Console Parallelization package.
5+
*
6+
* (c) Webmozarts GmbH <office@webmozarts.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
declare(strict_types=1);
13+
14+
namespace Webmozarts\Console\Parallelization\Logger;
15+
16+
use Psr\Log\LoggerInterface;
17+
use function sprintf;
18+
use function str_pad;
19+
use const STR_PAD_BOTH;
20+
use function str_replace;
21+
use Symfony\Component\Console\Helper\ProgressBar;
22+
use Symfony\Component\Console\Output\OutputInterface;
23+
use Webmozart\Assert\Assert;
24+
25+
final class StandardLogger implements Logger
26+
{
27+
private OutputInterface $output;
28+
private string $advancementChar;
29+
private int $terminalWidth;
30+
private ProgressBar $progressBar;
31+
private ProgressBarFactory $progressBarFactory;
32+
private LoggerInterface $logger;
33+
34+
public function __construct(
35+
OutputInterface $output,
36+
string $advancementCharacter,
37+
int $terminalWidth,
38+
ProgressBarFactory $progressBarFactory,
39+
LoggerInterface $logger
40+
) {
41+
$this->output = $output;
42+
$this->advancementChar = $advancementCharacter;
43+
$this->terminalWidth = $terminalWidth;
44+
$this->progressBarFactory = $progressBarFactory;
45+
$this->logger = $logger;
46+
}
47+
48+
public function logConfiguration(
49+
int $segmentSize,
50+
int $batchSize,
51+
int $numberOfItems,
52+
int $numberOfSegments,
53+
int $numberOfBatches,
54+
int $numberOfProcesses,
55+
string $itemName
56+
): void {
57+
$this->output->writeln(sprintf(
58+
'Processing %d %s in segments of %d, batches of %d, %d %s, %d %s in %d %s',
59+
$numberOfItems,
60+
$itemName,
61+
$segmentSize,
62+
$batchSize,
63+
$numberOfSegments,
64+
1 === $numberOfSegments ? 'round' : 'rounds',
65+
$numberOfBatches,
66+
1 === $numberOfBatches ? 'batch' : 'batches',
67+
$numberOfProcesses,
68+
1 === $numberOfProcesses ? 'process' : 'processes',
69+
));
70+
$this->output->writeln('');
71+
}
72+
73+
public function startProgress(int $numberOfItems): void
74+
{
75+
Assert::false(
76+
isset($this->progressBar),
77+
'Cannot start the progress: already started.',
78+
);
79+
80+
$this->progressBar = $this->progressBarFactory->create(
81+
$this->output,
82+
$numberOfItems,
83+
);
84+
}
85+
86+
public function advance(int $steps = 1): void
87+
{
88+
Assert::notNull(
89+
$this->progressBar,
90+
'Expected the progress to be started.',
91+
);
92+
93+
$this->progressBar->advance($steps);
94+
}
95+
96+
public function finish(string $itemName): void
97+
{
98+
$progressBar = $this->progressBar;
99+
Assert::notNull(
100+
$progressBar,
101+
'Expected the progress to be started.',
102+
);
103+
104+
$progressBar->finish();
105+
106+
$this->output->writeln('');
107+
$this->output->writeln('');
108+
$this->output->writeln(sprintf(
109+
'Processed %d %s.',
110+
$progressBar->getMaxSteps(),
111+
$itemName,
112+
));
113+
114+
unset($this->progressBar);
115+
}
116+
117+
public function logUnexpectedOutput(string $buffer): void
118+
{
119+
$this->output->writeln('');
120+
$this->output->writeln(sprintf(
121+
'<comment>%s</comment>',
122+
str_pad(
123+
' Process Output ',
124+
$this->terminalWidth,
125+
'=',
126+
STR_PAD_BOTH,
127+
),
128+
));
129+
$this->output->writeln(str_replace($this->advancementChar, '', $buffer));
130+
$this->output->writeln('');
131+
}
132+
133+
public function logCommandStarted(string $commandName): void
134+
{
135+
$this->logger->debug('Command started: '.$commandName);
136+
}
137+
138+
public function logCommandFinished(string $commandName): void
139+
{
140+
$this->logger->debug('Command finished: '.$commandName);
141+
}
142+
}

‎src/Parallelization.php

+25-45
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
use function sprintf;
2525
use const STDIN;
2626
use Symfony\Component\Console\Command\Command;
27-
use Symfony\Component\Console\Helper\ProgressBar;
2827
use Symfony\Component\Console\Input\InputInterface;
2928
use Symfony\Component\Console\Logger\ConsoleLogger;
3029
use Symfony\Component\Console\Output\OutputInterface;
@@ -37,6 +36,9 @@
3736
use Throwable;
3837
use function trim;
3938
use Webmozart\Assert\Assert;
39+
use Webmozarts\Console\Parallelization\Logger\DebugProgressBarFactory;
40+
use Webmozarts\Console\Parallelization\Logger\Logger;
41+
use Webmozarts\Console\Parallelization\Logger\StandardLogger;
4042

4143
/**
4244
* Adds parallelization capabilities to console commands.
@@ -276,25 +278,27 @@ protected function executeMasterProcess(
276278

277279
$numberOfSegments = $config->getNumberOfSegments();
278280
$numberOfBatches = $config->getNumberOfBatches();
281+
$itemName = $this->getItemName($numberOfItems);
282+
283+
$logger = new StandardLogger(
284+
$output,
285+
self::getProgressSymbol(),
286+
(new Terminal())->getWidth(),
287+
new DebugProgressBarFactory(),
288+
new ConsoleLogger($output),
289+
);
279290

280-
$output->writeln(sprintf(
281-
'Processing %d %s in segments of %d, batches of %d, %d %s, %d %s in %d %s',
282-
$numberOfItems,
283-
$this->getItemName($numberOfItems),
291+
$logger->logConfiguration(
284292
$segmentSize,
285293
$batchSize,
294+
$numberOfItems,
286295
$numberOfSegments,
287-
1 === $numberOfSegments ? 'round' : 'rounds',
288296
$numberOfBatches,
289-
1 === $numberOfBatches ? 'batch' : 'batches',
290297
$numberOfProcesses,
291-
1 === $numberOfProcesses ? 'process' : 'processes',
292-
));
293-
$output->writeln('');
298+
$itemName,
299+
);
294300

295-
$progressBar = new ProgressBar($output, $numberOfItems);
296-
$progressBar->setFormat('debug');
297-
$progressBar->start();
301+
$logger->startProgress($numberOfItems);
298302

299303
if ($numberOfItems <= $segmentSize
300304
|| (1 === $numberOfProcesses && !$parallelizationInput->isNumberOfProcessesDefined())
@@ -307,7 +311,7 @@ protected function executeMasterProcess(
307311
foreach ($items as $item) {
308312
$this->runTolerantSingleCommand($item, $input, $output);
309313

310-
$progressBar->advance();
314+
$logger->advance();
311315
}
312316

313317
$this->runAfterBatch($input, $output, $items);
@@ -344,8 +348,6 @@ protected function executeMasterProcess(
344348
),
345349
);
346350

347-
$terminalWidth = (new Terminal())->getWidth();
348-
349351
// @TODO: can be removed once ProcessLauncher accepts command arrays
350352
$tempProcess = new Process($commandTemplate);
351353
$commandString = $tempProcess->getCommandLine();
@@ -357,24 +359,14 @@ protected function executeMasterProcess(
357359
$numberOfProcesses,
358360
$segmentSize,
359361
// TODO: offer a way to create the process launcher in a different manner
360-
new ConsoleLogger($output),
361-
function (string $type, string $buffer) use ($progressBar, $output, $terminalWidth) {
362-
$this->processChildOutput($buffer, $progressBar, $output, $terminalWidth);
363-
},
362+
$logger,
363+
fn (string $type, string $buffer) => $this->processChildOutput($buffer, $logger),
364364
);
365365

366366
$processLauncher->run($itemIterator->getItems());
367367
}
368368

369-
$progressBar->finish();
370-
371-
$output->writeln('');
372-
$output->writeln('');
373-
$output->writeln(sprintf(
374-
'Processed %d %s.',
375-
$numberOfItems,
376-
$this->getItemName($numberOfItems),
377-
));
369+
$logger->finish($itemName);
378370

379371
$this->runAfterLastCommand($input, $output);
380372
}
@@ -496,33 +488,21 @@ private static function getWorkingDirectory(ContainerInterface $container): stri
496488
/**
497489
* Called whenever data is received in the master process from a child process.
498490
*
499-
* @param string $buffer The received data
500-
* @param ProgressBar $progressBar The progress bar
501-
* @param OutputInterface $output The output of the master process
502-
* @param int $terminalWidth The width of the terminal window
503-
* in characters
491+
* @param string $buffer The received data
504492
*/
505493
private function processChildOutput(
506494
string $buffer,
507-
ProgressBar $progressBar,
508-
OutputInterface $output,
509-
int $terminalWidth
495+
Logger $logger
510496
): void {
511497
$advancementChar = self::getProgressSymbol();
512498
$chars = mb_substr_count($buffer, $advancementChar);
513499

514500
// Display unexpected output
515501
if ($chars !== mb_strlen($buffer)) {
516-
$output->writeln('');
517-
$output->writeln(sprintf(
518-
'<comment>%s</comment>',
519-
str_pad(' Process Output ', $terminalWidth, '=', STR_PAD_BOTH),
520-
));
521-
$output->writeln(str_replace($advancementChar, '', $buffer));
522-
$output->writeln('');
502+
$logger->logUnexpectedOutput($buffer);
523503
}
524504

525-
$progressBar->advance($chars);
505+
$logger->advance($chars);
526506
}
527507

528508
private function runTolerantSingleCommand(

0 commit comments

Comments
 (0)