Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions config/app.example.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
'workerLifetime' => 60, // 1 minutes
// Legacy: 'workermaxruntime' is deprecated but still supported

// optional random offset (0-N seconds) added per worker to stagger shutdowns in a fleet (0 = disabled)
'workerLifetimeJitter' => 0,

// seconds of running time after which the PHP process will terminate, null uses workerLifetime * 2
'workerPhpTimeout' => null,
// Legacy: 'workertimeout' is deprecated but still supported
Expand Down
8 changes: 8 additions & 0 deletions docs/sections/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ You may create a file called `app_queue.php` inside your `config` folder (NOT th
bin/cake queue run --max-runtime 300 # Run for 5 minutes
```

- Optional per-worker jitter (in seconds) added to the worker lifetime:

```php
$config['Queue']['workerLifetimeJitter'] = 30; // up to +30s random offset per worker
```

Each worker picks a random offset in `[0, workerLifetimeJitter]` at startup and adds it to its effective lifetime. Useful when many workers are spawned simultaneously (e.g. ECS/Kubernetes) to stagger shutdowns and avoid a thundering herd of concurrent restarts. Defaults to `0` (no jitter).

- Seconds of running time after which the PHP process of the worker will terminate:

```php
Expand Down
25 changes: 25 additions & 0 deletions src/Queue/Processor.php
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,10 @@ public function run(array $args): int {
$this->exit = false;

$startTime = time();
$jitterOffset = $this->computeLifetimeJitterOffset();
if ($jitterOffset > 0) {
$this->io->out('Applying worker lifetime jitter: +' . $jitterOffset . ' seconds');
}

while (!$this->exit) {
$this->setPhpTimeout($config['maxruntime']);
Expand Down Expand Up @@ -237,6 +241,9 @@ public function run(array $args): int {
throw new RuntimeException('Queue.workerLifetime (or deprecated workermaxruntime) config is required');
}
$maxRuntime = $config['maxruntime'] ?? (int)$workerLifetime;
if ($maxRuntime > 0 && $jitterOffset > 0) {
$maxRuntime += $jitterOffset;
}
// check if we are over the maximum runtime and end processing if so.
if ($maxRuntime > 0 && (time() - $startTime) >= $maxRuntime) {
$this->exit = true;
Expand Down Expand Up @@ -626,6 +633,24 @@ protected function setPhpTimeout(?int $maxruntime): void {
set_time_limit($timeLimit);
}

/**
* Compute the per-worker lifetime jitter offset in seconds.
*
* Returns a random integer in [0, Queue.workerLifetimeJitter]. Used to stagger
* worker shutdowns so a fleet spawned at the same moment does not all exit
* on the same tick (thundering herd).
*
* @return int
*/
protected function computeLifetimeJitterOffset(): int {
$jitter = (int)Configure::read('Queue.workerLifetimeJitter', 0);
if ($jitter <= 0) {
return 0;
}

return mt_rand(0, $jitter);
}

/**
* @param array<string, mixed> $args
*
Expand Down
47 changes: 47 additions & 0 deletions tests/TestCase/Queue/ProcessorTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -442,4 +442,51 @@ public function testSetPhpTimeoutWithDeprecatedConfig() {
Configure::delete('Queue.workertimeout');
}

/**
* @return void
*/
public function testComputeLifetimeJitterOffsetDefaultsToZero() {
$processor = new Processor(new Io(new ConsoleIo()), new NullLogger());

Configure::delete('Queue.workerLifetimeJitter');
$result = $this->invokeMethod($processor, 'computeLifetimeJitterOffset');
$this->assertSame(0, $result);

Configure::write('Queue.workerLifetimeJitter', 0);
$result = $this->invokeMethod($processor, 'computeLifetimeJitterOffset');
$this->assertSame(0, $result);

Configure::delete('Queue.workerLifetimeJitter');
}

/**
* @return void
*/
public function testComputeLifetimeJitterOffsetWithinBounds() {
$processor = new Processor(new Io(new ConsoleIo()), new NullLogger());

Configure::write('Queue.workerLifetimeJitter', 15);
for ($i = 0; $i < 50; $i++) {
$result = $this->invokeMethod($processor, 'computeLifetimeJitterOffset');
$this->assertIsInt($result);
$this->assertGreaterThanOrEqual(0, $result);
$this->assertLessThanOrEqual(15, $result);
}

Configure::delete('Queue.workerLifetimeJitter');
}

/**
* @return void
*/
public function testComputeLifetimeJitterOffsetIgnoresNegative() {
$processor = new Processor(new Io(new ConsoleIo()), new NullLogger());

Configure::write('Queue.workerLifetimeJitter', -10);
$result = $this->invokeMethod($processor, 'computeLifetimeJitterOffset');
$this->assertSame(0, $result);

Configure::delete('Queue.workerLifetimeJitter');
}

}
Loading