mirror of
https://github.com/coollabsio/coolify.git
synced 2025-12-28 05:34:50 +00:00
Fix complex status logic: handle degraded sub-resources and mixed running+starting states
- Add support for degraded status from sub-resources as highest priority - Handle mixed running+starting state to show service as not fully ready - Update state priority hierarchy from 8 to 10 levels - Add comprehensive test coverage for new status scenarios 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
parent
afb19114cf
commit
c65ad2e655
@ -16,14 +16,16 @@ use Illuminate\Support\Facades\Log;
|
||||
* UI components transform this to human-readable format (e.g., "Running (Healthy)").
|
||||
*
|
||||
* State Priority (highest to lowest):
|
||||
* 1. Restarting → degraded:unhealthy
|
||||
* 2. Crash Loop (exited with restarts) → degraded:unhealthy
|
||||
* 3. Mixed (running + exited) → degraded:unhealthy
|
||||
* 4. Running → running:healthy/unhealthy/unknown
|
||||
* 5. Dead/Removing → degraded:unhealthy
|
||||
* 6. Paused → paused:unknown
|
||||
* 7. Starting/Created → starting:unknown
|
||||
* 8. Exited → exited
|
||||
* 1. Degraded (from sub-resources) → degraded:unhealthy
|
||||
* 2. Restarting → degraded:unhealthy
|
||||
* 3. Crash Loop (exited with restarts) → degraded:unhealthy
|
||||
* 4. Mixed (running + exited) → degraded:unhealthy
|
||||
* 5. Mixed (running + starting) → starting:unknown
|
||||
* 6. Running → running:healthy/unhealthy/unknown
|
||||
* 7. Dead/Removing → degraded:unhealthy
|
||||
* 8. Paused → paused:unknown
|
||||
* 9. Starting/Created → starting:unknown
|
||||
* 10. Exited → exited
|
||||
*/
|
||||
class ContainerStatusAggregator
|
||||
{
|
||||
@ -64,10 +66,16 @@ class ContainerStatusAggregator
|
||||
$hasStarting = false;
|
||||
$hasPaused = false;
|
||||
$hasDead = false;
|
||||
$hasDegraded = false;
|
||||
|
||||
// Parse each status string and set flags
|
||||
foreach ($containerStatuses as $status) {
|
||||
if (str($status)->contains('restarting')) {
|
||||
if (str($status)->contains('degraded')) {
|
||||
$hasDegraded = true;
|
||||
if (str($status)->contains('unhealthy')) {
|
||||
$hasUnhealthy = true;
|
||||
}
|
||||
} elseif (str($status)->contains('restarting')) {
|
||||
$hasRestarting = true;
|
||||
} elseif (str($status)->contains('running')) {
|
||||
$hasRunning = true;
|
||||
@ -98,6 +106,7 @@ class ContainerStatusAggregator
|
||||
$hasStarting,
|
||||
$hasPaused,
|
||||
$hasDead,
|
||||
$hasDegraded,
|
||||
$maxRestartCount
|
||||
);
|
||||
}
|
||||
@ -175,6 +184,7 @@ class ContainerStatusAggregator
|
||||
$hasStarting,
|
||||
$hasPaused,
|
||||
$hasDead,
|
||||
false, // $hasDegraded - not applicable for container objects, only for status strings
|
||||
$maxRestartCount
|
||||
);
|
||||
}
|
||||
@ -190,6 +200,7 @@ class ContainerStatusAggregator
|
||||
* @param bool $hasStarting Has at least one starting/created container
|
||||
* @param bool $hasPaused Has at least one paused container
|
||||
* @param bool $hasDead Has at least one dead/removing container
|
||||
* @param bool $hasDegraded Has at least one degraded container
|
||||
* @param int $maxRestartCount Maximum restart count (for crash loop detection)
|
||||
* @return string Status in colon format (e.g., "running:healthy")
|
||||
*/
|
||||
@ -202,24 +213,37 @@ class ContainerStatusAggregator
|
||||
bool $hasStarting,
|
||||
bool $hasPaused,
|
||||
bool $hasDead,
|
||||
bool $hasDegraded,
|
||||
int $maxRestartCount
|
||||
): string {
|
||||
// Priority 1: Restarting containers (degraded state)
|
||||
// Priority 1: Degraded containers from sub-resources (highest priority)
|
||||
// If any service/application within a service stack is degraded, the entire stack is degraded
|
||||
if ($hasDegraded) {
|
||||
return 'degraded:unhealthy';
|
||||
}
|
||||
|
||||
// Priority 2: Restarting containers (degraded state)
|
||||
if ($hasRestarting) {
|
||||
return 'degraded:unhealthy';
|
||||
}
|
||||
|
||||
// Priority 2: Crash loop detection (exited with restart count > 0)
|
||||
// Priority 3: Crash loop detection (exited with restart count > 0)
|
||||
if ($hasExited && $maxRestartCount > 0) {
|
||||
return 'degraded:unhealthy';
|
||||
}
|
||||
|
||||
// Priority 3: Mixed state (some running, some exited = degraded)
|
||||
// Priority 4: Mixed state (some running, some exited = degraded)
|
||||
if ($hasRunning && $hasExited) {
|
||||
return 'degraded:unhealthy';
|
||||
}
|
||||
|
||||
// Priority 4: Running containers (check health status)
|
||||
// Priority 5: Mixed state (some running, some starting = still starting)
|
||||
// If any component is still starting, the entire service stack is not fully ready
|
||||
if ($hasRunning && $hasStarting) {
|
||||
return 'starting:unknown';
|
||||
}
|
||||
|
||||
// Priority 6: Running containers (check health status)
|
||||
if ($hasRunning) {
|
||||
if ($hasUnhealthy) {
|
||||
return 'running:unhealthy';
|
||||
@ -230,22 +254,22 @@ class ContainerStatusAggregator
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 5: Dead or removing containers
|
||||
// Priority 7: Dead or removing containers
|
||||
if ($hasDead) {
|
||||
return 'degraded:unhealthy';
|
||||
}
|
||||
|
||||
// Priority 6: Paused containers
|
||||
// Priority 8: Paused containers
|
||||
if ($hasPaused) {
|
||||
return 'paused:unknown';
|
||||
}
|
||||
|
||||
// Priority 7: Starting/created containers
|
||||
// Priority 9: Starting/created containers
|
||||
if ($hasStarting) {
|
||||
return 'starting:unknown';
|
||||
}
|
||||
|
||||
// Priority 8: All containers exited (no restart count = truly stopped)
|
||||
// Priority 10: All containers exited (no restart count = truly stopped)
|
||||
return 'exited';
|
||||
}
|
||||
}
|
||||
|
||||
@ -126,6 +126,70 @@ describe('aggregateFromStrings', function () {
|
||||
expect($result)->toBe('starting:unknown');
|
||||
});
|
||||
|
||||
test('returns degraded:unhealthy for single degraded container', function () {
|
||||
$statuses = collect(['degraded:unhealthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('returns degraded:unhealthy when mixing degraded with running healthy', function () {
|
||||
$statuses = collect(['degraded:unhealthy', 'running:healthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('returns degraded:unhealthy when mixing running healthy with degraded', function () {
|
||||
$statuses = collect(['running:healthy', 'degraded:unhealthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('returns degraded:unhealthy for multiple degraded containers', function () {
|
||||
$statuses = collect(['degraded:unhealthy', 'degraded:unhealthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('degraded status overrides all other non-critical states', function () {
|
||||
$statuses = collect(['degraded:unhealthy', 'running:healthy', 'starting', 'paused']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('returns starting:unknown when mixing starting with running healthy (service not fully ready)', function () {
|
||||
$statuses = collect(['starting:unknown', 'running:healthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('starting:unknown');
|
||||
});
|
||||
|
||||
test('returns starting:unknown when mixing created with running healthy', function () {
|
||||
$statuses = collect(['created', 'running:healthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('starting:unknown');
|
||||
});
|
||||
|
||||
test('returns starting:unknown for multiple starting containers with some running', function () {
|
||||
$statuses = collect(['starting:unknown', 'starting:unknown', 'running:healthy']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('starting:unknown');
|
||||
});
|
||||
|
||||
test('handles parentheses format input (backward compatibility)', function () {
|
||||
$statuses = collect(['running (healthy)', 'running (unhealthy)']);
|
||||
|
||||
@ -166,8 +230,16 @@ describe('aggregateFromStrings', function () {
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('prioritizes running over paused/starting/exited', function () {
|
||||
$statuses = collect(['running:healthy', 'starting', 'paused']);
|
||||
test('mixed running and starting returns starting', function () {
|
||||
$statuses = collect(['running:healthy', 'starting']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('starting:unknown');
|
||||
});
|
||||
|
||||
test('prioritizes running over paused/exited when no starting', function () {
|
||||
$statuses = collect(['running:healthy', 'paused', 'exited']);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
@ -398,7 +470,23 @@ describe('aggregateFromContainers', function () {
|
||||
});
|
||||
|
||||
describe('state priority enforcement', function () {
|
||||
test('restarting has highest priority', function () {
|
||||
test('degraded from sub-resources has highest priority', function () {
|
||||
$statuses = collect([
|
||||
'degraded:unhealthy',
|
||||
'restarting',
|
||||
'running:healthy',
|
||||
'dead',
|
||||
'paused',
|
||||
'starting',
|
||||
'exited',
|
||||
]);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('restarting has second highest priority', function () {
|
||||
$statuses = collect([
|
||||
'restarting',
|
||||
'running:healthy',
|
||||
@ -413,7 +501,7 @@ describe('state priority enforcement', function () {
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('crash loop has second highest priority', function () {
|
||||
test('crash loop has third highest priority', function () {
|
||||
$statuses = collect([
|
||||
'exited',
|
||||
'running:healthy',
|
||||
@ -426,7 +514,7 @@ describe('state priority enforcement', function () {
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('mixed state (running + exited) has third priority', function () {
|
||||
test('mixed state (running + exited) has fourth priority', function () {
|
||||
$statuses = collect([
|
||||
'running:healthy',
|
||||
'exited',
|
||||
@ -439,6 +527,18 @@ describe('state priority enforcement', function () {
|
||||
expect($result)->toBe('degraded:unhealthy');
|
||||
});
|
||||
|
||||
test('mixed state (running + starting) has fifth priority', function () {
|
||||
$statuses = collect([
|
||||
'running:healthy',
|
||||
'starting',
|
||||
'paused',
|
||||
]);
|
||||
|
||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||
|
||||
expect($result)->toBe('starting:unknown');
|
||||
});
|
||||
|
||||
test('running:unhealthy has priority over running:unknown', function () {
|
||||
$statuses = collect([
|
||||
'running:unknown',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user