mirror of
https://github.com/coollabsio/coolify.git
synced 2025-12-28 13:41:51 +00:00
debug: add comprehensive status change logging
Added detailed debug logging to all status update paths to help
diagnose why "unhealthy" status appears in the UI.
## Logging Added
### 1. PushServerUpdateJob (Sentinel updates)
**Location**: Lines 303-315
**Logs**: Status changes from Sentinel push updates
**Data tracked**:
- Old vs new status
- Container statuses that led to aggregation
- Status flags (hasRunning, hasUnhealthy, hasUnknown)
### 2. GetContainersStatus (SSH updates)
**Location**: Lines 441-449, 346-354, 358-365
**Logs**: Status changes from SSH-based checks
**Scenarios**:
- Normal status aggregation
- Recently restarted containers (kept as degraded)
- Applications not running (set to exited)
**Data tracked**:
- Old vs new status
- Container statuses
- Restart count and timing
- Whether containers exist
### 3. Application Model Status Accessor
**Location**: Lines 706-712, 726-732
**Logs**: When status is set without explicit health information
**Issue**: Highlights cases where health defaults to "unhealthy"
**Data tracked**:
- Raw value passed to setter
- Final result after default applied
## How to Use
### Enable Debug Logging
Edit `.env` or `config/logging.php` to set log level to debug:
```
LOG_LEVEL=debug
```
### Monitor Logs
```bash
tail -f storage/logs/laravel.log | grep STATUS-DEBUG
```
### Log Format
All logs use `[STATUS-DEBUG]` prefix for easy filtering:
```
[2025-11-19 13:00:00] local.DEBUG: [STATUS-DEBUG] Sentinel status change
{
"source": "PushServerUpdateJob",
"app_id": 123,
"app_name": "my-app",
"old_status": "running:unknown",
"new_status": "running:healthy",
"container_statuses": [...],
"flags": {...}
}
```
## What to Look For
1. **Default to unhealthy**: Check Application model accessor logs
2. **Status flipping**: Compare timestamps between Sentinel and SSH updates
3. **Incorrect aggregation**: Check flags and container_statuses
4. **Stale database values**: Check if old_status persists across multiple logs
## Next Steps
After gathering logs, we can:
1. Identify the exact source of "unhealthy" status
2. Determine if it's a default issue, aggregation bug, or timing problem
3. Apply targeted fix based on evidence
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
128c0b00ec
commit
d2d9c1b2bc
@ -11,6 +11,7 @@ use App\Models\ServiceDatabase;
|
||||
use Illuminate\Support\Arr;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
|
||||
class GetContainersStatus
|
||||
@ -342,9 +343,26 @@ class GetContainersStatus
|
||||
|
||||
if ($recentlyRestarted) {
|
||||
// Keep it as degraded if it was recently in a crash loop
|
||||
Log::debug('[STATUS-DEBUG] Recently restarted - keeping degraded', [
|
||||
'source' => 'GetContainersStatus (not running)',
|
||||
'app_id' => $application->id,
|
||||
'app_name' => $application->name,
|
||||
'old_status' => $application->status,
|
||||
'new_status' => 'degraded (unhealthy)',
|
||||
'restart_count' => $application->restart_count,
|
||||
'last_restart_at' => $application->last_restart_at,
|
||||
]);
|
||||
$application->update(['status' => 'degraded (unhealthy)']);
|
||||
} else {
|
||||
// Reset restart count when application exits completely
|
||||
Log::debug('[STATUS-DEBUG] Application not running', [
|
||||
'source' => 'GetContainersStatus (not running)',
|
||||
'app_id' => $application->id,
|
||||
'app_name' => $application->name,
|
||||
'old_status' => $application->status,
|
||||
'new_status' => 'exited',
|
||||
'containers_exist' => ! $this->containers->isEmpty(),
|
||||
]);
|
||||
$application->update([
|
||||
'status' => 'exited',
|
||||
'restart_count' => 0,
|
||||
@ -437,6 +455,15 @@ class GetContainersStatus
|
||||
if ($aggregatedStatus) {
|
||||
$statusFromDb = $application->status;
|
||||
if ($statusFromDb !== $aggregatedStatus) {
|
||||
Log::debug('[STATUS-DEBUG] SSH status change', [
|
||||
'source' => 'GetContainersStatus',
|
||||
'app_id' => $application->id,
|
||||
'app_name' => $application->name,
|
||||
'old_status' => $statusFromDb,
|
||||
'new_status' => $aggregatedStatus,
|
||||
'container_statuses' => $containerStatuses->toArray(),
|
||||
'max_restart_count' => $maxRestartCount,
|
||||
]);
|
||||
$application->update(['status' => $aggregatedStatus]);
|
||||
} else {
|
||||
$application->update(['last_online_at' => now()]);
|
||||
|
||||
@ -21,6 +21,7 @@ use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Horizon\Contracts\Silenced;
|
||||
|
||||
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
@ -299,6 +300,19 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
|
||||
// Update application status with aggregated result
|
||||
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
|
||||
Log::debug('[STATUS-DEBUG] Sentinel status change', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'app_id' => $application->id,
|
||||
'app_name' => $application->name,
|
||||
'old_status' => $application->status,
|
||||
'new_status' => $aggregatedStatus,
|
||||
'container_statuses' => $relevantStatuses->toArray(),
|
||||
'flags' => [
|
||||
'hasRunning' => $hasRunning,
|
||||
'hasUnhealthy' => $hasUnhealthy,
|
||||
'hasUnknown' => $hasUnknown,
|
||||
],
|
||||
]);
|
||||
$application->status = $aggregatedStatus;
|
||||
$application->save();
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||
use Illuminate\Database\Eloquent\SoftDeletes;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Validator;
|
||||
use Illuminate\Support\Str;
|
||||
use OpenApi\Attributes as OA;
|
||||
@ -702,6 +703,13 @@ class Application extends BaseModel
|
||||
} else {
|
||||
$status = $value;
|
||||
$health = 'unhealthy';
|
||||
Log::debug('[STATUS-DEBUG] Status set without health - defaulting to unhealthy', [
|
||||
'source' => 'Application model accessor',
|
||||
'app_id' => $this->id,
|
||||
'app_name' => $this->name,
|
||||
'raw_value' => $value,
|
||||
'result' => "$status:$health",
|
||||
]);
|
||||
}
|
||||
|
||||
return "$status:$health";
|
||||
@ -715,6 +723,13 @@ class Application extends BaseModel
|
||||
} else {
|
||||
$status = $value;
|
||||
$health = 'unhealthy';
|
||||
Log::debug('[STATUS-DEBUG] Status set without health (multi-server) - defaulting to unhealthy', [
|
||||
'source' => 'Application model accessor',
|
||||
'app_id' => $this->id,
|
||||
'app_name' => $this->name,
|
||||
'raw_value' => $value,
|
||||
'result' => "$status:$health",
|
||||
]);
|
||||
}
|
||||
|
||||
return "$status:$health";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user