debug: add comprehensive status change logging

Added detailed debug logging to all status update paths to help
diagnose why "unhealthy" status appears in the UI.

## Logging Added

### 1. PushServerUpdateJob (Sentinel updates)
**Location**: Lines 303-315
**Logs**: Status changes from Sentinel push updates
**Data tracked**:
- Old vs new status
- Container statuses that led to aggregation
- Status flags (hasRunning, hasUnhealthy, hasUnknown)

### 2. GetContainersStatus (SSH updates)
**Location**: Lines 441-449, 346-354, 358-365
**Logs**: Status changes from SSH-based checks
**Scenarios**:
- Normal status aggregation
- Recently restarted containers (kept as degraded)
- Applications not running (set to exited)
**Data tracked**:
- Old vs new status
- Container statuses
- Restart count and timing
- Whether containers exist

### 3. Application Model Status Accessor
**Location**: Lines 706-712, 726-732
**Logs**: When status is set without explicit health information
**Issue**: Highlights cases where health defaults to "unhealthy"
**Data tracked**:
- Raw value passed to setter
- Final result after default applied

## How to Use

### Enable Debug Logging
Edit `.env` or `config/logging.php` to set log level to debug:
```
LOG_LEVEL=debug
```

### Monitor Logs
```bash
tail -f storage/logs/laravel.log | grep STATUS-DEBUG
```

### Log Format
All logs use `[STATUS-DEBUG]` prefix for easy filtering:
```
[2025-11-19 13:00:00] local.DEBUG: [STATUS-DEBUG] Sentinel status change
{
  "source": "PushServerUpdateJob",
  "app_id": 123,
  "app_name": "my-app",
  "old_status": "running:unknown",
  "new_status": "running:healthy",
  "container_statuses": [...],
  "flags": {...}
}
```

## What to Look For

1. **Default to unhealthy**: Check Application model accessor logs
2. **Status flipping**: Compare timestamps between Sentinel and SSH updates
3. **Incorrect aggregation**: Check flags and container_statuses
4. **Stale database values**: Check if old_status persists across multiple logs

## Next Steps

After gathering logs, we can:
1. Identify the exact source of "unhealthy" status
2. Determine if it's a default issue, aggregation bug, or timing problem
3. Apply targeted fix based on evidence

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai 2025-11-19 13:52:08 +01:00
parent 128c0b00ec
commit d2d9c1b2bc
3 changed files with 56 additions and 0 deletions

View File

@ -11,6 +11,7 @@ use App\Models\ServiceDatabase;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Lorisleiva\Actions\Concerns\AsAction;
class GetContainersStatus
@ -342,9 +343,26 @@ class GetContainersStatus
if ($recentlyRestarted) {
// Keep it as degraded if it was recently in a crash loop
Log::debug('[STATUS-DEBUG] Recently restarted - keeping degraded', [
'source' => 'GetContainersStatus (not running)',
'app_id' => $application->id,
'app_name' => $application->name,
'old_status' => $application->status,
'new_status' => 'degraded (unhealthy)',
'restart_count' => $application->restart_count,
'last_restart_at' => $application->last_restart_at,
]);
$application->update(['status' => 'degraded (unhealthy)']);
} else {
// Reset restart count when application exits completely
Log::debug('[STATUS-DEBUG] Application not running', [
'source' => 'GetContainersStatus (not running)',
'app_id' => $application->id,
'app_name' => $application->name,
'old_status' => $application->status,
'new_status' => 'exited',
'containers_exist' => ! $this->containers->isEmpty(),
]);
$application->update([
'status' => 'exited',
'restart_count' => 0,
@ -437,6 +455,15 @@ class GetContainersStatus
if ($aggregatedStatus) {
$statusFromDb = $application->status;
if ($statusFromDb !== $aggregatedStatus) {
Log::debug('[STATUS-DEBUG] SSH status change', [
'source' => 'GetContainersStatus',
'app_id' => $application->id,
'app_name' => $application->name,
'old_status' => $statusFromDb,
'new_status' => $aggregatedStatus,
'container_statuses' => $containerStatuses->toArray(),
'max_restart_count' => $maxRestartCount,
]);
$application->update(['status' => $aggregatedStatus]);
} else {
$application->update(['last_online_at' => now()]);

View File

@ -21,6 +21,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@ -299,6 +300,19 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
Log::debug('[STATUS-DEBUG] Sentinel status change', [
'source' => 'PushServerUpdateJob',
'app_id' => $application->id,
'app_name' => $application->name,
'old_status' => $application->status,
'new_status' => $aggregatedStatus,
'container_statuses' => $relevantStatuses->toArray(),
'flags' => [
'hasRunning' => $hasRunning,
'hasUnhealthy' => $hasUnhealthy,
'hasUnknown' => $hasUnknown,
],
]);
$application->status = $aggregatedStatus;
$application->save();
}

View File

@ -12,6 +12,7 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use OpenApi\Attributes as OA;
@ -702,6 +703,13 @@ class Application extends BaseModel
} else {
$status = $value;
$health = 'unhealthy';
Log::debug('[STATUS-DEBUG] Status set without health - defaulting to unhealthy', [
'source' => 'Application model accessor',
'app_id' => $this->id,
'app_name' => $this->name,
'raw_value' => $value,
'result' => "$status:$health",
]);
}
return "$status:$health";
@ -715,6 +723,13 @@ class Application extends BaseModel
} else {
$status = $value;
$health = 'unhealthy';
Log::debug('[STATUS-DEBUG] Status set without health (multi-server) - defaulting to unhealthy', [
'source' => 'Application model accessor',
'app_id' => $this->id,
'app_name' => $this->name,
'raw_value' => $value,
'result' => "$status:$health",
]);
}
return "$status:$health";