From 0cfce0686906715dfa4450a3cdb76765fdbfd333 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 11 Nov 2025 12:32:52 +0100 Subject: [PATCH] feat(Cleanup): implement failure marking for stuck scheduled tasks and database backups during startup --- app/Console/Commands/Dev.php | 31 +++ app/Console/Commands/Init.php | 31 +++ tests/Feature/StartupExecutionCleanupTest.php | 216 ++++++++++++++++++ tests/Unit/StartupExecutionCleanupTest.php | 116 ++++++++++ 4 files changed, 394 insertions(+) create mode 100644 tests/Feature/StartupExecutionCleanupTest.php create mode 100644 tests/Unit/StartupExecutionCleanupTest.php diff --git a/app/Console/Commands/Dev.php b/app/Console/Commands/Dev.php index f04d67482..acc6dc2f9 100644 --- a/app/Console/Commands/Dev.php +++ b/app/Console/Commands/Dev.php @@ -4,6 +4,9 @@ namespace App\Console\Commands; use App\Jobs\CheckHelperImageJob; use App\Models\InstanceSettings; +use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledTaskExecution; +use Carbon\Carbon; use Illuminate\Console\Command; use Illuminate\Support\Facades\Artisan; @@ -55,6 +58,34 @@ class Dev extends Command echo "Error in cleanup:redis: {$e->getMessage()}\n"; } + try { + $updatedTaskCount = ScheduledTaskExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + if ($updatedTaskCount > 0) { + echo "Marked {$updatedTaskCount} stuck scheduled task executions as failed\n"; + } + } catch (\Throwable $e) { + echo "Could not cleanup stuck scheduled task executions: {$e->getMessage()}\n"; + } + + try { + $updatedBackupCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + if ($updatedBackupCount > 0) { + echo "Marked {$updatedBackupCount} stuck database backup executions as failed\n"; + } + } catch (\Throwable $e) { + echo "Could not cleanup stuck database backup executions: {$e->getMessage()}\n"; + } + CheckHelperImageJob::dispatch(); } } diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index 80b053ef7..66cb77838 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -10,9 +10,12 @@ use App\Models\ApplicationDeploymentQueue; use App\Models\Environment; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledTaskExecution; use App\Models\Server; use App\Models\StandalonePostgresql; use App\Models\User; +use Carbon\Carbon; use Illuminate\Console\Command; use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\File; @@ -103,6 +106,34 @@ class Init extends Command echo "Could not cleanup inprogress deployments: {$e->getMessage()}\n"; } + try { + $updatedTaskCount = ScheduledTaskExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + if ($updatedTaskCount > 0) { + echo "Marked {$updatedTaskCount} stuck scheduled task executions as failed\n"; + } + } catch (\Throwable $e) { + echo "Could not cleanup stuck scheduled task executions: {$e->getMessage()}\n"; + } + + try { + $updatedBackupCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + if ($updatedBackupCount > 0) { + echo "Marked {$updatedBackupCount} stuck database backup executions as failed\n"; + } + } catch (\Throwable $e) { + echo "Could not cleanup stuck database backup executions: {$e->getMessage()}\n"; + } + try { $localhost = $this->servers->where('id', 0)->first(); if ($localhost) { diff --git a/tests/Feature/StartupExecutionCleanupTest.php b/tests/Feature/StartupExecutionCleanupTest.php new file mode 100644 index 000000000..3a6b00208 --- /dev/null +++ b/tests/Feature/StartupExecutionCleanupTest.php @@ -0,0 +1,216 @@ +create(); + + // Create a scheduled task + $scheduledTask = ScheduledTask::factory()->create([ + 'team_id' => $team->id, + ]); + + // Create multiple task executions with 'running' status + $runningExecution1 = ScheduledTaskExecution::create([ + 'scheduled_task_id' => $scheduledTask->id, + 'status' => 'running', + 'started_at' => Carbon::now()->subMinutes(10), + ]); + + $runningExecution2 = ScheduledTaskExecution::create([ + 'scheduled_task_id' => $scheduledTask->id, + 'status' => 'running', + 'started_at' => Carbon::now()->subMinutes(5), + ]); + + // Create a completed execution (should not be affected) + $completedExecution = ScheduledTaskExecution::create([ + 'scheduled_task_id' => $scheduledTask->id, + 'status' => 'success', + 'started_at' => Carbon::now()->subMinutes(15), + 'finished_at' => Carbon::now()->subMinutes(14), + ]); + + // Run the app:init command + Artisan::call('app:init'); + + // Refresh models from database + $runningExecution1->refresh(); + $runningExecution2->refresh(); + $completedExecution->refresh(); + + // Assert running executions are now failed + expect($runningExecution1->status)->toBe('failed') + ->and($runningExecution1->message)->toBe('Marked as failed during Coolify startup - job was interrupted') + ->and($runningExecution1->finished_at)->not->toBeNull() + ->and($runningExecution1->finished_at->toDateTimeString())->toBe('2025-01-15 12:00:00'); + + expect($runningExecution2->status)->toBe('failed') + ->and($runningExecution2->message)->toBe('Marked as failed during Coolify startup - job was interrupted') + ->and($runningExecution2->finished_at)->not->toBeNull(); + + // Assert completed execution is unchanged + expect($completedExecution->status)->toBe('success') + ->and($completedExecution->message)->toBeNull(); + + // Assert NO notifications were sent + Notification::assertNothingSent(); +}); + +test('app:init marks stuck database backup executions as failed', function () { + // Create a team for the scheduled backup + $team = Team::factory()->create(); + + // Create a database + $database = StandalonePostgresql::factory()->create([ + 'team_id' => $team->id, + ]); + + // Create a scheduled backup + $scheduledBackup = ScheduledDatabaseBackup::factory()->create([ + 'team_id' => $team->id, + 'database_id' => $database->id, + 'database_type' => StandalonePostgresql::class, + ]); + + // Create multiple backup executions with 'running' status + $runningBackup1 = ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $scheduledBackup->id, + 'status' => 'running', + 'database_name' => 'test_db', + ]); + + $runningBackup2 = ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $scheduledBackup->id, + 'status' => 'running', + 'database_name' => 'test_db_2', + ]); + + // Create a successful backup (should not be affected) + $successfulBackup = ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $scheduledBackup->id, + 'status' => 'success', + 'database_name' => 'test_db_3', + 'finished_at' => Carbon::now()->subMinutes(20), + ]); + + // Run the app:init command + Artisan::call('app:init'); + + // Refresh models from database + $runningBackup1->refresh(); + $runningBackup2->refresh(); + $successfulBackup->refresh(); + + // Assert running backups are now failed + expect($runningBackup1->status)->toBe('failed') + ->and($runningBackup1->message)->toBe('Marked as failed during Coolify startup - job was interrupted') + ->and($runningBackup1->finished_at)->not->toBeNull() + ->and($runningBackup1->finished_at->toDateTimeString())->toBe('2025-01-15 12:00:00'); + + expect($runningBackup2->status)->toBe('failed') + ->and($runningBackup2->message)->toBe('Marked as failed during Coolify startup - job was interrupted') + ->and($runningBackup2->finished_at)->not->toBeNull(); + + // Assert successful backup is unchanged + expect($successfulBackup->status)->toBe('success') + ->and($successfulBackup->message)->toBeNull(); + + // Assert NO notifications were sent + Notification::assertNothingSent(); +}); + +test('app:init handles cleanup when no stuck executions exist', function () { + // Create a team + $team = Team::factory()->create(); + + // Create a scheduled task + $scheduledTask = ScheduledTask::factory()->create([ + 'team_id' => $team->id, + ]); + + // Create only completed executions + ScheduledTaskExecution::create([ + 'scheduled_task_id' => $scheduledTask->id, + 'status' => 'success', + 'started_at' => Carbon::now()->subMinutes(10), + 'finished_at' => Carbon::now()->subMinutes(9), + ]); + + ScheduledTaskExecution::create([ + 'scheduled_task_id' => $scheduledTask->id, + 'status' => 'failed', + 'started_at' => Carbon::now()->subMinutes(20), + 'finished_at' => Carbon::now()->subMinutes(19), + ]); + + // Run the app:init command (should not fail) + $exitCode = Artisan::call('app:init'); + + // Assert command succeeded + expect($exitCode)->toBe(0); + + // Assert all executions remain unchanged + expect(ScheduledTaskExecution::where('status', 'running')->count())->toBe(0) + ->and(ScheduledTaskExecution::where('status', 'success')->count())->toBe(1) + ->and(ScheduledTaskExecution::where('status', 'failed')->count())->toBe(1); + + // Assert NO notifications were sent + Notification::assertNothingSent(); +}); + +test('cleanup does not send notifications even when team has notification settings', function () { + // Create a team with notification settings enabled + $team = Team::factory()->create([ + 'smtp_enabled' => true, + 'smtp_from_address' => 'test@example.com', + ]); + + // Create a scheduled task + $scheduledTask = ScheduledTask::factory()->create([ + 'team_id' => $team->id, + ]); + + // Create a running execution + $runningExecution = ScheduledTaskExecution::create([ + 'scheduled_task_id' => $scheduledTask->id, + 'status' => 'running', + 'started_at' => Carbon::now()->subMinutes(5), + ]); + + // Run the app:init command + Artisan::call('app:init'); + + // Refresh model + $runningExecution->refresh(); + + // Assert execution is failed + expect($runningExecution->status)->toBe('failed'); + + // Assert NO notifications were sent despite team having notification settings + Notification::assertNothingSent(); +}); diff --git a/tests/Unit/StartupExecutionCleanupTest.php b/tests/Unit/StartupExecutionCleanupTest.php new file mode 100644 index 000000000..1fae590eb --- /dev/null +++ b/tests/Unit/StartupExecutionCleanupTest.php @@ -0,0 +1,116 @@ +shouldReceive('where') + ->once() + ->with('status', 'running') + ->andReturnSelf(); + + // Expect update to be called with correct parameters + $mockBuilder->shouldReceive('update') + ->once() + ->with([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]) + ->andReturn(2); // Simulate 2 records updated + + // Execute the cleanup logic directly + $updatedCount = ScheduledTaskExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + // Assert the count is correct + expect($updatedCount)->toBe(2); +}); + +it('marks stuck database backup executions as failed without triggering notifications', function () { + // Mock the ScheduledDatabaseBackupExecution model + $mockBuilder = \Mockery::mock('alias:'.ScheduledDatabaseBackupExecution::class); + + // Expect where clause to be called with 'running' status + $mockBuilder->shouldReceive('where') + ->once() + ->with('status', 'running') + ->andReturnSelf(); + + // Expect update to be called with correct parameters + $mockBuilder->shouldReceive('update') + ->once() + ->with([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]) + ->andReturn(3); // Simulate 3 records updated + + // Execute the cleanup logic directly + $updatedCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + // Assert the count is correct + expect($updatedCount)->toBe(3); +}); + +it('handles cleanup when no stuck executions exist', function () { + // Mock the ScheduledTaskExecution model + $mockBuilder = \Mockery::mock('alias:'.ScheduledTaskExecution::class); + + $mockBuilder->shouldReceive('where') + ->once() + ->with('status', 'running') + ->andReturnSelf(); + + $mockBuilder->shouldReceive('update') + ->once() + ->andReturn(0); // No records updated + + $updatedCount = ScheduledTaskExecution::where('status', 'running')->update([ + 'status' => 'failed', + 'message' => 'Marked as failed during Coolify startup - job was interrupted', + 'finished_at' => Carbon::now(), + ]); + + expect($updatedCount)->toBe(0); +}); + +it('uses correct failure message for interrupted jobs', function () { + $expectedMessage = 'Marked as failed during Coolify startup - job was interrupted'; + + // Verify the message clearly indicates the job was interrupted during startup + expect($expectedMessage) + ->toContain('Coolify startup') + ->toContain('interrupted') + ->toContain('failed'); +}); + +it('sets finished_at timestamp when marking executions as failed', function () { + $now = Carbon::now(); + + // Verify Carbon::now() is used for finished_at + expect($now)->toBeInstanceOf(Carbon::class) + ->and($now->toDateTimeString())->toBe('2025-01-15 12:00:00'); +});