583 lines
20 KiB
PHP
583 lines
20 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Jobs\Middleware\TrackOperationRun;
|
|
use App\Models\BackupSchedule;
|
|
use App\Models\BackupScheduleRun;
|
|
use App\Models\OperationRun;
|
|
use App\Models\Tenant;
|
|
use App\Services\BackupScheduling\PolicyTypeResolver;
|
|
use App\Services\BackupScheduling\RunErrorMapper;
|
|
use App\Services\BackupScheduling\ScheduleTimeService;
|
|
use App\Services\Intune\AuditLogger;
|
|
use App\Services\Intune\BackupService;
|
|
use App\Services\Intune\PolicySyncService;
|
|
use App\Services\OperationRunService;
|
|
use App\Support\OperationRunLinks;
|
|
use Carbon\CarbonImmutable;
|
|
use Filament\Actions\Action;
|
|
use Filament\Notifications\Notification;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Arr;
|
|
use Illuminate\Support\Facades\Bus;
|
|
use Illuminate\Support\Facades\Cache;
|
|
|
|
class RunBackupScheduleJob implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
public int $tries = 3;
|
|
|
|
public ?OperationRun $operationRun = null;
|
|
|
|
public function __construct(
|
|
public int $backupScheduleRunId,
|
|
?OperationRun $operationRun = null,
|
|
) {
|
|
$this->operationRun = $operationRun;
|
|
}
|
|
|
|
public function middleware(): array
|
|
{
|
|
return [new TrackOperationRun];
|
|
}
|
|
|
|
public function handle(
|
|
PolicySyncService $policySyncService,
|
|
BackupService $backupService,
|
|
PolicyTypeResolver $policyTypeResolver,
|
|
ScheduleTimeService $scheduleTimeService,
|
|
AuditLogger $auditLogger,
|
|
RunErrorMapper $errorMapper,
|
|
): void {
|
|
$run = BackupScheduleRun::query()
|
|
->with(['schedule', 'tenant', 'user'])
|
|
->find($this->backupScheduleRunId);
|
|
|
|
if (! $run) {
|
|
if ($this->operationRun) {
|
|
$this->markOperationRunFailed(
|
|
run: $this->operationRun,
|
|
summaryCounts: [],
|
|
reasonCode: 'run_not_found',
|
|
reason: 'Backup schedule run not found.',
|
|
);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
$tenant = $run->tenant;
|
|
|
|
if ($tenant instanceof Tenant) {
|
|
$this->resolveOperationRunFromContext($tenant, $run);
|
|
}
|
|
|
|
if ($this->operationRun) {
|
|
$this->operationRun->update([
|
|
'context' => array_merge($this->operationRun->context ?? [], [
|
|
'backup_schedule_id' => (int) $run->backup_schedule_id,
|
|
'backup_schedule_run_id' => (int) $run->getKey(),
|
|
]),
|
|
]);
|
|
|
|
/** @var OperationRunService $operationRunService */
|
|
$operationRunService = app(OperationRunService::class);
|
|
|
|
if ($this->operationRun->status === 'queued') {
|
|
$operationRunService->updateRun($this->operationRun, 'running');
|
|
}
|
|
}
|
|
|
|
$schedule = $run->schedule;
|
|
|
|
if (! $schedule instanceof BackupSchedule) {
|
|
$run->update([
|
|
'status' => BackupScheduleRun::STATUS_FAILED,
|
|
'error_code' => RunErrorMapper::ERROR_UNKNOWN,
|
|
'error_message' => 'Schedule not found.',
|
|
'finished_at' => CarbonImmutable::now('UTC'),
|
|
]);
|
|
|
|
if ($this->operationRun) {
|
|
$this->markOperationRunFailed(
|
|
run: $this->operationRun,
|
|
summaryCounts: [
|
|
'total' => 0,
|
|
'processed' => 0,
|
|
'failed' => 1,
|
|
],
|
|
reasonCode: 'schedule_not_found',
|
|
reason: 'Schedule not found.',
|
|
);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (! $tenant) {
|
|
$run->update([
|
|
'status' => BackupScheduleRun::STATUS_FAILED,
|
|
'error_code' => RunErrorMapper::ERROR_UNKNOWN,
|
|
'error_message' => 'Tenant not found.',
|
|
'finished_at' => CarbonImmutable::now('UTC'),
|
|
]);
|
|
|
|
if ($this->operationRun) {
|
|
$this->markOperationRunFailed(
|
|
run: $this->operationRun,
|
|
summaryCounts: [
|
|
'total' => 0,
|
|
'processed' => 0,
|
|
'failed' => 1,
|
|
],
|
|
reasonCode: 'tenant_not_found',
|
|
reason: 'Tenant not found.',
|
|
);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
$lock = Cache::lock("backup_schedule:{$schedule->id}", 900);
|
|
|
|
if (! $lock->get()) {
|
|
$this->finishRun(
|
|
run: $run,
|
|
schedule: $schedule,
|
|
status: BackupScheduleRun::STATUS_SKIPPED,
|
|
errorCode: 'CONCURRENT_RUN',
|
|
errorMessage: 'Another run is already in progress for this schedule.',
|
|
summary: ['reason' => 'concurrent_run'],
|
|
scheduleTimeService: $scheduleTimeService,
|
|
);
|
|
|
|
$this->syncOperationRunFromRun(
|
|
tenant: $tenant,
|
|
schedule: $schedule,
|
|
run: $run->refresh(),
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
try {
|
|
$nowUtc = CarbonImmutable::now('UTC');
|
|
|
|
$run->forceFill([
|
|
'started_at' => $run->started_at ?? $nowUtc,
|
|
'status' => BackupScheduleRun::STATUS_RUNNING,
|
|
])->save();
|
|
|
|
$this->notifyRunStarted($run, $schedule);
|
|
|
|
$auditLogger->log(
|
|
tenant: $tenant,
|
|
action: 'backup_schedule.run_started',
|
|
context: [
|
|
'metadata' => [
|
|
'backup_schedule_id' => $schedule->id,
|
|
'backup_schedule_run_id' => $run->id,
|
|
'scheduled_for' => $run->scheduled_for?->toDateTimeString(),
|
|
],
|
|
],
|
|
resourceType: 'backup_schedule_run',
|
|
resourceId: (string) $run->id,
|
|
status: 'success'
|
|
);
|
|
|
|
$runtime = $policyTypeResolver->resolveRuntime((array) ($schedule->policy_types ?? []));
|
|
$validTypes = $runtime['valid'];
|
|
$unknownTypes = $runtime['unknown'];
|
|
|
|
if (empty($validTypes)) {
|
|
$this->finishRun(
|
|
run: $run,
|
|
schedule: $schedule,
|
|
status: BackupScheduleRun::STATUS_SKIPPED,
|
|
errorCode: 'UNKNOWN_POLICY_TYPE',
|
|
errorMessage: 'All configured policy types are unknown.',
|
|
summary: [
|
|
'unknown_policy_types' => $unknownTypes,
|
|
],
|
|
scheduleTimeService: $scheduleTimeService,
|
|
);
|
|
|
|
$this->syncOperationRunFromRun(
|
|
tenant: $tenant,
|
|
schedule: $schedule,
|
|
run: $run->refresh(),
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
$supported = array_values(array_filter(
|
|
config('tenantpilot.supported_policy_types', []),
|
|
fn (array $typeConfig): bool => in_array($typeConfig['type'] ?? null, $validTypes, true),
|
|
));
|
|
|
|
$syncReport = $policySyncService->syncPoliciesWithReport($tenant, $supported);
|
|
|
|
$policyIds = $syncReport['synced'] ?? [];
|
|
$syncFailures = $syncReport['failures'] ?? [];
|
|
|
|
$backupSet = $backupService->createBackupSet(
|
|
tenant: $tenant,
|
|
policyIds: $policyIds,
|
|
actorEmail: null,
|
|
actorName: null,
|
|
name: 'Scheduled backup: '.$schedule->name,
|
|
includeAssignments: false,
|
|
includeScopeTags: false,
|
|
includeFoundations: (bool) ($schedule->include_foundations ?? false),
|
|
);
|
|
|
|
$status = match ($backupSet->status) {
|
|
'completed' => BackupScheduleRun::STATUS_SUCCESS,
|
|
'partial' => BackupScheduleRun::STATUS_PARTIAL,
|
|
'failed' => BackupScheduleRun::STATUS_FAILED,
|
|
default => BackupScheduleRun::STATUS_SUCCESS,
|
|
};
|
|
|
|
$errorCode = null;
|
|
$errorMessage = null;
|
|
|
|
$summary = [
|
|
'policies_total' => count($policyIds),
|
|
'policies_backed_up' => (int) ($backupSet->item_count ?? 0),
|
|
'sync_failures' => $syncFailures,
|
|
];
|
|
|
|
if (! empty($unknownTypes)) {
|
|
$status = BackupScheduleRun::STATUS_PARTIAL;
|
|
$errorCode = 'UNKNOWN_POLICY_TYPE';
|
|
$errorMessage = 'Some configured policy types are unknown and were skipped.';
|
|
$summary['unknown_policy_types'] = $unknownTypes;
|
|
}
|
|
|
|
$this->finishRun(
|
|
run: $run,
|
|
schedule: $schedule,
|
|
status: $status,
|
|
errorCode: $errorCode,
|
|
errorMessage: $errorMessage,
|
|
summary: $summary,
|
|
scheduleTimeService: $scheduleTimeService,
|
|
backupSetId: (string) $backupSet->id,
|
|
);
|
|
|
|
$this->syncOperationRunFromRun(
|
|
tenant: $tenant,
|
|
schedule: $schedule,
|
|
run: $run->refresh(),
|
|
);
|
|
|
|
$auditLogger->log(
|
|
tenant: $tenant,
|
|
action: 'backup_schedule.run_finished',
|
|
context: [
|
|
'metadata' => [
|
|
'backup_schedule_id' => $schedule->id,
|
|
'backup_schedule_run_id' => $run->id,
|
|
'status' => $status,
|
|
'error_code' => $errorCode,
|
|
],
|
|
],
|
|
resourceType: 'backup_schedule_run',
|
|
resourceId: (string) $run->id,
|
|
status: in_array($status, [BackupScheduleRun::STATUS_SUCCESS], true) ? 'success' : 'partial'
|
|
);
|
|
} catch (\Throwable $throwable) {
|
|
$attempt = (int) method_exists($this, 'attempts') ? $this->attempts() : 1;
|
|
$mapped = $errorMapper->map($throwable, $attempt, $this->tries);
|
|
|
|
if ($mapped['shouldRetry']) {
|
|
if ($this->operationRun) {
|
|
/** @var OperationRunService $operationRunService */
|
|
$operationRunService = app(OperationRunService::class);
|
|
$operationRunService->updateRun($this->operationRun, 'running', 'pending');
|
|
}
|
|
|
|
$this->release($mapped['delay']);
|
|
|
|
return;
|
|
}
|
|
|
|
$this->finishRun(
|
|
run: $run,
|
|
schedule: $schedule,
|
|
status: BackupScheduleRun::STATUS_FAILED,
|
|
errorCode: $mapped['error_code'],
|
|
errorMessage: $mapped['error_message'],
|
|
summary: [
|
|
'exception' => get_class($throwable),
|
|
'attempt' => $attempt,
|
|
],
|
|
scheduleTimeService: $scheduleTimeService,
|
|
);
|
|
|
|
$this->syncOperationRunFromRun(
|
|
tenant: $tenant,
|
|
schedule: $schedule,
|
|
run: $run->refresh(),
|
|
);
|
|
|
|
$auditLogger->log(
|
|
tenant: $tenant,
|
|
action: 'backup_schedule.run_failed',
|
|
context: [
|
|
'metadata' => [
|
|
'backup_schedule_id' => $schedule->id,
|
|
'backup_schedule_run_id' => $run->id,
|
|
'error_code' => $mapped['error_code'],
|
|
],
|
|
],
|
|
resourceType: 'backup_schedule_run',
|
|
resourceId: (string) $run->id,
|
|
status: 'failed'
|
|
);
|
|
} finally {
|
|
optional($lock)->release();
|
|
}
|
|
}
|
|
|
|
private function notifyRunStarted(BackupScheduleRun $run, BackupSchedule $schedule): void
|
|
{
|
|
$user = $run->user;
|
|
|
|
if (! $user) {
|
|
return;
|
|
}
|
|
|
|
$notification = Notification::make()
|
|
->title('Backup started')
|
|
->body(sprintf('Schedule "%s" has started.', $schedule->name))
|
|
->info()
|
|
->actions([
|
|
Action::make('view_run')
|
|
->label('View run')
|
|
->url($this->operationRun ? OperationRunLinks::view($this->operationRun, $run->tenant) : OperationRunLinks::index($run->tenant)),
|
|
]);
|
|
|
|
$notification->sendToDatabase($user);
|
|
}
|
|
|
|
private function notifyRunFinished(BackupScheduleRun $run, BackupSchedule $schedule): void
|
|
{
|
|
$user = $run->user;
|
|
|
|
if (! $user) {
|
|
return;
|
|
}
|
|
|
|
$title = match ($run->status) {
|
|
BackupScheduleRun::STATUS_SUCCESS => 'Backup completed',
|
|
BackupScheduleRun::STATUS_PARTIAL => 'Backup completed (partial)',
|
|
BackupScheduleRun::STATUS_SKIPPED => 'Backup skipped',
|
|
default => 'Backup failed',
|
|
};
|
|
|
|
$notification = Notification::make()
|
|
->title($title)
|
|
->body(sprintf('Schedule "%s" finished with status: %s.', $schedule->name, $run->status));
|
|
|
|
if (filled($run->error_message)) {
|
|
$notification->body($notification->getBody()."\n".$run->error_message);
|
|
}
|
|
|
|
match ($run->status) {
|
|
BackupScheduleRun::STATUS_SUCCESS => $notification->success(),
|
|
BackupScheduleRun::STATUS_PARTIAL, BackupScheduleRun::STATUS_SKIPPED => $notification->warning(),
|
|
default => $notification->danger(),
|
|
};
|
|
|
|
$notification
|
|
->actions([
|
|
Action::make('view_run')
|
|
->label('View run')
|
|
->url($this->operationRun ? OperationRunLinks::view($this->operationRun, $run->tenant) : OperationRunLinks::index($run->tenant)),
|
|
])
|
|
->sendToDatabase($user);
|
|
}
|
|
|
|
private function syncOperationRunFromRun(
|
|
Tenant $tenant,
|
|
BackupSchedule $schedule,
|
|
BackupScheduleRun $run,
|
|
): void {
|
|
if (! $this->operationRun) {
|
|
return;
|
|
}
|
|
|
|
$outcome = match ($run->status) {
|
|
BackupScheduleRun::STATUS_SUCCESS => 'succeeded',
|
|
BackupScheduleRun::STATUS_PARTIAL => 'partially_succeeded',
|
|
// Note: 'cancelled' is a reserved OperationRun outcome token.
|
|
// We treat schedule SKIPPED/CANCELED as terminal failures with a failure entry.
|
|
BackupScheduleRun::STATUS_SKIPPED,
|
|
BackupScheduleRun::STATUS_CANCELED => 'failed',
|
|
default => 'failed',
|
|
};
|
|
|
|
$summary = is_array($run->summary) ? $run->summary : [];
|
|
$syncFailures = $summary['sync_failures'] ?? [];
|
|
|
|
$policiesTotal = (int) ($summary['policies_total'] ?? 0);
|
|
$policiesBackedUp = (int) ($summary['policies_backed_up'] ?? 0);
|
|
$syncFailureCount = is_array($syncFailures) ? count($syncFailures) : 0;
|
|
|
|
$failedCount = max(0, $policiesTotal - $policiesBackedUp);
|
|
|
|
$summaryCounts = [
|
|
'total' => $policiesTotal,
|
|
'processed' => $policiesTotal,
|
|
'succeeded' => $policiesBackedUp,
|
|
'failed' => $failedCount,
|
|
'skipped' => 0,
|
|
'created' => filled($run->backup_set_id) ? 1 : 0,
|
|
'updated' => $policiesBackedUp,
|
|
'items' => $policiesTotal,
|
|
];
|
|
|
|
$failures = [];
|
|
|
|
if (filled($run->error_message) || filled($run->error_code)) {
|
|
$failures[] = [
|
|
'code' => strtolower((string) ($run->error_code ?: 'backup_schedule_error')),
|
|
'message' => (string) ($run->error_message ?: 'Backup schedule run failed.'),
|
|
];
|
|
}
|
|
|
|
if (is_array($syncFailures)) {
|
|
foreach ($syncFailures as $failure) {
|
|
if (! is_array($failure)) {
|
|
continue;
|
|
}
|
|
|
|
$policyType = (string) ($failure['policy_type'] ?? 'unknown');
|
|
$status = is_numeric($failure['status'] ?? null) ? (int) $failure['status'] : null;
|
|
$errors = $failure['errors'] ?? null;
|
|
|
|
$firstErrorMessage = null;
|
|
if (is_array($errors) && isset($errors[0]) && is_array($errors[0])) {
|
|
$firstErrorMessage = $errors[0]['message'] ?? null;
|
|
}
|
|
|
|
$message = $status !== null
|
|
? "{$policyType}: Graph returned {$status}"
|
|
: "{$policyType}: Graph request failed";
|
|
|
|
if (is_string($firstErrorMessage) && $firstErrorMessage !== '') {
|
|
$message .= ' - '.trim($firstErrorMessage);
|
|
}
|
|
|
|
$failures[] = [
|
|
'code' => $status !== null ? 'graph_http_'.(string) $status : 'graph_error',
|
|
'message' => $message,
|
|
];
|
|
}
|
|
}
|
|
|
|
/** @var OperationRunService $operationRunService */
|
|
$operationRunService = app(OperationRunService::class);
|
|
|
|
$this->operationRun->update([
|
|
'context' => array_merge($this->operationRun->context ?? [], [
|
|
'backup_schedule_id' => (int) $schedule->getKey(),
|
|
'backup_schedule_run_id' => (int) $run->getKey(),
|
|
'backup_set_id' => $run->backup_set_id ? (int) $run->backup_set_id : null,
|
|
]),
|
|
]);
|
|
|
|
$operationRunService->updateRun(
|
|
$this->operationRun,
|
|
status: 'completed',
|
|
outcome: $outcome,
|
|
summaryCounts: $summaryCounts,
|
|
failures: $failures,
|
|
);
|
|
}
|
|
|
|
private function markOperationRunFailed(
|
|
OperationRun $run,
|
|
array $summaryCounts,
|
|
string $reasonCode,
|
|
string $reason,
|
|
): void {
|
|
/** @var OperationRunService $operationRunService */
|
|
$operationRunService = app(OperationRunService::class);
|
|
|
|
$operationRunService->updateRun(
|
|
$run,
|
|
status: 'completed',
|
|
outcome: 'failed',
|
|
summaryCounts: $summaryCounts,
|
|
failures: [
|
|
[
|
|
'code' => $reasonCode,
|
|
'message' => $reason,
|
|
],
|
|
],
|
|
);
|
|
}
|
|
|
|
private function resolveOperationRunFromContext(Tenant $tenant, BackupScheduleRun $run): void
|
|
{
|
|
if ($this->operationRun) {
|
|
return;
|
|
}
|
|
|
|
$operationRun = OperationRun::query()
|
|
->where('tenant_id', $tenant->getKey())
|
|
->whereIn('type', ['backup_schedule.run_now', 'backup_schedule.retry'])
|
|
->whereIn('status', ['queued', 'running'])
|
|
->where('context->backup_schedule_run_id', (int) $run->getKey())
|
|
->latest('id')
|
|
->first();
|
|
|
|
if ($operationRun instanceof OperationRun) {
|
|
$this->operationRun = $operationRun;
|
|
}
|
|
}
|
|
|
|
private function finishRun(
|
|
BackupScheduleRun $run,
|
|
BackupSchedule $schedule,
|
|
string $status,
|
|
?string $errorCode,
|
|
?string $errorMessage,
|
|
array $summary,
|
|
ScheduleTimeService $scheduleTimeService,
|
|
?string $backupSetId = null,
|
|
): void {
|
|
$nowUtc = CarbonImmutable::now('UTC');
|
|
|
|
$run->forceFill([
|
|
'status' => $status,
|
|
'error_code' => $errorCode,
|
|
'error_message' => $errorMessage,
|
|
'summary' => Arr::wrap($summary),
|
|
'finished_at' => $nowUtc,
|
|
'backup_set_id' => $backupSetId,
|
|
])->save();
|
|
|
|
$schedule->forceFill([
|
|
'last_run_at' => $nowUtc,
|
|
'last_run_status' => $status,
|
|
'next_run_at' => $scheduleTimeService->nextRunFor($schedule, $nowUtc),
|
|
])->saveQuietly();
|
|
|
|
$this->notifyRunFinished($run, $schedule);
|
|
|
|
if ($backupSetId && in_array($status, [BackupScheduleRun::STATUS_SUCCESS, BackupScheduleRun::STATUS_PARTIAL], true)) {
|
|
Bus::dispatch(new ApplyBackupScheduleRetentionJob($schedule->id));
|
|
}
|
|
}
|
|
}
|