Summary This PR introduces Unified Operations Runs + Monitoring Hub (053). Goal: Standardize how long-running operations are tracked and monitored using the existing tenant-scoped run record (BulkOperationRun) as the canonical “operation run”, and surface it in a single Monitoring → Operations hub (view-only, tenant-scoped, role-aware). Phase 1 adoption scope (per spec): • Drift generation (drift.generate) • Backup Set “Add Policies” (backup_set.add_policies) Note: This PR does not convert every run type yet (e.g. GroupSyncRuns / InventorySyncRuns remain separate for now). This is intentionally incremental. ⸻ What changed Monitoring / Operations hub • Moved/organized run monitoring under Monitoring → Operations • Added: • status buckets (queued / running / succeeded / partially succeeded / failed) • filters (run type, status bucket, time range) • run detail “Related” links (e.g. Drift findings, Backup Set context) • All hub pages are DB-only and view-only (no rerun/cancel/delete actions) Canonical run semantics • Added canonical helpers on BulkOperationRun: • runType() (resource.action) • statusBucket() derived from status + counts (testable semantics) Drift integration (Phase 1) • Drift generation start behavior now: • creates/reuses a BulkOperationRun with drift context payload (scope_key + baseline/current run ids) • dispatches generation job • emits DB notifications including “View run” link • On generation failure: stores sanitized failure entries + sends failure notification Permissions / tenant isolation • Monitoring run list/view is tenant-scoped and returns 403 for cross-tenant access • Readonly can view runs but cannot start drift generation ⸻ Tests Added/updated Pest coverage: • BulkOperationRunStatusBucketTest.php • DriftGenerationDispatchTest.php • GenerateDriftFindingsJobNotificationTest.php • RunAuthorizationTenantIsolationTest.php Validation run locally: • ./vendor/bin/pint --dirty • targeted tests from feature quickstart / drift monitoring tests ⸻ Manual QA 1. Go to Monitoring → Operations • verify filters (run type / status / time range) • verify run detail shows counts + sanitized failures + “Related” links 2. Open Drift Landing • with >=2 successful inventory runs for scope: should queue drift generation + show notification with “View run” • as readonly: should not start generation 3. Run detail • drift.generate runs show “Drift findings” related link • failure entries are sanitized (no secrets/tokens/raw payload dumps) ⸻ Notes / Ops • Queue workers must be restarted after deploy so they load the new code: • php artisan queue:restart (or Sail equivalent) • This PR standardizes monitoring for Phase 1 producers only; follow-ups will migrate additional run types into the unified pattern. ⸻ Spec / Docs • SpecKit artifacts added under specs/053-unify-runs-monitoring/ • Checklists are complete: • requirements checklist PASS • writing checklist PASS Co-authored-by: Ahmed Darrazi <ahmeddarrazi@adsmac.local> Reviewed-on: #60
185 lines
6.3 KiB
PHP
185 lines
6.3 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Models\BulkOperationRun;
|
|
use App\Models\InventorySyncRun;
|
|
use App\Models\Tenant;
|
|
use App\Notifications\RunStatusChangedNotification;
|
|
use App\Services\BulkOperationService;
|
|
use App\Services\Drift\DriftFindingGenerator;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Log;
|
|
use RuntimeException;
|
|
use Throwable;
|
|
|
|
class GenerateDriftFindingsJob implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
public function __construct(
|
|
public int $tenantId,
|
|
public int $userId,
|
|
public int $baselineRunId,
|
|
public int $currentRunId,
|
|
public string $scopeKey,
|
|
public int $bulkOperationRunId,
|
|
) {}
|
|
|
|
/**
|
|
* Execute the job.
|
|
*/
|
|
public function handle(DriftFindingGenerator $generator, BulkOperationService $bulkOperationService): void
|
|
{
|
|
Log::info('GenerateDriftFindingsJob: started', [
|
|
'tenant_id' => $this->tenantId,
|
|
'baseline_run_id' => $this->baselineRunId,
|
|
'current_run_id' => $this->currentRunId,
|
|
'scope_key' => $this->scopeKey,
|
|
'bulk_operation_run_id' => $this->bulkOperationRunId,
|
|
]);
|
|
|
|
$tenant = Tenant::query()->find($this->tenantId);
|
|
if (! $tenant instanceof Tenant) {
|
|
throw new RuntimeException('Tenant not found.');
|
|
}
|
|
|
|
$baseline = InventorySyncRun::query()->find($this->baselineRunId);
|
|
if (! $baseline instanceof InventorySyncRun) {
|
|
throw new RuntimeException('Baseline run not found.');
|
|
}
|
|
|
|
$current = InventorySyncRun::query()->find($this->currentRunId);
|
|
if (! $current instanceof InventorySyncRun) {
|
|
throw new RuntimeException('Current run not found.');
|
|
}
|
|
|
|
$run = BulkOperationRun::query()
|
|
->where('tenant_id', $tenant->getKey())
|
|
->find($this->bulkOperationRunId);
|
|
|
|
if (! $run instanceof BulkOperationRun) {
|
|
throw new RuntimeException('Bulk operation run not found.');
|
|
}
|
|
|
|
$bulkOperationService->start($run);
|
|
|
|
try {
|
|
$created = $generator->generate(
|
|
tenant: $tenant,
|
|
baseline: $baseline,
|
|
current: $current,
|
|
scopeKey: $this->scopeKey,
|
|
);
|
|
|
|
Log::info('GenerateDriftFindingsJob: completed', [
|
|
'tenant_id' => $this->tenantId,
|
|
'baseline_run_id' => $this->baselineRunId,
|
|
'current_run_id' => $this->currentRunId,
|
|
'scope_key' => $this->scopeKey,
|
|
'bulk_operation_run_id' => $this->bulkOperationRunId,
|
|
'created_findings_count' => $created,
|
|
]);
|
|
|
|
$bulkOperationService->recordSuccess($run);
|
|
$bulkOperationService->complete($run);
|
|
|
|
$this->notifyStatus($run->refresh());
|
|
} catch (Throwable $e) {
|
|
Log::error('GenerateDriftFindingsJob: failed', [
|
|
'tenant_id' => $this->tenantId,
|
|
'baseline_run_id' => $this->baselineRunId,
|
|
'current_run_id' => $this->currentRunId,
|
|
'scope_key' => $this->scopeKey,
|
|
'bulk_operation_run_id' => $this->bulkOperationRunId,
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
|
|
$bulkOperationService->recordFailure(
|
|
run: $run,
|
|
itemId: $this->scopeKey,
|
|
reason: $e->getMessage(),
|
|
reasonCode: 'unknown',
|
|
);
|
|
|
|
$bulkOperationService->fail($run, $e->getMessage());
|
|
$this->notifyStatus($run->refresh());
|
|
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
private function notifyStatus(BulkOperationRun $run): void
|
|
{
|
|
try {
|
|
if (! $run->relationLoaded('user')) {
|
|
$run->loadMissing('user');
|
|
}
|
|
|
|
if (! $run->user) {
|
|
return;
|
|
}
|
|
|
|
$status = 'failed';
|
|
|
|
try {
|
|
$status = $run->statusBucket();
|
|
} catch (Throwable) {
|
|
$failureEntries = $run->failures ?? [];
|
|
$hasNonSkippedFailure = false;
|
|
foreach ($failureEntries as $entry) {
|
|
if (! is_array($entry)) {
|
|
continue;
|
|
}
|
|
|
|
if (($entry['type'] ?? 'failed') !== 'skipped') {
|
|
$hasNonSkippedFailure = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
$failedCount = (int) ($run->failed ?? 0);
|
|
$succeededCount = (int) ($run->succeeded ?? 0);
|
|
$hasFailures = $failedCount > 0 || $hasNonSkippedFailure;
|
|
|
|
if ($succeededCount > 0 && $hasFailures) {
|
|
$status = 'partially succeeded';
|
|
} elseif ($succeededCount === 0 && $hasFailures) {
|
|
$status = 'failed';
|
|
} else {
|
|
$status = match ($run->status) {
|
|
'pending' => 'queued',
|
|
'running' => 'running',
|
|
'completed', 'completed_with_errors' => 'succeeded',
|
|
default => 'failed',
|
|
};
|
|
}
|
|
}
|
|
|
|
$run->user->notify(new RunStatusChangedNotification([
|
|
'tenant_id' => (int) $run->tenant_id,
|
|
'run_type' => 'bulk_operation',
|
|
'run_id' => (int) $run->getKey(),
|
|
'status' => $status,
|
|
'counts' => [
|
|
'total' => (int) $run->total_items,
|
|
'processed' => (int) $run->processed_items,
|
|
'succeeded' => (int) $run->succeeded,
|
|
'failed' => (int) $run->failed,
|
|
'skipped' => (int) $run->skipped,
|
|
],
|
|
]));
|
|
} catch (Throwable $e) {
|
|
Log::warning('GenerateDriftFindingsJob: status notification failed', [
|
|
'tenant_id' => (int) $run->tenant_id,
|
|
'bulk_operation_run_id' => (int) $run->getKey(),
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
}
|