feat/040-inventory-core (#43)
Summary Implements Inventory Core (Spec 040): a tenant-scoped, mutable “last observed” inventory catalog + sync run logging, with deterministic selection hashing and safe derived “missing” semantics. This establishes the foundation for Inventory UI (041), Dependencies Graph (042), Compare/Promotion (043), and Drift (044). What’s included • DB schema • inventory_items (unique: tenant_id + policy_type + external_id; indexes; last_seen_at, last_seen_run_id) • inventory_sync_runs (tenant_id, selection_hash/payload, status, started/finished, counts, error_codes, correlation_id) • Selection hashing • Deterministic selection_hash via canonical JSON (sorted keys + sorted arrays) + sha256 • Sync semantics • Idempotent upsert (no duplicates) • Updates last_seen_* when observed • Enforces tenant scoping for all reads/writes • Guardrail: inventory sync does not create snapshots/backups • Missing semantics (derived) • “missing” computed relative to latest completed run for same (tenant_id, selection_hash) • Low confidence when latest run is partial/failed or had_errors=true • Selection isolation (runs for other selections don’t affect missing) • deleted is reserved (not produced here) • Safety • meta_jsonb whitelist enforced (unknown keys dropped; never fail sync) • Safe error persistence (no bearer tokens / secrets) • Locking to prevent overlapping runs for same tenant+selection • Concurrency limiter (global + per-tenant) and throttling resilience (429/503 backoff + jitter) Tests Added Pest coverage for: • selection_hash determinism (array order invariant) • upsert idempotency + last_seen updates • missing derived semantics + selection isolation • low confidence missing on partial/had_errors • meta whitelist drop (no exception) • lock prevents overlapping runs • no snapshots/backups side effects • safe error persistence (no bearer tokens) Non-goals • Inventory UI pages/resources (Spec 041) • Dependency graph hydration (Spec 042) • Cross-tenant compare/promotion flows (Spec 043) • Drift analysis dashboards (Spec 044) Review focus • Data model correctness + indexes/constraints • Selection hash canonicalization (determinism) • Missing semantics (latest completed run + confidence rule) • Guardrails (no snapshot/backups side effects) • Safety: error_code taxonomy + safe persistence/logging Co-authored-by: Ahmed Darrazi <ahmeddarrazi@adsmac.local> Reviewed-on: #43
This commit is contained in:
parent
dedca3c612
commit
8ae7a7234e
30
app/Models/InventoryItem.php
Normal file
30
app/Models/InventoryItem.php
Normal file
@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
class InventoryItem extends Model
|
||||
{
|
||||
/** @use HasFactory<\Database\Factories\InventoryItemFactory> */
|
||||
use HasFactory;
|
||||
|
||||
protected $guarded = [];
|
||||
|
||||
protected $casts = [
|
||||
'meta_jsonb' => 'array',
|
||||
'last_seen_at' => 'datetime',
|
||||
];
|
||||
|
||||
public function tenant(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(Tenant::class);
|
||||
}
|
||||
|
||||
public function lastSeenRun(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(InventorySyncRun::class, 'last_seen_run_id');
|
||||
}
|
||||
}
|
||||
52
app/Models/InventorySyncRun.php
Normal file
52
app/Models/InventorySyncRun.php
Normal file
@ -0,0 +1,52 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Builder;
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
class InventorySyncRun extends Model
|
||||
{
|
||||
/** @use HasFactory<\Database\Factories\InventorySyncRunFactory> */
|
||||
use HasFactory;
|
||||
|
||||
public const STATUS_RUNNING = 'running';
|
||||
|
||||
public const STATUS_SUCCESS = 'success';
|
||||
|
||||
public const STATUS_PARTIAL = 'partial';
|
||||
|
||||
public const STATUS_FAILED = 'failed';
|
||||
|
||||
public const STATUS_SKIPPED = 'skipped';
|
||||
|
||||
protected $guarded = [];
|
||||
|
||||
protected $casts = [
|
||||
'selection_payload' => 'array',
|
||||
'had_errors' => 'boolean',
|
||||
'error_codes' => 'array',
|
||||
'error_context' => 'array',
|
||||
'started_at' => 'datetime',
|
||||
'finished_at' => 'datetime',
|
||||
];
|
||||
|
||||
public function tenant(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(Tenant::class);
|
||||
}
|
||||
|
||||
public function scopeCompleted(Builder $query): Builder
|
||||
{
|
||||
return $query
|
||||
->whereIn('status', [
|
||||
self::STATUS_SUCCESS,
|
||||
self::STATUS_PARTIAL,
|
||||
self::STATUS_FAILED,
|
||||
self::STATUS_SKIPPED,
|
||||
])
|
||||
->whereNotNull('finished_at');
|
||||
}
|
||||
}
|
||||
40
app/Services/Inventory/InventoryConcurrencyLimiter.php
Normal file
40
app/Services/Inventory/InventoryConcurrencyLimiter.php
Normal file
@ -0,0 +1,40 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Inventory;
|
||||
|
||||
use Illuminate\Contracts\Cache\Lock;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
|
||||
class InventoryConcurrencyLimiter
|
||||
{
|
||||
public function __construct(private readonly int $lockTtlSeconds = 900) {}
|
||||
|
||||
public function acquireGlobalSlot(): ?Lock
|
||||
{
|
||||
$max = (int) config('tenantpilot.inventory_sync.concurrency.global_max', 2);
|
||||
$max = max(0, $max);
|
||||
|
||||
return $this->acquireSlot('inventory_sync:global:slot:', $max);
|
||||
}
|
||||
|
||||
public function acquireTenantSlot(int $tenantId): ?Lock
|
||||
{
|
||||
$max = (int) config('tenantpilot.inventory_sync.concurrency.per_tenant_max', 1);
|
||||
$max = max(0, $max);
|
||||
|
||||
return $this->acquireSlot("inventory_sync:tenant:{$tenantId}:slot:", $max);
|
||||
}
|
||||
|
||||
private function acquireSlot(string $prefix, int $max): ?Lock
|
||||
{
|
||||
for ($slot = 0; $slot < $max; $slot++) {
|
||||
$lock = Cache::lock($prefix.$slot, $this->lockTtlSeconds);
|
||||
|
||||
if ($lock->get()) {
|
||||
return $lock;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
101
app/Services/Inventory/InventoryMetaSanitizer.php
Normal file
101
app/Services/Inventory/InventoryMetaSanitizer.php
Normal file
@ -0,0 +1,101 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Inventory;
|
||||
|
||||
class InventoryMetaSanitizer
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $meta
|
||||
* @return array{odata_type?: string, etag?: string|null, scope_tag_ids?: list<string>, assignment_target_count?: int|null, warnings?: list<string>}
|
||||
*/
|
||||
public function sanitize(array $meta): array
|
||||
{
|
||||
$sanitized = [];
|
||||
|
||||
$odataType = $meta['odata_type'] ?? null;
|
||||
if (is_string($odataType) && trim($odataType) !== '') {
|
||||
$sanitized['odata_type'] = trim($odataType);
|
||||
}
|
||||
|
||||
$etag = $meta['etag'] ?? null;
|
||||
if ($etag === null || is_string($etag)) {
|
||||
$sanitized['etag'] = $etag === null ? null : trim($etag);
|
||||
}
|
||||
|
||||
$scopeTagIds = $meta['scope_tag_ids'] ?? null;
|
||||
if (is_array($scopeTagIds)) {
|
||||
$sanitized['scope_tag_ids'] = $this->stringList($scopeTagIds);
|
||||
}
|
||||
|
||||
$assignmentTargetCount = $meta['assignment_target_count'] ?? null;
|
||||
if (is_int($assignmentTargetCount)) {
|
||||
$sanitized['assignment_target_count'] = $assignmentTargetCount;
|
||||
} elseif (is_numeric($assignmentTargetCount)) {
|
||||
$sanitized['assignment_target_count'] = (int) $assignmentTargetCount;
|
||||
} elseif ($assignmentTargetCount === null) {
|
||||
$sanitized['assignment_target_count'] = null;
|
||||
}
|
||||
|
||||
$warnings = $meta['warnings'] ?? null;
|
||||
if (is_array($warnings)) {
|
||||
$sanitized['warnings'] = $this->boundedStringList($warnings, 25, 200);
|
||||
}
|
||||
|
||||
return array_filter(
|
||||
$sanitized,
|
||||
static fn (mixed $value): bool => $value !== null && $value !== [] && $value !== ''
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<mixed> $values
|
||||
* @return list<string>
|
||||
*/
|
||||
private function stringList(array $values): array
|
||||
{
|
||||
$result = [];
|
||||
|
||||
foreach ($values as $value) {
|
||||
if (! is_string($value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$value = trim($value);
|
||||
if ($value === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$result[] = $value;
|
||||
}
|
||||
|
||||
return array_values(array_unique($result));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<mixed> $values
|
||||
* @return list<string>
|
||||
*/
|
||||
private function boundedStringList(array $values, int $maxItems, int $maxLen): array
|
||||
{
|
||||
$items = [];
|
||||
|
||||
foreach ($values as $value) {
|
||||
if (count($items) >= $maxItems) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (! is_string($value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$value = trim($value);
|
||||
if ($value === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$items[] = mb_substr($value, 0, $maxLen);
|
||||
}
|
||||
|
||||
return array_values(array_unique($items));
|
||||
}
|
||||
}
|
||||
66
app/Services/Inventory/InventoryMissingService.php
Normal file
66
app/Services/Inventory/InventoryMissingService.php
Normal file
@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Inventory;
|
||||
|
||||
use App\Models\InventoryItem;
|
||||
use App\Models\InventorySyncRun;
|
||||
use App\Models\Tenant;
|
||||
use App\Services\BackupScheduling\PolicyTypeResolver;
|
||||
use Illuminate\Database\Eloquent\Collection;
|
||||
|
||||
class InventoryMissingService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly InventorySelectionHasher $selectionHasher,
|
||||
private readonly PolicyTypeResolver $policyTypeResolver,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $selectionPayload
|
||||
* @return array{latestRun: InventorySyncRun|null, missing: Collection<int, InventoryItem>, lowConfidence: bool}
|
||||
*/
|
||||
public function missingForSelection(Tenant $tenant, array $selectionPayload): array
|
||||
{
|
||||
$normalized = $this->selectionHasher->normalize($selectionPayload);
|
||||
$normalized['policy_types'] = $this->policyTypeResolver->filterRuntime($normalized['policy_types']);
|
||||
$selectionHash = $this->selectionHasher->hash($normalized);
|
||||
|
||||
$latestRun = InventorySyncRun::query()
|
||||
->where('tenant_id', $tenant->getKey())
|
||||
->where('selection_hash', $selectionHash)
|
||||
->whereIn('status', [
|
||||
InventorySyncRun::STATUS_SUCCESS,
|
||||
InventorySyncRun::STATUS_PARTIAL,
|
||||
InventorySyncRun::STATUS_FAILED,
|
||||
InventorySyncRun::STATUS_SKIPPED,
|
||||
])
|
||||
->orderByDesc('finished_at')
|
||||
->orderByDesc('id')
|
||||
->first();
|
||||
|
||||
if (! $latestRun) {
|
||||
return [
|
||||
'latestRun' => null,
|
||||
'missing' => InventoryItem::query()->whereRaw('1 = 0')->get(),
|
||||
'lowConfidence' => true,
|
||||
];
|
||||
}
|
||||
|
||||
$missingQuery = InventoryItem::query()
|
||||
->where('tenant_id', $tenant->getKey())
|
||||
->whereIn('policy_type', $normalized['policy_types'])
|
||||
->where(function ($query) use ($latestRun): void {
|
||||
$query
|
||||
->whereNull('last_seen_run_id')
|
||||
->orWhere('last_seen_run_id', '!=', $latestRun->getKey());
|
||||
});
|
||||
|
||||
$lowConfidence = $latestRun->status !== InventorySyncRun::STATUS_SUCCESS || (bool) ($latestRun->had_errors ?? false);
|
||||
|
||||
return [
|
||||
'latestRun' => $latestRun,
|
||||
'missing' => $missingQuery->get(),
|
||||
'lowConfidence' => $lowConfidence,
|
||||
];
|
||||
}
|
||||
}
|
||||
89
app/Services/Inventory/InventorySelectionHasher.php
Normal file
89
app/Services/Inventory/InventorySelectionHasher.php
Normal file
@ -0,0 +1,89 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Inventory;
|
||||
|
||||
class InventorySelectionHasher
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $selectionPayload
|
||||
* @return array{policy_types: list<string>, categories: list<string>, include_foundations: bool, include_dependencies: bool}
|
||||
*/
|
||||
public function normalize(array $selectionPayload): array
|
||||
{
|
||||
$policyTypes = $this->stringList($selectionPayload['policy_types'] ?? []);
|
||||
sort($policyTypes);
|
||||
|
||||
$categories = $this->stringList($selectionPayload['categories'] ?? []);
|
||||
sort($categories);
|
||||
|
||||
return [
|
||||
'policy_types' => $policyTypes,
|
||||
'categories' => $categories,
|
||||
'include_foundations' => (bool) ($selectionPayload['include_foundations'] ?? false),
|
||||
'include_dependencies' => (bool) ($selectionPayload['include_dependencies'] ?? false),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $selectionPayload
|
||||
*/
|
||||
public function canonicalJson(array $selectionPayload): string
|
||||
{
|
||||
$normalized = $this->normalize($selectionPayload);
|
||||
$normalized = $this->ksortRecursive($normalized);
|
||||
|
||||
return (string) json_encode($normalized, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $selectionPayload
|
||||
*/
|
||||
public function hash(array $selectionPayload): string
|
||||
{
|
||||
return hash('sha256', $this->canonicalJson($selectionPayload));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<string>
|
||||
*/
|
||||
private function stringList(mixed $value): array
|
||||
{
|
||||
if (! is_array($value)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$result = [];
|
||||
foreach ($value as $item) {
|
||||
if (! is_string($item)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$item = trim($item);
|
||||
if ($item === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$result[] = $item;
|
||||
}
|
||||
|
||||
return array_values(array_unique($result));
|
||||
}
|
||||
|
||||
private function ksortRecursive(mixed $value): mixed
|
||||
{
|
||||
if (! is_array($value)) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
$isList = array_is_list($value);
|
||||
if (! $isList) {
|
||||
ksort($value);
|
||||
}
|
||||
|
||||
foreach ($value as $key => $child) {
|
||||
$value[$key] = $this->ksortRecursive($child);
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
}
|
||||
323
app/Services/Inventory/InventorySyncService.php
Normal file
323
app/Services/Inventory/InventorySyncService.php
Normal file
@ -0,0 +1,323 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Inventory;
|
||||
|
||||
use App\Models\InventoryItem;
|
||||
use App\Models\InventorySyncRun;
|
||||
use App\Models\Tenant;
|
||||
use App\Services\BackupScheduling\PolicyTypeResolver;
|
||||
use App\Services\Graph\GraphClientInterface;
|
||||
use App\Services\Graph\GraphResponse;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Contracts\Cache\Lock;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Throwable;
|
||||
|
||||
class InventorySyncService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly GraphClientInterface $graphClient,
|
||||
private readonly PolicyTypeResolver $policyTypeResolver,
|
||||
private readonly InventorySelectionHasher $selectionHasher,
|
||||
private readonly InventoryMetaSanitizer $metaSanitizer,
|
||||
private readonly InventoryConcurrencyLimiter $concurrencyLimiter,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Runs an inventory sync inline (no queue), enforcing locks/concurrency and creating an observable run record.
|
||||
*
|
||||
* @param array<string, mixed> $selectionPayload
|
||||
*/
|
||||
public function syncNow(Tenant $tenant, array $selectionPayload): InventorySyncRun
|
||||
{
|
||||
$normalizedSelection = $this->selectionHasher->normalize($selectionPayload);
|
||||
$normalizedSelection['policy_types'] = $this->policyTypeResolver->filterRuntime($normalizedSelection['policy_types']);
|
||||
$selectionHash = $this->selectionHasher->hash($normalizedSelection);
|
||||
|
||||
$now = CarbonImmutable::now('UTC');
|
||||
|
||||
$globalSlot = $this->concurrencyLimiter->acquireGlobalSlot();
|
||||
if (! $globalSlot instanceof Lock) {
|
||||
return $this->createSkippedRun(
|
||||
tenant: $tenant,
|
||||
selectionHash: $selectionHash,
|
||||
selectionPayload: $normalizedSelection,
|
||||
now: $now,
|
||||
errorCode: 'concurrency_limit_global',
|
||||
);
|
||||
}
|
||||
|
||||
$tenantSlot = $this->concurrencyLimiter->acquireTenantSlot((int) $tenant->id);
|
||||
if (! $tenantSlot instanceof Lock) {
|
||||
$globalSlot->release();
|
||||
|
||||
return $this->createSkippedRun(
|
||||
tenant: $tenant,
|
||||
selectionHash: $selectionHash,
|
||||
selectionPayload: $normalizedSelection,
|
||||
now: $now,
|
||||
errorCode: 'concurrency_limit_tenant',
|
||||
);
|
||||
}
|
||||
|
||||
$selectionLock = Cache::lock($this->selectionLockKey($tenant, $selectionHash), 900);
|
||||
if (! $selectionLock->get()) {
|
||||
$tenantSlot->release();
|
||||
$globalSlot->release();
|
||||
|
||||
return $this->createSkippedRun(
|
||||
tenant: $tenant,
|
||||
selectionHash: $selectionHash,
|
||||
selectionPayload: $normalizedSelection,
|
||||
now: $now,
|
||||
errorCode: 'lock_contended',
|
||||
);
|
||||
}
|
||||
|
||||
$run = InventorySyncRun::query()->create([
|
||||
'tenant_id' => $tenant->getKey(),
|
||||
'selection_hash' => $selectionHash,
|
||||
'selection_payload' => $normalizedSelection,
|
||||
'status' => InventorySyncRun::STATUS_RUNNING,
|
||||
'had_errors' => false,
|
||||
'error_codes' => [],
|
||||
'error_context' => null,
|
||||
'started_at' => $now,
|
||||
'finished_at' => null,
|
||||
'items_observed_count' => 0,
|
||||
'items_upserted_count' => 0,
|
||||
'errors_count' => 0,
|
||||
]);
|
||||
|
||||
try {
|
||||
return $this->executeRun($run, $tenant, $normalizedSelection);
|
||||
} finally {
|
||||
$selectionLock->release();
|
||||
$tenantSlot->release();
|
||||
$globalSlot->release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{policy_types: list<string>, categories: list<string>, include_foundations: bool, include_dependencies: bool} $normalizedSelection
|
||||
*/
|
||||
private function executeRun(InventorySyncRun $run, Tenant $tenant, array $normalizedSelection): InventorySyncRun
|
||||
{
|
||||
$observed = 0;
|
||||
$upserted = 0;
|
||||
$errors = 0;
|
||||
$errorCodes = [];
|
||||
$hadErrors = false;
|
||||
|
||||
try {
|
||||
$typesConfig = $this->supportedTypeConfigByType();
|
||||
|
||||
foreach ($normalizedSelection['policy_types'] as $policyType) {
|
||||
$typeConfig = $typesConfig[$policyType] ?? null;
|
||||
|
||||
if (! is_array($typeConfig)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$response = $this->listPoliciesWithRetry($policyType, [
|
||||
'tenant' => $tenant->tenant_id ?? $tenant->external_id,
|
||||
'client_id' => $tenant->app_client_id,
|
||||
'client_secret' => $tenant->app_client_secret,
|
||||
'platform' => $typeConfig['platform'] ?? null,
|
||||
'filter' => $typeConfig['filter'] ?? null,
|
||||
]);
|
||||
|
||||
if ($response->failed()) {
|
||||
$hadErrors = true;
|
||||
$errors++;
|
||||
$errorCodes[] = $this->mapGraphFailureToErrorCode($response);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($response->data as $policyData) {
|
||||
if (! is_array($policyData)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$externalId = $policyData['id'] ?? $policyData['external_id'] ?? null;
|
||||
if (! is_string($externalId) || $externalId === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$observed++;
|
||||
|
||||
$displayName = $policyData['displayName'] ?? $policyData['name'] ?? null;
|
||||
$displayName = is_string($displayName) ? $displayName : null;
|
||||
|
||||
$scopeTagIds = $policyData['roleScopeTagIds'] ?? null;
|
||||
$assignmentTargetCount = null;
|
||||
$assignments = $policyData['assignments'] ?? null;
|
||||
if (is_array($assignments)) {
|
||||
$assignmentTargetCount = count($assignments);
|
||||
}
|
||||
|
||||
$meta = $this->metaSanitizer->sanitize([
|
||||
'odata_type' => $policyData['@odata.type'] ?? $policyData['@OData.Type'] ?? null,
|
||||
'etag' => $policyData['@odata.etag'] ?? null,
|
||||
'scope_tag_ids' => is_array($scopeTagIds) ? $scopeTagIds : null,
|
||||
'assignment_target_count' => $assignmentTargetCount,
|
||||
'warnings' => [],
|
||||
]);
|
||||
|
||||
InventoryItem::query()->updateOrCreate(
|
||||
[
|
||||
'tenant_id' => $tenant->getKey(),
|
||||
'policy_type' => $policyType,
|
||||
'external_id' => $externalId,
|
||||
],
|
||||
[
|
||||
'display_name' => $displayName,
|
||||
'category' => $typeConfig['category'] ?? null,
|
||||
'platform' => $typeConfig['platform'] ?? null,
|
||||
'meta_jsonb' => $meta,
|
||||
'last_seen_at' => now(),
|
||||
'last_seen_run_id' => $run->getKey(),
|
||||
]
|
||||
);
|
||||
|
||||
$upserted++;
|
||||
}
|
||||
}
|
||||
|
||||
$status = $hadErrors ? InventorySyncRun::STATUS_PARTIAL : InventorySyncRun::STATUS_SUCCESS;
|
||||
|
||||
$run->update([
|
||||
'status' => $status,
|
||||
'had_errors' => $hadErrors,
|
||||
'error_codes' => array_values(array_unique($errorCodes)),
|
||||
'error_context' => null,
|
||||
'items_observed_count' => $observed,
|
||||
'items_upserted_count' => $upserted,
|
||||
'errors_count' => $errors,
|
||||
'finished_at' => CarbonImmutable::now('UTC'),
|
||||
]);
|
||||
|
||||
return $run->refresh();
|
||||
} catch (Throwable $throwable) {
|
||||
$run->update([
|
||||
'status' => InventorySyncRun::STATUS_FAILED,
|
||||
'had_errors' => true,
|
||||
'error_codes' => ['unexpected_exception'],
|
||||
'error_context' => $this->safeErrorContext($throwable),
|
||||
'items_observed_count' => $observed,
|
||||
'items_upserted_count' => $upserted,
|
||||
'errors_count' => $errors + 1,
|
||||
'finished_at' => CarbonImmutable::now('UTC'),
|
||||
]);
|
||||
|
||||
return $run->refresh();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, array<string, mixed>>
|
||||
*/
|
||||
private function supportedTypeConfigByType(): array
|
||||
{
|
||||
/** @var array<int, array<string, mixed>> $supported */
|
||||
$supported = config('tenantpilot.supported_policy_types', []);
|
||||
|
||||
$byType = [];
|
||||
foreach ($supported as $config) {
|
||||
$type = $config['type'] ?? null;
|
||||
if (is_string($type) && $type !== '') {
|
||||
$byType[$type] = $config;
|
||||
}
|
||||
}
|
||||
|
||||
return $byType;
|
||||
}
|
||||
|
||||
private function selectionLockKey(Tenant $tenant, string $selectionHash): string
|
||||
{
|
||||
return sprintf('inventory_sync:tenant:%s:selection:%s', (string) $tenant->getKey(), $selectionHash);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $selectionPayload
|
||||
*/
|
||||
private function createSkippedRun(
|
||||
Tenant $tenant,
|
||||
string $selectionHash,
|
||||
array $selectionPayload,
|
||||
CarbonImmutable $now,
|
||||
string $errorCode,
|
||||
): InventorySyncRun {
|
||||
return InventorySyncRun::query()->create([
|
||||
'tenant_id' => $tenant->getKey(),
|
||||
'selection_hash' => $selectionHash,
|
||||
'selection_payload' => $selectionPayload,
|
||||
'status' => InventorySyncRun::STATUS_SKIPPED,
|
||||
'had_errors' => true,
|
||||
'error_codes' => [$errorCode],
|
||||
'error_context' => null,
|
||||
'started_at' => $now,
|
||||
'finished_at' => $now,
|
||||
'items_observed_count' => 0,
|
||||
'items_upserted_count' => 0,
|
||||
'errors_count' => 0,
|
||||
]);
|
||||
}
|
||||
|
||||
private function mapGraphFailureToErrorCode(GraphResponse $response): string
|
||||
{
|
||||
$status = (int) ($response->status ?? 0);
|
||||
|
||||
return match ($status) {
|
||||
403 => 'graph_forbidden',
|
||||
429 => 'graph_throttled',
|
||||
503 => 'graph_transient',
|
||||
default => 'graph_transient',
|
||||
};
|
||||
}
|
||||
|
||||
private function listPoliciesWithRetry(string $policyType, array $options): GraphResponse
|
||||
{
|
||||
$maxAttempts = 3;
|
||||
|
||||
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
|
||||
$response = $this->graphClient->listPolicies($policyType, $options);
|
||||
|
||||
if (! $response->failed()) {
|
||||
return $response;
|
||||
}
|
||||
|
||||
$status = (int) ($response->status ?? 0);
|
||||
if (! in_array($status, [429, 503], true)) {
|
||||
return $response;
|
||||
}
|
||||
|
||||
if ($attempt >= $maxAttempts) {
|
||||
return $response;
|
||||
}
|
||||
|
||||
$baseMs = 250 * (2 ** ($attempt - 1));
|
||||
$jitterMs = random_int(0, 250);
|
||||
usleep(($baseMs + $jitterMs) * 1000);
|
||||
}
|
||||
|
||||
return new GraphResponse(false, [], null, ['error' => ['code' => 'unexpected_exception', 'message' => 'retry loop failed']]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function safeErrorContext(Throwable $throwable): array
|
||||
{
|
||||
$message = $throwable->getMessage();
|
||||
|
||||
$message = preg_replace('/Bearer\s+[A-Za-z0-9\-\._~\+\/]+=*/', 'Bearer [REDACTED]', (string) $message);
|
||||
$message = mb_substr((string) $message, 0, 500);
|
||||
|
||||
return [
|
||||
'exception_class' => get_class($throwable),
|
||||
'message' => $message,
|
||||
];
|
||||
}
|
||||
}
|
||||
@ -320,6 +320,13 @@
|
||||
'poll_interval_seconds' => (int) env('TENANTPILOT_BULK_POLL_INTERVAL_SECONDS', 3),
|
||||
],
|
||||
|
||||
'inventory_sync' => [
|
||||
'concurrency' => [
|
||||
'global_max' => (int) env('TENANTPILOT_INVENTORY_SYNC_CONCURRENCY_GLOBAL_MAX', 2),
|
||||
'per_tenant_max' => (int) env('TENANTPILOT_INVENTORY_SYNC_CONCURRENCY_PER_TENANT_MAX', 1),
|
||||
],
|
||||
],
|
||||
|
||||
'display' => [
|
||||
'show_script_content' => (bool) env('TENANTPILOT_SHOW_SCRIPT_CONTENT', false),
|
||||
'max_script_content_chars' => (int) env('TENANTPILOT_MAX_SCRIPT_CONTENT_CHARS', 5000),
|
||||
|
||||
38
database/factories/InventoryItemFactory.php
Normal file
38
database/factories/InventoryItemFactory.php
Normal file
@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
namespace Database\Factories;
|
||||
|
||||
use App\Models\Tenant;
|
||||
use Illuminate\Database\Eloquent\Factories\Factory;
|
||||
|
||||
/**
|
||||
* @extends \Illuminate\Database\Eloquent\Factories\Factory<\App\Models\InventoryItem>
|
||||
*/
|
||||
class InventoryItemFactory extends Factory
|
||||
{
|
||||
/**
|
||||
* Define the model's default state.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function definition(): array
|
||||
{
|
||||
return [
|
||||
'tenant_id' => Tenant::factory(),
|
||||
'policy_type' => 'deviceConfiguration',
|
||||
'external_id' => fake()->uuid(),
|
||||
'display_name' => fake()->words(3, true),
|
||||
'category' => 'Configuration',
|
||||
'platform' => fake()->randomElement(['android', 'iOS', 'macOS', 'windows10', 'windows']),
|
||||
'meta_jsonb' => [
|
||||
'odata_type' => '#microsoft.graph.deviceConfiguration',
|
||||
'etag' => null,
|
||||
'scope_tag_ids' => [],
|
||||
'assignment_target_count' => null,
|
||||
'warnings' => [],
|
||||
],
|
||||
'last_seen_at' => now(),
|
||||
'last_seen_run_id' => null,
|
||||
];
|
||||
}
|
||||
}
|
||||
43
database/factories/InventorySyncRunFactory.php
Normal file
43
database/factories/InventorySyncRunFactory.php
Normal file
@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
namespace Database\Factories;
|
||||
|
||||
use App\Models\InventorySyncRun;
|
||||
use App\Models\Tenant;
|
||||
use Illuminate\Database\Eloquent\Factories\Factory;
|
||||
|
||||
/**
|
||||
* @extends \Illuminate\Database\Eloquent\Factories\Factory<\App\Models\InventorySyncRun>
|
||||
*/
|
||||
class InventorySyncRunFactory extends Factory
|
||||
{
|
||||
/**
|
||||
* Define the model's default state.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function definition(): array
|
||||
{
|
||||
$selectionPayload = [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
return [
|
||||
'tenant_id' => Tenant::factory(),
|
||||
'selection_hash' => hash('sha256', (string) json_encode($selectionPayload)),
|
||||
'selection_payload' => $selectionPayload,
|
||||
'status' => InventorySyncRun::STATUS_SUCCESS,
|
||||
'had_errors' => false,
|
||||
'error_codes' => [],
|
||||
'error_context' => null,
|
||||
'started_at' => now()->subMinute(),
|
||||
'finished_at' => now(),
|
||||
'items_observed_count' => 0,
|
||||
'items_upserted_count' => 0,
|
||||
'errors_count' => 0,
|
||||
];
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
// No-op: replaced by 2026_01_07_142720_create_inventory_items_table.php
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
// No-op
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,46 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('inventory_sync_runs', function (Blueprint $table) {
|
||||
$table->id();
|
||||
|
||||
$table->foreignId('tenant_id')->constrained();
|
||||
|
||||
$table->string('selection_hash', 64);
|
||||
$table->jsonb('selection_payload')->nullable();
|
||||
|
||||
$table->string('status');
|
||||
$table->boolean('had_errors')->default(false);
|
||||
$table->jsonb('error_codes')->nullable();
|
||||
$table->jsonb('error_context')->nullable();
|
||||
|
||||
$table->timestampTz('started_at')->nullable();
|
||||
$table->timestampTz('finished_at')->nullable();
|
||||
|
||||
$table->unsignedInteger('items_observed_count')->default(0);
|
||||
$table->unsignedInteger('items_upserted_count')->default(0);
|
||||
$table->unsignedInteger('errors_count')->default(0);
|
||||
|
||||
$table->timestamps();
|
||||
|
||||
$table->index(['tenant_id', 'selection_hash']);
|
||||
$table->index(['tenant_id', 'status']);
|
||||
$table->index(['tenant_id', 'finished_at']);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::dropIfExists('inventory_sync_runs');
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('inventory_items', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->foreignId('tenant_id')->constrained();
|
||||
$table->string('policy_type');
|
||||
$table->string('external_id');
|
||||
$table->string('display_name')->nullable();
|
||||
$table->string('category')->nullable();
|
||||
$table->string('platform')->nullable();
|
||||
$table->jsonb('meta_jsonb')->nullable();
|
||||
$table->timestampTz('last_seen_at')->nullable();
|
||||
$table->foreignId('last_seen_run_id')
|
||||
->nullable()
|
||||
->constrained('inventory_sync_runs')
|
||||
->nullOnDelete();
|
||||
$table->timestamps();
|
||||
|
||||
$table->unique(['tenant_id', 'policy_type', 'external_id']);
|
||||
$table->index(['tenant_id', 'policy_type']);
|
||||
$table->index(['tenant_id', 'category']);
|
||||
$table->index('last_seen_at');
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::dropIfExists('inventory_items');
|
||||
}
|
||||
};
|
||||
45
specs/040-inventory-core/checklists/requirements.md
Normal file
45
specs/040-inventory-core/checklists/requirements.md
Normal file
@ -0,0 +1,45 @@
|
||||
# Requirements Checklist — Inventory Core (040)
|
||||
|
||||
## Scope
|
||||
|
||||
- [x] This checklist applies only to Spec 040 (Inventory Core).
|
||||
|
||||
## Constitution Gates
|
||||
|
||||
- [x] Inventory-first: inventory stores “last observed” only (no snapshot/backup side effects)
|
||||
- [x] Read/write separation: no Intune write paths introduced
|
||||
- [x] Single contract path to Graph: Graph access only via GraphClientInterface + contracts (if used)
|
||||
- [x] Tenant isolation: all reads/writes tenant-scoped
|
||||
- [x] Automation is idempotent & observable: run records + locks + stable error codes
|
||||
- [x] Data minimization & safe logging: no secrets/tokens; DB run records are the source of truth
|
||||
|
||||
## Functional Requirements Coverage
|
||||
|
||||
- [x] FR-001 Inventory catalog exists (inventory_items)
|
||||
- [x] FR-002 Stable upsert identity prevents duplicates
|
||||
- [x] FR-003 Sync runs recorded (inventory_sync_runs)
|
||||
- [x] FR-004 Tenant isolation enforced
|
||||
- [x] FR-005 Deterministic selection_hash implemented and tested
|
||||
- [x] FR-006 Sync does NOT create snapshots/backups (explicitly tested)
|
||||
- [x] FR-007 Missing is derived (not persisted) relative to latestRun(tenant_id, selection_hash)
|
||||
- [x] FR-008 meta_jsonb whitelist enforced; unknown keys dropped; never fails sync
|
||||
- [x] FR-009 Locks/idempotency/observable failures implemented
|
||||
|
||||
## Non-Functional Requirements Coverage
|
||||
|
||||
- [x] NFR-001 Global + per-tenant concurrency limits (source + defaults + behavior defined)
|
||||
- [x] NFR-002 Throttling resilience (429/503 backoff+jitter) with stable error codes
|
||||
- [x] NFR-003 Deterministic behavior is testable (Pest)
|
||||
- [x] NFR-004 Data minimization enforced (no raw payload storage)
|
||||
- [x] NFR-005 Safe logging: no secrets/tokens; rely on run records not log parsing
|
||||
|
||||
## Tests (Pest)
|
||||
|
||||
- [x] Upsert prevents duplicates + updates last_seen fields
|
||||
- [x] selection_hash determinism (order invariant)
|
||||
- [x] Missing semantics per latest completed run and selection isolation
|
||||
- [x] Low-confidence missing when latest run partial/failed/had_errors
|
||||
- [x] meta whitelist drops unknown keys
|
||||
- [x] Lock prevents overlapping runs for same tenant+selection
|
||||
- [x] No snapshot/backup tables are written during sync
|
||||
- [x] Error reporting uses stable error codes; no secrets/tokens persisted in run error details
|
||||
@ -1,32 +1,126 @@
|
||||
# Implementation Plan: Inventory Core (040)
|
||||
|
||||
**Branch**: `feat/040-inventory-core` | **Date**: 2026-01-07 | **Spec**: `specs/040-inventory-core/spec.md`
|
||||
**Branch**: `spec/040-inventory-core` | **Date**: 2026-01-07 | **Spec**: `specs/040-inventory-core/spec.md`
|
||||
**Scope (this step)**: Produce a clean, implementable `plan.md` + consistent `tasks.md` for Spec 040 only.
|
||||
|
||||
## Summary
|
||||
|
||||
Implement a tenant-scoped inventory catalog (“last observed”) and an observable sync run system with deterministic selection scoping. Ensure no snapshots/backups are created by sync.
|
||||
Implement tenant-scoped Inventory + Sync Run tracking as the foundational substrate for later Inventory UI and higher-order features.
|
||||
|
||||
Key outcomes:
|
||||
|
||||
- Inventory is “last observed” (not backup), stored as metadata + whitelisted `meta_jsonb`.
|
||||
- Sync runs are observable, selection-scoped via deterministic `selection_hash`.
|
||||
- “Missing” is derived relative to latest completed run for the same `(tenant_id, selection_hash)`.
|
||||
- Automation is safe: locks, idempotency, throttling handling, global+per-tenant concurrency limits.
|
||||
|
||||
## Technical Context
|
||||
|
||||
- **Language/Version**: PHP 8.4
|
||||
- **Framework**: Laravel 12
|
||||
- **Admin UI**: Filament v4 + Livewire v3
|
||||
- **Storage**: PostgreSQL (JSONB available)
|
||||
- **Queue/Locks**: Laravel queue + cache/Redis locks (as configured)
|
||||
- **Testing**: Pest v4 (`php artisan test`)
|
||||
- **Target Platform**: Sail-first local dev; container deploy (Dokploy)
|
||||
|
||||
## Constitution Check
|
||||
|
||||
- Inventory-first, snapshots-second (sync never creates snapshots)
|
||||
- Read/write separation (sync is read-only; any future writes require preview/confirmation/audit/tests)
|
||||
- Single contract path to Graph (Graph access only through existing abstractions/contracts)
|
||||
- Deterministic capabilities (capabilities resolver output testable)
|
||||
- Tenant isolation (non-negotiable)
|
||||
- Automation observable + idempotent (locks, run records, stable error codes, 429/503 handling)
|
||||
- Data minimization + safe logging
|
||||
- Inventory-first: inventory stores last observed state only (no snapshot/backup side effects).
|
||||
- Read/write separation: this feature introduces no Intune write paths.
|
||||
- Single contract path to Graph: Graph reads (if needed) go via Graph abstraction and contracts.
|
||||
- Tenant isolation: all reads/writes tenant-scoped; no cross-tenant shortcuts.
|
||||
- Automation: locked + idempotent + observable; handle 429/503 with backoff+jitter.
|
||||
- Data minimization: no payload-heavy storage; safe logs.
|
||||
|
||||
## Deliverables (Phase-friendly)
|
||||
No constitution violations expected.
|
||||
|
||||
- Data model for inventory items and sync runs
|
||||
- Sync engine orchestration and locking strategy
|
||||
- Deterministic selection hashing
|
||||
- Capabilities resolver output snapshot tests
|
||||
- Minimal Filament/CLI surface to trigger and observe sync runs (if required by tasks)
|
||||
## Project Structure (Impacted Areas)
|
||||
|
||||
## Out of Scope
|
||||
```text
|
||||
specs/040-inventory-core/
|
||||
├── spec.md
|
||||
├── plan.md
|
||||
└── tasks.md
|
||||
|
||||
- Dependency graph hydration (spec 042)
|
||||
- Cross-tenant promotion (spec 043)
|
||||
- Drift reporting (spec 044)
|
||||
- Lifecycle “deleted” semantics (feature 900)
|
||||
app/
|
||||
├── Models/
|
||||
├── Jobs/
|
||||
├── Services/
|
||||
└── Support/
|
||||
|
||||
database/migrations/
|
||||
tests/Feature/
|
||||
tests/Unit/
|
||||
```
|
||||
|
||||
## Implementation Approach
|
||||
|
||||
### Phase A — Data Model + Migrations
|
||||
|
||||
1. Add `inventory_items` table
|
||||
- Identity: unique constraint to prevent duplicates, recommended:
|
||||
- `(tenant_id, policy_type, external_id)`
|
||||
- Fields: `display_name`, `platform`/`category` (if applicable), `meta_jsonb`, `last_seen_at`, `last_seen_run_id`.
|
||||
- Indexing: indexes supporting tenant/type listing; consider partials as needed.
|
||||
|
||||
2. Add `inventory_sync_runs` table
|
||||
- Identity: `tenant_id`, `selection_hash`
|
||||
- Status fields: `status`, `started_at`, `finished_at`, `had_errors`
|
||||
- Counters: `items_observed_count`, `items_upserted_count`, `errors_count`
|
||||
- Error reporting: stable error code(s) list or summary field.
|
||||
|
||||
### Phase B — Selection Hash (Deterministic)
|
||||
|
||||
Implement canonicalization exactly as Spec Appendix:
|
||||
|
||||
- Only include scope-affecting keys in `selection_payload`.
|
||||
- Sort object keys; sort `policy_types[]` and `categories[]` arrays.
|
||||
- Compute `selection_hash = sha256(canonical_json(selection_payload))`.
|
||||
|
||||
### Phase C — Sync Run Lifecycle + Upsert
|
||||
|
||||
- Create a service that:
|
||||
- acquires a lock for `(tenant_id, selection_hash)`
|
||||
- creates a run record
|
||||
- enumerates selected policy types
|
||||
- upserts inventory items by identity key
|
||||
- updates `last_seen_at` and `last_seen_run_id` per observed item
|
||||
- finalizes run status + counters
|
||||
- never creates/modifies snapshot/backup records (`policy_versions`, `backup_*`)
|
||||
|
||||
### Phase D — Derived “Missing” Semantics
|
||||
|
||||
- Implement “missing” as a computed state relative to `latestRun(tenant_id, selection_hash)`.
|
||||
- Do not persist “missing” or “deleted”.
|
||||
- Mark missing as low-confidence when `latestRun.status != success` or `latestRun.had_errors = true`.
|
||||
|
||||
### Phase E — Meta Whitelist
|
||||
|
||||
- Define a whitelist of allowed `meta_jsonb` keys.
|
||||
- Enforce by dropping unknown keys (never fail sync).
|
||||
|
||||
### Phase F — Concurrency Limits
|
||||
|
||||
- Enforce global concurrency (across tenants) and per-tenant concurrency.
|
||||
- The implementation may be via queue worker limits, semaphore/lock strategy, or both; the behavior must be testable.
|
||||
- When limits are hit, create an observable run record with `status=skipped`, `had_errors=true`, and stable error code(s).
|
||||
|
||||
## Test Plan (Pest)
|
||||
|
||||
Minimum required coverage aligned to Spec test cases:
|
||||
|
||||
- Upsert identity prevents duplicates; `last_seen_*` updates.
|
||||
- `selection_hash` determinism (array ordering invariant).
|
||||
- Missing derived per latest completed run for same `(tenant_id, selection_hash)`.
|
||||
- Low-confidence missing when latest run is partial/failed or had_errors.
|
||||
- Meta whitelist drops unknown keys.
|
||||
- Lock prevents overlapping runs per tenant+selection.
|
||||
- No snapshot/backup rows are created/modified by inventory sync.
|
||||
- Error reporting uses stable `error_codes` and stores no secrets/tokens.
|
||||
|
||||
## Out of Scope (Explicit)
|
||||
|
||||
- Any UI (covered by Spec 041)
|
||||
- Any snapshot/backup creation
|
||||
- Any restore/promotion/remediation write paths
|
||||
|
||||
@ -140,11 +140,64 @@ ### Meta Whitelist (Fail-safe)
|
||||
- `meta_jsonb` has a documented whitelist of allowed keys.
|
||||
- **AC:** Unknown `meta_jsonb` keys are dropped (not persisted) and MUST NOT cause sync to fail.
|
||||
|
||||
#### Initial `meta_jsonb` whitelist (v1)
|
||||
|
||||
Allowed keys (all optional; if not applicable for a type, omit):
|
||||
|
||||
- `odata_type`: string (copied from Graph `@odata.type`)
|
||||
- `etag`: string|null (Graph etag if available; never treated as a secret)
|
||||
- `scope_tag_ids`: array<string> (IDs only; no display names required)
|
||||
- `assignment_target_count`: int|null (count only; no target details)
|
||||
- `warnings`: array<string> (bounded, human-readable, no secrets)
|
||||
|
||||
**AC:** Any other key is dropped silently (not persisted) and MUST NOT fail sync.
|
||||
|
||||
### Observed Run
|
||||
|
||||
- `inventory_items.last_seen_run_id` and `inventory_items.last_seen_at` are updated when an item is observed.
|
||||
- `last_seen_run_id` implies the selection via `sync_runs.selection_hash`; no per-item selection hash is required for core.
|
||||
|
||||
### Run Error Codes (taxonomy)
|
||||
|
||||
Sync runs record:
|
||||
|
||||
- `status`: one of `success|partial|failed|skipped`
|
||||
- `had_errors`: bool (true if any non-ideal condition occurred)
|
||||
- `error_codes[]`: array of stable machine-readable codes (no secrets)
|
||||
|
||||
Minimal taxonomy (3–8 codes):
|
||||
|
||||
- `lock_contended` (a run could not start because the per-tenant+selection lock is held)
|
||||
- `concurrency_limit_global` (global concurrency limit reached; run skipped)
|
||||
- `concurrency_limit_tenant` (per-tenant concurrency limit reached; run skipped)
|
||||
- `graph_throttled` (429 encountered; run partial/failed depending on recovery)
|
||||
- `graph_transient` (503/timeout/other transient errors)
|
||||
- `graph_forbidden` (403/insufficient permission)
|
||||
- `unexpected_exception` (unexpected failure; message must be safe/redacted)
|
||||
|
||||
**Rule:** Run records MUST store codes (and safe, bounded context) rather than raw exception dumps or tokens.
|
||||
|
||||
### Concurrency Limits (source, defaults, behavior)
|
||||
|
||||
**Source:** Config (recommended keys):
|
||||
|
||||
- `tenantpilot.inventory_sync.concurrency.global_max`
|
||||
- `tenantpilot.inventory_sync.concurrency.per_tenant_max`
|
||||
|
||||
**Defaults (if not configured):**
|
||||
|
||||
- global_max = 2
|
||||
- per_tenant_max = 1
|
||||
|
||||
**Behavior when limits are hit:**
|
||||
|
||||
- The system MUST create a Sync Run record with:
|
||||
- `status = skipped`
|
||||
- `had_errors = true` (so missing stays low-confidence for that selection)
|
||||
- `error_codes[]` includes `concurrency_limit_global` or `concurrency_limit_tenant`
|
||||
- `started_at`/`finished_at` set (observable)
|
||||
- No inventory items are mutated in a skipped run.
|
||||
|
||||
## Testing Guidance (non-implementation)
|
||||
|
||||
These are test cases expressed in behavior terms (not code).
|
||||
@ -154,6 +207,7 @@ ### Test Cases — Sync and Upsert
|
||||
- **TC-001**: Sync creates or updates inventory items and sets `last_seen_at`.
|
||||
- **TC-002**: Re-running sync for the same tenant+selection updates existing records and does not create duplicates.
|
||||
- **TC-003**: Inventory queries scoped to Tenant A never return Tenant B’s items.
|
||||
- **TC-004**: Inventory sync does not create or modify snapshot/backup records (e.g., no new rows in `policy_versions`, `backup_sets`, `backup_items`, `backup_schedules`, `backup_schedule_runs`).
|
||||
|
||||
### Test Cases — Selection Hash Determinism
|
||||
|
||||
|
||||
@ -4,29 +4,35 @@ # Tasks: Inventory Core (040)
|
||||
|
||||
## P1 — MVP (US1/US2)
|
||||
|
||||
- [ ] T001 [US1] Define Inventory Item data model (tenant-scoped identity + last_seen fields)
|
||||
- [ ] T002 [US1] Define Sync Run data model (tenant_id, selection_hash, status, timestamps, counts, stable error codes)
|
||||
- [ ] T003 [US1] Implement deterministic selection hashing (canonical json + sha256)
|
||||
- [ ] T004 [US1] Implement inventory upsert semantics (no duplicates)
|
||||
- [ ] T005 [US1] Enforce tenant isolation in all inventory/run queries
|
||||
- [ ] T006 [US2] Implement derived “missing” computation relative to latest completed run (tenant_id + selection_hash)
|
||||
- [ ] T007 [US2] Ensure low-confidence missing when latestRun is partial/failed or had_errors
|
||||
- [ ] T008 [US2] Implement meta_jsonb whitelist enforcement (drop unknown keys, never fail sync)
|
||||
- [X] T001 [US1] Add migrations: `inventory_items` (unique: tenant_id+policy_type+external_id; indexes; last_seen fields)
|
||||
- [X] T002 [US1] Add migrations: `inventory_sync_runs` (tenant_id, selection_hash, status, started/finished, counts, stable error codes)
|
||||
- [X] T003 [US1] Implement deterministic `selection_hash` (canonical JSON: sorted keys + sorted arrays; sha256)
|
||||
- [X] T004 [US1] Implement inventory upsert semantics (idempotent, no duplicates)
|
||||
- [X] T005 [US1] Enforce tenant isolation for all inventory + run read/write paths
|
||||
- [X] T006 [US2] Implement derived “missing” query semantics vs latest completed run for same (tenant_id, selection_hash)
|
||||
- [X] T007 [US2] Missing confidence rule: partial/failed or had_errors => low confidence
|
||||
- [X] T008 [US2] Enforce `meta_jsonb` whitelist (drop unknown keys; never fail sync)
|
||||
- [X] T009 [US1] Guardrail: inventory sync must not create snapshots/backups (no writes to `policy_versions`/`backup_*`)
|
||||
|
||||
## P2 — Observability & Safety (US3 + NFR)
|
||||
|
||||
- [ ] T009 [US3] Ensure run records include stable error codes and counts
|
||||
- [ ] T010 [NFR] Add idempotency + locks to prevent overlapping runs per tenant+selection
|
||||
- [ ] T011 [NFR] Add global + per-tenant concurrency limiting strategy
|
||||
- [ ] T012 [NFR] Implement throttling handling strategy (backoff + jitter for transient Graph failures)
|
||||
- [X] T010 [US3] Run lifecycle: ensure run records include counts + stable error codes (visible and actionable)
|
||||
- [X] T011 [NFR] Locking/idempotency: prevent overlapping runs per (tenant_id, selection_hash)
|
||||
- [X] T012 [NFR] Concurrency: enforce global + per-tenant limits (queue/semaphore strategy)
|
||||
- [X] T013 [NFR] Throttling resilience: backoff + jitter for transient Graph failures (429/503)
|
||||
- [X] T014 [NFR] Safe logging & safe persistence: store only stable `error_codes` + bounded safe context in run records (no secrets/tokens; no log parsing required)
|
||||
|
||||
## Tests (Required for runtime behavior)
|
||||
|
||||
- [ ] T020 [US1] Tests: upsert does not create duplicates; last_seen updated
|
||||
- [ ] T021 [US2] Tests: missing derived per latestRun(selection_hash); selection isolation
|
||||
- [ ] T022 [US2] Tests: partial/failed run => low confidence missing
|
||||
- [ ] T023 [US2] Tests: meta whitelist drops unknown keys without failing
|
||||
- [ ] T024 [NFR] Tests: selection_hash determinism (array ordering)
|
||||
- [X] T020 [US1] Pest: upsert prevents duplicates; `last_seen_at` and `last_seen_run_id` update
|
||||
- [X] T021 [US2] Pest: missing derived per latest completed run for same (tenant_id, selection_hash)
|
||||
- [X] T022 [US2] Pest: selection isolation (run for selection Y does not affect selection X)
|
||||
- [X] T023 [US2] Pest: partial/failed/had_errors => missing is low confidence
|
||||
- [X] T024 [US2] Pest: meta whitelist drops unknown keys (no exception; no persistence)
|
||||
- [X] T025 [NFR] Pest: selection_hash determinism (array ordering invariant)
|
||||
- [X] T026 [NFR] Pest: lock prevents overlapping runs for same (tenant_id, selection_hash)
|
||||
- [X] T027 [NFR] Pest: run error persistence contains no secrets/tokens (assert error context is bounded; no “Bearer ” / access token patterns; prefer error_codes)
|
||||
- [X] T028 [US1] Pest: inventory sync creates no rows in `policy_versions` and `backup_*` tables (assert counts unchanged)
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
296
tests/Feature/Inventory/InventorySyncServiceTest.php
Normal file
296
tests/Feature/Inventory/InventorySyncServiceTest.php
Normal file
@ -0,0 +1,296 @@
|
||||
<?php
|
||||
|
||||
use App\Models\BackupItem;
|
||||
use App\Models\BackupSchedule;
|
||||
use App\Models\BackupScheduleRun;
|
||||
use App\Models\BackupSet;
|
||||
use App\Models\PolicyVersion;
|
||||
use App\Models\Tenant;
|
||||
use App\Services\Graph\GraphClientInterface;
|
||||
use App\Services\Graph\GraphResponse;
|
||||
use App\Services\Inventory\InventoryMetaSanitizer;
|
||||
use App\Services\Inventory\InventoryMissingService;
|
||||
use App\Services\Inventory\InventorySyncService;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
function fakeGraphClient(array $policiesByType = [], array $failedTypes = [], ?Throwable $throwable = null): GraphClientInterface
|
||||
{
|
||||
return new class($policiesByType, $failedTypes, $throwable) implements GraphClientInterface
|
||||
{
|
||||
public function __construct(
|
||||
private readonly array $policiesByType,
|
||||
private readonly array $failedTypes,
|
||||
private readonly ?Throwable $throwable,
|
||||
) {}
|
||||
|
||||
public function listPolicies(string $policyType, array $options = []): GraphResponse
|
||||
{
|
||||
if ($this->throwable instanceof Throwable) {
|
||||
throw $this->throwable;
|
||||
}
|
||||
|
||||
if (in_array($policyType, $this->failedTypes, true)) {
|
||||
return new GraphResponse(false, [], 403, ['error' => ['code' => 'Forbidden', 'message' => 'forbidden']], [], []);
|
||||
}
|
||||
|
||||
return new GraphResponse(true, $this->policiesByType[$policyType] ?? []);
|
||||
}
|
||||
|
||||
public function getPolicy(string $policyType, string $policyId, array $options = []): GraphResponse
|
||||
{
|
||||
return new GraphResponse(true, []);
|
||||
}
|
||||
|
||||
public function getOrganization(array $options = []): GraphResponse
|
||||
{
|
||||
return new GraphResponse(true, []);
|
||||
}
|
||||
|
||||
public function applyPolicy(string $policyType, string $policyId, array $payload, array $options = []): GraphResponse
|
||||
{
|
||||
return new GraphResponse(true, []);
|
||||
}
|
||||
|
||||
public function getServicePrincipalPermissions(array $options = []): GraphResponse
|
||||
{
|
||||
return new GraphResponse(true, []);
|
||||
}
|
||||
|
||||
public function request(string $method, string $path, array $options = []): GraphResponse
|
||||
{
|
||||
return new GraphResponse(true, []);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
test('inventory sync upserts and updates last_seen fields without duplicates', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [
|
||||
['id' => 'cfg-1', 'displayName' => 'Config 1', '@odata.type' => '#microsoft.graph.deviceConfiguration'],
|
||||
],
|
||||
]));
|
||||
|
||||
$service = app(InventorySyncService::class);
|
||||
|
||||
$selection = [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
$runA = $service->syncNow($tenant, $selection);
|
||||
expect($runA->status)->toBe('success');
|
||||
|
||||
$item = \App\Models\InventoryItem::query()->where('tenant_id', $tenant->id)->first();
|
||||
expect($item)->not->toBeNull();
|
||||
expect($item->external_id)->toBe('cfg-1');
|
||||
expect($item->last_seen_run_id)->toBe($runA->id);
|
||||
|
||||
$runB = $service->syncNow($tenant, $selection);
|
||||
|
||||
$items = \App\Models\InventoryItem::query()->where('tenant_id', $tenant->id)->get();
|
||||
expect($items)->toHaveCount(1);
|
||||
|
||||
$items->first()->refresh();
|
||||
expect($items->first()->last_seen_run_id)->toBe($runB->id);
|
||||
});
|
||||
|
||||
test('meta whitelist drops unknown keys without failing', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
$sanitizer = app(InventoryMetaSanitizer::class);
|
||||
|
||||
$meta = $sanitizer->sanitize([
|
||||
'odata_type' => '#microsoft.graph.deviceConfiguration',
|
||||
'etag' => 'W/\"123\"',
|
||||
'scope_tag_ids' => ['0', 'tag-1'],
|
||||
'assignment_target_count' => '5',
|
||||
'warnings' => ['ok'],
|
||||
'unknown_key' => 'should_not_persist',
|
||||
]);
|
||||
|
||||
$item = \App\Models\InventoryItem::query()->create([
|
||||
'tenant_id' => $tenant->id,
|
||||
'policy_type' => 'deviceConfiguration',
|
||||
'external_id' => 'cfg-1',
|
||||
'display_name' => 'Config 1',
|
||||
'meta_jsonb' => $meta,
|
||||
'last_seen_at' => now(),
|
||||
'last_seen_run_id' => null,
|
||||
]);
|
||||
|
||||
$item->refresh();
|
||||
|
||||
$stored = is_array($item->meta_jsonb) ? $item->meta_jsonb : [];
|
||||
|
||||
expect($stored)->not->toHaveKey('unknown_key');
|
||||
expect($stored['assignment_target_count'] ?? null)->toBe(5);
|
||||
});
|
||||
|
||||
test('inventory missing is derived from latest completed run and low confidence on partial runs', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
$selection = [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [
|
||||
['id' => 'cfg-1', 'displayName' => 'Config 1', '@odata.type' => '#microsoft.graph.deviceConfiguration'],
|
||||
],
|
||||
]));
|
||||
|
||||
app(InventorySyncService::class)->syncNow($tenant, $selection);
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [],
|
||||
]));
|
||||
|
||||
app(InventorySyncService::class)->syncNow($tenant, $selection);
|
||||
|
||||
$missingService = app(InventoryMissingService::class);
|
||||
$result = $missingService->missingForSelection($tenant, $selection);
|
||||
|
||||
expect($result['missing'])->toHaveCount(1);
|
||||
expect($result['lowConfidence'])->toBeFalse();
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [],
|
||||
], failedTypes: ['deviceConfiguration']));
|
||||
|
||||
app(InventorySyncService::class)->syncNow($tenant, $selection);
|
||||
|
||||
$result2 = $missingService->missingForSelection($tenant, $selection);
|
||||
expect($result2['missing'])->toHaveCount(1);
|
||||
expect($result2['lowConfidence'])->toBeTrue();
|
||||
});
|
||||
|
||||
test('selection isolation: run for selection Y does not affect selection X missing', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
$selectionX = [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
$selectionY = [
|
||||
'policy_types' => ['deviceCompliancePolicy'],
|
||||
'categories' => ['Compliance'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [
|
||||
['id' => 'cfg-1', 'displayName' => 'Config 1', '@odata.type' => '#microsoft.graph.deviceConfiguration'],
|
||||
],
|
||||
'deviceCompliancePolicy' => [
|
||||
['id' => 'cmp-1', 'displayName' => 'Compliance 1', '@odata.type' => '#microsoft.graph.deviceCompliancePolicy'],
|
||||
],
|
||||
]));
|
||||
|
||||
$service = app(InventorySyncService::class);
|
||||
$service->syncNow($tenant, $selectionX);
|
||||
|
||||
$service->syncNow($tenant, $selectionY);
|
||||
|
||||
$missingService = app(InventoryMissingService::class);
|
||||
$resultX = $missingService->missingForSelection($tenant, $selectionX);
|
||||
|
||||
expect($resultX['missing'])->toHaveCount(0);
|
||||
});
|
||||
|
||||
test('lock prevents overlapping runs for same tenant and selection', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [],
|
||||
]));
|
||||
|
||||
$service = app(InventorySyncService::class);
|
||||
|
||||
$selection = [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
$hash = app(\App\Services\Inventory\InventorySelectionHasher::class)->hash($selection);
|
||||
$lock = Cache::lock("inventory_sync:tenant:{$tenant->id}:selection:{$hash}", 900);
|
||||
expect($lock->get())->toBeTrue();
|
||||
|
||||
$run = $service->syncNow($tenant, $selection);
|
||||
|
||||
expect($run->status)->toBe('skipped');
|
||||
expect($run->error_codes)->toContain('lock_contended');
|
||||
|
||||
$lock->release();
|
||||
});
|
||||
|
||||
test('inventory sync does not create snapshot or backup rows', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
$baseline = [
|
||||
'policy_versions' => PolicyVersion::query()->count(),
|
||||
'backup_sets' => BackupSet::query()->count(),
|
||||
'backup_items' => BackupItem::query()->count(),
|
||||
'backup_schedules' => BackupSchedule::query()->count(),
|
||||
'backup_schedule_runs' => BackupScheduleRun::query()->count(),
|
||||
];
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient([
|
||||
'deviceConfiguration' => [],
|
||||
]));
|
||||
|
||||
$service = app(InventorySyncService::class);
|
||||
|
||||
$service->syncNow($tenant, [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
]);
|
||||
|
||||
expect(PolicyVersion::query()->count())->toBe($baseline['policy_versions']);
|
||||
expect(BackupSet::query()->count())->toBe($baseline['backup_sets']);
|
||||
expect(BackupItem::query()->count())->toBe($baseline['backup_items']);
|
||||
expect(BackupSchedule::query()->count())->toBe($baseline['backup_schedules']);
|
||||
expect(BackupScheduleRun::query()->count())->toBe($baseline['backup_schedule_runs']);
|
||||
});
|
||||
|
||||
test('run error persistence is safe and does not include bearer tokens', function () {
|
||||
$tenant = Tenant::factory()->create();
|
||||
|
||||
$throwable = new RuntimeException('Graph failed: Bearer abc.def.ghi');
|
||||
|
||||
app()->instance(GraphClientInterface::class, fakeGraphClient(throwable: $throwable));
|
||||
|
||||
$service = app(InventorySyncService::class);
|
||||
|
||||
$run = $service->syncNow($tenant, [
|
||||
'policy_types' => ['deviceConfiguration'],
|
||||
'categories' => ['Configuration'],
|
||||
'include_foundations' => false,
|
||||
'include_dependencies' => false,
|
||||
]);
|
||||
|
||||
expect($run->status)->toBe('failed');
|
||||
|
||||
$context = is_array($run->error_context) ? $run->error_context : [];
|
||||
$message = (string) ($context['message'] ?? '');
|
||||
|
||||
expect($message)->not->toContain('abc.def.ghi');
|
||||
expect($message)->toContain('Bearer [REDACTED]');
|
||||
});
|
||||
23
tests/Unit/Inventory/InventorySelectionHasherTest.php
Normal file
23
tests/Unit/Inventory/InventorySelectionHasherTest.php
Normal file
@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
use App\Services\Inventory\InventorySelectionHasher;
|
||||
|
||||
it('computes the same selection_hash regardless of array ordering', function () {
|
||||
$hasher = app(InventorySelectionHasher::class);
|
||||
|
||||
$payloadA = [
|
||||
'policy_types' => ['deviceCompliancePolicy', 'deviceConfiguration'],
|
||||
'categories' => ['Compliance', 'Configuration'],
|
||||
'include_foundations' => true,
|
||||
'include_dependencies' => false,
|
||||
];
|
||||
|
||||
$payloadB = [
|
||||
'include_dependencies' => false,
|
||||
'include_foundations' => true,
|
||||
'categories' => ['Configuration', 'Compliance'],
|
||||
'policy_types' => ['deviceConfiguration', 'deviceCompliancePolicy'],
|
||||
];
|
||||
|
||||
expect($hasher->hash($payloadA))->toBe($hasher->hash($payloadB));
|
||||
});
|
||||
Loading…
Reference in New Issue
Block a user