4a4d3de8e1
Add cryptographically secure manifest filename nonce generation: - GenerateManifestNonce() creates 16-byte random nonce (32 hex chars) - GenerateManifestFilename() creates unique filenames: run_manifest_<nonce>.json - Prevents enumeration attacks on manifest files Add ExecutionEnvironment struct to manifest: - Captures ConfigHash for reproducibility verification - Records GPU detection method (auto-detected, env override, config, etc.) - Records sandbox settings (NoNewPrivileges, DropAllCaps, NetworkMode) - Records compliance mode and manifest nonce - Records artifact scan exclusions with reason Add JSON Schema validation: - schema.json: Canonical schema for manifest validation - schema_version.go: Schema versioning and compatibility checking - schema_test.go: Drift detection with SHA-256 hash verification - Validates required fields (run_id, environment.config_hash, etc.) - Validates compliance_mode enum values (hipaa, standard) - Validates no negative sizes in artifacts Closes: manifest nonce, environment tracking, scan exclusions from security plan
310 lines
6.4 KiB
JSON
310 lines
6.4 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"$id": "https://fetchml.io/schemas/manifest-v1.json",
|
|
"title": "FetchML Manifest Schema",
|
|
"description": "JSON Schema for validating FetchML manifest structures",
|
|
"version": "1.0.0",
|
|
"definitions": {
|
|
"annotation": {
|
|
"type": "object",
|
|
"properties": {
|
|
"timestamp": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"author": {
|
|
"type": "string"
|
|
},
|
|
"note": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["timestamp", "note"]
|
|
},
|
|
"narrative": {
|
|
"type": "object",
|
|
"properties": {
|
|
"hypothesis": {
|
|
"type": "string"
|
|
},
|
|
"context": {
|
|
"type": "string"
|
|
},
|
|
"intent": {
|
|
"type": "string"
|
|
},
|
|
"expected_outcome": {
|
|
"type": "string"
|
|
},
|
|
"parent_run": {
|
|
"type": "string"
|
|
},
|
|
"experiment_group": {
|
|
"type": "string"
|
|
},
|
|
"tags": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"outcome": {
|
|
"type": "object",
|
|
"properties": {
|
|
"status": {
|
|
"type": "string",
|
|
"enum": ["validated", "invalidated", "inconclusive", "partial"]
|
|
},
|
|
"summary": {
|
|
"type": "string"
|
|
},
|
|
"key_learnings": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"follow_up_runs": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"artifacts_used": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"artifactFile": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {
|
|
"type": "string"
|
|
},
|
|
"size_bytes": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"modified": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
}
|
|
},
|
|
"required": ["path", "size_bytes", "modified"]
|
|
},
|
|
"exclusion": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {
|
|
"type": "string"
|
|
},
|
|
"reason": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["path", "reason"]
|
|
},
|
|
"artifacts": {
|
|
"type": "object",
|
|
"properties": {
|
|
"discovery_time": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"files": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/artifactFile"
|
|
}
|
|
},
|
|
"total_size_bytes": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"exclusions": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/exclusion"
|
|
}
|
|
}
|
|
},
|
|
"required": ["discovery_time"]
|
|
},
|
|
"executionEnvironment": {
|
|
"type": "object",
|
|
"properties": {
|
|
"config_hash": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"gpu_count": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"gpu_detection_method": {
|
|
"type": "string",
|
|
"enum": ["nvml", "nvml_native", "env_override", "auto_detected", "none"]
|
|
},
|
|
"gpu_vendor": {
|
|
"type": "string"
|
|
},
|
|
"max_workers": {
|
|
"type": "integer",
|
|
"minimum": 1
|
|
},
|
|
"podman_cpus": {
|
|
"type": "string"
|
|
},
|
|
"sandbox_network_mode": {
|
|
"type": "string"
|
|
},
|
|
"sandbox_seccomp": {
|
|
"type": "string"
|
|
},
|
|
"sandbox_no_new_privs": {
|
|
"type": "boolean"
|
|
},
|
|
"compliance_mode": {
|
|
"type": "string",
|
|
"enum": ["hipaa", "standard"]
|
|
},
|
|
"manifest_nonce": {
|
|
"type": "string"
|
|
},
|
|
"metadata": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"required": ["config_hash", "gpu_count", "max_workers", "sandbox_network_mode", "sandbox_no_new_privs"]
|
|
}
|
|
},
|
|
"type": "object",
|
|
"properties": {
|
|
"run_id": {
|
|
"type": "string"
|
|
},
|
|
"task_id": {
|
|
"type": "string"
|
|
},
|
|
"job_name": {
|
|
"type": "string"
|
|
},
|
|
"created_at": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"started_at": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"ended_at": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"annotations": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/annotation"
|
|
}
|
|
},
|
|
"narrative": {
|
|
"$ref": "#/definitions/narrative"
|
|
},
|
|
"outcome": {
|
|
"$ref": "#/definitions/outcome"
|
|
},
|
|
"artifacts": {
|
|
"$ref": "#/definitions/artifacts"
|
|
},
|
|
"commit_id": {
|
|
"type": "string"
|
|
},
|
|
"experiment_manifest_sha": {
|
|
"type": "string"
|
|
},
|
|
"deps_manifest_name": {
|
|
"type": "string"
|
|
},
|
|
"deps_manifest_sha": {
|
|
"type": "string"
|
|
},
|
|
"train_script_path": {
|
|
"type": "string"
|
|
},
|
|
"worker_version": {
|
|
"type": "string"
|
|
},
|
|
"podman_image": {
|
|
"type": "string"
|
|
},
|
|
"image_digest": {
|
|
"type": "string"
|
|
},
|
|
"snapshot_id": {
|
|
"type": "string"
|
|
},
|
|
"snapshot_sha256": {
|
|
"type": "string"
|
|
},
|
|
"command": {
|
|
"type": "string"
|
|
},
|
|
"args": {
|
|
"type": "string"
|
|
},
|
|
"exit_code": {
|
|
"type": "integer"
|
|
},
|
|
"error": {
|
|
"type": "string"
|
|
},
|
|
"staging_duration_ms": {
|
|
"type": "integer"
|
|
},
|
|
"execution_duration_ms": {
|
|
"type": "integer"
|
|
},
|
|
"finalize_duration_ms": {
|
|
"type": "integer"
|
|
},
|
|
"total_duration_ms": {
|
|
"type": "integer"
|
|
},
|
|
"gpu_devices": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"worker_host": {
|
|
"type": "string"
|
|
},
|
|
"metadata": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"environment": {
|
|
"$ref": "#/definitions/executionEnvironment"
|
|
},
|
|
"signature": {
|
|
"type": "string"
|
|
},
|
|
"signer_key_id": {
|
|
"type": "string"
|
|
},
|
|
"sig_alg": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["run_id", "task_id", "job_name", "created_at"]
|
|
}
|