fix(assessment): keep release origin unproven
Some checks failed
Dependency Audit / dependency-audit (push) Failing after 15s
Security Audit / security-audit (push) Failing after 14s

This commit is contained in:
2026-07-22 19:55:35 +02:00
parent 4c16069f88
commit ddee5c00dc
6 changed files with 689 additions and 121 deletions

View File

@@ -55,7 +55,7 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
"docs/capability-fit-proof-authority-keyring.schema.json"
)
def test_matching_installed_composition_proves_versions_records_and_commits(
def test_matching_installed_composition_proves_local_consistency_not_origin(
self,
) -> None:
catalog, keyring = signed_catalog(self.assessment)
@@ -71,6 +71,17 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
self.assertTrue(report["proof_scope"]["installed_artifacts"]["valid"])
self.assertTrue(report["proof_scope"]["installed_record_integrity"]["valid"])
self.assertTrue(report["proof_scope"]["installed_source_provenance"]["valid"])
self.assertEqual(
"local-pep610-metadata",
report["proof_scope"]["installed_source_provenance"]["basis"],
)
self.assertFalse(
report["proof_scope"]["installed_source_provenance"][
"release_origin_bound"
]
)
self.assertFalse(report["proof_scope"]["installed_release_origin"]["checked"])
self.assertIsNone(report["proof_scope"]["installed_release_origin"]["valid"])
self.assertFalse(report["proof_scope"]["runtime_activation"]["checked"])
self.assertFalse(report["proof_scope"]["target_environment"]["checked"])
self.assertEqual(
@@ -78,6 +89,40 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
report["proof_scope"]["installed_artifacts"]["evidence_sha256"],
)
def test_catalog_trust_blocker_invalidates_dependent_evidence_scopes(
self,
) -> None:
catalog, keyring = signed_catalog(self.assessment)
catalog["signatures"][0]["value"] = "AAAA"
installed = matching_installed_evidence(self.assessment)
proof, authority = signed_boundary_evidence(
assessment=self.assessment,
installed=installed,
claims=[boundary_claim("target_environment", "passed")],
allowed_scopes=["target_environment"],
)
report = self.review(
catalog=catalog,
keyring=keyring,
installed_evidence=installed,
boundary_evidence=proof,
authority_keyring=authority,
)
self.assertEqual("blocked", report["status"])
self.assertFalse(
report["proof_scope"]["catalog_signature_and_keyring"]["valid"]
)
self.assertFalse(report["proof_scope"]["installed_artifacts"]["checked"])
self.assertIsNone(report["proof_scope"]["installed_artifacts"]["valid"])
self.assertFalse(report["proof_scope"]["target_environment"]["checked"])
self.assertIsNone(report["proof_scope"]["target_environment"]["valid"])
self.assertIn(
"installed_catalog_trust_unavailable",
{item["code"] for item in report["findings"]},
)
def test_missing_extra_and_version_drift_have_deterministic_review_targets(
self,
) -> None:
@@ -137,7 +182,10 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
for item in evidence["artifacts"]
if item["package_name"] == "govoplan-campaign"
)
campaign["source_provenance"] = {"kind": "editable-local"}
campaign["source_provenance"] = {
"basis": "local-pep610-metadata",
"kind": "editable-local",
}
campaign["record_integrity"] = {
**campaign["record_integrity"],
"status": "partial",
@@ -212,6 +260,7 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
schema=self.installed_schema,
observation_mode="direct_local",
verification_time=datetime(2026, 7, 23, 12, tzinfo=UTC),
catalog_trusted=True,
)
codes = {item.code for item in result.findings}
@@ -317,6 +366,10 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
)
observed = evidence["artifacts"][0]
self.assertEqual("editable-local", observed["source_provenance"]["kind"])
self.assertEqual(
"local-pep610-metadata",
observed["source_provenance"]["basis"],
)
self.assertEqual("verified", observed["record_integrity"]["status"])
encoded = json.dumps(evidence, sort_keys=True)
self.assertNotIn(str(root), encoded)
@@ -338,6 +391,108 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
tampered["artifacts"][0]["record_integrity"]["mismatched_file_count"],
)
def test_collector_detects_payload_declared_in_record_but_missing(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
distribution, payload_file = create_distribution(root)
payload_file.unlink()
with mock.patch.object(sys, "path", [str(root), *sys.path]):
evidence = collect_installed_composition(
assessment=self.assessment,
collected_at=datetime(2026, 7, 22, 12, tzinfo=UTC),
distributions=[distribution],
)
integrity = evidence["artifacts"][0]["record_integrity"]
self.assertEqual("mismatch", integrity["status"])
self.assertEqual(1, integrity["missing_file_count"])
def test_collector_rejects_duplicate_record_declarations(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
distribution, _ = create_distribution(root)
record_path = root / "govoplan_demo-1.0.0.dist-info" / "RECORD"
rows = record_path.read_text(encoding="utf-8").splitlines()
record_path.write_text(
"\n".join((*rows, rows[0])) + "\n",
encoding="utf-8",
)
with mock.patch.object(sys, "path", [str(root), *sys.path]):
evidence = collect_installed_composition(
assessment=self.assessment,
collected_at=datetime(2026, 7, 22, 12, tzinfo=UTC),
distributions=[distribution],
)
integrity = evidence["artifacts"][0]["record_integrity"]
self.assertEqual("partial", integrity["status"])
self.assertGreater(integrity["unverifiable_file_count"], 0)
def test_collector_rejects_malformed_record_declarations(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
distribution, _ = create_distribution(root)
record_path = root / "govoplan_demo-1.0.0.dist-info" / "RECORD"
record_path.write_text(
record_path.read_text(encoding="utf-8") + "malformed,row\n",
encoding="utf-8",
)
with mock.patch.object(sys, "path", [str(root), *sys.path]):
evidence = collect_installed_composition(
assessment=self.assessment,
collected_at=datetime(2026, 7, 22, 12, tzinfo=UTC),
distributions=[distribution],
)
integrity = evidence["artifacts"][0]["record_integrity"]
self.assertEqual("partial", integrity["status"])
self.assertGreater(integrity["unverifiable_file_count"], 0)
def test_collector_verifies_declared_record_sizes(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
distribution, _ = create_distribution(root)
record_path = root / "govoplan_demo-1.0.0.dist-info" / "RECORD"
rows = record_path.read_text(encoding="utf-8").splitlines()
path, digest, size = rows[0].split(",")
rows[0] = f"{path},{digest},{int(size) + 1}"
record_path.write_text("\n".join(rows) + "\n", encoding="utf-8")
with mock.patch.object(sys, "path", [str(root), *sys.path]):
evidence = collect_installed_composition(
assessment=self.assessment,
collected_at=datetime(2026, 7, 22, 12, tzinfo=UTC),
distributions=[distribution],
)
integrity = evidence["artifacts"][0]["record_integrity"]
self.assertEqual("mismatch", integrity["status"])
self.assertEqual(1, integrity["mismatched_file_count"])
def test_collector_rejects_symlinked_distribution_metadata_root(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
create_distribution(root)
metadata_root = root / "govoplan_demo-1.0.0.dist-info"
real_metadata_root = root / "real-govoplan-demo.dist-info"
metadata_root.rename(real_metadata_root)
metadata_root.symlink_to(real_metadata_root, target_is_directory=True)
distribution = metadata.PathDistribution(metadata_root)
with mock.patch.object(sys, "path", [str(root), *sys.path]):
evidence = collect_installed_composition(
assessment=self.assessment,
collected_at=datetime(2026, 7, 22, 12, tzinfo=UTC),
distributions=[distribution],
)
integrity = evidence["artifacts"][0]["record_integrity"]
self.assertEqual("unavailable", integrity["status"])
self.assertEqual(0, integrity["hashed_file_count"])
def test_collector_accepts_real_core_wheel_payload_scripts_data_and_pyc(
self,
) -> None:
@@ -604,7 +759,9 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
json.dumps(reversed_order, sort_keys=True),
)
def test_signed_scope_authority_can_validate_only_its_external_claim(self) -> None:
def test_signed_boundary_claim_waits_for_anchored_installed_release_origin(
self,
) -> None:
catalog, keyring = signed_catalog(self.assessment)
installed = matching_installed_evidence(self.assessment)
proof, authority = signed_boundary_evidence(
@@ -622,10 +779,15 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
authority_keyring=authority,
)
self.assertEqual("current", report["status"])
self.assertTrue(report["proof_scope"]["target_environment"]["valid"])
self.assertEqual("blocked", report["status"])
self.assertFalse(report["proof_scope"]["target_environment"]["checked"])
self.assertIsNone(report["proof_scope"]["target_environment"]["valid"])
self.assertFalse(report["proof_scope"]["external_providers"]["checked"])
self.assertFalse(report["proof_scope"]["production_approval"]["checked"])
self.assertIn(
"boundary_installed_release_origin_untrusted",
{item["code"] for item in report["findings"]},
)
self.assertEqual(
self.assessment["deployment_profile"]["id"],
report["proof_scope"]["target_environment"]["expected_subject_id"],
@@ -666,10 +828,15 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
"boundary_provider_subject_unconfigured",
{item["code"] for item in unconfigured["findings"]},
)
self.assertEqual("current", accepted["status"])
self.assertEqual("blocked", accepted["status"])
provider_scope = accepted["proof_scope"]["external_providers"]
self.assertFalse(provider_scope["checked"])
self.assertEqual("provider:test", provider_scope["expected_subject_id"])
self.assertEqual("provider:test", provider_scope["observed_subject_id"])
self.assertIn(
"boundary_installed_release_origin_untrusted",
{item["code"] for item in accepted["findings"]},
)
def test_boundary_subject_mismatch_blocks_without_echoing_invalid_config(
self,
@@ -711,7 +878,7 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
self.assertNotIn("private.invalid", encoded)
self.assertNotIn("secret", encoded)
def test_authorized_negative_claim_creates_scope_specific_review_target(
def test_authorized_negative_claim_waits_for_installed_origin_binding(
self,
) -> None:
catalog, keyring = signed_catalog(self.assessment)
@@ -734,14 +901,12 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
authority_keyring=authority,
)
targets = {item["id"] for item in report["review_targets"]}
self.assertIn(
self.assertEqual("blocked", report["status"])
self.assertFalse(report["proof_scope"]["target_environment"]["checked"])
self.assertFalse(report["proof_scope"]["production_approval"]["checked"])
self.assertNotIn(
"boundary.target_environment.production-like-dev",
targets,
)
self.assertIn(
"boundary.production_approval.production-like-dev",
targets,
{item["id"] for item in report["review_targets"]},
)
def test_proof_authorities_must_be_independent_and_key_ids_unambiguous(
@@ -818,6 +983,55 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
{item["code"] for item in report["findings"]},
)
def test_malformed_authority_members_invalidate_the_whole_keyring(self) -> None:
catalog, keyring = signed_catalog(self.assessment)
installed = matching_installed_evidence(self.assessment)
proof, authority = signed_boundary_evidence(
assessment=self.assessment,
installed=installed,
claims=[boundary_claim("target_environment", "passed")],
allowed_scopes=["target_environment"],
)
malformed = deepcopy(authority)
malformed["keys"].append(
{
"key_id": "authority:malformed",
"status": "active",
"public_key": "not-base64",
"allowed_scopes": ["target_environment"],
}
)
inverted = deepcopy(authority)
inverted["keys"][0]["not_before"] = "2026-08-01T00:00:00Z"
inverted["keys"][0]["not_after"] = "2026-07-01T00:00:00Z"
cases = (
(malformed, "boundary_authority_key_invalid"),
(inverted, "boundary_authority_key_interval_invalid"),
)
for supplied_keyring, expected_code in cases:
with self.subTest(expected_code=expected_code):
report = self.review(
catalog=catalog,
keyring=keyring,
installed_evidence=installed,
boundary_evidence=proof,
authority_keyring=supplied_keyring,
)
self.assertEqual("blocked", report["status"])
self.assertIn(
expected_code,
{item["code"] for item in report["findings"]},
)
self.assertIn(
"boundary_authority_untrusted",
{item["code"] for item in report["findings"]},
)
self.assertFalse(
report["proof_scope"]["target_environment"]["checked"]
)
def test_historical_verification_and_generated_runtime_gap_are_visible(
self,
) -> None:
@@ -855,6 +1069,7 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
observation_mode="direct_local",
verification_time=datetime(2026, 7, 23, 12, tzinfo=UTC),
verification_mode="https://private.invalid/secret",
catalog_trusted=True,
)
encoded = json.dumps(result.proof_scope, sort_keys=True)
@@ -981,7 +1196,7 @@ class CapabilityFitEvidenceTests(unittest.TestCase):
def matching_installed_evidence(assessment: dict[str, object]) -> dict[str, object]:
return {
"$schema": "./installed-composition-evidence.schema.json",
"schema_version": "0.2.0",
"schema_version": "0.3.0",
"evidence_kind": "govoplan.installed-composition",
"assessment_id": assessment["assessment_id"],
"assessment_release": assessment["release"]["ref"],
@@ -1010,7 +1225,11 @@ def artifact(
"modules": []
if module_id == "core"
else [{"module_id": module_id, "manifest_version": version}],
"source_provenance": {"kind": "vcs-commit", "commit": commit},
"source_provenance": {
"basis": "local-pep610-metadata",
"kind": "vcs-commit",
"commit": commit,
},
"record_integrity": {
"status": "verified",
"hashed_file_count": 1,

View File

@@ -360,7 +360,8 @@ class CapabilityFitReviewTests(unittest.TestCase):
self.assertIn("Capability fit rerun: current", rendered)
self.assertIn(
"No installed-artifact, target-provider, or production proof", rendered
"No installed-composition, installed-release-origin, target-environment, external-provider, production-approval proof",
rendered,
)
def test_public_fetch_failure_is_generic_and_blocking(self) -> None: