Expose object modification evidence
This commit is contained in:
@@ -28,6 +28,11 @@ module artifacts. It provides bounded read/write/list/stat/delete operations
|
|||||||
for local and S3-compatible storage. Modules own their object-key namespace and
|
for local and S3-compatible storage. Modules own their object-key namespace and
|
||||||
business metadata; Core does not interpret module files.
|
business metadata; Core does not interpret module files.
|
||||||
|
|
||||||
|
`stat` and `list_objects` return object size plus a UTC `modified_at` value when
|
||||||
|
the backend can prove it. Reconciliation and retention code may use that value
|
||||||
|
for conservative grace periods, but must treat a missing timestamp as
|
||||||
|
ineligible for automatic deletion rather than guessing an age.
|
||||||
|
|
||||||
Rules for modules:
|
Rules for modules:
|
||||||
|
|
||||||
- Store only opaque object keys in business records, never local absolute
|
- Store only opaque object keys in business records, never local absolute
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timezone
|
||||||
from heapq import nsmallest
|
from heapq import nsmallest
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -27,6 +28,7 @@ class StorageObjectMissing(StorageBackendError):
|
|||||||
class StorageObjectInfo:
|
class StorageObjectInfo:
|
||||||
key: str
|
key: str
|
||||||
size_bytes: int
|
size_bytes: int
|
||||||
|
modified_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -165,9 +167,14 @@ class LocalFilesystemStorageBackend:
|
|||||||
|
|
||||||
def stat(self, key: str) -> StorageObjectInfo:
|
def stat(self, key: str) -> StorageObjectInfo:
|
||||||
path = self._readable_path(key)
|
path = self._readable_path(key)
|
||||||
|
metadata = path.stat()
|
||||||
return StorageObjectInfo(
|
return StorageObjectInfo(
|
||||||
key=normalize_storage_key(key),
|
key=normalize_storage_key(key),
|
||||||
size_bytes=path.stat().st_size,
|
size_bytes=metadata.st_size,
|
||||||
|
modified_at=datetime.fromtimestamp(
|
||||||
|
metadata.st_mtime,
|
||||||
|
tz=timezone.utc,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def list_objects(
|
def list_objects(
|
||||||
@@ -188,9 +195,14 @@ class LocalFilesystemStorageBackend:
|
|||||||
normalized_after is not None and key <= normalized_after
|
normalized_after is not None and key <= normalized_after
|
||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
|
metadata = path.stat()
|
||||||
yield StorageObjectInfo(
|
yield StorageObjectInfo(
|
||||||
key=key,
|
key=key,
|
||||||
size_bytes=path.stat().st_size,
|
size_bytes=metadata.st_size,
|
||||||
|
modified_at=datetime.fromtimestamp(
|
||||||
|
metadata.st_mtime,
|
||||||
|
tz=timezone.utc,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
candidates = nsmallest(
|
candidates = nsmallest(
|
||||||
@@ -381,7 +393,11 @@ class S3StorageBackend:
|
|||||||
raise StorageBackendError(
|
raise StorageBackendError(
|
||||||
"S3 object metadata did not include a valid size"
|
"S3 object metadata did not include a valid size"
|
||||||
) from exc
|
) from exc
|
||||||
return StorageObjectInfo(key=normalized, size_bytes=size)
|
return StorageObjectInfo(
|
||||||
|
key=normalized,
|
||||||
|
size_bytes=size,
|
||||||
|
modified_at=_storage_modified_at(response.get("LastModified")),
|
||||||
|
)
|
||||||
|
|
||||||
def list_objects(
|
def list_objects(
|
||||||
self,
|
self,
|
||||||
@@ -407,6 +423,7 @@ class S3StorageBackend:
|
|||||||
StorageObjectInfo(
|
StorageObjectInfo(
|
||||||
key=str(item["Key"]),
|
key=str(item["Key"]),
|
||||||
size_bytes=int(item.get("Size") or 0),
|
size_bytes=int(item.get("Size") or 0),
|
||||||
|
modified_at=_storage_modified_at(item.get("LastModified")),
|
||||||
)
|
)
|
||||||
for item in response.get("Contents", ())
|
for item in response.get("Contents", ())
|
||||||
if isinstance(item, dict) and item.get("Key")
|
if isinstance(item, dict) and item.get("Key")
|
||||||
@@ -418,6 +435,14 @@ class S3StorageBackend:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _storage_modified_at(value: object) -> datetime | None:
|
||||||
|
if not isinstance(value, datetime):
|
||||||
|
return None
|
||||||
|
if value.tzinfo is None:
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def configured_storage_backend(settings: object) -> StorageBackend:
|
def configured_storage_backend(settings: object) -> StorageBackend:
|
||||||
"""Build the deployment-wide object store from Core settings.
|
"""Build the deployment-wide object store from Core settings.
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -26,10 +27,12 @@ class _S3Error(RuntimeError):
|
|||||||
class _FakeS3Client:
|
class _FakeS3Client:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.objects: dict[str, bytes] = {}
|
self.objects: dict[str, bytes] = {}
|
||||||
|
self.modified_at: dict[str, datetime] = {}
|
||||||
self.head_error: Exception | None = None
|
self.head_error: Exception | None = None
|
||||||
|
|
||||||
def put_object(self, *, Key: str, Body: bytes, **_kwargs) -> None:
|
def put_object(self, *, Key: str, Body: bytes, **_kwargs) -> None:
|
||||||
self.objects[Key] = Body
|
self.objects[Key] = Body
|
||||||
|
self.modified_at[Key] = datetime.now(timezone.utc)
|
||||||
|
|
||||||
def get_object(self, *, Key: str, **_kwargs):
|
def get_object(self, *, Key: str, **_kwargs):
|
||||||
try:
|
try:
|
||||||
@@ -45,10 +48,40 @@ class _FakeS3Client:
|
|||||||
payload = self.objects[Key]
|
payload = self.objects[Key]
|
||||||
except KeyError as exc:
|
except KeyError as exc:
|
||||||
raise _S3Error("NotFound", 404) from exc
|
raise _S3Error("NotFound", 404) from exc
|
||||||
return {"ContentLength": len(payload)}
|
return {
|
||||||
|
"ContentLength": len(payload),
|
||||||
|
"LastModified": self.modified_at[Key],
|
||||||
|
}
|
||||||
|
|
||||||
def delete_object(self, *, Key: str, **_kwargs) -> None:
|
def delete_object(self, *, Key: str, **_kwargs) -> None:
|
||||||
self.objects.pop(Key, None)
|
self.objects.pop(Key, None)
|
||||||
|
self.modified_at.pop(Key, None)
|
||||||
|
|
||||||
|
def list_objects_v2(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
Prefix: str,
|
||||||
|
MaxKeys: int,
|
||||||
|
StartAfter: str | None = None,
|
||||||
|
**_kwargs,
|
||||||
|
):
|
||||||
|
keys = [
|
||||||
|
key
|
||||||
|
for key in sorted(self.objects)
|
||||||
|
if key.startswith(Prefix) and (StartAfter is None or key > StartAfter)
|
||||||
|
]
|
||||||
|
selected = keys[:MaxKeys]
|
||||||
|
return {
|
||||||
|
"Contents": [
|
||||||
|
{
|
||||||
|
"Key": key,
|
||||||
|
"Size": len(self.objects[key]),
|
||||||
|
"LastModified": self.modified_at[key],
|
||||||
|
}
|
||||||
|
for key in selected
|
||||||
|
],
|
||||||
|
"IsTruncated": len(keys) > len(selected),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class ObjectStorageTests(unittest.TestCase):
|
class ObjectStorageTests(unittest.TestCase):
|
||||||
@@ -67,6 +100,7 @@ class ObjectStorageTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertEqual(b"a", backend.get_bytes("campaign/a.eml"))
|
self.assertEqual(b"a", backend.get_bytes("campaign/a.eml"))
|
||||||
self.assertEqual(1, backend.stat("campaign/a.eml").size_bytes)
|
self.assertEqual(1, backend.stat("campaign/a.eml").size_bytes)
|
||||||
|
self.assertIsNotNone(backend.stat("campaign/a.eml").modified_at)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
("campaign/a.eml",), tuple(item.key for item in first.objects)
|
("campaign/a.eml",), tuple(item.key for item in first.objects)
|
||||||
)
|
)
|
||||||
@@ -134,6 +168,10 @@ class ObjectStorageTests(unittest.TestCase):
|
|||||||
backend.put_bytes("campaign/message.eml", b"message/rfc822")
|
backend.put_bytes("campaign/message.eml", b"message/rfc822")
|
||||||
self.assertTrue(backend.exists("campaign/message.eml"))
|
self.assertTrue(backend.exists("campaign/message.eml"))
|
||||||
self.assertEqual(b"message/rfc822", backend.get_bytes("campaign/message.eml"))
|
self.assertEqual(b"message/rfc822", backend.get_bytes("campaign/message.eml"))
|
||||||
|
self.assertIsNotNone(backend.stat("campaign/message.eml").modified_at)
|
||||||
|
self.assertIsNotNone(
|
||||||
|
backend.list_objects(prefix="campaign/").objects[0].modified_at
|
||||||
|
)
|
||||||
self.assertFalse(backend.exists("campaign/missing.eml"))
|
self.assertFalse(backend.exists("campaign/missing.eml"))
|
||||||
|
|
||||||
client.head_error = _S3Error("AccessDenied", 403)
|
client.head_error = _S3Error("AccessDenied", 403)
|
||||||
|
|||||||
Reference in New Issue
Block a user