469 lines
14 KiB
Python
469 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import (
|
|
Boolean,
|
|
DateTime,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
JSON,
|
|
LargeBinary,
|
|
String,
|
|
Text,
|
|
UniqueConstraint,
|
|
)
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from govoplan_core.db.base import Base, TimestampMixin
|
|
|
|
|
|
def new_uuid() -> str:
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
class ConnectorTabularSource(Base, TimestampMixin):
|
|
__tablename__ = "connector_tabular_sources"
|
|
__table_args__ = (
|
|
UniqueConstraint("tenant_id", "source_name", name="uq_connector_tabular_source_name"),
|
|
Index("ix_connector_tabular_sources_tenant_status", "tenant_id", "status"),
|
|
Index("ix_connector_tabular_sources_tenant_updated", "tenant_id", "updated_at"),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
|
provider: Mapped[str] = mapped_column(String(50), default="snapshot", nullable=False, index=True)
|
|
source_name: Mapped[str] = mapped_column(String(120), nullable=False)
|
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
|
description: Mapped[str | None] = mapped_column(Text)
|
|
status: Mapped[str] = mapped_column(String(30), default="active", nullable=False, index=True)
|
|
schema_version: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
|
schema_: Mapped[list[dict[str, Any]]] = mapped_column("schema", JSON, default=list, nullable=False)
|
|
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
|
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
|
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
|
|
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
|
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
|
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
|
|
|
|
|
class ConnectorSanctionsAcquisitionRun(Base, TimestampMixin):
|
|
__tablename__ = "connector_sanctions_acquisition_runs"
|
|
__table_args__ = (
|
|
Index(
|
|
"ix_connector_sanctions_run_health",
|
|
"tenant_id",
|
|
"provider_id",
|
|
"status",
|
|
"started_at",
|
|
),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36),
|
|
primary_key=True,
|
|
default=new_uuid,
|
|
)
|
|
tenant_id: Mapped[str] = mapped_column(
|
|
String(36),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
provider_id: Mapped[str] = mapped_column(
|
|
String(100),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
source_id: Mapped[str] = mapped_column(
|
|
String(200),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
status: Mapped[str] = mapped_column(
|
|
String(40),
|
|
default="running",
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
attempt_count: Mapped[int] = mapped_column(
|
|
Integer,
|
|
default=0,
|
|
nullable=False,
|
|
)
|
|
request_evidence: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
response_evidence: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
started_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
finished_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True),
|
|
nullable=True,
|
|
)
|
|
snapshot_id: Mapped[str | None] = mapped_column(
|
|
String(36),
|
|
nullable=True,
|
|
index=True,
|
|
)
|
|
error: Mapped[str | None] = mapped_column(
|
|
Text,
|
|
nullable=True,
|
|
)
|
|
created_by: Mapped[str | None] = mapped_column(
|
|
String(255),
|
|
nullable=True,
|
|
index=True,
|
|
)
|
|
|
|
|
|
class ConnectorSanctionsSnapshot(Base, TimestampMixin):
|
|
__tablename__ = "connector_sanctions_snapshots"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"connector_run_id",
|
|
name="uq_connector_sanctions_snapshot_run",
|
|
),
|
|
Index(
|
|
"ix_connector_sanctions_snapshot_source",
|
|
"tenant_id",
|
|
"provider_id",
|
|
"acquired_at",
|
|
),
|
|
Index(
|
|
"ix_connector_sanctions_snapshot_version",
|
|
"provider_id",
|
|
"source_id",
|
|
"source_version",
|
|
),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36),
|
|
primary_key=True,
|
|
default=new_uuid,
|
|
)
|
|
tenant_id: Mapped[str] = mapped_column(
|
|
String(36),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
provider_id: Mapped[str] = mapped_column(
|
|
String(100),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
publisher: Mapped[str] = mapped_column(
|
|
String(300),
|
|
nullable=False,
|
|
)
|
|
jurisdiction: Mapped[str] = mapped_column(
|
|
String(100),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
list_type: Mapped[str] = mapped_column(
|
|
String(100),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
source_id: Mapped[str] = mapped_column(
|
|
String(200),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
source_version: Mapped[str] = mapped_column(
|
|
String(255),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
publication_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True),
|
|
nullable=True,
|
|
)
|
|
effective_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True),
|
|
nullable=True,
|
|
)
|
|
acquired_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
source_url: Mapped[str | None] = mapped_column(
|
|
String(1500),
|
|
nullable=True,
|
|
)
|
|
content_type: Mapped[str] = mapped_column(
|
|
String(200),
|
|
nullable=False,
|
|
)
|
|
byte_count: Mapped[int] = mapped_column(
|
|
Integer,
|
|
nullable=False,
|
|
)
|
|
sha256: Mapped[str] = mapped_column(
|
|
String(64),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
signature_evidence: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
parser_version: Mapped[str] = mapped_column(
|
|
String(100),
|
|
nullable=False,
|
|
)
|
|
licence_notes: Mapped[str | None] = mapped_column(
|
|
Text,
|
|
nullable=True,
|
|
)
|
|
trust_notes: Mapped[str | None] = mapped_column(
|
|
Text,
|
|
nullable=True,
|
|
)
|
|
connector_run_id: Mapped[str] = mapped_column(
|
|
ForeignKey(
|
|
"connector_sanctions_acquisition_runs.id",
|
|
ondelete="RESTRICT",
|
|
),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
transport_evidence: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
raw_content: Mapped[bytes] = mapped_column(
|
|
LargeBinary,
|
|
nullable=False,
|
|
)
|
|
|
|
|
|
class ConnectorDefinition(Base, TimestampMixin):
|
|
__tablename__ = "connector_definitions"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"tenant_id",
|
|
"definition_key",
|
|
name="uq_connector_definition_tenant_key",
|
|
),
|
|
Index(
|
|
"ix_connector_definitions_tenant_status",
|
|
"tenant_id",
|
|
"status",
|
|
),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
|
definition_key: Mapped[str] = mapped_column(String(160), nullable=False)
|
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
|
description: Mapped[str | None] = mapped_column(Text)
|
|
status: Mapped[str] = mapped_column(
|
|
String(30),
|
|
default="active",
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
current_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
source_package: Mapped[str | None] = mapped_column(String(300))
|
|
local_definition: Mapped[bool] = mapped_column(
|
|
Boolean,
|
|
default=False,
|
|
nullable=False,
|
|
)
|
|
|
|
|
|
class ConnectorDefinitionRevision(Base, TimestampMixin):
|
|
__tablename__ = "connector_definition_revisions"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"definition_id",
|
|
"revision",
|
|
name="uq_connector_definition_revision",
|
|
),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
|
definition_id: Mapped[str] = mapped_column(
|
|
ForeignKey("connector_definitions.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
specification: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
definition_hash: Mapped[str] = mapped_column(
|
|
String(64),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
origin: Mapped[str] = mapped_column(String(30), nullable=False)
|
|
package_ref: Mapped[str | None] = mapped_column(String(300))
|
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
|
|
|
|
|
class ConnectorConfiguration(Base, TimestampMixin):
|
|
__tablename__ = "connector_configurations"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"tenant_id",
|
|
"name",
|
|
name="uq_connector_configuration_tenant_name",
|
|
),
|
|
Index(
|
|
"ix_connector_configurations_tenant_status",
|
|
"tenant_id",
|
|
"status",
|
|
),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
|
definition_id: Mapped[str] = mapped_column(
|
|
ForeignKey("connector_definitions.id", ondelete="RESTRICT"),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
|
status: Mapped[str] = mapped_column(
|
|
String(30),
|
|
default="draft",
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
endpoint_url: Mapped[str | None] = mapped_column(String(1500))
|
|
credential_ref: Mapped[str | None] = mapped_column(String(500))
|
|
base_definition_revision: Mapped[int] = mapped_column(
|
|
Integer,
|
|
nullable=False,
|
|
)
|
|
local_overrides: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
protected_paths: Mapped[list[str]] = mapped_column(
|
|
JSON,
|
|
default=list,
|
|
nullable=False,
|
|
)
|
|
effective_configuration: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
effective_hash: Mapped[str] = mapped_column(
|
|
String(64),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
resource_revision: Mapped[int] = mapped_column(
|
|
Integer,
|
|
default=1,
|
|
nullable=False,
|
|
)
|
|
ambiguity_policy: Mapped[str] = mapped_column(
|
|
String(30),
|
|
default="manual_review",
|
|
nullable=False,
|
|
)
|
|
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
|
|
|
|
|
class ConnectorSimulationRun(Base, TimestampMixin):
|
|
__tablename__ = "connector_simulation_runs"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"tenant_id",
|
|
"configuration_id",
|
|
"mode",
|
|
"idempotency_key",
|
|
name="uq_connector_simulation_run_idempotency",
|
|
),
|
|
Index(
|
|
"ix_connector_simulation_runs_review",
|
|
"tenant_id",
|
|
"review_state",
|
|
"created_at",
|
|
),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
|
configuration_id: Mapped[str] = mapped_column(
|
|
ForeignKey("connector_configurations.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
mode: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
|
review_state: Mapped[str] = mapped_column(
|
|
String(30),
|
|
default="not_required",
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
definition_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
configuration_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
configuration_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
input_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
summary: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
effects: Mapped[list[dict[str, Any]]] = mapped_column(
|
|
JSON,
|
|
default=list,
|
|
nullable=False,
|
|
)
|
|
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
|
JSON,
|
|
default=list,
|
|
nullable=False,
|
|
)
|
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
|
JSON,
|
|
default=dict,
|
|
nullable=False,
|
|
)
|
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
|
reviewed_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
|
reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
review_reason: Mapped[str | None] = mapped_column(Text)
|
|
|
|
|
|
__all__ = [
|
|
"ConnectorConfiguration",
|
|
"ConnectorDefinition",
|
|
"ConnectorDefinitionRevision",
|
|
"ConnectorSanctionsAcquisitionRun",
|
|
"ConnectorSanctionsSnapshot",
|
|
"ConnectorSimulationRun",
|
|
"ConnectorTabularSource",
|
|
"new_uuid",
|
|
]
|