Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6ab8291ec | ||
|
|
de8e74b866 | ||
|
|
0f61fe4607 | ||
|
|
3964a2e4e7 | ||
|
|
b88f1b2c2d | ||
|
|
32cac8835b | ||
|
|
c69d68f1da | ||
|
|
1caae6e49e | ||
|
|
84c9bb7711 | ||
|
|
20146ef8fe | ||
|
|
c33380b957 | ||
|
|
dfa717b9ba | ||
|
|
26f8898d11 | ||
|
|
7be93785a2 | ||
|
|
52fe33568c | ||
|
|
c5a43b3dae | ||
|
|
27302f0c39 | ||
|
|
ba5ccea5b0 | ||
|
|
dd45d9bd36 | ||
|
|
853d12151f | ||
|
|
d2ba4ce4d8 |
@@ -0,0 +1,270 @@
|
|||||||
|
name: Module Package Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
release_tag:
|
||||||
|
description: Existing protected version tag to publish
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-packages:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Select and validate protected release tag
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
REQUESTED_TAG: ${{ inputs.release_tag }}
|
||||||
|
TRIGGER_TAG: ${{ gitea.ref_name }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
|
||||||
|
case "$tag" in
|
||||||
|
v[0-9]*.[0-9]*.[0-9]*) ;;
|
||||||
|
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
|
||||||
|
tag_commit="$(git rev-list -n 1 "$tag")"
|
||||||
|
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
|
||||||
|
echo "Release tag is not contained in main" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
git checkout --detach "$tag"
|
||||||
|
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
|
||||||
|
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
|
||||||
|
- name: Validate package versions
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
tag = os.environ["RELEASE_TAG"]
|
||||||
|
expected = tag.removeprefix("v")
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
if project.get("version") != expected:
|
||||||
|
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
|
||||||
|
raise SystemExit("Python distribution name must use the govoplan-* namespace")
|
||||||
|
webui = Path("webui/package.json")
|
||||||
|
if webui.is_file():
|
||||||
|
package = json.loads(webui.read_text(encoding="utf-8"))
|
||||||
|
if package.get("version") != expected:
|
||||||
|
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
|
||||||
|
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
|
||||||
|
release = Path("webui/package.release.json")
|
||||||
|
if release.is_file():
|
||||||
|
release_package = json.loads(release.read_text(encoding="utf-8"))
|
||||||
|
if (
|
||||||
|
release_package.get("name") != package.get("name")
|
||||||
|
or release_package.get("version") != expected
|
||||||
|
):
|
||||||
|
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
|
||||||
|
PY
|
||||||
|
- name: Build immutable package artifacts
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||||
|
rm -rf dist .package-webui
|
||||||
|
python -m build --wheel --outdir dist
|
||||||
|
python -m twine check dist/*.whl
|
||||||
|
if [[ -f webui/package.json ]]; then
|
||||||
|
mkdir .package-webui
|
||||||
|
cp -a webui/. .package-webui/
|
||||||
|
rm -rf .package-webui/node_modules .package-webui/dist
|
||||||
|
if [[ -f .package-webui/package.release.json ]]; then
|
||||||
|
cp .package-webui/package.release.json .package-webui/package.json
|
||||||
|
fi
|
||||||
|
node <<'NODE'
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = ".package-webui/package.json";
|
||||||
|
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||||
|
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
|
||||||
|
for (const group of groups) {
|
||||||
|
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
|
||||||
|
if (!name.startsWith("@govoplan/")) continue;
|
||||||
|
if (typeof specifier !== "string") {
|
||||||
|
throw new Error(`${group}.${name} must use a string version`);
|
||||||
|
}
|
||||||
|
const packageSlug = name.slice("@govoplan/".length);
|
||||||
|
if (!packageSlug.endsWith("-webui")) {
|
||||||
|
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
|
||||||
|
}
|
||||||
|
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
|
||||||
|
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
const gitTag = specifier.match(
|
||||||
|
new RegExp(
|
||||||
|
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (gitTag) {
|
||||||
|
packageJson[group][name] = gitTag[1];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
|
||||||
|
throw new Error(
|
||||||
|
`${group}.${name} must resolve to an exact registry version for publication`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete packageJson.private;
|
||||||
|
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
|
||||||
|
NODE
|
||||||
|
npm pkg delete private --prefix .package-webui
|
||||||
|
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
|
||||||
|
fi
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
artifacts = []
|
||||||
|
for path in sorted(Path("dist").iterdir()):
|
||||||
|
if path.suffix not in {".whl", ".tgz"}:
|
||||||
|
continue
|
||||||
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
|
||||||
|
payload = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"repository": os.environ["GITEA_REPOSITORY"],
|
||||||
|
"tag": os.environ["RELEASE_TAG"],
|
||||||
|
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
Path("dist/package-artifacts.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
- name: Retain package hash evidence
|
||||||
|
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
|
||||||
|
with:
|
||||||
|
name: module-packages-${{ gitea.ref_name }}
|
||||||
|
path: dist/package-artifacts.json
|
||||||
|
- name: Check immutable registry state
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tomllib
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
from urllib.parse import quote
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
|
||||||
|
token = os.environ["PACKAGE_TOKEN"]
|
||||||
|
|
||||||
|
def should_publish(kind, name, version, path):
|
||||||
|
package_url = "/".join(
|
||||||
|
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
|
||||||
|
)
|
||||||
|
request = Request(
|
||||||
|
package_url,
|
||||||
|
headers={"Accept": "application/json", "Authorization": f"token {token}"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urlopen(request, timeout=30) as response:
|
||||||
|
files = json.load(response)
|
||||||
|
except HTTPError as exc:
|
||||||
|
if exc.code == 404:
|
||||||
|
print(f"{kind} package {name}=={version} is not published yet")
|
||||||
|
return True
|
||||||
|
raise
|
||||||
|
if not isinstance(files, list) or len(files) != 1:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} has an unexpected file set"
|
||||||
|
)
|
||||||
|
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
if files[0].get("sha256") != expected_sha256:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
|
||||||
|
)
|
||||||
|
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
|
||||||
|
return False
|
||||||
|
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
wheels = tuple(Path("dist").glob("*.whl"))
|
||||||
|
if len(wheels) != 1:
|
||||||
|
raise SystemExit("release build must contain exactly one wheel")
|
||||||
|
publish_pypi = should_publish(
|
||||||
|
"pypi", str(project["name"]), str(project["version"]), wheels[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
tarballs = tuple(Path("dist").glob("*.tgz"))
|
||||||
|
if len(tarballs) > 1:
|
||||||
|
raise SystemExit("release build must contain at most one npm package")
|
||||||
|
publish_npm = False
|
||||||
|
if tarballs:
|
||||||
|
webui = json.loads(
|
||||||
|
Path(".package-webui/package.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
publish_npm = should_publish(
|
||||||
|
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
|
||||||
|
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
|
||||||
|
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
|
||||||
|
PY
|
||||||
|
- name: Publish wheel and WebUI package
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_USERNAME"
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
if [[ "$PUBLISH_PYPI" == 1 ]]; then
|
||||||
|
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||||
|
python -m twine upload --non-interactive \
|
||||||
|
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||||
|
dist/*.whl
|
||||||
|
else
|
||||||
|
echo "Exact wheel is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
|
shopt -s nullglob
|
||||||
|
webui_packages=(dist/*.tgz)
|
||||||
|
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
|
||||||
|
npmrc="$(mktemp)"
|
||||||
|
trap 'rm -f "$npmrc"' EXIT
|
||||||
|
chmod 600 "$npmrc"
|
||||||
|
printf '%s\n' \
|
||||||
|
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
|
||||||
|
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
|
||||||
|
> "$npmrc"
|
||||||
|
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
|
||||||
|
--ignore-scripts --access public \
|
||||||
|
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
|
||||||
|
elif (( ${#webui_packages[@]} )); then
|
||||||
|
echo "Exact WebUI package is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
+276
@@ -0,0 +1,276 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.egg-info/
|
||||||
|
.pytest_cache/
|
||||||
|
.mypy_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.venv/
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
node_modules/
|
||||||
|
webui/node_modules/
|
||||||
|
webui/dist/
|
||||||
|
*.tsbuildinfo
|
||||||
|
.component-test-build/
|
||||||
|
.module-test-build/
|
||||||
|
.policy-test-build/
|
||||||
|
.template-preview-test-build/
|
||||||
|
.import-test-build/
|
||||||
|
webui/.component-test-build/
|
||||||
|
webui/.module-test-build/
|
||||||
|
webui/.policy-test-build/
|
||||||
|
webui/.template-preview-test-build/
|
||||||
|
webui/.import-test-build/
|
||||||
|
|
||||||
|
# GovOPlaN shared ignore rules from govoplan-core
|
||||||
|
# ---> Node
|
||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
.pnpm-debug.log*
|
||||||
|
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||||
|
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||||
|
# Runtime data
|
||||||
|
pids
|
||||||
|
*.pid
|
||||||
|
*.seed
|
||||||
|
*.pid.lock
|
||||||
|
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||||
|
lib-cov
|
||||||
|
# Coverage directory used by tools like istanbul
|
||||||
|
coverage
|
||||||
|
*.lcov
|
||||||
|
# nyc test coverage
|
||||||
|
.nyc_output
|
||||||
|
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||||
|
.grunt
|
||||||
|
# Bower dependency directory (https://bower.io/)
|
||||||
|
bower_components
|
||||||
|
# node-waf configuration
|
||||||
|
.lock-wscript
|
||||||
|
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||||
|
build/Release
|
||||||
|
# Dependency directories
|
||||||
|
jspm_packages/
|
||||||
|
# Snowpack dependency directory (https://snowpack.dev/)
|
||||||
|
web_modules/
|
||||||
|
# TypeScript cache
|
||||||
|
# Optional npm cache directory
|
||||||
|
.npm
|
||||||
|
# Optional eslint cache
|
||||||
|
.eslintcache
|
||||||
|
# Optional stylelint cache
|
||||||
|
.stylelintcache
|
||||||
|
# Microbundle cache
|
||||||
|
.rpt2_cache/
|
||||||
|
.rts2_cache_cjs/
|
||||||
|
.rts2_cache_es/
|
||||||
|
.rts2_cache_umd/
|
||||||
|
# Optional REPL history
|
||||||
|
.node_repl_history
|
||||||
|
# Output of 'npm pack'
|
||||||
|
*.tgz
|
||||||
|
# Yarn Integrity file
|
||||||
|
.yarn-integrity
|
||||||
|
# dotenv environment variable files
|
||||||
|
.env
|
||||||
|
.env.development.local
|
||||||
|
.env.test.local
|
||||||
|
.env.production.local
|
||||||
|
.env.local
|
||||||
|
# parcel-bundler cache (https://parceljs.org/)
|
||||||
|
.cache
|
||||||
|
.parcel-cache
|
||||||
|
# Next.js build output
|
||||||
|
.next
|
||||||
|
out
|
||||||
|
# Nuxt.js build / generate output
|
||||||
|
.nuxt
|
||||||
|
dist
|
||||||
|
# Gatsby files
|
||||||
|
.cache/
|
||||||
|
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||||
|
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||||
|
# public
|
||||||
|
# vuepress build output
|
||||||
|
.vuepress/dist
|
||||||
|
# vuepress v2.x temp and cache directory
|
||||||
|
.temp
|
||||||
|
.cache
|
||||||
|
# vitepress build output
|
||||||
|
**/.vitepress/dist
|
||||||
|
# vitepress cache directory
|
||||||
|
**/.vitepress/cache
|
||||||
|
# Docusaurus cache and generated files
|
||||||
|
.docusaurus
|
||||||
|
# Serverless directories
|
||||||
|
.serverless/
|
||||||
|
# FuseBox cache
|
||||||
|
.fusebox/
|
||||||
|
# DynamoDB Local files
|
||||||
|
.dynamodb/
|
||||||
|
# TernJS port file
|
||||||
|
.tern-port
|
||||||
|
# Stores VSCode versions used for testing VSCode extensions
|
||||||
|
.vscode-test
|
||||||
|
# yarn v2
|
||||||
|
.yarn/cache
|
||||||
|
.yarn/unplugged
|
||||||
|
.yarn/build-state.yml
|
||||||
|
.yarn/install-state.gz
|
||||||
|
.pnp.*
|
||||||
|
# Local WebUI test/build scratch directories
|
||||||
|
# ---> Python
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
*$py.class
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
develop-eggs/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
cover/
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
#uv.lock
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
#poetry.lock
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
#pdm.lock
|
||||||
|
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||||
|
# in version control.
|
||||||
|
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||||
|
.pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
# mypy
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
#.idea/
|
||||||
|
# Ruff stuff:
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
# ---> VisualStudioCode
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
!.vscode/*.code-snippets
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
# Built Visual Studio Code Extensions
|
||||||
|
*.vsix
|
||||||
|
*.db
|
||||||
|
# GovOPlaN local runtime state
|
||||||
|
runtime/
|
||||||
|
# GovOPlaN WebUI test output
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# GovOPlaN Connectors Codex Guide
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This repository owns reusable external connection profiles, protocol adapters, governed snapshots, and connector capability contracts.
|
||||||
|
|
||||||
|
## Documentation Contract
|
||||||
|
|
||||||
|
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
|
||||||
|
- Keep feature content here; `govoplan-docs` projects it without importing Connectors internals.
|
||||||
|
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
- Domain modules own business semantics; connectors own transport, credentials, synchronization, and diagnostics.
|
||||||
|
- Keep optional adapters behind capabilities and enforce egress and peer-validation policy.
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
# govoplan-connectors
|
||||||
|
|
||||||
|
<!-- govoplan-repository-type:start -->
|
||||||
|
**Repository type:** connector (connector-hub).
|
||||||
|
<!-- govoplan-repository-type:end -->
|
||||||
|
|
||||||
|
`govoplan-connectors` owns integration catalogues and generic external system
|
||||||
|
connection patterns for GovOPlaN.
|
||||||
|
|
||||||
|
The module should make external systems discoverable, testable, and usable
|
||||||
|
without taking ownership of their business semantics. Domain-specific modules
|
||||||
|
remain responsible for case, file, workflow, payment, mail, identity, document,
|
||||||
|
or reporting behavior.
|
||||||
|
|
||||||
|
## Executable First Slice
|
||||||
|
|
||||||
|
The first executable connector capability provides tenant-isolated tabular
|
||||||
|
origins. Operators can import bounded JSON or CSV snapshots, inspect inferred
|
||||||
|
schemas, and expose immutable source references and content fingerprints
|
||||||
|
through `connectors.datasource_origins@0.1.0`. Preview reads enforce provider
|
||||||
|
ceilings for rows, serialized bytes, and elapsed time and report the effective
|
||||||
|
limits and any truncation as structured diagnostics.
|
||||||
|
|
||||||
|
Connectors owns acquisition, connection profiles, credentials, discovery, and
|
||||||
|
provider health. `govoplan-datasources` registers an origin as a governed live
|
||||||
|
or cached datasource and owns staging, materializations, frozen states, and
|
||||||
|
consumer access. Dataflow consumes that Datasources contract and never imports
|
||||||
|
connector implementations or stores connector credentials.
|
||||||
|
|
||||||
|
Each origin declares whether it is live, cached, file-backed, or static, its
|
||||||
|
structured health state, and which projection, filter, aggregation, sorting,
|
||||||
|
and pagination operations it can push down. The immutable snapshot provider
|
||||||
|
currently supports projection and pagination only; consumers must keep other
|
||||||
|
operations in Dataflow rather than assuming transport-side execution.
|
||||||
|
|
||||||
|
Database, REST/HTTP, directory, managed-file, and warehouse providers can
|
||||||
|
implement the same origin contract without changing Datasources or Dataflow.
|
||||||
|
|
||||||
|
Governed sanctions and feed snapshot acquisitions use Core recovery operations.
|
||||||
|
The source revision/cursor, redacted dry-run decision, canonical request digest,
|
||||||
|
and distributed lease are durable before network I/O. Immutable snapshot rows
|
||||||
|
and the terminal recovery checkpoint commit atomically, and an
|
||||||
|
`Idempotency-Key` replays the committed result without contacting the provider.
|
||||||
|
Current acquisition transports are read-only. The exported external-mutation
|
||||||
|
recovery contract requires stable idempotency, provider verification, and
|
||||||
|
operator reconciliation, but no connector currently claims a production write
|
||||||
|
or delete path.
|
||||||
|
|
||||||
|
RSS and Atom emission is a bounded renderer, not an authority shortcut. Every
|
||||||
|
selected entry declares whether it came from a GovOPlaN event, publication,
|
||||||
|
case, or report and carries an opaque owning-module reference and optional
|
||||||
|
revision. Public-feed permission can render only public entries. Tenant and
|
||||||
|
private audiences require a separate restricted-feed permission, and the API
|
||||||
|
derives the allowed visibility set from that audience instead of accepting a
|
||||||
|
caller-controlled allow-list. Portal or Reporting remains responsible for any
|
||||||
|
durable public or authenticated route and must re-authorize restricted access.
|
||||||
|
|
||||||
|
The governed connector runtime adds immutable definition revisions,
|
||||||
|
revision-pinned tenant configurations, protected local overrides, explicit
|
||||||
|
package-update adoption, bounded dry-runs and simulations, redacted provenance,
|
||||||
|
idempotency, and configurable ambiguity handling through review, quarantine,
|
||||||
|
or rejection. Its administration surface is contributed to the shared system
|
||||||
|
administration workspace. Provider-specific adapters still own live writes.
|
||||||
|
|
||||||
|
Development:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/mnt/DATA/git/govoplan/.venv/bin/python -m pip install -e .
|
||||||
|
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest discover -s tests
|
||||||
|
```
|
||||||
|
|
||||||
|
See:
|
||||||
|
|
||||||
|
- [Connector concept](docs/CONCEPT.md)
|
||||||
|
- [Public-sector integration catalogue](docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md)
|
||||||
|
- [Connector source lifecycle](docs/CONNECTOR_SOURCE_LIFECYCLE.md)
|
||||||
|
- [OpenProject connector concept](docs/OPENPROJECT_CONNECTOR.md)
|
||||||
|
- [OpenDesk integration map](docs/OPENDESK_INTEGRATION_MAP.md)
|
||||||
|
- [Governed connector configuration](docs/GOVERNED_CONNECTOR_CONFIGURATION.md)
|
||||||
+197
@@ -0,0 +1,197 @@
|
|||||||
|
# govoplan-connectors Concept
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`govoplan-connectors` is the integration catalogue and connector coordination
|
||||||
|
module. It helps GovOPlaN connect to existing public-sector and organizational
|
||||||
|
systems without pretending to replace every specialist platform.
|
||||||
|
|
||||||
|
The module owns connector metadata, connection profiles, health checks, test
|
||||||
|
results, credential references, and generic integration events. Protocol-heavy
|
||||||
|
or domain-heavy integrations may live in dedicated modules once their scope is
|
||||||
|
clear.
|
||||||
|
|
||||||
|
Connector capability is not source ownership. Each configured binding also
|
||||||
|
declares whether GovOPlaN is authoritative, the external system is
|
||||||
|
authoritative, GovOPlaN keeps a mirror, both sides use governed synchronization,
|
||||||
|
GovOPlaN supplies only a governance overlay, or the object is link-only. The
|
||||||
|
same connector may be configured differently by tenant, service, object type,
|
||||||
|
or field group.
|
||||||
|
|
||||||
|
Detailed follow-up documents:
|
||||||
|
|
||||||
|
- [Public-sector integration catalogue](PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md)
|
||||||
|
- [Connector source lifecycle](CONNECTOR_SOURCE_LIFECYCLE.md)
|
||||||
|
- [OpenProject connector concept](OPENPROJECT_CONNECTOR.md)
|
||||||
|
- [OpenDesk integration map](OPENDESK_INTEGRATION_MAP.md)
|
||||||
|
|
||||||
|
## Shared runtime boundary
|
||||||
|
|
||||||
|
Connector transports use the versioned Core runtime contract for bounded dry
|
||||||
|
runs and diagnostics. Connectors owns endpoint discovery, authentication
|
||||||
|
hand-off, protocol reads, retry/backoff, and source health. A consuming module
|
||||||
|
owns domain mappings and mutations: Addresses, for example, owns contact and
|
||||||
|
vCard semantics even when Connectors supplies reusable LDAP, CardDAV, Exchange,
|
||||||
|
or Google transport patterns.
|
||||||
|
|
||||||
|
Dry runs carry an immutable input hash, source revision and fingerprint,
|
||||||
|
redacted effects, diagnostics, truncation state, and an apply token. Apply must
|
||||||
|
reject a changed input or source revision. URLs and diagnostics never contain
|
||||||
|
credential material; they retain only credential-envelope references.
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
|
||||||
|
The module owns:
|
||||||
|
|
||||||
|
- connector catalogue entries and capability metadata
|
||||||
|
- connection profiles and endpoint configuration
|
||||||
|
- credential references and test diagnostics
|
||||||
|
- generic webhook/polling/job coordination metadata
|
||||||
|
- connector health status and last-test evidence
|
||||||
|
- operator-visible integration inventory
|
||||||
|
- cross-module discovery of available external capabilities
|
||||||
|
- supported integration maturity, source-authority modes, operation limits,
|
||||||
|
and effect/reconciliation behavior for each connector type
|
||||||
|
|
||||||
|
The module does not own:
|
||||||
|
|
||||||
|
- governed datasource identity, staging, materializations, frozen states, or
|
||||||
|
consumer read semantics, owned by `govoplan-datasources`
|
||||||
|
- file storage semantics, owned by files/DMS
|
||||||
|
- identity provisioning semantics, owned by IDM/access
|
||||||
|
- mail/calendar semantics, owned by mail/calendar
|
||||||
|
- case/workflow/task/domain records
|
||||||
|
- payment, ledger, XRechnung, XTA/OSCI, FIT-Connect, or XOE/V protocol
|
||||||
|
semantics once those are dedicated modules
|
||||||
|
|
||||||
|
## Connector Categories
|
||||||
|
|
||||||
|
Initial catalogue categories:
|
||||||
|
|
||||||
|
- project management and task systems such as OpenProject
|
||||||
|
- DMS/e-file/archive systems
|
||||||
|
- file providers such as Nextcloud, Seafile, WebDAV, SMB/NFS, object storage
|
||||||
|
- identity providers such as LDAP, Active Directory, OIDC, SAML, OpenDesk IDM
|
||||||
|
- groupware such as Open-Xchange mail/calendar
|
||||||
|
- ERP, finance, accounting, payment, and cash-register systems
|
||||||
|
- public-sector protocols such as FIT-Connect, XTA/OSCI, XRechnung, XOE/V
|
||||||
|
- reporting, BI, RSS/API publication, and open-data endpoints
|
||||||
|
|
||||||
|
## Core Contracts
|
||||||
|
|
||||||
|
The module should integrate through:
|
||||||
|
|
||||||
|
- module manifest metadata, route factories, permissions, and migrations
|
||||||
|
- a connector catalogue API for listing available connector types
|
||||||
|
- a connection profile API with secret references, not plaintext secrets
|
||||||
|
- a provider declaration that composes authority mode, maturity, supported
|
||||||
|
operations, revisions/freshness, health, limits, idempotency, conflicts,
|
||||||
|
evidence, and reconciliation behavior
|
||||||
|
- capability declarations such as `connectors.catalog`,
|
||||||
|
`connectors.profileTester`, and `connectors.health`
|
||||||
|
- events such as `connector.profile_created`, `connector.test_succeeded`,
|
||||||
|
`connector.test_failed`, and `connector.health_changed`
|
||||||
|
- configuration-package fragments for required external systems
|
||||||
|
|
||||||
|
Domain modules should ask whether a connector capability exists and request a
|
||||||
|
profile/test result through core-mediated capabilities. They must not import
|
||||||
|
connector implementation modules directly.
|
||||||
|
|
||||||
|
Data-oriented consumers use a two-layer path: Connectors publishes a
|
||||||
|
provider-specific datasource origin, then Datasources registers and governs it.
|
||||||
|
Dataflow, Workflow, Reporting, and other consumers use Datasources rather than
|
||||||
|
calling the connector origin directly.
|
||||||
|
|
||||||
|
## Reference Journeys
|
||||||
|
|
||||||
|
### OpenProject Connector First
|
||||||
|
|
||||||
|
1. Operator registers an OpenProject connection profile.
|
||||||
|
2. Connector tests API reachability and authentication.
|
||||||
|
3. A future project-management decision can use the connector before a native
|
||||||
|
`govoplan-projects` module exists.
|
||||||
|
4. Cases/tasks/workflow may link to external project/task references through
|
||||||
|
stable external-reference DTOs.
|
||||||
|
|
||||||
|
### Public-Sector Integration Catalogue
|
||||||
|
|
||||||
|
1. Operator records which external systems exist in an organization.
|
||||||
|
2. GovOPlaN identifies common protocols and missing connectors.
|
||||||
|
3. Configuration packages can declare required connector profiles.
|
||||||
|
4. Health/status pages show whether required integrations are ready.
|
||||||
|
|
||||||
|
### OpenDesk Profile
|
||||||
|
|
||||||
|
1. Operator records OpenDesk component profiles for identity, mail, calendar,
|
||||||
|
files/documents, and OpenProject where present.
|
||||||
|
2. Connectors shows which components are configured, tested, degraded, or
|
||||||
|
missing.
|
||||||
|
3. Domain modules enable optional behavior by checking capabilities through
|
||||||
|
core, not by importing connector or OpenDesk-specific implementation code.
|
||||||
|
|
||||||
|
## MVP Slice
|
||||||
|
|
||||||
|
The first implementation should provide:
|
||||||
|
|
||||||
|
- connector type registry
|
||||||
|
- connection profile CRUD with secret references
|
||||||
|
- connection test result records
|
||||||
|
- WebUI catalogue and profile pages
|
||||||
|
- configuration-package fragment support
|
||||||
|
- generic external-reference DTOs
|
||||||
|
- source-authority binding and provider-operation metadata
|
||||||
|
- health summary provider
|
||||||
|
|
||||||
|
## Permissions
|
||||||
|
|
||||||
|
Candidate scopes:
|
||||||
|
|
||||||
|
- `connectors:catalog:read`
|
||||||
|
- `connectors:profile:read`
|
||||||
|
- `connectors:profile:write`
|
||||||
|
- `connectors:profile:test`
|
||||||
|
- `connectors:secret:manage`
|
||||||
|
- `connectors:admin`
|
||||||
|
|
||||||
|
## Data Model Sketch
|
||||||
|
|
||||||
|
Candidate tables:
|
||||||
|
|
||||||
|
- `connector_types`
|
||||||
|
- `connector_profiles`
|
||||||
|
- `connector_profile_tests`
|
||||||
|
- `connector_health_status`
|
||||||
|
- `external_references`
|
||||||
|
|
||||||
|
Plaintext credentials must never be stored in connector tables. Use secret
|
||||||
|
references and the platform secret contract.
|
||||||
|
|
||||||
|
## WebUI
|
||||||
|
|
||||||
|
Initial route contributions:
|
||||||
|
|
||||||
|
- `/connectors`
|
||||||
|
- `/connectors/profiles/:profileId`
|
||||||
|
|
||||||
|
The UI should show profile status, last test result, capability labels, required
|
||||||
|
configuration-package dependencies, and external-reference search where a
|
||||||
|
connector supports it.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Minimum tests:
|
||||||
|
|
||||||
|
- core starts with connectors installed and no domain modules present
|
||||||
|
- profile validation rejects plaintext secret echoing
|
||||||
|
- connection tests record success/failure diagnostics without leaking secrets
|
||||||
|
- configuration package can require a connector profile
|
||||||
|
- domain-module optional behavior can detect connector capabilities without
|
||||||
|
imports
|
||||||
|
|
||||||
|
## Open Decisions
|
||||||
|
|
||||||
|
- Whether protocol-specific connector modules depend on `govoplan-connectors`
|
||||||
|
or only share kernel contracts.
|
||||||
|
- Which connector type should be first after OpenProject.
|
||||||
|
- How much polling/webhook scheduling belongs here versus workflow/ops.
|
||||||
|
- Whether external-reference indexing should move to search/dataflow later.
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
# Connector Source Lifecycle
|
||||||
|
|
||||||
|
GovOPlaN modules should treat external systems as sources with explicit
|
||||||
|
lifecycle state. A connector profile can consume records from a source, publish
|
||||||
|
records into a source, or do both. The lifecycle below keeps connectors
|
||||||
|
predictable and avoids hidden module imports.
|
||||||
|
|
||||||
|
## Source Directions
|
||||||
|
|
||||||
|
- `consume`: GovOPlaN reads external records, normalizes them, and exposes them
|
||||||
|
to modules as external references, events, or staged imports.
|
||||||
|
- `publish`: GovOPlaN creates or updates external records and stores the external
|
||||||
|
identifiers as immutable references.
|
||||||
|
- `bidirectional`: GovOPlaN supports both directions with conflict detection and
|
||||||
|
reconciliation rules.
|
||||||
|
|
||||||
|
Direction describes transport. Every binding also needs a source-authority
|
||||||
|
mode:
|
||||||
|
|
||||||
|
- `native_authoritative`
|
||||||
|
- `external_authoritative`
|
||||||
|
- `external_mirror`
|
||||||
|
- `governed_sync`
|
||||||
|
- `governance_overlay`
|
||||||
|
- `linked_reference`
|
||||||
|
|
||||||
|
The authority mode and the connector's integration maturity are orthogonal. A
|
||||||
|
bidirectional connector may be configured as an external mirror, and a native
|
||||||
|
GovOPlaN object may publish to an external target without transferring
|
||||||
|
authority. The effective binding must identify its scope and provenance rather
|
||||||
|
than relying on a profile-wide `sync` boolean.
|
||||||
|
|
||||||
|
## Source Data Lifecycle
|
||||||
|
|
||||||
|
Connector profiles have operational states, while individual external records
|
||||||
|
or source datasets move through a data lifecycle:
|
||||||
|
|
||||||
|
1. `discovered`
|
||||||
|
A source, record, file, feed item, webhook event, or remote object is known
|
||||||
|
but not yet trusted for domain use.
|
||||||
|
2. `connected`
|
||||||
|
GovOPlaN can authenticate and fetch or publish against the source profile.
|
||||||
|
3. `imported`
|
||||||
|
Minimal source data has been staged with external id, version/ETag, source
|
||||||
|
timestamp, and provenance.
|
||||||
|
4. `validated`
|
||||||
|
Shape, permissions, freshness, and required fields passed connector and
|
||||||
|
domain validation.
|
||||||
|
5. `transformed`
|
||||||
|
A dataflow, workflow, or domain module normalized the staged payload into a
|
||||||
|
domain-specific form.
|
||||||
|
6. `published`
|
||||||
|
GovOPlaN exposed or wrote an output through API, RSS, report, export, or a
|
||||||
|
downstream connector.
|
||||||
|
7. `archived`
|
||||||
|
The source/output is no longer active but remains available under retention,
|
||||||
|
audit, and external-reference rules.
|
||||||
|
8. `deprecated`
|
||||||
|
The source/output remains readable for history but must not be used for new
|
||||||
|
workflows.
|
||||||
|
|
||||||
|
Every transition must preserve provenance, permissions context, freshness, and
|
||||||
|
audit trace. Domain modules may add stricter states, but they should map back to
|
||||||
|
this lifecycle when a connector publishes status.
|
||||||
|
|
||||||
|
## Lifecycle States
|
||||||
|
|
||||||
|
1. `draft`
|
||||||
|
Profile exists but is not used by runtime jobs.
|
||||||
|
2. `configured`
|
||||||
|
Required endpoint and credential references are present.
|
||||||
|
3. `tested`
|
||||||
|
A health/test run succeeded and recorded non-secret diagnostics.
|
||||||
|
4. `active`
|
||||||
|
Runtime jobs may consume or publish data.
|
||||||
|
5. `degraded`
|
||||||
|
The connector is active but health checks or recent jobs show failures.
|
||||||
|
6. `paused`
|
||||||
|
Operators intentionally stop scheduled connector activity.
|
||||||
|
7. `retiring`
|
||||||
|
The connector is being removed from active workflows while references remain
|
||||||
|
readable.
|
||||||
|
8. `retired`
|
||||||
|
No new runtime activity is allowed. Historical references remain available.
|
||||||
|
|
||||||
|
## State Transition Gates
|
||||||
|
|
||||||
|
| Transition | Required Evidence | Blockers |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `draft` -> `configured` | endpoint fields are valid, credential references exist, owner/tenant scope is set | plaintext secret in profile payload, unsupported connector type |
|
||||||
|
| `configured` -> `tested` | latest profile test succeeded and diagnostics were redacted | failed auth, unreachable endpoint, TLS/policy error |
|
||||||
|
| `tested` -> `active` | operator enabled runtime use, required modules/capabilities are present, schedule/webhook is valid | missing module, missing permission, no idempotency strategy for publish jobs |
|
||||||
|
| `active` -> `degraded` | health check or job telemetry reports failures | none; this is automatic diagnostic state |
|
||||||
|
| `degraded` -> `active` | health/test succeeds or failed jobs are reconciled | unresolved conflict or repeated failure threshold |
|
||||||
|
| any running state -> `paused` | operator pause request or maintenance preflight | active critical transaction that cannot be interrupted |
|
||||||
|
| `paused` -> `active` | successful re-test when credentials/endpoints changed | failed profile test |
|
||||||
|
| any state -> `retiring` | uninstall/disable plan accepted, schedulers/workers stopped | active domain references that require operator decision |
|
||||||
|
| `retiring` -> `retired` | non-destructive retirement complete, references remain readable | destructive retirement requested without provider and backup |
|
||||||
|
|
||||||
|
## Consume Flow
|
||||||
|
|
||||||
|
1. Discover changes through polling, webhook, batch upload, or manual operator
|
||||||
|
action.
|
||||||
|
2. Fetch only the minimal remote data required for the declared use case.
|
||||||
|
3. Normalize into a connector-owned staging payload.
|
||||||
|
4. Validate shape, required fields, and source trust level.
|
||||||
|
5. Publish data-shaped inputs as versioned datasource origins.
|
||||||
|
6. Let Datasources register live/cached origins or stage immutable snapshots.
|
||||||
|
7. Let domain modules consume governed datasource references through
|
||||||
|
capabilities, not imports.
|
||||||
|
8. Emit a core-mediated event such as `connector.record_discovered`.
|
||||||
|
9. Store external references with source system, object type, object id, version
|
||||||
|
or ETag, and last-seen timestamp.
|
||||||
|
|
||||||
|
## Publish Flow
|
||||||
|
|
||||||
|
1. Domain module requests publish through a core-mediated connector capability.
|
||||||
|
2. Connector validates profile state, permission, idempotency key, and payload
|
||||||
|
shape.
|
||||||
|
3. Connector sends the remote request.
|
||||||
|
4. Connector stores the remote id, version/ETag, and response diagnostics.
|
||||||
|
5. A timeout or lost acknowledgement after dispatch becomes outcome-unknown,
|
||||||
|
not an ordinary failure or permission to duplicate the command.
|
||||||
|
6. Connector emits a confirmed, retryable, outcome-unknown, reconciled, or
|
||||||
|
corrected result event.
|
||||||
|
7. Domain module stores only the external-reference DTO and any domain result.
|
||||||
|
|
||||||
|
## Reconciliation
|
||||||
|
|
||||||
|
Every connector that writes to an external system needs a reconciliation story:
|
||||||
|
|
||||||
|
- idempotency key for create/update jobs
|
||||||
|
- remote object version, ETag, or last-modified value where available
|
||||||
|
- conflict state when local and remote records diverge
|
||||||
|
- retry policy for temporary failures
|
||||||
|
- explicit operator action for destructive overwrite or deletion
|
||||||
|
- audit trace from GovOPlaN record to external request and response summary
|
||||||
|
- explicit requested, approved, dispatched, possibly-executed, confirmed, and
|
||||||
|
reconciled/corrected effect states
|
||||||
|
|
||||||
|
## Durable recovery operations
|
||||||
|
|
||||||
|
Connectors declares two recovery classes. A read-only acquisition into an
|
||||||
|
immutable snapshot is `atomic`: the source revision or conditional cursor,
|
||||||
|
redacted dry-run decision, canonical request digest, and distributed
|
||||||
|
tenant/provider lease are durable before the fetch. The acquired domain rows
|
||||||
|
and terminal Core recovery checkpoint commit in one PostgreSQL transaction. A
|
||||||
|
caller-supplied `Idempotency-Key` replays that committed result without a second
|
||||||
|
provider request. A failed or stale transaction has no remote mutation and may
|
||||||
|
be repeated only as a new deliberate acquisition.
|
||||||
|
|
||||||
|
An external create, update, publish, or delete is `forward_recovery`. It must
|
||||||
|
start through the connector mutation recovery contract with a stable
|
||||||
|
idempotency key, SHA-256 request digest, source revision/cursor, and dry-run
|
||||||
|
evidence. Definitive rejection is terminal. A timeout or lost acknowledgement
|
||||||
|
after dispatch is `outcome_unknown` and blocks replay until the owning connector
|
||||||
|
verifies provider state. The contract and conformance tests exist; no current
|
||||||
|
connector advertises a production external mutation, so write/delete adoption
|
||||||
|
remains explicitly planned rather than implied.
|
||||||
|
|
||||||
|
## Provider Declaration
|
||||||
|
|
||||||
|
An executable connector type should publish machine-readable metadata for:
|
||||||
|
|
||||||
|
- owned object and field groups, plus supported authority modes;
|
||||||
|
- supported discovery, link, search, read, publish, synchronize, migrate, and
|
||||||
|
replacement maturity;
|
||||||
|
- read/write/delete/preview/dry-run operations and bounded response limits;
|
||||||
|
- revision/concurrency token, freshness, health, timeout, retry, and conflict
|
||||||
|
semantics;
|
||||||
|
- idempotency and outcome-unknown handling;
|
||||||
|
- evidence, rollback/compensation, correction, and reconciliation paths;
|
||||||
|
- classification, purpose, retention, secret, degraded, and outage behavior.
|
||||||
|
|
||||||
|
This declaration composes Core contracts. It does not move protocol behavior
|
||||||
|
or domain semantics into Core or Connectors.
|
||||||
|
|
||||||
|
## Capability Boundary
|
||||||
|
|
||||||
|
Domain modules must not import connector implementation packages directly. They
|
||||||
|
should ask core for capabilities such as:
|
||||||
|
|
||||||
|
- `connectors.catalog`
|
||||||
|
- `connectors.profileTester`
|
||||||
|
- `connectors.health`
|
||||||
|
- `connectors.externalReferences`
|
||||||
|
- `connectors.datasourceOrigins`
|
||||||
|
- `connectors.sourceConsumer`
|
||||||
|
- `connectors.sourcePublisher`
|
||||||
|
|
||||||
|
Connector payloads should be DTOs or protocol objects from kernel/core
|
||||||
|
contracts. Protocol-specific clients stay inside the connector module that owns
|
||||||
|
them.
|
||||||
|
|
||||||
|
## Safety Rules
|
||||||
|
|
||||||
|
- Secret values never leave the secret contract and are never stored in test
|
||||||
|
result payloads.
|
||||||
|
- Runtime jobs must include profile id, connector type, direction, idempotency
|
||||||
|
key, and triggering principal/system actor.
|
||||||
|
- Profile tests must redact tokens, passwords, cookies, authorization headers,
|
||||||
|
and remote personal data not needed for diagnostics.
|
||||||
|
- Deactivation must stop schedulers/workers before profile removal.
|
||||||
|
- Uninstall defaults to non-destructive retirement; domain data and external
|
||||||
|
references remain readable.
|
||||||
|
- Destructive retirement requires a module-owned retirement provider and an
|
||||||
|
explicit operator choice.
|
||||||
|
|
||||||
|
## Release Checklist
|
||||||
|
|
||||||
|
Before shipping an executable connector type:
|
||||||
|
|
||||||
|
- Add catalogue metadata and capability names.
|
||||||
|
- Add profile schema validation that rejects plaintext secrets.
|
||||||
|
- Add redaction tests for success and failure diagnostics.
|
||||||
|
- Add unavailable-optional-module tests for every consuming domain module.
|
||||||
|
- Add profile test and health status fixtures.
|
||||||
|
- Add external-reference DTO tests.
|
||||||
|
- Add source-authority and provider-declaration validation tests.
|
||||||
|
- Add lifecycle transition tests for pause, retry, retirement, and uninstall
|
||||||
|
guard behavior.
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
# Governed Connector Configuration
|
||||||
|
|
||||||
|
GovOPlaN connectors should make integration behavior inspectable and testable.
|
||||||
|
The target is not hardcoded glue hidden in module code, but governed connector
|
||||||
|
definitions with schemas, mappings, test runs, simulation, versioning, and
|
||||||
|
audit-visible execution.
|
||||||
|
|
||||||
|
## Connector Definition
|
||||||
|
|
||||||
|
A connector definition should describe:
|
||||||
|
|
||||||
|
- provider type and protocol
|
||||||
|
- endpoint and credential requirements
|
||||||
|
- supported capabilities
|
||||||
|
- input and output schemas
|
||||||
|
- mapping and transformation versions
|
||||||
|
- validation rules
|
||||||
|
- dry-run and test operations
|
||||||
|
- privacy and retention classification
|
||||||
|
- expected events and audit records
|
||||||
|
- operational limits and retry behavior
|
||||||
|
|
||||||
|
Provider-specific code may still be required, but the configured integration
|
||||||
|
logic should remain visible and reviewable.
|
||||||
|
|
||||||
|
## Runtime Expectations
|
||||||
|
|
||||||
|
Connectors supports the generic governed-definition and simulation portion of:
|
||||||
|
|
||||||
|
- discovery where possible
|
||||||
|
- typed configuration through UI-managed controls
|
||||||
|
- secret references instead of plaintext secrets
|
||||||
|
- dry-run plans before writes
|
||||||
|
- simulation with sample payloads
|
||||||
|
- provenance for consumed and produced data
|
||||||
|
- idempotent external writes where supported
|
||||||
|
- quarantine/manual-review state for unsafe or ambiguous results
|
||||||
|
|
||||||
|
Configuration packages may install connector definitions, but local overrides
|
||||||
|
must be protected from accidental package updates.
|
||||||
|
|
||||||
|
## Implemented Runtime Slice
|
||||||
|
|
||||||
|
The module now persists tenant-scoped connector definitions as immutable
|
||||||
|
revisions. A governed definition explicitly validates its provider, protocol,
|
||||||
|
capabilities, input/output schemas, mapping version and rules, validation,
|
||||||
|
preview metadata, audit expectations, classification, retention, limits, and
|
||||||
|
retry policy. Definitions record whether they are locally owned or supplied by
|
||||||
|
a named package.
|
||||||
|
|
||||||
|
Configurations pin a definition revision. They store an endpoint and a secret
|
||||||
|
reference, never credentials embedded in the URL. Tenant-local override values
|
||||||
|
are merged into the pinned definition and every overridden leaf is exposed as
|
||||||
|
a protected path. Installing a later package revision only marks the
|
||||||
|
configuration as having an update available. Adoption is an explicit,
|
||||||
|
optimistically locked action that reapplies the protected overrides over the
|
||||||
|
new package revision.
|
||||||
|
|
||||||
|
The generic execution surface supports bounded dry-runs and simulations. Each
|
||||||
|
run has a caller idempotency key and retains hashes of its inputs and effective
|
||||||
|
configuration together with definition, configuration, mapping, external
|
||||||
|
revision, actor, classification, and retention provenance. Samples redact the
|
||||||
|
definition's protected fields. Ambiguous uniqueness results follow the
|
||||||
|
configuration's policy and become either:
|
||||||
|
|
||||||
|
- `manual_review` with a pending decision;
|
||||||
|
- `quarantined` until an administrator decides; or
|
||||||
|
- `rejected` without a review queue entry.
|
||||||
|
|
||||||
|
Review decisions require a reason and are audited. The generic runtime stops
|
||||||
|
at deterministic preview evidence: provider-specific adapters remain
|
||||||
|
responsible for live external writes and must satisfy the Core connector
|
||||||
|
recovery contract before claiming write maturity.
|
||||||
|
|
||||||
|
The Connector governance administration page follows the shared workspace
|
||||||
|
archetype. Reload and Save stay in the semantic action bar, dirty navigation is
|
||||||
|
guarded, package adoption is a separate action, and simulation results and
|
||||||
|
review decisions remain visibly distinct from configuration editing.
|
||||||
|
|
||||||
|
## Relationship To Datasources And Dataflow
|
||||||
|
|
||||||
|
Recurring extraction and transformation should start as configuration across
|
||||||
|
connectors, files, workflow, reporting, and templates. Create dedicated
|
||||||
|
datasource or dataflow modules only when repeated source-catalog, lineage,
|
||||||
|
mapping, scheduling, or publication contracts clearly outgrow connector
|
||||||
|
ownership.
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
# OpenDesk Integration Map
|
||||||
|
|
||||||
|
OpenDesk is an integration profile across GovOPlaN modules, not a monolithic
|
||||||
|
GovOPlaN module. The profile should let an operator see which OpenDesk
|
||||||
|
components are connected, which GovOPlaN module owns each behavior, and which
|
||||||
|
optional capabilities are available.
|
||||||
|
|
||||||
|
The core boundary decision register is in
|
||||||
|
`/mnt/DATA/git/govoplan-core/docs/MODULE_ARCHITECTURE.md`.
|
||||||
|
|
||||||
|
## Component Routing
|
||||||
|
|
||||||
|
| OpenDesk area | Example component | GovOPlaN owner | Integration behavior |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Identity and directory | OpenDesk IDM, LDAP, AD, OIDC, SAML, SCIM | `govoplan-idm`, `govoplan-access` | integrate, synchronize selected accounts/groups, map principals and memberships |
|
||||||
|
| Mail/groupware | Open-Xchange mail | `govoplan-mail` | integrate, link profiles, test mailbox/send/append, keep mail semantics in mail |
|
||||||
|
| Calendar/groupware | Open-Xchange calendar, CalDAV/CardDAV | `govoplan-calendar` | integrate, free/busy lookup, selected event sync, resource calendars |
|
||||||
|
| Files/documents | Nextcloud/WebDAV/files, office integrations | `govoplan-files`, later `govoplan-dms` | integrate, import/link files, keep document lifecycle in DMS |
|
||||||
|
| Project management | OpenProject | `govoplan-connectors`, consumers in tasks/workflow/cases | connector-first, link/synchronize selected work packages |
|
||||||
|
| Portal/collaboration | Portal/chat/video/office services where present | `govoplan-portal`, `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow` | link/integrate only when a process needs it |
|
||||||
|
| Inventory and diagnostics | endpoint catalogue, profile health, version checks | `govoplan-connectors` | catalogue, profile test, health summary, optional capability discovery |
|
||||||
|
|
||||||
|
## Integration Behavior
|
||||||
|
|
||||||
|
- `integrate`: call a stable API or protocol for the component.
|
||||||
|
- `link`: store external references and open the external tool for
|
||||||
|
source-of-truth work.
|
||||||
|
- `import`: bring selected files/records into GovOPlaN-owned storage or
|
||||||
|
evidence.
|
||||||
|
- `synchronize`: keep selected records aligned through explicit source-of-truth
|
||||||
|
rules.
|
||||||
|
- `replace selected workflow`: only when GovOPlaN owns tighter governance,
|
||||||
|
audit, retention, or configuration-package state than the OpenDesk component.
|
||||||
|
|
||||||
|
## Shared Assumptions
|
||||||
|
|
||||||
|
- Identity is the first dependency. Mail, calendar, files, and project
|
||||||
|
connectors should record which identity profile or tenant mapping they expect.
|
||||||
|
- Connector profiles store references to secrets, never secret values.
|
||||||
|
- Module consumers discover optional behavior through core capabilities and
|
||||||
|
module metadata.
|
||||||
|
- The profile must work partially: an installation can have OpenProject without
|
||||||
|
Open-Xchange, or calendar without files.
|
||||||
|
|
||||||
|
## Candidate Profile Shape
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "opendesk-main",
|
||||||
|
"display_name": "OpenDesk",
|
||||||
|
"components": {
|
||||||
|
"identity": {"profile_id": "opendesk-idm", "owner": "govoplan-idm"},
|
||||||
|
"mail": {"profile_id": "ox-mail", "owner": "govoplan-mail"},
|
||||||
|
"calendar": {"profile_id": "ox-calendar", "owner": "govoplan-calendar"},
|
||||||
|
"files": {"profile_id": "nextcloud-main", "owner": "govoplan-files"},
|
||||||
|
"projects": {"profile_id": "openproject-main", "owner": "govoplan-connectors"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Follow-Up Implementation Issues
|
||||||
|
|
||||||
|
Existing high-priority module issues:
|
||||||
|
|
||||||
|
- `govoplan-idm#1`: LDAP, Active Directory, OpenDesk identity services.
|
||||||
|
- `govoplan-mail#5`: Open-Xchange mail/groupware adapter boundary.
|
||||||
|
- `govoplan-calendar#2`: Open-Xchange calendar adapter boundary.
|
||||||
|
- `govoplan-connectors#1`: OpenProject connector.
|
||||||
|
|
||||||
|
Future executable connector issues should be created in the owning module
|
||||||
|
repository when the first concrete API/profile slice is selected.
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
# OpenProject Connector Concept
|
||||||
|
|
||||||
|
OpenProject is the first proposed concrete connector for
|
||||||
|
`govoplan-connectors`. It gives GovOPlaN a public-sector-friendly project and
|
||||||
|
work-package integration target without making project management a core
|
||||||
|
platform dependency.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- Register OpenProject connection profiles.
|
||||||
|
- Test API reachability and authentication without exposing secrets.
|
||||||
|
- Read projects, users, statuses, and work packages for linking.
|
||||||
|
- Create or update work packages from GovOPlaN tasks/cases/workflows once those
|
||||||
|
modules request the capability.
|
||||||
|
- Receive or poll changes for external-reference synchronization.
|
||||||
|
- Keep all OpenProject-specific client code inside the connector module.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
- Replacing a future native GovOPlaN project-management module.
|
||||||
|
- Importing workflow, tasks, cases, or access implementation modules directly.
|
||||||
|
- Mirroring complete OpenProject project state into GovOPlaN by default.
|
||||||
|
- Storing OpenProject tokens outside the platform secret contract.
|
||||||
|
|
||||||
|
## Profile Fields
|
||||||
|
|
||||||
|
Candidate profile payload:
|
||||||
|
|
||||||
|
- `base_url`
|
||||||
|
- `api_version`, default `v3`
|
||||||
|
- `credential_ref`
|
||||||
|
- `verify_tls`
|
||||||
|
- `timeout_seconds`
|
||||||
|
- `allowed_project_ids`
|
||||||
|
- `default_project_id`
|
||||||
|
- `webhook_secret_ref`, optional
|
||||||
|
- `poll_interval_seconds`, optional
|
||||||
|
|
||||||
|
## Health Check
|
||||||
|
|
||||||
|
The connection test should:
|
||||||
|
|
||||||
|
1. Normalize and validate `base_url`.
|
||||||
|
2. Resolve the credential reference.
|
||||||
|
3. Call the OpenProject API root or a small read-only endpoint.
|
||||||
|
4. Record API version, authenticated principal where available, latency,
|
||||||
|
response status, and safe capability hints.
|
||||||
|
5. Redact token, Authorization headers, cookies, and any server-provided secret
|
||||||
|
fields from diagnostics.
|
||||||
|
|
||||||
|
## Candidate Capabilities
|
||||||
|
|
||||||
|
- `connectors.openproject.profileTester`
|
||||||
|
- `connectors.openproject.projects`
|
||||||
|
- `connectors.openproject.workPackages.read`
|
||||||
|
- `connectors.openproject.workPackages.write`
|
||||||
|
- `connectors.openproject.webhooks`
|
||||||
|
- `connectors.openproject.externalReferences`
|
||||||
|
|
||||||
|
Domain modules request these through core-mediated capabilities. For example,
|
||||||
|
`govoplan-tasks` can publish a task as an OpenProject work package without
|
||||||
|
importing OpenProject client code.
|
||||||
|
|
||||||
|
## Data Boundary Decision
|
||||||
|
|
||||||
|
OpenProject should be referenced live by default, not mirrored wholesale into
|
||||||
|
GovOPlaN. The connector stores stable external references and safe metadata:
|
||||||
|
|
||||||
|
- profile id and connector type
|
||||||
|
- project id and work-package id
|
||||||
|
- external URL
|
||||||
|
- remote version, ETag, or lock version where available
|
||||||
|
- last-seen timestamp and safe status/type labels
|
||||||
|
- GovOPlaN trace id for publish or synchronization jobs
|
||||||
|
|
||||||
|
GovOPlaN should import only the subset needed by a requesting domain module,
|
||||||
|
for example a work-package title/status for display, a link-back reference for a
|
||||||
|
task, or evidence that a publish operation succeeded. Full project state,
|
||||||
|
comments, attachments, membership lists, and custom fields remain remote unless
|
||||||
|
a future domain module explicitly owns that synchronization. This keeps cases,
|
||||||
|
tasks, workflow, and reporting decoupled from OpenProject while still allowing
|
||||||
|
link-out, link-back, selected publish, and selected read views.
|
||||||
|
|
||||||
|
## Runtime Events
|
||||||
|
|
||||||
|
- `openproject.profile_tested`
|
||||||
|
- `openproject.project_seen`
|
||||||
|
- `openproject.work_package_seen`
|
||||||
|
- `openproject.work_package_published`
|
||||||
|
- `openproject.webhook_received`
|
||||||
|
- `openproject.sync_failed`
|
||||||
|
|
||||||
|
Events should carry GovOPlaN ids, external ids, safe diagnostics, and trace
|
||||||
|
context. They must not contain credentials or raw personal data beyond what the
|
||||||
|
requesting domain module is authorized to process.
|
||||||
|
|
||||||
|
## Webhook And Polling Strategy
|
||||||
|
|
||||||
|
OpenProject supports API and webhook administration. The connector should allow
|
||||||
|
both:
|
||||||
|
|
||||||
|
- Webhook-first when an operator registers a webhook for selected project/work
|
||||||
|
package events.
|
||||||
|
- Polling fallback for installations where webhooks cannot be exposed.
|
||||||
|
|
||||||
|
The first implementation can start with manual test plus read-only project/work
|
||||||
|
package lookup, then add publishing, then webhook/polling synchronization.
|
||||||
|
|
||||||
|
## First Implementation Slice
|
||||||
|
|
||||||
|
1. Add connector type metadata for `openproject`.
|
||||||
|
2. Add connection profile CRUD using secret references.
|
||||||
|
3. Add a read-only test endpoint.
|
||||||
|
4. Add project/work-package lookup DTOs.
|
||||||
|
5. Add external-reference storage for linked OpenProject work packages.
|
||||||
|
6. Add a WebUI profile page with last-test diagnostics.
|
||||||
|
7. Add tests for redaction, unavailable connector behavior, and optional module
|
||||||
|
capability discovery.
|
||||||
|
|
||||||
|
## Minimum DTOs
|
||||||
|
|
||||||
|
Profile summary:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "openproject-main",
|
||||||
|
"connector_type": "openproject",
|
||||||
|
"name": "OpenProject",
|
||||||
|
"base_url": "https://openproject.example",
|
||||||
|
"state": "tested",
|
||||||
|
"last_test_at": "2026-07-09T10:00:00Z",
|
||||||
|
"last_test_status": "success"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
External reference:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"connector_type": "openproject",
|
||||||
|
"profile_id": "openproject-main",
|
||||||
|
"object_type": "work_package",
|
||||||
|
"external_id": "1234",
|
||||||
|
"external_url": "https://openproject.example/work_packages/1234",
|
||||||
|
"version": "etag-or-lock-version",
|
||||||
|
"metadata": {
|
||||||
|
"project_id": "42"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Diagnostics must be redacted and should include only endpoint, version,
|
||||||
|
authenticated principal label where safe, latency, status code, and capability
|
||||||
|
hints.
|
||||||
|
|
||||||
|
## First Tests To Add
|
||||||
|
|
||||||
|
- profile create/update rejects plaintext token fields
|
||||||
|
- profile test redacts Authorization, cookies, and token-like response fields
|
||||||
|
- lookup capabilities are absent when the connector module is disabled
|
||||||
|
- a task/workflow/case module can detect OpenProject capabilities without
|
||||||
|
importing connector internals
|
||||||
|
- external-reference round-trip stores profile id, object type, external id,
|
||||||
|
version/ETag, URL, and safe metadata
|
||||||
|
|
||||||
|
## Reference Sources
|
||||||
|
|
||||||
|
- OpenProject API v3 documentation: https://www.openproject.org/docs/api/
|
||||||
|
- OpenProject API introduction: https://www.openproject.org/docs/api/introduction/
|
||||||
|
- OpenProject API and webhooks administration: https://www.openproject.org/docs/system-admin-guide/api-and-webhooks/
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
# Public-Sector Integration Catalogue
|
||||||
|
|
||||||
|
`govoplan-connectors` should maintain an operator-visible catalogue of common
|
||||||
|
external systems, protocols, and integration patterns. The catalogue is not a
|
||||||
|
promise that GovOPlaN replaces those systems. It is the map that lets modules
|
||||||
|
discover what exists, test connections, and decide which optional behavior can
|
||||||
|
be enabled.
|
||||||
|
|
||||||
|
## Catalogue Entry Shape
|
||||||
|
|
||||||
|
Each connector type should define:
|
||||||
|
|
||||||
|
- stable connector type key, for example `openproject`, `fit-connect`,
|
||||||
|
`xrepository`, or `sap`
|
||||||
|
- category and owning GovOPlaN module, if any
|
||||||
|
- supported direction: consume, publish, or bidirectional
|
||||||
|
- supported trigger modes: manual test, polling, webhook, batch import, export
|
||||||
|
- credential method and whether secrets are stored through the platform secret
|
||||||
|
contract
|
||||||
|
- health check and diagnostic payload shape
|
||||||
|
- external reference shape for records created or linked through the connector
|
||||||
|
- required capabilities and optional module combinations
|
||||||
|
- lifecycle support: activate, pause, re-test, rotate credential, retire
|
||||||
|
|
||||||
|
## Initial Target Categories
|
||||||
|
|
||||||
|
The catalogue should be maintained as a ranked inventory. A target can start as
|
||||||
|
an inventory entry before there is executable connector code.
|
||||||
|
|
||||||
|
| Target | Scope/Jurisdiction | Category | Mode | Likely Owner | First Useful Capability | Priority |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| OpenProject | international/open source | Project/task management | link, synchronize selected records, publish tasks | `govoplan-connectors`, later tasks/workflow/projects | profile test, project/work-package lookup, external references | Wave 0 |
|
||||||
|
| Nextcloud/WebDAV/SMB/Seafile | broad public-sector/self-hosted | File providers | integrate, import, link | `govoplan-files` with connector inventory | profile health and managed-file provenance | Wave 0/in progress |
|
||||||
|
| OpenDesk IDM, LDAP, Active Directory, OIDC, SAML | Germany/EU and general enterprise | Identity | integrate, synchronize | `govoplan-idm`, `govoplan-access` | endpoint inventory, login/provisioning preflight | Wave 1 |
|
||||||
|
| Open-Xchange mail/calendar | Germany/OpenDesk and groupware deployments | Groupware | integrate, link | `govoplan-mail`, `govoplan-calendar` | profile test, mailbox/calendar diagnostics | Wave 1 |
|
||||||
|
| FIT-Connect | German public-sector transport | Public-sector transport | integrate, publish, receive | dedicated protocol module with connectors inventory | destination profile, test, receipt reference | Wave 1 |
|
||||||
|
| XRepository/XÖV lookup | German public-sector standards | Standards registry | link, import schema metadata | `govoplan-connectors`, later XÖV modules | read-only catalogue lookup/cache | Wave 1 |
|
||||||
|
| RSS/API publication | public data/external services | Publication/data exchange | consume, publish | `govoplan-connectors`, `govoplan-dataflow` | consume/publish feed profiles | Wave 2 |
|
||||||
|
| DMS/e-file/archive systems | German municipal/state/federal administration | DMS/records | link, import, synchronize selected metadata | `govoplan-dms`, `govoplan-files` | external document reference and health | Wave 2 |
|
||||||
|
| ERP/finance/procurement/payment systems | German municipal finance/procurement plus EU standards | ERP/payment | export, import, synchronize, replace only by domain decision | dedicated modules | profile inventory and export/import staging | Wave 2 |
|
||||||
|
|
||||||
|
### Project And Task Management
|
||||||
|
|
||||||
|
- OpenProject
|
||||||
|
- Jira or Jira-compatible APIs
|
||||||
|
- Redmine
|
||||||
|
- Microsoft Planner/Project where available through Microsoft Graph
|
||||||
|
|
||||||
|
GovOPlaN should start with OpenProject because it is open source, common in
|
||||||
|
public-sector environments, and has API/webhook documentation suitable for a
|
||||||
|
first connector.
|
||||||
|
|
||||||
|
### DMS, E-File, Records, And Archive
|
||||||
|
|
||||||
|
- d.velop/d.3
|
||||||
|
- enaio
|
||||||
|
- Fabasoft eGov-Suite
|
||||||
|
- ELO
|
||||||
|
- VIS/eAkte environments
|
||||||
|
- CMIS-capable repositories
|
||||||
|
- S3/object storage used as archive staging
|
||||||
|
|
||||||
|
These targets should usually be owned by DMS/files/records modules once a
|
||||||
|
domain module exists. `govoplan-connectors` should still provide inventory,
|
||||||
|
profiles, and generic health checks.
|
||||||
|
|
||||||
|
### File Providers
|
||||||
|
|
||||||
|
- SMB/CIFS
|
||||||
|
- WebDAV
|
||||||
|
- Nextcloud
|
||||||
|
- Seafile
|
||||||
|
- S3-compatible object storage
|
||||||
|
- SFTP
|
||||||
|
|
||||||
|
The files module owns file semantics. The connectors catalogue should record
|
||||||
|
profile metadata and health, but must not import files-module internals.
|
||||||
|
|
||||||
|
### Identity And Access
|
||||||
|
|
||||||
|
- LDAP
|
||||||
|
- Active Directory
|
||||||
|
- OIDC
|
||||||
|
- SAML
|
||||||
|
- OpenDesk IDM and comparable identity platforms
|
||||||
|
|
||||||
|
The access/IDM modules own principal synchronization and authorization effects.
|
||||||
|
Connectors own endpoint inventory and diagnostics.
|
||||||
|
|
||||||
|
### Mail, Calendar, And Collaboration
|
||||||
|
|
||||||
|
- Microsoft Exchange/M365
|
||||||
|
- Open-Xchange
|
||||||
|
- IMAP/SMTP where represented as external infrastructure
|
||||||
|
- CalDAV/CardDAV
|
||||||
|
- chat, video, and collaboration systems such as Matrix, Jitsi, BigBlueButton,
|
||||||
|
Nextcloud Talk, or Collabora/OnlyOffice environments
|
||||||
|
|
||||||
|
Mail/calendar/collaboration modules own business semantics. Connector profiles
|
||||||
|
can expose reachability and version diagnostics.
|
||||||
|
|
||||||
|
### ERP, Finance, Procurement, And Payment
|
||||||
|
|
||||||
|
- SAP
|
||||||
|
- MACH
|
||||||
|
- Infoma/new system
|
||||||
|
- DATEV interfaces
|
||||||
|
- XRechnung/Peppol access points
|
||||||
|
- XBestellung and procurement feeds
|
||||||
|
- payment providers and cash-register systems
|
||||||
|
|
||||||
|
Protocol-heavy parts should move into dedicated modules such as
|
||||||
|
`govoplan-xrechnung`, `govoplan-erp`, `govoplan-procurement`, or
|
||||||
|
`govoplan-payments`.
|
||||||
|
|
||||||
|
### Public-Sector Protocols And Registries
|
||||||
|
|
||||||
|
- FIT-Connect
|
||||||
|
- XTA/OSCI
|
||||||
|
- XÖV standards and XRepository lookup
|
||||||
|
- XRechnung/XBestellung
|
||||||
|
- register and Fachverfahren interfaces discovered by implementation projects
|
||||||
|
|
||||||
|
These are integration priorities because they model common administrative
|
||||||
|
processes. They should be represented as connector categories even when a
|
||||||
|
dedicated module later owns the actual protocol implementation.
|
||||||
|
|
||||||
|
## Wave 0 Catalogue Priorities
|
||||||
|
|
||||||
|
1. OpenProject connector concept and profile shape.
|
||||||
|
2. Generic connector profile, health, and secret-reference model.
|
||||||
|
3. Public-sector target inventory table with category, owner module, and
|
||||||
|
priority.
|
||||||
|
4. Consume/publish source lifecycle contract.
|
||||||
|
5. External-reference DTO shared through kernel/core contracts.
|
||||||
|
6. Configuration-package declaration for required connector profiles.
|
||||||
|
|
||||||
|
## Catalogue Maintenance Rules
|
||||||
|
|
||||||
|
- Prefer one stable connector type key per external product or protocol family.
|
||||||
|
- Record when GovOPlaN should integrate with an existing product instead of
|
||||||
|
replacing it.
|
||||||
|
- Keep protocol/client implementation in the owning connector or protocol
|
||||||
|
module; domain modules consume capabilities and DTOs only.
|
||||||
|
- Treat "inventory only" entries as useful: operators can document a landscape
|
||||||
|
before GovOPlaN can automate it.
|
||||||
|
- Every executable connector type needs a redaction-safe test plan, lifecycle
|
||||||
|
states, external-reference shape, and uninstall/retirement behavior.
|
||||||
|
|
||||||
|
## Reference Sources
|
||||||
|
|
||||||
|
- OpenProject API v3 documentation: https://www.openproject.org/docs/api/
|
||||||
|
- OpenProject API and webhooks administration: https://www.openproject.org/docs/system-admin-guide/api-and-webhooks/
|
||||||
|
- FIT-Connect Destination API documentation: https://docs.fitko.de/en/resources/fit-connect-destination-api/
|
||||||
|
- XÖV overview by KoSIT: https://www.xoev.de/xoev-4987
|
||||||
|
- XRepository overview: https://www.xrepository.de/
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=69", "wheel"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "govoplan-connectors"
|
||||||
|
version = "0.1.19"
|
||||||
|
description = "Governed connector catalogue and tabular source capabilities for GovOPlaN."
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
license = "AGPL-3.0-or-later"
|
||||||
|
authors = [{ name = "GovOPlaN" }]
|
||||||
|
dependencies = [
|
||||||
|
"defusedxml>=0.7,<1",
|
||||||
|
"govoplan-core>=0.1.18",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["src"]
|
||||||
|
|
||||||
|
[tool.setuptools.package-data]
|
||||||
|
govoplan_connectors = ["py.typed"]
|
||||||
|
|
||||||
|
[project.entry-points."govoplan.modules"]
|
||||||
|
connectors = "govoplan_connectors.backend.manifest:get_manifest"
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
"""GovOPlaN Connectors module."""
|
||||||
|
|
||||||
|
__all__: list[str] = []
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Connector backend package."""
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from govoplan_core.core.datasources import (
|
||||||
|
DatasourceAccessError,
|
||||||
|
DatasourceField,
|
||||||
|
DatasourceNotFoundError,
|
||||||
|
DatasourceOrigin,
|
||||||
|
DatasourceOriginReadRequest,
|
||||||
|
DatasourceOriginReadResult,
|
||||||
|
DatasourceUnavailableError,
|
||||||
|
DatasourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularSource,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceError,
|
||||||
|
TabularSourceNotFoundError,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.tabular_sources import SqlTabularSourceProvider
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDatasourceOriginProvider:
|
||||||
|
"""Expose connector-owned sources through the Datasources origin contract."""
|
||||||
|
|
||||||
|
def __init__(self, provider: SqlTabularSourceProvider | None = None) -> None:
|
||||||
|
self._provider = provider or SqlTabularSourceProvider()
|
||||||
|
|
||||||
|
def list_origins(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
query: str = "",
|
||||||
|
limit: int = 100,
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
rows = self._provider.list_sources(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
query=query,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _datasource_error(exc) from exc
|
||||||
|
return tuple(_origin(source) for source in rows)
|
||||||
|
|
||||||
|
def get_origin(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
origin_ref: str,
|
||||||
|
) -> DatasourceOrigin | None:
|
||||||
|
try:
|
||||||
|
source = self._provider.get_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
source_ref=origin_ref,
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _datasource_error(exc) from exc
|
||||||
|
return _origin(source) if source is not None else None
|
||||||
|
|
||||||
|
def read_origin(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: DatasourceOriginReadRequest,
|
||||||
|
) -> DatasourceOriginReadResult:
|
||||||
|
try:
|
||||||
|
result = self._provider.read_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=request.origin_ref,
|
||||||
|
limit=request.limit,
|
||||||
|
offset=request.offset,
|
||||||
|
columns=request.columns,
|
||||||
|
expected_fingerprint=request.expected_fingerprint,
|
||||||
|
max_bytes=request.max_bytes,
|
||||||
|
timeout_ms=request.timeout_ms,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _datasource_error(exc) from exc
|
||||||
|
return DatasourceOriginReadResult(
|
||||||
|
origin=_origin(result.source),
|
||||||
|
rows=result.rows,
|
||||||
|
total_rows=result.total_rows,
|
||||||
|
truncated=result.truncated,
|
||||||
|
returned_bytes=result.returned_bytes,
|
||||||
|
elapsed_ms=result.elapsed_ms,
|
||||||
|
effective_row_limit=result.effective_row_limit,
|
||||||
|
effective_byte_limit=result.effective_byte_limit,
|
||||||
|
effective_timeout_ms=result.effective_timeout_ms,
|
||||||
|
diagnostics=result.diagnostics,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _origin(source: TabularSource) -> DatasourceOrigin:
|
||||||
|
return DatasourceOrigin(
|
||||||
|
ref=source.ref,
|
||||||
|
source_name=source.source_name,
|
||||||
|
name=source.name,
|
||||||
|
description=source.description,
|
||||||
|
kind="upload",
|
||||||
|
shape="tabular",
|
||||||
|
supported_modes=("live", "cached"),
|
||||||
|
provider=f"connectors.{source.provider}",
|
||||||
|
schema=tuple(
|
||||||
|
DatasourceField(
|
||||||
|
name=column.name,
|
||||||
|
data_type=column.data_type,
|
||||||
|
nullable=column.nullable,
|
||||||
|
)
|
||||||
|
for column in source.schema
|
||||||
|
),
|
||||||
|
schema_version=source.schema_version,
|
||||||
|
fingerprint=source.fingerprint,
|
||||||
|
row_count=source.row_count,
|
||||||
|
byte_count=source.byte_count,
|
||||||
|
updated_at=source.updated_at,
|
||||||
|
capabilities=source.capabilities,
|
||||||
|
metadata=dict(source.metadata),
|
||||||
|
source_mode=source.source_mode,
|
||||||
|
pushdown=source.pushdown,
|
||||||
|
health=source.health,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _datasource_error(exc: TabularSourceError):
|
||||||
|
if isinstance(exc, TabularSourceAccessError):
|
||||||
|
return DatasourceAccessError(str(exc))
|
||||||
|
if isinstance(exc, TabularSourceNotFoundError):
|
||||||
|
return DatasourceNotFoundError(str(exc))
|
||||||
|
if isinstance(exc, TabularSourceUnavailableError):
|
||||||
|
return DatasourceUnavailableError(str(exc))
|
||||||
|
if isinstance(exc, TabularSourceValidationError):
|
||||||
|
return DatasourceValidationError(str(exc))
|
||||||
|
return DatasourceValidationError(str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["ConnectorDatasourceOriginProvider"]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
|
||||||
|
__all__ = ["ConnectorTabularSource"]
|
||||||
@@ -0,0 +1,468 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
JSON,
|
||||||
|
LargeBinary,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from govoplan_core.db.base import Base, TimestampMixin
|
||||||
|
|
||||||
|
|
||||||
|
def new_uuid() -> str:
|
||||||
|
return str(uuid.uuid4())
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorTabularSource(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_tabular_sources"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("tenant_id", "source_name", name="uq_connector_tabular_source_name"),
|
||||||
|
Index("ix_connector_tabular_sources_tenant_status", "tenant_id", "status"),
|
||||||
|
Index("ix_connector_tabular_sources_tenant_updated", "tenant_id", "updated_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
provider: Mapped[str] = mapped_column(String(50), default="snapshot", nullable=False, index=True)
|
||||||
|
source_name: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
|
status: Mapped[str] = mapped_column(String(30), default="active", nullable=False, index=True)
|
||||||
|
schema_version: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
schema_: Mapped[list[dict[str, Any]]] = mapped_column("schema", JSON, default=list, nullable=False)
|
||||||
|
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
|
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSanctionsAcquisitionRun(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_sanctions_acquisition_runs"
|
||||||
|
__table_args__ = (
|
||||||
|
Index(
|
||||||
|
"ix_connector_sanctions_run_health",
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"status",
|
||||||
|
"started_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
primary_key=True,
|
||||||
|
default=new_uuid,
|
||||||
|
)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
provider_id: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_id: Mapped[str] = mapped_column(
|
||||||
|
String(200),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(40),
|
||||||
|
default="running",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
attempt_count: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
default=0,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
request_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
response_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
snapshot_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
error: Mapped[str | None] = mapped_column(
|
||||||
|
Text,
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(
|
||||||
|
String(255),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSanctionsSnapshot(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_sanctions_snapshots"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"connector_run_id",
|
||||||
|
name="uq_connector_sanctions_snapshot_run",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_sanctions_snapshot_source",
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"acquired_at",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_sanctions_snapshot_version",
|
||||||
|
"provider_id",
|
||||||
|
"source_id",
|
||||||
|
"source_version",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
primary_key=True,
|
||||||
|
default=new_uuid,
|
||||||
|
)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
provider_id: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
publisher: Mapped[str] = mapped_column(
|
||||||
|
String(300),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
jurisdiction: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
list_type: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_id: Mapped[str] = mapped_column(
|
||||||
|
String(200),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_version: Mapped[str] = mapped_column(
|
||||||
|
String(255),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
publication_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
effective_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
acquired_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_url: Mapped[str | None] = mapped_column(
|
||||||
|
String(1500),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
content_type: Mapped[str] = mapped_column(
|
||||||
|
String(200),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
byte_count: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
sha256: Mapped[str] = mapped_column(
|
||||||
|
String(64),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
signature_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
parser_version: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
licence_notes: Mapped[str | None] = mapped_column(
|
||||||
|
Text,
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
trust_notes: Mapped[str | None] = mapped_column(
|
||||||
|
Text,
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
connector_run_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey(
|
||||||
|
"connector_sanctions_acquisition_runs.id",
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
transport_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
raw_content: Mapped[bytes] = mapped_column(
|
||||||
|
LargeBinary,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinition(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_definitions"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"definition_key",
|
||||||
|
name="uq_connector_definition_tenant_key",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_definitions_tenant_status",
|
||||||
|
"tenant_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
definition_key: Mapped[str] = mapped_column(String(160), nullable=False)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="active",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
current_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
source_package: Mapped[str | None] = mapped_column(String(300))
|
||||||
|
local_definition: Mapped[bool] = mapped_column(
|
||||||
|
Boolean,
|
||||||
|
default=False,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionRevision(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_definition_revisions"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"definition_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_connector_definition_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
definition_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_definitions.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
specification: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
definition_hash: Mapped[str] = mapped_column(
|
||||||
|
String(64),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
origin: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||||
|
package_ref: Mapped[str | None] = mapped_column(String(300))
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfiguration(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_configurations"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"name",
|
||||||
|
name="uq_connector_configuration_tenant_name",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_configurations_tenant_status",
|
||||||
|
"tenant_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
definition_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_definitions.id", ondelete="RESTRICT"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="draft",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
endpoint_url: Mapped[str | None] = mapped_column(String(1500))
|
||||||
|
credential_ref: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
base_definition_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
local_overrides: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
protected_paths: Mapped[list[str]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=list,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
effective_configuration: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
effective_hash: Mapped[str] = mapped_column(
|
||||||
|
String(64),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
resource_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
default=1,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
ambiguity_policy: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="manual_review",
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSimulationRun(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_simulation_runs"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
"mode",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_simulation_run_idempotency",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_simulation_runs_review",
|
||||||
|
"tenant_id",
|
||||||
|
"review_state",
|
||||||
|
"created_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
configuration_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_configurations.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
mode: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
review_state: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="not_required",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
definition_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
configuration_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
configuration_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
input_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
summary: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
effects: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=list,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=list,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
reviewed_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
review_reason: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConnectorConfiguration",
|
||||||
|
"ConnectorDefinition",
|
||||||
|
"ConnectorDefinitionRevision",
|
||||||
|
"ConnectorSanctionsAcquisitionRun",
|
||||||
|
"ConnectorSanctionsSnapshot",
|
||||||
|
"ConnectorSimulationRun",
|
||||||
|
"ConnectorTabularSource",
|
||||||
|
"new_uuid",
|
||||||
|
]
|
||||||
@@ -0,0 +1,482 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import (
|
||||||
|
DsarErasureActionRef,
|
||||||
|
DsarExecutionResultRef,
|
||||||
|
DsarRecordRef,
|
||||||
|
DsarSubjectRef,
|
||||||
|
dsar_capability_name,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
CONNECTORS_DSAR_CAPABILITY = dsar_capability_name("connectors")
|
||||||
|
_MAX_RECORDS = 5_000
|
||||||
|
_CONFLICT = object()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _SubjectSelectors:
|
||||||
|
account_id: str
|
||||||
|
source_id: str | None
|
||||||
|
acquisition_id: str | None
|
||||||
|
definition_id: str | None
|
||||||
|
configuration_id: str | None
|
||||||
|
simulation_id: str | None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def narrowed(self) -> bool:
|
||||||
|
return any(
|
||||||
|
(
|
||||||
|
self.source_id,
|
||||||
|
self.acquisition_id,
|
||||||
|
self.definition_id,
|
||||||
|
self.configuration_id,
|
||||||
|
self.simulation_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsDsarProvider:
|
||||||
|
provider_id = "connectors"
|
||||||
|
module_id = "connectors"
|
||||||
|
|
||||||
|
def search_subject(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
) -> Sequence[DsarRecordRef]:
|
||||||
|
db = _session(session)
|
||||||
|
selectors = _subject_selectors(subject)
|
||||||
|
if selectors is None:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
records: list[DsarRecordRef] = []
|
||||||
|
if not selectors.narrowed or selectors.source_id:
|
||||||
|
query = db.query(ConnectorTabularSource).filter(
|
||||||
|
ConnectorTabularSource.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
ConnectorTabularSource.created_by == selectors.account_id,
|
||||||
|
ConnectorTabularSource.updated_by == selectors.account_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if selectors.source_id:
|
||||||
|
query = query.filter(ConnectorTabularSource.id == selectors.source_id)
|
||||||
|
records.extend(
|
||||||
|
_source_attribution(row, selectors.account_id)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorTabularSource.created_at,
|
||||||
|
ConnectorTabularSource.id,
|
||||||
|
label="source attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.acquisition_id:
|
||||||
|
query = db.query(ConnectorSanctionsAcquisitionRun).filter(
|
||||||
|
ConnectorSanctionsAcquisitionRun.tenant_id == tenant_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.created_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.acquisition_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorSanctionsAcquisitionRun.id == selectors.acquisition_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_acquisition_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorSanctionsAcquisitionRun.started_at,
|
||||||
|
ConnectorSanctionsAcquisitionRun.id,
|
||||||
|
label="acquisition attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.definition_id:
|
||||||
|
query = (
|
||||||
|
db.query(ConnectorDefinitionRevision, ConnectorDefinition)
|
||||||
|
.join(
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinition.id == ConnectorDefinitionRevision.definition_id,
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.tenant_id == tenant_id,
|
||||||
|
ConnectorDefinitionRevision.created_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if selectors.definition_id:
|
||||||
|
query = query.filter(ConnectorDefinition.id == selectors.definition_id)
|
||||||
|
records.extend(
|
||||||
|
_definition_attribution(revision, definition)
|
||||||
|
for revision, definition in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorDefinitionRevision.created_at,
|
||||||
|
ConnectorDefinitionRevision.id,
|
||||||
|
label="definition attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.configuration_id:
|
||||||
|
query = db.query(ConnectorConfiguration).filter(
|
||||||
|
ConnectorConfiguration.tenant_id == tenant_id,
|
||||||
|
ConnectorConfiguration.updated_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.configuration_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorConfiguration.id == selectors.configuration_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_configuration_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorConfiguration.updated_at,
|
||||||
|
ConnectorConfiguration.id,
|
||||||
|
label="configuration attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.simulation_id:
|
||||||
|
query = db.query(ConnectorSimulationRun).filter(
|
||||||
|
ConnectorSimulationRun.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
ConnectorSimulationRun.created_by == selectors.account_id,
|
||||||
|
ConnectorSimulationRun.reviewed_by == selectors.account_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if selectors.simulation_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorSimulationRun.id == selectors.simulation_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_simulation_attribution(row, selectors.account_id)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorSimulationRun.created_at,
|
||||||
|
ConnectorSimulationRun.id,
|
||||||
|
label="simulation attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(records) > _MAX_RECORDS:
|
||||||
|
raise ValueError("Connectors DSAR result limit exceeded; narrow selectors.")
|
||||||
|
return tuple(
|
||||||
|
sorted(records, key=lambda item: (item.resource_type, item.resource_id))
|
||||||
|
)
|
||||||
|
|
||||||
|
def plan_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
records: Sequence[DsarRecordRef],
|
||||||
|
) -> Sequence[DsarErasureActionRef]:
|
||||||
|
del tenant_id
|
||||||
|
_session(session)
|
||||||
|
if _subject_selectors(subject) is None:
|
||||||
|
raise ValueError("Connectors DSAR subject selectors conflict.")
|
||||||
|
actions: list[DsarErasureActionRef] = []
|
||||||
|
for record in records:
|
||||||
|
_validate_record(record)
|
||||||
|
actions.append(
|
||||||
|
DsarErasureActionRef(
|
||||||
|
action_id=(
|
||||||
|
f"connectors:retain:{record.resource_type}:{record.resource_id}"
|
||||||
|
),
|
||||||
|
provider_id=self.provider_id,
|
||||||
|
module_id=self.module_id,
|
||||||
|
kind="retain",
|
||||||
|
resource_type=record.resource_type,
|
||||||
|
resource_id=record.resource_id,
|
||||||
|
title=f"Retain {record.title}",
|
||||||
|
rationale=(
|
||||||
|
record.retention_reason
|
||||||
|
or "Connector operator attribution remains governance evidence."
|
||||||
|
),
|
||||||
|
executable=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(actions)
|
||||||
|
|
||||||
|
def execute_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
actions: Sequence[DsarErasureActionRef],
|
||||||
|
request_id: str,
|
||||||
|
) -> Sequence[DsarExecutionResultRef]:
|
||||||
|
del tenant_id
|
||||||
|
_session(session)
|
||||||
|
if _subject_selectors(subject) is None:
|
||||||
|
raise ValueError("Connectors DSAR subject selectors conflict.")
|
||||||
|
results: list[DsarExecutionResultRef] = []
|
||||||
|
for action in actions:
|
||||||
|
_validate_action(action)
|
||||||
|
if action.executable or action.kind != "retain":
|
||||||
|
raise ValueError("Connectors DSAR publishes retain actions only.")
|
||||||
|
results.append(
|
||||||
|
DsarExecutionResultRef(
|
||||||
|
action_id=action.action_id,
|
||||||
|
status="blocked",
|
||||||
|
summary="Connector operator attribution remains governance evidence.",
|
||||||
|
evidence={"request_id": request_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
|
||||||
|
def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None:
|
||||||
|
references = subject.external_references
|
||||||
|
account = _coalesce(
|
||||||
|
subject.account_id,
|
||||||
|
references.get("connectors.account"),
|
||||||
|
references.get("access.account"),
|
||||||
|
)
|
||||||
|
values = {
|
||||||
|
"source_id": _coalesce(
|
||||||
|
references.get("connectors.source"),
|
||||||
|
references.get("connectors.source_id"),
|
||||||
|
),
|
||||||
|
"acquisition_id": _coalesce(
|
||||||
|
references.get("connectors.acquisition"),
|
||||||
|
references.get("connectors.acquisition_id"),
|
||||||
|
),
|
||||||
|
"definition_id": _coalesce(
|
||||||
|
references.get("connectors.definition"),
|
||||||
|
references.get("connectors.definition_id"),
|
||||||
|
),
|
||||||
|
"configuration_id": _coalesce(
|
||||||
|
references.get("connectors.configuration"),
|
||||||
|
references.get("connectors.configuration_id"),
|
||||||
|
),
|
||||||
|
"simulation_id": _coalesce(
|
||||||
|
references.get("connectors.simulation"),
|
||||||
|
references.get("connectors.simulation_id"),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if account is _CONFLICT or any(value is _CONFLICT for value in values.values()):
|
||||||
|
return None
|
||||||
|
account_id = _optional_string(account)
|
||||||
|
if not account_id:
|
||||||
|
return None
|
||||||
|
return _SubjectSelectors(
|
||||||
|
account_id=account_id,
|
||||||
|
source_id=_optional_string(values["source_id"]),
|
||||||
|
acquisition_id=_optional_string(values["acquisition_id"]),
|
||||||
|
definition_id=_optional_string(values["definition_id"]),
|
||||||
|
configuration_id=_optional_string(values["configuration_id"]),
|
||||||
|
simulation_id=_optional_string(values["simulation_id"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_attribution(row: ConnectorTabularSource, account_id: str) -> DsarRecordRef:
|
||||||
|
activities = []
|
||||||
|
if row.created_by == account_id:
|
||||||
|
activities.append("created_source_snapshot")
|
||||||
|
if row.updated_by == account_id:
|
||||||
|
activities.append("updated_source_snapshot")
|
||||||
|
return _record(
|
||||||
|
resource_type="source_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector source actor attribution",
|
||||||
|
data={
|
||||||
|
"source_id": row.id,
|
||||||
|
"provider": row.provider,
|
||||||
|
"status": row.status,
|
||||||
|
"schema_version": row.schema_version,
|
||||||
|
"row_count": row.row_count,
|
||||||
|
"activities": activities,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
"retired_at": _iso(row.deleted_at),
|
||||||
|
},
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _acquisition_attribution(row: ConnectorSanctionsAcquisitionRun) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="acquisition_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector acquisition actor attribution",
|
||||||
|
data={
|
||||||
|
"acquisition_id": row.id,
|
||||||
|
"provider_id": row.provider_id,
|
||||||
|
"source_id": row.source_id,
|
||||||
|
"status": row.status,
|
||||||
|
"attempt_count": row.attempt_count,
|
||||||
|
"started_at": _iso(row.started_at),
|
||||||
|
"finished_at": _iso(row.finished_at),
|
||||||
|
"activity": "started_source_acquisition",
|
||||||
|
},
|
||||||
|
observed_at=row.finished_at or row.started_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _definition_attribution(
|
||||||
|
row: ConnectorDefinitionRevision, definition: ConnectorDefinition
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="definition_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector definition actor attribution",
|
||||||
|
data={
|
||||||
|
"definition_id": definition.id,
|
||||||
|
"revision_id": row.id,
|
||||||
|
"revision": row.revision,
|
||||||
|
"origin": row.origin,
|
||||||
|
"activity": "created_definition_revision",
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
},
|
||||||
|
observed_at=row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_attribution(row: ConnectorConfiguration) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="configuration_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector configuration actor attribution",
|
||||||
|
data={
|
||||||
|
"configuration_id": row.id,
|
||||||
|
"definition_id": row.definition_id,
|
||||||
|
"status": row.status,
|
||||||
|
"base_definition_revision": row.base_definition_revision,
|
||||||
|
"resource_revision": row.resource_revision,
|
||||||
|
"ambiguity_policy": row.ambiguity_policy,
|
||||||
|
"activity": "updated_connector_configuration",
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
},
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _simulation_attribution(
|
||||||
|
row: ConnectorSimulationRun, account_id: str
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
activities = []
|
||||||
|
if row.created_by == account_id:
|
||||||
|
activities.append("created_simulation")
|
||||||
|
if row.reviewed_by == account_id:
|
||||||
|
activities.append("reviewed_simulation")
|
||||||
|
return _record(
|
||||||
|
resource_type="simulation_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector simulation actor attribution",
|
||||||
|
data={
|
||||||
|
"simulation_id": row.id,
|
||||||
|
"configuration_id": row.configuration_id,
|
||||||
|
"mode": row.mode,
|
||||||
|
"status": row.status,
|
||||||
|
"review_state": row.review_state,
|
||||||
|
"definition_revision": row.definition_revision,
|
||||||
|
"configuration_revision": row.configuration_revision,
|
||||||
|
"activities": activities,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"reviewed_at": _iso(row.reviewed_at),
|
||||||
|
},
|
||||||
|
observed_at=row.reviewed_at or row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _record(
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
title: str,
|
||||||
|
data: dict[str, object],
|
||||||
|
observed_at: datetime | None,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return DsarRecordRef(
|
||||||
|
provider_id="connectors",
|
||||||
|
module_id="connectors",
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
category="connector_governance_attribution",
|
||||||
|
title=title,
|
||||||
|
data=data,
|
||||||
|
observed_at=_aware(observed_at),
|
||||||
|
immutable_evidence=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Connector attribution is retained for configuration, review, and "
|
||||||
|
"external-operation accountability."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _limited(query, first, second, *, label: str):
|
||||||
|
rows = query.order_by(first, second).limit(_MAX_RECORDS + 1).all()
|
||||||
|
if len(rows) > _MAX_RECORDS:
|
||||||
|
raise ValueError(f"Connectors DSAR {label} limit exceeded; narrow selectors.")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _coalesce(*values: str | None) -> str | None | object:
|
||||||
|
normalized = {str(value).strip() for value in values if str(value or "").strip()}
|
||||||
|
if len(normalized) > 1:
|
||||||
|
return _CONFLICT
|
||||||
|
return next(iter(normalized), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_string(value: object) -> str | None:
|
||||||
|
return value if isinstance(value, str) and value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
aware = _aware(value)
|
||||||
|
return aware.isoformat() if aware else None
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None or value.tzinfo is not None:
|
||||||
|
return value
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Connectors DSAR requires a SQLAlchemy Session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
_RESOURCE_TYPES = {
|
||||||
|
"source_actor_attribution",
|
||||||
|
"acquisition_actor_attribution",
|
||||||
|
"definition_actor_attribution",
|
||||||
|
"configuration_actor_attribution",
|
||||||
|
"simulation_actor_attribution",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_record(record: DsarRecordRef) -> None:
|
||||||
|
if record.provider_id != "connectors" or record.module_id != "connectors":
|
||||||
|
raise ValueError("Connectors DSAR cannot plan a foreign provider record.")
|
||||||
|
if record.resource_type not in _RESOURCE_TYPES or not record.resource_id:
|
||||||
|
raise ValueError("Connectors DSAR record identity is invalid.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_action(action: DsarErasureActionRef) -> None:
|
||||||
|
if action.provider_id != "connectors" or action.module_id != "connectors":
|
||||||
|
raise ValueError("Connectors DSAR cannot execute a foreign provider action.")
|
||||||
|
if not action.action_id.startswith("connectors:retain:"):
|
||||||
|
raise ValueError("Connectors DSAR action identity is invalid.")
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["CONNECTORS_DSAR_CAPABILITY", "ConnectorsDsarProvider"]
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import re
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import replace
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from email.utils import format_datetime, parsedate_to_datetime
|
||||||
|
|
||||||
|
from defusedxml import ElementTree as SafeET
|
||||||
|
from defusedxml.common import DefusedXmlException
|
||||||
|
|
||||||
|
from govoplan_core.core.feeds import (
|
||||||
|
FeedCapabilityError,
|
||||||
|
FeedDocument,
|
||||||
|
FeedEntry,
|
||||||
|
FeedProvider,
|
||||||
|
FeedRenderRequest,
|
||||||
|
FeedRenderResult,
|
||||||
|
)
|
||||||
|
from govoplan_core.security.http_fetch import fetch_http
|
||||||
|
|
||||||
|
|
||||||
|
MAX_FEED_BYTES = 5_000_000
|
||||||
|
ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||||
|
FEED_PUBLISH_SCOPE = "connectors:feeds:publish"
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE = "connectors:feeds:publish_private"
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorFeedProvider(FeedProvider):
|
||||||
|
def fetch(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
timeout: float = 15,
|
||||||
|
max_entries: int = 2_000,
|
||||||
|
) -> FeedDocument:
|
||||||
|
try:
|
||||||
|
response = fetch_http(
|
||||||
|
url,
|
||||||
|
timeout=timeout,
|
||||||
|
label="RSS/Atom feed URL",
|
||||||
|
headers={
|
||||||
|
"Accept": (
|
||||||
|
"application/atom+xml, application/rss+xml, "
|
||||||
|
"application/xml;q=0.9, text/xml;q=0.8"
|
||||||
|
)
|
||||||
|
},
|
||||||
|
max_bytes=MAX_FEED_BYTES,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise FeedCapabilityError(f"Feed acquisition failed: {exc}") from exc
|
||||||
|
if response.status < 200 or response.status >= 300:
|
||||||
|
raise FeedCapabilityError(
|
||||||
|
f"Feed acquisition returned HTTP {response.status}."
|
||||||
|
)
|
||||||
|
content_type = _header(response.headers, "content-type")
|
||||||
|
document = self.parse(
|
||||||
|
response.body,
|
||||||
|
source_url=url,
|
||||||
|
content_type=content_type,
|
||||||
|
max_entries=max_entries,
|
||||||
|
)
|
||||||
|
acquired_at = datetime.now(timezone.utc)
|
||||||
|
return replace(
|
||||||
|
document,
|
||||||
|
acquired_at=acquired_at,
|
||||||
|
fresh_until=_fresh_until(response.headers, acquired_at),
|
||||||
|
etag=_header(response.headers, "etag"),
|
||||||
|
last_modified=_header(response.headers, "last-modified"),
|
||||||
|
metadata={
|
||||||
|
**dict(document.metadata),
|
||||||
|
"http_status": response.status,
|
||||||
|
"byte_count": len(response.body),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def parse(
|
||||||
|
self,
|
||||||
|
content: bytes,
|
||||||
|
*,
|
||||||
|
source_url: str,
|
||||||
|
content_type: str | None = None,
|
||||||
|
max_entries: int = 2_000,
|
||||||
|
) -> FeedDocument:
|
||||||
|
if not content:
|
||||||
|
raise FeedCapabilityError("Feed content is empty.")
|
||||||
|
if len(content) > MAX_FEED_BYTES:
|
||||||
|
raise FeedCapabilityError(
|
||||||
|
f"Feeds are limited to {MAX_FEED_BYTES // 1_000_000} MB."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
root = SafeET.fromstring(content)
|
||||||
|
except (ET.ParseError, DefusedXmlException) as exc:
|
||||||
|
raise FeedCapabilityError(f"Feed XML is not safe or valid: {exc}") from exc
|
||||||
|
local_name = _local_name(root.tag)
|
||||||
|
if local_name == "rss":
|
||||||
|
document = _parse_rss(root, source_url=source_url, max_entries=max_entries)
|
||||||
|
elif local_name == "feed":
|
||||||
|
document = _parse_atom(root, source_url=source_url, max_entries=max_entries)
|
||||||
|
else:
|
||||||
|
raise FeedCapabilityError("The document is neither an RSS nor an Atom feed.")
|
||||||
|
return replace(
|
||||||
|
document,
|
||||||
|
content_type=content_type,
|
||||||
|
sha256=hashlib.sha256(content).hexdigest(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def render(self, request: FeedRenderRequest) -> FeedRenderResult:
|
||||||
|
if not request.title.strip() or not request.feed_url.strip():
|
||||||
|
raise FeedCapabilityError("Feed title and feed URL are required.")
|
||||||
|
entries = tuple(
|
||||||
|
entry
|
||||||
|
for entry in request.entries
|
||||||
|
if entry.visibility in request.allowed_visibilities
|
||||||
|
)
|
||||||
|
root = (
|
||||||
|
_render_rss(request, entries)
|
||||||
|
if request.format == "rss"
|
||||||
|
else _render_atom(request, entries)
|
||||||
|
)
|
||||||
|
body = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
||||||
|
return FeedRenderResult(
|
||||||
|
format=request.format,
|
||||||
|
content_type=(
|
||||||
|
"application/rss+xml; charset=utf-8"
|
||||||
|
if request.format == "rss"
|
||||||
|
else "application/atom+xml; charset=utf-8"
|
||||||
|
),
|
||||||
|
body=body,
|
||||||
|
included_entries=len(entries),
|
||||||
|
excluded_entries=len(request.entries) - len(entries),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def feed_rows(document: FeedDocument) -> tuple[Mapping[str, object], ...]:
|
||||||
|
"""Map feed entries to the connector tabular shape used by Datasources."""
|
||||||
|
|
||||||
|
return tuple(
|
||||||
|
{
|
||||||
|
"id": entry.id,
|
||||||
|
"title": entry.title,
|
||||||
|
"url": entry.url,
|
||||||
|
"summary": entry.summary,
|
||||||
|
"content": entry.content,
|
||||||
|
"author": entry.author,
|
||||||
|
"published_at": (
|
||||||
|
entry.published_at.isoformat() if entry.published_at else None
|
||||||
|
),
|
||||||
|
"updated_at": entry.updated_at.isoformat() if entry.updated_at else None,
|
||||||
|
"categories": list(entry.categories),
|
||||||
|
"enclosures": [dict(item) for item in entry.enclosures],
|
||||||
|
}
|
||||||
|
for entry in document.entries
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_rss(root: ET.Element, *, source_url: str, max_entries: int) -> FeedDocument:
|
||||||
|
channel = _first_child(root, "channel")
|
||||||
|
if channel is None:
|
||||||
|
raise FeedCapabilityError("RSS feed is missing its channel element.")
|
||||||
|
entries: list[FeedEntry] = []
|
||||||
|
for item in _children(channel, "item"):
|
||||||
|
if len(entries) >= max_entries:
|
||||||
|
raise FeedCapabilityError(f"Feeds are limited to {max_entries:,} entries.")
|
||||||
|
url = _text(item, "link")
|
||||||
|
identifier = _text(item, "guid") or url or _entry_fallback_id(item)
|
||||||
|
entries.append(
|
||||||
|
FeedEntry(
|
||||||
|
id=identifier,
|
||||||
|
title=_text(item, "title") or "(Untitled)",
|
||||||
|
url=url,
|
||||||
|
summary=_text(item, "description"),
|
||||||
|
content=_text(item, "encoded"),
|
||||||
|
author=_text(item, "author") or _text(item, "creator"),
|
||||||
|
published_at=_parse_date(_text(item, "pubDate")),
|
||||||
|
categories=tuple(
|
||||||
|
value for child in _children(item, "category")
|
||||||
|
if (value := (child.text or "").strip())
|
||||||
|
),
|
||||||
|
enclosures=tuple(
|
||||||
|
{
|
||||||
|
"url": child.attrib.get("url"),
|
||||||
|
"media_type": child.attrib.get("type"),
|
||||||
|
"size_bytes": _integer(child.attrib.get("length")),
|
||||||
|
}
|
||||||
|
for child in _children(item, "enclosure")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return FeedDocument(
|
||||||
|
format="rss",
|
||||||
|
title=_text(channel, "title") or "Untitled feed",
|
||||||
|
source_url=source_url,
|
||||||
|
entries=tuple(entries),
|
||||||
|
description=_text(channel, "description"),
|
||||||
|
home_url=_text(channel, "link"),
|
||||||
|
language=_text(channel, "language"),
|
||||||
|
updated_at=_parse_date(
|
||||||
|
_text(channel, "lastBuildDate") or _text(channel, "pubDate")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_atom(root: ET.Element, *, source_url: str, max_entries: int) -> FeedDocument:
|
||||||
|
entries: list[FeedEntry] = []
|
||||||
|
for item in _children(root, "entry"):
|
||||||
|
if len(entries) >= max_entries:
|
||||||
|
raise FeedCapabilityError(f"Feeds are limited to {max_entries:,} entries.")
|
||||||
|
alternate = _atom_link(item, "alternate")
|
||||||
|
identifier = _text(item, "id") or alternate or _entry_fallback_id(item)
|
||||||
|
author = _first_child(item, "author")
|
||||||
|
entries.append(
|
||||||
|
FeedEntry(
|
||||||
|
id=identifier,
|
||||||
|
title=_text(item, "title") or "(Untitled)",
|
||||||
|
url=alternate,
|
||||||
|
summary=_text(item, "summary"),
|
||||||
|
content=_text(item, "content"),
|
||||||
|
author=_text(author, "name") if author is not None else None,
|
||||||
|
published_at=_parse_date(_text(item, "published")),
|
||||||
|
updated_at=_parse_date(_text(item, "updated")),
|
||||||
|
categories=tuple(
|
||||||
|
value for child in _children(item, "category")
|
||||||
|
if (value := (child.attrib.get("term") or "").strip())
|
||||||
|
),
|
||||||
|
enclosures=tuple(
|
||||||
|
{
|
||||||
|
"url": child.attrib.get("href"),
|
||||||
|
"media_type": child.attrib.get("type"),
|
||||||
|
"size_bytes": _integer(child.attrib.get("length")),
|
||||||
|
}
|
||||||
|
for child in _children(item, "link")
|
||||||
|
if child.attrib.get("rel") == "enclosure"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return FeedDocument(
|
||||||
|
format="atom",
|
||||||
|
title=_text(root, "title") or "Untitled feed",
|
||||||
|
source_url=source_url,
|
||||||
|
entries=tuple(entries),
|
||||||
|
description=_text(root, "subtitle"),
|
||||||
|
home_url=_atom_link(root, "alternate"),
|
||||||
|
updated_at=_parse_date(_text(root, "updated")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_rss(request: FeedRenderRequest, entries: tuple[FeedEntry, ...]) -> ET.Element:
|
||||||
|
ET.register_namespace("atom", ATOM_NS)
|
||||||
|
root = ET.Element("rss", {"version": "2.0"})
|
||||||
|
channel = ET.SubElement(root, "channel")
|
||||||
|
_element(channel, "title", request.title)
|
||||||
|
_element(channel, "link", request.home_url)
|
||||||
|
_element(channel, "description", request.description or request.title)
|
||||||
|
_element(channel, f"{{{ATOM_NS}}}link", None, {
|
||||||
|
"href": request.feed_url,
|
||||||
|
"rel": "self",
|
||||||
|
"type": "application/rss+xml",
|
||||||
|
})
|
||||||
|
if request.language:
|
||||||
|
_element(channel, "language", request.language)
|
||||||
|
for entry in entries:
|
||||||
|
item = ET.SubElement(channel, "item")
|
||||||
|
_element(item, "guid", entry.id, {"isPermaLink": "false"})
|
||||||
|
_element(item, "title", entry.title)
|
||||||
|
if entry.url:
|
||||||
|
_element(item, "link", entry.url)
|
||||||
|
if entry.summary or entry.content:
|
||||||
|
_element(item, "description", entry.summary or entry.content)
|
||||||
|
if entry.author:
|
||||||
|
_element(item, "author", entry.author)
|
||||||
|
date = entry.published_at or entry.updated_at
|
||||||
|
if date:
|
||||||
|
_element(item, "pubDate", format_datetime(_utc(date)))
|
||||||
|
for category in entry.categories:
|
||||||
|
_element(item, "category", category)
|
||||||
|
for enclosure in entry.enclosures:
|
||||||
|
attributes = {
|
||||||
|
"url": str(enclosure.get("url") or ""),
|
||||||
|
"type": str(enclosure.get("media_type") or "application/octet-stream"),
|
||||||
|
"length": str(enclosure.get("size_bytes") or 0),
|
||||||
|
}
|
||||||
|
if attributes["url"]:
|
||||||
|
_element(item, "enclosure", None, attributes)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _render_atom(request: FeedRenderRequest, entries: tuple[FeedEntry, ...]) -> ET.Element:
|
||||||
|
ET.register_namespace("", ATOM_NS)
|
||||||
|
root = ET.Element(f"{{{ATOM_NS}}}feed")
|
||||||
|
_element(root, f"{{{ATOM_NS}}}id", request.feed_url)
|
||||||
|
_element(root, f"{{{ATOM_NS}}}title", request.title)
|
||||||
|
_element(root, f"{{{ATOM_NS}}}link", None, {"href": request.home_url})
|
||||||
|
_element(
|
||||||
|
root,
|
||||||
|
f"{{{ATOM_NS}}}link",
|
||||||
|
None,
|
||||||
|
{"href": request.feed_url, "rel": "self", "type": "application/atom+xml"},
|
||||||
|
)
|
||||||
|
latest = max(
|
||||||
|
(date for entry in entries for date in (entry.updated_at, entry.published_at) if date),
|
||||||
|
default=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
_element(root, f"{{{ATOM_NS}}}updated", _utc(latest).isoformat().replace("+00:00", "Z"))
|
||||||
|
if request.description:
|
||||||
|
_element(root, f"{{{ATOM_NS}}}subtitle", request.description)
|
||||||
|
for value in entries:
|
||||||
|
entry = ET.SubElement(root, f"{{{ATOM_NS}}}entry")
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}id", value.id)
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}title", value.title)
|
||||||
|
if value.url:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}link", None, {"href": value.url})
|
||||||
|
if value.summary:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}summary", value.summary)
|
||||||
|
if value.content:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}content", value.content, {"type": "html"})
|
||||||
|
updated = value.updated_at or value.published_at or latest
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}updated", _utc(updated).isoformat().replace("+00:00", "Z"))
|
||||||
|
if value.published_at:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}published", _utc(value.published_at).isoformat().replace("+00:00", "Z"))
|
||||||
|
if value.author:
|
||||||
|
author = ET.SubElement(entry, f"{{{ATOM_NS}}}author")
|
||||||
|
_element(author, f"{{{ATOM_NS}}}name", value.author)
|
||||||
|
for category in value.categories:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}category", None, {"term": category})
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _children(element: ET.Element, name: str) -> tuple[ET.Element, ...]:
|
||||||
|
return tuple(child for child in element if _local_name(child.tag) == name)
|
||||||
|
|
||||||
|
|
||||||
|
def _first_child(element: ET.Element, name: str) -> ET.Element | None:
|
||||||
|
return next((child for child in element if _local_name(child.tag) == name), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _text(element: ET.Element | None, name: str) -> str | None:
|
||||||
|
if element is None:
|
||||||
|
return None
|
||||||
|
child = _first_child(element, name)
|
||||||
|
if child is None:
|
||||||
|
return None
|
||||||
|
value = "".join(child.itertext()).strip()
|
||||||
|
return value or None
|
||||||
|
|
||||||
|
|
||||||
|
def _local_name(tag: str) -> str:
|
||||||
|
return tag.rsplit("}", 1)[-1].split(":", 1)[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def _atom_link(element: ET.Element, relation: str) -> str | None:
|
||||||
|
for child in _children(element, "link"):
|
||||||
|
if (child.attrib.get("rel") or "alternate") == relation:
|
||||||
|
return child.attrib.get("href")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str | None) -> datetime | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = parsedate_to_datetime(value)
|
||||||
|
except (TypeError, ValueError, OverflowError):
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return _utc(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def _utc(value: datetime) -> datetime:
|
||||||
|
if value.tzinfo is None:
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _integer(value: str | None) -> int | None:
|
||||||
|
try:
|
||||||
|
return int(value) if value is not None else None
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_fallback_id(element: ET.Element) -> str:
|
||||||
|
body = ET.tostring(element, encoding="utf-8")
|
||||||
|
return f"urn:sha256:{hashlib.sha256(body).hexdigest()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _header(headers: Mapping[str, str], name: str) -> str | None:
|
||||||
|
lowered = name.casefold()
|
||||||
|
return next((value for key, value in headers.items() if key.casefold() == lowered), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _fresh_until(headers: Mapping[str, str], acquired_at: datetime) -> datetime | None:
|
||||||
|
cache_control = _header(headers, "cache-control") or ""
|
||||||
|
match = re.search(r"(?:^|,)\s*max-age\s*=\s*(\d+)", cache_control, re.IGNORECASE)
|
||||||
|
if match:
|
||||||
|
return acquired_at + timedelta(seconds=int(match.group(1)))
|
||||||
|
return _parse_date(_header(headers, "expires"))
|
||||||
|
|
||||||
|
|
||||||
|
def _element(
|
||||||
|
parent: ET.Element,
|
||||||
|
tag: str,
|
||||||
|
text: str | None,
|
||||||
|
attributes: Mapping[str, str] | None = None,
|
||||||
|
) -> ET.Element:
|
||||||
|
child = ET.SubElement(parent, tag, dict(attributes or {}))
|
||||||
|
child.text = text
|
||||||
|
return child
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConnectorFeedProvider",
|
||||||
|
"FEED_PRIVATE_PUBLISH_SCOPE",
|
||||||
|
"FEED_PUBLISH_SCOPE",
|
||||||
|
"MAX_FEED_BYTES",
|
||||||
|
"feed_rows",
|
||||||
|
]
|
||||||
@@ -0,0 +1,921 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.connector_runtime import ConnectorContractError, ConnectorEndpoint
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_schemas import (
|
||||||
|
ConnectorConfigurationCreateRequest,
|
||||||
|
ConnectorConfigurationItem,
|
||||||
|
ConnectorConfigurationUpdateRequest,
|
||||||
|
ConnectorDefinitionItem,
|
||||||
|
ConnectorDefinitionUpsertRequest,
|
||||||
|
ConnectorReviewRequest,
|
||||||
|
ConnectorRunItem,
|
||||||
|
ConnectorRunRequest,
|
||||||
|
GovernedConnectorSpecification,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedConnectorError(ValueError):
|
||||||
|
def __init__(self, code: str, message: str) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
def list_definitions(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
) -> list[ConnectorDefinitionItem]:
|
||||||
|
rows = (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.tenant_id == tenant_id,
|
||||||
|
ConnectorDefinition.status == "active",
|
||||||
|
)
|
||||||
|
.order_by(ConnectorDefinition.name.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
definition_item(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
_definition_revision(session, row, row.current_revision),
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_definition(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
payload: ConnectorDefinitionUpsertRequest,
|
||||||
|
) -> ConnectorDefinitionItem:
|
||||||
|
specification = payload.specification.model_dump(mode="json")
|
||||||
|
definition_hash = _hash(specification)
|
||||||
|
definition = (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.tenant_id == principal.tenant_id,
|
||||||
|
ConnectorDefinition.definition_key == payload.definition_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
expected_local = payload.origin == "local"
|
||||||
|
if definition is None:
|
||||||
|
definition = ConnectorDefinition(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_key=payload.definition_key,
|
||||||
|
name=payload.name.strip(),
|
||||||
|
description=_optional_text(payload.description),
|
||||||
|
status="active",
|
||||||
|
current_revision=0,
|
||||||
|
source_package=payload.package_ref,
|
||||||
|
local_definition=expected_local,
|
||||||
|
)
|
||||||
|
session.add(definition)
|
||||||
|
session.flush()
|
||||||
|
elif definition.local_definition != expected_local:
|
||||||
|
if definition.local_definition:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"local_definition_protected",
|
||||||
|
"A package update cannot replace a locally owned connector definition.",
|
||||||
|
)
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"package_definition_requires_overrides",
|
||||||
|
"Use configuration overrides instead of converting a package definition into a local definition.",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
current = _definition_revision(
|
||||||
|
session,
|
||||||
|
definition,
|
||||||
|
definition.current_revision,
|
||||||
|
)
|
||||||
|
if current.definition_hash == definition_hash:
|
||||||
|
return definition_item(session, definition, current)
|
||||||
|
|
||||||
|
definition.name = payload.name.strip()
|
||||||
|
definition.description = _optional_text(payload.description)
|
||||||
|
definition.source_package = payload.package_ref
|
||||||
|
definition.current_revision += 1
|
||||||
|
revision = ConnectorDefinitionRevision(
|
||||||
|
definition_id=definition.id,
|
||||||
|
revision=definition.current_revision,
|
||||||
|
specification=specification,
|
||||||
|
definition_hash=definition_hash,
|
||||||
|
origin=payload.origin,
|
||||||
|
package_ref=payload.package_ref,
|
||||||
|
created_by=principal.user.id,
|
||||||
|
)
|
||||||
|
session.add(revision)
|
||||||
|
session.flush()
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="connectors.definition.revision_created",
|
||||||
|
object_type="connector_definition",
|
||||||
|
object_id=definition.id,
|
||||||
|
details={
|
||||||
|
"definition_key": definition.definition_key,
|
||||||
|
"revision": revision.revision,
|
||||||
|
"definition_hash": definition_hash,
|
||||||
|
"origin": payload.origin,
|
||||||
|
"package_ref": payload.package_ref,
|
||||||
|
"provider": payload.specification.provider,
|
||||||
|
"protocol": payload.specification.protocol,
|
||||||
|
"capabilities": sorted(payload.specification.capabilities),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return definition_item(session, definition, revision)
|
||||||
|
|
||||||
|
|
||||||
|
def list_configurations(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
) -> list[ConnectorConfigurationItem]:
|
||||||
|
rows = (
|
||||||
|
session.query(ConnectorConfiguration)
|
||||||
|
.filter(ConnectorConfiguration.tenant_id == tenant_id)
|
||||||
|
.order_by(ConnectorConfiguration.name.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [configuration_item(session, row) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def create_configuration(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
payload: ConnectorConfigurationCreateRequest,
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
definition = _tenant_definition(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_id=payload.definition_id,
|
||||||
|
)
|
||||||
|
_validate_endpoint(payload.endpoint_url, payload.credential_ref)
|
||||||
|
revision = _definition_revision(session, definition, definition.current_revision)
|
||||||
|
overrides = copy.deepcopy(payload.local_overrides)
|
||||||
|
effective = _effective_specification(revision.specification, overrides)
|
||||||
|
item = ConnectorConfiguration(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_id=definition.id,
|
||||||
|
name=payload.name.strip(),
|
||||||
|
status=payload.status,
|
||||||
|
endpoint_url=_optional_text(payload.endpoint_url),
|
||||||
|
credential_ref=_optional_text(payload.credential_ref),
|
||||||
|
base_definition_revision=definition.current_revision,
|
||||||
|
local_overrides=overrides,
|
||||||
|
protected_paths=_protected_paths(overrides),
|
||||||
|
effective_configuration=effective,
|
||||||
|
effective_hash=_hash(effective),
|
||||||
|
resource_revision=1,
|
||||||
|
ambiguity_policy=payload.ambiguity_policy,
|
||||||
|
updated_by=principal.user.id,
|
||||||
|
)
|
||||||
|
session.add(item)
|
||||||
|
session.flush()
|
||||||
|
_audit_configuration(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
item,
|
||||||
|
action="connectors.configuration.created",
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return configuration_item(session, item)
|
||||||
|
|
||||||
|
|
||||||
|
def update_configuration(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
payload: ConnectorConfigurationUpdateRequest,
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
item = _tenant_configuration(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
if item.resource_revision != payload.expected_revision:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"configuration_conflict",
|
||||||
|
"The connector configuration changed; reload it before saving.",
|
||||||
|
)
|
||||||
|
supplied = payload.model_fields_set
|
||||||
|
definition = _tenant_definition(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_id=item.definition_id,
|
||||||
|
)
|
||||||
|
if "name" in supplied and payload.name is not None:
|
||||||
|
item.name = payload.name.strip()
|
||||||
|
if "endpoint_url" in supplied:
|
||||||
|
item.endpoint_url = _optional_text(payload.endpoint_url)
|
||||||
|
if "credential_ref" in supplied:
|
||||||
|
item.credential_ref = _optional_text(payload.credential_ref)
|
||||||
|
if "local_overrides" in supplied and payload.local_overrides is not None:
|
||||||
|
item.local_overrides = copy.deepcopy(payload.local_overrides)
|
||||||
|
if payload.ambiguity_policy is not None:
|
||||||
|
item.ambiguity_policy = payload.ambiguity_policy
|
||||||
|
if payload.status is not None:
|
||||||
|
item.status = payload.status
|
||||||
|
if payload.adopt_latest_definition:
|
||||||
|
item.base_definition_revision = definition.current_revision
|
||||||
|
_validate_endpoint(item.endpoint_url, item.credential_ref)
|
||||||
|
base = _definition_revision(
|
||||||
|
session,
|
||||||
|
definition,
|
||||||
|
item.base_definition_revision,
|
||||||
|
)
|
||||||
|
effective = _effective_specification(base.specification, item.local_overrides)
|
||||||
|
item.protected_paths = _protected_paths(item.local_overrides)
|
||||||
|
item.effective_configuration = effective
|
||||||
|
item.effective_hash = _hash(effective)
|
||||||
|
item.resource_revision += 1
|
||||||
|
item.updated_by = principal.user.id
|
||||||
|
_audit_configuration(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
item,
|
||||||
|
action="connectors.configuration.updated",
|
||||||
|
extra={"adopted_latest_definition": payload.adopt_latest_definition},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return configuration_item(session, item)
|
||||||
|
|
||||||
|
|
||||||
|
def execute_run(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
mode: str,
|
||||||
|
payload: ConnectorRunRequest,
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
if mode not in {"dry_run", "simulation"}:
|
||||||
|
raise GovernedConnectorError("invalid_mode", "Unsupported connector run mode.")
|
||||||
|
configuration = _tenant_configuration(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
if configuration.status == "disabled":
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"configuration_disabled",
|
||||||
|
"Disabled connector configurations cannot be executed.",
|
||||||
|
)
|
||||||
|
specification = GovernedConnectorSpecification.model_validate(
|
||||||
|
configuration.effective_configuration
|
||||||
|
)
|
||||||
|
if mode == "dry_run" and not specification.dry_run.supported:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"dry_run_unsupported",
|
||||||
|
"This connector definition does not support dry runs.",
|
||||||
|
)
|
||||||
|
if mode == "simulation" and not specification.dry_run.simulation_supported:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"simulation_unsupported",
|
||||||
|
"This connector definition does not support simulation.",
|
||||||
|
)
|
||||||
|
rows = (
|
||||||
|
list(payload.input_rows)
|
||||||
|
if payload.input_rows is not None
|
||||||
|
else list(specification.dry_run.sample_rows)
|
||||||
|
)
|
||||||
|
request_payload = {
|
||||||
|
"mode": mode,
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"external_revision": payload.external_revision,
|
||||||
|
"input_rows": rows,
|
||||||
|
}
|
||||||
|
request_hash = _hash(request_payload)
|
||||||
|
existing = (
|
||||||
|
session.query(ConnectorSimulationRun)
|
||||||
|
.filter(
|
||||||
|
ConnectorSimulationRun.tenant_id == principal.tenant_id,
|
||||||
|
ConnectorSimulationRun.configuration_id == configuration.id,
|
||||||
|
ConnectorSimulationRun.mode == mode,
|
||||||
|
ConnectorSimulationRun.idempotency_key == payload.idempotency_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
if existing.request_hash != request_hash:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"idempotency_conflict",
|
||||||
|
"The idempotency key was already used with different run inputs.",
|
||||||
|
)
|
||||||
|
return run_item(existing)
|
||||||
|
|
||||||
|
limit = specification.dry_run.max_items
|
||||||
|
truncated = len(rows) > limit
|
||||||
|
bounded_rows = rows[:limit]
|
||||||
|
effects, diagnostics, ambiguous_count = _simulate(
|
||||||
|
bounded_rows,
|
||||||
|
specification,
|
||||||
|
)
|
||||||
|
if truncated:
|
||||||
|
diagnostics.append(
|
||||||
|
{
|
||||||
|
"severity": "warning",
|
||||||
|
"code": "connectors.run.truncated",
|
||||||
|
"message": f"The run was limited to {limit} input rows.",
|
||||||
|
"stage": "planning",
|
||||||
|
"retryable": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
errors = sum(1 for item in diagnostics if item["severity"] == "error")
|
||||||
|
if ambiguous_count:
|
||||||
|
status_value, review_state = {
|
||||||
|
"manual_review": ("manual_review", "pending"),
|
||||||
|
"quarantine": ("quarantined", "quarantined"),
|
||||||
|
"reject": ("rejected", "not_required"),
|
||||||
|
}[configuration.ambiguity_policy]
|
||||||
|
elif errors:
|
||||||
|
status_value, review_state = "invalid", "not_required"
|
||||||
|
else:
|
||||||
|
status_value, review_state = "ready", "not_required"
|
||||||
|
input_hash = _hash(bounded_rows)
|
||||||
|
summary = {
|
||||||
|
"total": len(effects),
|
||||||
|
"creates": sum(1 for item in effects if item["effect"] == "create"),
|
||||||
|
"updates": 0,
|
||||||
|
"deletes": 0,
|
||||||
|
"conflicts": sum(1 for item in effects if item["effect"] == "conflict"),
|
||||||
|
"unchanged": 0,
|
||||||
|
"ignored": sum(1 for item in effects if item["effect"] == "ignored"),
|
||||||
|
"ambiguous": ambiguous_count,
|
||||||
|
"errors": errors,
|
||||||
|
"truncated": truncated,
|
||||||
|
}
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
provenance = {
|
||||||
|
"contract_version": "1.0",
|
||||||
|
"definition_id": configuration.definition_id,
|
||||||
|
"definition_revision": configuration.base_definition_revision,
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"mapping_version": specification.mapping.version,
|
||||||
|
"input_hash": input_hash,
|
||||||
|
"external_revision": payload.external_revision,
|
||||||
|
"generated_at": now.isoformat(),
|
||||||
|
"actor_id": principal.user.id,
|
||||||
|
"mode": mode,
|
||||||
|
"provider": specification.provider,
|
||||||
|
"protocol": specification.protocol,
|
||||||
|
"privacy_classification": specification.privacy_classification,
|
||||||
|
"retention_class": specification.retention_class,
|
||||||
|
}
|
||||||
|
run = ConnectorSimulationRun(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode=mode,
|
||||||
|
idempotency_key=payload.idempotency_key,
|
||||||
|
request_hash=request_hash,
|
||||||
|
status=status_value,
|
||||||
|
review_state=review_state,
|
||||||
|
definition_revision=configuration.base_definition_revision,
|
||||||
|
configuration_revision=configuration.resource_revision,
|
||||||
|
configuration_hash=configuration.effective_hash,
|
||||||
|
input_hash=input_hash,
|
||||||
|
summary=summary,
|
||||||
|
effects=effects,
|
||||||
|
diagnostics=diagnostics,
|
||||||
|
provenance=provenance,
|
||||||
|
created_by=principal.user.id,
|
||||||
|
)
|
||||||
|
session.add(run)
|
||||||
|
session.flush()
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=f"connectors.configuration.{mode}_completed",
|
||||||
|
object_type="connector_simulation_run",
|
||||||
|
object_id=run.id,
|
||||||
|
details={
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"definition_revision": configuration.base_definition_revision,
|
||||||
|
"input_hash": input_hash,
|
||||||
|
"status": status_value,
|
||||||
|
"review_state": review_state,
|
||||||
|
"summary": summary,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return run_item(run)
|
||||||
|
|
||||||
|
|
||||||
|
def list_runs(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
configuration_id: str | None = None,
|
||||||
|
review_state: str | None = None,
|
||||||
|
limit: int = 100,
|
||||||
|
) -> list[ConnectorRunItem]:
|
||||||
|
query = session.query(ConnectorSimulationRun).filter(
|
||||||
|
ConnectorSimulationRun.tenant_id == tenant_id
|
||||||
|
)
|
||||||
|
if configuration_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorSimulationRun.configuration_id == configuration_id
|
||||||
|
)
|
||||||
|
if review_state:
|
||||||
|
query = query.filter(ConnectorSimulationRun.review_state == review_state)
|
||||||
|
rows = (
|
||||||
|
query.order_by(ConnectorSimulationRun.created_at.desc())
|
||||||
|
.limit(max(1, min(int(limit), 500)))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [run_item(row) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def review_run(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
run_id: str,
|
||||||
|
payload: ConnectorReviewRequest,
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
run = (
|
||||||
|
session.query(ConnectorSimulationRun)
|
||||||
|
.filter(
|
||||||
|
ConnectorSimulationRun.id == run_id,
|
||||||
|
ConnectorSimulationRun.tenant_id == principal.tenant_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if run is None:
|
||||||
|
raise GovernedConnectorError("run_not_found", "Connector run not found.")
|
||||||
|
if run.review_state not in {"pending", "quarantined"}:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"run_not_reviewable",
|
||||||
|
"Only pending or quarantined connector results can be reviewed.",
|
||||||
|
)
|
||||||
|
run.review_state = payload.decision
|
||||||
|
run.status = f"review_{payload.decision}"
|
||||||
|
run.reviewed_by = principal.user.id
|
||||||
|
run.reviewed_at = datetime.now(UTC)
|
||||||
|
run.review_reason = payload.reason.strip()
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=f"connectors.simulation.{payload.decision}",
|
||||||
|
object_type="connector_simulation_run",
|
||||||
|
object_id=run.id,
|
||||||
|
details={
|
||||||
|
"configuration_id": run.configuration_id,
|
||||||
|
"input_hash": run.input_hash,
|
||||||
|
"configuration_hash": run.configuration_hash,
|
||||||
|
"decision": payload.decision,
|
||||||
|
"reason": run.review_reason,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return run_item(run)
|
||||||
|
|
||||||
|
|
||||||
|
def definition_item(
|
||||||
|
session: Session,
|
||||||
|
definition: ConnectorDefinition,
|
||||||
|
revision: ConnectorDefinitionRevision,
|
||||||
|
) -> ConnectorDefinitionItem:
|
||||||
|
del session
|
||||||
|
return ConnectorDefinitionItem(
|
||||||
|
id=definition.id,
|
||||||
|
tenant_id=definition.tenant_id,
|
||||||
|
definition_key=definition.definition_key,
|
||||||
|
name=definition.name,
|
||||||
|
description=definition.description,
|
||||||
|
status=definition.status,
|
||||||
|
current_revision=definition.current_revision,
|
||||||
|
source_package=definition.source_package,
|
||||||
|
local_definition=definition.local_definition,
|
||||||
|
revision_id=revision.id,
|
||||||
|
definition_hash=revision.definition_hash,
|
||||||
|
origin=revision.origin,
|
||||||
|
package_ref=revision.package_ref,
|
||||||
|
specification=GovernedConnectorSpecification.model_validate(
|
||||||
|
revision.specification
|
||||||
|
),
|
||||||
|
created_at=definition.created_at,
|
||||||
|
updated_at=definition.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def configuration_item(
|
||||||
|
session: Session,
|
||||||
|
item: ConnectorConfiguration,
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
definition = _tenant_definition(
|
||||||
|
session,
|
||||||
|
tenant_id=item.tenant_id,
|
||||||
|
definition_id=item.definition_id,
|
||||||
|
)
|
||||||
|
return ConnectorConfigurationItem(
|
||||||
|
id=item.id,
|
||||||
|
tenant_id=item.tenant_id,
|
||||||
|
definition_id=item.definition_id,
|
||||||
|
definition_key=definition.definition_key,
|
||||||
|
definition_name=definition.name,
|
||||||
|
name=item.name,
|
||||||
|
status=item.status,
|
||||||
|
endpoint_url=item.endpoint_url,
|
||||||
|
credential_ref=item.credential_ref,
|
||||||
|
base_definition_revision=item.base_definition_revision,
|
||||||
|
latest_definition_revision=definition.current_revision,
|
||||||
|
update_available=definition.current_revision > item.base_definition_revision,
|
||||||
|
local_overrides=dict(item.local_overrides or {}),
|
||||||
|
protected_paths=list(item.protected_paths or []),
|
||||||
|
effective_configuration=dict(item.effective_configuration or {}),
|
||||||
|
effective_hash=item.effective_hash,
|
||||||
|
resource_revision=item.resource_revision,
|
||||||
|
ambiguity_policy=item.ambiguity_policy,
|
||||||
|
updated_at=item.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_item(run: ConnectorSimulationRun) -> ConnectorRunItem:
|
||||||
|
return ConnectorRunItem(
|
||||||
|
id=run.id,
|
||||||
|
tenant_id=run.tenant_id,
|
||||||
|
configuration_id=run.configuration_id,
|
||||||
|
mode=run.mode, # type: ignore[arg-type]
|
||||||
|
idempotency_key=run.idempotency_key,
|
||||||
|
status=run.status,
|
||||||
|
review_state=run.review_state,
|
||||||
|
definition_revision=run.definition_revision,
|
||||||
|
configuration_revision=run.configuration_revision,
|
||||||
|
configuration_hash=run.configuration_hash,
|
||||||
|
input_hash=run.input_hash,
|
||||||
|
summary=dict(run.summary or {}),
|
||||||
|
effects=list(run.effects or []),
|
||||||
|
diagnostics=list(run.diagnostics or []),
|
||||||
|
provenance=dict(run.provenance or {}),
|
||||||
|
reviewed_by=run.reviewed_by,
|
||||||
|
reviewed_at=run.reviewed_at,
|
||||||
|
review_reason=run.review_reason,
|
||||||
|
created_at=run.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _simulate(
|
||||||
|
rows: Sequence[Mapping[str, Any]],
|
||||||
|
specification: GovernedConnectorSpecification,
|
||||||
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
|
||||||
|
mapped_rows: list[dict[str, Any]] = []
|
||||||
|
row_errors: dict[int, list[dict[str, Any]]] = {}
|
||||||
|
diagnostics: list[dict[str, Any]] = []
|
||||||
|
for index, row in enumerate(rows):
|
||||||
|
mapped: dict[str, Any] = {}
|
||||||
|
for rule in specification.mapping.rules:
|
||||||
|
value, found = _path_value(row, rule.source)
|
||||||
|
if not found:
|
||||||
|
value = rule.default
|
||||||
|
if rule.required and value in (None, ""):
|
||||||
|
diagnostic = _diagnostic(
|
||||||
|
"error",
|
||||||
|
"connectors.mapping.required_source_missing",
|
||||||
|
f"Required source field {rule.source!r} is missing.",
|
||||||
|
"mapping",
|
||||||
|
index=index,
|
||||||
|
field=rule.source,
|
||||||
|
)
|
||||||
|
row_errors.setdefault(index, []).append(diagnostic)
|
||||||
|
diagnostics.append(diagnostic)
|
||||||
|
_set_path(mapped, rule.target, value)
|
||||||
|
mapped_rows.append(mapped)
|
||||||
|
|
||||||
|
ambiguous_indexes: set[int] = set()
|
||||||
|
for rule in specification.validation_rules:
|
||||||
|
if rule.kind == "unique":
|
||||||
|
seen: dict[str, list[int]] = {}
|
||||||
|
for index, row in enumerate(mapped_rows):
|
||||||
|
value, found = _path_value(row, rule.field)
|
||||||
|
if found and value not in (None, ""):
|
||||||
|
seen.setdefault(_stable_value(value), []).append(index)
|
||||||
|
for indexes in seen.values():
|
||||||
|
if len(indexes) > 1:
|
||||||
|
ambiguous_indexes.update(indexes)
|
||||||
|
for index in indexes:
|
||||||
|
diagnostic = _diagnostic(
|
||||||
|
rule.severity,
|
||||||
|
rule.code,
|
||||||
|
rule.message,
|
||||||
|
"validation",
|
||||||
|
index=index,
|
||||||
|
field=rule.field,
|
||||||
|
)
|
||||||
|
row_errors.setdefault(index, []).append(diagnostic)
|
||||||
|
diagnostics.append(diagnostic)
|
||||||
|
continue
|
||||||
|
for index, row in enumerate(mapped_rows):
|
||||||
|
value, found = _path_value(row, rule.field)
|
||||||
|
invalid = (
|
||||||
|
rule.kind == "required" and (not found or value in (None, ""))
|
||||||
|
) or (
|
||||||
|
rule.kind == "one_of" and found and value not in rule.values
|
||||||
|
)
|
||||||
|
if invalid:
|
||||||
|
diagnostic = _diagnostic(
|
||||||
|
rule.severity,
|
||||||
|
rule.code,
|
||||||
|
rule.message,
|
||||||
|
"validation",
|
||||||
|
index=index,
|
||||||
|
field=rule.field,
|
||||||
|
)
|
||||||
|
row_errors.setdefault(index, []).append(diagnostic)
|
||||||
|
diagnostics.append(diagnostic)
|
||||||
|
|
||||||
|
redacted = set(specification.dry_run.redacted_fields)
|
||||||
|
effects: list[dict[str, Any]] = []
|
||||||
|
for index, mapped in enumerate(mapped_rows):
|
||||||
|
errors = row_errors.get(index, [])
|
||||||
|
has_error = any(item["severity"] == "error" for item in errors)
|
||||||
|
effect = "conflict" if has_error or index in ambiguous_indexes else "create"
|
||||||
|
effects.append(
|
||||||
|
{
|
||||||
|
"effect": effect,
|
||||||
|
"source_object_ref": _source_ref(rows[index], index),
|
||||||
|
"target_object_ref": None,
|
||||||
|
"changed_fields": sorted(_leaf_paths(mapped)),
|
||||||
|
"sample": _redact_fields(mapped, redacted),
|
||||||
|
"reason_code": (
|
||||||
|
"ambiguous_external_result"
|
||||||
|
if index in ambiguous_indexes
|
||||||
|
else errors[0]["code"] if errors else None
|
||||||
|
),
|
||||||
|
"outcome": "preview",
|
||||||
|
"revision": specification.mapping.version,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return effects, diagnostics, len(ambiguous_indexes)
|
||||||
|
|
||||||
|
|
||||||
|
def _audit_configuration(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
item: ConnectorConfiguration,
|
||||||
|
*,
|
||||||
|
action: str,
|
||||||
|
extra: Mapping[str, Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=action,
|
||||||
|
object_type="connector_configuration",
|
||||||
|
object_id=item.id,
|
||||||
|
details={
|
||||||
|
"definition_id": item.definition_id,
|
||||||
|
"base_definition_revision": item.base_definition_revision,
|
||||||
|
"resource_revision": item.resource_revision,
|
||||||
|
"effective_hash": item.effective_hash,
|
||||||
|
"protected_paths": list(item.protected_paths or []),
|
||||||
|
"ambiguity_policy": item.ambiguity_policy,
|
||||||
|
"status": item.status,
|
||||||
|
"credential_reference_present": bool(item.credential_ref),
|
||||||
|
**dict(extra or {}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_definition(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
definition_id: str,
|
||||||
|
) -> ConnectorDefinition:
|
||||||
|
item = (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.id == definition_id,
|
||||||
|
ConnectorDefinition.tenant_id == tenant_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"definition_not_found",
|
||||||
|
"Connector definition not found.",
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_configuration(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
configuration_id: str,
|
||||||
|
) -> ConnectorConfiguration:
|
||||||
|
item = (
|
||||||
|
session.query(ConnectorConfiguration)
|
||||||
|
.filter(
|
||||||
|
ConnectorConfiguration.id == configuration_id,
|
||||||
|
ConnectorConfiguration.tenant_id == tenant_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"configuration_not_found",
|
||||||
|
"Connector configuration not found.",
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _definition_revision(
|
||||||
|
session: Session,
|
||||||
|
definition: ConnectorDefinition,
|
||||||
|
revision: int,
|
||||||
|
) -> ConnectorDefinitionRevision:
|
||||||
|
item = (
|
||||||
|
session.query(ConnectorDefinitionRevision)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinitionRevision.definition_id == definition.id,
|
||||||
|
ConnectorDefinitionRevision.revision == revision,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"definition_revision_not_found",
|
||||||
|
"Connector definition revision not found.",
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _effective_specification(
|
||||||
|
base: Mapping[str, Any],
|
||||||
|
overrides: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
merged = _deep_merge(base, overrides)
|
||||||
|
return GovernedConnectorSpecification.model_validate(merged).model_dump(mode="json")
|
||||||
|
|
||||||
|
|
||||||
|
def _deep_merge(
|
||||||
|
base: Mapping[str, Any],
|
||||||
|
overrides: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result = copy.deepcopy(dict(base))
|
||||||
|
for key, value in overrides.items():
|
||||||
|
if isinstance(value, Mapping) and isinstance(result.get(key), Mapping):
|
||||||
|
result[key] = _deep_merge(result[key], value) # type: ignore[arg-type]
|
||||||
|
else:
|
||||||
|
result[key] = copy.deepcopy(value)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _protected_paths(value: Mapping[str, Any], prefix: str = "") -> list[str]:
|
||||||
|
paths: list[str] = []
|
||||||
|
for key in sorted(value):
|
||||||
|
path = f"{prefix}.{key}" if prefix else str(key)
|
||||||
|
item = value[key]
|
||||||
|
if isinstance(item, Mapping) and item:
|
||||||
|
paths.extend(_protected_paths(item, path))
|
||||||
|
else:
|
||||||
|
paths.append(path)
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def _leaf_paths(value: Mapping[str, Any], prefix: str = "") -> set[str]:
|
||||||
|
paths: set[str] = set()
|
||||||
|
for key, item in value.items():
|
||||||
|
path = f"{prefix}.{key}" if prefix else str(key)
|
||||||
|
if isinstance(item, Mapping) and item:
|
||||||
|
paths.update(_leaf_paths(item, path))
|
||||||
|
else:
|
||||||
|
paths.add(path)
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def _path_value(value: Mapping[str, Any], path: str) -> tuple[Any, bool]:
|
||||||
|
current: Any = value
|
||||||
|
for part in path.split("."):
|
||||||
|
if not isinstance(current, Mapping) or part not in current:
|
||||||
|
return None, False
|
||||||
|
current = current[part]
|
||||||
|
return current, True
|
||||||
|
|
||||||
|
|
||||||
|
def _set_path(target: dict[str, Any], path: str, value: Any) -> None:
|
||||||
|
parts = path.split(".")
|
||||||
|
current = target
|
||||||
|
for part in parts[:-1]:
|
||||||
|
nested = current.get(part)
|
||||||
|
if not isinstance(nested, dict):
|
||||||
|
nested = {}
|
||||||
|
current[part] = nested
|
||||||
|
current = nested
|
||||||
|
current[parts[-1]] = value
|
||||||
|
|
||||||
|
|
||||||
|
def _source_ref(row: Mapping[str, Any], index: int) -> str:
|
||||||
|
for key in ("id", "external_id", "source_id"):
|
||||||
|
value = row.get(key)
|
||||||
|
if value not in (None, ""):
|
||||||
|
return f"sample:{value}"
|
||||||
|
return f"sample:row:{index + 1}"
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_fields(value: Mapping[str, Any], fields: set[str]) -> dict[str, Any]:
|
||||||
|
result = copy.deepcopy(dict(value))
|
||||||
|
for path in fields:
|
||||||
|
parts = path.split(".")
|
||||||
|
current: Any = result
|
||||||
|
for part in parts[:-1]:
|
||||||
|
if not isinstance(current, dict):
|
||||||
|
break
|
||||||
|
current = current.get(part)
|
||||||
|
else:
|
||||||
|
if isinstance(current, dict) and parts[-1] in current:
|
||||||
|
current[parts[-1]] = "<redacted>"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _diagnostic(
|
||||||
|
severity: str,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
stage: str,
|
||||||
|
**details: Any,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"severity": severity,
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
"stage": stage,
|
||||||
|
"retryable": False,
|
||||||
|
"details": details,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_endpoint(
|
||||||
|
endpoint_url: str | None,
|
||||||
|
credential_ref: str | None,
|
||||||
|
) -> None:
|
||||||
|
if endpoint_url:
|
||||||
|
try:
|
||||||
|
ConnectorEndpoint(
|
||||||
|
url=endpoint_url,
|
||||||
|
credential_ref=_optional_text(credential_ref),
|
||||||
|
)
|
||||||
|
except ConnectorContractError as exc:
|
||||||
|
raise GovernedConnectorError("invalid_endpoint", str(exc)) from exc
|
||||||
|
elif credential_ref:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"credential_without_endpoint",
|
||||||
|
"A credential reference requires a configured endpoint.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _stable_value(value: Any) -> str:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||||
|
|
||||||
|
|
||||||
|
def _hash(value: Any) -> str:
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: object | None) -> str | None:
|
||||||
|
normalized = str(value or "").strip()
|
||||||
|
return normalized or None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"GovernedConnectorError",
|
||||||
|
"configuration_item",
|
||||||
|
"create_configuration",
|
||||||
|
"execute_run",
|
||||||
|
"list_configurations",
|
||||||
|
"list_definitions",
|
||||||
|
"list_runs",
|
||||||
|
"review_run",
|
||||||
|
"run_item",
|
||||||
|
"update_configuration",
|
||||||
|
"upsert_definition",
|
||||||
|
]
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorMappingRule(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
source: str = Field(min_length=1, max_length=255)
|
||||||
|
target: str = Field(min_length=1, max_length=255)
|
||||||
|
required: bool = False
|
||||||
|
default: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorMappingDefinition(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
version: str = Field(min_length=1, max_length=100)
|
||||||
|
rules: list[ConnectorMappingRule] = Field(default_factory=list, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorValidationRule(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
kind: Literal["required", "one_of", "unique"]
|
||||||
|
field: str = Field(min_length=1, max_length=255)
|
||||||
|
values: list[Any] = Field(default_factory=list, max_length=500)
|
||||||
|
severity: Literal["warning", "error"] = "error"
|
||||||
|
code: str = Field(min_length=1, max_length=120)
|
||||||
|
message: str = Field(min_length=1, max_length=500)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_values(self) -> "ConnectorValidationRule":
|
||||||
|
if self.kind == "one_of" and not self.values:
|
||||||
|
raise ValueError("one_of validation requires allowed values")
|
||||||
|
if self.kind != "one_of" and self.values:
|
||||||
|
raise ValueError("Only one_of validation accepts values")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDryRunMetadata(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
supported: bool = True
|
||||||
|
simulation_supported: bool = True
|
||||||
|
sample_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=500)
|
||||||
|
max_items: int = Field(default=500, ge=1, le=10_000)
|
||||||
|
redacted_fields: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorAuditMetadata(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
event_prefix: str = Field(min_length=1, max_length=120)
|
||||||
|
expected_events: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
evidence_fields: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedConnectorSpecification(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
provider: str = Field(min_length=1, max_length=120)
|
||||||
|
protocol: str = Field(min_length=1, max_length=80)
|
||||||
|
capabilities: list[str] = Field(min_length=1, max_length=100)
|
||||||
|
input_schema: dict[str, Any]
|
||||||
|
output_schema: dict[str, Any]
|
||||||
|
mapping: ConnectorMappingDefinition
|
||||||
|
validation_rules: list[ConnectorValidationRule] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
max_length=500,
|
||||||
|
)
|
||||||
|
dry_run: ConnectorDryRunMetadata
|
||||||
|
audit: ConnectorAuditMetadata
|
||||||
|
privacy_classification: Literal[
|
||||||
|
"public",
|
||||||
|
"internal",
|
||||||
|
"confidential",
|
||||||
|
"restricted",
|
||||||
|
] = "internal"
|
||||||
|
retention_class: str = Field(min_length=1, max_length=120)
|
||||||
|
operational_limits: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
retry_policy: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionUpsertRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
definition_key: str = Field(pattern=r"^[a-z0-9][a-z0-9._-]{0,159}$")
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
origin: Literal["package", "local"] = "local"
|
||||||
|
package_ref: str | None = Field(default=None, max_length=300)
|
||||||
|
specification: GovernedConnectorSpecification
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def package_provenance(self) -> "ConnectorDefinitionUpsertRequest":
|
||||||
|
if self.origin == "package" and not self.package_ref:
|
||||||
|
raise ValueError("Package definitions require package_ref")
|
||||||
|
if self.origin == "local" and self.package_ref:
|
||||||
|
raise ValueError("Local definitions cannot claim package_ref")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
definition_key: str
|
||||||
|
name: str
|
||||||
|
description: str | None = None
|
||||||
|
status: str
|
||||||
|
current_revision: int
|
||||||
|
source_package: str | None = None
|
||||||
|
local_definition: bool
|
||||||
|
revision_id: str
|
||||||
|
definition_hash: str
|
||||||
|
origin: str
|
||||||
|
package_ref: str | None = None
|
||||||
|
specification: GovernedConnectorSpecification
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionListResponse(BaseModel):
|
||||||
|
items: list[ConnectorDefinitionItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationCreateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
definition_id: str = Field(min_length=1, max_length=36)
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
endpoint_url: str | None = Field(default=None, max_length=1500)
|
||||||
|
credential_ref: str | None = Field(default=None, max_length=500)
|
||||||
|
local_overrides: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
ambiguity_policy: Literal["manual_review", "quarantine", "reject"] = (
|
||||||
|
"manual_review"
|
||||||
|
)
|
||||||
|
status: Literal["draft", "active", "disabled"] = "draft"
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationUpdateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
name: str | None = Field(default=None, min_length=1, max_length=300)
|
||||||
|
endpoint_url: str | None = Field(default=None, max_length=1500)
|
||||||
|
credential_ref: str | None = Field(default=None, max_length=500)
|
||||||
|
local_overrides: dict[str, Any] | None = None
|
||||||
|
ambiguity_policy: Literal["manual_review", "quarantine", "reject"] | None = None
|
||||||
|
status: Literal["draft", "active", "disabled"] | None = None
|
||||||
|
adopt_latest_definition: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
definition_id: str
|
||||||
|
definition_key: str
|
||||||
|
definition_name: str
|
||||||
|
name: str
|
||||||
|
status: str
|
||||||
|
endpoint_url: str | None = None
|
||||||
|
credential_ref: str | None = None
|
||||||
|
base_definition_revision: int
|
||||||
|
latest_definition_revision: int
|
||||||
|
update_available: bool
|
||||||
|
local_overrides: dict[str, Any]
|
||||||
|
protected_paths: list[str]
|
||||||
|
effective_configuration: dict[str, Any]
|
||||||
|
effective_hash: str
|
||||||
|
resource_revision: int
|
||||||
|
ambiguity_policy: str
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationListResponse(BaseModel):
|
||||||
|
items: list[ConnectorConfigurationItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRunRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
input_rows: list[dict[str, Any]] | None = Field(default=None, max_length=500)
|
||||||
|
external_revision: str | None = Field(default=None, max_length=255)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRunItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
configuration_id: str
|
||||||
|
mode: Literal["dry_run", "simulation"]
|
||||||
|
idempotency_key: str
|
||||||
|
status: str
|
||||||
|
review_state: str
|
||||||
|
definition_revision: int
|
||||||
|
configuration_revision: int
|
||||||
|
configuration_hash: str
|
||||||
|
input_hash: str
|
||||||
|
summary: dict[str, Any]
|
||||||
|
effects: list[dict[str, Any]]
|
||||||
|
diagnostics: list[dict[str, Any]]
|
||||||
|
provenance: dict[str, Any]
|
||||||
|
reviewed_by: str | None = None
|
||||||
|
reviewed_at: datetime | None = None
|
||||||
|
review_reason: str | None = None
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRunListResponse(BaseModel):
|
||||||
|
items: list[ConnectorRunItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorReviewRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
decision: Literal["approved", "rejected"]
|
||||||
|
reason: str = Field(min_length=5, max_length=1000)
|
||||||
@@ -0,0 +1,726 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from govoplan_core.core.access import (
|
||||||
|
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||||
|
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.module_guards import (
|
||||||
|
drop_table_retirement_provider,
|
||||||
|
persistent_table_uninstall_guard,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.datasources import CAPABILITY_DATASOURCE_ORIGINS
|
||||||
|
from govoplan_core.core.feeds import CAPABILITY_CONNECTORS_FEEDS
|
||||||
|
from govoplan_core.core.modules import (
|
||||||
|
CapabilityDocumentation,
|
||||||
|
DocumentationTopic,
|
||||||
|
FrontendModule,
|
||||||
|
MigrationSpec,
|
||||||
|
ModuleInterfaceProvider,
|
||||||
|
ModuleManifest,
|
||||||
|
PermissionDefinition,
|
||||||
|
RoleTemplate,
|
||||||
|
ViewSurface,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.provider_governance import (
|
||||||
|
ExternalProviderDeclaration,
|
||||||
|
ExternalProviderStateProviderRegistration,
|
||||||
|
ModuleArchitectureDeclaration,
|
||||||
|
ModuleArchitectureDocumentation,
|
||||||
|
ModuleMaturityEvidence,
|
||||||
|
ProviderBehaviorDeclaration,
|
||||||
|
ProviderObjectDeclaration,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.sanctions import (
|
||||||
|
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.dsar_provider import (
|
||||||
|
CONNECTORS_DSAR_CAPABILITY,
|
||||||
|
ConnectorsDsarProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.sanctions_sources import (
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
SqlSanctionsSnapshotProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.tabular_sources import (
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
READ_SCOPE,
|
||||||
|
WRITE_SCOPE,
|
||||||
|
SqlTabularSourceProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.datasource_origins import (
|
||||||
|
ConnectorDatasourceOriginProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.feeds import (
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
ConnectorFeedProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.provider_state import (
|
||||||
|
SANCTIONS_PROVIDER_ID,
|
||||||
|
TABULAR_PROVIDER_ID,
|
||||||
|
sanctions_provider_states,
|
||||||
|
tabular_provider_states,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
MODULE_ID = "connectors"
|
||||||
|
MODULE_VERSION = "0.1.19"
|
||||||
|
TABULAR_SOURCE_INTERFACE_VERSION = "0.1.0"
|
||||||
|
DATASOURCE_ORIGIN_INTERFACE_VERSION = "0.1.0"
|
||||||
|
SANCTIONS_SNAPSHOT_INTERFACE_VERSION = "1.0.0"
|
||||||
|
FEED_INTERFACE_VERSION = "0.2.0"
|
||||||
|
CONNECTOR_RUNTIME_INTERFACE_VERSION = "1.0.0"
|
||||||
|
|
||||||
|
ARCHITECTURE = ModuleArchitectureDeclaration(
|
||||||
|
layer="data_reporting_integration",
|
||||||
|
kind="integration",
|
||||||
|
maturity="vertical_slice",
|
||||||
|
evidence=(
|
||||||
|
ModuleMaturityEvidence(
|
||||||
|
kind="test",
|
||||||
|
reference="tests/test_tabular_sources.py",
|
||||||
|
summary="Exercises tenant-safe immutable tabular snapshots and bounded reads.",
|
||||||
|
),
|
||||||
|
ModuleMaturityEvidence(
|
||||||
|
kind="test",
|
||||||
|
reference="tests/test_sanctions_sources.py",
|
||||||
|
summary="Exercises source acquisition health, checksums, retries, and immutable evidence.",
|
||||||
|
),
|
||||||
|
ModuleMaturityEvidence(
|
||||||
|
kind="recovery",
|
||||||
|
reference="tests/test_recovery.py",
|
||||||
|
summary="Proves atomic snapshot commits, idempotent replay, distributed fences, tamper rejection, and unknown external-effect handling.",
|
||||||
|
),
|
||||||
|
ModuleMaturityEvidence(
|
||||||
|
kind="test",
|
||||||
|
reference="tests/test_governed_runtime.py",
|
||||||
|
summary="Exercises immutable definition revisions, protected local overrides, idempotent simulations, and explicit ambiguity review.",
|
||||||
|
),
|
||||||
|
ModuleMaturityEvidence(
|
||||||
|
kind="documentation",
|
||||||
|
reference="docs/CONNECTOR_SOURCE_LIFECYCLE.md",
|
||||||
|
summary="Defines source lifecycle, authority, evidence, and outage boundaries.",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
known_limits=(
|
||||||
|
"The executable generic datasource origin is an immutable tabular snapshot; database and arbitrary REST profiles remain future providers.",
|
||||||
|
"Feed publication renders a governed document but does not yet push it to an external publishing endpoint.",
|
||||||
|
"The generic governed runtime simulates deterministic mapping and validation; provider-specific live writes remain owned by explicit connector adapters.",
|
||||||
|
),
|
||||||
|
supported_authority_modes=(
|
||||||
|
"external_authoritative",
|
||||||
|
"external_mirror",
|
||||||
|
"linked_reference",
|
||||||
|
),
|
||||||
|
owned_concepts=(
|
||||||
|
"external transport profiles",
|
||||||
|
"protocol interaction",
|
||||||
|
"immutable connector snapshots",
|
||||||
|
"connector acquisition health",
|
||||||
|
),
|
||||||
|
non_owned_concepts=(
|
||||||
|
"datasource catalogue identity and lifecycle",
|
||||||
|
"domain records and business semantics",
|
||||||
|
"data transformations",
|
||||||
|
"screening dispositions",
|
||||||
|
),
|
||||||
|
target_tested_providers=(
|
||||||
|
TABULAR_PROVIDER_ID,
|
||||||
|
SANCTIONS_PROVIDER_ID,
|
||||||
|
),
|
||||||
|
documentation=ModuleArchitectureDocumentation(
|
||||||
|
migration=("src/govoplan_connectors/backend/migrations/versions",),
|
||||||
|
upgrade=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
|
||||||
|
recovery=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
|
||||||
|
security=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
|
||||||
|
operations=("docs/CONNECTOR_SOURCE_LIFECYCLE.md",),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
EXTERNAL_PROVIDERS = (
|
||||||
|
ExternalProviderDeclaration(
|
||||||
|
id=TABULAR_PROVIDER_ID,
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
label="Immutable tabular snapshot provider",
|
||||||
|
maturity="read",
|
||||||
|
operations=("discover", "search", "read", "preview", "dry_run"),
|
||||||
|
objects=(
|
||||||
|
ProviderObjectDeclaration(
|
||||||
|
object_type="tabular_source_snapshot",
|
||||||
|
field_groups=("identity", "schema", "rows", "source_provenance"),
|
||||||
|
authority_modes=("external_authoritative", "external_mirror"),
|
||||||
|
default_authority_mode="external_mirror",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
behavior=ProviderBehaviorDeclaration(
|
||||||
|
revision_tokens="Source fingerprints and immutable snapshot ids are retained.",
|
||||||
|
concurrency="Reads may require the expected fingerprint; snapshots never mutate in place.",
|
||||||
|
freshness="Snapshot acquisition time and source timestamp are exposed.",
|
||||||
|
health="Import validation and source-read failures are explicit.",
|
||||||
|
max_read_items=1000,
|
||||||
|
idempotency="Feed imports accept a caller request key and replay the same committed immutable source without refetching.",
|
||||||
|
retry="Read-only acquisition may be retried only as a new deliberate request after a failed atomic operation.",
|
||||||
|
outcome_unknown="Provider reads do not mutate remote state; an uncertain database commit is resolved by the atomic recovery transaction.",
|
||||||
|
outcome_unknown_supported=False,
|
||||||
|
evidence="Rows, schema, fingerprint, source metadata, and acquisition provenance remain linked.",
|
||||||
|
correction="Import a replacement snapshot; retain the prior snapshot as evidence.",
|
||||||
|
rollback="Snapshot rows and the terminal recovery checkpoint commit or roll back together.",
|
||||||
|
reconciliation="Compare source and snapshot fingerprints before selecting a new current state.",
|
||||||
|
outage="Existing snapshots remain available and visibly stale; no live-source claim is made.",
|
||||||
|
classifications=("internal", "confidential", "restricted"),
|
||||||
|
purposes=("governed import", "dataflow input", "evidence reconstruction"),
|
||||||
|
retention="Datasources or the consuming domain supplies retention and hold policy.",
|
||||||
|
secret_handling="Generic snapshots contain no connector credential; transport credentials stay in credential envelopes.",
|
||||||
|
),
|
||||||
|
capability_names=(
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||||
|
CAPABILITY_DATASOURCE_ORIGINS,
|
||||||
|
),
|
||||||
|
interface_names=(
|
||||||
|
"connectors.tabular_sources",
|
||||||
|
"connectors.datasource_origins",
|
||||||
|
),
|
||||||
|
documentation_topic_ids=(
|
||||||
|
"connectors.authority-and-effects",
|
||||||
|
"connectors.tabular-sources",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
ExternalProviderDeclaration(
|
||||||
|
id=SANCTIONS_PROVIDER_ID,
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
label="Sanctions source snapshot provider",
|
||||||
|
maturity="read",
|
||||||
|
operations=("discover", "search", "read", "preview"),
|
||||||
|
objects=(
|
||||||
|
ProviderObjectDeclaration(
|
||||||
|
object_type="sanctions_source_snapshot",
|
||||||
|
field_groups=(
|
||||||
|
"source_identity",
|
||||||
|
"raw_evidence",
|
||||||
|
"entries",
|
||||||
|
"acquisition_health",
|
||||||
|
),
|
||||||
|
authority_modes=("external_authoritative", "external_mirror"),
|
||||||
|
default_authority_mode="external_mirror",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
behavior=ProviderBehaviorDeclaration(
|
||||||
|
revision_tokens="Provider source version, ETag, Last-Modified, and SHA-256 digest are retained when available.",
|
||||||
|
concurrency="Refreshes use conditional source requests, a distributed per-tenant/provider fence, and immutable snapshots.",
|
||||||
|
freshness="Latest successful acquisition, source timestamp, and stale health are reported.",
|
||||||
|
health="Transport, parsing, source-change, and malformed-source states are explicit.",
|
||||||
|
max_read_items=5000,
|
||||||
|
idempotency="A caller request key identifies one acquisition run and replays its committed result without contacting the source again.",
|
||||||
|
retry="Bounded HTTP retries are safe because acquisition is read-only; failed runs require a new deliberate request key.",
|
||||||
|
timeout_seconds=30,
|
||||||
|
outcome_unknown="The external operation is read-only; snapshot rows and recovery evidence commit atomically.",
|
||||||
|
outcome_unknown_supported=False,
|
||||||
|
evidence="Raw source bytes, checksum, acquisition run, parser result, and normalized entry count are linked.",
|
||||||
|
correction="A corrected source creates a new immutable snapshot and acquisition run.",
|
||||||
|
rollback="A failed database transaction leaves no snapshot and the stale atomic fence resolves as failed.",
|
||||||
|
reconciliation="Compare source version and digest, then preserve both prior and corrected evidence.",
|
||||||
|
outage="The latest accepted snapshot stays usable with stale/unavailable source health.",
|
||||||
|
classifications=("public", "internal"),
|
||||||
|
purposes=("sanctions source acquisition", "compliance screening evidence"),
|
||||||
|
retention="Risk and Records policies determine accepted snapshot retention and legal holds.",
|
||||||
|
secret_handling="Public sources require no subject data or source credential; configured proxy secrets remain external to snapshots.",
|
||||||
|
),
|
||||||
|
capability_names=(CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,),
|
||||||
|
interface_names=("connectors.sanctions_snapshots",),
|
||||||
|
documentation_topic_ids=(
|
||||||
|
"connectors.authority-and-effects",
|
||||||
|
"connectors.sanctions-snapshots",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
|
||||||
|
module_id, resource, action = scope.split(":", 2)
|
||||||
|
return PermissionDefinition(
|
||||||
|
scope=scope,
|
||||||
|
label=label,
|
||||||
|
description=description,
|
||||||
|
category="Connectors",
|
||||||
|
level="tenant",
|
||||||
|
module_id=module_id,
|
||||||
|
resource=resource,
|
||||||
|
action=action,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
PERMISSIONS = (
|
||||||
|
_permission(
|
||||||
|
READ_SCOPE,
|
||||||
|
"View tabular sources",
|
||||||
|
"Discover and preview policy-visible tabular connector sources.",
|
||||||
|
),
|
||||||
|
_permission(
|
||||||
|
WRITE_SCOPE,
|
||||||
|
"Manage tabular sources",
|
||||||
|
"Import and retire bounded tabular snapshots.",
|
||||||
|
),
|
||||||
|
_permission(
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
"Administer connector sources",
|
||||||
|
"Manage tenant connector sources, versioned definitions, protected overrides, and review policies.",
|
||||||
|
),
|
||||||
|
_permission(
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
"Publish public feeds",
|
||||||
|
"Render provenance-bearing public RSS or Atom output from selected GovOPlaN objects.",
|
||||||
|
),
|
||||||
|
_permission(
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
"Publish restricted feeds",
|
||||||
|
"Render tenant or private RSS or Atom output after the owning product surface has authorized every selected object.",
|
||||||
|
),
|
||||||
|
_permission(
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
"View sanctions source evidence",
|
||||||
|
"Inspect immutable sanctions snapshots and acquisition health.",
|
||||||
|
),
|
||||||
|
_permission(
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
"Refresh sanctions sources",
|
||||||
|
"Acquire a new immutable sanctions source snapshot.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
ROLE_TEMPLATES = (
|
||||||
|
RoleTemplate(
|
||||||
|
slug="connector_administrator",
|
||||||
|
name="Connector administrator",
|
||||||
|
description="Govern connector definitions, local configurations, simulations, and manual review.",
|
||||||
|
permissions=(
|
||||||
|
READ_SCOPE,
|
||||||
|
WRITE_SCOPE,
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
RoleTemplate(
|
||||||
|
slug="connector_source_manager",
|
||||||
|
name="Connector source manager",
|
||||||
|
description="Discover, import, preview, and retire tabular sources.",
|
||||||
|
permissions=(
|
||||||
|
READ_SCOPE,
|
||||||
|
WRITE_SCOPE,
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
RoleTemplate(
|
||||||
|
slug="connector_source_reader",
|
||||||
|
name="Connector source reader",
|
||||||
|
description="Discover and preview tabular connector sources.",
|
||||||
|
permissions=(READ_SCOPE, SANCTIONS_READ_SCOPE),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _router(_context):
|
||||||
|
from govoplan_connectors.backend.router import router
|
||||||
|
|
||||||
|
return router
|
||||||
|
|
||||||
|
|
||||||
|
def _provider(_context) -> SqlTabularSourceProvider:
|
||||||
|
return SqlTabularSourceProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _datasource_origin_provider(_context) -> ConnectorDatasourceOriginProvider:
|
||||||
|
return ConnectorDatasourceOriginProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _sanctions_snapshot_provider(
|
||||||
|
_context,
|
||||||
|
) -> SqlSanctionsSnapshotProvider:
|
||||||
|
return SqlSanctionsSnapshotProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _feed_provider(_context) -> ConnectorFeedProvider:
|
||||||
|
return ConnectorFeedProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _dsar_provider(_context) -> ConnectorsDsarProvider:
|
||||||
|
return ConnectorsDsarProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
||||||
|
return {
|
||||||
|
"connector_definitions": (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(ConnectorDefinition.tenant_id == tenant_id)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"connector_configurations": (
|
||||||
|
session.query(ConnectorConfiguration)
|
||||||
|
.filter(ConnectorConfiguration.tenant_id == tenant_id)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"connector_simulation_runs": (
|
||||||
|
session.query(ConnectorSimulationRun)
|
||||||
|
.filter(ConnectorSimulationRun.tenant_id == tenant_id)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"connector_tabular_sources": (
|
||||||
|
session.query(ConnectorTabularSource)
|
||||||
|
.filter(
|
||||||
|
ConnectorTabularSource.tenant_id == tenant_id,
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"connector_sanctions_snapshots": (
|
||||||
|
session.query(ConnectorSanctionsSnapshot)
|
||||||
|
.filter(ConnectorSanctionsSnapshot.tenant_id == tenant_id)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"connector_sanctions_runs": (
|
||||||
|
session.query(ConnectorSanctionsAcquisitionRun)
|
||||||
|
.filter(ConnectorSanctionsAcquisitionRun.tenant_id == tenant_id)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
manifest = ModuleManifest(
|
||||||
|
id=MODULE_ID,
|
||||||
|
name="Connectors",
|
||||||
|
version=MODULE_VERSION,
|
||||||
|
optional_dependencies=(
|
||||||
|
"access",
|
||||||
|
"audit",
|
||||||
|
"files",
|
||||||
|
"policy",
|
||||||
|
"datasources",
|
||||||
|
"portal",
|
||||||
|
"reporting",
|
||||||
|
"risk_compliance",
|
||||||
|
),
|
||||||
|
required_capabilities=(
|
||||||
|
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||||
|
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||||
|
),
|
||||||
|
provides_interfaces=(
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="connectors.tabular_sources",
|
||||||
|
version=TABULAR_SOURCE_INTERFACE_VERSION,
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="connectors.tabular_snapshot_writer",
|
||||||
|
version=TABULAR_SOURCE_INTERFACE_VERSION,
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="connectors.datasource_origins",
|
||||||
|
version=DATASOURCE_ORIGIN_INTERFACE_VERSION,
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="connectors.sanctions_snapshots",
|
||||||
|
version=SANCTIONS_SNAPSHOT_INTERFACE_VERSION,
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="connectors.feeds",
|
||||||
|
version=FEED_INTERFACE_VERSION,
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="connectors.runtime_contract",
|
||||||
|
version=CONNECTOR_RUNTIME_INTERFACE_VERSION,
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(name=CONNECTORS_DSAR_CAPABILITY, version="0.1.0"),
|
||||||
|
),
|
||||||
|
permissions=PERMISSIONS,
|
||||||
|
role_templates=ROLE_TEMPLATES,
|
||||||
|
route_factory=_router,
|
||||||
|
frontend=FrontendModule(
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
package_name="@govoplan/connectors-webui",
|
||||||
|
view_surfaces=(
|
||||||
|
ViewSurface(
|
||||||
|
id="connectors.admin.governed-configurations",
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
kind="section",
|
||||||
|
label="Connector governance",
|
||||||
|
order=45,
|
||||||
|
),
|
||||||
|
ViewSurface(
|
||||||
|
id="connectors.admin.simulation-review",
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
kind="section",
|
||||||
|
label="Connector simulation review",
|
||||||
|
parent_id="connectors.admin.governed-configurations",
|
||||||
|
order=20,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
capability_factories={
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES: _provider,
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER: _provider,
|
||||||
|
CAPABILITY_DATASOURCE_ORIGINS: _datasource_origin_provider,
|
||||||
|
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS: (_sanctions_snapshot_provider),
|
||||||
|
CAPABILITY_CONNECTORS_FEEDS: _feed_provider,
|
||||||
|
CONNECTORS_DSAR_CAPABILITY: _dsar_provider,
|
||||||
|
},
|
||||||
|
capability_documentation={
|
||||||
|
CONNECTORS_DSAR_CAPABILITY: CapabilityDocumentation(
|
||||||
|
label="Connector data-subject request provider",
|
||||||
|
summary=(
|
||||||
|
"Exports minimized operator attribution without connector secrets, "
|
||||||
|
"external payloads, or transport evidence."
|
||||||
|
),
|
||||||
|
contract_version="0.1.0",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
tenant_summary_providers=(_tenant_summary,),
|
||||||
|
architecture=ARCHITECTURE,
|
||||||
|
external_providers=EXTERNAL_PROVIDERS,
|
||||||
|
external_provider_state_providers=(
|
||||||
|
ExternalProviderStateProviderRegistration(
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
provider_id=TABULAR_PROVIDER_ID,
|
||||||
|
provider=tabular_provider_states,
|
||||||
|
),
|
||||||
|
ExternalProviderStateProviderRegistration(
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
provider_id=SANCTIONS_PROVIDER_ID,
|
||||||
|
provider=sanctions_provider_states,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migration_spec=MigrationSpec(
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
metadata=Base.metadata,
|
||||||
|
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||||
|
retirement_supported=True,
|
||||||
|
retirement_provider=drop_table_retirement_provider(
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
label="Connectors",
|
||||||
|
),
|
||||||
|
retirement_notes=(
|
||||||
|
"Destructive retirement drops connector-owned source snapshots after "
|
||||||
|
"the installer captures a database snapshot."
|
||||||
|
),
|
||||||
|
),
|
||||||
|
uninstall_guard_providers=(
|
||||||
|
persistent_table_uninstall_guard(
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
label="Connectors",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
documentation=(
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.data-subject-requests",
|
||||||
|
title="Connector data-subject requests",
|
||||||
|
summary=(
|
||||||
|
"Export accountable connector activity without disclosing credentials "
|
||||||
|
"or external data."
|
||||||
|
),
|
||||||
|
body=(
|
||||||
|
"Connectors correlates only an exact tenant account identifier and can "
|
||||||
|
"narrow an already verified search to one source, acquisition, "
|
||||||
|
"definition, configuration, or simulation. The export identifies the "
|
||||||
|
"subject's configuration, acquisition, simulation, and review activity "
|
||||||
|
"using bounded lifecycle metadata. It never includes credential or "
|
||||||
|
"endpoint references, source rows, external responses, request payloads, "
|
||||||
|
"mapping and configuration documents, diagnostics, provenance, hashes, "
|
||||||
|
"or transport evidence. Connector attribution remains immutable "
|
||||||
|
"governance and external-operation evidence and is retained rather than "
|
||||||
|
"automatically erased. Email or object identifiers without a verified "
|
||||||
|
"account identifier do not establish a match."
|
||||||
|
),
|
||||||
|
layer="configured",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("user", "operator", "module_admin", "auditor"),
|
||||||
|
related_modules=("core", "access", "audit", "policy"),
|
||||||
|
order=37,
|
||||||
|
metadata={
|
||||||
|
"help_contexts": ["privacy.data-subject-requests"],
|
||||||
|
"consequence_classes": {
|
||||||
|
"export_operator_attribution": (
|
||||||
|
"Returns minimized connector activity for the exact account."
|
||||||
|
),
|
||||||
|
"exclude_connector_secrets": (
|
||||||
|
"Never returns credentials, endpoints, external rows, or evidence payloads."
|
||||||
|
),
|
||||||
|
"retain_connector_evidence": (
|
||||||
|
"Preserves configuration and external-operation accountability."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.governed-configuration",
|
||||||
|
title="Govern connector definitions and simulations",
|
||||||
|
summary="Version connector schemas and mappings while preserving tenant-local overrides and review evidence.",
|
||||||
|
body=(
|
||||||
|
"Connector administrators create package-managed or local definitions that explicitly declare provider, protocol, capabilities, schemas, mapping rules, validation, preview support, audit expectations, privacy, retention, limits, and retry metadata. Every definition change creates an immutable revision. A tenant configuration pins one revision and stores only a credential reference; package updates remain available but do not change the effective configuration until an administrator adopts them. Local override leaf paths are displayed as protected and are reapplied when an update is adopted. Dry-runs and simulations are bounded, redact configured fields, are idempotent by caller key, and retain configuration, mapping, input, and external revision provenance. Ambiguous results follow the configuration policy: manual review, quarantine, or rejection. Pending and quarantined evidence requires an explicit approve or reject decision with a reason. Provider-specific live writes are not implied by a successful generic simulation."
|
||||||
|
),
|
||||||
|
layer="configured",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("operator", "module_admin", "integration_admin"),
|
||||||
|
related_modules=("policy", "audit", "dataflow", "ops"),
|
||||||
|
order=38,
|
||||||
|
metadata={
|
||||||
|
"kind": "guide",
|
||||||
|
"help_contexts": ["connectors.admin.governed-configurations"],
|
||||||
|
"prerequisites": [
|
||||||
|
"A connector definition has been installed or authored.",
|
||||||
|
"Credential material is stored outside the connector URL and referenced by an approved secret identifier.",
|
||||||
|
],
|
||||||
|
"outcome": "The active connector behavior is inspectable, version-pinned, testable, and reviewable before any provider-specific write.",
|
||||||
|
"verification": "Reload the configuration, inspect protected paths and effective hash, run a simulation with a new idempotency key, and resolve any pending review result.",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.authority-and-effects",
|
||||||
|
title="Connector authority and effect behavior",
|
||||||
|
summary="Connector direction, technical maturity, and configured source authority are separate and must remain visible.",
|
||||||
|
body=(
|
||||||
|
"A connector can consume, publish, or work bidirectionally and can mature from discovery through replacement. "
|
||||||
|
"Each binding separately states whether GovOPlaN is authoritative, follows an external authority, keeps a mirror, synchronizes under conflict rules, adds a governance overlay, or retains only a link. "
|
||||||
|
"Writable providers must explain revisions, limits, idempotency, outcome-unknown handling, evidence, reconciliation, correction, outage behavior, and secret requirements."
|
||||||
|
),
|
||||||
|
layer="available",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("operator", "module_admin", "power_user", "product_owner"),
|
||||||
|
related_modules=("datasources", "dataflow", "ops", "policy", "audit"),
|
||||||
|
order=39,
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.runtime-preview-contract",
|
||||||
|
title="Connector previews and diagnostics",
|
||||||
|
summary="Use one bounded, redacted dry-run shape across external transports.",
|
||||||
|
body=(
|
||||||
|
"Connectors owns endpoint discovery, authentication hand-off, transport limits, retries, and protocol health. "
|
||||||
|
"Domain modules own field mapping, validation, reconciliation, and record mutation. The shared Core runtime "
|
||||||
|
"contract reports redacted effects and diagnostics with source revisions, fingerprints, and immutable input hashes. "
|
||||||
|
"Tabular previews enforce effective row, serialized-byte, and elapsed-time ceilings and report limit truncation "
|
||||||
|
"as structured diagnostics. A commit must reject stale, truncated, conflicting, or error-bearing previews, and "
|
||||||
|
"credentials never appear in URLs or samples."
|
||||||
|
),
|
||||||
|
layer="available",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("operator", "module_admin", "power_user"),
|
||||||
|
related_modules=("addresses", "datasources", "dataflow", "policy", "audit"),
|
||||||
|
order=40,
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.tabular-sources",
|
||||||
|
title="Governed tabular sources",
|
||||||
|
summary="Provider-neutral source discovery and bounded reads for Dataflow.",
|
||||||
|
body=(
|
||||||
|
"Connectors owns source configuration, access checks, schema discovery, "
|
||||||
|
"fingerprints, and bounded reads. Dataflow stores only opaque source "
|
||||||
|
"references and expected fingerprints. Each source declares its live, "
|
||||||
|
"cached, file-backed, or static mode, structured health, and supported "
|
||||||
|
"projection, filter, aggregation, sorting, and pagination pushdown. The "
|
||||||
|
"first executable provider imports immutable JSON or CSV snapshots, "
|
||||||
|
"supports projection and pagination, and exposes them as Datasource "
|
||||||
|
"origins. Database and API providers can implement the same origin "
|
||||||
|
"contract without changing Datasources or Dataflow."
|
||||||
|
),
|
||||||
|
layer="available",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("operator", "module_admin", "power_user"),
|
||||||
|
related_modules=("dataflow", "files", "reporting", "risk_compliance"),
|
||||||
|
order=40,
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.rss-atom",
|
||||||
|
title="RSS and Atom feeds",
|
||||||
|
summary="Import governed feed snapshots and emit visibility-filtered feeds.",
|
||||||
|
body=(
|
||||||
|
"Connectors owns bounded, SSRF-protected RSS/Atom transport and XML "
|
||||||
|
"parsing. Imported entries become immutable tabular snapshots exposed "
|
||||||
|
"through Datasources, including acquisition, freshness, ETag, content "
|
||||||
|
"digest, and source provenance. Emission accepts only provenance-bearing "
|
||||||
|
"event, publication, case, or report selections from an owning surface. "
|
||||||
|
"The requested audience determines the visibility ceiling: public output "
|
||||||
|
"contains only public entries, while tenant or private output requires a "
|
||||||
|
"separate restricted-feed permission. Callers cannot supply their own "
|
||||||
|
"visibility allow-list. Portal or Reporting owns durable publication "
|
||||||
|
"routes and must re-authorize access on every restricted feed request. "
|
||||||
|
"A separate RSS module is only warranted if GovOPlaN later needs a "
|
||||||
|
"dedicated feed-reader product surface."
|
||||||
|
),
|
||||||
|
layer="available",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("operator", "module_admin", "power_user"),
|
||||||
|
related_modules=("datasources", "dataflow", "portal", "reporting"),
|
||||||
|
order=42,
|
||||||
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="connectors.sanctions-snapshots",
|
||||||
|
title="Sanctions source snapshots",
|
||||||
|
summary=(
|
||||||
|
"Acquire immutable, checksum-verifiable sanctions list "
|
||||||
|
"evidence without transmitting screening subjects."
|
||||||
|
),
|
||||||
|
body=(
|
||||||
|
"Connectors provides a deterministic synthetic fixture and "
|
||||||
|
"the official United Nations Security Council consolidated "
|
||||||
|
"XML source. Each fetch records conditional transport "
|
||||||
|
"evidence, bounded retries, health state, source metadata, "
|
||||||
|
"raw evidence, and a SHA-256 checksum. Refreshes acquire a "
|
||||||
|
"distributed recovery fence before provider I/O; the immutable "
|
||||||
|
"snapshot and terminal recovery checkpoint then commit in one "
|
||||||
|
"transaction. A repeated request key returns the same result. "
|
||||||
|
"Risk Compliance owns "
|
||||||
|
"normalization, matching, legal review, and dispositions."
|
||||||
|
),
|
||||||
|
layer="available",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("operator", "module_admin", "compliance_reviewer"),
|
||||||
|
related_modules=("risk_compliance", "dataflow"),
|
||||||
|
order=41,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_manifest() -> ModuleManifest:
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"MODULE_ID",
|
||||||
|
"MODULE_VERSION",
|
||||||
|
"DATASOURCE_ORIGIN_INTERFACE_VERSION",
|
||||||
|
"SANCTIONS_SNAPSHOT_INTERFACE_VERSION",
|
||||||
|
"TABULAR_SOURCE_INTERFACE_VERSION",
|
||||||
|
"get_manifest",
|
||||||
|
"manifest",
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Connectors migrations."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Connectors migration revisions."""
|
||||||
+214
@@ -0,0 +1,214 @@
|
|||||||
|
"""governed connector definitions and simulation evidence
|
||||||
|
|
||||||
|
Revision ID: a8d9e0f1b2c3
|
||||||
|
Revises: f7c8d9e0a1b2
|
||||||
|
Create Date: 2026-08-20 12:30:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a8d9e0f1b2c3"
|
||||||
|
down_revision = "f7c8d9e0a1b2"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_definitions",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("definition_key", sa.String(length=160), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("description", sa.Text(), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("current_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_package", sa.String(length=300), nullable=True),
|
||||||
|
sa.Column("local_definition", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_definitions")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"definition_key",
|
||||||
|
name="uq_connector_definition_tenant_key",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definitions_tenant_id"),
|
||||||
|
"connector_definitions",
|
||||||
|
["tenant_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definitions_status"),
|
||||||
|
"connector_definitions",
|
||||||
|
["status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_definitions_tenant_status",
|
||||||
|
"connector_definitions",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_definition_revisions",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("definition_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("specification", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("definition_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("origin", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("package_ref", sa.String(length=300), nullable=True),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["definition_id"],
|
||||||
|
["connector_definitions.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_definition_revisions_definition_id_connector_definitions"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_definition_revisions"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"definition_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_connector_definition_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definition_revisions_definition_id"),
|
||||||
|
"connector_definition_revisions",
|
||||||
|
["definition_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definition_revisions_definition_hash"),
|
||||||
|
"connector_definition_revisions",
|
||||||
|
["definition_hash"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definition_revisions_created_by"),
|
||||||
|
"connector_definition_revisions",
|
||||||
|
["created_by"],
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"connector_configurations",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("definition_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("endpoint_url", sa.String(length=1500), nullable=True),
|
||||||
|
sa.Column("credential_ref", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("base_definition_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("local_overrides", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("protected_paths", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effective_configuration", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effective_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("ambiguity_policy", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["definition_id"],
|
||||||
|
["connector_definitions.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_configurations_definition_id_connector_definitions"
|
||||||
|
),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_configurations")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"name",
|
||||||
|
name="uq_connector_configuration_tenant_name",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_configurations_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_configurations_definition_id", ["definition_id"]),
|
||||||
|
("ix_connector_configurations_status", ["status"]),
|
||||||
|
("ix_connector_configurations_effective_hash", ["effective_hash"]),
|
||||||
|
("ix_connector_configurations_updated_by", ["updated_by"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_configurations", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_configurations_tenant_status",
|
||||||
|
"connector_configurations",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_simulation_runs",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("configuration_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("mode", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("review_state", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("definition_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("configuration_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("configuration_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("input_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("summary", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effects", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("reviewed_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("review_reason", sa.Text(), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["configuration_id"],
|
||||||
|
["connector_configurations.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_simulation_runs_configuration_id_connector_configurations"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_simulation_runs"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
"mode",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_simulation_run_idempotency",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_simulation_runs_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_simulation_runs_configuration_id", ["configuration_id"]),
|
||||||
|
("ix_connector_simulation_runs_mode", ["mode"]),
|
||||||
|
("ix_connector_simulation_runs_status", ["status"]),
|
||||||
|
("ix_connector_simulation_runs_review_state", ["review_state"]),
|
||||||
|
("ix_connector_simulation_runs_created_by", ["created_by"]),
|
||||||
|
("ix_connector_simulation_runs_reviewed_by", ["reviewed_by"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_simulation_runs", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_simulation_runs_review",
|
||||||
|
"connector_simulation_runs",
|
||||||
|
["tenant_id", "review_state", "created_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("connector_simulation_runs")
|
||||||
|
op.drop_table("connector_configurations")
|
||||||
|
op.drop_table("connector_definition_revisions")
|
||||||
|
op.drop_table("connector_definitions")
|
||||||
+141
@@ -0,0 +1,141 @@
|
|||||||
|
"""v0.1.14 Connectors baseline
|
||||||
|
|
||||||
|
Revision ID: e6b7c8d9f0a1
|
||||||
|
Revises: None
|
||||||
|
Create Date: 2026-07-28 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "e6b7c8d9f0a1"
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_tabular_sources",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("provider", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("source_name", sa.String(length=120), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("description", sa.Text(), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("schema_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("schema", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("rows", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("fingerprint", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("row_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("byte_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("metadata", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_tabular_sources")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"source_name",
|
||||||
|
name="uq_connector_tabular_source_name",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_created_by"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["created_by"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_deleted_at"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["deleted_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_fingerprint"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["fingerprint"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_provider"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["provider"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_status"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["status"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_tenant_id"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["tenant_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_updated_by"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["updated_by"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_status",
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_updated",
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["tenant_id", "updated_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_updated",
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_status",
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_updated_by"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_tenant_id"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_status"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_provider"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_fingerprint"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_deleted_at"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_created_by"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_table("connector_tabular_sources")
|
||||||
+192
@@ -0,0 +1,192 @@
|
|||||||
|
"""Add immutable sanctions source snapshots.
|
||||||
|
|
||||||
|
Revision ID: f7c8d9e0a1b2
|
||||||
|
Revises: e6b7c8d9f0a1
|
||||||
|
Create Date: 2026-07-29
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "f7c8d9e0a1b2"
|
||||||
|
down_revision = "e6b7c8d9f0a1"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("provider_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(length=200), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("request_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("response_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"started_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"finished_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("snapshot_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_sanctions_acquisition_runs"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for column in (
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"source_id",
|
||||||
|
"status",
|
||||||
|
"started_at",
|
||||||
|
"snapshot_id",
|
||||||
|
"created_by",
|
||||||
|
):
|
||||||
|
op.create_index(
|
||||||
|
op.f(
|
||||||
|
"ix_connector_sanctions_acquisition_runs_"
|
||||||
|
f"{column}"
|
||||||
|
),
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
[column],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_sanctions_run_health",
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
["tenant_id", "provider_id", "status", "started_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("provider_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("publisher", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("jurisdiction", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("list_type", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(length=200), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"source_version",
|
||||||
|
sa.String(length=255),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"publication_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"effective_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"acquired_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("source_url", sa.String(length=1500), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"content_type",
|
||||||
|
sa.String(length=200),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("byte_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("signature_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"parser_version",
|
||||||
|
sa.String(length=100),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("licence_notes", sa.Text(), nullable=True),
|
||||||
|
sa.Column("trust_notes", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"connector_run_id",
|
||||||
|
sa.String(length=36),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("transport_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("raw_content", sa.LargeBinary(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["connector_run_id"],
|
||||||
|
["connector_sanctions_acquisition_runs.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_sanctions_snapshots_connector_run_id_"
|
||||||
|
"connector_sanctions_acquisition_runs"
|
||||||
|
),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_sanctions_snapshots"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"connector_run_id",
|
||||||
|
name="uq_connector_sanctions_snapshot_run",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for column in (
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"jurisdiction",
|
||||||
|
"list_type",
|
||||||
|
"source_id",
|
||||||
|
"source_version",
|
||||||
|
"acquired_at",
|
||||||
|
"sha256",
|
||||||
|
"connector_run_id",
|
||||||
|
):
|
||||||
|
op.create_index(
|
||||||
|
op.f(f"ix_connector_sanctions_snapshots_{column}"),
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
[column],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_sanctions_snapshot_source",
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
["tenant_id", "provider_id", "acquired_at"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_sanctions_snapshot_version",
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
["provider_id", "source_id", "source_version"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("connector_sanctions_snapshots")
|
||||||
|
op.drop_table("connector_sanctions_acquisition_runs")
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from hashlib import sha256
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.provider_governance import (
|
||||||
|
ExternalProviderRuntimeState,
|
||||||
|
ExternalProviderStateContext,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
TABULAR_PROVIDER_ID = "connectors.tabular_snapshot"
|
||||||
|
SANCTIONS_PROVIDER_ID = "connectors.sanctions_snapshot"
|
||||||
|
|
||||||
|
|
||||||
|
def tabular_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
session = _session(context)
|
||||||
|
statement = select(ConnectorTabularSource).where(
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None)
|
||||||
|
)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
ConnectorTabularSource.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
sources = tuple(
|
||||||
|
session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
ConnectorTabularSource.tenant_id,
|
||||||
|
ConnectorTabularSource.id,
|
||||||
|
).limit(context.max_items + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(_tabular_state(item, observed_at=observed_at) for item in sources)
|
||||||
|
|
||||||
|
|
||||||
|
def sanctions_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
session = _session(context)
|
||||||
|
statement = select(ConnectorSanctionsAcquisitionRun)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
ConnectorSanctionsAcquisitionRun.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
runs = tuple(
|
||||||
|
session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
ConnectorSanctionsAcquisitionRun.tenant_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.provider_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.source_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.started_at.desc(),
|
||||||
|
).limit(max(context.max_items * 10, context.max_items + 1))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
latest_by_binding: dict[tuple[str, str, str], ConnectorSanctionsAcquisitionRun] = {}
|
||||||
|
for run in runs:
|
||||||
|
key = (run.tenant_id, run.provider_id, run.source_id)
|
||||||
|
latest_by_binding.setdefault(key, run)
|
||||||
|
if len(latest_by_binding) >= context.max_items + 1:
|
||||||
|
break
|
||||||
|
|
||||||
|
snapshot_counts = _snapshot_counts(
|
||||||
|
session,
|
||||||
|
binding_keys=tuple(latest_by_binding),
|
||||||
|
)
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(
|
||||||
|
_sanctions_state(
|
||||||
|
run,
|
||||||
|
observed_at=observed_at,
|
||||||
|
snapshot_count=snapshot_counts.get(key, 0),
|
||||||
|
)
|
||||||
|
for key, run in latest_by_binding.items()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(context: ExternalProviderStateContext) -> Session:
|
||||||
|
if not isinstance(context.session, Session):
|
||||||
|
raise RuntimeError("Connectors provider state requires a database session.")
|
||||||
|
return context.session
|
||||||
|
|
||||||
|
|
||||||
|
def _tabular_state(
|
||||||
|
source: ConnectorTabularSource,
|
||||||
|
*,
|
||||||
|
observed_at: datetime,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
active = source.status == "active"
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=TABULAR_PROVIDER_ID,
|
||||||
|
binding_ref=f"connectors:tabular-source:{source.id}",
|
||||||
|
authority_mode="external_mirror",
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=active,
|
||||||
|
health="healthy" if active else "inactive",
|
||||||
|
freshness="not_applicable",
|
||||||
|
conflict="not_applicable",
|
||||||
|
recovery="ready" if active else "not_applicable",
|
||||||
|
last_success_at=_aware(source.updated_at or source.created_at),
|
||||||
|
detail=(
|
||||||
|
"Immutable tabular snapshot is available."
|
||||||
|
if active
|
||||||
|
else "Immutable tabular snapshot is inactive."
|
||||||
|
),
|
||||||
|
metrics={
|
||||||
|
"row_count": int(source.row_count),
|
||||||
|
"byte_count": int(source.byte_count),
|
||||||
|
"schema_version": int(source.schema_version),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_counts(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
binding_keys: tuple[tuple[str, str, str], ...],
|
||||||
|
) -> dict[tuple[str, str, str], int]:
|
||||||
|
if not binding_keys:
|
||||||
|
return {}
|
||||||
|
tenant_ids = {item[0] for item in binding_keys}
|
||||||
|
rows = session.execute(
|
||||||
|
select(
|
||||||
|
ConnectorSanctionsSnapshot.tenant_id,
|
||||||
|
ConnectorSanctionsSnapshot.provider_id,
|
||||||
|
ConnectorSanctionsSnapshot.source_id,
|
||||||
|
func.count(ConnectorSanctionsSnapshot.id),
|
||||||
|
)
|
||||||
|
.where(ConnectorSanctionsSnapshot.tenant_id.in_(tenant_ids))
|
||||||
|
.group_by(
|
||||||
|
ConnectorSanctionsSnapshot.tenant_id,
|
||||||
|
ConnectorSanctionsSnapshot.provider_id,
|
||||||
|
ConnectorSanctionsSnapshot.source_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
(str(tenant_id), str(provider_id), str(source_id)): int(count)
|
||||||
|
for tenant_id, provider_id, source_id, count in rows
|
||||||
|
if (str(tenant_id), str(provider_id), str(source_id)) in binding_keys
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _sanctions_state(
|
||||||
|
run: ConnectorSanctionsAcquisitionRun,
|
||||||
|
*,
|
||||||
|
observed_at: datetime,
|
||||||
|
snapshot_count: int,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
status = str(run.status)
|
||||||
|
success = status in {"succeeded", "success", "not_modified"}
|
||||||
|
running = status in {"running", "pending", "retry"}
|
||||||
|
has_snapshot = bool(run.snapshot_id) or snapshot_count > 0
|
||||||
|
health = "healthy" if success else "warning" if running else "error"
|
||||||
|
binding_digest = sha256(
|
||||||
|
f"{run.tenant_id}\0{run.provider_id}\0{run.source_id}".encode("utf-8")
|
||||||
|
).hexdigest()[:24]
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=SANCTIONS_PROVIDER_ID,
|
||||||
|
binding_ref=f"connectors:sanctions-source:{binding_digest}",
|
||||||
|
authority_mode="external_mirror",
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=True,
|
||||||
|
health=health,
|
||||||
|
freshness="unknown",
|
||||||
|
conflict="not_applicable",
|
||||||
|
recovery="ready" if success and has_snapshot else "attention",
|
||||||
|
last_success_at=_aware(run.finished_at) if success else None,
|
||||||
|
detail=(
|
||||||
|
"Latest sanctions acquisition completed."
|
||||||
|
if success
|
||||||
|
else "Sanctions acquisition is in progress."
|
||||||
|
if running
|
||||||
|
else "Latest sanctions acquisition failed; prior accepted snapshots remain separate evidence."
|
||||||
|
),
|
||||||
|
metrics={
|
||||||
|
"latest_status": status,
|
||||||
|
"attempt_count": int(run.attempt_count),
|
||||||
|
"accepted_snapshots": int(snapshot_count),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"SANCTIONS_PROVIDER_ID",
|
||||||
|
"TABULAR_PROVIDER_ID",
|
||||||
|
"sanctions_provider_states",
|
||||||
|
"tabular_provider_states",
|
||||||
|
]
|
||||||
@@ -0,0 +1,381 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
from typing import Any
|
||||||
|
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
DurableRecoveryOperation,
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRecoveryError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConnectorRecoveryDeclaration:
|
||||||
|
operation_type: str
|
||||||
|
mode: RecoveryMode
|
||||||
|
provider_mutation: bool
|
||||||
|
idempotency: str
|
||||||
|
verification: tuple[str, ...]
|
||||||
|
recovery: tuple[str, ...]
|
||||||
|
implemented: bool
|
||||||
|
|
||||||
|
|
||||||
|
CONNECTOR_RECOVERY_OPERATIONS = (
|
||||||
|
ConnectorRecoveryDeclaration(
|
||||||
|
operation_type="read-snapshot",
|
||||||
|
mode=RecoveryMode.ATOMIC,
|
||||||
|
provider_mutation=False,
|
||||||
|
idempotency=(
|
||||||
|
"Caller-supplied request keys replay a committed immutable snapshot; "
|
||||||
|
"otherwise each deliberate acquisition receives a generated key."
|
||||||
|
),
|
||||||
|
verification=(
|
||||||
|
"provider revision or conditional cursor is recorded before fetch",
|
||||||
|
"domain snapshot and terminal recovery checkpoint commit together",
|
||||||
|
"stored bytes and provider evidence are checksum verified",
|
||||||
|
),
|
||||||
|
recovery=(
|
||||||
|
"a stale running transaction is failed after its database transaction rolls back",
|
||||||
|
"a new deliberate acquisition may then use a new request key",
|
||||||
|
),
|
||||||
|
implemented=True,
|
||||||
|
),
|
||||||
|
ConnectorRecoveryDeclaration(
|
||||||
|
operation_type="external-mutation",
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
provider_mutation=True,
|
||||||
|
idempotency="A stable caller key and canonical request digest are mandatory.",
|
||||||
|
verification=(
|
||||||
|
"record the remote revision and bounded provider result",
|
||||||
|
"verify the provider state before reporting success",
|
||||||
|
),
|
||||||
|
recovery=(
|
||||||
|
"unknown outcomes remain unresolved until provider-backed reconciliation",
|
||||||
|
"never retry the same remote effect solely to reconstruct local state",
|
||||||
|
),
|
||||||
|
implemented=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def connector_session_factory(session: Session) -> sessionmaker[Session]:
|
||||||
|
bind = session.get_bind()
|
||||||
|
if bind is None:
|
||||||
|
raise ConnectorRecoveryError("Connector recovery requires a bound database session")
|
||||||
|
return sessionmaker(bind=bind, expire_on_commit=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _digest(value: str) -> str:
|
||||||
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_key(value: str | None) -> str:
|
||||||
|
clean = str(value or "").strip()
|
||||||
|
if clean and len(clean) > 500:
|
||||||
|
raise ConnectorRecoveryError("Connector idempotency keys are limited to 500 characters")
|
||||||
|
return clean or str(uuid4())
|
||||||
|
|
||||||
|
|
||||||
|
def _stable_resource_id(
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
provider_id: str,
|
||||||
|
operation_type: str,
|
||||||
|
request_key: str,
|
||||||
|
) -> str:
|
||||||
|
return str(
|
||||||
|
uuid5(
|
||||||
|
NAMESPACE_URL,
|
||||||
|
f"govoplan:{tenant_id}:{provider_id}:{operation_type}:{request_key}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ConnectorReadSnapshotRecovery:
|
||||||
|
operation: DurableRecoveryOperation | None
|
||||||
|
operation_id: str
|
||||||
|
request_key: str
|
||||||
|
resource_id: str
|
||||||
|
replayed: bool
|
||||||
|
|
||||||
|
def commit_success(self, session: Session, *, evidence: dict[str, Any]) -> None:
|
||||||
|
if self.operation is None:
|
||||||
|
raise ConnectorRecoveryError("A replayed connector read cannot be committed again")
|
||||||
|
try:
|
||||||
|
self.operation.commit_atomic_success(session, evidence=evidence)
|
||||||
|
except Exception as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector snapshot and recovery evidence did not commit atomically"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def commit_failure(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
summary: str,
|
||||||
|
evidence: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
if self.operation is None:
|
||||||
|
raise ConnectorRecoveryError("A replayed connector read cannot be failed again")
|
||||||
|
try:
|
||||||
|
self.operation.commit_atomic_failure(
|
||||||
|
session,
|
||||||
|
summary=summary,
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector failure evidence did not commit atomically"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def fail_without_projection(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
summary: str,
|
||||||
|
code: str,
|
||||||
|
) -> None:
|
||||||
|
if self.operation is None:
|
||||||
|
return
|
||||||
|
self.operation.fail(
|
||||||
|
summary=summary,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {
|
||||||
|
"provider_mutation": False,
|
||||||
|
"projection_committed": False,
|
||||||
|
"failure_code": code,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def begin_connector_read_snapshot(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
provider_id: str,
|
||||||
|
idempotency_key: str | None,
|
||||||
|
source_revision: str | None,
|
||||||
|
cursor: str | None,
|
||||||
|
dry_run_evidence: dict[str, Any],
|
||||||
|
request_metadata: dict[str, Any] | None = None,
|
||||||
|
resource_type: str = "connector_sync_run",
|
||||||
|
) -> ConnectorReadSnapshotRecovery:
|
||||||
|
request_key = _clean_key(idempotency_key)
|
||||||
|
resource_id = _stable_resource_id(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
provider_id=provider_id,
|
||||||
|
operation_type="read-snapshot",
|
||||||
|
request_key=request_key,
|
||||||
|
)
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
"request_key_sha256": _digest(request_key),
|
||||||
|
**dict(request_metadata or {}),
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
connector_session_factory(session),
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="connectors",
|
||||||
|
operation_type="read-snapshot",
|
||||||
|
idempotency_key=f"connector-read:{_digest(f'{tenant_id}:{provider_id}:{request_key}')}",
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.ATOMIC,
|
||||||
|
preconditions=(
|
||||||
|
"the actor is authorized for the connector source",
|
||||||
|
"the provider request is read-only",
|
||||||
|
"the source revision, cursor, and dry-run decision are durable",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"validate the bounded provider response and source revision",
|
||||||
|
"commit the immutable snapshot and terminal checkpoint atomically",
|
||||||
|
"compare the stored content digest with the acquired bytes",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"cursor_sha256": _digest(cursor) if cursor else None,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
"provider_mutation": False,
|
||||||
|
},
|
||||||
|
lease_resource_key=f"connectors:read:{tenant_id}:{_digest(provider_id)[:40]}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "external-provider"],
|
||||||
|
"provider_mutation": False,
|
||||||
|
"recovery_declaration": "read-snapshot",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except RecoveryOperationBusy as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"Another runtime is already acquiring this connector source"
|
||||||
|
) from exc
|
||||||
|
except RecoveryOperationStateConflict as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"This connector request is active or unresolved; reconcile it before retrying"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector recovery ledger is unavailable; the provider was not contacted"
|
||||||
|
) from exc
|
||||||
|
return ConnectorReadSnapshotRecovery(
|
||||||
|
operation=started.operation,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
request_key=request_key,
|
||||||
|
resource_id=resource_id,
|
||||||
|
replayed=started.replayed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ConnectorExternalMutationRecovery:
|
||||||
|
operation: DurableRecoveryOperation | None
|
||||||
|
operation_id: str
|
||||||
|
replayed: bool
|
||||||
|
|
||||||
|
def succeed(self, *, provider_evidence: dict[str, Any]) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.succeed(evidence=provider_evidence)
|
||||||
|
|
||||||
|
def reject(self, *, summary: str, provider_code: str) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.reject(
|
||||||
|
summary=summary,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {"provider_rejection": provider_code},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def outcome_unknown(self, *, summary: str, provider_code: str) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary=summary,
|
||||||
|
evidence={"effect_started": True, "provider_code": provider_code},
|
||||||
|
failure_summary="Inspect provider state before any retry",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def begin_connector_external_mutation(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
provider_id: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
request_sha256: str,
|
||||||
|
source_revision: str | None,
|
||||||
|
cursor: str | None,
|
||||||
|
dry_run_evidence: dict[str, Any],
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
) -> ConnectorExternalMutationRecovery:
|
||||||
|
if not str(idempotency_key or "").strip():
|
||||||
|
raise ConnectorRecoveryError("External connector mutations require an idempotency key")
|
||||||
|
request_key = _clean_key(idempotency_key)
|
||||||
|
if len(request_sha256) != 64 or any(
|
||||||
|
character not in "0123456789abcdefABCDEF" for character in request_sha256
|
||||||
|
):
|
||||||
|
raise ConnectorRecoveryError("External connector mutations require a SHA-256 request digest")
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
connector_session_factory(session),
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="connectors",
|
||||||
|
operation_type="external-mutation",
|
||||||
|
idempotency_key=f"connector-write:{_digest(f'{tenant_id}:{provider_id}:{request_key}')}",
|
||||||
|
request={
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"request_sha256": request_sha256,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"cursor_sha256": _digest(cursor) if cursor else None,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
},
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the actor and effective connector policy authorize the mutation",
|
||||||
|
"a stable idempotency key and canonical request digest are present",
|
||||||
|
"the dry-run and source revision evidence are durable",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"inspect provider state without repeating the mutation",
|
||||||
|
"record whether the provider accepted the requested revision",
|
||||||
|
"retry only under a new deliberate key when absence is proven",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"compare provider identity and revision with the canonical request",
|
||||||
|
"verify the consuming domain state independently",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"request_sha256": request_sha256,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"cursor_sha256": _digest(cursor) if cursor else None,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
"provider_mutation": True,
|
||||||
|
},
|
||||||
|
lease_resource_key=(
|
||||||
|
f"connectors:write:{tenant_id}:{_digest(provider_id)[:24]}:"
|
||||||
|
f"{_digest(resource_id)[:24]}"
|
||||||
|
),
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "queue", "external-provider"],
|
||||||
|
"provider_mutation": True,
|
||||||
|
"recovery_declaration": "external-mutation",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"This external connector effect is active or unresolved"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector recovery ledger is unavailable; no external mutation started"
|
||||||
|
) from exc
|
||||||
|
return ConnectorExternalMutationRecovery(
|
||||||
|
operation=started.operation,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
replayed=started.replayed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CONNECTOR_RECOVERY_OPERATIONS",
|
||||||
|
"ConnectorExternalMutationRecovery",
|
||||||
|
"ConnectorReadSnapshotRecovery",
|
||||||
|
"ConnectorRecoveryDeclaration",
|
||||||
|
"ConnectorRecoveryError",
|
||||||
|
"begin_connector_external_mutation",
|
||||||
|
"begin_connector_read_snapshot",
|
||||||
|
"connector_session_factory",
|
||||||
|
]
|
||||||
@@ -0,0 +1,964 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict
|
||||||
|
import hashlib
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.audit.logging import audit_event
|
||||||
|
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularSnapshotInput,
|
||||||
|
TabularSource,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceError,
|
||||||
|
TabularSourceNotFoundError,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.feeds import (
|
||||||
|
FeedCapabilityError,
|
||||||
|
FeedEntry,
|
||||||
|
FeedRenderRequest,
|
||||||
|
FeedVisibility,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.sanctions import SanctionsSnapshotReference
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_connectors.backend.schemas import (
|
||||||
|
FeedAcquireRequest,
|
||||||
|
FeedDocumentResponse,
|
||||||
|
FeedImportRequest,
|
||||||
|
FeedPublicationEntryPayload,
|
||||||
|
FeedRenderPayload,
|
||||||
|
SanctionsAcquisitionRunListResponse,
|
||||||
|
SanctionsAcquisitionRunResponse,
|
||||||
|
SanctionsRefreshResponse,
|
||||||
|
SanctionsSnapshotListResponse,
|
||||||
|
SanctionsSnapshotResponse,
|
||||||
|
SanctionsSourceListResponse,
|
||||||
|
SanctionsSourceResponse,
|
||||||
|
SnapshotCreateRequest,
|
||||||
|
TabularColumnResponse,
|
||||||
|
TabularHealthResponse,
|
||||||
|
TabularPreviewDiagnosticResponse,
|
||||||
|
TabularPushdownResponse,
|
||||||
|
TabularSourceDeleteResponse,
|
||||||
|
TabularSourceListResponse,
|
||||||
|
TabularSourcePreviewResponse,
|
||||||
|
TabularSourceResponse,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.feeds import (
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
ConnectorFeedProvider,
|
||||||
|
feed_rows,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_runtime import (
|
||||||
|
GovernedConnectorError,
|
||||||
|
create_configuration,
|
||||||
|
execute_run,
|
||||||
|
list_configurations,
|
||||||
|
list_definitions,
|
||||||
|
list_runs,
|
||||||
|
review_run,
|
||||||
|
update_configuration,
|
||||||
|
upsert_definition,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_schemas import (
|
||||||
|
ConnectorConfigurationCreateRequest,
|
||||||
|
ConnectorConfigurationItem,
|
||||||
|
ConnectorConfigurationListResponse,
|
||||||
|
ConnectorConfigurationUpdateRequest,
|
||||||
|
ConnectorDefinitionItem,
|
||||||
|
ConnectorDefinitionListResponse,
|
||||||
|
ConnectorDefinitionUpsertRequest,
|
||||||
|
ConnectorReviewRequest,
|
||||||
|
ConnectorRunItem,
|
||||||
|
ConnectorRunListResponse,
|
||||||
|
ConnectorRunRequest,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.recovery import (
|
||||||
|
ConnectorRecoveryError,
|
||||||
|
begin_connector_read_snapshot,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.sanctions_sources import (
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
SanctionsSourceAccessError,
|
||||||
|
SanctionsSourceError,
|
||||||
|
SanctionsSourceNotFoundError,
|
||||||
|
SqlSanctionsSnapshotProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.tabular_sources import (
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
READ_SCOPE,
|
||||||
|
WRITE_SCOPE,
|
||||||
|
SqlTabularSourceProvider,
|
||||||
|
parse_csv_snapshot,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/connectors", tags=["connectors"])
|
||||||
|
provider = SqlTabularSourceProvider()
|
||||||
|
sanctions_provider = SqlSanctionsSnapshotProvider()
|
||||||
|
feed_transport = ConnectorFeedProvider()
|
||||||
|
|
||||||
|
|
||||||
|
def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None:
|
||||||
|
if any(has_scope(principal, scope) for scope in scopes):
|
||||||
|
return
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail=f"Missing one of the required scopes: {', '.join(scopes)}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _http_error(exc: TabularSourceError) -> HTTPException:
|
||||||
|
if isinstance(exc, TabularSourceNotFoundError):
|
||||||
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||||
|
if isinstance(exc, TabularSourceAccessError):
|
||||||
|
return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
|
||||||
|
if isinstance(exc, TabularSourceUnavailableError):
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
return HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
def _sanctions_http_error(
|
||||||
|
exc: SanctionsSourceError,
|
||||||
|
) -> HTTPException:
|
||||||
|
if isinstance(exc, SanctionsSourceNotFoundError):
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
if isinstance(exc, SanctionsSourceAccessError):
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _feed_http_error(exc: FeedCapabilityError) -> HTTPException:
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _recovery_http_error(exc: ConnectorRecoveryError) -> HTTPException:
|
||||||
|
detail = str(exc)
|
||||||
|
return HTTPException(
|
||||||
|
status_code=(
|
||||||
|
status.HTTP_409_CONFLICT
|
||||||
|
if "already" in detail.casefold() or "active" in detail.casefold()
|
||||||
|
else status.HTTP_503_SERVICE_UNAVAILABLE
|
||||||
|
),
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _governed_http_error(exc: GovernedConnectorError) -> HTTPException:
|
||||||
|
if exc.code.endswith("_not_found"):
|
||||||
|
status_code = status.HTTP_404_NOT_FOUND
|
||||||
|
elif exc.code in {
|
||||||
|
"configuration_conflict",
|
||||||
|
"configuration_disabled",
|
||||||
|
"idempotency_conflict",
|
||||||
|
"local_definition_protected",
|
||||||
|
"package_definition_requires_overrides",
|
||||||
|
"run_not_reviewable",
|
||||||
|
}:
|
||||||
|
status_code = status.HTTP_409_CONFLICT
|
||||||
|
else:
|
||||||
|
status_code = status.HTTP_422_UNPROCESSABLE_CONTENT
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status_code,
|
||||||
|
detail={"code": exc.code, "message": str(exc)},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds/preview", response_model=FeedDocumentResponse)
|
||||||
|
def api_preview_feed(
|
||||||
|
payload: FeedAcquireRequest,
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> FeedDocumentResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
document = feed_transport.fetch(
|
||||||
|
payload.url,
|
||||||
|
max_entries=payload.max_entries,
|
||||||
|
)
|
||||||
|
except FeedCapabilityError as exc:
|
||||||
|
raise _feed_http_error(exc) from exc
|
||||||
|
return FeedDocumentResponse.model_validate(asdict(document))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/feeds/import",
|
||||||
|
response_model=TabularSourceResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def api_import_feed_snapshot(
|
||||||
|
payload: FeedImportRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
idempotency_key: Annotated[
|
||||||
|
str | None,
|
||||||
|
Header(alias="Idempotency-Key", max_length=500),
|
||||||
|
] = None,
|
||||||
|
) -> TabularSourceResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
recovery = begin_connector_read_snapshot(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
provider_id="connectors.feed_snapshot",
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
source_revision=None,
|
||||||
|
cursor=None,
|
||||||
|
dry_run_evidence={
|
||||||
|
"performed": False,
|
||||||
|
"reason": "read-only acquisition into an immutable snapshot",
|
||||||
|
},
|
||||||
|
request_metadata={
|
||||||
|
"source_url_sha256": hashlib.sha256(
|
||||||
|
payload.url.encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"source_name": payload.source_name,
|
||||||
|
"max_entries": payload.max_entries,
|
||||||
|
},
|
||||||
|
resource_type="connector_tabular_source",
|
||||||
|
)
|
||||||
|
except ConnectorRecoveryError as exc:
|
||||||
|
raise _recovery_http_error(exc) from exc
|
||||||
|
if recovery.replayed:
|
||||||
|
try:
|
||||||
|
source = provider.get_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
source_ref=f"snapshot:{recovery.resource_id}",
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return _source_response(source)
|
||||||
|
try:
|
||||||
|
document = feed_transport.fetch(
|
||||||
|
payload.url,
|
||||||
|
max_entries=payload.max_entries,
|
||||||
|
)
|
||||||
|
source = provider.create_snapshot(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name=payload.name,
|
||||||
|
source_name=payload.source_name,
|
||||||
|
description=payload.description or document.description,
|
||||||
|
rows=feed_rows(document),
|
||||||
|
metadata={
|
||||||
|
"import_format": document.format,
|
||||||
|
"feed": {
|
||||||
|
"source_url": document.source_url,
|
||||||
|
"home_url": document.home_url,
|
||||||
|
"acquired_at": (
|
||||||
|
document.acquired_at.isoformat()
|
||||||
|
if document.acquired_at
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"fresh_until": (
|
||||||
|
document.fresh_until.isoformat()
|
||||||
|
if document.fresh_until
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"etag": document.etag,
|
||||||
|
"last_modified": document.last_modified,
|
||||||
|
"content_type": document.content_type,
|
||||||
|
"sha256": document.sha256,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
source_id=recovery.resource_id,
|
||||||
|
)
|
||||||
|
except (FeedCapabilityError, TabularSourceError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
recovery.fail_without_projection(
|
||||||
|
summary="The read-only feed import failed before a snapshot committed",
|
||||||
|
code=exc.__class__.__name__,
|
||||||
|
)
|
||||||
|
if isinstance(exc, FeedCapabilityError):
|
||||||
|
raise _feed_http_error(exc) from exc
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="connectors.feed_snapshot.created",
|
||||||
|
object_type="connector_tabular_source",
|
||||||
|
object_id=source.ref,
|
||||||
|
details={
|
||||||
|
"source_url": document.source_url,
|
||||||
|
"format": document.format,
|
||||||
|
"sha256": document.sha256,
|
||||||
|
"row_count": source.row_count,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
recovery.commit_success(
|
||||||
|
session,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {
|
||||||
|
"snapshot_ref": source.ref,
|
||||||
|
"snapshot_fingerprint": source.fingerprint,
|
||||||
|
"feed_sha256": document.sha256,
|
||||||
|
"row_count": source.row_count,
|
||||||
|
"provider_mutation": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except ConnectorRecoveryError as exc:
|
||||||
|
raise _recovery_http_error(exc) from exc
|
||||||
|
return _source_response(source)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds/render")
|
||||||
|
def api_render_feed(
|
||||||
|
payload: FeedRenderPayload,
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> Response:
|
||||||
|
_require_any_scope(principal, FEED_PUBLISH_SCOPE, ADMIN_SCOPE)
|
||||||
|
if payload.audience != "public":
|
||||||
|
_require_any_scope(principal, FEED_PRIVATE_PUBLISH_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
rendered = feed_transport.render(
|
||||||
|
FeedRenderRequest(
|
||||||
|
format=payload.format,
|
||||||
|
title=payload.title,
|
||||||
|
feed_url=payload.feed_url,
|
||||||
|
home_url=payload.home_url,
|
||||||
|
description=payload.description,
|
||||||
|
language=payload.language,
|
||||||
|
entries=tuple(
|
||||||
|
_publication_feed_entry(item) for item in payload.entries
|
||||||
|
),
|
||||||
|
allowed_visibilities=_feed_audience_visibilities(payload.audience),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except FeedCapabilityError as exc:
|
||||||
|
raise _feed_http_error(exc) from exc
|
||||||
|
return Response(
|
||||||
|
content=rendered.body,
|
||||||
|
media_type=rendered.content_type,
|
||||||
|
headers={
|
||||||
|
"X-GovOPlaN-Feed-Included": str(rendered.included_entries),
|
||||||
|
"X-GovOPlaN-Feed-Excluded": str(rendered.excluded_entries),
|
||||||
|
"X-GovOPlaN-Feed-Audience": payload.audience,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _feed_audience_visibilities(
|
||||||
|
audience: str,
|
||||||
|
) -> frozenset[FeedVisibility]:
|
||||||
|
if audience == "public":
|
||||||
|
return frozenset({"public"})
|
||||||
|
if audience == "tenant":
|
||||||
|
return frozenset({"public", "tenant"})
|
||||||
|
return frozenset({"public", "tenant", "private"})
|
||||||
|
|
||||||
|
|
||||||
|
def _publication_feed_entry(item: FeedPublicationEntryPayload) -> FeedEntry:
|
||||||
|
values = item.model_dump(
|
||||||
|
exclude={"source_kind", "source_module", "source_ref", "source_revision"}
|
||||||
|
)
|
||||||
|
values["categories"] = tuple(values["categories"])
|
||||||
|
values["enclosures"] = tuple(values["enclosures"])
|
||||||
|
values["metadata"] = {
|
||||||
|
**values["metadata"],
|
||||||
|
"govoplan_source": {
|
||||||
|
"kind": item.source_kind,
|
||||||
|
"module": item.source_module,
|
||||||
|
"reference": item.source_ref,
|
||||||
|
"revision": item.source_revision,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return FeedEntry(**values)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/tabular-sources", response_model=TabularSourceListResponse)
|
||||||
|
def api_list_tabular_sources(
|
||||||
|
query: str = Query(default="", max_length=200),
|
||||||
|
limit: int = Query(default=100, ge=1, le=100),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> TabularSourceListResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||||
|
sources = provider.list_sources(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
query=query,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
return TabularSourceListResponse(sources=[_source_response(source) for source in sources])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/tabular-sources/snapshots",
|
||||||
|
response_model=TabularSourceResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def api_create_tabular_snapshot(
|
||||||
|
payload: SnapshotCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> TabularSourceResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
rows = (
|
||||||
|
tuple(payload.rows or ())
|
||||||
|
if payload.format == "json"
|
||||||
|
else parse_csv_snapshot(payload.csv_text or "", delimiter=payload.delimiter)
|
||||||
|
)
|
||||||
|
source = provider.create_snapshot(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name=payload.name,
|
||||||
|
source_name=payload.source_name,
|
||||||
|
description=payload.description,
|
||||||
|
rows=rows,
|
||||||
|
metadata={"import_format": payload.format},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="connectors.tabular_snapshot.created",
|
||||||
|
object_type="connector_tabular_source",
|
||||||
|
object_id=source.ref,
|
||||||
|
details={
|
||||||
|
"provider": source.provider,
|
||||||
|
"source_name": source.source_name,
|
||||||
|
"fingerprint": source.fingerprint,
|
||||||
|
"row_count": source.row_count,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _source_response(source)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/tabular-sources/{source_id}/preview",
|
||||||
|
response_model=TabularSourcePreviewResponse,
|
||||||
|
)
|
||||||
|
def api_preview_tabular_source(
|
||||||
|
source_id: str,
|
||||||
|
limit: int = Query(default=100, ge=1, le=500),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
|
max_bytes: int = Query(default=1_000_000, ge=2, le=5_000_000),
|
||||||
|
timeout_ms: int = Query(default=2_000, ge=1, le=10_000),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> TabularSourcePreviewResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
result = provider.read_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=f"snapshot:{source_id}",
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
max_bytes=max_bytes,
|
||||||
|
timeout_ms=timeout_ms,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return TabularSourcePreviewResponse(
|
||||||
|
source=_source_response(result.source),
|
||||||
|
rows=[dict(row) for row in result.rows],
|
||||||
|
total_rows=result.total_rows,
|
||||||
|
truncated=result.truncated,
|
||||||
|
returned_bytes=result.returned_bytes,
|
||||||
|
elapsed_ms=result.elapsed_ms,
|
||||||
|
effective_row_limit=result.effective_row_limit,
|
||||||
|
effective_byte_limit=result.effective_byte_limit,
|
||||||
|
effective_timeout_ms=result.effective_timeout_ms,
|
||||||
|
diagnostics=[
|
||||||
|
TabularPreviewDiagnosticResponse(
|
||||||
|
severity=item.severity,
|
||||||
|
code=item.code,
|
||||||
|
message=item.message,
|
||||||
|
details=dict(item.details),
|
||||||
|
)
|
||||||
|
for item in result.diagnostics
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete(
|
||||||
|
"/tabular-sources/{source_id}",
|
||||||
|
response_model=TabularSourceDeleteResponse,
|
||||||
|
)
|
||||||
|
def api_delete_tabular_source(
|
||||||
|
source_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> TabularSourceDeleteResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
source_ref = f"snapshot:{source_id}"
|
||||||
|
try:
|
||||||
|
source = provider.delete_snapshot(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
source_ref=source_ref,
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="connectors.tabular_snapshot.deleted",
|
||||||
|
object_type="connector_tabular_source",
|
||||||
|
object_id=source_ref,
|
||||||
|
details={"source_name": source.source_name, "fingerprint": source.fingerprint},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return TabularSourceDeleteResponse(deleted=True, source_ref=source_ref)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/sanctions/sources",
|
||||||
|
response_model=SanctionsSourceListResponse,
|
||||||
|
)
|
||||||
|
def api_list_sanctions_sources(
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> SanctionsSourceListResponse:
|
||||||
|
_require_any_scope(
|
||||||
|
principal,
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
)
|
||||||
|
return SanctionsSourceListResponse(
|
||||||
|
sources=[
|
||||||
|
SanctionsSourceResponse.model_validate(
|
||||||
|
source,
|
||||||
|
from_attributes=True,
|
||||||
|
)
|
||||||
|
for source in sanctions_provider.available_sources()
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/sanctions/sources/{provider_id}/refresh",
|
||||||
|
response_model=SanctionsRefreshResponse,
|
||||||
|
)
|
||||||
|
def api_refresh_sanctions_source(
|
||||||
|
provider_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
idempotency_key: Annotated[
|
||||||
|
str | None,
|
||||||
|
Header(alias="Idempotency-Key", max_length=500),
|
||||||
|
] = None,
|
||||||
|
) -> SanctionsRefreshResponse:
|
||||||
|
_require_any_scope(
|
||||||
|
principal,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = sanctions_provider.refresh_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider_id=provider_id,
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
)
|
||||||
|
except ConnectorRecoveryError as exc:
|
||||||
|
raise _recovery_http_error(exc) from exc
|
||||||
|
except SanctionsSourceError as exc:
|
||||||
|
raise _sanctions_http_error(exc) from exc
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="connectors.sanctions_source.refreshed",
|
||||||
|
object_type="connector_sanctions_acquisition_run",
|
||||||
|
object_id=result.run_id,
|
||||||
|
details={
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"status": result.status,
|
||||||
|
"snapshot_ref": (
|
||||||
|
result.snapshot.ref
|
||||||
|
if result.snapshot is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return SanctionsRefreshResponse(
|
||||||
|
run_id=result.run_id,
|
||||||
|
provider_id=result.provider_id,
|
||||||
|
status=result.status,
|
||||||
|
snapshot=(
|
||||||
|
_sanctions_snapshot_response(result.snapshot)
|
||||||
|
if result.snapshot is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
error=result.error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/sanctions/snapshots",
|
||||||
|
response_model=SanctionsSnapshotListResponse,
|
||||||
|
)
|
||||||
|
def api_list_sanctions_snapshots(
|
||||||
|
limit: int = Query(default=100, ge=1, le=500),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> SanctionsSnapshotListResponse:
|
||||||
|
_require_any_scope(
|
||||||
|
principal,
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
snapshots = sanctions_provider.list_snapshots(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
except SanctionsSourceError as exc:
|
||||||
|
raise _sanctions_http_error(exc) from exc
|
||||||
|
return SanctionsSnapshotListResponse(
|
||||||
|
snapshots=[
|
||||||
|
_sanctions_snapshot_response(item)
|
||||||
|
for item in snapshots
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/sanctions/runs",
|
||||||
|
response_model=SanctionsAcquisitionRunListResponse,
|
||||||
|
)
|
||||||
|
def api_list_sanctions_runs(
|
||||||
|
limit: int = Query(default=100, ge=1, le=500),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> SanctionsAcquisitionRunListResponse:
|
||||||
|
_require_any_scope(
|
||||||
|
principal,
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
ADMIN_SCOPE,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
runs = sanctions_provider.list_runs(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
except SanctionsSourceError as exc:
|
||||||
|
raise _sanctions_http_error(exc) from exc
|
||||||
|
return SanctionsAcquisitionRunListResponse(
|
||||||
|
runs=[
|
||||||
|
SanctionsAcquisitionRunResponse.model_validate(
|
||||||
|
item,
|
||||||
|
from_attributes=True,
|
||||||
|
)
|
||||||
|
for item in runs
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/governed/definitions",
|
||||||
|
response_model=ConnectorDefinitionListResponse,
|
||||||
|
)
|
||||||
|
def api_list_governed_definitions(
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorDefinitionListResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
items = list_definitions(session, tenant_id=principal.tenant_id)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
return ConnectorDefinitionListResponse(items=items)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/governed/definitions",
|
||||||
|
response_model=ConnectorDefinitionItem,
|
||||||
|
)
|
||||||
|
def api_upsert_governed_definition(
|
||||||
|
payload: ConnectorDefinitionUpsertRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorDefinitionItem:
|
||||||
|
_require_any_scope(principal, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
return upsert_definition(session, principal, payload)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/governed/configurations",
|
||||||
|
response_model=ConnectorConfigurationListResponse,
|
||||||
|
)
|
||||||
|
def api_list_governed_configurations(
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorConfigurationListResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
items = list_configurations(session, tenant_id=principal.tenant_id)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
return ConnectorConfigurationListResponse(items=items)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/governed/configurations",
|
||||||
|
response_model=ConnectorConfigurationItem,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def api_create_governed_configuration(
|
||||||
|
payload: ConnectorConfigurationCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
_require_any_scope(principal, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
return create_configuration(session, principal, payload)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/governed/configurations/{configuration_id}",
|
||||||
|
response_model=ConnectorConfigurationItem,
|
||||||
|
)
|
||||||
|
def api_update_governed_configuration(
|
||||||
|
configuration_id: str,
|
||||||
|
payload: ConnectorConfigurationUpdateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
_require_any_scope(principal, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
return update_configuration(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _api_execute_governed_run(
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
mode: str,
|
||||||
|
payload: ConnectorRunRequest,
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
return execute_run(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
mode=mode,
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/governed/configurations/{configuration_id}/dry-runs",
|
||||||
|
response_model=ConnectorRunItem,
|
||||||
|
)
|
||||||
|
def api_dry_run_governed_configuration(
|
||||||
|
configuration_id: str,
|
||||||
|
payload: ConnectorRunRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
return _api_execute_governed_run(
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
mode="dry_run",
|
||||||
|
payload=payload,
|
||||||
|
session=session,
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/governed/configurations/{configuration_id}/simulations",
|
||||||
|
response_model=ConnectorRunItem,
|
||||||
|
)
|
||||||
|
def api_simulate_governed_configuration(
|
||||||
|
configuration_id: str,
|
||||||
|
payload: ConnectorRunRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
return _api_execute_governed_run(
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=payload,
|
||||||
|
session=session,
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/governed/runs",
|
||||||
|
response_model=ConnectorRunListResponse,
|
||||||
|
)
|
||||||
|
def api_list_governed_runs(
|
||||||
|
configuration_id: str | None = Query(default=None),
|
||||||
|
review_state: str | None = Query(default=None),
|
||||||
|
limit: int = Query(default=100, ge=1, le=500),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorRunListResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
items = list_runs(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
review_state=review_state,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
return ConnectorRunListResponse(items=items)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/governed/runs/{run_id}/review",
|
||||||
|
response_model=ConnectorRunItem,
|
||||||
|
)
|
||||||
|
def api_review_governed_run(
|
||||||
|
run_id: str,
|
||||||
|
payload: ConnectorReviewRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
_require_any_scope(principal, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
return review_run(session, principal, run_id=run_id, payload=payload)
|
||||||
|
except GovernedConnectorError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _governed_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _source_response(source: TabularSource) -> TabularSourceResponse:
|
||||||
|
return TabularSourceResponse(
|
||||||
|
ref=source.ref,
|
||||||
|
provider=source.provider,
|
||||||
|
source_name=source.source_name,
|
||||||
|
name=source.name,
|
||||||
|
description=source.description,
|
||||||
|
columns=[
|
||||||
|
TabularColumnResponse(
|
||||||
|
name=column.name,
|
||||||
|
data_type=column.data_type,
|
||||||
|
nullable=column.nullable,
|
||||||
|
)
|
||||||
|
for column in source.schema
|
||||||
|
],
|
||||||
|
schema_version=source.schema_version,
|
||||||
|
fingerprint=source.fingerprint,
|
||||||
|
row_count=source.row_count,
|
||||||
|
byte_count=source.byte_count,
|
||||||
|
updated_at=source.updated_at.isoformat() if source.updated_at else None,
|
||||||
|
capabilities=list(source.capabilities),
|
||||||
|
metadata=dict(source.metadata),
|
||||||
|
source_mode=source.source_mode,
|
||||||
|
pushdown=TabularPushdownResponse(
|
||||||
|
projections=source.pushdown.projections,
|
||||||
|
pagination=source.pushdown.pagination,
|
||||||
|
filters=list(source.pushdown.filters),
|
||||||
|
aggregations=list(source.pushdown.aggregations),
|
||||||
|
sorting=list(source.pushdown.sorting),
|
||||||
|
),
|
||||||
|
health=TabularHealthResponse(
|
||||||
|
status=source.health.status,
|
||||||
|
code=source.health.code,
|
||||||
|
summary=source.health.summary,
|
||||||
|
checked_at=(
|
||||||
|
source.health.checked_at.isoformat()
|
||||||
|
if source.health.checked_at
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
details=dict(source.health.details),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sanctions_snapshot_response(
|
||||||
|
snapshot: SanctionsSnapshotReference,
|
||||||
|
) -> SanctionsSnapshotResponse:
|
||||||
|
return SanctionsSnapshotResponse.model_validate(
|
||||||
|
{
|
||||||
|
"ref": snapshot.ref,
|
||||||
|
"provider_id": snapshot.provider_id,
|
||||||
|
"publisher": snapshot.publisher,
|
||||||
|
"jurisdiction": snapshot.jurisdiction,
|
||||||
|
"list_type": snapshot.list_type,
|
||||||
|
"source_id": snapshot.source_id,
|
||||||
|
"source_version": snapshot.source_version,
|
||||||
|
"publication_at": snapshot.publication_at,
|
||||||
|
"effective_at": snapshot.effective_at,
|
||||||
|
"acquired_at": snapshot.acquired_at,
|
||||||
|
"content_type": snapshot.content_type,
|
||||||
|
"byte_count": snapshot.byte_count,
|
||||||
|
"sha256": snapshot.sha256,
|
||||||
|
"parser_version": snapshot.parser_version,
|
||||||
|
"raw_evidence_ref": snapshot.raw_evidence_ref,
|
||||||
|
"connector_run_id": snapshot.connector_run_id,
|
||||||
|
"signature_evidence": dict(
|
||||||
|
snapshot.signature_evidence
|
||||||
|
),
|
||||||
|
"licence_notes": snapshot.licence_notes,
|
||||||
|
"trust_notes": snapshot.trust_notes,
|
||||||
|
"transport_evidence": dict(
|
||||||
|
snapshot.transport_evidence
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["router"]
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,269 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
|
class FeedAcquireRequest(BaseModel):
|
||||||
|
url: str = Field(min_length=1, max_length=2000)
|
||||||
|
max_entries: int = Field(default=2_000, ge=1, le=10_000)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedImportRequest(FeedAcquireRequest):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
source_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=120,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||||
|
)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedEntryPayload(BaseModel):
|
||||||
|
id: str = Field(min_length=1, max_length=2000)
|
||||||
|
title: str = Field(min_length=1, max_length=1000)
|
||||||
|
url: str | None = Field(default=None, max_length=2000)
|
||||||
|
summary: str | None = None
|
||||||
|
content: str | None = None
|
||||||
|
author: str | None = Field(default=None, max_length=500)
|
||||||
|
published_at: datetime | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
categories: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
enclosures: list[dict[str, Any]] = Field(default_factory=list, max_length=100)
|
||||||
|
visibility: Literal["public", "tenant", "private"] = "public"
|
||||||
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedPublicationEntryPayload(FeedEntryPayload):
|
||||||
|
source_kind: Literal["event", "publication", "case", "report"]
|
||||||
|
source_module: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=100,
|
||||||
|
pattern=r"^[a-z][a-z0-9_]*$",
|
||||||
|
)
|
||||||
|
source_ref: str = Field(min_length=1, max_length=500)
|
||||||
|
source_revision: str | None = Field(default=None, max_length=200)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedDocumentResponse(BaseModel):
|
||||||
|
format: Literal["rss", "atom"]
|
||||||
|
title: str
|
||||||
|
source_url: str
|
||||||
|
description: str | None = None
|
||||||
|
home_url: str | None = None
|
||||||
|
language: str | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
acquired_at: datetime | None = None
|
||||||
|
fresh_until: datetime | None = None
|
||||||
|
etag: str | None = None
|
||||||
|
last_modified: str | None = None
|
||||||
|
content_type: str | None = None
|
||||||
|
sha256: str
|
||||||
|
entries: list[FeedEntryPayload]
|
||||||
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedRenderPayload(BaseModel):
|
||||||
|
format: Literal["rss", "atom"]
|
||||||
|
title: str = Field(min_length=1, max_length=1000)
|
||||||
|
feed_url: str = Field(min_length=1, max_length=2000)
|
||||||
|
home_url: str = Field(min_length=1, max_length=2000)
|
||||||
|
description: str | None = None
|
||||||
|
language: str | None = Field(default=None, max_length=100)
|
||||||
|
entries: list[FeedPublicationEntryPayload] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
max_length=10_000,
|
||||||
|
)
|
||||||
|
audience: Literal["public", "tenant", "private"] = "public"
|
||||||
|
|
||||||
|
|
||||||
|
class SnapshotCreateRequest(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
source_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=120,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||||
|
)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
format: Literal["json", "csv"] = "json"
|
||||||
|
rows: list[dict[str, Any]] | None = Field(default=None, max_length=10_000)
|
||||||
|
csv_text: str | None = Field(default=None, max_length=5_000_000)
|
||||||
|
delimiter: Literal[",", ";", "\t", "|"] = ","
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_payload(self) -> "SnapshotCreateRequest":
|
||||||
|
if self.format == "json" and self.rows is None:
|
||||||
|
raise ValueError("JSON snapshots require rows.")
|
||||||
|
if self.format == "json" and self.csv_text is not None:
|
||||||
|
raise ValueError("JSON snapshots cannot include CSV text.")
|
||||||
|
if self.format == "csv" and not self.csv_text:
|
||||||
|
raise ValueError("CSV snapshots require CSV text.")
|
||||||
|
if self.format == "csv" and self.rows is not None:
|
||||||
|
raise ValueError("CSV snapshots cannot include JSON rows.")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class TabularColumnResponse(BaseModel):
|
||||||
|
name: str
|
||||||
|
data_type: str
|
||||||
|
nullable: bool
|
||||||
|
|
||||||
|
|
||||||
|
class TabularPushdownResponse(BaseModel):
|
||||||
|
projections: bool
|
||||||
|
pagination: bool
|
||||||
|
filters: list[str]
|
||||||
|
aggregations: list[str]
|
||||||
|
sorting: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularHealthResponse(BaseModel):
|
||||||
|
status: Literal["healthy", "warning", "error", "unknown"]
|
||||||
|
code: str
|
||||||
|
summary: str
|
||||||
|
checked_at: str | None
|
||||||
|
details: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularPreviewDiagnosticResponse(BaseModel):
|
||||||
|
severity: Literal["info", "warning", "error"]
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
details: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourceResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
provider: str
|
||||||
|
source_name: str
|
||||||
|
name: str
|
||||||
|
description: str | None
|
||||||
|
columns: list[TabularColumnResponse]
|
||||||
|
schema_version: str
|
||||||
|
fingerprint: str
|
||||||
|
row_count: int | None
|
||||||
|
byte_count: int | None
|
||||||
|
updated_at: str | None
|
||||||
|
capabilities: list[str]
|
||||||
|
metadata: dict[str, Any]
|
||||||
|
source_mode: Literal["live", "cached", "file_backed", "static"]
|
||||||
|
pushdown: TabularPushdownResponse
|
||||||
|
health: TabularHealthResponse
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourceListResponse(BaseModel):
|
||||||
|
sources: list[TabularSourceResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourcePreviewResponse(BaseModel):
|
||||||
|
source: TabularSourceResponse
|
||||||
|
rows: list[dict[str, Any]]
|
||||||
|
total_rows: int
|
||||||
|
truncated: bool
|
||||||
|
returned_bytes: int
|
||||||
|
elapsed_ms: int
|
||||||
|
effective_row_limit: int
|
||||||
|
effective_byte_limit: int
|
||||||
|
effective_timeout_ms: int
|
||||||
|
diagnostics: list[TabularPreviewDiagnosticResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourceDeleteResponse(BaseModel):
|
||||||
|
deleted: bool
|
||||||
|
source_ref: str
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSourceResponse(BaseModel):
|
||||||
|
provider_id: str
|
||||||
|
publisher: str
|
||||||
|
jurisdiction: str
|
||||||
|
list_type: str
|
||||||
|
source_id: str
|
||||||
|
source_url: str | None
|
||||||
|
parser_version: str
|
||||||
|
licence_notes: str
|
||||||
|
trust_notes: str
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSourceListResponse(BaseModel):
|
||||||
|
sources: list[SanctionsSourceResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSnapshotResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
provider_id: str
|
||||||
|
publisher: str
|
||||||
|
jurisdiction: str
|
||||||
|
list_type: str
|
||||||
|
source_id: str
|
||||||
|
source_version: str
|
||||||
|
publication_at: datetime | None
|
||||||
|
effective_at: datetime | None
|
||||||
|
acquired_at: datetime
|
||||||
|
content_type: str
|
||||||
|
byte_count: int
|
||||||
|
sha256: str
|
||||||
|
parser_version: str
|
||||||
|
raw_evidence_ref: str
|
||||||
|
connector_run_id: str
|
||||||
|
signature_evidence: dict[str, Any]
|
||||||
|
licence_notes: str | None
|
||||||
|
trust_notes: str | None
|
||||||
|
transport_evidence: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSnapshotListResponse(BaseModel):
|
||||||
|
snapshots: list[SanctionsSnapshotResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsAcquisitionRunResponse(BaseModel):
|
||||||
|
id: str
|
||||||
|
provider_id: str
|
||||||
|
source_id: str
|
||||||
|
status: str
|
||||||
|
attempt_count: int
|
||||||
|
request_evidence: dict[str, Any]
|
||||||
|
response_evidence: dict[str, Any]
|
||||||
|
started_at: datetime
|
||||||
|
finished_at: datetime | None
|
||||||
|
snapshot_id: str | None
|
||||||
|
error: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsAcquisitionRunListResponse(BaseModel):
|
||||||
|
runs: list[SanctionsAcquisitionRunResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsRefreshResponse(BaseModel):
|
||||||
|
run_id: str
|
||||||
|
provider_id: str
|
||||||
|
status: str
|
||||||
|
snapshot: SanctionsSnapshotResponse | None
|
||||||
|
error: str | None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FeedAcquireRequest",
|
||||||
|
"FeedDocumentResponse",
|
||||||
|
"FeedEntryPayload",
|
||||||
|
"FeedImportRequest",
|
||||||
|
"FeedRenderPayload",
|
||||||
|
"SnapshotCreateRequest",
|
||||||
|
"SanctionsAcquisitionRunListResponse",
|
||||||
|
"SanctionsAcquisitionRunResponse",
|
||||||
|
"SanctionsRefreshResponse",
|
||||||
|
"SanctionsSnapshotListResponse",
|
||||||
|
"SanctionsSnapshotResponse",
|
||||||
|
"SanctionsSourceListResponse",
|
||||||
|
"SanctionsSourceResponse",
|
||||||
|
"TabularColumnResponse",
|
||||||
|
"TabularHealthResponse",
|
||||||
|
"TabularPreviewDiagnosticResponse",
|
||||||
|
"TabularPushdownResponse",
|
||||||
|
"TabularSourceDeleteResponse",
|
||||||
|
"TabularSourceListResponse",
|
||||||
|
"TabularSourcePreviewResponse",
|
||||||
|
"TabularSourceResponse",
|
||||||
|
]
|
||||||
@@ -0,0 +1,498 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
|
from sqlalchemy import or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, has_scope
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularColumn,
|
||||||
|
TabularPreviewDiagnostic,
|
||||||
|
TabularPushdown,
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularReadResult,
|
||||||
|
TabularSnapshotInput,
|
||||||
|
TabularSource,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceNotFoundError,
|
||||||
|
TabularSourceHealth,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
parse_tabular_csv,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import utcnow
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
|
||||||
|
|
||||||
|
READ_SCOPE = "connectors:source:read"
|
||||||
|
WRITE_SCOPE = "connectors:source:write"
|
||||||
|
ADMIN_SCOPE = "connectors:source:admin"
|
||||||
|
MAX_SNAPSHOT_ROWS = 10_000
|
||||||
|
MAX_SNAPSHOT_BYTES = 5_000_000
|
||||||
|
MAX_READ_ROWS = 500
|
||||||
|
MAX_READ_BYTES = 1_000_000
|
||||||
|
MAX_READ_TIMEOUT_MS = 2_000
|
||||||
|
|
||||||
|
|
||||||
|
class SqlTabularSourceProvider:
|
||||||
|
def __init__(self, *, clock: Callable[[], float] = time.monotonic) -> None:
|
||||||
|
self._clock = clock
|
||||||
|
|
||||||
|
def list_sources(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
query: str = "",
|
||||||
|
limit: int = 100,
|
||||||
|
) -> Sequence[TabularSource]:
|
||||||
|
db, api_principal = _context(session, principal, READ_SCOPE)
|
||||||
|
normalized_query = str(query or "").strip()
|
||||||
|
statement = select(ConnectorTabularSource).where(
|
||||||
|
ConnectorTabularSource.tenant_id == api_principal.tenant_id,
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None),
|
||||||
|
ConnectorTabularSource.status == "active",
|
||||||
|
)
|
||||||
|
if normalized_query:
|
||||||
|
pattern = f"%{_escape_like(normalized_query)}%"
|
||||||
|
statement = statement.where(
|
||||||
|
or_(
|
||||||
|
ConnectorTabularSource.name.ilike(pattern, escape="\\"),
|
||||||
|
ConnectorTabularSource.source_name.ilike(pattern, escape="\\"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
statement = statement.order_by(
|
||||||
|
ConnectorTabularSource.updated_at.desc(),
|
||||||
|
ConnectorTabularSource.name,
|
||||||
|
).limit(max(1, min(int(limit), 100)))
|
||||||
|
return tuple(_source_dto(item) for item in db.scalars(statement))
|
||||||
|
|
||||||
|
def get_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
source_ref: str,
|
||||||
|
) -> TabularSource | None:
|
||||||
|
db, api_principal = _context(session, principal, READ_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=source_ref,
|
||||||
|
)
|
||||||
|
return _source_dto(item) if item is not None else None
|
||||||
|
|
||||||
|
def read_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: TabularReadRequest,
|
||||||
|
) -> TabularReadResult:
|
||||||
|
db, api_principal = _context(session, principal, READ_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=request.source_ref,
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise TabularSourceNotFoundError("Tabular source not found.")
|
||||||
|
if request.expected_fingerprint and request.expected_fingerprint != item.fingerprint:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The source fingerprint changed; refresh the source node before running it."
|
||||||
|
)
|
||||||
|
|
||||||
|
started = self._clock()
|
||||||
|
limit = max(1, min(int(request.limit), MAX_READ_ROWS))
|
||||||
|
byte_limit = max(2, min(int(request.max_bytes), MAX_READ_BYTES))
|
||||||
|
timeout_ms = max(1, min(int(request.timeout_ms), MAX_READ_TIMEOUT_MS))
|
||||||
|
offset = max(0, int(request.offset))
|
||||||
|
diagnostics: list[TabularPreviewDiagnostic] = []
|
||||||
|
if limit != request.limit:
|
||||||
|
diagnostics.append(
|
||||||
|
_preview_diagnostic(
|
||||||
|
"preview.row_limit_tightened",
|
||||||
|
"The provider tightened the requested row limit.",
|
||||||
|
requested=request.limit,
|
||||||
|
effective=limit,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if byte_limit != request.max_bytes:
|
||||||
|
diagnostics.append(
|
||||||
|
_preview_diagnostic(
|
||||||
|
"preview.byte_limit_tightened",
|
||||||
|
"The provider tightened the requested byte limit.",
|
||||||
|
requested=request.max_bytes,
|
||||||
|
effective=byte_limit,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if timeout_ms != request.timeout_ms:
|
||||||
|
diagnostics.append(
|
||||||
|
_preview_diagnostic(
|
||||||
|
"preview.timeout_tightened",
|
||||||
|
"The provider tightened the requested time limit.",
|
||||||
|
requested=request.timeout_ms,
|
||||||
|
effective=timeout_ms,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
selected_columns = tuple(dict.fromkeys(request.columns))
|
||||||
|
known_columns = {column["name"] for column in item.schema_}
|
||||||
|
unknown_columns = [column for column in selected_columns if column not in known_columns]
|
||||||
|
if unknown_columns:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Unknown source columns: {', '.join(unknown_columns)}"
|
||||||
|
)
|
||||||
|
rows: list[dict[str, object]] = []
|
||||||
|
returned_bytes = 2
|
||||||
|
stopped_for = ""
|
||||||
|
for row in item.rows[offset:]:
|
||||||
|
if len(rows) >= limit:
|
||||||
|
stopped_for = "rows"
|
||||||
|
break
|
||||||
|
elapsed_ms = int(max(0.0, self._clock() - started) * 1_000)
|
||||||
|
if elapsed_ms >= timeout_ms:
|
||||||
|
if not rows:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Tabular source preview exceeded its time budget."
|
||||||
|
)
|
||||||
|
stopped_for = "time"
|
||||||
|
break
|
||||||
|
selected = {
|
||||||
|
key: value
|
||||||
|
for key, value in row.items()
|
||||||
|
if not selected_columns or key in selected_columns
|
||||||
|
}
|
||||||
|
row_bytes = len(
|
||||||
|
json.dumps(
|
||||||
|
selected,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
default=str,
|
||||||
|
).encode("utf-8")
|
||||||
|
)
|
||||||
|
additional_bytes = row_bytes + (1 if rows else 0)
|
||||||
|
if returned_bytes + additional_bytes > byte_limit:
|
||||||
|
if not rows:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"A single source row exceeds the preview byte limit."
|
||||||
|
)
|
||||||
|
stopped_for = "bytes"
|
||||||
|
break
|
||||||
|
rows.append(selected)
|
||||||
|
returned_bytes += additional_bytes
|
||||||
|
elapsed_ms = int(max(0.0, self._clock() - started) * 1_000)
|
||||||
|
if stopped_for:
|
||||||
|
labels = {
|
||||||
|
"rows": ("preview.row_limit_reached", "row"),
|
||||||
|
"bytes": ("preview.byte_limit_reached", "byte"),
|
||||||
|
"time": ("preview.timeout_reached", "time"),
|
||||||
|
}
|
||||||
|
code, label = labels[stopped_for]
|
||||||
|
diagnostics.append(
|
||||||
|
TabularPreviewDiagnostic(
|
||||||
|
severity="warning",
|
||||||
|
code=code,
|
||||||
|
message=f"The preview stopped at its effective {label} limit.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return TabularReadResult(
|
||||||
|
source=_source_dto(item),
|
||||||
|
rows=tuple(rows),
|
||||||
|
total_rows=item.row_count,
|
||||||
|
truncated=offset + len(rows) < item.row_count,
|
||||||
|
returned_bytes=returned_bytes,
|
||||||
|
elapsed_ms=elapsed_ms,
|
||||||
|
effective_row_limit=limit,
|
||||||
|
effective_byte_limit=byte_limit,
|
||||||
|
effective_timeout_ms=timeout_ms,
|
||||||
|
diagnostics=tuple(diagnostics),
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_snapshot(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
snapshot: TabularSnapshotInput,
|
||||||
|
source_id: str | None = None,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
name = snapshot.name.strip()
|
||||||
|
source_name = snapshot.source_name.strip()
|
||||||
|
if not name:
|
||||||
|
raise TabularSourceValidationError("Snapshot name is required.")
|
||||||
|
if not source_name:
|
||||||
|
raise TabularSourceValidationError("Snapshot source name is required.")
|
||||||
|
if len(snapshot.rows) > MAX_SNAPSHOT_ROWS:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Snapshots are limited to {MAX_SNAPSHOT_ROWS:,} rows."
|
||||||
|
)
|
||||||
|
rows = [_json_row(row) for row in snapshot.rows]
|
||||||
|
encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
||||||
|
if len(encoded) > MAX_SNAPSHOT_BYTES:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Snapshots are limited to {MAX_SNAPSHOT_BYTES // 1_000_000} MB."
|
||||||
|
)
|
||||||
|
existing = db.scalar(
|
||||||
|
select(ConnectorTabularSource.id).where(
|
||||||
|
ConnectorTabularSource.tenant_id == api_principal.tenant_id,
|
||||||
|
ConnectorTabularSource.source_name == source_name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"A tabular source named {source_name!r} already exists."
|
||||||
|
)
|
||||||
|
schema = infer_schema(rows)
|
||||||
|
fingerprint = snapshot_fingerprint(rows, schema)
|
||||||
|
actor_id = _actor_id(api_principal)
|
||||||
|
item = ConnectorTabularSource(
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
provider="snapshot",
|
||||||
|
source_name=source_name,
|
||||||
|
name=name,
|
||||||
|
description=_clean_optional(snapshot.description),
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[_column_payload(column) for column in schema],
|
||||||
|
rows=rows,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
row_count=len(rows),
|
||||||
|
byte_count=len(encoded),
|
||||||
|
metadata_=dict(snapshot.metadata),
|
||||||
|
created_by=actor_id,
|
||||||
|
updated_by=actor_id,
|
||||||
|
)
|
||||||
|
if source_id:
|
||||||
|
item.id = source_id
|
||||||
|
db.add(item)
|
||||||
|
db.flush()
|
||||||
|
return _source_dto(item)
|
||||||
|
|
||||||
|
def delete_snapshot(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
source_ref: str,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=source_ref,
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise TabularSourceNotFoundError("Tabular source not found.")
|
||||||
|
item.deleted_at = utcnow()
|
||||||
|
item.status = "retired"
|
||||||
|
item.updated_by = _actor_id(api_principal)
|
||||||
|
db.flush()
|
||||||
|
return _source_dto(item)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_csv_snapshot(csv_text: str, *, delimiter: str) -> tuple[Mapping[str, object], ...]:
|
||||||
|
return parse_tabular_csv(
|
||||||
|
csv_text,
|
||||||
|
delimiter=delimiter,
|
||||||
|
max_rows=MAX_SNAPSHOT_ROWS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[TabularColumn, ...]:
|
||||||
|
names: list[str] = []
|
||||||
|
for row in rows:
|
||||||
|
for name in row:
|
||||||
|
if name not in names:
|
||||||
|
names.append(name)
|
||||||
|
result: list[TabularColumn] = []
|
||||||
|
for name in names:
|
||||||
|
values = [row.get(name) for row in rows]
|
||||||
|
concrete = [value for value in values if value is not None]
|
||||||
|
data_type = _type_name(concrete[0]) if concrete else "unknown"
|
||||||
|
if any(_type_name(value) != data_type for value in concrete[1:]):
|
||||||
|
data_type = "mixed"
|
||||||
|
result.append(
|
||||||
|
TabularColumn(
|
||||||
|
name=name,
|
||||||
|
data_type=data_type,
|
||||||
|
nullable=len(concrete) != len(values),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_fingerprint(
|
||||||
|
rows: Sequence[Mapping[str, object]],
|
||||||
|
schema: Sequence[TabularColumn],
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"schema": [_column_payload(column) for column in schema],
|
||||||
|
"rows": [dict(row) for row in rows],
|
||||||
|
}
|
||||||
|
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
|
||||||
|
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _source_record(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
source_ref: str,
|
||||||
|
) -> ConnectorTabularSource | None:
|
||||||
|
source_id = source_ref.removeprefix("snapshot:")
|
||||||
|
if not source_id or source_id == source_ref:
|
||||||
|
return None
|
||||||
|
return session.scalar(
|
||||||
|
select(ConnectorTabularSource).where(
|
||||||
|
ConnectorTabularSource.id == source_id,
|
||||||
|
ConnectorTabularSource.tenant_id == tenant_id,
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_dto(item: ConnectorTabularSource) -> TabularSource:
|
||||||
|
return TabularSource(
|
||||||
|
ref=f"snapshot:{item.id}",
|
||||||
|
provider=item.provider,
|
||||||
|
source_name=item.source_name,
|
||||||
|
name=item.name,
|
||||||
|
description=item.description,
|
||||||
|
schema=tuple(TabularColumn(**column) for column in item.schema_),
|
||||||
|
schema_version=str(item.schema_version),
|
||||||
|
fingerprint=item.fingerprint,
|
||||||
|
row_count=item.row_count,
|
||||||
|
byte_count=item.byte_count,
|
||||||
|
updated_at=item.updated_at,
|
||||||
|
capabilities=("read", "preview"),
|
||||||
|
metadata=dict(item.metadata_),
|
||||||
|
source_mode="cached",
|
||||||
|
pushdown=TabularPushdown(
|
||||||
|
projections=True,
|
||||||
|
pagination=True,
|
||||||
|
),
|
||||||
|
health=TabularSourceHealth(
|
||||||
|
status="healthy",
|
||||||
|
code="snapshot.ready",
|
||||||
|
summary="The immutable connector snapshot is ready.",
|
||||||
|
checked_at=item.updated_at,
|
||||||
|
details={"immutable": True},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _preview_diagnostic(
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
requested: int,
|
||||||
|
effective: int,
|
||||||
|
) -> TabularPreviewDiagnostic:
|
||||||
|
return TabularPreviewDiagnostic(
|
||||||
|
severity="info",
|
||||||
|
code=code,
|
||||||
|
message=message,
|
||||||
|
details={"requested": requested, "effective": effective},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _context(
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
required_scope: str,
|
||||||
|
) -> tuple[Session, ApiPrincipal]:
|
||||||
|
if not isinstance(session, Session):
|
||||||
|
raise TypeError("Tabular source providers require a SQLAlchemy session.")
|
||||||
|
if not isinstance(principal, ApiPrincipal):
|
||||||
|
raise TabularSourceAccessError("A tenant API principal is required.")
|
||||||
|
accepted_scopes = {required_scope, ADMIN_SCOPE}
|
||||||
|
if required_scope == READ_SCOPE:
|
||||||
|
accepted_scopes.update(
|
||||||
|
{
|
||||||
|
WRITE_SCOPE,
|
||||||
|
"datasources:catalogue:read",
|
||||||
|
"datasources:source:write",
|
||||||
|
"datasources:source:admin",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not any(has_scope(principal, scope) for scope in accepted_scopes):
|
||||||
|
raise TabularSourceAccessError(f"Missing scope: {required_scope}")
|
||||||
|
return session, principal
|
||||||
|
|
||||||
|
|
||||||
|
def _json_row(row: Mapping[str, object]) -> dict[str, Any]:
|
||||||
|
normalized = {str(key).strip(): value for key, value in row.items()}
|
||||||
|
if not normalized or any(not key for key in normalized):
|
||||||
|
raise TabularSourceValidationError("Every snapshot row needs named columns.")
|
||||||
|
try:
|
||||||
|
json.dumps(normalized, default=_unsupported_json)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise TabularSourceValidationError(f"Snapshot values must be JSON compatible: {exc}") from exc
|
||||||
|
return json.loads(json.dumps(normalized, default=_unsupported_json))
|
||||||
|
|
||||||
|
|
||||||
|
def _unsupported_json(value: object) -> object:
|
||||||
|
if isinstance(value, (datetime, Decimal)):
|
||||||
|
return str(value)
|
||||||
|
raise TypeError(f"{type(value).__name__} is not JSON serializable")
|
||||||
|
|
||||||
|
|
||||||
|
def _type_name(value: object) -> str:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "boolean"
|
||||||
|
if isinstance(value, int):
|
||||||
|
return "integer"
|
||||||
|
if isinstance(value, (float, Decimal)):
|
||||||
|
return "number"
|
||||||
|
if isinstance(value, str):
|
||||||
|
return "string"
|
||||||
|
if isinstance(value, list):
|
||||||
|
return "array"
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return "object"
|
||||||
|
return type(value).__name__.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _column_payload(column: TabularColumn) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"name": column.name,
|
||||||
|
"data_type": column.data_type,
|
||||||
|
"nullable": column.nullable,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _actor_id(principal: ApiPrincipal) -> str | None:
|
||||||
|
return principal.account_id or principal.membership_id or principal.identity_id
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_optional(value: str | None) -> str | None:
|
||||||
|
cleaned = str(value or "").strip()
|
||||||
|
return cleaned or None
|
||||||
|
|
||||||
|
|
||||||
|
def _escape_like(value: str) -> str:
|
||||||
|
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ADMIN_SCOPE",
|
||||||
|
"MAX_READ_ROWS",
|
||||||
|
"MAX_READ_BYTES",
|
||||||
|
"MAX_READ_TIMEOUT_MS",
|
||||||
|
"MAX_SNAPSHOT_BYTES",
|
||||||
|
"MAX_SNAPSHOT_ROWS",
|
||||||
|
"READ_SCOPE",
|
||||||
|
"SqlTabularSourceProvider",
|
||||||
|
"WRITE_SCOPE",
|
||||||
|
"infer_schema",
|
||||||
|
"parse_csv_snapshot",
|
||||||
|
"snapshot_fingerprint",
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.datasources import DatasourceOriginReadRequest
|
||||||
|
from govoplan_core.core.tabular_sources import TabularSnapshotInput
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.datasource_origins import (
|
||||||
|
ConnectorDatasourceOriginProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.tabular_sources import (
|
||||||
|
WRITE_SCOPE,
|
||||||
|
SqlTabularSourceProvider,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(*, scopes: tuple[str, ...]) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDatasourceOriginTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[ConnectorTabularSource.__table__],
|
||||||
|
)
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
self.session = self.Session()
|
||||||
|
provider = SqlTabularSourceProvider()
|
||||||
|
self.source = provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=(WRITE_SCOPE,)),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Imported cases",
|
||||||
|
source_name="imported_cases",
|
||||||
|
rows=({"id": 1, "name": "Ada"},),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.origins = ConnectorDatasourceOriginProvider(provider)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
Base.metadata.drop_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[ConnectorTabularSource.__table__],
|
||||||
|
)
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_datasource_reader_can_discover_and_read_connector_origin(self) -> None:
|
||||||
|
datasource_principal = principal(
|
||||||
|
scopes=("datasources:catalogue:read",),
|
||||||
|
)
|
||||||
|
|
||||||
|
origins = self.origins.list_origins(
|
||||||
|
self.session,
|
||||||
|
datasource_principal,
|
||||||
|
)
|
||||||
|
result = self.origins.read_origin(
|
||||||
|
self.session,
|
||||||
|
datasource_principal,
|
||||||
|
request=DatasourceOriginReadRequest(origin_ref=self.source.ref),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((self.source.ref,), tuple(item.ref for item in origins))
|
||||||
|
self.assertEqual(("live", "cached"), origins[0].supported_modes)
|
||||||
|
self.assertEqual(({"id": 1, "name": "Ada"},), result.rows)
|
||||||
|
self.assertEqual("cached", origins[0].source_mode)
|
||||||
|
self.assertTrue(origins[0].pushdown.projections)
|
||||||
|
self.assertEqual("healthy", origins[0].health.status)
|
||||||
|
self.assertGreater(result.returned_bytes, 2)
|
||||||
|
self.assertEqual(1_000_000, result.effective_byte_limit)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.privacy.dsar_workflow import (
|
||||||
|
create_data_subject_request,
|
||||||
|
search_data_subject_request,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.dsar_provider import (
|
||||||
|
CONNECTORS_DSAR_CAPABILITY,
|
||||||
|
ConnectorsDsarProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
NOW = datetime(2026, 8, 22, 9, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, provider: ConnectorsDsarProvider) -> None:
|
||||||
|
self.provider = provider
|
||||||
|
|
||||||
|
def capability_names(self):
|
||||||
|
return (CONNECTORS_DSAR_CAPABILITY,)
|
||||||
|
|
||||||
|
def capability_owner(self, name):
|
||||||
|
if name != CONNECTORS_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
return "connectors"
|
||||||
|
|
||||||
|
def tenant_entitlement_resolver(self):
|
||||||
|
class _Resolver:
|
||||||
|
@staticmethod
|
||||||
|
def resolve(session, tenant_id):
|
||||||
|
del session, tenant_id
|
||||||
|
return type("State", (), {"effective_modules": ("connectors",)})()
|
||||||
|
|
||||||
|
return _Resolver()
|
||||||
|
|
||||||
|
def require_tenant_capability(self, name, session, **kwargs):
|
||||||
|
del session, kwargs
|
||||||
|
if name != CONNECTORS_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
return self.provider
|
||||||
|
|
||||||
|
def manifests(self):
|
||||||
|
return (type("Manifest", (), {"id": "connectors"})(),)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsDsarProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.provider = ConnectorsDsarProvider()
|
||||||
|
self.assertIsInstance(self.provider, DsarProvider)
|
||||||
|
self._seed()
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _seed(self) -> None:
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
ConnectorTabularSource(
|
||||||
|
id="source-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider="snapshot",
|
||||||
|
source_name="people",
|
||||||
|
name="People import",
|
||||||
|
description="Do not export this business description",
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[{"name": "email"}],
|
||||||
|
rows=[{"email": "third-party@example.test"}],
|
||||||
|
fingerprint="source-fingerprint-do-not-export",
|
||||||
|
row_count=1,
|
||||||
|
byte_count=100,
|
||||||
|
metadata_={"secret": "source-metadata-do-not-export"},
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorTabularSource(
|
||||||
|
id="source-other",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
provider="snapshot",
|
||||||
|
source_name="other",
|
||||||
|
name="Other tenant",
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[],
|
||||||
|
rows=[],
|
||||||
|
fingerprint="other-fingerprint",
|
||||||
|
row_count=0,
|
||||||
|
byte_count=0,
|
||||||
|
metadata_={},
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
),
|
||||||
|
ConnectorSanctionsAcquisitionRun(
|
||||||
|
id="acquisition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="un",
|
||||||
|
source_id="consolidated",
|
||||||
|
status="complete",
|
||||||
|
attempt_count=1,
|
||||||
|
request_evidence={"secret": "request-evidence-do-not-export"},
|
||||||
|
response_evidence={"secret": "response-evidence-do-not-export"},
|
||||||
|
started_at=NOW,
|
||||||
|
finished_at=NOW,
|
||||||
|
snapshot_id="snapshot-1",
|
||||||
|
error="transport-detail-do-not-export",
|
||||||
|
created_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorDefinition(
|
||||||
|
id="definition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_key="address-reader",
|
||||||
|
name="Address reader",
|
||||||
|
description="Definition detail",
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
source_package="package-secret-do-not-export",
|
||||||
|
local_definition=True,
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorDefinitionRevision(
|
||||||
|
id="definition-revision-1",
|
||||||
|
definition_id="definition-1",
|
||||||
|
revision=1,
|
||||||
|
specification={"secret": "specification-do-not-export"},
|
||||||
|
definition_hash="definition-hash-do-not-export",
|
||||||
|
origin="local",
|
||||||
|
package_ref="package-ref-do-not-export",
|
||||||
|
created_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorConfiguration(
|
||||||
|
id="configuration-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id="definition-1",
|
||||||
|
name="Production addresses",
|
||||||
|
status="active",
|
||||||
|
endpoint_url="https://secret.example.test/api",
|
||||||
|
credential_ref="vault://secret-do-not-export",
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={"secret": "override-do-not-export"},
|
||||||
|
protected_paths=["secret"],
|
||||||
|
effective_configuration={"secret": "effective-do-not-export"},
|
||||||
|
effective_hash="configuration-hash-do-not-export",
|
||||||
|
resource_revision=2,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
updated_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorSimulationRun(
|
||||||
|
id="simulation-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
configuration_id="configuration-1",
|
||||||
|
mode="dry_run",
|
||||||
|
idempotency_key="simulation-idempotency-do-not-export",
|
||||||
|
request_hash="simulation-request-hash-do-not-export",
|
||||||
|
status="complete",
|
||||||
|
review_state="approved",
|
||||||
|
definition_revision=1,
|
||||||
|
configuration_revision=2,
|
||||||
|
configuration_hash="simulation-config-hash-do-not-export",
|
||||||
|
input_hash="simulation-input-hash-do-not-export",
|
||||||
|
summary={"secret": "summary-do-not-export"},
|
||||||
|
effects=[{"secret": "effect-do-not-export"}],
|
||||||
|
diagnostics=[{"secret": "diagnostic-do-not-export"}],
|
||||||
|
provenance={"secret": "provenance-do-not-export"},
|
||||||
|
created_by="account-1",
|
||||||
|
reviewed_by="account-1",
|
||||||
|
reviewed_at=NOW,
|
||||||
|
review_reason="review-reason-do-not-export",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _subject() -> DsarSubjectRef:
|
||||||
|
return DsarSubjectRef(account_id="account-1")
|
||||||
|
|
||||||
|
def test_search_exports_minimized_operator_attribution(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session, tenant_id="tenant-1", subject=self._subject()
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
"source_actor_attribution",
|
||||||
|
"acquisition_actor_attribution",
|
||||||
|
"definition_actor_attribution",
|
||||||
|
"configuration_actor_attribution",
|
||||||
|
"simulation_actor_attribution",
|
||||||
|
},
|
||||||
|
{record.resource_type for record in records},
|
||||||
|
)
|
||||||
|
exported = json.dumps([record.to_dict() for record in records])
|
||||||
|
for excluded in (
|
||||||
|
"third-party@example.test",
|
||||||
|
"source-fingerprint-do-not-export",
|
||||||
|
"source-metadata-do-not-export",
|
||||||
|
"request-evidence-do-not-export",
|
||||||
|
"response-evidence-do-not-export",
|
||||||
|
"transport-detail-do-not-export",
|
||||||
|
"specification-do-not-export",
|
||||||
|
"definition-hash-do-not-export",
|
||||||
|
"package-ref-do-not-export",
|
||||||
|
"https://secret.example.test/api",
|
||||||
|
"vault://secret-do-not-export",
|
||||||
|
"override-do-not-export",
|
||||||
|
"effective-do-not-export",
|
||||||
|
"configuration-hash-do-not-export",
|
||||||
|
"simulation-idempotency-do-not-export",
|
||||||
|
"simulation-request-hash-do-not-export",
|
||||||
|
"simulation-config-hash-do-not-export",
|
||||||
|
"simulation-input-hash-do-not-export",
|
||||||
|
"summary-do-not-export",
|
||||||
|
"effect-do-not-export",
|
||||||
|
"diagnostic-do-not-export",
|
||||||
|
"provenance-do-not-export",
|
||||||
|
"review-reason-do-not-export",
|
||||||
|
):
|
||||||
|
self.assertNotIn(excluded, exported)
|
||||||
|
|
||||||
|
def test_requires_exact_account_and_enforces_tenant(self) -> None:
|
||||||
|
email_only = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(email="operator@example.test"),
|
||||||
|
)
|
||||||
|
conflict = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
external_references={"connectors.account": "account-other"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session, tenant_id="tenant-1", subject=self._subject()
|
||||||
|
)
|
||||||
|
self.assertEqual((), email_only)
|
||||||
|
self.assertEqual((), conflict)
|
||||||
|
self.assertNotIn("source-other", {record.resource_id for record in records})
|
||||||
|
|
||||||
|
def test_object_narrowing_does_not_broaden_the_search(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
external_references={"connectors.simulation": "simulation-1"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[("simulation_actor_attribution", "simulation-1")],
|
||||||
|
[(record.resource_type, record.resource_id) for record in records],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_erasure_retains_external_operation_evidence(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session, tenant_id="tenant-1", subject=self._subject()
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self._subject(),
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
self.assertTrue(actions)
|
||||||
|
self.assertTrue(
|
||||||
|
all(action.kind == "retain" and not action.executable for action in actions)
|
||||||
|
)
|
||||||
|
results = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self._subject(),
|
||||||
|
actions=actions,
|
||||||
|
request_id="dsar-connectors-1",
|
||||||
|
)
|
||||||
|
self.assertTrue(all(result.status == "blocked" for result in results))
|
||||||
|
|
||||||
|
def test_manifest_and_core_workflow_discover_provider(self) -> None:
|
||||||
|
self.assertIn(CONNECTORS_DSAR_CAPABILITY, manifest.capability_factories)
|
||||||
|
self.assertIn(
|
||||||
|
"connectors.data-subject-requests",
|
||||||
|
{topic.id for topic in manifest.documentation},
|
||||||
|
)
|
||||||
|
row = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-CONNECTORS-1",
|
||||||
|
request_kind="access",
|
||||||
|
subject=self._subject(),
|
||||||
|
purpose="Connector attribution access request",
|
||||||
|
legal_basis=None,
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="operator-1",
|
||||||
|
)
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=_Registry(self.provider),
|
||||||
|
row=row,
|
||||||
|
expected_revision=row.resource_revision,
|
||||||
|
)
|
||||||
|
self.assertEqual("searched", row.status)
|
||||||
|
self.assertEqual(5, row.search_result["record_count"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from defusedxml import ElementTree as SafeET
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.feeds import (
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
ConnectorFeedProvider,
|
||||||
|
feed_rows,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.router import api_render_feed
|
||||||
|
from govoplan_connectors.backend.schemas import FeedRenderPayload
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.feeds import (
|
||||||
|
FeedCapabilityError,
|
||||||
|
FeedEntry,
|
||||||
|
FeedRenderRequest,
|
||||||
|
)
|
||||||
|
from govoplan_core.security.http_fetch import HttpFetchResponse
|
||||||
|
|
||||||
|
|
||||||
|
RSS = b"""<?xml version="1.0"?>
|
||||||
|
<rss version="2.0"><channel>
|
||||||
|
<title>Decisions</title><link>https://example.test/</link>
|
||||||
|
<description>Published decisions</description>
|
||||||
|
<item><guid>decision-1</guid><title>Decision one</title>
|
||||||
|
<link>https://example.test/1</link>
|
||||||
|
<pubDate>Fri, 31 Jul 2026 10:00:00 GMT</pubDate>
|
||||||
|
<category>planning</category>
|
||||||
|
</item>
|
||||||
|
</channel></rss>"""
|
||||||
|
|
||||||
|
ATOM = b"""<?xml version="1.0"?>
|
||||||
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||||
|
<id>https://example.test/feed</id><title>Updates</title>
|
||||||
|
<updated>2026-07-31T10:00:00Z</updated>
|
||||||
|
<link href="https://example.test/" />
|
||||||
|
<entry><id>update-1</id><title>Update one</title>
|
||||||
|
<updated>2026-07-31T10:00:00Z</updated>
|
||||||
|
<link href="https://example.test/update-1" />
|
||||||
|
</entry>
|
||||||
|
</feed>"""
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorFeedProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.provider = ConnectorFeedProvider()
|
||||||
|
|
||||||
|
def test_rss_and_atom_are_normalized_to_tabular_entries(self) -> None:
|
||||||
|
rss = self.provider.parse(RSS, source_url="https://example.test/rss")
|
||||||
|
atom = self.provider.parse(ATOM, source_url="https://example.test/atom")
|
||||||
|
|
||||||
|
self.assertEqual("rss", rss.format)
|
||||||
|
self.assertEqual("decision-1", rss.entries[0].id)
|
||||||
|
self.assertEqual("atom", atom.format)
|
||||||
|
self.assertEqual("https://example.test/update-1", atom.entries[0].url)
|
||||||
|
self.assertEqual("decision-1", feed_rows(rss)[0]["id"])
|
||||||
|
self.assertEqual(64, len(rss.sha256))
|
||||||
|
|
||||||
|
def test_fetch_records_transport_freshness_and_provenance(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_connectors.backend.feeds.fetch_http",
|
||||||
|
return_value=HttpFetchResponse(
|
||||||
|
status=200,
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/rss+xml",
|
||||||
|
"Cache-Control": "public, max-age=600",
|
||||||
|
"ETag": '"feed-1"',
|
||||||
|
},
|
||||||
|
body=RSS,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
document = self.provider.fetch("https://example.test/rss")
|
||||||
|
|
||||||
|
self.assertEqual('"feed-1"', document.etag)
|
||||||
|
self.assertIsNotNone(document.acquired_at)
|
||||||
|
self.assertEqual(600, int((document.fresh_until - document.acquired_at).total_seconds()))
|
||||||
|
self.assertEqual(len(RSS), document.metadata["byte_count"])
|
||||||
|
|
||||||
|
def test_render_filters_entries_by_explicit_visibility(self) -> None:
|
||||||
|
request = FeedRenderRequest(
|
||||||
|
format="atom",
|
||||||
|
title="Public updates",
|
||||||
|
feed_url="https://example.test/feed.atom",
|
||||||
|
home_url="https://example.test/",
|
||||||
|
entries=(
|
||||||
|
FeedEntry(id="public-1", title="Public", visibility="public"),
|
||||||
|
FeedEntry(id="tenant-1", title="Tenant", visibility="tenant"),
|
||||||
|
),
|
||||||
|
allowed_visibilities=frozenset({"public"}),
|
||||||
|
)
|
||||||
|
result = self.provider.render(request)
|
||||||
|
root = SafeET.fromstring(result.body)
|
||||||
|
|
||||||
|
self.assertEqual(1, result.included_entries)
|
||||||
|
self.assertEqual(1, result.excluded_entries)
|
||||||
|
self.assertIn(b"Public", result.body)
|
||||||
|
self.assertNotIn(b"Tenant", result.body)
|
||||||
|
self.assertTrue(root.tag.endswith("feed"))
|
||||||
|
|
||||||
|
def test_unsafe_xml_is_rejected(self) -> None:
|
||||||
|
payload = b'<!DOCTYPE x [<!ENTITY y SYSTEM "file:///etc/passwd">]><rss>&y;</rss>'
|
||||||
|
with self.assertRaisesRegex(FeedCapabilityError, "not safe or valid"):
|
||||||
|
self.provider.parse(payload, source_url="https://example.test/rss")
|
||||||
|
|
||||||
|
def test_render_api_derives_visibility_from_audience_and_permissions(self) -> None:
|
||||||
|
entries = [
|
||||||
|
{
|
||||||
|
"id": visibility,
|
||||||
|
"title": visibility.title(),
|
||||||
|
"visibility": visibility,
|
||||||
|
"source_kind": source_kind,
|
||||||
|
"source_module": source_module,
|
||||||
|
"source_ref": f"{source_kind}:{visibility}",
|
||||||
|
"source_revision": "7",
|
||||||
|
}
|
||||||
|
for visibility, source_kind, source_module in (
|
||||||
|
("public", "publication", "docs"),
|
||||||
|
("tenant", "case", "cases"),
|
||||||
|
("private", "report", "reporting"),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
public_payload = FeedRenderPayload(
|
||||||
|
format="rss",
|
||||||
|
title="Selected GovOPlaN updates",
|
||||||
|
feed_url="https://example.test/feed.xml",
|
||||||
|
home_url="https://example.test/",
|
||||||
|
audience="public",
|
||||||
|
entries=entries,
|
||||||
|
)
|
||||||
|
public = api_render_feed(
|
||||||
|
public_payload,
|
||||||
|
principal=_principal(FEED_PUBLISH_SCOPE),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("public", public.headers["x-govoplan-feed-audience"])
|
||||||
|
self.assertEqual("1", public.headers["x-govoplan-feed-included"])
|
||||||
|
self.assertEqual("2", public.headers["x-govoplan-feed-excluded"])
|
||||||
|
self.assertIn(b"Public", public.body)
|
||||||
|
self.assertNotIn(b"Tenant", public.body)
|
||||||
|
|
||||||
|
restricted_payload = public_payload.model_copy(update={"audience": "private"})
|
||||||
|
with self.assertRaises(HTTPException) as denied:
|
||||||
|
api_render_feed(
|
||||||
|
restricted_payload,
|
||||||
|
principal=_principal(FEED_PUBLISH_SCOPE),
|
||||||
|
)
|
||||||
|
self.assertEqual(403, denied.exception.status_code)
|
||||||
|
|
||||||
|
restricted = api_render_feed(
|
||||||
|
restricted_payload,
|
||||||
|
principal=_principal(
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("3", restricted.headers["x-govoplan-feed-included"])
|
||||||
|
self.assertIn(b"Private", restricted.body)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal(*scopes: str) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_runtime import (
|
||||||
|
GovernedConnectorError,
|
||||||
|
create_configuration,
|
||||||
|
execute_run,
|
||||||
|
list_configurations,
|
||||||
|
review_run,
|
||||||
|
update_configuration,
|
||||||
|
upsert_definition,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_schemas import (
|
||||||
|
ConnectorConfigurationCreateRequest,
|
||||||
|
ConnectorConfigurationUpdateRequest,
|
||||||
|
ConnectorDefinitionUpsertRequest,
|
||||||
|
ConnectorReviewRequest,
|
||||||
|
ConnectorRunRequest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(tenant_id: str = "tenant-1") -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(
|
||||||
|
{
|
||||||
|
"connectors:source:read",
|
||||||
|
"connectors:source:write",
|
||||||
|
"connectors:source:admin",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def definition_payload(
|
||||||
|
*,
|
||||||
|
mapping_version: str = "1",
|
||||||
|
package_ref: str = "municipal-addresses@1",
|
||||||
|
timeout_seconds: int = 10,
|
||||||
|
) -> ConnectorDefinitionUpsertRequest:
|
||||||
|
return ConnectorDefinitionUpsertRequest.model_validate(
|
||||||
|
{
|
||||||
|
"definition_key": "municipal.addresses",
|
||||||
|
"name": "Municipal addresses",
|
||||||
|
"description": "A package-managed reference connector.",
|
||||||
|
"origin": "package",
|
||||||
|
"package_ref": package_ref,
|
||||||
|
"specification": {
|
||||||
|
"provider": "municipal-directory",
|
||||||
|
"protocol": "rest",
|
||||||
|
"capabilities": ["discover", "read", "dry_run"],
|
||||||
|
"input_schema": {"type": "object"},
|
||||||
|
"output_schema": {"type": "object"},
|
||||||
|
"mapping": {
|
||||||
|
"version": mapping_version,
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"source": "external_id",
|
||||||
|
"target": "address.external_id",
|
||||||
|
"required": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "street",
|
||||||
|
"target": "address.street",
|
||||||
|
"required": True,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"validation_rules": [
|
||||||
|
{
|
||||||
|
"kind": "unique",
|
||||||
|
"field": "address.external_id",
|
||||||
|
"severity": "error",
|
||||||
|
"code": "addresses.external_id.ambiguous",
|
||||||
|
"message": "The external identifier is not unique.",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"dry_run": {
|
||||||
|
"supported": True,
|
||||||
|
"simulation_supported": True,
|
||||||
|
"max_items": 50,
|
||||||
|
"redacted_fields": ["address.street"],
|
||||||
|
"sample_rows": [
|
||||||
|
{"external_id": "A-1", "street": "Sample street"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"audit": {
|
||||||
|
"event_prefix": "connectors.municipal_addresses",
|
||||||
|
"expected_events": ["simulation.completed"],
|
||||||
|
"evidence_fields": ["input_hash", "configuration_hash"],
|
||||||
|
},
|
||||||
|
"privacy_classification": "confidential",
|
||||||
|
"retention_class": "connector-preview-30d",
|
||||||
|
"operational_limits": {"timeout_seconds": timeout_seconds},
|
||||||
|
"retry_policy": {"max_attempts": 2},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedConnectorRuntimeTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
self.tables = [
|
||||||
|
ConnectorDefinition.__table__,
|
||||||
|
ConnectorDefinitionRevision.__table__,
|
||||||
|
ConnectorConfiguration.__table__,
|
||||||
|
ConnectorSimulationRun.__table__,
|
||||||
|
]
|
||||||
|
Base.metadata.create_all(self.engine, tables=self.tables)
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
self.session = self.Session()
|
||||||
|
self.audit = patch(
|
||||||
|
"govoplan_connectors.backend.governed_runtime.audit_from_principal"
|
||||||
|
)
|
||||||
|
self.audit_mock = self.audit.start()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.audit.stop()
|
||||||
|
self.session.close()
|
||||||
|
Base.metadata.drop_all(self.engine, tables=reversed(self.tables))
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _configuration(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
ambiguity_policy: str = "manual_review",
|
||||||
|
):
|
||||||
|
definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(),
|
||||||
|
)
|
||||||
|
return create_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
ConnectorConfigurationCreateRequest(
|
||||||
|
definition_id=definition.id,
|
||||||
|
name=f"Address import {ambiguity_policy}",
|
||||||
|
endpoint_url="https://directory.example.invalid/v1",
|
||||||
|
credential_ref="vault://connectors/address-reader",
|
||||||
|
local_overrides={"retry_policy": {"max_attempts": 5}},
|
||||||
|
ambiguity_policy=ambiguity_policy,
|
||||||
|
status="active",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_package_update_is_explicit_and_preserves_local_overrides(self) -> None:
|
||||||
|
configuration = self._configuration()
|
||||||
|
|
||||||
|
updated_definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(
|
||||||
|
mapping_version="2",
|
||||||
|
package_ref="municipal-addresses@2",
|
||||||
|
timeout_seconds=20,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
unchanged = next(
|
||||||
|
item
|
||||||
|
for item in list_configurations(self.session, tenant_id="tenant-1")
|
||||||
|
if item.id == configuration.id
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, updated_definition.current_revision)
|
||||||
|
self.assertTrue(unchanged.update_available)
|
||||||
|
self.assertEqual("1", unchanged.effective_configuration["mapping"]["version"])
|
||||||
|
self.assertEqual(5, unchanged.effective_configuration["retry_policy"]["max_attempts"])
|
||||||
|
self.assertEqual(["retry_policy.max_attempts"], unchanged.protected_paths)
|
||||||
|
|
||||||
|
adopted = update_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
payload=ConnectorConfigurationUpdateRequest(
|
||||||
|
expected_revision=configuration.resource_revision,
|
||||||
|
adopt_latest_definition=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(adopted.update_available)
|
||||||
|
self.assertEqual("2", adopted.effective_configuration["mapping"]["version"])
|
||||||
|
self.assertEqual(
|
||||||
|
20,
|
||||||
|
adopted.effective_configuration["operational_limits"]["timeout_seconds"],
|
||||||
|
)
|
||||||
|
self.assertEqual(5, adopted.effective_configuration["retry_policy"]["max_attempts"])
|
||||||
|
self.assertEqual(["retry_policy.max_attempts"], adopted.protected_paths)
|
||||||
|
|
||||||
|
def test_ambiguous_simulation_requires_review_and_is_idempotent(self) -> None:
|
||||||
|
configuration = self._configuration()
|
||||||
|
payload = ConnectorRunRequest(
|
||||||
|
idempotency_key="simulation-1",
|
||||||
|
external_revision="directory-etag-22",
|
||||||
|
input_rows=[
|
||||||
|
{"external_id": "duplicate", "street": "First"},
|
||||||
|
{"external_id": "duplicate", "street": "Second"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
created = execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
replayed = execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(created.id, replayed.id)
|
||||||
|
self.assertEqual("manual_review", created.status)
|
||||||
|
self.assertEqual("pending", created.review_state)
|
||||||
|
self.assertEqual(2, created.summary["ambiguous"])
|
||||||
|
self.assertEqual("<redacted>", created.effects[0]["sample"]["address"]["street"])
|
||||||
|
self.assertEqual("directory-etag-22", created.provenance["external_revision"])
|
||||||
|
|
||||||
|
reviewed = review_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
run_id=created.id,
|
||||||
|
payload=ConnectorReviewRequest(
|
||||||
|
decision="approved",
|
||||||
|
reason="The duplicate rows represent an approved upstream alias.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("approved", reviewed.review_state)
|
||||||
|
self.assertEqual("review_approved", reviewed.status)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(GovernedConnectorError, "different run inputs"):
|
||||||
|
execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=ConnectorRunRequest(
|
||||||
|
idempotency_key="simulation-1",
|
||||||
|
input_rows=[{"external_id": "other", "street": "Other"}],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_ambiguity_policy_can_quarantine_or_reject(self) -> None:
|
||||||
|
for policy, expected_status, expected_review in (
|
||||||
|
("quarantine", "quarantined", "quarantined"),
|
||||||
|
("reject", "rejected", "not_required"),
|
||||||
|
):
|
||||||
|
configuration = self._configuration(ambiguity_policy=policy)
|
||||||
|
result = execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="dry_run",
|
||||||
|
payload=ConnectorRunRequest(
|
||||||
|
idempotency_key=f"{policy}-1",
|
||||||
|
input_rows=[
|
||||||
|
{"external_id": "same", "street": "First"},
|
||||||
|
{"external_id": "same", "street": "Second"},
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(expected_status, result.status)
|
||||||
|
self.assertEqual(expected_review, result.review_state)
|
||||||
|
|
||||||
|
def test_endpoint_credentials_and_stale_saves_are_rejected(self) -> None:
|
||||||
|
definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(GovernedConnectorError, "must not contain credentials"):
|
||||||
|
create_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
ConnectorConfigurationCreateRequest(
|
||||||
|
definition_id=definition.id,
|
||||||
|
name="Unsafe",
|
||||||
|
endpoint_url="https://user:secret@example.invalid/v1",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
configuration = self._configuration()
|
||||||
|
with self.assertRaisesRegex(GovernedConnectorError, "reload it before saving"):
|
||||||
|
update_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
payload=ConnectorConfigurationUpdateRequest(
|
||||||
|
expected_revision=configuration.resource_revision + 1,
|
||||||
|
name="Stale",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_definition_ownership_cannot_change_implicitly(self) -> None:
|
||||||
|
package_definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(),
|
||||||
|
)
|
||||||
|
local_payload = definition_payload().model_copy(
|
||||||
|
update={"origin": "local", "package_ref": None}
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
GovernedConnectorError,
|
||||||
|
"configuration overrides",
|
||||||
|
):
|
||||||
|
upsert_definition(self.session, principal(), local_payload)
|
||||||
|
|
||||||
|
self.assertFalse(package_definition.local_definition)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.sanctions import (
|
||||||
|
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsManifestTests(unittest.TestCase):
|
||||||
|
def test_manifest_exposes_versioned_tabular_capabilities(self) -> None:
|
||||||
|
self.assertEqual("connectors", manifest.id)
|
||||||
|
self.assertIn("access", manifest.optional_dependencies)
|
||||||
|
self.assertIn(
|
||||||
|
"connectors.tabular_sources",
|
||||||
|
{interface.name for interface in manifest.provides_interfaces},
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||||
|
manifest.capability_factories,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||||
|
manifest.capability_factories,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||||
|
manifest.capability_factories,
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(manifest.migration_spec)
|
||||||
|
self.assertEqual(
|
||||||
|
"@govoplan/connectors-webui",
|
||||||
|
manifest.frontend.package_name,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"connectors.governed-configuration",
|
||||||
|
{topic.id for topic in manifest.documentation},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from alembic.runtime.migration import MigrationContext
|
||||||
|
from sqlalchemy import create_engine, inspect
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.manifest import get_manifest
|
||||||
|
from govoplan_core.db.migrations import migrate_database
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsMigrationTests(unittest.TestCase):
|
||||||
|
def test_baseline_creates_connector_tables_and_head(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-connectors-migration-") as directory:
|
||||||
|
url = f"sqlite:///{Path(directory) / 'connectors.db'}"
|
||||||
|
migrate_database(
|
||||||
|
database_url=url,
|
||||||
|
enabled_modules=("connectors",),
|
||||||
|
manifest_factories=(get_manifest,),
|
||||||
|
)
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
self.assertIn(
|
||||||
|
"a8d9e0f1b2c3",
|
||||||
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
{
|
||||||
|
"connector_tabular_sources",
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
"connector_definitions",
|
||||||
|
"connector_definition_revisions",
|
||||||
|
"connector_configurations",
|
||||||
|
"connector_simulation_runs",
|
||||||
|
}.issubset(inspect(connection).get_table_names())
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.manifest import manifest
|
||||||
|
from govoplan_connectors.backend.provider_state import (
|
||||||
|
SANCTIONS_PROVIDER_ID,
|
||||||
|
TABULAR_PROVIDER_ID,
|
||||||
|
sanctions_provider_states,
|
||||||
|
tabular_provider_states,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.provider_governance import ExternalProviderStateContext
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsProviderStateTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
ConnectorTabularSource.__table__,
|
||||||
|
ConnectorSanctionsAcquisitionRun.__table__,
|
||||||
|
ConnectorSanctionsSnapshot.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_immutable_tabular_snapshot_reports_ready_state(self) -> None:
|
||||||
|
self.session.add(
|
||||||
|
ConnectorTabularSource(
|
||||||
|
id="tabular-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
source_name="monthly",
|
||||||
|
name="Monthly input",
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[{"name": "id", "type": "string"}],
|
||||||
|
rows=[{"id": "1"}],
|
||||||
|
fingerprint="a" * 64,
|
||||||
|
row_count=1,
|
||||||
|
byte_count=10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
state = tabular_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("healthy", state.health)
|
||||||
|
self.assertEqual("ready", state.recovery)
|
||||||
|
self.assertEqual("not_applicable", state.freshness)
|
||||||
|
|
||||||
|
def test_sanctions_state_hashes_binding_and_manifest_registers_state(self) -> None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
run = ConnectorSanctionsAcquisitionRun(
|
||||||
|
id="run-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="eu",
|
||||||
|
source_id="secret-source-name",
|
||||||
|
status="succeeded",
|
||||||
|
attempt_count=1,
|
||||||
|
started_at=now,
|
||||||
|
finished_at=now,
|
||||||
|
)
|
||||||
|
snapshot = ConnectorSanctionsSnapshot(
|
||||||
|
id="snapshot-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="eu",
|
||||||
|
publisher="European Union",
|
||||||
|
jurisdiction="EU",
|
||||||
|
list_type="sanctions",
|
||||||
|
source_id="secret-source-name",
|
||||||
|
source_version="2026-08-01",
|
||||||
|
acquired_at=now,
|
||||||
|
source_url="https://source.example.test/list.xml",
|
||||||
|
content_type="application/xml",
|
||||||
|
byte_count=8,
|
||||||
|
sha256="b" * 64,
|
||||||
|
parser_version="1",
|
||||||
|
connector_run_id=run.id,
|
||||||
|
raw_content=b"<list/>",
|
||||||
|
)
|
||||||
|
run.snapshot_id = snapshot.id
|
||||||
|
self.session.add_all((run, snapshot))
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
state = sanctions_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("healthy", state.health)
|
||||||
|
self.assertEqual("ready", state.recovery)
|
||||||
|
rendered = str(state.to_dict())
|
||||||
|
self.assertNotIn("secret-source-name", rendered)
|
||||||
|
self.assertNotIn("source.example.test", rendered)
|
||||||
|
self.assertEqual(
|
||||||
|
{TABULAR_PROVIDER_ID, SANCTIONS_PROVIDER_ID},
|
||||||
|
{item.provider_id for item in manifest.external_provider_state_providers},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,263 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.feeds import ConnectorFeedProvider
|
||||||
|
from govoplan_connectors.backend.recovery import (
|
||||||
|
CONNECTOR_RECOVERY_OPERATIONS,
|
||||||
|
ConnectorRecoveryError,
|
||||||
|
begin_connector_external_mutation,
|
||||||
|
begin_connector_read_snapshot,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.router import api_import_feed_snapshot
|
||||||
|
from govoplan_connectors.backend.schemas import FeedImportRequest
|
||||||
|
from govoplan_connectors.backend.tabular_sources import WRITE_SCOPE
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryCheckpoint,
|
||||||
|
RecoveryOperation,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
claim_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import TabularSnapshotInput
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
DistributedLease,
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.tabular_sources import SqlTabularSourceProvider
|
||||||
|
|
||||||
|
|
||||||
|
RSS = b"""<?xml version="1.0"?>
|
||||||
|
<rss version="2.0"><channel><title>Updates</title>
|
||||||
|
<link>https://example.test/</link><description>Updates</description>
|
||||||
|
<item><guid>1</guid><title>One</title></item></channel></rss>"""
|
||||||
|
|
||||||
|
|
||||||
|
def _identity(node: str, incarnation: str) -> RuntimeIdentity:
|
||||||
|
return RuntimeIdentity(
|
||||||
|
installation_id="connector-recovery-tests",
|
||||||
|
node_id=node,
|
||||||
|
incarnation=incarnation,
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="a" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal() -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset({WRITE_SCOPE}),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRecoveryTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
DistributedLease.__table__,
|
||||||
|
RecoveryOperation.__table__,
|
||||||
|
RecoveryCheckpoint.__table__,
|
||||||
|
ConnectorTabularSource.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine, expire_on_commit=False)
|
||||||
|
bind_process_runtime_identity(_identity("node-1", "incarnation-1"))
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_feed_snapshot_and_recovery_checkpoint_commit_atomically_and_replay(self) -> None:
|
||||||
|
document = ConnectorFeedProvider().parse(
|
||||||
|
RSS,
|
||||||
|
source_url="https://example.test/feed.xml",
|
||||||
|
)
|
||||||
|
payload = FeedImportRequest(
|
||||||
|
url="https://example.test/feed.xml",
|
||||||
|
name="Updates",
|
||||||
|
source_name="updates",
|
||||||
|
)
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_connectors.backend.router.feed_transport.fetch",
|
||||||
|
return_value=document,
|
||||||
|
) as fetch,
|
||||||
|
patch("govoplan_connectors.backend.router.audit_event"),
|
||||||
|
):
|
||||||
|
first = api_import_feed_snapshot(
|
||||||
|
payload,
|
||||||
|
session=self.session,
|
||||||
|
principal=_principal(),
|
||||||
|
idempotency_key="feed-import-1",
|
||||||
|
)
|
||||||
|
replay = api_import_feed_snapshot(
|
||||||
|
payload,
|
||||||
|
session=self.session,
|
||||||
|
principal=_principal(),
|
||||||
|
idempotency_key="feed-import-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(first.ref, replay.ref)
|
||||||
|
fetch.assert_called_once()
|
||||||
|
operation = self.session.scalar(select(RecoveryOperation))
|
||||||
|
assert operation is not None
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||||
|
self.assertEqual(
|
||||||
|
first.ref.removeprefix("snapshot:"),
|
||||||
|
operation.resource_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_recovery_metadata_distinguishes_reads_from_external_mutations(self) -> None:
|
||||||
|
declarations = {
|
||||||
|
item.operation_type: item for item in CONNECTOR_RECOVERY_OPERATIONS
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertFalse(declarations["read-snapshot"].provider_mutation)
|
||||||
|
self.assertTrue(declarations["read-snapshot"].implemented)
|
||||||
|
self.assertTrue(declarations["external-mutation"].provider_mutation)
|
||||||
|
self.assertFalse(declarations["external-mutation"].implemented)
|
||||||
|
|
||||||
|
def test_stale_atomic_connector_fence_fails_without_claiming_an_effect(self) -> None:
|
||||||
|
recovery = begin_connector_read_snapshot(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="provider-1",
|
||||||
|
idempotency_key="read-1",
|
||||||
|
source_revision="revision-1",
|
||||||
|
cursor="cursor-1",
|
||||||
|
dry_run_evidence={"performed": True, "approved": True},
|
||||||
|
)
|
||||||
|
lease = self.session.scalar(select(DistributedLease))
|
||||||
|
assert lease is not None
|
||||||
|
lease.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
|
||||||
|
self.session.commit()
|
||||||
|
bind_process_runtime_identity(_identity("node-2", "incarnation-2"))
|
||||||
|
|
||||||
|
with self.assertRaises(RecoveryOperationStateConflict):
|
||||||
|
claim_durable_recovery_operation(
|
||||||
|
recovery.operation.session_factory,
|
||||||
|
identity=_identity("node-2", "incarnation-2"),
|
||||||
|
operation_id=recovery.operation_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.FAILED.value, operation.status)
|
||||||
|
|
||||||
|
def test_external_mutation_unknown_outcome_blocks_blind_retry(self) -> None:
|
||||||
|
kwargs = {
|
||||||
|
"tenant_id": "tenant-1",
|
||||||
|
"provider_id": "provider-1",
|
||||||
|
"idempotency_key": "publish-1",
|
||||||
|
"request_sha256": "b" * 64,
|
||||||
|
"source_revision": "revision-1",
|
||||||
|
"cursor": None,
|
||||||
|
"dry_run_evidence": {"performed": True, "approved": True},
|
||||||
|
"resource_type": "external_record",
|
||||||
|
"resource_id": "record-1",
|
||||||
|
}
|
||||||
|
recovery = begin_connector_external_mutation(self.session, **kwargs)
|
||||||
|
recovery.outcome_unknown(
|
||||||
|
summary="The provider connection closed after dispatch",
|
||||||
|
provider_code="connection_closed",
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(ConnectorRecoveryError):
|
||||||
|
begin_connector_external_mutation(self.session, **kwargs)
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
|
||||||
|
|
||||||
|
def test_tampered_chain_rolls_back_the_atomic_connector_projection(self) -> None:
|
||||||
|
recovery = begin_connector_read_snapshot(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="provider-1",
|
||||||
|
idempotency_key="tampered-read",
|
||||||
|
source_revision=None,
|
||||||
|
cursor=None,
|
||||||
|
dry_run_evidence={"performed": False, "reason": "read-only"},
|
||||||
|
)
|
||||||
|
checkpoint = self.session.scalar(
|
||||||
|
select(RecoveryCheckpoint)
|
||||||
|
.where(RecoveryCheckpoint.operation_id == recovery.operation_id)
|
||||||
|
.order_by(RecoveryCheckpoint.sequence)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
assert checkpoint is not None
|
||||||
|
checkpoint.summary = "tampered"
|
||||||
|
self.session.commit()
|
||||||
|
source = SqlTabularSourceProvider().create_snapshot(
|
||||||
|
self.session,
|
||||||
|
_principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Tampered",
|
||||||
|
source_name="tampered",
|
||||||
|
rows=({"id": 1},),
|
||||||
|
),
|
||||||
|
source_id=recovery.resource_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(ConnectorRecoveryError):
|
||||||
|
recovery.commit_success(
|
||||||
|
self.session,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {"snapshot_ref": source.ref},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(
|
||||||
|
self.session.get(ConnectorTabularSource, recovery.resource_id)
|
||||||
|
)
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.RUNNING.value, operation.status)
|
||||||
|
|
||||||
|
def test_definitive_external_rejection_is_terminal(self) -> None:
|
||||||
|
recovery = begin_connector_external_mutation(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="provider-1",
|
||||||
|
idempotency_key="publish-rejected",
|
||||||
|
request_sha256="c" * 64,
|
||||||
|
source_revision="revision-1",
|
||||||
|
cursor=None,
|
||||||
|
dry_run_evidence={"performed": True, "approved": True},
|
||||||
|
resource_type="external_record",
|
||||||
|
resource_id="record-2",
|
||||||
|
)
|
||||||
|
recovery.reject(
|
||||||
|
summary="The provider rejected the requested revision",
|
||||||
|
provider_code="revision_conflict",
|
||||||
|
)
|
||||||
|
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.REJECTED.value, operation.status)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
import hashlib
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from urllib.error import URLError
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.sanctions_sources import (
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
SOURCE_DEFINITIONS,
|
||||||
|
SYNTHETIC_PROVIDER_ID,
|
||||||
|
SYNTHETIC_UN_XML,
|
||||||
|
SanctionsSourceError,
|
||||||
|
SqlSanctionsSnapshotProvider,
|
||||||
|
TransportResponse,
|
||||||
|
UNSC_PROVIDER_ID,
|
||||||
|
UrllibSanctionsTransport,
|
||||||
|
)
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryCheckpoint,
|
||||||
|
RecoveryOperation,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
DistributedLease,
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base, utcnow
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
scopes: tuple[str, ...] = (
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Transport:
|
||||||
|
def __init__(self, responses):
|
||||||
|
self.responses = list(responses)
|
||||||
|
self.headers = []
|
||||||
|
|
||||||
|
def fetch(self, definition, *, headers):
|
||||||
|
del definition
|
||||||
|
self.headers.append(dict(headers))
|
||||||
|
response = self.responses.pop(0)
|
||||||
|
if isinstance(response, Exception):
|
||||||
|
raise response
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def response(
|
||||||
|
content: bytes = SYNTHETIC_UN_XML,
|
||||||
|
*,
|
||||||
|
status: int = 200,
|
||||||
|
content_type: str = "application/xml",
|
||||||
|
etag: str = '"fixture-v1"',
|
||||||
|
) -> TransportResponse:
|
||||||
|
return TransportResponse(
|
||||||
|
status=status,
|
||||||
|
final_url="https://scsanctions.un.org/consolidated.xml",
|
||||||
|
headers={
|
||||||
|
"content-type": content_type,
|
||||||
|
"etag": etag,
|
||||||
|
},
|
||||||
|
content=content,
|
||||||
|
attempts=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSourcesTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
DistributedLease.__table__,
|
||||||
|
RecoveryOperation.__table__,
|
||||||
|
RecoveryCheckpoint.__table__,
|
||||||
|
ConnectorSanctionsAcquisitionRun.__table__,
|
||||||
|
ConnectorSanctionsSnapshot.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
bind_process_runtime_identity(
|
||||||
|
RuntimeIdentity(
|
||||||
|
installation_id="connectors-tests",
|
||||||
|
node_id="connectors-test-node",
|
||||||
|
incarnation="connectors-test-incarnation",
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="a" * 64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_fixture_refreshes_are_immutable_and_evidence_is_readable(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider()
|
||||||
|
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
second = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertEqual("succeeded", first.status)
|
||||||
|
self.assertEqual("succeeded", second.status)
|
||||||
|
self.assertNotEqual(first.snapshot.ref, second.snapshot.ref)
|
||||||
|
self.assertEqual(
|
||||||
|
hashlib.sha256(SYNTHETIC_UN_XML).hexdigest(),
|
||||||
|
first.snapshot.sha256,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
SYNTHETIC_UN_XML,
|
||||||
|
provider.read_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot_ref=first.snapshot.ref,
|
||||||
|
).content,
|
||||||
|
)
|
||||||
|
runs = provider.list_runs(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
all(
|
||||||
|
run.request_evidence["subject_data_transmitted"]
|
||||||
|
is False
|
||||||
|
for run in runs
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_conditional_fetch_reuses_prior_immutable_snapshot(self) -> None:
|
||||||
|
transport = _Transport(
|
||||||
|
(
|
||||||
|
response(),
|
||||||
|
response(b"", status=304),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
provider = SqlSanctionsSnapshotProvider(transport)
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
second = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("not_modified", second.status)
|
||||||
|
self.assertEqual(first.snapshot.ref, second.snapshot.ref)
|
||||||
|
self.assertEqual(
|
||||||
|
{'If-None-Match': '"fixture-v1"'},
|
||||||
|
transport.headers[1],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
1,
|
||||||
|
self.session.query(ConnectorSanctionsSnapshot).count(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_malformed_and_changed_sources_have_explicit_health(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
cases = (
|
||||||
|
(
|
||||||
|
b"<CONSOLIDATED_LIST>",
|
||||||
|
"application/xml",
|
||||||
|
"malformed",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
b"<DIFFERENT><INDIVIDUALS/><ENTITIES/></DIFFERENT>",
|
||||||
|
"application/xml",
|
||||||
|
"unexpected_change",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
SYNTHETIC_UN_XML,
|
||||||
|
"text/html",
|
||||||
|
"unexpected_change",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for payload, content_type, expected in cases:
|
||||||
|
with self.subTest(expected=expected, content_type=content_type):
|
||||||
|
provider = SqlSanctionsSnapshotProvider(
|
||||||
|
_Transport(
|
||||||
|
(
|
||||||
|
response(
|
||||||
|
payload,
|
||||||
|
content_type=content_type,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
self.assertEqual(expected, result.status)
|
||||||
|
self.assertIsNotNone(result.error)
|
||||||
|
|
||||||
|
def test_unavailable_source_becomes_stale_when_evidence_is_old(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider(
|
||||||
|
_Transport(
|
||||||
|
(
|
||||||
|
response(),
|
||||||
|
SanctionsSourceError("offline"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
record = self.session.get(
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
first.snapshot.ref.removeprefix("sanctions-snapshot:"),
|
||||||
|
)
|
||||||
|
record.acquired_at = utcnow() - timedelta(days=3)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
result = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("stale", result.status)
|
||||||
|
self.assertEqual(first.snapshot.ref, result.snapshot.ref)
|
||||||
|
|
||||||
|
def test_idempotent_refresh_replays_the_committed_acquisition(self) -> None:
|
||||||
|
transport = _Transport((response(),))
|
||||||
|
provider = SqlSanctionsSnapshotProvider(transport)
|
||||||
|
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
idempotency_key="scheduled-refresh-1",
|
||||||
|
)
|
||||||
|
replay = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
idempotency_key="scheduled-refresh-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(first.run_id, replay.run_id)
|
||||||
|
self.assertEqual(first.snapshot.ref, replay.snapshot.ref)
|
||||||
|
self.assertEqual([], transport.responses)
|
||||||
|
operation = self.session.query(RecoveryOperation).one()
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||||
|
|
||||||
|
def test_provider_failure_commits_failed_run_and_terminal_recovery(self) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider(
|
||||||
|
_Transport((SanctionsSourceError("offline"),))
|
||||||
|
)
|
||||||
|
|
||||||
|
result = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("unavailable", result.status)
|
||||||
|
operation = self.session.query(RecoveryOperation).one()
|
||||||
|
self.assertEqual(RecoveryStatus.FAILED.value, operation.status)
|
||||||
|
self.assertEqual(
|
||||||
|
result.run_id,
|
||||||
|
self.session.query(ConnectorSanctionsAcquisitionRun).one().id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_snapshot_access_is_tenant_and_scope_isolated(self) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider()
|
||||||
|
created = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(
|
||||||
|
provider.get_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
snapshot_ref=created.snapshot.ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(Exception, "Missing scope"):
|
||||||
|
provider.list_snapshots(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_transport_retries_transient_network_failures(self) -> None:
|
||||||
|
class _Headers(dict):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class _Response:
|
||||||
|
status = 200
|
||||||
|
headers = _Headers(
|
||||||
|
{
|
||||||
|
"Content-Type": "application/xml",
|
||||||
|
"Content-Length": str(len(SYNTHETIC_UN_XML)),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def geturl(self):
|
||||||
|
return (
|
||||||
|
"https://scsanctions.un.org/"
|
||||||
|
"resources/xml/en/consolidated.xml"
|
||||||
|
)
|
||||||
|
|
||||||
|
def read(self, size):
|
||||||
|
del size
|
||||||
|
if hasattr(self, "_read"):
|
||||||
|
return b""
|
||||||
|
self._read = True
|
||||||
|
return SYNTHETIC_UN_XML
|
||||||
|
|
||||||
|
opener = unittest.mock.Mock()
|
||||||
|
opener.open.side_effect = (
|
||||||
|
URLError("temporary"),
|
||||||
|
URLError("temporary"),
|
||||||
|
_Response(),
|
||||||
|
)
|
||||||
|
sleeps = []
|
||||||
|
transport = UrllibSanctionsTransport(
|
||||||
|
sleeper=sleeps.append
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_connectors.backend.sanctions_sources.build_opener",
|
||||||
|
return_value=opener,
|
||||||
|
):
|
||||||
|
fetched = transport.fetch(
|
||||||
|
SOURCE_DEFINITIONS[UNSC_PROVIDER_ID],
|
||||||
|
headers={},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(3, fetched.attempts)
|
||||||
|
self.assertEqual([1.0, 2.0], sleeps)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,268 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularSnapshotInput,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.router import api_create_tabular_snapshot
|
||||||
|
from govoplan_connectors.backend.schemas import SnapshotCreateRequest
|
||||||
|
from govoplan_connectors.backend.tabular_sources import (
|
||||||
|
READ_SCOPE,
|
||||||
|
WRITE_SCOPE,
|
||||||
|
SqlTabularSourceProvider,
|
||||||
|
parse_csv_snapshot,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
scopes: tuple[str, ...] = (READ_SCOPE, WRITE_SCOPE),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsTabularSourceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine, tables=[ConnectorTabularSource.__table__])
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
self.session = self.Session()
|
||||||
|
self.provider = SqlTabularSourceProvider()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
Base.metadata.drop_all(self.engine, tables=[ConnectorTabularSource.__table__])
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_snapshot_round_trip_preserves_schema_fingerprint_and_bounds(self) -> None:
|
||||||
|
created = self.provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Monthly cases",
|
||||||
|
source_name="monthly_cases_2026_07",
|
||||||
|
rows=(
|
||||||
|
{"case_id": "A-1", "amount": 12, "active": True},
|
||||||
|
{"case_id": "A-2", "amount": None, "active": False},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
listed = self.provider.list_sources(self.session, principal())
|
||||||
|
preview = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
limit=1,
|
||||||
|
expected_fingerprint=created.fingerprint,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((created.ref,), tuple(source.ref for source in listed))
|
||||||
|
self.assertEqual(
|
||||||
|
["case_id", "amount", "active"],
|
||||||
|
[column.name for column in created.schema],
|
||||||
|
)
|
||||||
|
self.assertEqual(2, preview.total_rows)
|
||||||
|
self.assertEqual(1, len(preview.rows))
|
||||||
|
self.assertTrue(preview.truncated)
|
||||||
|
self.assertEqual(created.fingerprint, preview.source.fingerprint)
|
||||||
|
self.assertEqual("cached", preview.source.source_mode)
|
||||||
|
self.assertTrue(preview.source.pushdown.projections)
|
||||||
|
self.assertTrue(preview.source.pushdown.pagination)
|
||||||
|
self.assertEqual("healthy", preview.source.health.status)
|
||||||
|
self.assertGreater(preview.returned_bytes, 2)
|
||||||
|
self.assertEqual(1, preview.effective_row_limit)
|
||||||
|
self.assertEqual("preview.row_limit_reached", preview.diagnostics[0].code)
|
||||||
|
|
||||||
|
def test_preview_enforces_byte_time_and_provider_ceiling_budgets(self) -> None:
|
||||||
|
created = self.provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Bounded",
|
||||||
|
source_name="bounded",
|
||||||
|
rows=(
|
||||||
|
{"id": 1, "value": "first"},
|
||||||
|
{"id": 2, "value": "second"},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
bounded = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
limit=500,
|
||||||
|
max_bytes=35,
|
||||||
|
timeout_ms=2_000,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(bounded.rows))
|
||||||
|
self.assertTrue(bounded.truncated)
|
||||||
|
self.assertEqual(
|
||||||
|
"preview.byte_limit_reached",
|
||||||
|
bounded.diagnostics[-1].code,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceValidationError,
|
||||||
|
"single source row exceeds",
|
||||||
|
):
|
||||||
|
self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
max_bytes=2,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
tightened = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
limit=5_000,
|
||||||
|
max_bytes=5_000_000,
|
||||||
|
timeout_ms=10_000,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(500, tightened.effective_row_limit)
|
||||||
|
self.assertEqual(1_000_000, tightened.effective_byte_limit)
|
||||||
|
self.assertEqual(2_000, tightened.effective_timeout_ms)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
"preview.row_limit_tightened",
|
||||||
|
"preview.byte_limit_tightened",
|
||||||
|
"preview.timeout_tightened",
|
||||||
|
},
|
||||||
|
{item.code for item in tightened.diagnostics},
|
||||||
|
)
|
||||||
|
|
||||||
|
times = iter((0.0, 0.01))
|
||||||
|
timeout_provider = SqlTabularSourceProvider(clock=lambda: next(times))
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
"time budget",
|
||||||
|
):
|
||||||
|
timeout_provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
timeout_ms=1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_tenant_and_scope_isolation_are_enforced(self) -> None:
|
||||||
|
created = self.provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Private",
|
||||||
|
source_name="private_source",
|
||||||
|
rows=({"id": 1},),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertEqual((), self.provider.list_sources(self.session, principal("tenant-2")))
|
||||||
|
self.assertIsNone(
|
||||||
|
self.provider.get_source(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
source_ref=created.ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with self.assertRaises(TabularSourceAccessError):
|
||||||
|
self.provider.list_sources(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_duplicate_source_name_and_stale_fingerprint_are_rejected(self) -> None:
|
||||||
|
snapshot = TabularSnapshotInput(
|
||||||
|
name="Cases",
|
||||||
|
source_name="cases",
|
||||||
|
rows=({"id": 1},),
|
||||||
|
)
|
||||||
|
created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
expected_fingerprint="stale",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_csv_parser_infers_scalar_values_and_rejects_duplicate_headers(self) -> None:
|
||||||
|
rows = parse_csv_snapshot(
|
||||||
|
"\ufeffid;amount;active;note\n0012;12.5;true;\n2;7;false;ok\n",
|
||||||
|
delimiter=";",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
(
|
||||||
|
{"id": "0012", "amount": 12.5, "active": True, "note": None},
|
||||||
|
{"id": 2, "amount": 7, "active": False, "note": "ok"},
|
||||||
|
),
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
parse_csv_snapshot("id,id\n1,2\n", delimiter=",")
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
parse_csv_snapshot("id,name\n1,Ada,extra\n", delimiter=",")
|
||||||
|
|
||||||
|
def test_malformed_csv_api_request_is_reported_as_validation_error(self) -> None:
|
||||||
|
payload = SnapshotCreateRequest(
|
||||||
|
name="Malformed",
|
||||||
|
source_name="malformed",
|
||||||
|
format="csv",
|
||||||
|
csv_text="id,name\n1,Ada,extra\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(HTTPException) as raised:
|
||||||
|
api_create_tabular_snapshot(
|
||||||
|
payload,
|
||||||
|
session=self.session,
|
||||||
|
principal=principal(),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(422, raised.exception.status_code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"name": "@govoplan/connectors-webui",
|
||||||
|
"version": "0.1.19",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"main": "src/index.ts",
|
||||||
|
"module": "src/index.ts",
|
||||||
|
"types": "src/index.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"types": "./src/index.ts",
|
||||||
|
"import": "./src/index.ts"
|
||||||
|
},
|
||||||
|
"./styles/connectors.css": "./src/styles/connectors.css"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"@govoplan/core-webui": "^0.1.18",
|
||||||
|
"react": ">=19.2.7 <20",
|
||||||
|
"react-dom": ">=19.2.7 <20"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"@govoplan/core-webui": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"test:connector-governance-ui": "node tests/connector-governance-ui-structure.test.mjs"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
import { apiFetch, apiPath, type ApiSettings } from "@govoplan/core-webui";
|
||||||
|
|
||||||
|
export type GovernedConnectorSpecification = {
|
||||||
|
provider: string;
|
||||||
|
protocol: string;
|
||||||
|
capabilities: string[];
|
||||||
|
input_schema: Record<string, unknown>;
|
||||||
|
output_schema: Record<string, unknown>;
|
||||||
|
mapping: {
|
||||||
|
version: string;
|
||||||
|
rules: Array<{
|
||||||
|
source: string;
|
||||||
|
target: string;
|
||||||
|
required?: boolean;
|
||||||
|
default?: unknown;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
validation_rules: Array<{
|
||||||
|
kind: "required" | "one_of" | "unique";
|
||||||
|
field: string;
|
||||||
|
values?: unknown[];
|
||||||
|
severity: "warning" | "error";
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
}>;
|
||||||
|
dry_run: {
|
||||||
|
supported: boolean;
|
||||||
|
simulation_supported: boolean;
|
||||||
|
sample_rows: Array<Record<string, unknown>>;
|
||||||
|
max_items: number;
|
||||||
|
redacted_fields: string[];
|
||||||
|
};
|
||||||
|
audit: {
|
||||||
|
event_prefix: string;
|
||||||
|
expected_events: string[];
|
||||||
|
evidence_fields: string[];
|
||||||
|
};
|
||||||
|
privacy_classification: "public" | "internal" | "confidential" | "restricted";
|
||||||
|
retention_class: string;
|
||||||
|
operational_limits: Record<string, unknown>;
|
||||||
|
retry_policy: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorDefinition = {
|
||||||
|
id: string;
|
||||||
|
definition_key: string;
|
||||||
|
name: string;
|
||||||
|
description?: string | null;
|
||||||
|
status: string;
|
||||||
|
current_revision: number;
|
||||||
|
source_package?: string | null;
|
||||||
|
local_definition: boolean;
|
||||||
|
definition_hash: string;
|
||||||
|
specification: GovernedConnectorSpecification;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorConfiguration = {
|
||||||
|
id: string;
|
||||||
|
definition_id: string;
|
||||||
|
definition_key: string;
|
||||||
|
definition_name: string;
|
||||||
|
name: string;
|
||||||
|
status: "draft" | "active" | "disabled";
|
||||||
|
endpoint_url?: string | null;
|
||||||
|
credential_ref?: string | null;
|
||||||
|
base_definition_revision: number;
|
||||||
|
latest_definition_revision: number;
|
||||||
|
update_available: boolean;
|
||||||
|
local_overrides: Record<string, unknown>;
|
||||||
|
protected_paths: string[];
|
||||||
|
effective_configuration: GovernedConnectorSpecification;
|
||||||
|
effective_hash: string;
|
||||||
|
resource_revision: number;
|
||||||
|
ambiguity_policy: "manual_review" | "quarantine" | "reject";
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorRun = {
|
||||||
|
id: string;
|
||||||
|
configuration_id: string;
|
||||||
|
mode: "dry_run" | "simulation";
|
||||||
|
idempotency_key: string;
|
||||||
|
status: string;
|
||||||
|
review_state: string;
|
||||||
|
definition_revision: number;
|
||||||
|
configuration_revision: number;
|
||||||
|
configuration_hash: string;
|
||||||
|
input_hash: string;
|
||||||
|
summary: Record<string, number | boolean>;
|
||||||
|
effects: Array<Record<string, unknown>>;
|
||||||
|
diagnostics: Array<Record<string, unknown>>;
|
||||||
|
provenance: Record<string, unknown>;
|
||||||
|
reviewed_by?: string | null;
|
||||||
|
reviewed_at?: string | null;
|
||||||
|
review_reason?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorConfigurationDraft = {
|
||||||
|
name: string;
|
||||||
|
status: ConnectorConfiguration["status"];
|
||||||
|
endpoint_url: string;
|
||||||
|
credential_ref: string;
|
||||||
|
local_overrides: string;
|
||||||
|
ambiguity_policy: ConnectorConfiguration["ambiguity_policy"];
|
||||||
|
};
|
||||||
|
|
||||||
|
const ROOT = "/api/v1/connectors/governed";
|
||||||
|
|
||||||
|
export async function listConnectorDefinitions(
|
||||||
|
settings: ApiSettings
|
||||||
|
): Promise<ConnectorDefinition[]> {
|
||||||
|
const response = await apiFetch<{ items: ConnectorDefinition[] }>(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/definitions`
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function upsertConnectorDefinition(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorDefinition> {
|
||||||
|
return apiFetch(settings, `${ROOT}/definitions`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listConnectorConfigurations(
|
||||||
|
settings: ApiSettings
|
||||||
|
): Promise<ConnectorConfiguration[]> {
|
||||||
|
const response = await apiFetch<{ items: ConnectorConfiguration[] }>(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/configurations`
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createConnectorConfiguration(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorConfiguration> {
|
||||||
|
return apiFetch(settings, `${ROOT}/configurations`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function updateConnectorConfiguration(
|
||||||
|
settings: ApiSettings,
|
||||||
|
configurationId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorConfiguration> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/configurations/${encodeURIComponent(configurationId)}`,
|
||||||
|
{ method: "PUT", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listConnectorRuns(
|
||||||
|
settings: ApiSettings,
|
||||||
|
configurationId?: string
|
||||||
|
): Promise<ConnectorRun[]> {
|
||||||
|
const response = await apiFetch<{ items: ConnectorRun[] }>(
|
||||||
|
settings,
|
||||||
|
apiPath(`${ROOT}/runs`, {
|
||||||
|
configuration_id: configurationId || undefined,
|
||||||
|
limit: 100
|
||||||
|
})
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function executeConnectorRun(
|
||||||
|
settings: ApiSettings,
|
||||||
|
configurationId: string,
|
||||||
|
mode: "dry-runs" | "simulations",
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorRun> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/configurations/${encodeURIComponent(configurationId)}/${mode}`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reviewConnectorRun(
|
||||||
|
settings: ApiSettings,
|
||||||
|
runId: string,
|
||||||
|
decision: "approved" | "rejected",
|
||||||
|
reason: string
|
||||||
|
): Promise<ConnectorRun> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/runs/${encodeURIComponent(runId)}/review`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ decision, reason })
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,661 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
AdminPageLayout,
|
||||||
|
Button,
|
||||||
|
Card,
|
||||||
|
DataGrid,
|
||||||
|
Dialog,
|
||||||
|
FilterBar,
|
||||||
|
FormField,
|
||||||
|
FormGrid,
|
||||||
|
MetricCard,
|
||||||
|
MetricGrid,
|
||||||
|
PageActionBar,
|
||||||
|
SelectionList,
|
||||||
|
SelectionListItem,
|
||||||
|
SelectionListItemContent,
|
||||||
|
StatePanel,
|
||||||
|
StatusBadge,
|
||||||
|
TableActionGroup,
|
||||||
|
WorkspaceLayout,
|
||||||
|
formatDateTime,
|
||||||
|
hasScope,
|
||||||
|
useUnsavedChanges,
|
||||||
|
useUnsavedDraftGuard,
|
||||||
|
type ApiSettings,
|
||||||
|
type AuthInfo,
|
||||||
|
type DataGridColumn
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import {
|
||||||
|
createConnectorConfiguration,
|
||||||
|
executeConnectorRun,
|
||||||
|
listConnectorConfigurations,
|
||||||
|
listConnectorDefinitions,
|
||||||
|
listConnectorRuns,
|
||||||
|
reviewConnectorRun,
|
||||||
|
updateConnectorConfiguration,
|
||||||
|
upsertConnectorDefinition,
|
||||||
|
type ConnectorConfiguration,
|
||||||
|
type ConnectorConfigurationDraft,
|
||||||
|
type ConnectorDefinition,
|
||||||
|
type ConnectorRun
|
||||||
|
} from "../api/governedConnectors";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
settings: ApiSettings;
|
||||||
|
auth: AuthInfo;
|
||||||
|
};
|
||||||
|
|
||||||
|
const EMPTY_DRAFT: ConnectorConfigurationDraft = {
|
||||||
|
name: "",
|
||||||
|
status: "draft",
|
||||||
|
endpoint_url: "",
|
||||||
|
credential_ref: "",
|
||||||
|
local_overrides: "{}",
|
||||||
|
ambiguity_policy: "manual_review"
|
||||||
|
};
|
||||||
|
|
||||||
|
const EXAMPLE_DEFINITION = JSON.stringify({
|
||||||
|
definition_key: "example.reference-data",
|
||||||
|
name: "Example reference data",
|
||||||
|
description: "Locally governed example connector",
|
||||||
|
origin: "local",
|
||||||
|
specification: {
|
||||||
|
provider: "example-provider",
|
||||||
|
protocol: "rest",
|
||||||
|
capabilities: ["discover", "read", "dry_run"],
|
||||||
|
input_schema: { type: "object" },
|
||||||
|
output_schema: { type: "object" },
|
||||||
|
mapping: {
|
||||||
|
version: "1",
|
||||||
|
rules: [{ source: "id", target: "record.id", required: true }]
|
||||||
|
},
|
||||||
|
validation_rules: [{
|
||||||
|
kind: "unique",
|
||||||
|
field: "record.id",
|
||||||
|
severity: "error",
|
||||||
|
code: "record.id.ambiguous",
|
||||||
|
message: "The record identifier is not unique."
|
||||||
|
}],
|
||||||
|
dry_run: {
|
||||||
|
supported: true,
|
||||||
|
simulation_supported: true,
|
||||||
|
sample_rows: [{ id: "sample-1" }],
|
||||||
|
max_items: 500,
|
||||||
|
redacted_fields: []
|
||||||
|
},
|
||||||
|
audit: {
|
||||||
|
event_prefix: "connectors.example",
|
||||||
|
expected_events: ["simulation.completed"],
|
||||||
|
evidence_fields: ["input_hash", "configuration_hash"]
|
||||||
|
},
|
||||||
|
privacy_classification: "internal",
|
||||||
|
retention_class: "connector-preview-30d",
|
||||||
|
operational_limits: { timeout_seconds: 30 },
|
||||||
|
retry_policy: { max_attempts: 2 }
|
||||||
|
}
|
||||||
|
}, null, 2);
|
||||||
|
|
||||||
|
export default function ConnectorGovernancePage({ settings, auth }: Props) {
|
||||||
|
const [definitions, setDefinitions] = useState<ConnectorDefinition[]>([]);
|
||||||
|
const [configurations, setConfigurations] = useState<ConnectorConfiguration[]>([]);
|
||||||
|
const [runs, setRuns] = useState<ConnectorRun[]>([]);
|
||||||
|
const [selectedId, setSelectedId] = useState("");
|
||||||
|
const [draft, setDraft] = useState<ConnectorConfigurationDraft>(EMPTY_DRAFT);
|
||||||
|
const [savedKey, setSavedKey] = useState("");
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
const [definitionOpen, setDefinitionOpen] = useState(false);
|
||||||
|
const [definitionJson, setDefinitionJson] = useState(EXAMPLE_DEFINITION);
|
||||||
|
const [configurationOpen, setConfigurationOpen] = useState(false);
|
||||||
|
const [newDefinitionId, setNewDefinitionId] = useState("");
|
||||||
|
const [newDraft, setNewDraft] = useState<ConnectorConfigurationDraft>(EMPTY_DRAFT);
|
||||||
|
const [sampleJson, setSampleJson] = useState("[]");
|
||||||
|
const [externalRevision, setExternalRevision] = useState("");
|
||||||
|
const [reviewRun, setReviewRun] = useState<ConnectorRun | null>(null);
|
||||||
|
const [reviewReason, setReviewReason] = useState("");
|
||||||
|
const { requestDiscard } = useUnsavedChanges();
|
||||||
|
|
||||||
|
const selected = configurations.find((item) => item.id === selectedId) ?? null;
|
||||||
|
const canAdmin = hasScope(auth, "connectors:source:admin");
|
||||||
|
const canExecute = canAdmin || hasScope(auth, "connectors:source:write");
|
||||||
|
const dirty = Boolean(selected && draftKey(draft) !== savedKey);
|
||||||
|
|
||||||
|
const applyConfiguration = useCallback((item: ConnectorConfiguration | null) => {
|
||||||
|
const next = item ? draftFromConfiguration(item) : EMPTY_DRAFT;
|
||||||
|
setDraft(next);
|
||||||
|
setSavedKey(item ? draftKey(next) : "");
|
||||||
|
setSampleJson(JSON.stringify(
|
||||||
|
item?.effective_configuration.dry_run.sample_rows ?? [],
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const reload = useCallback(async (preferredId?: string) => {
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const [nextDefinitions, nextConfigurations] = await Promise.all([
|
||||||
|
listConnectorDefinitions(settings),
|
||||||
|
listConnectorConfigurations(settings)
|
||||||
|
]);
|
||||||
|
const nextId = preferredId && nextConfigurations.some((item) => item.id === preferredId)
|
||||||
|
? preferredId
|
||||||
|
: nextConfigurations.some((item) => item.id === selectedId)
|
||||||
|
? selectedId
|
||||||
|
: nextConfigurations[0]?.id ?? "";
|
||||||
|
const nextRuns = await listConnectorRuns(settings, nextId || undefined);
|
||||||
|
setDefinitions(nextDefinitions);
|
||||||
|
setConfigurations(nextConfigurations);
|
||||||
|
setRuns(nextRuns);
|
||||||
|
setSelectedId(nextId);
|
||||||
|
applyConfiguration(nextConfigurations.find((item) => item.id === nextId) ?? null);
|
||||||
|
if (!newDefinitionId && nextDefinitions[0]) setNewDefinitionId(nextDefinitions[0].id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}, [applyConfiguration, newDefinitionId, selectedId, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void reload();
|
||||||
|
}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
||||||
|
|
||||||
|
const visibleConfigurations = useMemo(() => {
|
||||||
|
const needle = search.trim().toLocaleLowerCase();
|
||||||
|
return configurations.filter((item) => !needle ||
|
||||||
|
`${item.name} ${item.definition_name} ${item.status}`
|
||||||
|
.toLocaleLowerCase()
|
||||||
|
.includes(needle));
|
||||||
|
}, [configurations, search]);
|
||||||
|
|
||||||
|
const save = async (): Promise<boolean> => {
|
||||||
|
if (!selected || !canAdmin) return false;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const overrides = parseObject(draft.local_overrides, "Local overrides");
|
||||||
|
const updated = await updateConnectorConfiguration(settings, selected.id, {
|
||||||
|
expected_revision: selected.resource_revision,
|
||||||
|
name: draft.name.trim(),
|
||||||
|
status: draft.status,
|
||||||
|
endpoint_url: draft.endpoint_url.trim() || null,
|
||||||
|
credential_ref: draft.credential_ref.trim() || null,
|
||||||
|
local_overrides: overrides,
|
||||||
|
ambiguity_policy: draft.ambiguity_policy
|
||||||
|
});
|
||||||
|
setSuccess("Connector configuration saved.");
|
||||||
|
await reload(updated.id);
|
||||||
|
return true;
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useUnsavedDraftGuard({
|
||||||
|
dirty,
|
||||||
|
onSave: save,
|
||||||
|
onDiscard: () => applyConfiguration(selected),
|
||||||
|
title: "Unsaved connector changes",
|
||||||
|
message: "Save or discard the current connector changes before continuing."
|
||||||
|
});
|
||||||
|
|
||||||
|
const selectConfiguration = (item: ConnectorConfiguration) => {
|
||||||
|
if (item.id === selectedId) return;
|
||||||
|
requestDiscard(() => {
|
||||||
|
setSelectedId(item.id);
|
||||||
|
applyConfiguration(item);
|
||||||
|
setRuns([]);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
void listConnectorRuns(settings, item.id).then(setRuns).catch((caught) => {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const createDefinition = async () => {
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const payload = parseObject(definitionJson, "Definition");
|
||||||
|
const created = await upsertConnectorDefinition(settings, payload);
|
||||||
|
setDefinitionOpen(false);
|
||||||
|
setSuccess("Connector definition revision saved.");
|
||||||
|
setNewDefinitionId(created.id);
|
||||||
|
await reload(selectedId);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const createConfiguration = async () => {
|
||||||
|
if (!newDefinitionId || !newDraft.name.trim()) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const created = await createConnectorConfiguration(settings, {
|
||||||
|
definition_id: newDefinitionId,
|
||||||
|
name: newDraft.name.trim(),
|
||||||
|
status: newDraft.status,
|
||||||
|
endpoint_url: newDraft.endpoint_url.trim() || null,
|
||||||
|
credential_ref: newDraft.credential_ref.trim() || null,
|
||||||
|
local_overrides: parseObject(newDraft.local_overrides, "Local overrides"),
|
||||||
|
ambiguity_policy: newDraft.ambiguity_policy
|
||||||
|
});
|
||||||
|
setConfigurationOpen(false);
|
||||||
|
setNewDraft(EMPTY_DRAFT);
|
||||||
|
setSuccess("Connector configuration created.");
|
||||||
|
await reload(created.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const adoptUpdate = async () => {
|
||||||
|
if (!selected || dirty || !selected.update_available) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const updated = await updateConnectorConfiguration(settings, selected.id, {
|
||||||
|
expected_revision: selected.resource_revision,
|
||||||
|
adopt_latest_definition: true
|
||||||
|
});
|
||||||
|
setSuccess("Package revision adopted; protected local overrides were reapplied.");
|
||||||
|
await reload(updated.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const run = async (mode: "dry-runs" | "simulations") => {
|
||||||
|
if (!selected || dirty) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const inputRows = parseRows(sampleJson);
|
||||||
|
const created = await executeConnectorRun(settings, selected.id, mode, {
|
||||||
|
idempotency_key: `${mode}-${crypto.randomUUID()}`,
|
||||||
|
input_rows: inputRows,
|
||||||
|
external_revision: externalRevision.trim() || null
|
||||||
|
});
|
||||||
|
setSuccess(`${mode === "dry-runs" ? "Dry-run" : "Simulation"} completed with status ${created.status}.`);
|
||||||
|
setRuns(await listConnectorRuns(settings, selected.id));
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const decideReview = async (decision: "approved" | "rejected") => {
|
||||||
|
if (!reviewRun || reviewReason.trim().length < 5) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
await reviewConnectorRun(settings, reviewRun.id, decision, reviewReason.trim());
|
||||||
|
setReviewRun(null);
|
||||||
|
setReviewReason("");
|
||||||
|
setSuccess(`Simulation ${decision}.`);
|
||||||
|
setRuns(await listConnectorRuns(settings, selectedId));
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const runColumns = useMemo<DataGridColumn<ConnectorRun>[]>(() => [
|
||||||
|
{
|
||||||
|
id: "created",
|
||||||
|
header: "Run",
|
||||||
|
width: 190,
|
||||||
|
sortable: true,
|
||||||
|
value: (row) => row.created_at,
|
||||||
|
render: (row) => <>{row.mode}<br /><span className="muted">{formatDateTime(row.created_at)}</span></>
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "status",
|
||||||
|
header: "Status",
|
||||||
|
width: 150,
|
||||||
|
sortable: true,
|
||||||
|
value: (row) => row.status,
|
||||||
|
render: (row) => <StatusBadge status={row.status} label={row.status.replaceAll("_", " ")} />
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "summary",
|
||||||
|
header: "Effects",
|
||||||
|
width: "1fr",
|
||||||
|
minWidth: 220,
|
||||||
|
render: (row) => `${row.summary.total ?? 0} total · ${row.summary.ambiguous ?? 0} ambiguous · ${row.summary.errors ?? 0} errors`
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "revision",
|
||||||
|
header: "Evidence",
|
||||||
|
width: 170,
|
||||||
|
render: (row) => <code title={row.configuration_hash}>r{row.configuration_revision} · {row.input_hash.slice(0, 8)}</code>
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
header: "Actions",
|
||||||
|
width: 90,
|
||||||
|
sticky: "end",
|
||||||
|
align: "right",
|
||||||
|
render: (row) => <TableActionGroup actions={[{
|
||||||
|
id: "review",
|
||||||
|
label: "Review result",
|
||||||
|
icon: <span>✓</span>,
|
||||||
|
applicable: ["pending", "quarantined"].includes(row.review_state),
|
||||||
|
disabled: !canAdmin || busy,
|
||||||
|
disabledReason: !canAdmin ? "Connector administration permission is required." : undefined,
|
||||||
|
onClick: () => setReviewRun(row)
|
||||||
|
}]} />
|
||||||
|
}
|
||||||
|
], [busy, canAdmin]);
|
||||||
|
|
||||||
|
const actionBar = <PageActionBar
|
||||||
|
variant="editor"
|
||||||
|
state={busy ? "saving" : dirty ? "dirty" : "clean"}
|
||||||
|
refreshable
|
||||||
|
reloadAction={{ onReload: () => void reload(selectedId), loading }}
|
||||||
|
primaryActions={<>
|
||||||
|
<Button onClick={() => setDefinitionOpen(true)} disabled={!canAdmin || busy}>
|
||||||
|
New definition
|
||||||
|
</Button>
|
||||||
|
<Button onClick={() => setConfigurationOpen(true)} disabled={!canAdmin || busy || !definitions.length}>
|
||||||
|
New configuration
|
||||||
|
</Button>
|
||||||
|
{selected?.update_available ? <Button
|
||||||
|
variant="secondary"
|
||||||
|
onClick={() => void adoptUpdate()}
|
||||||
|
disabled={!canAdmin || busy || dirty}
|
||||||
|
disabledReason={dirty ? "Save or discard local edits before adopting a package update." : undefined}
|
||||||
|
>
|
||||||
|
Adopt package revision {selected.latest_definition_revision}
|
||||||
|
</Button> : null}
|
||||||
|
</>}
|
||||||
|
discardAction={{
|
||||||
|
label: "Discard changes",
|
||||||
|
disabled: !selected,
|
||||||
|
onClick: () => applyConfiguration(selected)
|
||||||
|
}}
|
||||||
|
saveAction={{
|
||||||
|
label: "Save",
|
||||||
|
disabled: !selected || !canAdmin || busy,
|
||||||
|
disabledReason: !canAdmin ? "Connector administration permission is required." : undefined,
|
||||||
|
onClick: () => void save()
|
||||||
|
}}
|
||||||
|
/>;
|
||||||
|
|
||||||
|
return <AdminPageLayout
|
||||||
|
archetype="workspace"
|
||||||
|
title="Connector governance"
|
||||||
|
description="Version schemas and mappings, protect local overrides, and review deterministic simulations before provider-specific writes."
|
||||||
|
loading={loading && !configurations.length}
|
||||||
|
error={error}
|
||||||
|
success={success}
|
||||||
|
actions={actionBar}
|
||||||
|
className="connector-governance-page"
|
||||||
|
helpContextId="connectors.admin.governed-configurations"
|
||||||
|
>
|
||||||
|
<MetricGrid columns={4} density="compact" minimum="compact">
|
||||||
|
<MetricCard label="Definitions" value={definitions.length} />
|
||||||
|
<MetricCard label="Configurations" value={configurations.length} />
|
||||||
|
<MetricCard label="Updates available" value={configurations.filter((item) => item.update_available).length} tone="warning" />
|
||||||
|
<MetricCard label="Awaiting review" value={runs.filter((item) => ["pending", "quarantined"].includes(item.review_state)).length} tone="warning" />
|
||||||
|
</MetricGrid>
|
||||||
|
|
||||||
|
<WorkspaceLayout
|
||||||
|
variant="split"
|
||||||
|
primarySize="compact"
|
||||||
|
surface="contained"
|
||||||
|
primaryScrollable={false}
|
||||||
|
contentScrollable={false}
|
||||||
|
primaryLabel="Connector configurations"
|
||||||
|
contentLabel="Configuration details"
|
||||||
|
primary={<div className="connector-governance-list">
|
||||||
|
<FilterBar surface="panel">
|
||||||
|
<input
|
||||||
|
type="search"
|
||||||
|
value={search}
|
||||||
|
onChange={(event) => setSearch(event.target.value)}
|
||||||
|
placeholder="Search configurations"
|
||||||
|
aria-label="Search connector configurations"
|
||||||
|
/>
|
||||||
|
</FilterBar>
|
||||||
|
<SelectionList variant="navigation" label="Connector configurations">
|
||||||
|
{visibleConfigurations.map((item) => <SelectionListItem
|
||||||
|
key={item.id}
|
||||||
|
selected={item.id === selectedId}
|
||||||
|
onClick={() => selectConfiguration(item)}
|
||||||
|
>
|
||||||
|
<SelectionListItemContent
|
||||||
|
title={item.name}
|
||||||
|
description={`${item.definition_name} · revision ${item.base_definition_revision}`}
|
||||||
|
/>
|
||||||
|
<StatusBadge status={item.update_available ? "warning" : item.status} />
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!visibleConfigurations.length
|
||||||
|
? <StatePanel size="compact" description="No matching configurations." />
|
||||||
|
: null}
|
||||||
|
</SelectionList>
|
||||||
|
</div>}
|
||||||
|
>
|
||||||
|
{!selected ? <StatePanel
|
||||||
|
size="fill"
|
||||||
|
title="Connector configurations"
|
||||||
|
description="Create or select a configuration to inspect its pinned definition and simulation evidence."
|
||||||
|
/> : <div className="connector-governance-detail">
|
||||||
|
<Card title={selected.name}>
|
||||||
|
<div className="connector-revision-line">
|
||||||
|
<StatusBadge status={selected.status} />
|
||||||
|
<span>Definition revision {selected.base_definition_revision}</span>
|
||||||
|
{selected.update_available
|
||||||
|
? <StatusBadge status="warning" label={`Revision ${selected.latest_definition_revision} available`} />
|
||||||
|
: null}
|
||||||
|
<code title={selected.effective_hash}>{selected.effective_hash.slice(0, 12)}</code>
|
||||||
|
</div>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Name">
|
||||||
|
<input value={draft.name} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, name: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Status">
|
||||||
|
<select value={draft.status} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, status: event.target.value as ConnectorConfigurationDraft["status"] })}>
|
||||||
|
<option value="draft">Draft</option>
|
||||||
|
<option value="active">Active</option>
|
||||||
|
<option value="disabled">Disabled</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Endpoint URL" hint="Credentials are rejected in URLs.">
|
||||||
|
<input value={draft.endpoint_url} disabled={!canAdmin || busy} placeholder="https://provider.example/api" onChange={(event) => setDraft({ ...draft, endpoint_url: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Credential reference" hint="Reference an approved secret; do not paste a secret.">
|
||||||
|
<input value={draft.credential_ref} disabled={!canAdmin || busy} placeholder="vault://connectors/provider" onChange={(event) => setDraft({ ...draft, credential_ref: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Ambiguous-result policy">
|
||||||
|
<select value={draft.ambiguity_policy} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, ambiguity_policy: event.target.value as ConnectorConfigurationDraft["ambiguity_policy"] })}>
|
||||||
|
<option value="manual_review">Manual review</option>
|
||||||
|
<option value="quarantine">Quarantine</option>
|
||||||
|
<option value="reject">Reject</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Protected local overrides" hint="JSON object leaf paths remain protected when package revisions are adopted.">
|
||||||
|
<textarea rows={8} value={draft.local_overrides} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, local_overrides: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
<p className="muted">
|
||||||
|
Protected paths: {selected.protected_paths.length ? selected.protected_paths.join(", ") : "none"}
|
||||||
|
</p>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Dry-run and simulation">
|
||||||
|
<p className="muted">Runs never apply changes. They retain redacted, revision-pinned evidence for review.</p>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Sample input rows" hint="JSON array, bounded by the definition's maximum.">
|
||||||
|
<textarea rows={9} value={sampleJson} disabled={!canExecute || busy} onChange={(event) => setSampleJson(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="External revision" hint="Optional provider ETag, cursor, or snapshot revision.">
|
||||||
|
<input value={externalRevision} disabled={!canExecute || busy} onChange={(event) => setExternalRevision(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
<div className="connector-run-actions">
|
||||||
|
<Button onClick={() => void run("dry-runs")} disabled={!canExecute || busy || dirty}>Run dry-run</Button>
|
||||||
|
<Button variant="primary" onClick={() => void run("simulations")} disabled={!canExecute || busy || dirty}>Run simulation</Button>
|
||||||
|
{dirty ? <span className="muted">Save or discard configuration changes before running.</span> : null}
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Simulation evidence">
|
||||||
|
<DataGrid
|
||||||
|
id="connector-simulation-runs"
|
||||||
|
rows={runs}
|
||||||
|
columns={runColumns}
|
||||||
|
getRowKey={(row) => row.id}
|
||||||
|
initialFit="container"
|
||||||
|
emptyText="No dry-runs or simulations have been recorded."
|
||||||
|
/>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Effective governed definition">
|
||||||
|
<pre className="connector-json-preview">{JSON.stringify(selected.effective_configuration, null, 2)}</pre>
|
||||||
|
</Card>
|
||||||
|
</div>}
|
||||||
|
</WorkspaceLayout>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={definitionOpen}
|
||||||
|
title="Create or revise connector definition"
|
||||||
|
onClose={() => !busy && setDefinitionOpen(false)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={<>
|
||||||
|
<Button onClick={() => setDefinitionOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createDefinition()} disabled={!canAdmin || busy}>Save definition revision</Button>
|
||||||
|
</>}
|
||||||
|
>
|
||||||
|
<p className="muted">The definition is schema-validated and every changed specification creates an immutable revision. Package definitions must name their package reference.</p>
|
||||||
|
<FormField label="Governed definition JSON">
|
||||||
|
<textarea className="connector-definition-editor" rows={24} value={definitionJson} disabled={busy} onChange={(event) => setDefinitionJson(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={configurationOpen}
|
||||||
|
title="Create connector configuration"
|
||||||
|
onClose={() => !busy && setConfigurationOpen(false)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={<>
|
||||||
|
<Button onClick={() => setConfigurationOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createConfiguration()} disabled={busy || !newDefinitionId || !newDraft.name.trim()}>Create configuration</Button>
|
||||||
|
</>}
|
||||||
|
>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Definition">
|
||||||
|
<select value={newDefinitionId} disabled={busy} onChange={(event) => setNewDefinitionId(event.target.value)}>
|
||||||
|
{definitions.map((item) => <option key={item.id} value={item.id}>{item.name} · revision {item.current_revision}</option>)}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Name">
|
||||||
|
<input value={newDraft.name} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, name: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Endpoint URL">
|
||||||
|
<input value={newDraft.endpoint_url} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, endpoint_url: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Credential reference">
|
||||||
|
<input value={newDraft.credential_ref} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, credential_ref: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Ambiguous-result policy">
|
||||||
|
<select value={newDraft.ambiguity_policy} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, ambiguity_policy: event.target.value as ConnectorConfigurationDraft["ambiguity_policy"] })}>
|
||||||
|
<option value="manual_review">Manual review</option>
|
||||||
|
<option value="quarantine">Quarantine</option>
|
||||||
|
<option value="reject">Reject</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Local overrides">
|
||||||
|
<textarea rows={7} value={newDraft.local_overrides} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, local_overrides: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={Boolean(reviewRun)}
|
||||||
|
title="Review ambiguous connector result"
|
||||||
|
onClose={() => !busy && setReviewRun(null)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={<>
|
||||||
|
<Button onClick={() => setReviewRun(null)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="danger" onClick={() => void decideReview("rejected")} disabled={busy || reviewReason.trim().length < 5}>Reject</Button>
|
||||||
|
<Button variant="primary" onClick={() => void decideReview("approved")} disabled={busy || reviewReason.trim().length < 5}>Approve</Button>
|
||||||
|
</>}
|
||||||
|
>
|
||||||
|
<p>Review {reviewRun?.summary.ambiguous ?? 0} ambiguous effects against the retained input and configuration hashes before deciding.</p>
|
||||||
|
<FormField label="Decision reason">
|
||||||
|
<textarea rows={4} value={reviewReason} disabled={busy} onChange={(event) => setReviewReason(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
<pre className="connector-json-preview">{JSON.stringify(reviewRun?.diagnostics ?? [], null, 2)}</pre>
|
||||||
|
</Dialog>
|
||||||
|
</AdminPageLayout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftFromConfiguration(item: ConnectorConfiguration): ConnectorConfigurationDraft {
|
||||||
|
return {
|
||||||
|
name: item.name,
|
||||||
|
status: item.status,
|
||||||
|
endpoint_url: item.endpoint_url ?? "",
|
||||||
|
credential_ref: item.credential_ref ?? "",
|
||||||
|
local_overrides: JSON.stringify(item.local_overrides, null, 2),
|
||||||
|
ambiguity_policy: item.ambiguity_policy
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftKey(value: ConnectorConfigurationDraft): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
name: value.name.trim(),
|
||||||
|
status: value.status,
|
||||||
|
endpoint_url: value.endpoint_url.trim(),
|
||||||
|
credential_ref: value.credential_ref.trim(),
|
||||||
|
local_overrides: normalizeJson(value.local_overrides),
|
||||||
|
ambiguity_policy: value.ambiguity_policy
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeJson(value: string): unknown {
|
||||||
|
try {
|
||||||
|
return JSON.parse(value);
|
||||||
|
} catch {
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseObject(value: string, label: string): Record<string, unknown> {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") {
|
||||||
|
throw new Error(`${label} must be a JSON object.`);
|
||||||
|
}
|
||||||
|
return parsed as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRows(value: string): Array<Record<string, unknown>> {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!Array.isArray(parsed) || parsed.some((item) => !item || Array.isArray(item) || typeof item !== "object")) {
|
||||||
|
throw new Error("Sample input must be a JSON array of objects.");
|
||||||
|
}
|
||||||
|
return parsed as Array<Record<string, unknown>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
export { connectorsModule as default, connectorsModule } from "./module";
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { createElement, lazy } from "react";
|
||||||
|
import type {
|
||||||
|
AdminSectionsUiCapability,
|
||||||
|
PlatformWebModule
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import "./styles/connectors.css";
|
||||||
|
|
||||||
|
const ConnectorGovernancePage = lazy(
|
||||||
|
() => import("./features/ConnectorGovernancePage")
|
||||||
|
);
|
||||||
|
|
||||||
|
const readScopes = [
|
||||||
|
"connectors:source:read",
|
||||||
|
"connectors:source:admin"
|
||||||
|
];
|
||||||
|
|
||||||
|
const adminSections: AdminSectionsUiCapability = {
|
||||||
|
sections: [
|
||||||
|
{
|
||||||
|
id: "connector-governance",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "management",
|
||||||
|
surfaceId: "connectors.admin.governed-configurations",
|
||||||
|
label: "Connector governance",
|
||||||
|
group: "SYSTEM",
|
||||||
|
order: 45,
|
||||||
|
anyOf: readScopes,
|
||||||
|
render: ({ settings, auth }) =>
|
||||||
|
createElement(ConnectorGovernancePage, { settings, auth })
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
export const connectorsModule: PlatformWebModule = {
|
||||||
|
id: "connectors",
|
||||||
|
label: "Connectors",
|
||||||
|
version: "0.1.18",
|
||||||
|
dependencies: [],
|
||||||
|
optionalDependencies: ["access", "audit", "policy", "ops"],
|
||||||
|
viewSurfaces: [
|
||||||
|
{
|
||||||
|
id: "connectors.admin.governed-configurations",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "section",
|
||||||
|
label: "Connector governance",
|
||||||
|
order: 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "connectors.admin.simulation-review",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "section",
|
||||||
|
label: "Connector simulation review",
|
||||||
|
parentId: "connectors.admin.governed-configurations",
|
||||||
|
order: 20
|
||||||
|
}
|
||||||
|
],
|
||||||
|
uiCapabilities: {
|
||||||
|
"admin.sections": adminSections
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export default connectorsModule;
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
.connector-governance-page .connector-governance-list,
|
||||||
|
.connector-governance-page .connector-governance-detail {
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page .connector-revision-line,
|
||||||
|
.connector-governance-page .connector-run-actions {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: var(--space-3, 0.75rem);
|
||||||
|
margin-bottom: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page textarea {
|
||||||
|
font-family: var(--font-family-mono, ui-monospace, monospace);
|
||||||
|
resize: vertical;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page .connector-definition-editor,
|
||||||
|
.connector-governance-page .connector-json-preview {
|
||||||
|
max-height: 34rem;
|
||||||
|
overflow: auto;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page .connector-json-preview {
|
||||||
|
background: var(--surface-subtle);
|
||||||
|
border-radius: var(--radius-md, 0.5rem);
|
||||||
|
padding: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const moduleSource = readFileSync("src/module.ts", "utf8");
|
||||||
|
const page = readFileSync("src/features/ConnectorGovernancePage.tsx", "utf8");
|
||||||
|
const api = readFileSync("src/api/governedConnectors.ts", "utf8");
|
||||||
|
|
||||||
|
assert.match(moduleSource, /"admin.sections": adminSections/);
|
||||||
|
assert.match(moduleSource, /connectors\.admin\.governed-configurations/);
|
||||||
|
assert.match(page, /<AdminPageLayout/);
|
||||||
|
assert.match(page, /<PageActionBar/);
|
||||||
|
assert.match(page, /refreshable/);
|
||||||
|
assert.match(page, /saveAction=/);
|
||||||
|
assert.match(page, /useUnsavedDraftGuard/);
|
||||||
|
assert.match(page, /<WorkspaceLayout/);
|
||||||
|
assert.match(page, /Adopt package revision/);
|
||||||
|
assert.match(page, /Protected paths/);
|
||||||
|
assert.match(page, /manual_review/);
|
||||||
|
assert.match(page, /quarantine/);
|
||||||
|
assert.match(page, /Run simulation/);
|
||||||
|
assert.match(page, /Review ambiguous connector result/);
|
||||||
|
assert.match(api, /idempotency_key/);
|
||||||
|
assert.match(api, /credential_ref/);
|
||||||
|
|
||||||
|
console.log("Connector governance UI structural contract passed.");
|
||||||
Reference in New Issue
Block a user