Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b277e8d7ac | ||
|
|
79e2315e89 | ||
|
|
2c2b11f860 | ||
|
|
e6ab8291ec | ||
|
|
de8e74b866 | ||
|
|
0f61fe4607 | ||
|
|
3964a2e4e7 | ||
|
|
b88f1b2c2d | ||
|
|
32cac8835b | ||
|
|
c69d68f1da | ||
|
|
1caae6e49e | ||
|
|
84c9bb7711 | ||
|
|
20146ef8fe | ||
|
|
c33380b957 | ||
|
|
dfa717b9ba | ||
|
|
26f8898d11 | ||
|
|
7be93785a2 | ||
|
|
52fe33568c | ||
|
|
c5a43b3dae | ||
|
|
27302f0c39 | ||
|
|
ba5ccea5b0 | ||
|
|
dd45d9bd36 | ||
|
|
853d12151f | ||
|
|
d2ba4ce4d8 |
@@ -0,0 +1,270 @@
|
|||||||
|
name: Module Package Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
release_tag:
|
||||||
|
description: Existing protected version tag to publish
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-packages:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Select and validate protected release tag
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
REQUESTED_TAG: ${{ inputs.release_tag }}
|
||||||
|
TRIGGER_TAG: ${{ gitea.ref_name }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
|
||||||
|
case "$tag" in
|
||||||
|
v[0-9]*.[0-9]*.[0-9]*) ;;
|
||||||
|
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
|
||||||
|
tag_commit="$(git rev-list -n 1 "$tag")"
|
||||||
|
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
|
||||||
|
echo "Release tag is not contained in main" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
git checkout --detach "$tag"
|
||||||
|
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
|
||||||
|
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
|
||||||
|
- name: Validate package versions
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
tag = os.environ["RELEASE_TAG"]
|
||||||
|
expected = tag.removeprefix("v")
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
if project.get("version") != expected:
|
||||||
|
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
|
||||||
|
raise SystemExit("Python distribution name must use the govoplan-* namespace")
|
||||||
|
webui = Path("webui/package.json")
|
||||||
|
if webui.is_file():
|
||||||
|
package = json.loads(webui.read_text(encoding="utf-8"))
|
||||||
|
if package.get("version") != expected:
|
||||||
|
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
|
||||||
|
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
|
||||||
|
release = Path("webui/package.release.json")
|
||||||
|
if release.is_file():
|
||||||
|
release_package = json.loads(release.read_text(encoding="utf-8"))
|
||||||
|
if (
|
||||||
|
release_package.get("name") != package.get("name")
|
||||||
|
or release_package.get("version") != expected
|
||||||
|
):
|
||||||
|
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
|
||||||
|
PY
|
||||||
|
- name: Build immutable package artifacts
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||||
|
rm -rf dist .package-webui
|
||||||
|
python -m build --wheel --outdir dist
|
||||||
|
python -m twine check dist/*.whl
|
||||||
|
if [[ -f webui/package.json ]]; then
|
||||||
|
mkdir .package-webui
|
||||||
|
cp -a webui/. .package-webui/
|
||||||
|
rm -rf .package-webui/node_modules .package-webui/dist
|
||||||
|
if [[ -f .package-webui/package.release.json ]]; then
|
||||||
|
cp .package-webui/package.release.json .package-webui/package.json
|
||||||
|
fi
|
||||||
|
node <<'NODE'
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = ".package-webui/package.json";
|
||||||
|
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||||
|
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
|
||||||
|
for (const group of groups) {
|
||||||
|
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
|
||||||
|
if (!name.startsWith("@govoplan/")) continue;
|
||||||
|
if (typeof specifier !== "string") {
|
||||||
|
throw new Error(`${group}.${name} must use a string version`);
|
||||||
|
}
|
||||||
|
const packageSlug = name.slice("@govoplan/".length);
|
||||||
|
if (!packageSlug.endsWith("-webui")) {
|
||||||
|
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
|
||||||
|
}
|
||||||
|
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
|
||||||
|
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
const gitTag = specifier.match(
|
||||||
|
new RegExp(
|
||||||
|
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (gitTag) {
|
||||||
|
packageJson[group][name] = gitTag[1];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
|
||||||
|
throw new Error(
|
||||||
|
`${group}.${name} must resolve to an exact registry version for publication`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete packageJson.private;
|
||||||
|
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
|
||||||
|
NODE
|
||||||
|
npm pkg delete private --prefix .package-webui
|
||||||
|
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
|
||||||
|
fi
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
artifacts = []
|
||||||
|
for path in sorted(Path("dist").iterdir()):
|
||||||
|
if path.suffix not in {".whl", ".tgz"}:
|
||||||
|
continue
|
||||||
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
|
||||||
|
payload = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"repository": os.environ["GITEA_REPOSITORY"],
|
||||||
|
"tag": os.environ["RELEASE_TAG"],
|
||||||
|
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
Path("dist/package-artifacts.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
- name: Retain package hash evidence
|
||||||
|
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
|
||||||
|
with:
|
||||||
|
name: module-packages-${{ gitea.ref_name }}
|
||||||
|
path: dist/package-artifacts.json
|
||||||
|
- name: Check immutable registry state
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tomllib
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
from urllib.parse import quote
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
|
||||||
|
token = os.environ["PACKAGE_TOKEN"]
|
||||||
|
|
||||||
|
def should_publish(kind, name, version, path):
|
||||||
|
package_url = "/".join(
|
||||||
|
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
|
||||||
|
)
|
||||||
|
request = Request(
|
||||||
|
package_url,
|
||||||
|
headers={"Accept": "application/json", "Authorization": f"token {token}"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urlopen(request, timeout=30) as response:
|
||||||
|
files = json.load(response)
|
||||||
|
except HTTPError as exc:
|
||||||
|
if exc.code == 404:
|
||||||
|
print(f"{kind} package {name}=={version} is not published yet")
|
||||||
|
return True
|
||||||
|
raise
|
||||||
|
if not isinstance(files, list) or len(files) != 1:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} has an unexpected file set"
|
||||||
|
)
|
||||||
|
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
if files[0].get("sha256") != expected_sha256:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
|
||||||
|
)
|
||||||
|
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
|
||||||
|
return False
|
||||||
|
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
wheels = tuple(Path("dist").glob("*.whl"))
|
||||||
|
if len(wheels) != 1:
|
||||||
|
raise SystemExit("release build must contain exactly one wheel")
|
||||||
|
publish_pypi = should_publish(
|
||||||
|
"pypi", str(project["name"]), str(project["version"]), wheels[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
tarballs = tuple(Path("dist").glob("*.tgz"))
|
||||||
|
if len(tarballs) > 1:
|
||||||
|
raise SystemExit("release build must contain at most one npm package")
|
||||||
|
publish_npm = False
|
||||||
|
if tarballs:
|
||||||
|
webui = json.loads(
|
||||||
|
Path(".package-webui/package.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
publish_npm = should_publish(
|
||||||
|
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
|
||||||
|
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
|
||||||
|
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
|
||||||
|
PY
|
||||||
|
- name: Publish wheel and WebUI package
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_USERNAME"
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
if [[ "$PUBLISH_PYPI" == 1 ]]; then
|
||||||
|
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||||
|
python -m twine upload --non-interactive \
|
||||||
|
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||||
|
dist/*.whl
|
||||||
|
else
|
||||||
|
echo "Exact wheel is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
|
shopt -s nullglob
|
||||||
|
webui_packages=(dist/*.tgz)
|
||||||
|
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
|
||||||
|
npmrc="$(mktemp)"
|
||||||
|
trap 'rm -f "$npmrc"' EXIT
|
||||||
|
chmod 600 "$npmrc"
|
||||||
|
printf '%s\n' \
|
||||||
|
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
|
||||||
|
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
|
||||||
|
> "$npmrc"
|
||||||
|
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
|
||||||
|
--ignore-scripts --access public \
|
||||||
|
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
|
||||||
|
elif (( ${#webui_packages[@]} )); then
|
||||||
|
echo "Exact WebUI package is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
+276
@@ -0,0 +1,276 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.egg-info/
|
||||||
|
.pytest_cache/
|
||||||
|
.mypy_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.venv/
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
node_modules/
|
||||||
|
webui/node_modules/
|
||||||
|
webui/dist/
|
||||||
|
*.tsbuildinfo
|
||||||
|
.component-test-build/
|
||||||
|
.module-test-build/
|
||||||
|
.policy-test-build/
|
||||||
|
.template-preview-test-build/
|
||||||
|
.import-test-build/
|
||||||
|
webui/.component-test-build/
|
||||||
|
webui/.module-test-build/
|
||||||
|
webui/.policy-test-build/
|
||||||
|
webui/.template-preview-test-build/
|
||||||
|
webui/.import-test-build/
|
||||||
|
|
||||||
|
# GovOPlaN shared ignore rules from govoplan-core
|
||||||
|
# ---> Node
|
||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
.pnpm-debug.log*
|
||||||
|
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||||
|
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||||
|
# Runtime data
|
||||||
|
pids
|
||||||
|
*.pid
|
||||||
|
*.seed
|
||||||
|
*.pid.lock
|
||||||
|
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||||
|
lib-cov
|
||||||
|
# Coverage directory used by tools like istanbul
|
||||||
|
coverage
|
||||||
|
*.lcov
|
||||||
|
# nyc test coverage
|
||||||
|
.nyc_output
|
||||||
|
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||||
|
.grunt
|
||||||
|
# Bower dependency directory (https://bower.io/)
|
||||||
|
bower_components
|
||||||
|
# node-waf configuration
|
||||||
|
.lock-wscript
|
||||||
|
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||||
|
build/Release
|
||||||
|
# Dependency directories
|
||||||
|
jspm_packages/
|
||||||
|
# Snowpack dependency directory (https://snowpack.dev/)
|
||||||
|
web_modules/
|
||||||
|
# TypeScript cache
|
||||||
|
# Optional npm cache directory
|
||||||
|
.npm
|
||||||
|
# Optional eslint cache
|
||||||
|
.eslintcache
|
||||||
|
# Optional stylelint cache
|
||||||
|
.stylelintcache
|
||||||
|
# Microbundle cache
|
||||||
|
.rpt2_cache/
|
||||||
|
.rts2_cache_cjs/
|
||||||
|
.rts2_cache_es/
|
||||||
|
.rts2_cache_umd/
|
||||||
|
# Optional REPL history
|
||||||
|
.node_repl_history
|
||||||
|
# Output of 'npm pack'
|
||||||
|
*.tgz
|
||||||
|
# Yarn Integrity file
|
||||||
|
.yarn-integrity
|
||||||
|
# dotenv environment variable files
|
||||||
|
.env
|
||||||
|
.env.development.local
|
||||||
|
.env.test.local
|
||||||
|
.env.production.local
|
||||||
|
.env.local
|
||||||
|
# parcel-bundler cache (https://parceljs.org/)
|
||||||
|
.cache
|
||||||
|
.parcel-cache
|
||||||
|
# Next.js build output
|
||||||
|
.next
|
||||||
|
out
|
||||||
|
# Nuxt.js build / generate output
|
||||||
|
.nuxt
|
||||||
|
dist
|
||||||
|
# Gatsby files
|
||||||
|
.cache/
|
||||||
|
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||||
|
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||||
|
# public
|
||||||
|
# vuepress build output
|
||||||
|
.vuepress/dist
|
||||||
|
# vuepress v2.x temp and cache directory
|
||||||
|
.temp
|
||||||
|
.cache
|
||||||
|
# vitepress build output
|
||||||
|
**/.vitepress/dist
|
||||||
|
# vitepress cache directory
|
||||||
|
**/.vitepress/cache
|
||||||
|
# Docusaurus cache and generated files
|
||||||
|
.docusaurus
|
||||||
|
# Serverless directories
|
||||||
|
.serverless/
|
||||||
|
# FuseBox cache
|
||||||
|
.fusebox/
|
||||||
|
# DynamoDB Local files
|
||||||
|
.dynamodb/
|
||||||
|
# TernJS port file
|
||||||
|
.tern-port
|
||||||
|
# Stores VSCode versions used for testing VSCode extensions
|
||||||
|
.vscode-test
|
||||||
|
# yarn v2
|
||||||
|
.yarn/cache
|
||||||
|
.yarn/unplugged
|
||||||
|
.yarn/build-state.yml
|
||||||
|
.yarn/install-state.gz
|
||||||
|
.pnp.*
|
||||||
|
# Local WebUI test/build scratch directories
|
||||||
|
# ---> Python
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
*$py.class
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
develop-eggs/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
cover/
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
#uv.lock
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
#poetry.lock
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
#pdm.lock
|
||||||
|
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||||
|
# in version control.
|
||||||
|
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||||
|
.pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
# mypy
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
#.idea/
|
||||||
|
# Ruff stuff:
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
# ---> VisualStudioCode
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
!.vscode/*.code-snippets
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
# Built Visual Studio Code Extensions
|
||||||
|
*.vsix
|
||||||
|
*.db
|
||||||
|
# GovOPlaN local runtime state
|
||||||
|
runtime/
|
||||||
|
# GovOPlaN WebUI test output
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# GovOPlaN Connectors Codex Guide
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This repository owns reusable external connection profiles, protocol adapters, governed snapshots, and connector capability contracts.
|
||||||
|
|
||||||
|
## Documentation Contract
|
||||||
|
|
||||||
|
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
|
||||||
|
- Keep feature content here; `govoplan-docs` projects it without importing Connectors internals.
|
||||||
|
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
- Domain modules own business semantics; connectors own transport, credentials, synchronization, and diagnostics.
|
||||||
|
- Keep optional adapters behind capabilities and enforce egress and peer-validation policy.
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
# govoplan-connectors
|
||||||
|
|
||||||
|
<!-- govoplan-repository-type:start -->
|
||||||
|
**Repository type:** connector (connector-hub).
|
||||||
|
<!-- govoplan-repository-type:end -->
|
||||||
|
|
||||||
|
`govoplan-connectors` owns integration catalogues and generic external system
|
||||||
|
connection patterns for GovOPlaN.
|
||||||
|
|
||||||
|
The module should make external systems discoverable, testable, and usable
|
||||||
|
without taking ownership of their business semantics. Domain-specific modules
|
||||||
|
remain responsible for case, file, workflow, payment, mail, identity, document,
|
||||||
|
or reporting behavior.
|
||||||
|
|
||||||
|
## Executable First Slice
|
||||||
|
|
||||||
|
The executable connector capability provides tenant-isolated tabular origins.
|
||||||
|
Operators can import bounded JSON or CSV snapshots, bind an exact managed Files
|
||||||
|
CSV/XLSX version, or discover a table through an active governed PostgreSQL
|
||||||
|
configuration. Every origin exposes a reviewed schema, opaque reference, and
|
||||||
|
content/discovery fingerprint through `connectors.datasource_origins@0.1.0`.
|
||||||
|
Preview reads enforce provider ceilings for rows, serialized bytes, and elapsed
|
||||||
|
time and report the effective limits and any truncation as structured
|
||||||
|
diagnostics.
|
||||||
|
|
||||||
|
Connectors owns acquisition, connection profiles, credentials, discovery, and
|
||||||
|
provider health. `govoplan-datasources` registers an origin as a governed live
|
||||||
|
or cached datasource and owns staging, materializations, frozen states, and
|
||||||
|
consumer access. Dataflow consumes that Datasources contract and never imports
|
||||||
|
connector implementations or stores connector credentials.
|
||||||
|
|
||||||
|
Each origin declares whether it is live, cached, file-backed, or static, its
|
||||||
|
structured health state, and which projection, filter, aggregation, sorting,
|
||||||
|
and pagination operations it can push down. The snapshot, managed-file, and
|
||||||
|
PostgreSQL providers currently support projection and pagination only;
|
||||||
|
consumers must keep other operations in Dataflow rather than assuming
|
||||||
|
transport-side execution.
|
||||||
|
|
||||||
|
Managed-file sources are authorized and opened through
|
||||||
|
`files.tabular_content@1.0.0`; Files remains authoritative for ownership,
|
||||||
|
shares, download permission, exact versions, integrity, quarantine, encryption,
|
||||||
|
retention, and legal holds. CSV must be UTF-8. XLSX input is protected by
|
||||||
|
compressed-entry, expanded-byte, compression-ratio, row, and column limits.
|
||||||
|
A newer current version is reported but never silently replaces the pinned
|
||||||
|
version.
|
||||||
|
|
||||||
|
The PostgreSQL adapter accepts only an active governed connector configuration
|
||||||
|
whose secret-free endpoint uses the PostgreSQL driver. Authentication is
|
||||||
|
resolved from a tenant/scope/module/server-restricted Core credential envelope.
|
||||||
|
The adapter reflects a simple schema/table identifier, uses read-only
|
||||||
|
transactions and a statement timeout, and blocks configuration, credential, or
|
||||||
|
schema drift until an operator refreshes and reviews the source. Secrets are
|
||||||
|
never copied into source metadata or diagnostics. Other database, REST/HTTP,
|
||||||
|
directory, and warehouse providers can implement the same origin contract
|
||||||
|
without changing Datasources or Dataflow.
|
||||||
|
|
||||||
|
Governed sanctions and feed snapshot acquisitions use Core recovery operations.
|
||||||
|
The source revision/cursor, redacted dry-run decision, canonical request digest,
|
||||||
|
and distributed lease are durable before network I/O. Immutable snapshot rows
|
||||||
|
and the terminal recovery checkpoint commit atomically, and an
|
||||||
|
`Idempotency-Key` replays the committed result without contacting the provider.
|
||||||
|
Most acquisition transports are read-only. The MediaWiki/BlueSpice knowledge
|
||||||
|
adapter is the first governed publication path: it requires an expected remote
|
||||||
|
revision, a stable idempotency key, a scoped credential envelope, and durable
|
||||||
|
forward-recovery evidence. A timeout after dispatch becomes outcome-unknown and
|
||||||
|
blocks replay until the provider revision has been reconciled.
|
||||||
|
|
||||||
|
The knowledge adapter discovers MediaWiki or BlueSpice product/version and
|
||||||
|
capabilities, maps namespaces, pages, revisions, users, categories, links,
|
||||||
|
files, discussions, redirects, and permissions into identity-stable connector
|
||||||
|
snapshots, and consumes bounded full or recent-change deltas. Optional Search
|
||||||
|
integration indexes active pages and rechecks current profile state and ACLs on
|
||||||
|
every result. Migration into native Wiki is preview-only: conflicts, attachment
|
||||||
|
collisions, unsupported macros, truncation, and source fingerprints are
|
||||||
|
reported before a target-side write is considered.
|
||||||
|
|
||||||
|
The Znuny/OTRS-compatible service-desk adapter uses deployment-defined
|
||||||
|
GenericInterface REST routes. It supports identity-only links, bounded snapshot
|
||||||
|
imports, and ongoing synchronization with explicit authority, queue, ACL, and
|
||||||
|
dynamic-field mappings. Stable tickets, articles, and attachment references are
|
||||||
|
preserved with mapping-loss diagnostics; attachment bytes remain provider-side.
|
||||||
|
Optional Search integration rechecks the current profile, tenant, scope, and ACL
|
||||||
|
for every result. Revision-checked external updates are limited to governed-sync
|
||||||
|
profiles and use durable recovery evidence. Tickets, Helpdesk, and Cases remain
|
||||||
|
authoritative for their own business records and conversion workflows.
|
||||||
|
|
||||||
|
RSS and Atom emission is a bounded renderer, not an authority shortcut. Every
|
||||||
|
selected entry declares whether it came from a GovOPlaN event, publication,
|
||||||
|
case, or report and carries an opaque owning-module reference and optional
|
||||||
|
revision. Public-feed permission can render only public entries. Tenant and
|
||||||
|
private audiences require a separate restricted-feed permission, and the API
|
||||||
|
derives the allowed visibility set from that audience instead of accepting a
|
||||||
|
caller-controlled allow-list. Portal or Reporting remains responsible for any
|
||||||
|
durable public or authenticated route and must re-authorize restricted access.
|
||||||
|
|
||||||
|
The governed connector runtime adds immutable definition revisions,
|
||||||
|
revision-pinned tenant configurations, protected local overrides, explicit
|
||||||
|
package-update adoption, bounded dry-runs and simulations, redacted provenance,
|
||||||
|
idempotency, and configurable ambiguity handling through review, quarantine,
|
||||||
|
or rejection. Its administration surface is contributed to the shared system
|
||||||
|
administration workspace. Provider-specific adapters still own live writes.
|
||||||
|
|
||||||
|
Development:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/mnt/DATA/git/govoplan/.venv/bin/python -m pip install -e .
|
||||||
|
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest discover -s tests
|
||||||
|
```
|
||||||
|
|
||||||
|
See:
|
||||||
|
|
||||||
|
- [Connector concept](docs/CONCEPT.md)
|
||||||
|
- [Public-sector integration catalogue](docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md)
|
||||||
|
- [Connector source lifecycle](docs/CONNECTOR_SOURCE_LIFECYCLE.md)
|
||||||
|
- [OpenProject connector concept](docs/OPENPROJECT_CONNECTOR.md)
|
||||||
|
- [OpenDesk integration map](docs/OPENDESK_INTEGRATION_MAP.md)
|
||||||
|
- [Governed connector configuration](docs/GOVERNED_CONNECTOR_CONFIGURATION.md)
|
||||||
|
- [MediaWiki and BlueSpice connector](docs/MEDIAWIKI_BLUESPICE_CONNECTOR.md)
|
||||||
|
- [Znuny and OTRS-compatible service-desk connector](docs/ZNUNY_OTRS_CONNECTOR.md)
|
||||||
+197
@@ -0,0 +1,197 @@
|
|||||||
|
# govoplan-connectors Concept
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`govoplan-connectors` is the integration catalogue and connector coordination
|
||||||
|
module. It helps GovOPlaN connect to existing public-sector and organizational
|
||||||
|
systems without pretending to replace every specialist platform.
|
||||||
|
|
||||||
|
The module owns connector metadata, connection profiles, health checks, test
|
||||||
|
results, credential references, and generic integration events. Protocol-heavy
|
||||||
|
or domain-heavy integrations may live in dedicated modules once their scope is
|
||||||
|
clear.
|
||||||
|
|
||||||
|
Connector capability is not source ownership. Each configured binding also
|
||||||
|
declares whether GovOPlaN is authoritative, the external system is
|
||||||
|
authoritative, GovOPlaN keeps a mirror, both sides use governed synchronization,
|
||||||
|
GovOPlaN supplies only a governance overlay, or the object is link-only. The
|
||||||
|
same connector may be configured differently by tenant, service, object type,
|
||||||
|
or field group.
|
||||||
|
|
||||||
|
Detailed follow-up documents:
|
||||||
|
|
||||||
|
- [Public-sector integration catalogue](PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md)
|
||||||
|
- [Connector source lifecycle](CONNECTOR_SOURCE_LIFECYCLE.md)
|
||||||
|
- [OpenProject connector concept](OPENPROJECT_CONNECTOR.md)
|
||||||
|
- [OpenDesk integration map](OPENDESK_INTEGRATION_MAP.md)
|
||||||
|
|
||||||
|
## Shared runtime boundary
|
||||||
|
|
||||||
|
Connector transports use the versioned Core runtime contract for bounded dry
|
||||||
|
runs and diagnostics. Connectors owns endpoint discovery, authentication
|
||||||
|
hand-off, protocol reads, retry/backoff, and source health. A consuming module
|
||||||
|
owns domain mappings and mutations: Addresses, for example, owns contact and
|
||||||
|
vCard semantics even when Connectors supplies reusable LDAP, CardDAV, Exchange,
|
||||||
|
or Google transport patterns.
|
||||||
|
|
||||||
|
Dry runs carry an immutable input hash, source revision and fingerprint,
|
||||||
|
redacted effects, diagnostics, truncation state, and an apply token. Apply must
|
||||||
|
reject a changed input or source revision. URLs and diagnostics never contain
|
||||||
|
credential material; they retain only credential-envelope references.
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
|
||||||
|
The module owns:
|
||||||
|
|
||||||
|
- connector catalogue entries and capability metadata
|
||||||
|
- connection profiles and endpoint configuration
|
||||||
|
- credential references and test diagnostics
|
||||||
|
- generic webhook/polling/job coordination metadata
|
||||||
|
- connector health status and last-test evidence
|
||||||
|
- operator-visible integration inventory
|
||||||
|
- cross-module discovery of available external capabilities
|
||||||
|
- supported integration maturity, source-authority modes, operation limits,
|
||||||
|
and effect/reconciliation behavior for each connector type
|
||||||
|
|
||||||
|
The module does not own:
|
||||||
|
|
||||||
|
- governed datasource identity, staging, materializations, frozen states, or
|
||||||
|
consumer read semantics, owned by `govoplan-datasources`
|
||||||
|
- file storage semantics, owned by files/DMS
|
||||||
|
- identity provisioning semantics, owned by IDM/access
|
||||||
|
- mail/calendar semantics, owned by mail/calendar
|
||||||
|
- case/workflow/task/domain records
|
||||||
|
- payment, ledger, XRechnung, XTA/OSCI, FIT-Connect, or XOE/V protocol
|
||||||
|
semantics once those are dedicated modules
|
||||||
|
|
||||||
|
## Connector Categories
|
||||||
|
|
||||||
|
Initial catalogue categories:
|
||||||
|
|
||||||
|
- project management and task systems such as OpenProject
|
||||||
|
- DMS/e-file/archive systems
|
||||||
|
- file providers such as Nextcloud, Seafile, WebDAV, SMB/NFS, object storage
|
||||||
|
- identity providers such as LDAP, Active Directory, OIDC, SAML, OpenDesk IDM
|
||||||
|
- groupware such as Open-Xchange mail/calendar
|
||||||
|
- ERP, finance, accounting, payment, and cash-register systems
|
||||||
|
- public-sector protocols such as FIT-Connect, XTA/OSCI, XRechnung, XOE/V
|
||||||
|
- reporting, BI, RSS/API publication, and open-data endpoints
|
||||||
|
|
||||||
|
## Core Contracts
|
||||||
|
|
||||||
|
The module should integrate through:
|
||||||
|
|
||||||
|
- module manifest metadata, route factories, permissions, and migrations
|
||||||
|
- a connector catalogue API for listing available connector types
|
||||||
|
- a connection profile API with secret references, not plaintext secrets
|
||||||
|
- a provider declaration that composes authority mode, maturity, supported
|
||||||
|
operations, revisions/freshness, health, limits, idempotency, conflicts,
|
||||||
|
evidence, and reconciliation behavior
|
||||||
|
- capability declarations such as `connectors.catalog`,
|
||||||
|
`connectors.profileTester`, and `connectors.health`
|
||||||
|
- events such as `connector.profile_created`, `connector.test_succeeded`,
|
||||||
|
`connector.test_failed`, and `connector.health_changed`
|
||||||
|
- configuration-package fragments for required external systems
|
||||||
|
|
||||||
|
Domain modules should ask whether a connector capability exists and request a
|
||||||
|
profile/test result through core-mediated capabilities. They must not import
|
||||||
|
connector implementation modules directly.
|
||||||
|
|
||||||
|
Data-oriented consumers use a two-layer path: Connectors publishes a
|
||||||
|
provider-specific datasource origin, then Datasources registers and governs it.
|
||||||
|
Dataflow, Workflow, Reporting, and other consumers use Datasources rather than
|
||||||
|
calling the connector origin directly.
|
||||||
|
|
||||||
|
## Reference Journeys
|
||||||
|
|
||||||
|
### OpenProject Connector First
|
||||||
|
|
||||||
|
1. Operator registers an OpenProject connection profile.
|
||||||
|
2. Connector tests API reachability and authentication.
|
||||||
|
3. A future project-management decision can use the connector before a native
|
||||||
|
`govoplan-projects` module exists.
|
||||||
|
4. Cases/tasks/workflow may link to external project/task references through
|
||||||
|
stable external-reference DTOs.
|
||||||
|
|
||||||
|
### Public-Sector Integration Catalogue
|
||||||
|
|
||||||
|
1. Operator records which external systems exist in an organization.
|
||||||
|
2. GovOPlaN identifies common protocols and missing connectors.
|
||||||
|
3. Configuration packages can declare required connector profiles.
|
||||||
|
4. Health/status pages show whether required integrations are ready.
|
||||||
|
|
||||||
|
### OpenDesk Profile
|
||||||
|
|
||||||
|
1. Operator records OpenDesk component profiles for identity, mail, calendar,
|
||||||
|
files/documents, and OpenProject where present.
|
||||||
|
2. Connectors shows which components are configured, tested, degraded, or
|
||||||
|
missing.
|
||||||
|
3. Domain modules enable optional behavior by checking capabilities through
|
||||||
|
core, not by importing connector or OpenDesk-specific implementation code.
|
||||||
|
|
||||||
|
## MVP Slice
|
||||||
|
|
||||||
|
The first implementation should provide:
|
||||||
|
|
||||||
|
- connector type registry
|
||||||
|
- connection profile CRUD with secret references
|
||||||
|
- connection test result records
|
||||||
|
- WebUI catalogue and profile pages
|
||||||
|
- configuration-package fragment support
|
||||||
|
- generic external-reference DTOs
|
||||||
|
- source-authority binding and provider-operation metadata
|
||||||
|
- health summary provider
|
||||||
|
|
||||||
|
## Permissions
|
||||||
|
|
||||||
|
Candidate scopes:
|
||||||
|
|
||||||
|
- `connectors:catalog:read`
|
||||||
|
- `connectors:profile:read`
|
||||||
|
- `connectors:profile:write`
|
||||||
|
- `connectors:profile:test`
|
||||||
|
- `connectors:secret:manage`
|
||||||
|
- `connectors:admin`
|
||||||
|
|
||||||
|
## Data Model Sketch
|
||||||
|
|
||||||
|
Candidate tables:
|
||||||
|
|
||||||
|
- `connector_types`
|
||||||
|
- `connector_profiles`
|
||||||
|
- `connector_profile_tests`
|
||||||
|
- `connector_health_status`
|
||||||
|
- `external_references`
|
||||||
|
|
||||||
|
Plaintext credentials must never be stored in connector tables. Use secret
|
||||||
|
references and the platform secret contract.
|
||||||
|
|
||||||
|
## WebUI
|
||||||
|
|
||||||
|
Initial route contributions:
|
||||||
|
|
||||||
|
- `/connectors`
|
||||||
|
- `/connectors/profiles/:profileId`
|
||||||
|
|
||||||
|
The UI should show profile status, last test result, capability labels, required
|
||||||
|
configuration-package dependencies, and external-reference search where a
|
||||||
|
connector supports it.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Minimum tests:
|
||||||
|
|
||||||
|
- core starts with connectors installed and no domain modules present
|
||||||
|
- profile validation rejects plaintext secret echoing
|
||||||
|
- connection tests record success/failure diagnostics without leaking secrets
|
||||||
|
- configuration package can require a connector profile
|
||||||
|
- domain-module optional behavior can detect connector capabilities without
|
||||||
|
imports
|
||||||
|
|
||||||
|
## Open Decisions
|
||||||
|
|
||||||
|
- Whether protocol-specific connector modules depend on `govoplan-connectors`
|
||||||
|
or only share kernel contracts.
|
||||||
|
- Which connector type should be first after OpenProject.
|
||||||
|
- How much polling/webhook scheduling belongs here versus workflow/ops.
|
||||||
|
- Whether external-reference indexing should move to search/dataflow later.
|
||||||
@@ -0,0 +1,246 @@
|
|||||||
|
# Connector Source Lifecycle
|
||||||
|
|
||||||
|
GovOPlaN modules should treat external systems as sources with explicit
|
||||||
|
lifecycle state. A connector profile can consume records from a source, publish
|
||||||
|
records into a source, or do both. The lifecycle below keeps connectors
|
||||||
|
predictable and avoids hidden module imports.
|
||||||
|
|
||||||
|
## Source Directions
|
||||||
|
|
||||||
|
- `consume`: GovOPlaN reads external records, normalizes them, and exposes them
|
||||||
|
to modules as external references, events, or staged imports.
|
||||||
|
- `publish`: GovOPlaN creates or updates external records and stores the external
|
||||||
|
identifiers as immutable references.
|
||||||
|
- `bidirectional`: GovOPlaN supports both directions with conflict detection and
|
||||||
|
reconciliation rules.
|
||||||
|
|
||||||
|
Direction describes transport. Every binding also needs a source-authority
|
||||||
|
mode:
|
||||||
|
|
||||||
|
- `native_authoritative`
|
||||||
|
- `external_authoritative`
|
||||||
|
- `external_mirror`
|
||||||
|
- `governed_sync`
|
||||||
|
- `governance_overlay`
|
||||||
|
- `linked_reference`
|
||||||
|
|
||||||
|
The authority mode and the connector's integration maturity are orthogonal. A
|
||||||
|
bidirectional connector may be configured as an external mirror, and a native
|
||||||
|
GovOPlaN object may publish to an external target without transferring
|
||||||
|
authority. The effective binding must identify its scope and provenance rather
|
||||||
|
than relying on a profile-wide `sync` boolean.
|
||||||
|
|
||||||
|
## Source Data Lifecycle
|
||||||
|
|
||||||
|
Connector profiles have operational states, while individual external records
|
||||||
|
or source datasets move through a data lifecycle:
|
||||||
|
|
||||||
|
1. `discovered`
|
||||||
|
A source, record, file, feed item, webhook event, or remote object is known
|
||||||
|
but not yet trusted for domain use.
|
||||||
|
2. `connected`
|
||||||
|
GovOPlaN can authenticate and fetch or publish against the source profile.
|
||||||
|
3. `imported`
|
||||||
|
Minimal source data has been staged with external id, version/ETag, source
|
||||||
|
timestamp, and provenance.
|
||||||
|
4. `validated`
|
||||||
|
Shape, permissions, freshness, and required fields passed connector and
|
||||||
|
domain validation.
|
||||||
|
5. `transformed`
|
||||||
|
A dataflow, workflow, or domain module normalized the staged payload into a
|
||||||
|
domain-specific form.
|
||||||
|
6. `published`
|
||||||
|
GovOPlaN exposed or wrote an output through API, RSS, report, export, or a
|
||||||
|
downstream connector.
|
||||||
|
7. `archived`
|
||||||
|
The source/output is no longer active but remains available under retention,
|
||||||
|
audit, and external-reference rules.
|
||||||
|
8. `deprecated`
|
||||||
|
The source/output remains readable for history but must not be used for new
|
||||||
|
workflows.
|
||||||
|
|
||||||
|
Every transition must preserve provenance, permissions context, freshness, and
|
||||||
|
audit trace. Domain modules may add stricter states, but they should map back to
|
||||||
|
this lifecycle when a connector publishes status.
|
||||||
|
|
||||||
|
## Lifecycle States
|
||||||
|
|
||||||
|
1. `draft`
|
||||||
|
Profile exists but is not used by runtime jobs.
|
||||||
|
2. `configured`
|
||||||
|
Required endpoint and credential references are present.
|
||||||
|
3. `tested`
|
||||||
|
A health/test run succeeded and recorded non-secret diagnostics.
|
||||||
|
4. `active`
|
||||||
|
Runtime jobs may consume or publish data.
|
||||||
|
5. `degraded`
|
||||||
|
The connector is active but health checks or recent jobs show failures.
|
||||||
|
6. `paused`
|
||||||
|
Operators intentionally stop scheduled connector activity.
|
||||||
|
7. `retiring`
|
||||||
|
The connector is being removed from active workflows while references remain
|
||||||
|
readable.
|
||||||
|
8. `retired`
|
||||||
|
No new runtime activity is allowed. Historical references remain available.
|
||||||
|
|
||||||
|
## State Transition Gates
|
||||||
|
|
||||||
|
| Transition | Required Evidence | Blockers |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `draft` -> `configured` | endpoint fields are valid, credential references exist, owner/tenant scope is set | plaintext secret in profile payload, unsupported connector type |
|
||||||
|
| `configured` -> `tested` | latest profile test succeeded and diagnostics were redacted | failed auth, unreachable endpoint, TLS/policy error |
|
||||||
|
| `tested` -> `active` | operator enabled runtime use, required modules/capabilities are present, schedule/webhook is valid | missing module, missing permission, no idempotency strategy for publish jobs |
|
||||||
|
| `active` -> `degraded` | health check or job telemetry reports failures | none; this is automatic diagnostic state |
|
||||||
|
| `degraded` -> `active` | health/test succeeds or failed jobs are reconciled | unresolved conflict or repeated failure threshold |
|
||||||
|
| any running state -> `paused` | operator pause request or maintenance preflight | active critical transaction that cannot be interrupted |
|
||||||
|
| `paused` -> `active` | successful re-test when credentials/endpoints changed | failed profile test |
|
||||||
|
| any state -> `retiring` | uninstall/disable plan accepted, schedulers/workers stopped | active domain references that require operator decision |
|
||||||
|
| `retiring` -> `retired` | non-destructive retirement complete, references remain readable | destructive retirement requested without provider and backup |
|
||||||
|
|
||||||
|
## Consume Flow
|
||||||
|
|
||||||
|
1. Discover changes through polling, webhook, batch upload, or manual operator
|
||||||
|
action.
|
||||||
|
2. Fetch only the minimal remote data required for the declared use case.
|
||||||
|
3. Normalize into a connector-owned staging payload.
|
||||||
|
4. Validate shape, required fields, and source trust level.
|
||||||
|
5. Publish data-shaped inputs as versioned datasource origins.
|
||||||
|
6. Let Datasources register live/cached origins or stage immutable snapshots.
|
||||||
|
7. Let domain modules consume governed datasource references through
|
||||||
|
capabilities, not imports.
|
||||||
|
8. Emit a core-mediated event such as `connector.record_discovered`.
|
||||||
|
9. Store external references with source system, object type, object id, version
|
||||||
|
or ETag, and last-seen timestamp.
|
||||||
|
|
||||||
|
## Tabular source discovery and refresh
|
||||||
|
|
||||||
|
Immutable JSON/CSV snapshots, exact managed Files versions, and live PostgreSQL
|
||||||
|
tables share the same catalogue and bounded-preview contract. A managed-file
|
||||||
|
origin stores only the Files asset id, exact immutable version id, checksum,
|
||||||
|
parser settings, reviewed schema, and discovery fingerprint. Files re-authorizes
|
||||||
|
the current principal and verifies storage integrity and any encryption envelope
|
||||||
|
on every preview. A newer current version produces a warning; only an explicit
|
||||||
|
source refresh changes the pinned version and increments the discovery revision.
|
||||||
|
|
||||||
|
A PostgreSQL origin references an active governed connector configuration. Its
|
||||||
|
endpoint must contain no credentials. Connectors resolves the referenced Core
|
||||||
|
credential envelope for the current tenant, scope, module, and server, opens a
|
||||||
|
read-only connection, reflects a simple schema/table identifier, and records the
|
||||||
|
configuration hash/revision, credential revision, schema, and discovery
|
||||||
|
fingerprint. Configuration, credential, or schema drift blocks preview until an
|
||||||
|
explicit refresh. Missing Files capability, revoked file access, quarantine,
|
||||||
|
oversized or malformed content, inactive/stale credentials, unreachable SQL,
|
||||||
|
and timeout failures produce sanitized unavailable/validation diagnostics.
|
||||||
|
|
||||||
|
All three current providers declare projection and pagination pushdown only.
|
||||||
|
Filters, aggregations, and sorting remain in Dataflow until an adapter explicitly
|
||||||
|
declares and tests those operations.
|
||||||
|
|
||||||
|
## Publish Flow
|
||||||
|
|
||||||
|
1. Domain module requests publish through a core-mediated connector capability.
|
||||||
|
2. Connector validates profile state, permission, idempotency key, and payload
|
||||||
|
shape.
|
||||||
|
3. Connector sends the remote request.
|
||||||
|
4. Connector stores the remote id, version/ETag, and response diagnostics.
|
||||||
|
5. A timeout or lost acknowledgement after dispatch becomes outcome-unknown,
|
||||||
|
not an ordinary failure or permission to duplicate the command.
|
||||||
|
6. Connector emits a confirmed, retryable, outcome-unknown, reconciled, or
|
||||||
|
corrected result event.
|
||||||
|
7. Domain module stores only the external-reference DTO and any domain result.
|
||||||
|
|
||||||
|
## Reconciliation
|
||||||
|
|
||||||
|
Every connector that writes to an external system needs a reconciliation story:
|
||||||
|
|
||||||
|
- idempotency key for create/update jobs
|
||||||
|
- remote object version, ETag, or last-modified value where available
|
||||||
|
- conflict state when local and remote records diverge
|
||||||
|
- retry policy for temporary failures
|
||||||
|
- explicit operator action for destructive overwrite or deletion
|
||||||
|
- audit trace from GovOPlaN record to external request and response summary
|
||||||
|
- explicit requested, approved, dispatched, possibly-executed, confirmed, and
|
||||||
|
reconciled/corrected effect states
|
||||||
|
|
||||||
|
## Durable recovery operations
|
||||||
|
|
||||||
|
Connectors declares two recovery classes. A read-only acquisition into an
|
||||||
|
immutable snapshot is `atomic`: the source revision or conditional cursor,
|
||||||
|
redacted dry-run decision, canonical request digest, and distributed
|
||||||
|
tenant/provider lease are durable before the fetch. The acquired domain rows
|
||||||
|
and terminal Core recovery checkpoint commit in one PostgreSQL transaction. A
|
||||||
|
caller-supplied `Idempotency-Key` replays that committed result without a second
|
||||||
|
provider request. A failed or stale transaction has no remote mutation and may
|
||||||
|
be repeated only as a new deliberate acquisition.
|
||||||
|
|
||||||
|
An external create, update, publish, or delete is `forward_recovery`. It must
|
||||||
|
start through the connector mutation recovery contract with a stable
|
||||||
|
idempotency key, SHA-256 request digest, source revision/cursor, and dry-run
|
||||||
|
evidence. Definitive rejection is terminal. A timeout or lost acknowledgement
|
||||||
|
after dispatch is `outcome_unknown` and blocks replay until the owning connector
|
||||||
|
verifies provider state. The MediaWiki/BlueSpice page publisher implements this
|
||||||
|
path for revision-checked edits. Other connector types do not thereby acquire a
|
||||||
|
write capability; each adapter must declare and prove its own recovery and
|
||||||
|
reconciliation behavior.
|
||||||
|
|
||||||
|
## Provider Declaration
|
||||||
|
|
||||||
|
An executable connector type should publish machine-readable metadata for:
|
||||||
|
|
||||||
|
- owned object and field groups, plus supported authority modes;
|
||||||
|
- supported discovery, link, search, read, publish, synchronize, migrate, and
|
||||||
|
replacement maturity;
|
||||||
|
- read/write/delete/preview/dry-run operations and bounded response limits;
|
||||||
|
- revision/concurrency token, freshness, health, timeout, retry, and conflict
|
||||||
|
semantics;
|
||||||
|
- idempotency and outcome-unknown handling;
|
||||||
|
- evidence, rollback/compensation, correction, and reconciliation paths;
|
||||||
|
- classification, purpose, retention, secret, degraded, and outage behavior.
|
||||||
|
|
||||||
|
This declaration composes Core contracts. It does not move protocol behavior
|
||||||
|
or domain semantics into Core or Connectors.
|
||||||
|
|
||||||
|
## Capability Boundary
|
||||||
|
|
||||||
|
Domain modules must not import connector implementation packages directly. They
|
||||||
|
should ask core for capabilities such as:
|
||||||
|
|
||||||
|
- `connectors.catalog`
|
||||||
|
- `connectors.profileTester`
|
||||||
|
- `connectors.health`
|
||||||
|
- `connectors.externalReferences`
|
||||||
|
- `connectors.datasourceOrigins`
|
||||||
|
- `connectors.sourceConsumer`
|
||||||
|
- `connectors.sourcePublisher`
|
||||||
|
|
||||||
|
Connector payloads should be DTOs or protocol objects from kernel/core
|
||||||
|
contracts. Protocol-specific clients stay inside the connector module that owns
|
||||||
|
them.
|
||||||
|
|
||||||
|
## Safety Rules
|
||||||
|
|
||||||
|
- Secret values never leave the secret contract and are never stored in test
|
||||||
|
result payloads.
|
||||||
|
- Runtime jobs must include profile id, connector type, direction, idempotency
|
||||||
|
key, and triggering principal/system actor.
|
||||||
|
- Profile tests must redact tokens, passwords, cookies, authorization headers,
|
||||||
|
and remote personal data not needed for diagnostics.
|
||||||
|
- Deactivation must stop schedulers/workers before profile removal.
|
||||||
|
- Uninstall defaults to non-destructive retirement; domain data and external
|
||||||
|
references remain readable.
|
||||||
|
- Destructive retirement requires a module-owned retirement provider and an
|
||||||
|
explicit operator choice.
|
||||||
|
|
||||||
|
## Release Checklist
|
||||||
|
|
||||||
|
Before shipping an executable connector type:
|
||||||
|
|
||||||
|
- Add catalogue metadata and capability names.
|
||||||
|
- Add profile schema validation that rejects plaintext secrets.
|
||||||
|
- Add redaction tests for success and failure diagnostics.
|
||||||
|
- Add unavailable-optional-module tests for every consuming domain module.
|
||||||
|
- Add profile test and health status fixtures.
|
||||||
|
- Add external-reference DTO tests.
|
||||||
|
- Add source-authority and provider-declaration validation tests.
|
||||||
|
- Add lifecycle transition tests for pause, retry, retirement, and uninstall
|
||||||
|
guard behavior.
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
# Governed Connector Configuration
|
||||||
|
|
||||||
|
GovOPlaN connectors should make integration behavior inspectable and testable.
|
||||||
|
The target is not hardcoded glue hidden in module code, but governed connector
|
||||||
|
definitions with schemas, mappings, test runs, simulation, versioning, and
|
||||||
|
audit-visible execution.
|
||||||
|
|
||||||
|
## Connector Definition
|
||||||
|
|
||||||
|
A connector definition should describe:
|
||||||
|
|
||||||
|
- provider type and protocol
|
||||||
|
- endpoint and credential requirements
|
||||||
|
- supported capabilities
|
||||||
|
- input and output schemas
|
||||||
|
- mapping and transformation versions
|
||||||
|
- validation rules
|
||||||
|
- dry-run and test operations
|
||||||
|
- privacy and retention classification
|
||||||
|
- expected events and audit records
|
||||||
|
- operational limits and retry behavior
|
||||||
|
|
||||||
|
Provider-specific code may still be required, but the configured integration
|
||||||
|
logic should remain visible and reviewable.
|
||||||
|
|
||||||
|
## Runtime Expectations
|
||||||
|
|
||||||
|
Connectors supports the generic governed-definition and simulation portion of:
|
||||||
|
|
||||||
|
- discovery where possible
|
||||||
|
- typed configuration through UI-managed controls
|
||||||
|
- secret references instead of plaintext secrets
|
||||||
|
- dry-run plans before writes
|
||||||
|
- simulation with sample payloads
|
||||||
|
- provenance for consumed and produced data
|
||||||
|
- idempotent external writes where supported
|
||||||
|
- quarantine/manual-review state for unsafe or ambiguous results
|
||||||
|
|
||||||
|
Configuration packages may install connector definitions, but local overrides
|
||||||
|
must be protected from accidental package updates.
|
||||||
|
|
||||||
|
## Implemented Runtime Slice
|
||||||
|
|
||||||
|
The module now persists tenant-scoped connector definitions as immutable
|
||||||
|
revisions. A governed definition explicitly validates its provider, protocol,
|
||||||
|
capabilities, input/output schemas, mapping version and rules, validation,
|
||||||
|
preview metadata, audit expectations, classification, retention, limits, and
|
||||||
|
retry policy. Definitions record whether they are locally owned or supplied by
|
||||||
|
a named package.
|
||||||
|
|
||||||
|
Configurations pin a definition revision. They store an endpoint and a secret
|
||||||
|
reference, never credentials embedded in the URL. Tenant-local override values
|
||||||
|
are merged into the pinned definition and every overridden leaf is exposed as
|
||||||
|
a protected path. Installing a later package revision only marks the
|
||||||
|
configuration as having an update available. Adoption is an explicit,
|
||||||
|
optimistically locked action that reapplies the protected overrides over the
|
||||||
|
new package revision.
|
||||||
|
|
||||||
|
The generic execution surface supports bounded dry-runs and simulations. Each
|
||||||
|
run has a caller idempotency key and retains hashes of its inputs and effective
|
||||||
|
configuration together with definition, configuration, mapping, external
|
||||||
|
revision, actor, classification, and retention provenance. Samples redact the
|
||||||
|
definition's protected fields. Ambiguous uniqueness results follow the
|
||||||
|
configuration's policy and become either:
|
||||||
|
|
||||||
|
- `manual_review` with a pending decision;
|
||||||
|
- `quarantined` until an administrator decides; or
|
||||||
|
- `rejected` without a review queue entry.
|
||||||
|
|
||||||
|
Review decisions require a reason and are audited. The generic runtime stops
|
||||||
|
at deterministic preview evidence: provider-specific adapters remain
|
||||||
|
responsible for live external writes and must satisfy the Core connector
|
||||||
|
recovery contract before claiming write maturity.
|
||||||
|
|
||||||
|
The Connector governance administration page follows the shared workspace
|
||||||
|
archetype. Reload and Save stay in the semantic action bar, dirty navigation is
|
||||||
|
guarded, package adoption is a separate action, and simulation results and
|
||||||
|
review decisions remain visibly distinct from configuration editing.
|
||||||
|
|
||||||
|
## Relationship To Datasources And Dataflow
|
||||||
|
|
||||||
|
Recurring extraction and transformation should start as configuration across
|
||||||
|
connectors, files, workflow, reporting, and templates. Create dedicated
|
||||||
|
datasource or dataflow modules only when repeated source-catalog, lineage,
|
||||||
|
mapping, scheduling, or publication contracts clearly outgrow connector
|
||||||
|
ownership.
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
# MediaWiki and BlueSpice Knowledge Connector
|
||||||
|
|
||||||
|
The knowledge connector integrates an external MediaWiki or BlueSpice instance
|
||||||
|
without making Connectors the owner of native knowledge semantics. Connectors
|
||||||
|
owns endpoint and credential governance, Action API transport, discovery,
|
||||||
|
synchronization, identity-stable snapshots, diagnostics, and publication
|
||||||
|
recovery. Wiki continues to own GovOPlaN-native spaces, pages, drafts,
|
||||||
|
revisions, comments, and publishing policy.
|
||||||
|
|
||||||
|
## Configure and discover
|
||||||
|
|
||||||
|
Create an active governed connector definition and configuration with:
|
||||||
|
|
||||||
|
- provider `mediawiki` or `bluespice`;
|
||||||
|
- protocol `mediawiki`, `action_api`, or `mediawiki_action_api`;
|
||||||
|
- an absolute HTTP(S) endpoint without embedded credentials; and
|
||||||
|
- an optional Core credential-envelope reference. Publication requires the
|
||||||
|
credential; public read-only sources may omit it.
|
||||||
|
|
||||||
|
Core validates the endpoint and every redirect against the deployment egress,
|
||||||
|
DNS/IP-pinning, peer, downgrade, and response-size policies. Credentials can be
|
||||||
|
bearer/access tokens or username/password values resolved only for the current
|
||||||
|
tenant, actor, module, target scope, and server. They never enter connector
|
||||||
|
profiles, snapshots, diagnostics, URLs, or API responses.
|
||||||
|
|
||||||
|
A knowledge profile selects desired maturity and authority and maps each
|
||||||
|
included source namespace to a target Wiki space reference and path prefix.
|
||||||
|
Every profile also supplies a tenant or restricted fallback visibility. A
|
||||||
|
restricted fallback needs at least one normalized Search ACL token such as
|
||||||
|
`group:<id>`, `role:<id>`, `membership:<id>`, `account:<id>`,
|
||||||
|
`identity:<id>`, `function:<id>`, or `scope:<permission>`.
|
||||||
|
|
||||||
|
Discovery queries site and current-user metadata. It records the product,
|
||||||
|
version, API/PHP version, extensions, namespaces, advertised edit right,
|
||||||
|
capabilities, a stable discovery digest, and sanitized diagnostics. BlueSpice
|
||||||
|
permission and discussion extensions are detected where advertised. Standard
|
||||||
|
MediaWiki does not expose complete page ACL semantics through the base Action
|
||||||
|
API, so discovery reports that the configured namespace fallback will be used.
|
||||||
|
|
||||||
|
## Synchronize and preserve identity
|
||||||
|
|
||||||
|
Run one bounded full backfill and then bounded recent-change deltas. Every run
|
||||||
|
requires an idempotency key; exact replay returns the original evidence without
|
||||||
|
contacting the source, while reusing the key with different inputs is rejected.
|
||||||
|
The profile stores only its latest cursor and high-watermark. Runs retain
|
||||||
|
counts, redacted effects, diagnostics, configuration/discovery provenance, and
|
||||||
|
transport evidence.
|
||||||
|
|
||||||
|
Snapshots use the MediaWiki page id as stable identity and preserve revision
|
||||||
|
id, canonical URL, content digest, namespace, title, source timestamp, and
|
||||||
|
observed timestamp. Page content mapping covers categories, links, file
|
||||||
|
references, discussion references, revision author references, redirect
|
||||||
|
targets, permission metadata, target space/path, and detected wikitext macros.
|
||||||
|
Referenced files remain external references; binary transfer and attachment
|
||||||
|
ownership remain with Files or DMS.
|
||||||
|
|
||||||
|
Moves update the existing stable page. Deletions use the stable page id when the
|
||||||
|
provider supplies it and otherwise match a previously synchronized namespace
|
||||||
|
and title. An unmatched deletion becomes a separate tombstone with an explicit
|
||||||
|
diagnostic instead of silently deleting an unrelated page. Permission changes
|
||||||
|
are part of the content digest and therefore update the current snapshot even
|
||||||
|
when page text is unchanged.
|
||||||
|
|
||||||
|
Desired maturity is an operator ceiling. A read-only profile cannot be used for
|
||||||
|
synchronization, migration preview, or publication merely because the provider
|
||||||
|
advertises those capabilities.
|
||||||
|
|
||||||
|
## Search and access safety
|
||||||
|
|
||||||
|
When Search is installed, Connectors registers `connectors.mediawiki.pages` for
|
||||||
|
the `external_knowledge_page` resource type. Backfill includes only active,
|
||||||
|
non-deleted pages from active profiles. Restricted documents carry the current
|
||||||
|
snapshot ACL; tenant documents carry no narrower ACL.
|
||||||
|
|
||||||
|
Search performs a fail-closed authorization recheck for every result. It
|
||||||
|
requires the external-knowledge read permission, an exact tenant match, an
|
||||||
|
active profile, a current non-deleted page, and—when restricted—intersection
|
||||||
|
with the current account, membership, identity, group, role, function, or scope
|
||||||
|
tokens. Consequently, an ACL change takes effect even before a deferred index
|
||||||
|
update completes. Immediate index updates remove deleted pages and refresh ACLs;
|
||||||
|
the next Search rebuild reconciles any transient writer failure. Search also
|
||||||
|
removes source projections when Connectors is disabled.
|
||||||
|
|
||||||
|
## Publish and reconcile
|
||||||
|
|
||||||
|
Publication requires the publish permission, an active profile and
|
||||||
|
configuration, discovered provider publication capability, desired maturity at
|
||||||
|
least `publish`, an idempotency key, page title/body, and an optional expected
|
||||||
|
external revision. The Action API edit carries `baserevid` when supplied and
|
||||||
|
uses a credential-provided or freshly acquired CSRF token.
|
||||||
|
|
||||||
|
Before remote I/O, Connectors prepares a durable forward-recovery operation
|
||||||
|
with request digest, expected revision, cursor, profile/configuration revision,
|
||||||
|
and resource identity. A confirmed provider response stores the stable page and
|
||||||
|
new revision and completes the recovery evidence. A definitive provider
|
||||||
|
rejection is terminal. A timeout, invalid response, or server failure after
|
||||||
|
dispatch becomes outcome-unknown: do not retry with another key until an
|
||||||
|
operator compares the provider page/revision and reconciles the recovery
|
||||||
|
operation. Local rollback cannot undo a confirmed external edit.
|
||||||
|
|
||||||
|
## Migration preview into Wiki
|
||||||
|
|
||||||
|
Migration is intentionally a dry-run in this slice. The request names a target
|
||||||
|
space, supported macro set, existing target paths/source identities, and
|
||||||
|
existing attachment names. The preview is bounded to 500 source pages and
|
||||||
|
records a source fingerprint and high-watermark. It reports create, update, or
|
||||||
|
conflict effects plus:
|
||||||
|
|
||||||
|
- target path already owned by a different external page;
|
||||||
|
- attachment filename collisions;
|
||||||
|
- unsupported macros;
|
||||||
|
- pages outside the requested target space; and
|
||||||
|
- truncation at the configured limit.
|
||||||
|
|
||||||
|
`can_apply=true` means the bounded preview contains no error or conflict and is
|
||||||
|
not truncated. It does not write Wiki pages. A future target-side migration
|
||||||
|
worker must use the optional connector capability and Wiki-owned mutation
|
||||||
|
contract, revalidate the preview fingerprint, and keep native Wiki permissions
|
||||||
|
and revision history authoritative.
|
||||||
|
|
||||||
|
## Operations and privacy
|
||||||
|
|
||||||
|
Provider state exposes profile product/maturity, active-object count, latest run
|
||||||
|
status, health, recovery attention, and last success time without endpoint,
|
||||||
|
credential, title, ACL, or content data. Existing snapshots may remain
|
||||||
|
available during an outage only to principals still authorized by current local
|
||||||
|
profile and ACL state; their health and freshness remain explicit.
|
||||||
|
|
||||||
|
Data-subject exports include only minimized operator attribution for profile
|
||||||
|
updates and synchronization/migration/publication runs. They exclude endpoints,
|
||||||
|
credentials, page content, titles, ACLs, namespace mappings, idempotency keys,
|
||||||
|
request hashes, effects, diagnostics, provenance, and transport evidence.
|
||||||
|
Attribution and external-operation evidence is retained for governance rather
|
||||||
|
than automatically erased.
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
# OpenDesk Integration Map
|
||||||
|
|
||||||
|
OpenDesk is an integration profile across GovOPlaN modules, not a monolithic
|
||||||
|
GovOPlaN module. The profile should let an operator see which OpenDesk
|
||||||
|
components are connected, which GovOPlaN module owns each behavior, and which
|
||||||
|
optional capabilities are available.
|
||||||
|
|
||||||
|
The core boundary decision register is in
|
||||||
|
`/mnt/DATA/git/govoplan-core/docs/MODULE_ARCHITECTURE.md`.
|
||||||
|
|
||||||
|
## Component Routing
|
||||||
|
|
||||||
|
| OpenDesk area | Example component | GovOPlaN owner | Integration behavior |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Identity and directory | OpenDesk IDM, LDAP, AD, OIDC, SAML, SCIM | `govoplan-idm`, `govoplan-access` | integrate, synchronize selected accounts/groups, map principals and memberships |
|
||||||
|
| Mail/groupware | Open-Xchange mail | `govoplan-mail` | integrate, link profiles, test mailbox/send/append, keep mail semantics in mail |
|
||||||
|
| Calendar/groupware | Open-Xchange calendar, CalDAV/CardDAV | `govoplan-calendar` | integrate, free/busy lookup, selected event sync, resource calendars |
|
||||||
|
| Files/documents | Nextcloud/WebDAV/files, office integrations | `govoplan-files`, later `govoplan-dms` | integrate, import/link files, keep document lifecycle in DMS |
|
||||||
|
| Project management | OpenProject | `govoplan-connectors`, consumers in tasks/workflow/cases | connector-first, link/synchronize selected work packages |
|
||||||
|
| Portal/collaboration | Portal/chat/video/office services where present | `govoplan-portal`, `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow` | link/integrate only when a process needs it |
|
||||||
|
| Inventory and diagnostics | endpoint catalogue, profile health, version checks | `govoplan-connectors` | catalogue, profile test, health summary, optional capability discovery |
|
||||||
|
|
||||||
|
## Integration Behavior
|
||||||
|
|
||||||
|
- `integrate`: call a stable API or protocol for the component.
|
||||||
|
- `link`: store external references and open the external tool for
|
||||||
|
source-of-truth work.
|
||||||
|
- `import`: bring selected files/records into GovOPlaN-owned storage or
|
||||||
|
evidence.
|
||||||
|
- `synchronize`: keep selected records aligned through explicit source-of-truth
|
||||||
|
rules.
|
||||||
|
- `replace selected workflow`: only when GovOPlaN owns tighter governance,
|
||||||
|
audit, retention, or configuration-package state than the OpenDesk component.
|
||||||
|
|
||||||
|
## Shared Assumptions
|
||||||
|
|
||||||
|
- Identity is the first dependency. Mail, calendar, files, and project
|
||||||
|
connectors should record which identity profile or tenant mapping they expect.
|
||||||
|
- Connector profiles store references to secrets, never secret values.
|
||||||
|
- Module consumers discover optional behavior through core capabilities and
|
||||||
|
module metadata.
|
||||||
|
- The profile must work partially: an installation can have OpenProject without
|
||||||
|
Open-Xchange, or calendar without files.
|
||||||
|
|
||||||
|
## Candidate Profile Shape
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "opendesk-main",
|
||||||
|
"display_name": "OpenDesk",
|
||||||
|
"components": {
|
||||||
|
"identity": {"profile_id": "opendesk-idm", "owner": "govoplan-idm"},
|
||||||
|
"mail": {"profile_id": "ox-mail", "owner": "govoplan-mail"},
|
||||||
|
"calendar": {"profile_id": "ox-calendar", "owner": "govoplan-calendar"},
|
||||||
|
"files": {"profile_id": "nextcloud-main", "owner": "govoplan-files"},
|
||||||
|
"projects": {"profile_id": "openproject-main", "owner": "govoplan-connectors"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Follow-Up Implementation Issues
|
||||||
|
|
||||||
|
Existing high-priority module issues:
|
||||||
|
|
||||||
|
- `govoplan-idm#1`: LDAP, Active Directory, OpenDesk identity services.
|
||||||
|
- `govoplan-mail#5`: Open-Xchange mail/groupware adapter boundary.
|
||||||
|
- `govoplan-calendar#2`: Open-Xchange calendar adapter boundary.
|
||||||
|
- `govoplan-connectors#1`: OpenProject connector.
|
||||||
|
|
||||||
|
Future executable connector issues should be created in the owning module
|
||||||
|
repository when the first concrete API/profile slice is selected.
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
# OpenProject Connector Concept
|
||||||
|
|
||||||
|
OpenProject is the first proposed concrete connector for
|
||||||
|
`govoplan-connectors`. It gives GovOPlaN a public-sector-friendly project and
|
||||||
|
work-package integration target without making project management a core
|
||||||
|
platform dependency.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- Register OpenProject connection profiles.
|
||||||
|
- Test API reachability and authentication without exposing secrets.
|
||||||
|
- Read projects, users, statuses, and work packages for linking.
|
||||||
|
- Create or update work packages from GovOPlaN tasks/cases/workflows once those
|
||||||
|
modules request the capability.
|
||||||
|
- Receive or poll changes for external-reference synchronization.
|
||||||
|
- Keep all OpenProject-specific client code inside the connector module.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
- Replacing a future native GovOPlaN project-management module.
|
||||||
|
- Importing workflow, tasks, cases, or access implementation modules directly.
|
||||||
|
- Mirroring complete OpenProject project state into GovOPlaN by default.
|
||||||
|
- Storing OpenProject tokens outside the platform secret contract.
|
||||||
|
|
||||||
|
## Profile Fields
|
||||||
|
|
||||||
|
Candidate profile payload:
|
||||||
|
|
||||||
|
- `base_url`
|
||||||
|
- `api_version`, default `v3`
|
||||||
|
- `credential_ref`
|
||||||
|
- `verify_tls`
|
||||||
|
- `timeout_seconds`
|
||||||
|
- `allowed_project_ids`
|
||||||
|
- `default_project_id`
|
||||||
|
- `webhook_secret_ref`, optional
|
||||||
|
- `poll_interval_seconds`, optional
|
||||||
|
|
||||||
|
## Health Check
|
||||||
|
|
||||||
|
The connection test should:
|
||||||
|
|
||||||
|
1. Normalize and validate `base_url`.
|
||||||
|
2. Resolve the credential reference.
|
||||||
|
3. Call the OpenProject API root or a small read-only endpoint.
|
||||||
|
4. Record API version, authenticated principal where available, latency,
|
||||||
|
response status, and safe capability hints.
|
||||||
|
5. Redact token, Authorization headers, cookies, and any server-provided secret
|
||||||
|
fields from diagnostics.
|
||||||
|
|
||||||
|
## Candidate Capabilities
|
||||||
|
|
||||||
|
- `connectors.openproject.profileTester`
|
||||||
|
- `connectors.openproject.projects`
|
||||||
|
- `connectors.openproject.workPackages.read`
|
||||||
|
- `connectors.openproject.workPackages.write`
|
||||||
|
- `connectors.openproject.webhooks`
|
||||||
|
- `connectors.openproject.externalReferences`
|
||||||
|
|
||||||
|
Domain modules request these through core-mediated capabilities. For example,
|
||||||
|
`govoplan-tasks` can publish a task as an OpenProject work package without
|
||||||
|
importing OpenProject client code.
|
||||||
|
|
||||||
|
## Data Boundary Decision
|
||||||
|
|
||||||
|
OpenProject should be referenced live by default, not mirrored wholesale into
|
||||||
|
GovOPlaN. The connector stores stable external references and safe metadata:
|
||||||
|
|
||||||
|
- profile id and connector type
|
||||||
|
- project id and work-package id
|
||||||
|
- external URL
|
||||||
|
- remote version, ETag, or lock version where available
|
||||||
|
- last-seen timestamp and safe status/type labels
|
||||||
|
- GovOPlaN trace id for publish or synchronization jobs
|
||||||
|
|
||||||
|
GovOPlaN should import only the subset needed by a requesting domain module,
|
||||||
|
for example a work-package title/status for display, a link-back reference for a
|
||||||
|
task, or evidence that a publish operation succeeded. Full project state,
|
||||||
|
comments, attachments, membership lists, and custom fields remain remote unless
|
||||||
|
a future domain module explicitly owns that synchronization. This keeps cases,
|
||||||
|
tasks, workflow, and reporting decoupled from OpenProject while still allowing
|
||||||
|
link-out, link-back, selected publish, and selected read views.
|
||||||
|
|
||||||
|
## Runtime Events
|
||||||
|
|
||||||
|
- `openproject.profile_tested`
|
||||||
|
- `openproject.project_seen`
|
||||||
|
- `openproject.work_package_seen`
|
||||||
|
- `openproject.work_package_published`
|
||||||
|
- `openproject.webhook_received`
|
||||||
|
- `openproject.sync_failed`
|
||||||
|
|
||||||
|
Events should carry GovOPlaN ids, external ids, safe diagnostics, and trace
|
||||||
|
context. They must not contain credentials or raw personal data beyond what the
|
||||||
|
requesting domain module is authorized to process.
|
||||||
|
|
||||||
|
## Webhook And Polling Strategy
|
||||||
|
|
||||||
|
OpenProject supports API and webhook administration. The connector should allow
|
||||||
|
both:
|
||||||
|
|
||||||
|
- Webhook-first when an operator registers a webhook for selected project/work
|
||||||
|
package events.
|
||||||
|
- Polling fallback for installations where webhooks cannot be exposed.
|
||||||
|
|
||||||
|
The first implementation can start with manual test plus read-only project/work
|
||||||
|
package lookup, then add publishing, then webhook/polling synchronization.
|
||||||
|
|
||||||
|
## First Implementation Slice
|
||||||
|
|
||||||
|
1. Add connector type metadata for `openproject`.
|
||||||
|
2. Add connection profile CRUD using secret references.
|
||||||
|
3. Add a read-only test endpoint.
|
||||||
|
4. Add project/work-package lookup DTOs.
|
||||||
|
5. Add external-reference storage for linked OpenProject work packages.
|
||||||
|
6. Add a WebUI profile page with last-test diagnostics.
|
||||||
|
7. Add tests for redaction, unavailable connector behavior, and optional module
|
||||||
|
capability discovery.
|
||||||
|
|
||||||
|
## Minimum DTOs
|
||||||
|
|
||||||
|
Profile summary:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "openproject-main",
|
||||||
|
"connector_type": "openproject",
|
||||||
|
"name": "OpenProject",
|
||||||
|
"base_url": "https://openproject.example",
|
||||||
|
"state": "tested",
|
||||||
|
"last_test_at": "2026-07-09T10:00:00Z",
|
||||||
|
"last_test_status": "success"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
External reference:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"connector_type": "openproject",
|
||||||
|
"profile_id": "openproject-main",
|
||||||
|
"object_type": "work_package",
|
||||||
|
"external_id": "1234",
|
||||||
|
"external_url": "https://openproject.example/work_packages/1234",
|
||||||
|
"version": "etag-or-lock-version",
|
||||||
|
"metadata": {
|
||||||
|
"project_id": "42"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Diagnostics must be redacted and should include only endpoint, version,
|
||||||
|
authenticated principal label where safe, latency, status code, and capability
|
||||||
|
hints.
|
||||||
|
|
||||||
|
## First Tests To Add
|
||||||
|
|
||||||
|
- profile create/update rejects plaintext token fields
|
||||||
|
- profile test redacts Authorization, cookies, and token-like response fields
|
||||||
|
- lookup capabilities are absent when the connector module is disabled
|
||||||
|
- a task/workflow/case module can detect OpenProject capabilities without
|
||||||
|
importing connector internals
|
||||||
|
- external-reference round-trip stores profile id, object type, external id,
|
||||||
|
version/ETag, URL, and safe metadata
|
||||||
|
|
||||||
|
## Reference Sources
|
||||||
|
|
||||||
|
- OpenProject API v3 documentation: https://www.openproject.org/docs/api/
|
||||||
|
- OpenProject API introduction: https://www.openproject.org/docs/api/introduction/
|
||||||
|
- OpenProject API and webhooks administration: https://www.openproject.org/docs/system-admin-guide/api-and-webhooks/
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
# Public-Sector Integration Catalogue
|
||||||
|
|
||||||
|
`govoplan-connectors` should maintain an operator-visible catalogue of common
|
||||||
|
external systems, protocols, and integration patterns. The catalogue is not a
|
||||||
|
promise that GovOPlaN replaces those systems. It is the map that lets modules
|
||||||
|
discover what exists, test connections, and decide which optional behavior can
|
||||||
|
be enabled.
|
||||||
|
|
||||||
|
## Catalogue Entry Shape
|
||||||
|
|
||||||
|
Each connector type should define:
|
||||||
|
|
||||||
|
- stable connector type key, for example `openproject`, `fit-connect`,
|
||||||
|
`xrepository`, or `sap`
|
||||||
|
- category and owning GovOPlaN module, if any
|
||||||
|
- supported direction: consume, publish, or bidirectional
|
||||||
|
- supported trigger modes: manual test, polling, webhook, batch import, export
|
||||||
|
- credential method and whether secrets are stored through the platform secret
|
||||||
|
contract
|
||||||
|
- health check and diagnostic payload shape
|
||||||
|
- external reference shape for records created or linked through the connector
|
||||||
|
- required capabilities and optional module combinations
|
||||||
|
- lifecycle support: activate, pause, re-test, rotate credential, retire
|
||||||
|
|
||||||
|
## Initial Target Categories
|
||||||
|
|
||||||
|
The catalogue should be maintained as a ranked inventory. A target can start as
|
||||||
|
an inventory entry before there is executable connector code.
|
||||||
|
|
||||||
|
| Target | Scope/Jurisdiction | Category | Mode | Likely Owner | First Useful Capability | Priority |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| OpenProject | international/open source | Project/task management | link, synchronize selected records, publish tasks | `govoplan-connectors`, later tasks/workflow/projects | profile test, project/work-package lookup, external references | Wave 0 |
|
||||||
|
| Nextcloud/WebDAV/SMB/Seafile | broad public-sector/self-hosted | File providers | integrate, import, link | `govoplan-files` with connector inventory | profile health and managed-file provenance | Wave 0/in progress |
|
||||||
|
| OpenDesk IDM, LDAP, Active Directory, OIDC, SAML | Germany/EU and general enterprise | Identity | integrate, synchronize | `govoplan-idm`, `govoplan-access` | endpoint inventory, login/provisioning preflight | Wave 1 |
|
||||||
|
| Open-Xchange mail/calendar | Germany/OpenDesk and groupware deployments | Groupware | integrate, link | `govoplan-mail`, `govoplan-calendar` | profile test, mailbox/calendar diagnostics | Wave 1 |
|
||||||
|
| FIT-Connect | German public-sector transport | Public-sector transport | integrate, publish, receive | dedicated protocol module with connectors inventory | destination profile, test, receipt reference | Wave 1 |
|
||||||
|
| XRepository/XÖV lookup | German public-sector standards | Standards registry | link, import schema metadata | `govoplan-connectors`, later XÖV modules | read-only catalogue lookup/cache | Wave 1 |
|
||||||
|
| RSS/API publication | public data/external services | Publication/data exchange | consume, publish | `govoplan-connectors`, `govoplan-dataflow` | consume/publish feed profiles | Wave 2 |
|
||||||
|
| DMS/e-file/archive systems | German municipal/state/federal administration | DMS/records | link, import, synchronize selected metadata | `govoplan-dms`, `govoplan-files` | external document reference and health | Wave 2 |
|
||||||
|
| ERP/finance/procurement/payment systems | German municipal finance/procurement plus EU standards | ERP/payment | export, import, synchronize, replace only by domain decision | dedicated modules | profile inventory and export/import staging | Wave 2 |
|
||||||
|
|
||||||
|
### Project And Task Management
|
||||||
|
|
||||||
|
- OpenProject
|
||||||
|
- Jira or Jira-compatible APIs
|
||||||
|
- Redmine
|
||||||
|
- Microsoft Planner/Project where available through Microsoft Graph
|
||||||
|
|
||||||
|
GovOPlaN should start with OpenProject because it is open source, common in
|
||||||
|
public-sector environments, and has API/webhook documentation suitable for a
|
||||||
|
first connector.
|
||||||
|
|
||||||
|
### DMS, E-File, Records, And Archive
|
||||||
|
|
||||||
|
- d.velop/d.3
|
||||||
|
- enaio
|
||||||
|
- Fabasoft eGov-Suite
|
||||||
|
- ELO
|
||||||
|
- VIS/eAkte environments
|
||||||
|
- CMIS-capable repositories
|
||||||
|
- S3/object storage used as archive staging
|
||||||
|
|
||||||
|
These targets should usually be owned by DMS/files/records modules once a
|
||||||
|
domain module exists. `govoplan-connectors` should still provide inventory,
|
||||||
|
profiles, and generic health checks.
|
||||||
|
|
||||||
|
### File Providers
|
||||||
|
|
||||||
|
- SMB/CIFS
|
||||||
|
- WebDAV
|
||||||
|
- Nextcloud
|
||||||
|
- Seafile
|
||||||
|
- S3-compatible object storage
|
||||||
|
- SFTP
|
||||||
|
|
||||||
|
The files module owns file semantics. The connectors catalogue should record
|
||||||
|
profile metadata and health, but must not import files-module internals.
|
||||||
|
|
||||||
|
### Identity And Access
|
||||||
|
|
||||||
|
- LDAP
|
||||||
|
- Active Directory
|
||||||
|
- OIDC
|
||||||
|
- SAML
|
||||||
|
- OpenDesk IDM and comparable identity platforms
|
||||||
|
|
||||||
|
The access/IDM modules own principal synchronization and authorization effects.
|
||||||
|
Connectors own endpoint inventory and diagnostics.
|
||||||
|
|
||||||
|
### Mail, Calendar, And Collaboration
|
||||||
|
|
||||||
|
- Microsoft Exchange/M365
|
||||||
|
- Open-Xchange
|
||||||
|
- IMAP/SMTP where represented as external infrastructure
|
||||||
|
- CalDAV/CardDAV
|
||||||
|
- chat, video, and collaboration systems such as Matrix, Jitsi, BigBlueButton,
|
||||||
|
Nextcloud Talk, or Collabora/OnlyOffice environments
|
||||||
|
|
||||||
|
Mail/calendar/collaboration modules own business semantics. Connector profiles
|
||||||
|
can expose reachability and version diagnostics.
|
||||||
|
|
||||||
|
### ERP, Finance, Procurement, And Payment
|
||||||
|
|
||||||
|
- SAP
|
||||||
|
- MACH
|
||||||
|
- Infoma/new system
|
||||||
|
- DATEV interfaces
|
||||||
|
- XRechnung/Peppol access points
|
||||||
|
- XBestellung and procurement feeds
|
||||||
|
- payment providers and cash-register systems
|
||||||
|
|
||||||
|
Protocol-heavy parts should move into dedicated modules such as
|
||||||
|
`govoplan-xrechnung`, `govoplan-erp`, `govoplan-procurement`, or
|
||||||
|
`govoplan-payments`.
|
||||||
|
|
||||||
|
### Public-Sector Protocols And Registries
|
||||||
|
|
||||||
|
- FIT-Connect
|
||||||
|
- XTA/OSCI
|
||||||
|
- XÖV standards and XRepository lookup
|
||||||
|
- XRechnung/XBestellung
|
||||||
|
- register and Fachverfahren interfaces discovered by implementation projects
|
||||||
|
|
||||||
|
These are integration priorities because they model common administrative
|
||||||
|
processes. They should be represented as connector categories even when a
|
||||||
|
dedicated module later owns the actual protocol implementation.
|
||||||
|
|
||||||
|
## Wave 0 Catalogue Priorities
|
||||||
|
|
||||||
|
1. OpenProject connector concept and profile shape.
|
||||||
|
2. Generic connector profile, health, and secret-reference model.
|
||||||
|
3. Public-sector target inventory table with category, owner module, and
|
||||||
|
priority.
|
||||||
|
4. Consume/publish source lifecycle contract.
|
||||||
|
5. External-reference DTO shared through kernel/core contracts.
|
||||||
|
6. Configuration-package declaration for required connector profiles.
|
||||||
|
|
||||||
|
## Catalogue Maintenance Rules
|
||||||
|
|
||||||
|
- Prefer one stable connector type key per external product or protocol family.
|
||||||
|
- Record when GovOPlaN should integrate with an existing product instead of
|
||||||
|
replacing it.
|
||||||
|
- Keep protocol/client implementation in the owning connector or protocol
|
||||||
|
module; domain modules consume capabilities and DTOs only.
|
||||||
|
- Treat "inventory only" entries as useful: operators can document a landscape
|
||||||
|
before GovOPlaN can automate it.
|
||||||
|
- Every executable connector type needs a redaction-safe test plan, lifecycle
|
||||||
|
states, external-reference shape, and uninstall/retirement behavior.
|
||||||
|
|
||||||
|
## Reference Sources
|
||||||
|
|
||||||
|
- OpenProject API v3 documentation: https://www.openproject.org/docs/api/
|
||||||
|
- OpenProject API and webhooks administration: https://www.openproject.org/docs/system-admin-guide/api-and-webhooks/
|
||||||
|
- FIT-Connect Destination API documentation: https://docs.fitko.de/en/resources/fit-connect-destination-api/
|
||||||
|
- XÖV overview by KoSIT: https://www.xoev.de/xoev-4987
|
||||||
|
- XRepository overview: https://www.xrepository.de/
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
# Znuny and OTRS-compatible service-desk connector
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
Connectors owns the governed endpoint profile, GenericInterface REST transport,
|
||||||
|
credential hand-off, provider discovery, bounded synchronization, external
|
||||||
|
references, mapping diagnostics, health, and recovery evidence. It does not own
|
||||||
|
ticket, helpdesk, or case semantics. A synchronized provider ticket remains an
|
||||||
|
external service-desk ticket; creating or relating a GovOPlaN Ticket, Helpdesk
|
||||||
|
item, or Case is the responsibility of the corresponding optional module.
|
||||||
|
|
||||||
|
The provider interface is `connectors.external_service_desk@1.0.0`, and the
|
||||||
|
target-tested provider declaration is `connectors.znuny.tickets`.
|
||||||
|
|
||||||
|
## Governed configuration and routes
|
||||||
|
|
||||||
|
Create an active connector definition/configuration with:
|
||||||
|
|
||||||
|
- `provider`: `znuny`, `otrs`, or `znuny_otrs`;
|
||||||
|
- `protocol`: `generic_interface_rest` (the aliases `rest` and
|
||||||
|
`otrs_generic_interface_rest` are accepted);
|
||||||
|
- an HTTP(S) endpoint that passes the central outbound-request policy; and
|
||||||
|
- an optional scoped Core credential-envelope reference.
|
||||||
|
|
||||||
|
Znuny GenericInterface routes are configured by each deployment rather than
|
||||||
|
being one universal product API. The profile therefore governs relative search,
|
||||||
|
ticket-read, and optional update paths and their supported
|
||||||
|
methods. Ticket and update paths must contain `{ticket_id}`. An optional
|
||||||
|
absolute HTTP(S) browser URL template may contain `{ticket_id}` or
|
||||||
|
`{ticket_number}`. `search_filters` carries up to 100 deployment-supported,
|
||||||
|
secret-free GenericTicket search criteria such as queue identifiers. It cannot
|
||||||
|
override synchronization bounds, ordering, change cursors, or authentication.
|
||||||
|
Routes cannot change authority or contain credentials.
|
||||||
|
|
||||||
|
Header authentication is the recommended default. The adapter supports the
|
||||||
|
documented `X-OTRS-Header-UserLogin`, `X-OTRS-Header-Password`,
|
||||||
|
`X-OTRS-Header-SessionID`, and customer-login headers. A legacy credential may
|
||||||
|
declare `auth_mode: body`, but then every affected route must use POST. Secret
|
||||||
|
fields are never placed in a GET URL, persisted projection, diagnostic, or API
|
||||||
|
response. Core also treats these provider headers as redirect-sensitive and
|
||||||
|
removes them before following any cross-origin redirect.
|
||||||
|
|
||||||
|
Reference configuration examples:
|
||||||
|
|
||||||
|
- [Znuny GenericTicketConnectorREST example](https://doc.znuny.org/znuny/admin/webservices/examples/GenericTicketConnectorREST/index.html)
|
||||||
|
- [Znuny provider and header authentication](https://doc.znuny.org/znuny-7_1/admin/webservices/provider/)
|
||||||
|
- [Znuny web-service configuration](https://doc.znuny.org/znuny-7_3/admin/webservices/config/index.html)
|
||||||
|
|
||||||
|
## Profile policy
|
||||||
|
|
||||||
|
The integration and authority choices are intentionally separate but bounded:
|
||||||
|
|
||||||
|
| Integration mode | Allowed authority | Maturity |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `link` | `linked_reference` | `link` through `search` |
|
||||||
|
| `import` | `external_authoritative`, `external_mirror` | exactly `read` |
|
||||||
|
| `synchronize` | `external_authoritative`, `governed_sync` | exactly `synchronize` |
|
||||||
|
|
||||||
|
`link` retains stable identity and routing facts and asks the provider not to
|
||||||
|
return articles, attachments, or dynamic fields; any unexpectedly returned
|
||||||
|
content is still discarded. `import` is a deliberate bounded full snapshot and
|
||||||
|
never silently changes to delta synchronization.
|
||||||
|
`synchronize` completes a full reconciliation and then advances to overlap-safe
|
||||||
|
change-time deltas.
|
||||||
|
|
||||||
|
Queue mappings govern inclusion, an optional opaque target queue reference, and
|
||||||
|
tenant or restricted visibility with ACL tokens. A restricted default requires
|
||||||
|
at least one ACL token. Provider-supplied `GovOPlaNVisibility` and
|
||||||
|
`GovOPlaNACL` fields win when valid. Otherwise a reviewed queue mapping wins,
|
||||||
|
then the profile default. Standard GenericInterface installations do not expose
|
||||||
|
a portable ticket-ACL contract, so every fallback is visible as a diagnostic.
|
||||||
|
|
||||||
|
Dynamic-field mappings govern source name, optional target name, inclusion, and
|
||||||
|
`string`, `number`, `boolean`, `date`, or `json` conversion. Conversion loss,
|
||||||
|
unreturned configured fields, provider-specific ticket fields, synthesized
|
||||||
|
identities, and truncation are structured diagnostics rather than silent loss.
|
||||||
|
|
||||||
|
## Discovery and synchronization
|
||||||
|
|
||||||
|
Discovery performs a bounded ticket search and records health, API family,
|
||||||
|
route hash, the exact governed configuration revision/hash, product/version
|
||||||
|
evidence, capabilities, maturity, and diagnostics. A changed endpoint,
|
||||||
|
configuration revision, or route map invalidates that evidence: synchronization,
|
||||||
|
and updates fail closed until discovery is repeated, while prior projections are
|
||||||
|
invalidated and Search stays closed until a new full reconciliation verifies
|
||||||
|
them. Integration, route, queue, or dynamic-field mapping changes also reset the
|
||||||
|
cursor and require a full reconciliation. Recognized Znuny or OTRS major
|
||||||
|
versions 6 and later can reach synchronization maturity. An unverified
|
||||||
|
product/version stays at read maturity. An update route adds the technical
|
||||||
|
`publish` capability, but does not override profile authority.
|
||||||
|
|
||||||
|
Full synchronization first obtains a stable ordered identity set, then reads at
|
||||||
|
most 500 tickets per call. A continued full run stores its offset, identity-set
|
||||||
|
fingerprint, and cumulative high-watermark. If the provider identity set changes
|
||||||
|
mid-run, the cursor is rejected and the operator must restart the full run. A
|
||||||
|
completed full run reconciles local removals and, for `synchronize` mode,
|
||||||
|
transitions to an overlap-safe delta cursor.
|
||||||
|
|
||||||
|
An explicit `full` request always restarts at the beginning; `auto` continues a
|
||||||
|
committed full cursor or advances a completed delta cursor. A caller-supplied
|
||||||
|
cursor must exactly match the profile's committed cursor, and delta mode cannot
|
||||||
|
bootstrap a profile that has not completed its full synchronization.
|
||||||
|
|
||||||
|
Delta synchronization reads the bounded candidate set, orders changes by
|
||||||
|
provider change time and ticket id, and suppresses only the exact ticket
|
||||||
|
revisions already observed at the current timestamp boundary. A previously seen
|
||||||
|
ticket that has changed again is therefore not lost. Every run requires a
|
||||||
|
profile-wide idempotency key. An exact replay returns the committed run without
|
||||||
|
contacting the provider; reuse for a different request or changed profile policy
|
||||||
|
is rejected.
|
||||||
|
|
||||||
|
Operational bounds:
|
||||||
|
|
||||||
|
- at most 10,000 ticket identities per profile search;
|
||||||
|
- at most 500 ticket reads per API call;
|
||||||
|
- at most 10 MB per provider response;
|
||||||
|
- a 20-second outbound timeout; and
|
||||||
|
- a 4,000-character cursor, including timestamp-boundary identities.
|
||||||
|
|
||||||
|
Partition larger or unusually bursty providers into queue-scoped profiles by
|
||||||
|
combining provider-side queue `search_filters` with matching queue mappings. A
|
||||||
|
full-run fingerprint conflict requires a restart. A timestamp-boundary overflow
|
||||||
|
requires a narrower partition. These are explicit safety stops, not partial
|
||||||
|
success claims.
|
||||||
|
|
||||||
|
## Mapping and attachment policy
|
||||||
|
|
||||||
|
Each ticket projection retains stable ticket id/number, title, type, queue,
|
||||||
|
target queue reference, state, priority, owner, responsible user, customer user,
|
||||||
|
organization, service, SLA, creation/change times, mapped dynamic fields,
|
||||||
|
articles, attachment metadata, permission source, ACLs, canonical URL, content
|
||||||
|
hash, provider revision, cursor, observation time, and source provenance.
|
||||||
|
|
||||||
|
Article and attachment references use Core `ExternalObjectReference` values.
|
||||||
|
Article bodies are capped at 200,000 characters. Attachment content is never
|
||||||
|
retained; only stable identity, filename, media type, size, disposition, content
|
||||||
|
id, article/ticket relationship, version, and provenance are mapped. Returned
|
||||||
|
bytes produce an `attachment_content_omitted` diagnostic.
|
||||||
|
|
||||||
|
## Search authorization
|
||||||
|
|
||||||
|
When Search is installed and both desired and discovered maturity permit it,
|
||||||
|
active non-deleted ticket projections are indexed. Search documents carry the
|
||||||
|
current visibility, ACL tokens, external reference, source revision, routing
|
||||||
|
metadata, article text, and bounded dynamic-field keywords.
|
||||||
|
|
||||||
|
Authorization always fails closed unless all of these remain true:
|
||||||
|
|
||||||
|
- the requesting principal belongs to the exact tenant;
|
||||||
|
- the principal has `connectors:service_desk:read`;
|
||||||
|
- the profile remains active and search-capable;
|
||||||
|
- the ticket remains active; and
|
||||||
|
- tenant visibility applies or a current account, membership, identity, group,
|
||||||
|
role, function, or scope ACL token intersects.
|
||||||
|
|
||||||
|
Pausing a profile or changing fallback/queue ACLs updates or removes Search
|
||||||
|
projections immediately. Search also rechecks the current database state for
|
||||||
|
every result, so a delayed index update does not grant access.
|
||||||
|
|
||||||
|
## Governed external updates and recovery
|
||||||
|
|
||||||
|
An external update is allowed only when the profile is active, authority is
|
||||||
|
`governed_sync`, discovery recorded the `publish` capability, and the caller has
|
||||||
|
`connectors:service_desk:update`. The request must include the synchronized
|
||||||
|
provider revision and a new idempotency key. Supported governed fields are
|
||||||
|
title, queue, state, priority, owner, responsible user, and explicitly mapped
|
||||||
|
dynamic fields.
|
||||||
|
|
||||||
|
Before dispatch, the adapter refetches the ticket and rejects a stale revision.
|
||||||
|
After dispatch, it refetches again and verifies both the changed revision and
|
||||||
|
every requested field value. A transport failure before a conclusive provider
|
||||||
|
response, an unchanged revision, or a requested value that cannot be confirmed
|
||||||
|
becomes `outcome_unknown`. Do not retry with another key. Inspect the provider
|
||||||
|
ticket and reconcile its accepted revision through the Core recovery evidence
|
||||||
|
first. Local database rollback cannot undo a remote provider mutation.
|
||||||
|
|
||||||
|
## Administrator verification
|
||||||
|
|
||||||
|
1. Create the governed definition/configuration and scoped credential envelope.
|
||||||
|
2. Create a restricted profile with reviewed routes, queue partitions, dynamic
|
||||||
|
fields, authority, and fallback ACLs.
|
||||||
|
3. Discover and confirm product/version, maturity, capabilities, and diagnostics.
|
||||||
|
4. Finish a keyed full run; continue while its cursor kind is `full`.
|
||||||
|
5. Run a new keyed automatic delta and inspect effects, losses, and health.
|
||||||
|
6. Verify one allowed and one denied Search principal against a restricted ticket.
|
||||||
|
7. If governed writes are enabled, update a non-production ticket with its
|
||||||
|
current revision, then verify provider and recovery evidence.
|
||||||
|
8. Reconcile every `outcome_unknown` run before any retry.
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=69", "wheel"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "govoplan-connectors"
|
||||||
|
version = "0.1.22"
|
||||||
|
description = "Governed connector catalogue and tabular source capabilities for GovOPlaN."
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
license = "AGPL-3.0-or-later"
|
||||||
|
authors = [{ name = "GovOPlaN" }]
|
||||||
|
dependencies = [
|
||||||
|
"defusedxml>=0.7,<1",
|
||||||
|
"govoplan-core>=0.1.33",
|
||||||
|
"openpyxl>=3.1.5,<4",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["src"]
|
||||||
|
|
||||||
|
[tool.setuptools.package-data]
|
||||||
|
govoplan_connectors = ["py.typed"]
|
||||||
|
|
||||||
|
[project.entry-points."govoplan.modules"]
|
||||||
|
connectors = "govoplan_connectors.backend.manifest:get_manifest"
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
"""GovOPlaN Connectors module."""
|
||||||
|
|
||||||
|
__all__: list[str] = []
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Connector backend package."""
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from govoplan_core.core.datasources import (
|
||||||
|
DatasourceAccessError,
|
||||||
|
DatasourceField,
|
||||||
|
DatasourceNotFoundError,
|
||||||
|
DatasourceOrigin,
|
||||||
|
DatasourceOriginReadRequest,
|
||||||
|
DatasourceOriginReadResult,
|
||||||
|
DatasourceUnavailableError,
|
||||||
|
DatasourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularSource,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceError,
|
||||||
|
TabularSourceNotFoundError,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.tabular_sources import SqlTabularSourceProvider
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDatasourceOriginProvider:
|
||||||
|
"""Expose connector-owned sources through the Datasources origin contract."""
|
||||||
|
|
||||||
|
def __init__(self, provider: SqlTabularSourceProvider | None = None) -> None:
|
||||||
|
self._provider = provider or SqlTabularSourceProvider()
|
||||||
|
|
||||||
|
def list_origins(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
query: str = "",
|
||||||
|
limit: int = 100,
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
rows = self._provider.list_sources(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
query=query,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _datasource_error(exc) from exc
|
||||||
|
return tuple(_origin(source) for source in rows)
|
||||||
|
|
||||||
|
def get_origin(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
origin_ref: str,
|
||||||
|
) -> DatasourceOrigin | None:
|
||||||
|
try:
|
||||||
|
source = self._provider.get_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
source_ref=origin_ref,
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _datasource_error(exc) from exc
|
||||||
|
return _origin(source) if source is not None else None
|
||||||
|
|
||||||
|
def read_origin(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: DatasourceOriginReadRequest,
|
||||||
|
) -> DatasourceOriginReadResult:
|
||||||
|
try:
|
||||||
|
result = self._provider.read_source(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=request.origin_ref,
|
||||||
|
limit=request.limit,
|
||||||
|
offset=request.offset,
|
||||||
|
columns=request.columns,
|
||||||
|
expected_fingerprint=request.expected_fingerprint,
|
||||||
|
max_bytes=request.max_bytes,
|
||||||
|
timeout_ms=request.timeout_ms,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except TabularSourceError as exc:
|
||||||
|
raise _datasource_error(exc) from exc
|
||||||
|
return DatasourceOriginReadResult(
|
||||||
|
origin=_origin(result.source),
|
||||||
|
rows=result.rows,
|
||||||
|
total_rows=result.total_rows,
|
||||||
|
truncated=result.truncated,
|
||||||
|
returned_bytes=result.returned_bytes,
|
||||||
|
elapsed_ms=result.elapsed_ms,
|
||||||
|
effective_row_limit=result.effective_row_limit,
|
||||||
|
effective_byte_limit=result.effective_byte_limit,
|
||||||
|
effective_timeout_ms=result.effective_timeout_ms,
|
||||||
|
diagnostics=result.diagnostics,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _origin(source: TabularSource) -> DatasourceOrigin:
|
||||||
|
kind = {
|
||||||
|
"managed_file": "file",
|
||||||
|
"postgresql": "database",
|
||||||
|
}.get(source.provider, "upload")
|
||||||
|
return DatasourceOrigin(
|
||||||
|
ref=source.ref,
|
||||||
|
source_name=source.source_name,
|
||||||
|
name=source.name,
|
||||||
|
description=source.description,
|
||||||
|
kind=kind,
|
||||||
|
shape="tabular",
|
||||||
|
supported_modes=("live", "cached"),
|
||||||
|
provider=f"connectors.{source.provider}",
|
||||||
|
schema=tuple(
|
||||||
|
DatasourceField(
|
||||||
|
name=column.name,
|
||||||
|
data_type=column.data_type,
|
||||||
|
nullable=column.nullable,
|
||||||
|
)
|
||||||
|
for column in source.schema
|
||||||
|
),
|
||||||
|
schema_version=source.schema_version,
|
||||||
|
fingerprint=source.fingerprint,
|
||||||
|
row_count=source.row_count,
|
||||||
|
byte_count=source.byte_count,
|
||||||
|
updated_at=source.updated_at,
|
||||||
|
capabilities=source.capabilities,
|
||||||
|
metadata=dict(source.metadata),
|
||||||
|
source_mode=source.source_mode,
|
||||||
|
pushdown=source.pushdown,
|
||||||
|
health=source.health,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _datasource_error(exc: TabularSourceError):
|
||||||
|
if isinstance(exc, TabularSourceAccessError):
|
||||||
|
return DatasourceAccessError(str(exc))
|
||||||
|
if isinstance(exc, TabularSourceNotFoundError):
|
||||||
|
return DatasourceNotFoundError(str(exc))
|
||||||
|
if isinstance(exc, TabularSourceUnavailableError):
|
||||||
|
return DatasourceUnavailableError(str(exc))
|
||||||
|
if isinstance(exc, TabularSourceValidationError):
|
||||||
|
return DatasourceValidationError(str(exc))
|
||||||
|
return DatasourceValidationError(str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["ConnectorDatasourceOriginProvider"]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
|
||||||
|
__all__ = ["ConnectorTabularSource"]
|
||||||
@@ -0,0 +1,831 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
JSON,
|
||||||
|
LargeBinary,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from govoplan_core.db.base import Base, TimestampMixin
|
||||||
|
|
||||||
|
|
||||||
|
def new_uuid() -> str:
|
||||||
|
return str(uuid.uuid4())
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorTabularSource(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_tabular_sources"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("tenant_id", "source_name", name="uq_connector_tabular_source_name"),
|
||||||
|
Index("ix_connector_tabular_sources_tenant_status", "tenant_id", "status"),
|
||||||
|
Index("ix_connector_tabular_sources_tenant_updated", "tenant_id", "updated_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
provider: Mapped[str] = mapped_column(String(50), default="snapshot", nullable=False, index=True)
|
||||||
|
source_name: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
|
status: Mapped[str] = mapped_column(String(30), default="active", nullable=False, index=True)
|
||||||
|
schema_version: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
schema_: Mapped[list[dict[str, Any]]] = mapped_column("schema", JSON, default=list, nullable=False)
|
||||||
|
rows: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
|
row_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
byte_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSanctionsAcquisitionRun(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_sanctions_acquisition_runs"
|
||||||
|
__table_args__ = (
|
||||||
|
Index(
|
||||||
|
"ix_connector_sanctions_run_health",
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"status",
|
||||||
|
"started_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
primary_key=True,
|
||||||
|
default=new_uuid,
|
||||||
|
)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
provider_id: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_id: Mapped[str] = mapped_column(
|
||||||
|
String(200),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(40),
|
||||||
|
default="running",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
attempt_count: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
default=0,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
request_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
response_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
snapshot_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
error: Mapped[str | None] = mapped_column(
|
||||||
|
Text,
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(
|
||||||
|
String(255),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSanctionsSnapshot(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_sanctions_snapshots"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"connector_run_id",
|
||||||
|
name="uq_connector_sanctions_snapshot_run",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_sanctions_snapshot_source",
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"acquired_at",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_sanctions_snapshot_version",
|
||||||
|
"provider_id",
|
||||||
|
"source_id",
|
||||||
|
"source_version",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
primary_key=True,
|
||||||
|
default=new_uuid,
|
||||||
|
)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(
|
||||||
|
String(36),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
provider_id: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
publisher: Mapped[str] = mapped_column(
|
||||||
|
String(300),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
jurisdiction: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
list_type: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_id: Mapped[str] = mapped_column(
|
||||||
|
String(200),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_version: Mapped[str] = mapped_column(
|
||||||
|
String(255),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
publication_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
effective_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
acquired_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
source_url: Mapped[str | None] = mapped_column(
|
||||||
|
String(1500),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
content_type: Mapped[str] = mapped_column(
|
||||||
|
String(200),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
byte_count: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
sha256: Mapped[str] = mapped_column(
|
||||||
|
String(64),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
signature_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
parser_version: Mapped[str] = mapped_column(
|
||||||
|
String(100),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
licence_notes: Mapped[str | None] = mapped_column(
|
||||||
|
Text,
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
trust_notes: Mapped[str | None] = mapped_column(
|
||||||
|
Text,
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
connector_run_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey(
|
||||||
|
"connector_sanctions_acquisition_runs.id",
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
transport_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
raw_content: Mapped[bytes] = mapped_column(
|
||||||
|
LargeBinary,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinition(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_definitions"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"definition_key",
|
||||||
|
name="uq_connector_definition_tenant_key",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_definitions_tenant_status",
|
||||||
|
"tenant_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
definition_key: Mapped[str] = mapped_column(String(160), nullable=False)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="active",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
current_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
source_package: Mapped[str | None] = mapped_column(String(300))
|
||||||
|
local_definition: Mapped[bool] = mapped_column(
|
||||||
|
Boolean,
|
||||||
|
default=False,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionRevision(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_definition_revisions"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"definition_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_connector_definition_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
definition_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_definitions.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
specification: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
definition_hash: Mapped[str] = mapped_column(
|
||||||
|
String(64),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
origin: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||||
|
package_ref: Mapped[str | None] = mapped_column(String(300))
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfiguration(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_configurations"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"name",
|
||||||
|
name="uq_connector_configuration_tenant_name",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_configurations_tenant_status",
|
||||||
|
"tenant_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
definition_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_definitions.id", ondelete="RESTRICT"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="draft",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
endpoint_url: Mapped[str | None] = mapped_column(String(1500))
|
||||||
|
credential_ref: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
base_definition_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
local_overrides: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
protected_paths: Mapped[list[str]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=list,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
effective_configuration: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
effective_hash: Mapped[str] = mapped_column(
|
||||||
|
String(64),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
resource_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
default=1,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
ambiguity_policy: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="manual_review",
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSimulationRun(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_simulation_runs"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
"mode",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_simulation_run_idempotency",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_simulation_runs_review",
|
||||||
|
"tenant_id",
|
||||||
|
"review_state",
|
||||||
|
"created_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
configuration_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_configurations.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
mode: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
review_state: Mapped[str] = mapped_column(
|
||||||
|
String(30),
|
||||||
|
default="not_required",
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
definition_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
configuration_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
configuration_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
input_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
summary: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
effects: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=list,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=list,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON,
|
||||||
|
default=dict,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
reviewed_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
review_reason: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorKnowledgeProfile(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_knowledge_profiles"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
name="uq_connector_knowledge_profile_configuration",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_knowledge_profiles_tenant_status",
|
||||||
|
"tenant_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
configuration_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_configurations.id", ondelete="RESTRICT"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="active", nullable=False, index=True
|
||||||
|
)
|
||||||
|
product: Mapped[str] = mapped_column(
|
||||||
|
String(50), default="unknown", nullable=False, index=True
|
||||||
|
)
|
||||||
|
product_version: Mapped[str | None] = mapped_column(String(100))
|
||||||
|
desired_maturity: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="read", nullable=False
|
||||||
|
)
|
||||||
|
discovered_maturity: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="discover", nullable=False
|
||||||
|
)
|
||||||
|
source_authority_mode: Mapped[str] = mapped_column(
|
||||||
|
String(40), default="external_mirror", nullable=False
|
||||||
|
)
|
||||||
|
default_visibility: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="restricted", nullable=False
|
||||||
|
)
|
||||||
|
default_acl_tokens: Mapped[list[str]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
namespace_mappings: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
capabilities: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
discovery_revision: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
discovery_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
health_status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="unknown", nullable=False, index=True
|
||||||
|
)
|
||||||
|
health_details: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
discovered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_sync_cursor: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
last_high_watermark: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorKnowledgeObject(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_knowledge_objects"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"profile_id",
|
||||||
|
"object_type",
|
||||||
|
"external_id",
|
||||||
|
name="uq_connector_knowledge_object_identity",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_knowledge_objects_tenant_profile_status",
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_knowledge_objects_tenant_updated",
|
||||||
|
"tenant_id",
|
||||||
|
"source_updated_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
profile_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_knowledge_profiles.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
object_type: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||||
|
external_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
external_page_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
external_revision_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
namespace_id: Mapped[int | None] = mapped_column(Integer, index=True)
|
||||||
|
title: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
|
canonical_url: Mapped[str | None] = mapped_column(String(1500))
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="active", nullable=False, index=True
|
||||||
|
)
|
||||||
|
redirect_target_external_id: Mapped[str | None] = mapped_column(String(255))
|
||||||
|
source_revision: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
|
visibility: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="restricted", nullable=False
|
||||||
|
)
|
||||||
|
acl_tokens: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
mapped_data: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
change_cursor: Mapped[str | None] = mapped_column(String(500), index=True)
|
||||||
|
source_updated_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), index=True
|
||||||
|
)
|
||||||
|
observed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True
|
||||||
|
)
|
||||||
|
resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorKnowledgeSyncRun(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_knowledge_sync_runs"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"mode",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_knowledge_sync_run_idempotency",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_knowledge_sync_runs_profile_started",
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"started_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
profile_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_knowledge_profiles.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
mode: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||||
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
cursor_before: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
cursor_after: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
high_watermark: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
counts: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||||
|
effects: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorServiceDeskProfile(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_service_desk_profiles"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
name="uq_connector_service_desk_profile_configuration",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_service_desk_profiles_tenant_status",
|
||||||
|
"tenant_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
configuration_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_configurations.id", ondelete="RESTRICT"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="active", nullable=False, index=True
|
||||||
|
)
|
||||||
|
integration_mode: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="synchronize", nullable=False, index=True
|
||||||
|
)
|
||||||
|
product: Mapped[str] = mapped_column(
|
||||||
|
String(50), default="unknown", nullable=False, index=True
|
||||||
|
)
|
||||||
|
product_version: Mapped[str | None] = mapped_column(String(100))
|
||||||
|
desired_maturity: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="synchronize", nullable=False
|
||||||
|
)
|
||||||
|
discovered_maturity: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="discover", nullable=False
|
||||||
|
)
|
||||||
|
source_authority_mode: Mapped[str] = mapped_column(
|
||||||
|
String(40), default="external_authoritative", nullable=False
|
||||||
|
)
|
||||||
|
default_visibility: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="restricted", nullable=False
|
||||||
|
)
|
||||||
|
default_acl_tokens: Mapped[list[str]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
routes: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||||
|
queue_mappings: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
dynamic_field_mappings: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
capabilities: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
discovery_revision: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
discovered_configuration_revision: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
discovered_configuration_hash: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
discovery_evidence: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
health_status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="unknown", nullable=False, index=True
|
||||||
|
)
|
||||||
|
health_details: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
discovered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_sync_cursor: Mapped[str | None] = mapped_column(String(4000))
|
||||||
|
last_high_watermark: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorServiceDeskObject(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_service_desk_objects"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"profile_id",
|
||||||
|
"object_type",
|
||||||
|
"external_id",
|
||||||
|
name="uq_connector_service_desk_object_identity",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_service_desk_objects_tenant_profile_status",
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"status",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_service_desk_objects_tenant_updated",
|
||||||
|
"tenant_id",
|
||||||
|
"source_updated_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
profile_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_service_desk_profiles.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
object_type: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||||
|
external_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
external_ticket_number: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
title: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
|
canonical_url: Mapped[str | None] = mapped_column(String(1500))
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="active", nullable=False, index=True
|
||||||
|
)
|
||||||
|
source_revision: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
|
visibility: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="restricted", nullable=False
|
||||||
|
)
|
||||||
|
acl_tokens: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||||
|
mapped_data: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
change_cursor: Mapped[str | None] = mapped_column(String(4000), index=True)
|
||||||
|
source_updated_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), index=True
|
||||||
|
)
|
||||||
|
observed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True
|
||||||
|
)
|
||||||
|
resource_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorServiceDeskSyncRun(Base, TimestampMixin):
|
||||||
|
__tablename__ = "connector_service_desk_sync_runs"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_service_desk_sync_run_idempotency",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_connector_service_desk_runs_profile_started",
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"started_at",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
profile_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("connector_service_desk_profiles.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
mode: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||||
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
cursor_before: Mapped[str | None] = mapped_column(String(4000))
|
||||||
|
cursor_after: Mapped[str | None] = mapped_column(String(4000))
|
||||||
|
high_watermark: Mapped[str | None] = mapped_column(String(500))
|
||||||
|
counts: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||||
|
effects: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
provenance: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConnectorConfiguration",
|
||||||
|
"ConnectorDefinition",
|
||||||
|
"ConnectorDefinitionRevision",
|
||||||
|
"ConnectorKnowledgeObject",
|
||||||
|
"ConnectorKnowledgeProfile",
|
||||||
|
"ConnectorKnowledgeSyncRun",
|
||||||
|
"ConnectorServiceDeskObject",
|
||||||
|
"ConnectorServiceDeskProfile",
|
||||||
|
"ConnectorServiceDeskSyncRun",
|
||||||
|
"ConnectorSanctionsAcquisitionRun",
|
||||||
|
"ConnectorSanctionsSnapshot",
|
||||||
|
"ConnectorSimulationRun",
|
||||||
|
"ConnectorTabularSource",
|
||||||
|
"new_uuid",
|
||||||
|
]
|
||||||
@@ -0,0 +1,680 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import (
|
||||||
|
DsarErasureActionRef,
|
||||||
|
DsarExecutionResultRef,
|
||||||
|
DsarRecordRef,
|
||||||
|
DsarSubjectRef,
|
||||||
|
dsar_capability_name,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeSyncRun,
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskSyncRun,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
CONNECTORS_DSAR_CAPABILITY = dsar_capability_name("connectors")
|
||||||
|
_MAX_RECORDS = 5_000
|
||||||
|
_CONFLICT = object()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _SubjectSelectors:
|
||||||
|
account_id: str
|
||||||
|
source_id: str | None
|
||||||
|
acquisition_id: str | None
|
||||||
|
definition_id: str | None
|
||||||
|
configuration_id: str | None
|
||||||
|
simulation_id: str | None
|
||||||
|
knowledge_profile_id: str | None
|
||||||
|
knowledge_run_id: str | None
|
||||||
|
service_desk_profile_id: str | None
|
||||||
|
service_desk_run_id: str | None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def narrowed(self) -> bool:
|
||||||
|
return any(
|
||||||
|
(
|
||||||
|
self.source_id,
|
||||||
|
self.acquisition_id,
|
||||||
|
self.definition_id,
|
||||||
|
self.configuration_id,
|
||||||
|
self.simulation_id,
|
||||||
|
self.knowledge_profile_id,
|
||||||
|
self.knowledge_run_id,
|
||||||
|
self.service_desk_profile_id,
|
||||||
|
self.service_desk_run_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsDsarProvider:
|
||||||
|
provider_id = "connectors"
|
||||||
|
module_id = "connectors"
|
||||||
|
|
||||||
|
def search_subject(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
) -> Sequence[DsarRecordRef]:
|
||||||
|
db = _session(session)
|
||||||
|
selectors = _subject_selectors(subject)
|
||||||
|
if selectors is None:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
records: list[DsarRecordRef] = []
|
||||||
|
if not selectors.narrowed or selectors.source_id:
|
||||||
|
query = db.query(ConnectorTabularSource).filter(
|
||||||
|
ConnectorTabularSource.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
ConnectorTabularSource.created_by == selectors.account_id,
|
||||||
|
ConnectorTabularSource.updated_by == selectors.account_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if selectors.source_id:
|
||||||
|
query = query.filter(ConnectorTabularSource.id == selectors.source_id)
|
||||||
|
records.extend(
|
||||||
|
_source_attribution(row, selectors.account_id)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorTabularSource.created_at,
|
||||||
|
ConnectorTabularSource.id,
|
||||||
|
label="source attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.acquisition_id:
|
||||||
|
query = db.query(ConnectorSanctionsAcquisitionRun).filter(
|
||||||
|
ConnectorSanctionsAcquisitionRun.tenant_id == tenant_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.created_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.acquisition_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorSanctionsAcquisitionRun.id == selectors.acquisition_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_acquisition_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorSanctionsAcquisitionRun.started_at,
|
||||||
|
ConnectorSanctionsAcquisitionRun.id,
|
||||||
|
label="acquisition attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.definition_id:
|
||||||
|
query = (
|
||||||
|
db.query(ConnectorDefinitionRevision, ConnectorDefinition)
|
||||||
|
.join(
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinition.id == ConnectorDefinitionRevision.definition_id,
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.tenant_id == tenant_id,
|
||||||
|
ConnectorDefinitionRevision.created_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if selectors.definition_id:
|
||||||
|
query = query.filter(ConnectorDefinition.id == selectors.definition_id)
|
||||||
|
records.extend(
|
||||||
|
_definition_attribution(revision, definition)
|
||||||
|
for revision, definition in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorDefinitionRevision.created_at,
|
||||||
|
ConnectorDefinitionRevision.id,
|
||||||
|
label="definition attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.configuration_id:
|
||||||
|
query = db.query(ConnectorConfiguration).filter(
|
||||||
|
ConnectorConfiguration.tenant_id == tenant_id,
|
||||||
|
ConnectorConfiguration.updated_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.configuration_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorConfiguration.id == selectors.configuration_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_configuration_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorConfiguration.updated_at,
|
||||||
|
ConnectorConfiguration.id,
|
||||||
|
label="configuration attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.simulation_id:
|
||||||
|
query = db.query(ConnectorSimulationRun).filter(
|
||||||
|
ConnectorSimulationRun.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
ConnectorSimulationRun.created_by == selectors.account_id,
|
||||||
|
ConnectorSimulationRun.reviewed_by == selectors.account_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if selectors.simulation_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorSimulationRun.id == selectors.simulation_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_simulation_attribution(row, selectors.account_id)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorSimulationRun.created_at,
|
||||||
|
ConnectorSimulationRun.id,
|
||||||
|
label="simulation attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.knowledge_profile_id:
|
||||||
|
query = db.query(ConnectorKnowledgeProfile).filter(
|
||||||
|
ConnectorKnowledgeProfile.tenant_id == tenant_id,
|
||||||
|
ConnectorKnowledgeProfile.updated_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.knowledge_profile_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorKnowledgeProfile.id == selectors.knowledge_profile_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_knowledge_profile_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorKnowledgeProfile.created_at,
|
||||||
|
ConnectorKnowledgeProfile.id,
|
||||||
|
label="knowledge profile attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.knowledge_run_id:
|
||||||
|
query = db.query(ConnectorKnowledgeSyncRun).filter(
|
||||||
|
ConnectorKnowledgeSyncRun.tenant_id == tenant_id,
|
||||||
|
ConnectorKnowledgeSyncRun.created_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.knowledge_run_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorKnowledgeSyncRun.id == selectors.knowledge_run_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_knowledge_run_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorKnowledgeSyncRun.started_at,
|
||||||
|
ConnectorKnowledgeSyncRun.id,
|
||||||
|
label="knowledge run attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.service_desk_profile_id:
|
||||||
|
query = db.query(ConnectorServiceDeskProfile).filter(
|
||||||
|
ConnectorServiceDeskProfile.tenant_id == tenant_id,
|
||||||
|
ConnectorServiceDeskProfile.updated_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.service_desk_profile_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorServiceDeskProfile.id
|
||||||
|
== selectors.service_desk_profile_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_service_desk_profile_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorServiceDeskProfile.created_at,
|
||||||
|
ConnectorServiceDeskProfile.id,
|
||||||
|
label="service-desk profile attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not selectors.narrowed or selectors.service_desk_run_id:
|
||||||
|
query = db.query(ConnectorServiceDeskSyncRun).filter(
|
||||||
|
ConnectorServiceDeskSyncRun.tenant_id == tenant_id,
|
||||||
|
ConnectorServiceDeskSyncRun.created_by == selectors.account_id,
|
||||||
|
)
|
||||||
|
if selectors.service_desk_run_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorServiceDeskSyncRun.id == selectors.service_desk_run_id
|
||||||
|
)
|
||||||
|
records.extend(
|
||||||
|
_service_desk_run_attribution(row)
|
||||||
|
for row in _limited(
|
||||||
|
query,
|
||||||
|
ConnectorServiceDeskSyncRun.started_at,
|
||||||
|
ConnectorServiceDeskSyncRun.id,
|
||||||
|
label="service-desk run attribution",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(records) > _MAX_RECORDS:
|
||||||
|
raise ValueError("Connectors DSAR result limit exceeded; narrow selectors.")
|
||||||
|
return tuple(
|
||||||
|
sorted(records, key=lambda item: (item.resource_type, item.resource_id))
|
||||||
|
)
|
||||||
|
|
||||||
|
def plan_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
records: Sequence[DsarRecordRef],
|
||||||
|
) -> Sequence[DsarErasureActionRef]:
|
||||||
|
del tenant_id
|
||||||
|
_session(session)
|
||||||
|
if _subject_selectors(subject) is None:
|
||||||
|
raise ValueError("Connectors DSAR subject selectors conflict.")
|
||||||
|
actions: list[DsarErasureActionRef] = []
|
||||||
|
for record in records:
|
||||||
|
_validate_record(record)
|
||||||
|
actions.append(
|
||||||
|
DsarErasureActionRef(
|
||||||
|
action_id=(
|
||||||
|
f"connectors:retain:{record.resource_type}:{record.resource_id}"
|
||||||
|
),
|
||||||
|
provider_id=self.provider_id,
|
||||||
|
module_id=self.module_id,
|
||||||
|
kind="retain",
|
||||||
|
resource_type=record.resource_type,
|
||||||
|
resource_id=record.resource_id,
|
||||||
|
title=f"Retain {record.title}",
|
||||||
|
rationale=(
|
||||||
|
record.retention_reason
|
||||||
|
or "Connector operator attribution remains governance evidence."
|
||||||
|
),
|
||||||
|
executable=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(actions)
|
||||||
|
|
||||||
|
def execute_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
actions: Sequence[DsarErasureActionRef],
|
||||||
|
request_id: str,
|
||||||
|
) -> Sequence[DsarExecutionResultRef]:
|
||||||
|
del tenant_id
|
||||||
|
_session(session)
|
||||||
|
if _subject_selectors(subject) is None:
|
||||||
|
raise ValueError("Connectors DSAR subject selectors conflict.")
|
||||||
|
results: list[DsarExecutionResultRef] = []
|
||||||
|
for action in actions:
|
||||||
|
_validate_action(action)
|
||||||
|
if action.executable or action.kind != "retain":
|
||||||
|
raise ValueError("Connectors DSAR publishes retain actions only.")
|
||||||
|
results.append(
|
||||||
|
DsarExecutionResultRef(
|
||||||
|
action_id=action.action_id,
|
||||||
|
status="blocked",
|
||||||
|
summary="Connector operator attribution remains governance evidence.",
|
||||||
|
evidence={"request_id": request_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
|
||||||
|
def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None:
|
||||||
|
references = subject.external_references
|
||||||
|
account = _coalesce(
|
||||||
|
subject.account_id,
|
||||||
|
references.get("connectors.account"),
|
||||||
|
references.get("access.account"),
|
||||||
|
)
|
||||||
|
values = {
|
||||||
|
"source_id": _coalesce(
|
||||||
|
references.get("connectors.source"),
|
||||||
|
references.get("connectors.source_id"),
|
||||||
|
),
|
||||||
|
"acquisition_id": _coalesce(
|
||||||
|
references.get("connectors.acquisition"),
|
||||||
|
references.get("connectors.acquisition_id"),
|
||||||
|
),
|
||||||
|
"definition_id": _coalesce(
|
||||||
|
references.get("connectors.definition"),
|
||||||
|
references.get("connectors.definition_id"),
|
||||||
|
),
|
||||||
|
"configuration_id": _coalesce(
|
||||||
|
references.get("connectors.configuration"),
|
||||||
|
references.get("connectors.configuration_id"),
|
||||||
|
),
|
||||||
|
"simulation_id": _coalesce(
|
||||||
|
references.get("connectors.simulation"),
|
||||||
|
references.get("connectors.simulation_id"),
|
||||||
|
),
|
||||||
|
"knowledge_profile_id": _coalesce(
|
||||||
|
references.get("connectors.knowledge_profile"),
|
||||||
|
references.get("connectors.knowledge_profile_id"),
|
||||||
|
),
|
||||||
|
"knowledge_run_id": _coalesce(
|
||||||
|
references.get("connectors.knowledge_run"),
|
||||||
|
references.get("connectors.knowledge_run_id"),
|
||||||
|
),
|
||||||
|
"service_desk_profile_id": _coalesce(
|
||||||
|
references.get("connectors.service_desk_profile"),
|
||||||
|
references.get("connectors.service_desk_profile_id"),
|
||||||
|
),
|
||||||
|
"service_desk_run_id": _coalesce(
|
||||||
|
references.get("connectors.service_desk_run"),
|
||||||
|
references.get("connectors.service_desk_run_id"),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if account is _CONFLICT or any(value is _CONFLICT for value in values.values()):
|
||||||
|
return None
|
||||||
|
account_id = _optional_string(account)
|
||||||
|
if not account_id:
|
||||||
|
return None
|
||||||
|
return _SubjectSelectors(
|
||||||
|
account_id=account_id,
|
||||||
|
source_id=_optional_string(values["source_id"]),
|
||||||
|
acquisition_id=_optional_string(values["acquisition_id"]),
|
||||||
|
definition_id=_optional_string(values["definition_id"]),
|
||||||
|
configuration_id=_optional_string(values["configuration_id"]),
|
||||||
|
simulation_id=_optional_string(values["simulation_id"]),
|
||||||
|
knowledge_profile_id=_optional_string(values["knowledge_profile_id"]),
|
||||||
|
knowledge_run_id=_optional_string(values["knowledge_run_id"]),
|
||||||
|
service_desk_profile_id=_optional_string(values["service_desk_profile_id"]),
|
||||||
|
service_desk_run_id=_optional_string(values["service_desk_run_id"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_attribution(row: ConnectorTabularSource, account_id: str) -> DsarRecordRef:
|
||||||
|
activities = []
|
||||||
|
if row.created_by == account_id:
|
||||||
|
activities.append("created_source_snapshot")
|
||||||
|
if row.updated_by == account_id:
|
||||||
|
activities.append("updated_source_snapshot")
|
||||||
|
return _record(
|
||||||
|
resource_type="source_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector source actor attribution",
|
||||||
|
data={
|
||||||
|
"source_id": row.id,
|
||||||
|
"provider": row.provider,
|
||||||
|
"status": row.status,
|
||||||
|
"schema_version": row.schema_version,
|
||||||
|
"row_count": row.row_count,
|
||||||
|
"activities": activities,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
"retired_at": _iso(row.deleted_at),
|
||||||
|
},
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _acquisition_attribution(row: ConnectorSanctionsAcquisitionRun) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="acquisition_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector acquisition actor attribution",
|
||||||
|
data={
|
||||||
|
"acquisition_id": row.id,
|
||||||
|
"provider_id": row.provider_id,
|
||||||
|
"source_id": row.source_id,
|
||||||
|
"status": row.status,
|
||||||
|
"attempt_count": row.attempt_count,
|
||||||
|
"started_at": _iso(row.started_at),
|
||||||
|
"finished_at": _iso(row.finished_at),
|
||||||
|
"activity": "started_source_acquisition",
|
||||||
|
},
|
||||||
|
observed_at=row.finished_at or row.started_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _definition_attribution(
|
||||||
|
row: ConnectorDefinitionRevision, definition: ConnectorDefinition
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="definition_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector definition actor attribution",
|
||||||
|
data={
|
||||||
|
"definition_id": definition.id,
|
||||||
|
"revision_id": row.id,
|
||||||
|
"revision": row.revision,
|
||||||
|
"origin": row.origin,
|
||||||
|
"activity": "created_definition_revision",
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
},
|
||||||
|
observed_at=row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_attribution(row: ConnectorConfiguration) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="configuration_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector configuration actor attribution",
|
||||||
|
data={
|
||||||
|
"configuration_id": row.id,
|
||||||
|
"definition_id": row.definition_id,
|
||||||
|
"status": row.status,
|
||||||
|
"base_definition_revision": row.base_definition_revision,
|
||||||
|
"resource_revision": row.resource_revision,
|
||||||
|
"ambiguity_policy": row.ambiguity_policy,
|
||||||
|
"activity": "updated_connector_configuration",
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
},
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _simulation_attribution(
|
||||||
|
row: ConnectorSimulationRun, account_id: str
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
activities = []
|
||||||
|
if row.created_by == account_id:
|
||||||
|
activities.append("created_simulation")
|
||||||
|
if row.reviewed_by == account_id:
|
||||||
|
activities.append("reviewed_simulation")
|
||||||
|
return _record(
|
||||||
|
resource_type="simulation_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="Connector simulation actor attribution",
|
||||||
|
data={
|
||||||
|
"simulation_id": row.id,
|
||||||
|
"configuration_id": row.configuration_id,
|
||||||
|
"mode": row.mode,
|
||||||
|
"status": row.status,
|
||||||
|
"review_state": row.review_state,
|
||||||
|
"definition_revision": row.definition_revision,
|
||||||
|
"configuration_revision": row.configuration_revision,
|
||||||
|
"activities": activities,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"reviewed_at": _iso(row.reviewed_at),
|
||||||
|
},
|
||||||
|
observed_at=row.reviewed_at or row.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _knowledge_profile_attribution(
|
||||||
|
row: ConnectorKnowledgeProfile,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="knowledge_profile_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="External knowledge profile actor attribution",
|
||||||
|
data={
|
||||||
|
"knowledge_profile_id": row.id,
|
||||||
|
"configuration_id": row.configuration_id,
|
||||||
|
"status": row.status,
|
||||||
|
"desired_maturity": row.desired_maturity,
|
||||||
|
"source_authority_mode": row.source_authority_mode,
|
||||||
|
"resource_revision": row.resource_revision,
|
||||||
|
"activity": "updated_external_knowledge_profile",
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
},
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _knowledge_run_attribution(
|
||||||
|
row: ConnectorKnowledgeSyncRun,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="knowledge_run_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="External knowledge operation actor attribution",
|
||||||
|
data={
|
||||||
|
"knowledge_run_id": row.id,
|
||||||
|
"knowledge_profile_id": row.profile_id,
|
||||||
|
"mode": row.mode,
|
||||||
|
"status": row.status,
|
||||||
|
"started_at": _iso(row.started_at),
|
||||||
|
"finished_at": _iso(row.finished_at),
|
||||||
|
"activity": "started_external_knowledge_operation",
|
||||||
|
},
|
||||||
|
observed_at=row.finished_at or row.started_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _service_desk_profile_attribution(
|
||||||
|
row: ConnectorServiceDeskProfile,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="service_desk_profile_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="External service-desk profile actor attribution",
|
||||||
|
data={
|
||||||
|
"service_desk_profile_id": row.id,
|
||||||
|
"configuration_id": row.configuration_id,
|
||||||
|
"status": row.status,
|
||||||
|
"integration_mode": row.integration_mode,
|
||||||
|
"desired_maturity": row.desired_maturity,
|
||||||
|
"source_authority_mode": row.source_authority_mode,
|
||||||
|
"resource_revision": row.resource_revision,
|
||||||
|
"activity": "updated_external_service_desk_profile",
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
},
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _service_desk_run_attribution(
|
||||||
|
row: ConnectorServiceDeskSyncRun,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return _record(
|
||||||
|
resource_type="service_desk_run_actor_attribution",
|
||||||
|
resource_id=row.id,
|
||||||
|
title="External service-desk operation actor attribution",
|
||||||
|
data={
|
||||||
|
"service_desk_run_id": row.id,
|
||||||
|
"service_desk_profile_id": row.profile_id,
|
||||||
|
"mode": row.mode,
|
||||||
|
"status": row.status,
|
||||||
|
"started_at": _iso(row.started_at),
|
||||||
|
"finished_at": _iso(row.finished_at),
|
||||||
|
"activity": "started_external_service_desk_operation",
|
||||||
|
},
|
||||||
|
observed_at=row.finished_at or row.started_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _record(
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
title: str,
|
||||||
|
data: dict[str, object],
|
||||||
|
observed_at: datetime | None,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return DsarRecordRef(
|
||||||
|
provider_id="connectors",
|
||||||
|
module_id="connectors",
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
category="connector_governance_attribution",
|
||||||
|
title=title,
|
||||||
|
data=data,
|
||||||
|
observed_at=_aware(observed_at),
|
||||||
|
immutable_evidence=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Connector attribution is retained for configuration, review, and "
|
||||||
|
"external-operation accountability."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _limited(query, first, second, *, label: str):
|
||||||
|
rows = query.order_by(first, second).limit(_MAX_RECORDS + 1).all()
|
||||||
|
if len(rows) > _MAX_RECORDS:
|
||||||
|
raise ValueError(f"Connectors DSAR {label} limit exceeded; narrow selectors.")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _coalesce(*values: str | None) -> str | None | object:
|
||||||
|
normalized = {str(value).strip() for value in values if str(value or "").strip()}
|
||||||
|
if len(normalized) > 1:
|
||||||
|
return _CONFLICT
|
||||||
|
return next(iter(normalized), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_string(value: object) -> str | None:
|
||||||
|
return value if isinstance(value, str) and value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
aware = _aware(value)
|
||||||
|
return aware.isoformat() if aware else None
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None or value.tzinfo is not None:
|
||||||
|
return value
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Connectors DSAR requires a SQLAlchemy Session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
_RESOURCE_TYPES = {
|
||||||
|
"source_actor_attribution",
|
||||||
|
"acquisition_actor_attribution",
|
||||||
|
"definition_actor_attribution",
|
||||||
|
"configuration_actor_attribution",
|
||||||
|
"simulation_actor_attribution",
|
||||||
|
"knowledge_profile_actor_attribution",
|
||||||
|
"knowledge_run_actor_attribution",
|
||||||
|
"service_desk_profile_actor_attribution",
|
||||||
|
"service_desk_run_actor_attribution",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_record(record: DsarRecordRef) -> None:
|
||||||
|
if record.provider_id != "connectors" or record.module_id != "connectors":
|
||||||
|
raise ValueError("Connectors DSAR cannot plan a foreign provider record.")
|
||||||
|
if record.resource_type not in _RESOURCE_TYPES or not record.resource_id:
|
||||||
|
raise ValueError("Connectors DSAR record identity is invalid.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_action(action: DsarErasureActionRef) -> None:
|
||||||
|
if action.provider_id != "connectors" or action.module_id != "connectors":
|
||||||
|
raise ValueError("Connectors DSAR cannot execute a foreign provider action.")
|
||||||
|
if not action.action_id.startswith("connectors:retain:"):
|
||||||
|
raise ValueError("Connectors DSAR action identity is invalid.")
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["CONNECTORS_DSAR_CAPABILITY", "ConnectorsDsarProvider"]
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import re
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import replace
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from email.utils import format_datetime, parsedate_to_datetime
|
||||||
|
|
||||||
|
from defusedxml import ElementTree as SafeET
|
||||||
|
from defusedxml.common import DefusedXmlException
|
||||||
|
|
||||||
|
from govoplan_core.core.feeds import (
|
||||||
|
FeedCapabilityError,
|
||||||
|
FeedDocument,
|
||||||
|
FeedEntry,
|
||||||
|
FeedProvider,
|
||||||
|
FeedRenderRequest,
|
||||||
|
FeedRenderResult,
|
||||||
|
)
|
||||||
|
from govoplan_core.security.http_fetch import fetch_http
|
||||||
|
|
||||||
|
|
||||||
|
MAX_FEED_BYTES = 5_000_000
|
||||||
|
ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||||
|
FEED_PUBLISH_SCOPE = "connectors:feeds:publish"
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE = "connectors:feeds:publish_private"
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorFeedProvider(FeedProvider):
|
||||||
|
def fetch(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
timeout: float = 15,
|
||||||
|
max_entries: int = 2_000,
|
||||||
|
) -> FeedDocument:
|
||||||
|
try:
|
||||||
|
response = fetch_http(
|
||||||
|
url,
|
||||||
|
timeout=timeout,
|
||||||
|
label="RSS/Atom feed URL",
|
||||||
|
headers={
|
||||||
|
"Accept": (
|
||||||
|
"application/atom+xml, application/rss+xml, "
|
||||||
|
"application/xml;q=0.9, text/xml;q=0.8"
|
||||||
|
)
|
||||||
|
},
|
||||||
|
max_bytes=MAX_FEED_BYTES,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise FeedCapabilityError(f"Feed acquisition failed: {exc}") from exc
|
||||||
|
if response.status < 200 or response.status >= 300:
|
||||||
|
raise FeedCapabilityError(
|
||||||
|
f"Feed acquisition returned HTTP {response.status}."
|
||||||
|
)
|
||||||
|
content_type = _header(response.headers, "content-type")
|
||||||
|
document = self.parse(
|
||||||
|
response.body,
|
||||||
|
source_url=url,
|
||||||
|
content_type=content_type,
|
||||||
|
max_entries=max_entries,
|
||||||
|
)
|
||||||
|
acquired_at = datetime.now(timezone.utc)
|
||||||
|
return replace(
|
||||||
|
document,
|
||||||
|
acquired_at=acquired_at,
|
||||||
|
fresh_until=_fresh_until(response.headers, acquired_at),
|
||||||
|
etag=_header(response.headers, "etag"),
|
||||||
|
last_modified=_header(response.headers, "last-modified"),
|
||||||
|
metadata={
|
||||||
|
**dict(document.metadata),
|
||||||
|
"http_status": response.status,
|
||||||
|
"byte_count": len(response.body),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def parse(
|
||||||
|
self,
|
||||||
|
content: bytes,
|
||||||
|
*,
|
||||||
|
source_url: str,
|
||||||
|
content_type: str | None = None,
|
||||||
|
max_entries: int = 2_000,
|
||||||
|
) -> FeedDocument:
|
||||||
|
if not content:
|
||||||
|
raise FeedCapabilityError("Feed content is empty.")
|
||||||
|
if len(content) > MAX_FEED_BYTES:
|
||||||
|
raise FeedCapabilityError(
|
||||||
|
f"Feeds are limited to {MAX_FEED_BYTES // 1_000_000} MB."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
root = SafeET.fromstring(content)
|
||||||
|
except (ET.ParseError, DefusedXmlException) as exc:
|
||||||
|
raise FeedCapabilityError(f"Feed XML is not safe or valid: {exc}") from exc
|
||||||
|
local_name = _local_name(root.tag)
|
||||||
|
if local_name == "rss":
|
||||||
|
document = _parse_rss(root, source_url=source_url, max_entries=max_entries)
|
||||||
|
elif local_name == "feed":
|
||||||
|
document = _parse_atom(root, source_url=source_url, max_entries=max_entries)
|
||||||
|
else:
|
||||||
|
raise FeedCapabilityError("The document is neither an RSS nor an Atom feed.")
|
||||||
|
return replace(
|
||||||
|
document,
|
||||||
|
content_type=content_type,
|
||||||
|
sha256=hashlib.sha256(content).hexdigest(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def render(self, request: FeedRenderRequest) -> FeedRenderResult:
|
||||||
|
if not request.title.strip() or not request.feed_url.strip():
|
||||||
|
raise FeedCapabilityError("Feed title and feed URL are required.")
|
||||||
|
entries = tuple(
|
||||||
|
entry
|
||||||
|
for entry in request.entries
|
||||||
|
if entry.visibility in request.allowed_visibilities
|
||||||
|
)
|
||||||
|
root = (
|
||||||
|
_render_rss(request, entries)
|
||||||
|
if request.format == "rss"
|
||||||
|
else _render_atom(request, entries)
|
||||||
|
)
|
||||||
|
body = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
||||||
|
return FeedRenderResult(
|
||||||
|
format=request.format,
|
||||||
|
content_type=(
|
||||||
|
"application/rss+xml; charset=utf-8"
|
||||||
|
if request.format == "rss"
|
||||||
|
else "application/atom+xml; charset=utf-8"
|
||||||
|
),
|
||||||
|
body=body,
|
||||||
|
included_entries=len(entries),
|
||||||
|
excluded_entries=len(request.entries) - len(entries),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def feed_rows(document: FeedDocument) -> tuple[Mapping[str, object], ...]:
|
||||||
|
"""Map feed entries to the connector tabular shape used by Datasources."""
|
||||||
|
|
||||||
|
return tuple(
|
||||||
|
{
|
||||||
|
"id": entry.id,
|
||||||
|
"title": entry.title,
|
||||||
|
"url": entry.url,
|
||||||
|
"summary": entry.summary,
|
||||||
|
"content": entry.content,
|
||||||
|
"author": entry.author,
|
||||||
|
"published_at": (
|
||||||
|
entry.published_at.isoformat() if entry.published_at else None
|
||||||
|
),
|
||||||
|
"updated_at": entry.updated_at.isoformat() if entry.updated_at else None,
|
||||||
|
"categories": list(entry.categories),
|
||||||
|
"enclosures": [dict(item) for item in entry.enclosures],
|
||||||
|
}
|
||||||
|
for entry in document.entries
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_rss(root: ET.Element, *, source_url: str, max_entries: int) -> FeedDocument:
|
||||||
|
channel = _first_child(root, "channel")
|
||||||
|
if channel is None:
|
||||||
|
raise FeedCapabilityError("RSS feed is missing its channel element.")
|
||||||
|
entries: list[FeedEntry] = []
|
||||||
|
for item in _children(channel, "item"):
|
||||||
|
if len(entries) >= max_entries:
|
||||||
|
raise FeedCapabilityError(f"Feeds are limited to {max_entries:,} entries.")
|
||||||
|
url = _text(item, "link")
|
||||||
|
identifier = _text(item, "guid") or url or _entry_fallback_id(item)
|
||||||
|
entries.append(
|
||||||
|
FeedEntry(
|
||||||
|
id=identifier,
|
||||||
|
title=_text(item, "title") or "(Untitled)",
|
||||||
|
url=url,
|
||||||
|
summary=_text(item, "description"),
|
||||||
|
content=_text(item, "encoded"),
|
||||||
|
author=_text(item, "author") or _text(item, "creator"),
|
||||||
|
published_at=_parse_date(_text(item, "pubDate")),
|
||||||
|
categories=tuple(
|
||||||
|
value for child in _children(item, "category")
|
||||||
|
if (value := (child.text or "").strip())
|
||||||
|
),
|
||||||
|
enclosures=tuple(
|
||||||
|
{
|
||||||
|
"url": child.attrib.get("url"),
|
||||||
|
"media_type": child.attrib.get("type"),
|
||||||
|
"size_bytes": _integer(child.attrib.get("length")),
|
||||||
|
}
|
||||||
|
for child in _children(item, "enclosure")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return FeedDocument(
|
||||||
|
format="rss",
|
||||||
|
title=_text(channel, "title") or "Untitled feed",
|
||||||
|
source_url=source_url,
|
||||||
|
entries=tuple(entries),
|
||||||
|
description=_text(channel, "description"),
|
||||||
|
home_url=_text(channel, "link"),
|
||||||
|
language=_text(channel, "language"),
|
||||||
|
updated_at=_parse_date(
|
||||||
|
_text(channel, "lastBuildDate") or _text(channel, "pubDate")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_atom(root: ET.Element, *, source_url: str, max_entries: int) -> FeedDocument:
|
||||||
|
entries: list[FeedEntry] = []
|
||||||
|
for item in _children(root, "entry"):
|
||||||
|
if len(entries) >= max_entries:
|
||||||
|
raise FeedCapabilityError(f"Feeds are limited to {max_entries:,} entries.")
|
||||||
|
alternate = _atom_link(item, "alternate")
|
||||||
|
identifier = _text(item, "id") or alternate or _entry_fallback_id(item)
|
||||||
|
author = _first_child(item, "author")
|
||||||
|
entries.append(
|
||||||
|
FeedEntry(
|
||||||
|
id=identifier,
|
||||||
|
title=_text(item, "title") or "(Untitled)",
|
||||||
|
url=alternate,
|
||||||
|
summary=_text(item, "summary"),
|
||||||
|
content=_text(item, "content"),
|
||||||
|
author=_text(author, "name") if author is not None else None,
|
||||||
|
published_at=_parse_date(_text(item, "published")),
|
||||||
|
updated_at=_parse_date(_text(item, "updated")),
|
||||||
|
categories=tuple(
|
||||||
|
value for child in _children(item, "category")
|
||||||
|
if (value := (child.attrib.get("term") or "").strip())
|
||||||
|
),
|
||||||
|
enclosures=tuple(
|
||||||
|
{
|
||||||
|
"url": child.attrib.get("href"),
|
||||||
|
"media_type": child.attrib.get("type"),
|
||||||
|
"size_bytes": _integer(child.attrib.get("length")),
|
||||||
|
}
|
||||||
|
for child in _children(item, "link")
|
||||||
|
if child.attrib.get("rel") == "enclosure"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return FeedDocument(
|
||||||
|
format="atom",
|
||||||
|
title=_text(root, "title") or "Untitled feed",
|
||||||
|
source_url=source_url,
|
||||||
|
entries=tuple(entries),
|
||||||
|
description=_text(root, "subtitle"),
|
||||||
|
home_url=_atom_link(root, "alternate"),
|
||||||
|
updated_at=_parse_date(_text(root, "updated")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_rss(request: FeedRenderRequest, entries: tuple[FeedEntry, ...]) -> ET.Element:
|
||||||
|
ET.register_namespace("atom", ATOM_NS)
|
||||||
|
root = ET.Element("rss", {"version": "2.0"})
|
||||||
|
channel = ET.SubElement(root, "channel")
|
||||||
|
_element(channel, "title", request.title)
|
||||||
|
_element(channel, "link", request.home_url)
|
||||||
|
_element(channel, "description", request.description or request.title)
|
||||||
|
_element(channel, f"{{{ATOM_NS}}}link", None, {
|
||||||
|
"href": request.feed_url,
|
||||||
|
"rel": "self",
|
||||||
|
"type": "application/rss+xml",
|
||||||
|
})
|
||||||
|
if request.language:
|
||||||
|
_element(channel, "language", request.language)
|
||||||
|
for entry in entries:
|
||||||
|
item = ET.SubElement(channel, "item")
|
||||||
|
_element(item, "guid", entry.id, {"isPermaLink": "false"})
|
||||||
|
_element(item, "title", entry.title)
|
||||||
|
if entry.url:
|
||||||
|
_element(item, "link", entry.url)
|
||||||
|
if entry.summary or entry.content:
|
||||||
|
_element(item, "description", entry.summary or entry.content)
|
||||||
|
if entry.author:
|
||||||
|
_element(item, "author", entry.author)
|
||||||
|
date = entry.published_at or entry.updated_at
|
||||||
|
if date:
|
||||||
|
_element(item, "pubDate", format_datetime(_utc(date)))
|
||||||
|
for category in entry.categories:
|
||||||
|
_element(item, "category", category)
|
||||||
|
for enclosure in entry.enclosures:
|
||||||
|
attributes = {
|
||||||
|
"url": str(enclosure.get("url") or ""),
|
||||||
|
"type": str(enclosure.get("media_type") or "application/octet-stream"),
|
||||||
|
"length": str(enclosure.get("size_bytes") or 0),
|
||||||
|
}
|
||||||
|
if attributes["url"]:
|
||||||
|
_element(item, "enclosure", None, attributes)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _render_atom(request: FeedRenderRequest, entries: tuple[FeedEntry, ...]) -> ET.Element:
|
||||||
|
ET.register_namespace("", ATOM_NS)
|
||||||
|
root = ET.Element(f"{{{ATOM_NS}}}feed")
|
||||||
|
_element(root, f"{{{ATOM_NS}}}id", request.feed_url)
|
||||||
|
_element(root, f"{{{ATOM_NS}}}title", request.title)
|
||||||
|
_element(root, f"{{{ATOM_NS}}}link", None, {"href": request.home_url})
|
||||||
|
_element(
|
||||||
|
root,
|
||||||
|
f"{{{ATOM_NS}}}link",
|
||||||
|
None,
|
||||||
|
{"href": request.feed_url, "rel": "self", "type": "application/atom+xml"},
|
||||||
|
)
|
||||||
|
latest = max(
|
||||||
|
(date for entry in entries for date in (entry.updated_at, entry.published_at) if date),
|
||||||
|
default=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
_element(root, f"{{{ATOM_NS}}}updated", _utc(latest).isoformat().replace("+00:00", "Z"))
|
||||||
|
if request.description:
|
||||||
|
_element(root, f"{{{ATOM_NS}}}subtitle", request.description)
|
||||||
|
for value in entries:
|
||||||
|
entry = ET.SubElement(root, f"{{{ATOM_NS}}}entry")
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}id", value.id)
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}title", value.title)
|
||||||
|
if value.url:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}link", None, {"href": value.url})
|
||||||
|
if value.summary:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}summary", value.summary)
|
||||||
|
if value.content:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}content", value.content, {"type": "html"})
|
||||||
|
updated = value.updated_at or value.published_at or latest
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}updated", _utc(updated).isoformat().replace("+00:00", "Z"))
|
||||||
|
if value.published_at:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}published", _utc(value.published_at).isoformat().replace("+00:00", "Z"))
|
||||||
|
if value.author:
|
||||||
|
author = ET.SubElement(entry, f"{{{ATOM_NS}}}author")
|
||||||
|
_element(author, f"{{{ATOM_NS}}}name", value.author)
|
||||||
|
for category in value.categories:
|
||||||
|
_element(entry, f"{{{ATOM_NS}}}category", None, {"term": category})
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _children(element: ET.Element, name: str) -> tuple[ET.Element, ...]:
|
||||||
|
return tuple(child for child in element if _local_name(child.tag) == name)
|
||||||
|
|
||||||
|
|
||||||
|
def _first_child(element: ET.Element, name: str) -> ET.Element | None:
|
||||||
|
return next((child for child in element if _local_name(child.tag) == name), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _text(element: ET.Element | None, name: str) -> str | None:
|
||||||
|
if element is None:
|
||||||
|
return None
|
||||||
|
child = _first_child(element, name)
|
||||||
|
if child is None:
|
||||||
|
return None
|
||||||
|
value = "".join(child.itertext()).strip()
|
||||||
|
return value or None
|
||||||
|
|
||||||
|
|
||||||
|
def _local_name(tag: str) -> str:
|
||||||
|
return tag.rsplit("}", 1)[-1].split(":", 1)[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def _atom_link(element: ET.Element, relation: str) -> str | None:
|
||||||
|
for child in _children(element, "link"):
|
||||||
|
if (child.attrib.get("rel") or "alternate") == relation:
|
||||||
|
return child.attrib.get("href")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str | None) -> datetime | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = parsedate_to_datetime(value)
|
||||||
|
except (TypeError, ValueError, OverflowError):
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return _utc(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def _utc(value: datetime) -> datetime:
|
||||||
|
if value.tzinfo is None:
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _integer(value: str | None) -> int | None:
|
||||||
|
try:
|
||||||
|
return int(value) if value is not None else None
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_fallback_id(element: ET.Element) -> str:
|
||||||
|
body = ET.tostring(element, encoding="utf-8")
|
||||||
|
return f"urn:sha256:{hashlib.sha256(body).hexdigest()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _header(headers: Mapping[str, str], name: str) -> str | None:
|
||||||
|
lowered = name.casefold()
|
||||||
|
return next((value for key, value in headers.items() if key.casefold() == lowered), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _fresh_until(headers: Mapping[str, str], acquired_at: datetime) -> datetime | None:
|
||||||
|
cache_control = _header(headers, "cache-control") or ""
|
||||||
|
match = re.search(r"(?:^|,)\s*max-age\s*=\s*(\d+)", cache_control, re.IGNORECASE)
|
||||||
|
if match:
|
||||||
|
return acquired_at + timedelta(seconds=int(match.group(1)))
|
||||||
|
return _parse_date(_header(headers, "expires"))
|
||||||
|
|
||||||
|
|
||||||
|
def _element(
|
||||||
|
parent: ET.Element,
|
||||||
|
tag: str,
|
||||||
|
text: str | None,
|
||||||
|
attributes: Mapping[str, str] | None = None,
|
||||||
|
) -> ET.Element:
|
||||||
|
child = ET.SubElement(parent, tag, dict(attributes or {}))
|
||||||
|
child.text = text
|
||||||
|
return child
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConnectorFeedProvider",
|
||||||
|
"FEED_PRIVATE_PUBLISH_SCOPE",
|
||||||
|
"FEED_PUBLISH_SCOPE",
|
||||||
|
"MAX_FEED_BYTES",
|
||||||
|
"feed_rows",
|
||||||
|
]
|
||||||
@@ -0,0 +1,921 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.connector_runtime import ConnectorContractError, ConnectorEndpoint
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_schemas import (
|
||||||
|
ConnectorConfigurationCreateRequest,
|
||||||
|
ConnectorConfigurationItem,
|
||||||
|
ConnectorConfigurationUpdateRequest,
|
||||||
|
ConnectorDefinitionItem,
|
||||||
|
ConnectorDefinitionUpsertRequest,
|
||||||
|
ConnectorReviewRequest,
|
||||||
|
ConnectorRunItem,
|
||||||
|
ConnectorRunRequest,
|
||||||
|
GovernedConnectorSpecification,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedConnectorError(ValueError):
|
||||||
|
def __init__(self, code: str, message: str) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
def list_definitions(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
) -> list[ConnectorDefinitionItem]:
|
||||||
|
rows = (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.tenant_id == tenant_id,
|
||||||
|
ConnectorDefinition.status == "active",
|
||||||
|
)
|
||||||
|
.order_by(ConnectorDefinition.name.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
definition_item(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
_definition_revision(session, row, row.current_revision),
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_definition(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
payload: ConnectorDefinitionUpsertRequest,
|
||||||
|
) -> ConnectorDefinitionItem:
|
||||||
|
specification = payload.specification.model_dump(mode="json")
|
||||||
|
definition_hash = _hash(specification)
|
||||||
|
definition = (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.tenant_id == principal.tenant_id,
|
||||||
|
ConnectorDefinition.definition_key == payload.definition_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
expected_local = payload.origin == "local"
|
||||||
|
if definition is None:
|
||||||
|
definition = ConnectorDefinition(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_key=payload.definition_key,
|
||||||
|
name=payload.name.strip(),
|
||||||
|
description=_optional_text(payload.description),
|
||||||
|
status="active",
|
||||||
|
current_revision=0,
|
||||||
|
source_package=payload.package_ref,
|
||||||
|
local_definition=expected_local,
|
||||||
|
)
|
||||||
|
session.add(definition)
|
||||||
|
session.flush()
|
||||||
|
elif definition.local_definition != expected_local:
|
||||||
|
if definition.local_definition:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"local_definition_protected",
|
||||||
|
"A package update cannot replace a locally owned connector definition.",
|
||||||
|
)
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"package_definition_requires_overrides",
|
||||||
|
"Use configuration overrides instead of converting a package definition into a local definition.",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
current = _definition_revision(
|
||||||
|
session,
|
||||||
|
definition,
|
||||||
|
definition.current_revision,
|
||||||
|
)
|
||||||
|
if current.definition_hash == definition_hash:
|
||||||
|
return definition_item(session, definition, current)
|
||||||
|
|
||||||
|
definition.name = payload.name.strip()
|
||||||
|
definition.description = _optional_text(payload.description)
|
||||||
|
definition.source_package = payload.package_ref
|
||||||
|
definition.current_revision += 1
|
||||||
|
revision = ConnectorDefinitionRevision(
|
||||||
|
definition_id=definition.id,
|
||||||
|
revision=definition.current_revision,
|
||||||
|
specification=specification,
|
||||||
|
definition_hash=definition_hash,
|
||||||
|
origin=payload.origin,
|
||||||
|
package_ref=payload.package_ref,
|
||||||
|
created_by=principal.user.id,
|
||||||
|
)
|
||||||
|
session.add(revision)
|
||||||
|
session.flush()
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="connectors.definition.revision_created",
|
||||||
|
object_type="connector_definition",
|
||||||
|
object_id=definition.id,
|
||||||
|
details={
|
||||||
|
"definition_key": definition.definition_key,
|
||||||
|
"revision": revision.revision,
|
||||||
|
"definition_hash": definition_hash,
|
||||||
|
"origin": payload.origin,
|
||||||
|
"package_ref": payload.package_ref,
|
||||||
|
"provider": payload.specification.provider,
|
||||||
|
"protocol": payload.specification.protocol,
|
||||||
|
"capabilities": sorted(payload.specification.capabilities),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return definition_item(session, definition, revision)
|
||||||
|
|
||||||
|
|
||||||
|
def list_configurations(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
) -> list[ConnectorConfigurationItem]:
|
||||||
|
rows = (
|
||||||
|
session.query(ConnectorConfiguration)
|
||||||
|
.filter(ConnectorConfiguration.tenant_id == tenant_id)
|
||||||
|
.order_by(ConnectorConfiguration.name.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [configuration_item(session, row) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def create_configuration(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
payload: ConnectorConfigurationCreateRequest,
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
definition = _tenant_definition(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_id=payload.definition_id,
|
||||||
|
)
|
||||||
|
_validate_endpoint(payload.endpoint_url, payload.credential_ref)
|
||||||
|
revision = _definition_revision(session, definition, definition.current_revision)
|
||||||
|
overrides = copy.deepcopy(payload.local_overrides)
|
||||||
|
effective = _effective_specification(revision.specification, overrides)
|
||||||
|
item = ConnectorConfiguration(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_id=definition.id,
|
||||||
|
name=payload.name.strip(),
|
||||||
|
status=payload.status,
|
||||||
|
endpoint_url=_optional_text(payload.endpoint_url),
|
||||||
|
credential_ref=_optional_text(payload.credential_ref),
|
||||||
|
base_definition_revision=definition.current_revision,
|
||||||
|
local_overrides=overrides,
|
||||||
|
protected_paths=_protected_paths(overrides),
|
||||||
|
effective_configuration=effective,
|
||||||
|
effective_hash=_hash(effective),
|
||||||
|
resource_revision=1,
|
||||||
|
ambiguity_policy=payload.ambiguity_policy,
|
||||||
|
updated_by=principal.user.id,
|
||||||
|
)
|
||||||
|
session.add(item)
|
||||||
|
session.flush()
|
||||||
|
_audit_configuration(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
item,
|
||||||
|
action="connectors.configuration.created",
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return configuration_item(session, item)
|
||||||
|
|
||||||
|
|
||||||
|
def update_configuration(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
payload: ConnectorConfigurationUpdateRequest,
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
item = _tenant_configuration(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
if item.resource_revision != payload.expected_revision:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"configuration_conflict",
|
||||||
|
"The connector configuration changed; reload it before saving.",
|
||||||
|
)
|
||||||
|
supplied = payload.model_fields_set
|
||||||
|
definition = _tenant_definition(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
definition_id=item.definition_id,
|
||||||
|
)
|
||||||
|
if "name" in supplied and payload.name is not None:
|
||||||
|
item.name = payload.name.strip()
|
||||||
|
if "endpoint_url" in supplied:
|
||||||
|
item.endpoint_url = _optional_text(payload.endpoint_url)
|
||||||
|
if "credential_ref" in supplied:
|
||||||
|
item.credential_ref = _optional_text(payload.credential_ref)
|
||||||
|
if "local_overrides" in supplied and payload.local_overrides is not None:
|
||||||
|
item.local_overrides = copy.deepcopy(payload.local_overrides)
|
||||||
|
if payload.ambiguity_policy is not None:
|
||||||
|
item.ambiguity_policy = payload.ambiguity_policy
|
||||||
|
if payload.status is not None:
|
||||||
|
item.status = payload.status
|
||||||
|
if payload.adopt_latest_definition:
|
||||||
|
item.base_definition_revision = definition.current_revision
|
||||||
|
_validate_endpoint(item.endpoint_url, item.credential_ref)
|
||||||
|
base = _definition_revision(
|
||||||
|
session,
|
||||||
|
definition,
|
||||||
|
item.base_definition_revision,
|
||||||
|
)
|
||||||
|
effective = _effective_specification(base.specification, item.local_overrides)
|
||||||
|
item.protected_paths = _protected_paths(item.local_overrides)
|
||||||
|
item.effective_configuration = effective
|
||||||
|
item.effective_hash = _hash(effective)
|
||||||
|
item.resource_revision += 1
|
||||||
|
item.updated_by = principal.user.id
|
||||||
|
_audit_configuration(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
item,
|
||||||
|
action="connectors.configuration.updated",
|
||||||
|
extra={"adopted_latest_definition": payload.adopt_latest_definition},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return configuration_item(session, item)
|
||||||
|
|
||||||
|
|
||||||
|
def execute_run(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
mode: str,
|
||||||
|
payload: ConnectorRunRequest,
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
if mode not in {"dry_run", "simulation"}:
|
||||||
|
raise GovernedConnectorError("invalid_mode", "Unsupported connector run mode.")
|
||||||
|
configuration = _tenant_configuration(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
if configuration.status == "disabled":
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"configuration_disabled",
|
||||||
|
"Disabled connector configurations cannot be executed.",
|
||||||
|
)
|
||||||
|
specification = GovernedConnectorSpecification.model_validate(
|
||||||
|
configuration.effective_configuration
|
||||||
|
)
|
||||||
|
if mode == "dry_run" and not specification.dry_run.supported:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"dry_run_unsupported",
|
||||||
|
"This connector definition does not support dry runs.",
|
||||||
|
)
|
||||||
|
if mode == "simulation" and not specification.dry_run.simulation_supported:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"simulation_unsupported",
|
||||||
|
"This connector definition does not support simulation.",
|
||||||
|
)
|
||||||
|
rows = (
|
||||||
|
list(payload.input_rows)
|
||||||
|
if payload.input_rows is not None
|
||||||
|
else list(specification.dry_run.sample_rows)
|
||||||
|
)
|
||||||
|
request_payload = {
|
||||||
|
"mode": mode,
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"external_revision": payload.external_revision,
|
||||||
|
"input_rows": rows,
|
||||||
|
}
|
||||||
|
request_hash = _hash(request_payload)
|
||||||
|
existing = (
|
||||||
|
session.query(ConnectorSimulationRun)
|
||||||
|
.filter(
|
||||||
|
ConnectorSimulationRun.tenant_id == principal.tenant_id,
|
||||||
|
ConnectorSimulationRun.configuration_id == configuration.id,
|
||||||
|
ConnectorSimulationRun.mode == mode,
|
||||||
|
ConnectorSimulationRun.idempotency_key == payload.idempotency_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
if existing.request_hash != request_hash:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"idempotency_conflict",
|
||||||
|
"The idempotency key was already used with different run inputs.",
|
||||||
|
)
|
||||||
|
return run_item(existing)
|
||||||
|
|
||||||
|
limit = specification.dry_run.max_items
|
||||||
|
truncated = len(rows) > limit
|
||||||
|
bounded_rows = rows[:limit]
|
||||||
|
effects, diagnostics, ambiguous_count = _simulate(
|
||||||
|
bounded_rows,
|
||||||
|
specification,
|
||||||
|
)
|
||||||
|
if truncated:
|
||||||
|
diagnostics.append(
|
||||||
|
{
|
||||||
|
"severity": "warning",
|
||||||
|
"code": "connectors.run.truncated",
|
||||||
|
"message": f"The run was limited to {limit} input rows.",
|
||||||
|
"stage": "planning",
|
||||||
|
"retryable": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
errors = sum(1 for item in diagnostics if item["severity"] == "error")
|
||||||
|
if ambiguous_count:
|
||||||
|
status_value, review_state = {
|
||||||
|
"manual_review": ("manual_review", "pending"),
|
||||||
|
"quarantine": ("quarantined", "quarantined"),
|
||||||
|
"reject": ("rejected", "not_required"),
|
||||||
|
}[configuration.ambiguity_policy]
|
||||||
|
elif errors:
|
||||||
|
status_value, review_state = "invalid", "not_required"
|
||||||
|
else:
|
||||||
|
status_value, review_state = "ready", "not_required"
|
||||||
|
input_hash = _hash(bounded_rows)
|
||||||
|
summary = {
|
||||||
|
"total": len(effects),
|
||||||
|
"creates": sum(1 for item in effects if item["effect"] == "create"),
|
||||||
|
"updates": 0,
|
||||||
|
"deletes": 0,
|
||||||
|
"conflicts": sum(1 for item in effects if item["effect"] == "conflict"),
|
||||||
|
"unchanged": 0,
|
||||||
|
"ignored": sum(1 for item in effects if item["effect"] == "ignored"),
|
||||||
|
"ambiguous": ambiguous_count,
|
||||||
|
"errors": errors,
|
||||||
|
"truncated": truncated,
|
||||||
|
}
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
provenance = {
|
||||||
|
"contract_version": "1.0",
|
||||||
|
"definition_id": configuration.definition_id,
|
||||||
|
"definition_revision": configuration.base_definition_revision,
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"mapping_version": specification.mapping.version,
|
||||||
|
"input_hash": input_hash,
|
||||||
|
"external_revision": payload.external_revision,
|
||||||
|
"generated_at": now.isoformat(),
|
||||||
|
"actor_id": principal.user.id,
|
||||||
|
"mode": mode,
|
||||||
|
"provider": specification.provider,
|
||||||
|
"protocol": specification.protocol,
|
||||||
|
"privacy_classification": specification.privacy_classification,
|
||||||
|
"retention_class": specification.retention_class,
|
||||||
|
}
|
||||||
|
run = ConnectorSimulationRun(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode=mode,
|
||||||
|
idempotency_key=payload.idempotency_key,
|
||||||
|
request_hash=request_hash,
|
||||||
|
status=status_value,
|
||||||
|
review_state=review_state,
|
||||||
|
definition_revision=configuration.base_definition_revision,
|
||||||
|
configuration_revision=configuration.resource_revision,
|
||||||
|
configuration_hash=configuration.effective_hash,
|
||||||
|
input_hash=input_hash,
|
||||||
|
summary=summary,
|
||||||
|
effects=effects,
|
||||||
|
diagnostics=diagnostics,
|
||||||
|
provenance=provenance,
|
||||||
|
created_by=principal.user.id,
|
||||||
|
)
|
||||||
|
session.add(run)
|
||||||
|
session.flush()
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=f"connectors.configuration.{mode}_completed",
|
||||||
|
object_type="connector_simulation_run",
|
||||||
|
object_id=run.id,
|
||||||
|
details={
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"definition_revision": configuration.base_definition_revision,
|
||||||
|
"input_hash": input_hash,
|
||||||
|
"status": status_value,
|
||||||
|
"review_state": review_state,
|
||||||
|
"summary": summary,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return run_item(run)
|
||||||
|
|
||||||
|
|
||||||
|
def list_runs(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
configuration_id: str | None = None,
|
||||||
|
review_state: str | None = None,
|
||||||
|
limit: int = 100,
|
||||||
|
) -> list[ConnectorRunItem]:
|
||||||
|
query = session.query(ConnectorSimulationRun).filter(
|
||||||
|
ConnectorSimulationRun.tenant_id == tenant_id
|
||||||
|
)
|
||||||
|
if configuration_id:
|
||||||
|
query = query.filter(
|
||||||
|
ConnectorSimulationRun.configuration_id == configuration_id
|
||||||
|
)
|
||||||
|
if review_state:
|
||||||
|
query = query.filter(ConnectorSimulationRun.review_state == review_state)
|
||||||
|
rows = (
|
||||||
|
query.order_by(ConnectorSimulationRun.created_at.desc())
|
||||||
|
.limit(max(1, min(int(limit), 500)))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [run_item(row) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def review_run(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
run_id: str,
|
||||||
|
payload: ConnectorReviewRequest,
|
||||||
|
) -> ConnectorRunItem:
|
||||||
|
run = (
|
||||||
|
session.query(ConnectorSimulationRun)
|
||||||
|
.filter(
|
||||||
|
ConnectorSimulationRun.id == run_id,
|
||||||
|
ConnectorSimulationRun.tenant_id == principal.tenant_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if run is None:
|
||||||
|
raise GovernedConnectorError("run_not_found", "Connector run not found.")
|
||||||
|
if run.review_state not in {"pending", "quarantined"}:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"run_not_reviewable",
|
||||||
|
"Only pending or quarantined connector results can be reviewed.",
|
||||||
|
)
|
||||||
|
run.review_state = payload.decision
|
||||||
|
run.status = f"review_{payload.decision}"
|
||||||
|
run.reviewed_by = principal.user.id
|
||||||
|
run.reviewed_at = datetime.now(UTC)
|
||||||
|
run.review_reason = payload.reason.strip()
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=f"connectors.simulation.{payload.decision}",
|
||||||
|
object_type="connector_simulation_run",
|
||||||
|
object_id=run.id,
|
||||||
|
details={
|
||||||
|
"configuration_id": run.configuration_id,
|
||||||
|
"input_hash": run.input_hash,
|
||||||
|
"configuration_hash": run.configuration_hash,
|
||||||
|
"decision": payload.decision,
|
||||||
|
"reason": run.review_reason,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return run_item(run)
|
||||||
|
|
||||||
|
|
||||||
|
def definition_item(
|
||||||
|
session: Session,
|
||||||
|
definition: ConnectorDefinition,
|
||||||
|
revision: ConnectorDefinitionRevision,
|
||||||
|
) -> ConnectorDefinitionItem:
|
||||||
|
del session
|
||||||
|
return ConnectorDefinitionItem(
|
||||||
|
id=definition.id,
|
||||||
|
tenant_id=definition.tenant_id,
|
||||||
|
definition_key=definition.definition_key,
|
||||||
|
name=definition.name,
|
||||||
|
description=definition.description,
|
||||||
|
status=definition.status,
|
||||||
|
current_revision=definition.current_revision,
|
||||||
|
source_package=definition.source_package,
|
||||||
|
local_definition=definition.local_definition,
|
||||||
|
revision_id=revision.id,
|
||||||
|
definition_hash=revision.definition_hash,
|
||||||
|
origin=revision.origin,
|
||||||
|
package_ref=revision.package_ref,
|
||||||
|
specification=GovernedConnectorSpecification.model_validate(
|
||||||
|
revision.specification
|
||||||
|
),
|
||||||
|
created_at=definition.created_at,
|
||||||
|
updated_at=definition.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def configuration_item(
|
||||||
|
session: Session,
|
||||||
|
item: ConnectorConfiguration,
|
||||||
|
) -> ConnectorConfigurationItem:
|
||||||
|
definition = _tenant_definition(
|
||||||
|
session,
|
||||||
|
tenant_id=item.tenant_id,
|
||||||
|
definition_id=item.definition_id,
|
||||||
|
)
|
||||||
|
return ConnectorConfigurationItem(
|
||||||
|
id=item.id,
|
||||||
|
tenant_id=item.tenant_id,
|
||||||
|
definition_id=item.definition_id,
|
||||||
|
definition_key=definition.definition_key,
|
||||||
|
definition_name=definition.name,
|
||||||
|
name=item.name,
|
||||||
|
status=item.status,
|
||||||
|
endpoint_url=item.endpoint_url,
|
||||||
|
credential_ref=item.credential_ref,
|
||||||
|
base_definition_revision=item.base_definition_revision,
|
||||||
|
latest_definition_revision=definition.current_revision,
|
||||||
|
update_available=definition.current_revision > item.base_definition_revision,
|
||||||
|
local_overrides=dict(item.local_overrides or {}),
|
||||||
|
protected_paths=list(item.protected_paths or []),
|
||||||
|
effective_configuration=dict(item.effective_configuration or {}),
|
||||||
|
effective_hash=item.effective_hash,
|
||||||
|
resource_revision=item.resource_revision,
|
||||||
|
ambiguity_policy=item.ambiguity_policy,
|
||||||
|
updated_at=item.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_item(run: ConnectorSimulationRun) -> ConnectorRunItem:
|
||||||
|
return ConnectorRunItem(
|
||||||
|
id=run.id,
|
||||||
|
tenant_id=run.tenant_id,
|
||||||
|
configuration_id=run.configuration_id,
|
||||||
|
mode=run.mode, # type: ignore[arg-type]
|
||||||
|
idempotency_key=run.idempotency_key,
|
||||||
|
status=run.status,
|
||||||
|
review_state=run.review_state,
|
||||||
|
definition_revision=run.definition_revision,
|
||||||
|
configuration_revision=run.configuration_revision,
|
||||||
|
configuration_hash=run.configuration_hash,
|
||||||
|
input_hash=run.input_hash,
|
||||||
|
summary=dict(run.summary or {}),
|
||||||
|
effects=list(run.effects or []),
|
||||||
|
diagnostics=list(run.diagnostics or []),
|
||||||
|
provenance=dict(run.provenance or {}),
|
||||||
|
reviewed_by=run.reviewed_by,
|
||||||
|
reviewed_at=run.reviewed_at,
|
||||||
|
review_reason=run.review_reason,
|
||||||
|
created_at=run.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _simulate(
|
||||||
|
rows: Sequence[Mapping[str, Any]],
|
||||||
|
specification: GovernedConnectorSpecification,
|
||||||
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
|
||||||
|
mapped_rows: list[dict[str, Any]] = []
|
||||||
|
row_errors: dict[int, list[dict[str, Any]]] = {}
|
||||||
|
diagnostics: list[dict[str, Any]] = []
|
||||||
|
for index, row in enumerate(rows):
|
||||||
|
mapped: dict[str, Any] = {}
|
||||||
|
for rule in specification.mapping.rules:
|
||||||
|
value, found = _path_value(row, rule.source)
|
||||||
|
if not found:
|
||||||
|
value = rule.default
|
||||||
|
if rule.required and value in (None, ""):
|
||||||
|
diagnostic = _diagnostic(
|
||||||
|
"error",
|
||||||
|
"connectors.mapping.required_source_missing",
|
||||||
|
f"Required source field {rule.source!r} is missing.",
|
||||||
|
"mapping",
|
||||||
|
index=index,
|
||||||
|
field=rule.source,
|
||||||
|
)
|
||||||
|
row_errors.setdefault(index, []).append(diagnostic)
|
||||||
|
diagnostics.append(diagnostic)
|
||||||
|
_set_path(mapped, rule.target, value)
|
||||||
|
mapped_rows.append(mapped)
|
||||||
|
|
||||||
|
ambiguous_indexes: set[int] = set()
|
||||||
|
for rule in specification.validation_rules:
|
||||||
|
if rule.kind == "unique":
|
||||||
|
seen: dict[str, list[int]] = {}
|
||||||
|
for index, row in enumerate(mapped_rows):
|
||||||
|
value, found = _path_value(row, rule.field)
|
||||||
|
if found and value not in (None, ""):
|
||||||
|
seen.setdefault(_stable_value(value), []).append(index)
|
||||||
|
for indexes in seen.values():
|
||||||
|
if len(indexes) > 1:
|
||||||
|
ambiguous_indexes.update(indexes)
|
||||||
|
for index in indexes:
|
||||||
|
diagnostic = _diagnostic(
|
||||||
|
rule.severity,
|
||||||
|
rule.code,
|
||||||
|
rule.message,
|
||||||
|
"validation",
|
||||||
|
index=index,
|
||||||
|
field=rule.field,
|
||||||
|
)
|
||||||
|
row_errors.setdefault(index, []).append(diagnostic)
|
||||||
|
diagnostics.append(diagnostic)
|
||||||
|
continue
|
||||||
|
for index, row in enumerate(mapped_rows):
|
||||||
|
value, found = _path_value(row, rule.field)
|
||||||
|
invalid = (
|
||||||
|
rule.kind == "required" and (not found or value in (None, ""))
|
||||||
|
) or (
|
||||||
|
rule.kind == "one_of" and found and value not in rule.values
|
||||||
|
)
|
||||||
|
if invalid:
|
||||||
|
diagnostic = _diagnostic(
|
||||||
|
rule.severity,
|
||||||
|
rule.code,
|
||||||
|
rule.message,
|
||||||
|
"validation",
|
||||||
|
index=index,
|
||||||
|
field=rule.field,
|
||||||
|
)
|
||||||
|
row_errors.setdefault(index, []).append(diagnostic)
|
||||||
|
diagnostics.append(diagnostic)
|
||||||
|
|
||||||
|
redacted = set(specification.dry_run.redacted_fields)
|
||||||
|
effects: list[dict[str, Any]] = []
|
||||||
|
for index, mapped in enumerate(mapped_rows):
|
||||||
|
errors = row_errors.get(index, [])
|
||||||
|
has_error = any(item["severity"] == "error" for item in errors)
|
||||||
|
effect = "conflict" if has_error or index in ambiguous_indexes else "create"
|
||||||
|
effects.append(
|
||||||
|
{
|
||||||
|
"effect": effect,
|
||||||
|
"source_object_ref": _source_ref(rows[index], index),
|
||||||
|
"target_object_ref": None,
|
||||||
|
"changed_fields": sorted(_leaf_paths(mapped)),
|
||||||
|
"sample": _redact_fields(mapped, redacted),
|
||||||
|
"reason_code": (
|
||||||
|
"ambiguous_external_result"
|
||||||
|
if index in ambiguous_indexes
|
||||||
|
else errors[0]["code"] if errors else None
|
||||||
|
),
|
||||||
|
"outcome": "preview",
|
||||||
|
"revision": specification.mapping.version,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return effects, diagnostics, len(ambiguous_indexes)
|
||||||
|
|
||||||
|
|
||||||
|
def _audit_configuration(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
item: ConnectorConfiguration,
|
||||||
|
*,
|
||||||
|
action: str,
|
||||||
|
extra: Mapping[str, Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=action,
|
||||||
|
object_type="connector_configuration",
|
||||||
|
object_id=item.id,
|
||||||
|
details={
|
||||||
|
"definition_id": item.definition_id,
|
||||||
|
"base_definition_revision": item.base_definition_revision,
|
||||||
|
"resource_revision": item.resource_revision,
|
||||||
|
"effective_hash": item.effective_hash,
|
||||||
|
"protected_paths": list(item.protected_paths or []),
|
||||||
|
"ambiguity_policy": item.ambiguity_policy,
|
||||||
|
"status": item.status,
|
||||||
|
"credential_reference_present": bool(item.credential_ref),
|
||||||
|
**dict(extra or {}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_definition(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
definition_id: str,
|
||||||
|
) -> ConnectorDefinition:
|
||||||
|
item = (
|
||||||
|
session.query(ConnectorDefinition)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinition.id == definition_id,
|
||||||
|
ConnectorDefinition.tenant_id == tenant_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"definition_not_found",
|
||||||
|
"Connector definition not found.",
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_configuration(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
configuration_id: str,
|
||||||
|
) -> ConnectorConfiguration:
|
||||||
|
item = (
|
||||||
|
session.query(ConnectorConfiguration)
|
||||||
|
.filter(
|
||||||
|
ConnectorConfiguration.id == configuration_id,
|
||||||
|
ConnectorConfiguration.tenant_id == tenant_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"configuration_not_found",
|
||||||
|
"Connector configuration not found.",
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _definition_revision(
|
||||||
|
session: Session,
|
||||||
|
definition: ConnectorDefinition,
|
||||||
|
revision: int,
|
||||||
|
) -> ConnectorDefinitionRevision:
|
||||||
|
item = (
|
||||||
|
session.query(ConnectorDefinitionRevision)
|
||||||
|
.filter(
|
||||||
|
ConnectorDefinitionRevision.definition_id == definition.id,
|
||||||
|
ConnectorDefinitionRevision.revision == revision,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"definition_revision_not_found",
|
||||||
|
"Connector definition revision not found.",
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _effective_specification(
|
||||||
|
base: Mapping[str, Any],
|
||||||
|
overrides: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
merged = _deep_merge(base, overrides)
|
||||||
|
return GovernedConnectorSpecification.model_validate(merged).model_dump(mode="json")
|
||||||
|
|
||||||
|
|
||||||
|
def _deep_merge(
|
||||||
|
base: Mapping[str, Any],
|
||||||
|
overrides: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result = copy.deepcopy(dict(base))
|
||||||
|
for key, value in overrides.items():
|
||||||
|
if isinstance(value, Mapping) and isinstance(result.get(key), Mapping):
|
||||||
|
result[key] = _deep_merge(result[key], value) # type: ignore[arg-type]
|
||||||
|
else:
|
||||||
|
result[key] = copy.deepcopy(value)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _protected_paths(value: Mapping[str, Any], prefix: str = "") -> list[str]:
|
||||||
|
paths: list[str] = []
|
||||||
|
for key in sorted(value):
|
||||||
|
path = f"{prefix}.{key}" if prefix else str(key)
|
||||||
|
item = value[key]
|
||||||
|
if isinstance(item, Mapping) and item:
|
||||||
|
paths.extend(_protected_paths(item, path))
|
||||||
|
else:
|
||||||
|
paths.append(path)
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def _leaf_paths(value: Mapping[str, Any], prefix: str = "") -> set[str]:
|
||||||
|
paths: set[str] = set()
|
||||||
|
for key, item in value.items():
|
||||||
|
path = f"{prefix}.{key}" if prefix else str(key)
|
||||||
|
if isinstance(item, Mapping) and item:
|
||||||
|
paths.update(_leaf_paths(item, path))
|
||||||
|
else:
|
||||||
|
paths.add(path)
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def _path_value(value: Mapping[str, Any], path: str) -> tuple[Any, bool]:
|
||||||
|
current: Any = value
|
||||||
|
for part in path.split("."):
|
||||||
|
if not isinstance(current, Mapping) or part not in current:
|
||||||
|
return None, False
|
||||||
|
current = current[part]
|
||||||
|
return current, True
|
||||||
|
|
||||||
|
|
||||||
|
def _set_path(target: dict[str, Any], path: str, value: Any) -> None:
|
||||||
|
parts = path.split(".")
|
||||||
|
current = target
|
||||||
|
for part in parts[:-1]:
|
||||||
|
nested = current.get(part)
|
||||||
|
if not isinstance(nested, dict):
|
||||||
|
nested = {}
|
||||||
|
current[part] = nested
|
||||||
|
current = nested
|
||||||
|
current[parts[-1]] = value
|
||||||
|
|
||||||
|
|
||||||
|
def _source_ref(row: Mapping[str, Any], index: int) -> str:
|
||||||
|
for key in ("id", "external_id", "source_id"):
|
||||||
|
value = row.get(key)
|
||||||
|
if value not in (None, ""):
|
||||||
|
return f"sample:{value}"
|
||||||
|
return f"sample:row:{index + 1}"
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_fields(value: Mapping[str, Any], fields: set[str]) -> dict[str, Any]:
|
||||||
|
result = copy.deepcopy(dict(value))
|
||||||
|
for path in fields:
|
||||||
|
parts = path.split(".")
|
||||||
|
current: Any = result
|
||||||
|
for part in parts[:-1]:
|
||||||
|
if not isinstance(current, dict):
|
||||||
|
break
|
||||||
|
current = current.get(part)
|
||||||
|
else:
|
||||||
|
if isinstance(current, dict) and parts[-1] in current:
|
||||||
|
current[parts[-1]] = "<redacted>"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _diagnostic(
|
||||||
|
severity: str,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
stage: str,
|
||||||
|
**details: Any,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"severity": severity,
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
"stage": stage,
|
||||||
|
"retryable": False,
|
||||||
|
"details": details,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_endpoint(
|
||||||
|
endpoint_url: str | None,
|
||||||
|
credential_ref: str | None,
|
||||||
|
) -> None:
|
||||||
|
if endpoint_url:
|
||||||
|
try:
|
||||||
|
ConnectorEndpoint(
|
||||||
|
url=endpoint_url,
|
||||||
|
credential_ref=_optional_text(credential_ref),
|
||||||
|
)
|
||||||
|
except ConnectorContractError as exc:
|
||||||
|
raise GovernedConnectorError("invalid_endpoint", str(exc)) from exc
|
||||||
|
elif credential_ref:
|
||||||
|
raise GovernedConnectorError(
|
||||||
|
"credential_without_endpoint",
|
||||||
|
"A credential reference requires a configured endpoint.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _stable_value(value: Any) -> str:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||||
|
|
||||||
|
|
||||||
|
def _hash(value: Any) -> str:
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: object | None) -> str | None:
|
||||||
|
normalized = str(value or "").strip()
|
||||||
|
return normalized or None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"GovernedConnectorError",
|
||||||
|
"configuration_item",
|
||||||
|
"create_configuration",
|
||||||
|
"execute_run",
|
||||||
|
"list_configurations",
|
||||||
|
"list_definitions",
|
||||||
|
"list_runs",
|
||||||
|
"review_run",
|
||||||
|
"run_item",
|
||||||
|
"update_configuration",
|
||||||
|
"upsert_definition",
|
||||||
|
]
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorMappingRule(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
source: str = Field(min_length=1, max_length=255)
|
||||||
|
target: str = Field(min_length=1, max_length=255)
|
||||||
|
required: bool = False
|
||||||
|
default: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorMappingDefinition(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
version: str = Field(min_length=1, max_length=100)
|
||||||
|
rules: list[ConnectorMappingRule] = Field(default_factory=list, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorValidationRule(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
kind: Literal["required", "one_of", "unique"]
|
||||||
|
field: str = Field(min_length=1, max_length=255)
|
||||||
|
values: list[Any] = Field(default_factory=list, max_length=500)
|
||||||
|
severity: Literal["warning", "error"] = "error"
|
||||||
|
code: str = Field(min_length=1, max_length=120)
|
||||||
|
message: str = Field(min_length=1, max_length=500)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_values(self) -> "ConnectorValidationRule":
|
||||||
|
if self.kind == "one_of" and not self.values:
|
||||||
|
raise ValueError("one_of validation requires allowed values")
|
||||||
|
if self.kind != "one_of" and self.values:
|
||||||
|
raise ValueError("Only one_of validation accepts values")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDryRunMetadata(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
supported: bool = True
|
||||||
|
simulation_supported: bool = True
|
||||||
|
sample_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=500)
|
||||||
|
max_items: int = Field(default=500, ge=1, le=10_000)
|
||||||
|
redacted_fields: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorAuditMetadata(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
event_prefix: str = Field(min_length=1, max_length=120)
|
||||||
|
expected_events: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
evidence_fields: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedConnectorSpecification(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
provider: str = Field(min_length=1, max_length=120)
|
||||||
|
protocol: str = Field(min_length=1, max_length=80)
|
||||||
|
capabilities: list[str] = Field(min_length=1, max_length=100)
|
||||||
|
input_schema: dict[str, Any]
|
||||||
|
output_schema: dict[str, Any]
|
||||||
|
mapping: ConnectorMappingDefinition
|
||||||
|
validation_rules: list[ConnectorValidationRule] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
max_length=500,
|
||||||
|
)
|
||||||
|
dry_run: ConnectorDryRunMetadata
|
||||||
|
audit: ConnectorAuditMetadata
|
||||||
|
privacy_classification: Literal[
|
||||||
|
"public",
|
||||||
|
"internal",
|
||||||
|
"confidential",
|
||||||
|
"restricted",
|
||||||
|
] = "internal"
|
||||||
|
retention_class: str = Field(min_length=1, max_length=120)
|
||||||
|
operational_limits: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
retry_policy: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionUpsertRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
definition_key: str = Field(pattern=r"^[a-z0-9][a-z0-9._-]{0,159}$")
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
origin: Literal["package", "local"] = "local"
|
||||||
|
package_ref: str | None = Field(default=None, max_length=300)
|
||||||
|
specification: GovernedConnectorSpecification
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def package_provenance(self) -> "ConnectorDefinitionUpsertRequest":
|
||||||
|
if self.origin == "package" and not self.package_ref:
|
||||||
|
raise ValueError("Package definitions require package_ref")
|
||||||
|
if self.origin == "local" and self.package_ref:
|
||||||
|
raise ValueError("Local definitions cannot claim package_ref")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
definition_key: str
|
||||||
|
name: str
|
||||||
|
description: str | None = None
|
||||||
|
status: str
|
||||||
|
current_revision: int
|
||||||
|
source_package: str | None = None
|
||||||
|
local_definition: bool
|
||||||
|
revision_id: str
|
||||||
|
definition_hash: str
|
||||||
|
origin: str
|
||||||
|
package_ref: str | None = None
|
||||||
|
specification: GovernedConnectorSpecification
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDefinitionListResponse(BaseModel):
|
||||||
|
items: list[ConnectorDefinitionItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationCreateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
definition_id: str = Field(min_length=1, max_length=36)
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
endpoint_url: str | None = Field(default=None, max_length=1500)
|
||||||
|
credential_ref: str | None = Field(default=None, max_length=500)
|
||||||
|
local_overrides: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
ambiguity_policy: Literal["manual_review", "quarantine", "reject"] = (
|
||||||
|
"manual_review"
|
||||||
|
)
|
||||||
|
status: Literal["draft", "active", "disabled"] = "draft"
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationUpdateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
name: str | None = Field(default=None, min_length=1, max_length=300)
|
||||||
|
endpoint_url: str | None = Field(default=None, max_length=1500)
|
||||||
|
credential_ref: str | None = Field(default=None, max_length=500)
|
||||||
|
local_overrides: dict[str, Any] | None = None
|
||||||
|
ambiguity_policy: Literal["manual_review", "quarantine", "reject"] | None = None
|
||||||
|
status: Literal["draft", "active", "disabled"] | None = None
|
||||||
|
adopt_latest_definition: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
definition_id: str
|
||||||
|
definition_key: str
|
||||||
|
definition_name: str
|
||||||
|
name: str
|
||||||
|
status: str
|
||||||
|
endpoint_url: str | None = None
|
||||||
|
credential_ref: str | None = None
|
||||||
|
base_definition_revision: int
|
||||||
|
latest_definition_revision: int
|
||||||
|
update_available: bool
|
||||||
|
local_overrides: dict[str, Any]
|
||||||
|
protected_paths: list[str]
|
||||||
|
effective_configuration: dict[str, Any]
|
||||||
|
effective_hash: str
|
||||||
|
resource_revision: int
|
||||||
|
ambiguity_policy: str
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorConfigurationListResponse(BaseModel):
|
||||||
|
items: list[ConnectorConfigurationItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRunRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
input_rows: list[dict[str, Any]] | None = Field(default=None, max_length=500)
|
||||||
|
external_revision: str | None = Field(default=None, max_length=255)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRunItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
configuration_id: str
|
||||||
|
mode: Literal["dry_run", "simulation"]
|
||||||
|
idempotency_key: str
|
||||||
|
status: str
|
||||||
|
review_state: str
|
||||||
|
definition_revision: int
|
||||||
|
configuration_revision: int
|
||||||
|
configuration_hash: str
|
||||||
|
input_hash: str
|
||||||
|
summary: dict[str, Any]
|
||||||
|
effects: list[dict[str, Any]]
|
||||||
|
diagnostics: list[dict[str, Any]]
|
||||||
|
provenance: dict[str, Any]
|
||||||
|
reviewed_by: str | None = None
|
||||||
|
reviewed_at: datetime | None = None
|
||||||
|
review_reason: str | None = None
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRunListResponse(BaseModel):
|
||||||
|
items: list[ConnectorRunItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorReviewRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
decision: Literal["approved", "rejected"]
|
||||||
|
reason: str = Field(min_length=5, max_length=1000)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,303 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||||
|
|
||||||
|
|
||||||
|
KnowledgeMaturity = Literal[
|
||||||
|
"discover",
|
||||||
|
"link",
|
||||||
|
"search",
|
||||||
|
"read",
|
||||||
|
"publish",
|
||||||
|
"synchronize",
|
||||||
|
"migrate",
|
||||||
|
]
|
||||||
|
KnowledgeVisibility = Literal["tenant", "restricted"]
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeNamespaceMapping(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
source_namespace_id: int
|
||||||
|
source_name: str = Field(default="", max_length=200)
|
||||||
|
target_space_ref: str = Field(min_length=1, max_length=255)
|
||||||
|
target_path_prefix: str = Field(default="", max_length=500)
|
||||||
|
include: bool = True
|
||||||
|
visibility: KnowledgeVisibility | None = None
|
||||||
|
acl_tokens: list[str] = Field(default_factory=list, max_length=500)
|
||||||
|
|
||||||
|
@field_validator("acl_tokens")
|
||||||
|
@classmethod
|
||||||
|
def normalize_acl_tokens(cls, values: list[str]) -> list[str]:
|
||||||
|
return _normalized_tokens(values)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def restricted_mapping_requires_acl(self) -> "KnowledgeNamespaceMapping":
|
||||||
|
if self.visibility == "restricted" and not self.acl_tokens:
|
||||||
|
raise ValueError("Restricted namespace mappings require ACL tokens")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeProfileCreateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
configuration_id: str = Field(min_length=1, max_length=36)
|
||||||
|
desired_maturity: KnowledgeMaturity = "read"
|
||||||
|
source_authority_mode: Literal[
|
||||||
|
"external_authoritative",
|
||||||
|
"external_mirror",
|
||||||
|
"governed_sync",
|
||||||
|
"linked_reference",
|
||||||
|
] = "external_mirror"
|
||||||
|
default_visibility: KnowledgeVisibility = "restricted"
|
||||||
|
default_acl_tokens: list[str] = Field(default_factory=list, max_length=500)
|
||||||
|
namespace_mappings: list[KnowledgeNamespaceMapping] = Field(
|
||||||
|
min_length=1, max_length=500
|
||||||
|
)
|
||||||
|
|
||||||
|
@field_validator("default_acl_tokens")
|
||||||
|
@classmethod
|
||||||
|
def normalize_acl_tokens(cls, values: list[str]) -> list[str]:
|
||||||
|
return _normalized_tokens(values)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def restricted_profile_requires_acl(self) -> "KnowledgeProfileCreateRequest":
|
||||||
|
if self.default_visibility == "restricted" and not self.default_acl_tokens:
|
||||||
|
raise ValueError("Restricted knowledge profiles require default ACL tokens")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeProfileUpdateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
expected_resource_revision: int = Field(ge=1)
|
||||||
|
status: Literal["active", "paused"] | None = None
|
||||||
|
desired_maturity: KnowledgeMaturity | None = None
|
||||||
|
source_authority_mode: Literal[
|
||||||
|
"external_authoritative",
|
||||||
|
"external_mirror",
|
||||||
|
"governed_sync",
|
||||||
|
"linked_reference",
|
||||||
|
] | None = None
|
||||||
|
default_visibility: KnowledgeVisibility | None = None
|
||||||
|
default_acl_tokens: list[str] | None = Field(default=None, max_length=500)
|
||||||
|
namespace_mappings: list[KnowledgeNamespaceMapping] | None = Field(
|
||||||
|
default=None, min_length=1, max_length=500
|
||||||
|
)
|
||||||
|
|
||||||
|
@field_validator("default_acl_tokens")
|
||||||
|
@classmethod
|
||||||
|
def normalize_acl_tokens(cls, values: list[str] | None) -> list[str] | None:
|
||||||
|
return _normalized_tokens(values) if values is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeProfileItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
configuration_id: str
|
||||||
|
status: str
|
||||||
|
product: str
|
||||||
|
product_version: str | None = None
|
||||||
|
desired_maturity: str
|
||||||
|
discovered_maturity: str
|
||||||
|
source_authority_mode: str
|
||||||
|
default_visibility: str
|
||||||
|
default_acl_tokens: list[str]
|
||||||
|
namespace_mappings: list[dict[str, Any]]
|
||||||
|
capabilities: list[str]
|
||||||
|
discovery_revision: str | None = None
|
||||||
|
health_status: str
|
||||||
|
health_details: dict[str, Any]
|
||||||
|
discovered_at: datetime | None = None
|
||||||
|
last_sync_cursor: str | None = None
|
||||||
|
last_high_watermark: str | None = None
|
||||||
|
resource_revision: int
|
||||||
|
credential_reference_present: bool = False
|
||||||
|
endpoint_configured: bool = False
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeProfileListResponse(BaseModel):
|
||||||
|
items: list[KnowledgeProfileItem]
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeDiagnostic(BaseModel):
|
||||||
|
severity: Literal["info", "warning", "error"]
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
object_ref: str | None = None
|
||||||
|
field: str | None = None
|
||||||
|
retryable: bool = False
|
||||||
|
details: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeDiscoveryResponse(BaseModel):
|
||||||
|
profile: KnowledgeProfileItem
|
||||||
|
product: str
|
||||||
|
product_version: str | None = None
|
||||||
|
api_version: str | None = None
|
||||||
|
capabilities: list[str]
|
||||||
|
namespaces: list[dict[str, Any]]
|
||||||
|
extensions: list[dict[str, Any]]
|
||||||
|
maturity: str
|
||||||
|
health_status: str
|
||||||
|
diagnostics: list[KnowledgeDiagnostic]
|
||||||
|
revision: str
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeSyncRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
cursor: str | None = Field(default=None, max_length=500)
|
||||||
|
force_full: bool = False
|
||||||
|
limit: int = Field(default=100, ge=1, le=500)
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeSyncRunItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
profile_id: str
|
||||||
|
mode: str
|
||||||
|
idempotency_key: str
|
||||||
|
status: str
|
||||||
|
cursor_before: str | None = None
|
||||||
|
cursor_after: str | None = None
|
||||||
|
high_watermark: str | None = None
|
||||||
|
counts: dict[str, Any]
|
||||||
|
effects: list[dict[str, Any]]
|
||||||
|
diagnostics: list[KnowledgeDiagnostic]
|
||||||
|
provenance: dict[str, Any]
|
||||||
|
started_at: datetime
|
||||||
|
finished_at: datetime | None = None
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeSyncRunListResponse(BaseModel):
|
||||||
|
items: list[KnowledgeSyncRunItem]
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeExternalReferenceResponse(BaseModel):
|
||||||
|
system: str
|
||||||
|
object_type: str
|
||||||
|
object_id: str
|
||||||
|
maturity: str
|
||||||
|
authority_mode: str
|
||||||
|
connector_id: str | None = None
|
||||||
|
canonical_url: str | None = None
|
||||||
|
version: str | None = None
|
||||||
|
etag: str | None = None
|
||||||
|
observed_at: str | None = None
|
||||||
|
metadata: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeObjectItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
profile_id: str
|
||||||
|
object_type: str
|
||||||
|
external_id: str
|
||||||
|
external_page_id: str | None = None
|
||||||
|
external_revision_id: str | None = None
|
||||||
|
namespace_id: int | None = None
|
||||||
|
title: str
|
||||||
|
canonical_url: str | None = None
|
||||||
|
status: str
|
||||||
|
redirect_target_external_id: str | None = None
|
||||||
|
source_revision: str
|
||||||
|
visibility: str
|
||||||
|
acl_tokens: list[str]
|
||||||
|
mapped_data: dict[str, Any]
|
||||||
|
external_reference: KnowledgeExternalReferenceResponse
|
||||||
|
source_updated_at: datetime | None = None
|
||||||
|
observed_at: datetime
|
||||||
|
resource_revision: int
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeObjectListResponse(BaseModel):
|
||||||
|
items: list[KnowledgeObjectItem]
|
||||||
|
next_cursor: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeMigrationTargetState(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
path: str = Field(min_length=1, max_length=500)
|
||||||
|
source_external_id: str | None = Field(default=None, max_length=255)
|
||||||
|
attachment_names: list[str] = Field(default_factory=list, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeMigrationDryRunRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
target_space_ref: str = Field(min_length=1, max_length=255)
|
||||||
|
max_items: int = Field(default=100, ge=1, le=500)
|
||||||
|
supported_macros: list[str] = Field(default_factory=list, max_length=200)
|
||||||
|
existing_targets: list[KnowledgeMigrationTargetState] = Field(
|
||||||
|
default_factory=list, max_length=5_000
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeMigrationDryRunResponse(BaseModel):
|
||||||
|
run: KnowledgeSyncRunItem
|
||||||
|
target_space_ref: str
|
||||||
|
source_revision: str
|
||||||
|
source_fingerprint: str
|
||||||
|
summary: dict[str, int]
|
||||||
|
effects: list[dict[str, Any]]
|
||||||
|
diagnostics: list[KnowledgeDiagnostic]
|
||||||
|
truncated: bool
|
||||||
|
can_apply: bool
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgePublishRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
title: str = Field(min_length=1, max_length=500)
|
||||||
|
body: str = Field(max_length=200_000)
|
||||||
|
summary: str = Field(default="", max_length=500)
|
||||||
|
expected_external_revision: str | None = Field(default=None, max_length=255)
|
||||||
|
minor: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgePublishResponse(BaseModel):
|
||||||
|
run: KnowledgeSyncRunItem
|
||||||
|
external_reference: KnowledgeExternalReferenceResponse
|
||||||
|
accepted: bool
|
||||||
|
outcome_unknown: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_tokens(values: list[str]) -> list[str]:
|
||||||
|
normalized = [str(item).strip() for item in values]
|
||||||
|
if any(not item or len(item) > 500 for item in normalized):
|
||||||
|
raise ValueError("ACL tokens must contain 1 to 500 characters")
|
||||||
|
if len(normalized) != len(set(normalized)):
|
||||||
|
raise ValueError("ACL tokens must be unique")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"KnowledgeDiagnostic",
|
||||||
|
"KnowledgeDiscoveryResponse",
|
||||||
|
"KnowledgeExternalReferenceResponse",
|
||||||
|
"KnowledgeMigrationDryRunRequest",
|
||||||
|
"KnowledgeMigrationDryRunResponse",
|
||||||
|
"KnowledgeMigrationTargetState",
|
||||||
|
"KnowledgeNamespaceMapping",
|
||||||
|
"KnowledgeObjectItem",
|
||||||
|
"KnowledgeObjectListResponse",
|
||||||
|
"KnowledgeProfileCreateRequest",
|
||||||
|
"KnowledgeProfileItem",
|
||||||
|
"KnowledgeProfileListResponse",
|
||||||
|
"KnowledgeProfileUpdateRequest",
|
||||||
|
"KnowledgePublishRequest",
|
||||||
|
"KnowledgePublishResponse",
|
||||||
|
"KnowledgeSyncRequest",
|
||||||
|
"KnowledgeSyncRunItem",
|
||||||
|
"KnowledgeSyncRunListResponse",
|
||||||
|
]
|
||||||
@@ -0,0 +1,269 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.external_references import ExternalObjectReference
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillPage,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
SearchDocument,
|
||||||
|
SearchResourceType,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorKnowledgeObject,
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.knowledge_connector import (
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
KNOWLEDGE_READ_SCOPE,
|
||||||
|
KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalKnowledgeSearchSource:
|
||||||
|
def resource_types(self) -> Sequence[SearchResourceType]:
|
||||||
|
return (
|
||||||
|
SearchResourceType(
|
||||||
|
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||||
|
module_id="connectors",
|
||||||
|
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
label="External knowledge pages",
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def backfill(
|
||||||
|
self, session: object, *, request: SearchBackfillRequest
|
||||||
|
) -> SearchBackfillPage:
|
||||||
|
_assert_source(request.provider_id, request.resource_type)
|
||||||
|
db = _session(session)
|
||||||
|
query = (
|
||||||
|
select(ConnectorKnowledgeObject, ConnectorKnowledgeProfile)
|
||||||
|
.join(
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeProfile.id == ConnectorKnowledgeObject.profile_id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorKnowledgeObject.tenant_id == request.tenant_id,
|
||||||
|
ConnectorKnowledgeObject.object_type == "page",
|
||||||
|
ConnectorKnowledgeObject.status != "deleted",
|
||||||
|
ConnectorKnowledgeProfile.tenant_id == request.tenant_id,
|
||||||
|
ConnectorKnowledgeProfile.status == "active",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if request.cursor:
|
||||||
|
query = query.where(ConnectorKnowledgeObject.id > request.cursor)
|
||||||
|
rows = tuple(
|
||||||
|
db.execute(
|
||||||
|
query.order_by(ConnectorKnowledgeObject.id.asc()).limit(
|
||||||
|
request.limit + 1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
has_more = len(rows) > request.limit
|
||||||
|
selected = rows[: request.limit]
|
||||||
|
watermark = db.scalar(
|
||||||
|
select(func.max(ConnectorKnowledgeObject.updated_at))
|
||||||
|
.join(
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeProfile.id == ConnectorKnowledgeObject.profile_id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorKnowledgeObject.tenant_id == request.tenant_id,
|
||||||
|
ConnectorKnowledgeObject.object_type == "page",
|
||||||
|
ConnectorKnowledgeObject.status != "deleted",
|
||||||
|
ConnectorKnowledgeProfile.tenant_id == request.tenant_id,
|
||||||
|
ConnectorKnowledgeProfile.status == "active",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return SearchBackfillPage(
|
||||||
|
documents=tuple(search_document(db, row, profile) for row, profile in selected),
|
||||||
|
next_cursor=selected[-1][0].id if has_more and selected else None,
|
||||||
|
complete=not has_more,
|
||||||
|
high_watermark=watermark.isoformat() if watermark else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def authorize(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
requests: Sequence[SearchAuthorizationRequest],
|
||||||
|
) -> Mapping[str, bool]:
|
||||||
|
db = _session(session)
|
||||||
|
tenant_id = str(getattr(principal, "tenant_id", "") or "")
|
||||||
|
can_read = _has_scope(principal, KNOWLEDGE_READ_SCOPE)
|
||||||
|
tokens = set(_principal_acl_tokens(principal))
|
||||||
|
decisions = {item.reference.key: False for item in requests}
|
||||||
|
if not tenant_id or not can_read:
|
||||||
|
return decisions
|
||||||
|
for request in requests:
|
||||||
|
reference = request.reference
|
||||||
|
if (
|
||||||
|
reference.tenant_id != tenant_id
|
||||||
|
or reference.module_id != "connectors"
|
||||||
|
or reference.resource_type != KNOWLEDGE_RESOURCE_TYPE
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
row = db.execute(
|
||||||
|
select(ConnectorKnowledgeObject, ConnectorKnowledgeProfile)
|
||||||
|
.join(
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeProfile.id
|
||||||
|
== ConnectorKnowledgeObject.profile_id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorKnowledgeObject.tenant_id == tenant_id,
|
||||||
|
ConnectorKnowledgeObject.id == reference.resource_id,
|
||||||
|
ConnectorKnowledgeObject.object_type == "page",
|
||||||
|
ConnectorKnowledgeObject.status != "deleted",
|
||||||
|
ConnectorKnowledgeProfile.tenant_id == tenant_id,
|
||||||
|
ConnectorKnowledgeProfile.status == "active",
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
if row is None:
|
||||||
|
continue
|
||||||
|
page, _profile = row
|
||||||
|
decisions[reference.key] = page.visibility == "tenant" or bool(
|
||||||
|
tokens.intersection(str(value) for value in page.acl_tokens or ())
|
||||||
|
)
|
||||||
|
return decisions
|
||||||
|
|
||||||
|
|
||||||
|
def create_external_knowledge_search_source(
|
||||||
|
_context: ModuleContext,
|
||||||
|
) -> ExternalKnowledgeSearchSource:
|
||||||
|
return ExternalKnowledgeSearchSource()
|
||||||
|
|
||||||
|
|
||||||
|
def search_document(
|
||||||
|
session: Session,
|
||||||
|
row: ConnectorKnowledgeObject,
|
||||||
|
profile: ConnectorKnowledgeProfile | None = None,
|
||||||
|
) -> SearchDocument:
|
||||||
|
if profile is None:
|
||||||
|
profile = session.scalar(
|
||||||
|
select(ConnectorKnowledgeProfile).where(
|
||||||
|
ConnectorKnowledgeProfile.tenant_id == row.tenant_id,
|
||||||
|
ConnectorKnowledgeProfile.id == row.profile_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if profile is None:
|
||||||
|
raise ValueError("External knowledge profile is unavailable.")
|
||||||
|
data = dict(row.mapped_data or {})
|
||||||
|
categories = tuple(str(value)[:200] for value in data.get("categories") or ())
|
||||||
|
links = tuple(
|
||||||
|
str(value.get("title") or value.get("external_id") or "")[:200]
|
||||||
|
for value in data.get("links") or ()
|
||||||
|
if isinstance(value, Mapping)
|
||||||
|
)
|
||||||
|
external_reference = ExternalObjectReference(
|
||||||
|
system=profile.product if profile.product != "unknown" else "mediawiki",
|
||||||
|
object_type=row.object_type,
|
||||||
|
object_id=row.external_id,
|
||||||
|
maturity=profile.discovered_maturity,
|
||||||
|
authority_mode=profile.source_authority_mode,
|
||||||
|
connector_id=profile.id,
|
||||||
|
canonical_url=row.canonical_url,
|
||||||
|
version=row.source_revision,
|
||||||
|
etag=row.content_hash,
|
||||||
|
observed_at=row.observed_at,
|
||||||
|
metadata={
|
||||||
|
"title": row.title,
|
||||||
|
"namespace_id": row.namespace_id,
|
||||||
|
"external_revision_id": row.external_revision_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return SearchDocument(
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
module_id="connectors",
|
||||||
|
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||||
|
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
resource_id=row.id,
|
||||||
|
title=row.title,
|
||||||
|
url=(
|
||||||
|
"/connectors/knowledge?profileId="
|
||||||
|
f"{quote(row.profile_id, safe='')}&objectId={quote(row.id, safe='')}"
|
||||||
|
),
|
||||||
|
summary=str(data.get("summary") or data.get("body") or "")[:4_000] or None,
|
||||||
|
body=str(data.get("body") or "")[:200_000] or None,
|
||||||
|
keywords=tuple(dict.fromkeys((*categories, *links)))[:100],
|
||||||
|
visibility=row.visibility,
|
||||||
|
acl_tokens=(
|
||||||
|
tuple(str(value) for value in row.acl_tokens or ())
|
||||||
|
if row.visibility == "restricted"
|
||||||
|
else ()
|
||||||
|
),
|
||||||
|
external_reference=external_reference,
|
||||||
|
metadata={
|
||||||
|
"profile_id": row.profile_id,
|
||||||
|
"external_page_id": row.external_page_id,
|
||||||
|
"namespace_id": row.namespace_id,
|
||||||
|
"target_space_ref": data.get("target_space_ref"),
|
||||||
|
"target_path": data.get("target_path"),
|
||||||
|
"status": row.status,
|
||||||
|
"redirect_target_external_id": row.redirect_target_external_id,
|
||||||
|
},
|
||||||
|
source_revision=row.source_revision,
|
||||||
|
change_cursor=row.change_cursor,
|
||||||
|
source_updated_at=row.source_updated_at or row.observed_at,
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal_acl_tokens(principal: object) -> tuple[str, ...]:
|
||||||
|
values: list[str] = []
|
||||||
|
for prefix, attribute in (
|
||||||
|
("account", "account_id"),
|
||||||
|
("membership", "membership_id"),
|
||||||
|
("identity", "identity_id"),
|
||||||
|
):
|
||||||
|
value = getattr(principal, attribute, None)
|
||||||
|
if value:
|
||||||
|
values.append(f"{prefix}:{value}")
|
||||||
|
for prefix, attribute in (
|
||||||
|
("group", "group_ids"),
|
||||||
|
("role", "role_ids"),
|
||||||
|
("function", "function_assignment_ids"),
|
||||||
|
("scope", "scopes"),
|
||||||
|
):
|
||||||
|
values.extend(
|
||||||
|
f"{prefix}:{value}"
|
||||||
|
for value in getattr(principal, attribute, ())
|
||||||
|
if value
|
||||||
|
)
|
||||||
|
return tuple(dict.fromkeys(values))[:500]
|
||||||
|
|
||||||
|
|
||||||
|
def _has_scope(principal: object, required: str) -> bool:
|
||||||
|
check = getattr(principal, "has", None)
|
||||||
|
if callable(check):
|
||||||
|
return bool(check(required))
|
||||||
|
return required in getattr(principal, "scopes", ())
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_source(provider_id: str, resource_type: str) -> None:
|
||||||
|
if (
|
||||||
|
provider_id != KNOWLEDGE_PROVIDER_ID
|
||||||
|
or resource_type != KNOWLEDGE_RESOURCE_TYPE
|
||||||
|
):
|
||||||
|
raise ValueError("Unsupported external knowledge search source.")
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("External knowledge Search requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ExternalKnowledgeSearchSource",
|
||||||
|
"create_external_knowledge_search_source",
|
||||||
|
"search_document",
|
||||||
|
]
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,503 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Protocol
|
||||||
|
from urllib.parse import quote, urlencode, urljoin, urlsplit, urlunsplit
|
||||||
|
|
||||||
|
from govoplan_core.security.http_fetch import fetch_http
|
||||||
|
|
||||||
|
|
||||||
|
MAX_MEDIAWIKI_RESPONSE_BYTES = 10_000_000
|
||||||
|
|
||||||
|
|
||||||
|
class MediaWikiTransportError(RuntimeError):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
retryable: bool = False,
|
||||||
|
outcome_unknown: bool = False,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
self.retryable = retryable
|
||||||
|
self.outcome_unknown = outcome_unknown
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class MediaWikiChangeBatch:
|
||||||
|
changes: tuple[Mapping[str, Any], ...]
|
||||||
|
next_cursor: str | None
|
||||||
|
complete: bool
|
||||||
|
high_watermark: str | None
|
||||||
|
evidence: Mapping[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class MediaWikiPublishResult:
|
||||||
|
page_id: str
|
||||||
|
revision_id: str
|
||||||
|
title: str
|
||||||
|
canonical_url: str | None
|
||||||
|
evidence: Mapping[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class MediaWikiTransport(Protocol):
|
||||||
|
def discover(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
) -> Mapping[str, Any]: ...
|
||||||
|
|
||||||
|
def changes(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
cursor: str | None,
|
||||||
|
limit: int,
|
||||||
|
force_full: bool,
|
||||||
|
) -> MediaWikiChangeBatch: ...
|
||||||
|
|
||||||
|
def publish(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
title: str,
|
||||||
|
body: str,
|
||||||
|
summary: str,
|
||||||
|
expected_revision: str | None,
|
||||||
|
minor: bool,
|
||||||
|
) -> MediaWikiPublishResult: ...
|
||||||
|
|
||||||
|
|
||||||
|
class HttpMediaWikiTransport:
|
||||||
|
"""Bounded MediaWiki Action API transport with Core outbound policy enforcement."""
|
||||||
|
|
||||||
|
def discover(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
) -> Mapping[str, Any]:
|
||||||
|
return self._request(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
params={
|
||||||
|
"action": "query",
|
||||||
|
"meta": "siteinfo|userinfo",
|
||||||
|
"siprop": "general|extensions|namespaces|namespacealiases|rightsinfo",
|
||||||
|
"uiprop": "rights|groups",
|
||||||
|
"format": "json",
|
||||||
|
"formatversion": "2",
|
||||||
|
"curtimestamp": "1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def changes(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
cursor: str | None,
|
||||||
|
limit: int,
|
||||||
|
force_full: bool,
|
||||||
|
) -> MediaWikiChangeBatch:
|
||||||
|
if force_full:
|
||||||
|
listing = self._request(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
params={
|
||||||
|
"action": "query",
|
||||||
|
"list": "allpages",
|
||||||
|
"aplimit": str(limit),
|
||||||
|
"apcontinue": cursor or "",
|
||||||
|
"apfilterredir": "all",
|
||||||
|
"format": "json",
|
||||||
|
"formatversion": "2",
|
||||||
|
"curtimestamp": "1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
rows = _list(listing, "query", "allpages")
|
||||||
|
page_ids = tuple(
|
||||||
|
str(item.get("pageid"))
|
||||||
|
for item in rows
|
||||||
|
if isinstance(item, Mapping) and item.get("pageid") is not None
|
||||||
|
)
|
||||||
|
changes = self._page_details(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
page_ids=page_ids,
|
||||||
|
)
|
||||||
|
next_cursor = _continue_token(listing, "apcontinue")
|
||||||
|
return MediaWikiChangeBatch(
|
||||||
|
changes=changes,
|
||||||
|
next_cursor=next_cursor,
|
||||||
|
complete=next_cursor is None,
|
||||||
|
high_watermark=_optional_text(listing.get("curtimestamp")),
|
||||||
|
evidence={
|
||||||
|
"mode": "backfill",
|
||||||
|
"listed": len(rows),
|
||||||
|
"resolved": len(changes),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
listing = self._request(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
params={
|
||||||
|
"action": "query",
|
||||||
|
"list": "recentchanges",
|
||||||
|
"rclimit": str(limit),
|
||||||
|
"rccontinue": cursor or "",
|
||||||
|
"rcdir": "newer",
|
||||||
|
"rcprop": "title|ids|sizes|flags|user|timestamp|loginfo|tags",
|
||||||
|
"rctype": "edit|new|log",
|
||||||
|
"format": "json",
|
||||||
|
"formatversion": "2",
|
||||||
|
"curtimestamp": "1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
recent = tuple(
|
||||||
|
item
|
||||||
|
for item in _list(listing, "query", "recentchanges")
|
||||||
|
if isinstance(item, Mapping)
|
||||||
|
)
|
||||||
|
page_ids = tuple(
|
||||||
|
dict.fromkeys(
|
||||||
|
str(item.get("pageid"))
|
||||||
|
for item in recent
|
||||||
|
if item.get("pageid") is not None
|
||||||
|
and str(item.get("logtype") or "") != "delete"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
resolved = {
|
||||||
|
str(item.get("pageid")): item
|
||||||
|
for item in self._page_details(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
page_ids=page_ids,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
changes: list[Mapping[str, Any]] = []
|
||||||
|
for item in recent:
|
||||||
|
page_id = _optional_text(item.get("pageid"))
|
||||||
|
if str(item.get("logtype") or "") == "delete":
|
||||||
|
changes.append(
|
||||||
|
{
|
||||||
|
"change_kind": "delete",
|
||||||
|
"pageid": page_id or f"log:{item.get('logid')}",
|
||||||
|
"title": _optional_text(item.get("title")) or "Deleted page",
|
||||||
|
"ns": item.get("ns"),
|
||||||
|
"timestamp": item.get("timestamp"),
|
||||||
|
"logid": item.get("logid"),
|
||||||
|
"change_cursor": _change_cursor(item),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
page = dict(resolved.get(page_id or "") or item)
|
||||||
|
page["change_kind"] = "upsert"
|
||||||
|
page["change_cursor"] = _change_cursor(item)
|
||||||
|
page["recent_change"] = _safe_recent_change(item)
|
||||||
|
changes.append(page)
|
||||||
|
next_cursor = _continue_token(listing, "rccontinue")
|
||||||
|
return MediaWikiChangeBatch(
|
||||||
|
changes=tuple(changes),
|
||||||
|
next_cursor=next_cursor,
|
||||||
|
complete=next_cursor is None,
|
||||||
|
high_watermark=_optional_text(listing.get("curtimestamp")),
|
||||||
|
evidence={
|
||||||
|
"mode": "delta",
|
||||||
|
"listed": len(recent),
|
||||||
|
"resolved": len(resolved),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def publish(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
title: str,
|
||||||
|
body: str,
|
||||||
|
summary: str,
|
||||||
|
expected_revision: str | None,
|
||||||
|
minor: bool,
|
||||||
|
) -> MediaWikiPublishResult:
|
||||||
|
if not credential:
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"credential_required",
|
||||||
|
"Publishing requires a governed credential envelope.",
|
||||||
|
)
|
||||||
|
csrf_token = _optional_text(credential.get("csrf_token"))
|
||||||
|
if not csrf_token:
|
||||||
|
token_payload = self._request(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
params={
|
||||||
|
"action": "query",
|
||||||
|
"meta": "tokens",
|
||||||
|
"type": "csrf",
|
||||||
|
"format": "json",
|
||||||
|
"formatversion": "2",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
query = token_payload.get("query")
|
||||||
|
tokens = query.get("tokens") if isinstance(query, Mapping) else None
|
||||||
|
csrf_token = (
|
||||||
|
_optional_text(tokens.get("csrftoken"))
|
||||||
|
if isinstance(tokens, Mapping)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
if not csrf_token:
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"csrf_token_unavailable",
|
||||||
|
"The provider did not issue a CSRF token for the configured credential.",
|
||||||
|
)
|
||||||
|
parameters: dict[str, str] = {
|
||||||
|
"action": "edit",
|
||||||
|
"title": title,
|
||||||
|
"text": body,
|
||||||
|
"summary": summary,
|
||||||
|
"token": csrf_token,
|
||||||
|
"format": "json",
|
||||||
|
"formatversion": "2",
|
||||||
|
}
|
||||||
|
if minor:
|
||||||
|
parameters["minor"] = "1"
|
||||||
|
if expected_revision:
|
||||||
|
parameters["baserevid"] = expected_revision
|
||||||
|
payload = self._request(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
params=parameters,
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
edit = payload.get("edit")
|
||||||
|
if not isinstance(edit, Mapping) or str(edit.get("result")) != "Success":
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"publish_rejected",
|
||||||
|
"MediaWiki rejected the page publication.",
|
||||||
|
)
|
||||||
|
page_id = _required_text(edit.get("pageid"), "MediaWiki omitted the page id")
|
||||||
|
revision_id = _required_text(
|
||||||
|
edit.get("newrevid"), "MediaWiki omitted the accepted revision id"
|
||||||
|
)
|
||||||
|
canonical_url = _canonical_page_url(endpoint_url, title)
|
||||||
|
return MediaWikiPublishResult(
|
||||||
|
page_id=page_id,
|
||||||
|
revision_id=revision_id,
|
||||||
|
title=_optional_text(edit.get("title")) or title,
|
||||||
|
canonical_url=canonical_url,
|
||||||
|
evidence={
|
||||||
|
"result": "Success",
|
||||||
|
"page_id": page_id,
|
||||||
|
"revision_id": revision_id,
|
||||||
|
"old_revision_id": _optional_text(edit.get("oldrevid")),
|
||||||
|
"new_page": bool(edit.get("new")),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _page_details(
|
||||||
|
self,
|
||||||
|
endpoint_url: str,
|
||||||
|
*,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
page_ids: Sequence[str],
|
||||||
|
) -> tuple[Mapping[str, Any], ...]:
|
||||||
|
if not page_ids:
|
||||||
|
return ()
|
||||||
|
payload = self._request(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
params={
|
||||||
|
"action": "query",
|
||||||
|
"pageids": "|".join(page_ids),
|
||||||
|
"prop": "info|revisions|categories|links|images|pageprops",
|
||||||
|
"inprop": "url|displaytitle",
|
||||||
|
"rvlimit": "1",
|
||||||
|
"rvprop": "ids|timestamp|user|comment|content|contentmodel|sha1|flags",
|
||||||
|
"rvslots": "main",
|
||||||
|
"cllimit": "max",
|
||||||
|
"pllimit": "max",
|
||||||
|
"imlimit": "max",
|
||||||
|
"format": "json",
|
||||||
|
"formatversion": "2",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return tuple(
|
||||||
|
item
|
||||||
|
for item in _list(payload, "query", "pages")
|
||||||
|
if isinstance(item, Mapping)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _request(
|
||||||
|
self,
|
||||||
|
endpoint_url: str,
|
||||||
|
*,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
params: Mapping[str, str],
|
||||||
|
method: str = "GET",
|
||||||
|
) -> Mapping[str, Any]:
|
||||||
|
api_url = _api_url(endpoint_url)
|
||||||
|
headers = {
|
||||||
|
"Accept": "application/json",
|
||||||
|
"User-Agent": "GovOPlaN-Connectors/MediaWiki",
|
||||||
|
**_auth_headers(credential),
|
||||||
|
}
|
||||||
|
body = None
|
||||||
|
target_url = api_url
|
||||||
|
if method == "POST":
|
||||||
|
body = urlencode(params).encode("utf-8")
|
||||||
|
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||||
|
else:
|
||||||
|
target_url = f"{api_url}?{urlencode({key: value for key, value in params.items() if value != ''})}"
|
||||||
|
try:
|
||||||
|
response = fetch_http(
|
||||||
|
target_url,
|
||||||
|
method=method,
|
||||||
|
headers=headers,
|
||||||
|
body=body,
|
||||||
|
max_bytes=MAX_MEDIAWIKI_RESPONSE_BYTES,
|
||||||
|
timeout=30,
|
||||||
|
label="MediaWiki Action API URL",
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"transport_unavailable",
|
||||||
|
"MediaWiki transport failed before a valid response was received.",
|
||||||
|
retryable=True,
|
||||||
|
outcome_unknown=method == "POST",
|
||||||
|
) from exc
|
||||||
|
if response.status < 200 or response.status >= 300:
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"http_error",
|
||||||
|
f"MediaWiki returned HTTP {response.status}.",
|
||||||
|
retryable=response.status >= 500,
|
||||||
|
outcome_unknown=method == "POST" and response.status >= 500,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
payload = json.loads(response.body)
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"invalid_json",
|
||||||
|
"MediaWiki returned an invalid JSON response.",
|
||||||
|
outcome_unknown=method == "POST",
|
||||||
|
) from exc
|
||||||
|
if not isinstance(payload, Mapping):
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
"invalid_response",
|
||||||
|
"MediaWiki returned an unsupported response shape.",
|
||||||
|
outcome_unknown=method == "POST",
|
||||||
|
)
|
||||||
|
error = payload.get("error")
|
||||||
|
if isinstance(error, Mapping):
|
||||||
|
code = _optional_text(error.get("code")) or "provider_error"
|
||||||
|
raise MediaWikiTransportError(
|
||||||
|
code,
|
||||||
|
_optional_text(error.get("info")) or "MediaWiki rejected the request.",
|
||||||
|
retryable=code in {"maxlag", "readonly", "ratelimited"},
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _api_url(endpoint_url: str) -> str:
|
||||||
|
normalized = endpoint_url.strip().rstrip("/")
|
||||||
|
parsed = urlsplit(normalized)
|
||||||
|
if parsed.path.endswith("/api.php"):
|
||||||
|
return normalized
|
||||||
|
path = f"{parsed.path.rstrip('/')}/api.php"
|
||||||
|
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_page_url(endpoint_url: str, title: str) -> str:
|
||||||
|
normalized = endpoint_url.strip().rstrip("/")
|
||||||
|
parsed = urlsplit(normalized)
|
||||||
|
root_path = parsed.path
|
||||||
|
if root_path.endswith("/api.php"):
|
||||||
|
root_path = root_path[: -len("/api.php")]
|
||||||
|
base = urlunsplit((parsed.scheme, parsed.netloc, f"{root_path.rstrip('/')}/", "", ""))
|
||||||
|
return urljoin(base, f"wiki/{quote(title.replace(' ', '_'), safe=':_-./~')}")
|
||||||
|
|
||||||
|
|
||||||
|
def _auth_headers(credential: Mapping[str, Any] | None) -> dict[str, str]:
|
||||||
|
if not credential:
|
||||||
|
return {}
|
||||||
|
token = _optional_text(
|
||||||
|
credential.get("bearer_token")
|
||||||
|
or credential.get("access_token")
|
||||||
|
or credential.get("token")
|
||||||
|
)
|
||||||
|
if token:
|
||||||
|
return {"Authorization": f"Bearer {token}"}
|
||||||
|
username = _optional_text(credential.get("username") or credential.get("user"))
|
||||||
|
password = _optional_text(credential.get("password"))
|
||||||
|
if username and password:
|
||||||
|
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||||
|
return {"Authorization": f"Basic {encoded}"}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _list(payload: Mapping[str, Any], *path: str) -> list[Any]:
|
||||||
|
value: Any = payload
|
||||||
|
for part in path:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
return []
|
||||||
|
value = value.get(part)
|
||||||
|
return list(value) if isinstance(value, list) else []
|
||||||
|
|
||||||
|
|
||||||
|
def _continue_token(payload: Mapping[str, Any], name: str) -> str | None:
|
||||||
|
value = payload.get("continue")
|
||||||
|
return _optional_text(value.get(name)) if isinstance(value, Mapping) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _change_cursor(change: Mapping[str, Any]) -> str:
|
||||||
|
for name in ("rcid", "logid", "revid", "old_revid"):
|
||||||
|
if change.get(name) is not None:
|
||||||
|
return f"{name}:{change[name]}"
|
||||||
|
return _optional_text(change.get("timestamp")) or "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_recent_change(change: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
allowed = (
|
||||||
|
"type",
|
||||||
|
"ns",
|
||||||
|
"title",
|
||||||
|
"pageid",
|
||||||
|
"revid",
|
||||||
|
"old_revid",
|
||||||
|
"timestamp",
|
||||||
|
"logtype",
|
||||||
|
"logaction",
|
||||||
|
"tags",
|
||||||
|
)
|
||||||
|
return {key: change[key] for key in allowed if key in change}
|
||||||
|
|
||||||
|
|
||||||
|
def _required_text(value: object, message: str) -> str:
|
||||||
|
normalized = _optional_text(value)
|
||||||
|
if not normalized:
|
||||||
|
raise MediaWikiTransportError("invalid_response", message)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: object) -> str | None:
|
||||||
|
normalized = str(value or "").strip()
|
||||||
|
return normalized or None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"HttpMediaWikiTransport",
|
||||||
|
"MAX_MEDIAWIKI_RESPONSE_BYTES",
|
||||||
|
"MediaWikiChangeBatch",
|
||||||
|
"MediaWikiPublishResult",
|
||||||
|
"MediaWikiTransport",
|
||||||
|
"MediaWikiTransportError",
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Connectors migrations."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Connectors migration revisions."""
|
||||||
+214
@@ -0,0 +1,214 @@
|
|||||||
|
"""governed connector definitions and simulation evidence
|
||||||
|
|
||||||
|
Revision ID: a8d9e0f1b2c3
|
||||||
|
Revises: f7c8d9e0a1b2
|
||||||
|
Create Date: 2026-08-20 12:30:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a8d9e0f1b2c3"
|
||||||
|
down_revision = "f7c8d9e0a1b2"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_definitions",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("definition_key", sa.String(length=160), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("description", sa.Text(), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("current_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_package", sa.String(length=300), nullable=True),
|
||||||
|
sa.Column("local_definition", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_definitions")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"definition_key",
|
||||||
|
name="uq_connector_definition_tenant_key",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definitions_tenant_id"),
|
||||||
|
"connector_definitions",
|
||||||
|
["tenant_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definitions_status"),
|
||||||
|
"connector_definitions",
|
||||||
|
["status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_definitions_tenant_status",
|
||||||
|
"connector_definitions",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_definition_revisions",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("definition_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("specification", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("definition_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("origin", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("package_ref", sa.String(length=300), nullable=True),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["definition_id"],
|
||||||
|
["connector_definitions.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_definition_revisions_definition_id_connector_definitions"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_definition_revisions"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"definition_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_connector_definition_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definition_revisions_definition_id"),
|
||||||
|
"connector_definition_revisions",
|
||||||
|
["definition_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definition_revisions_definition_hash"),
|
||||||
|
"connector_definition_revisions",
|
||||||
|
["definition_hash"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_definition_revisions_created_by"),
|
||||||
|
"connector_definition_revisions",
|
||||||
|
["created_by"],
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"connector_configurations",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("definition_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("endpoint_url", sa.String(length=1500), nullable=True),
|
||||||
|
sa.Column("credential_ref", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("base_definition_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("local_overrides", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("protected_paths", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effective_configuration", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effective_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("ambiguity_policy", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["definition_id"],
|
||||||
|
["connector_definitions.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_configurations_definition_id_connector_definitions"
|
||||||
|
),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_configurations")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"name",
|
||||||
|
name="uq_connector_configuration_tenant_name",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_configurations_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_configurations_definition_id", ["definition_id"]),
|
||||||
|
("ix_connector_configurations_status", ["status"]),
|
||||||
|
("ix_connector_configurations_effective_hash", ["effective_hash"]),
|
||||||
|
("ix_connector_configurations_updated_by", ["updated_by"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_configurations", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_configurations_tenant_status",
|
||||||
|
"connector_configurations",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_simulation_runs",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("configuration_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("mode", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("review_state", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("definition_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("configuration_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("configuration_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("input_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("summary", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effects", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("reviewed_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("review_reason", sa.Text(), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["configuration_id"],
|
||||||
|
["connector_configurations.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_simulation_runs_configuration_id_connector_configurations"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_simulation_runs"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
"mode",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_simulation_run_idempotency",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_simulation_runs_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_simulation_runs_configuration_id", ["configuration_id"]),
|
||||||
|
("ix_connector_simulation_runs_mode", ["mode"]),
|
||||||
|
("ix_connector_simulation_runs_status", ["status"]),
|
||||||
|
("ix_connector_simulation_runs_review_state", ["review_state"]),
|
||||||
|
("ix_connector_simulation_runs_created_by", ["created_by"]),
|
||||||
|
("ix_connector_simulation_runs_reviewed_by", ["reviewed_by"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_simulation_runs", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_simulation_runs_review",
|
||||||
|
"connector_simulation_runs",
|
||||||
|
["tenant_id", "review_state", "created_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("connector_simulation_runs")
|
||||||
|
op.drop_table("connector_configurations")
|
||||||
|
op.drop_table("connector_definition_revisions")
|
||||||
|
op.drop_table("connector_definitions")
|
||||||
+207
@@ -0,0 +1,207 @@
|
|||||||
|
"""MediaWiki and BlueSpice knowledge connector state
|
||||||
|
|
||||||
|
Revision ID: b9e0f1a2c3d4
|
||||||
|
Revises: a8d9e0f1b2c3
|
||||||
|
Create Date: 2026-08-22 14:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "b9e0f1a2c3d4"
|
||||||
|
down_revision = "a8d9e0f1b2c3"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_knowledge_profiles",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("configuration_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("product", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("product_version", sa.String(length=100), nullable=True),
|
||||||
|
sa.Column("desired_maturity", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("discovered_maturity", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("source_authority_mode", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("default_visibility", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("default_acl_tokens", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("namespace_mappings", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("capabilities", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("discovery_revision", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("discovery_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("health_status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("health_details", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_sync_cursor", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("last_high_watermark", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["configuration_id"],
|
||||||
|
["connector_configurations.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_knowledge_profiles_configuration_id_connector_configurations"
|
||||||
|
),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id", name=op.f("pk_connector_knowledge_profiles")
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
name="uq_connector_knowledge_profile_configuration",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_knowledge_profiles_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_knowledge_profiles_configuration_id", ["configuration_id"]),
|
||||||
|
("ix_connector_knowledge_profiles_status", ["status"]),
|
||||||
|
("ix_connector_knowledge_profiles_product", ["product"]),
|
||||||
|
("ix_connector_knowledge_profiles_discovery_revision", ["discovery_revision"]),
|
||||||
|
("ix_connector_knowledge_profiles_health_status", ["health_status"]),
|
||||||
|
("ix_connector_knowledge_profiles_updated_by", ["updated_by"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_knowledge_profiles", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_knowledge_profiles_tenant_status",
|
||||||
|
"connector_knowledge_profiles",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_knowledge_objects",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("profile_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("object_type", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("external_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("external_page_id", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("external_revision_id", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("namespace_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("title", sa.String(length=500), nullable=False),
|
||||||
|
sa.Column("canonical_url", sa.String(length=1500), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("redirect_target_external_id", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("source_revision", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("content_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("visibility", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("acl_tokens", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("mapped_data", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("change_cursor", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("source_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["profile_id"],
|
||||||
|
["connector_knowledge_profiles.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_knowledge_objects_profile_id_connector_knowledge_profiles"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id", name=op.f("pk_connector_knowledge_objects")
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"profile_id",
|
||||||
|
"object_type",
|
||||||
|
"external_id",
|
||||||
|
name="uq_connector_knowledge_object_identity",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_knowledge_objects_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_knowledge_objects_profile_id", ["profile_id"]),
|
||||||
|
("ix_connector_knowledge_objects_object_type", ["object_type"]),
|
||||||
|
("ix_connector_knowledge_objects_external_page_id", ["external_page_id"]),
|
||||||
|
("ix_connector_knowledge_objects_external_revision_id", ["external_revision_id"]),
|
||||||
|
("ix_connector_knowledge_objects_namespace_id", ["namespace_id"]),
|
||||||
|
("ix_connector_knowledge_objects_status", ["status"]),
|
||||||
|
("ix_connector_knowledge_objects_content_hash", ["content_hash"]),
|
||||||
|
("ix_connector_knowledge_objects_change_cursor", ["change_cursor"]),
|
||||||
|
("ix_connector_knowledge_objects_source_updated_at", ["source_updated_at"]),
|
||||||
|
("ix_connector_knowledge_objects_observed_at", ["observed_at"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_knowledge_objects", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_knowledge_objects_tenant_profile_status",
|
||||||
|
"connector_knowledge_objects",
|
||||||
|
["tenant_id", "profile_id", "status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_knowledge_objects_tenant_updated",
|
||||||
|
"connector_knowledge_objects",
|
||||||
|
["tenant_id", "source_updated_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_knowledge_sync_runs",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("profile_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("mode", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("cursor_before", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("cursor_after", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("high_watermark", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("counts", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effects", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["profile_id"],
|
||||||
|
["connector_knowledge_profiles.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_knowledge_sync_runs_profile_id_connector_knowledge_profiles"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id", name=op.f("pk_connector_knowledge_sync_runs")
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"mode",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_knowledge_sync_run_idempotency",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_knowledge_sync_runs_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_knowledge_sync_runs_profile_id", ["profile_id"]),
|
||||||
|
("ix_connector_knowledge_sync_runs_mode", ["mode"]),
|
||||||
|
("ix_connector_knowledge_sync_runs_status", ["status"]),
|
||||||
|
("ix_connector_knowledge_sync_runs_created_by", ["created_by"]),
|
||||||
|
("ix_connector_knowledge_sync_runs_started_at", ["started_at"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_knowledge_sync_runs", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_knowledge_sync_runs_profile_started",
|
||||||
|
"connector_knowledge_sync_runs",
|
||||||
|
["tenant_id", "profile_id", "started_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("connector_knowledge_sync_runs")
|
||||||
|
op.drop_table("connector_knowledge_objects")
|
||||||
|
op.drop_table("connector_knowledge_profiles")
|
||||||
+201
@@ -0,0 +1,201 @@
|
|||||||
|
"""Znuny and OTRS-compatible service-desk connector state
|
||||||
|
|
||||||
|
Revision ID: c0f1a2b3c4d5
|
||||||
|
Revises: b9e0f1a2c3d4
|
||||||
|
Create Date: 2026-08-22 15:15:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "c0f1a2b3c4d5"
|
||||||
|
down_revision = "b9e0f1a2c3d4"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_service_desk_profiles",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("configuration_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("integration_mode", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("product", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("product_version", sa.String(length=100), nullable=True),
|
||||||
|
sa.Column("desired_maturity", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("discovered_maturity", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("source_authority_mode", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("default_visibility", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("default_acl_tokens", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("routes", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("queue_mappings", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("dynamic_field_mappings", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("capabilities", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("discovery_revision", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("discovered_configuration_revision", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("discovered_configuration_hash", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("discovery_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("health_status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("health_details", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_sync_cursor", sa.String(length=4000), nullable=True),
|
||||||
|
sa.Column("last_high_watermark", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["configuration_id"],
|
||||||
|
["connector_configurations.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_service_desk_profiles_configuration_id_connector_configurations"
|
||||||
|
),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_service_desk_profiles")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"configuration_id",
|
||||||
|
name="uq_connector_service_desk_profile_configuration",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_service_desk_profiles_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_service_desk_profiles_configuration_id", ["configuration_id"]),
|
||||||
|
("ix_connector_service_desk_profiles_status", ["status"]),
|
||||||
|
("ix_connector_service_desk_profiles_integration_mode", ["integration_mode"]),
|
||||||
|
("ix_connector_service_desk_profiles_product", ["product"]),
|
||||||
|
("ix_connector_service_desk_profiles_discovery_revision", ["discovery_revision"]),
|
||||||
|
("ix_connector_service_desk_profiles_health_status", ["health_status"]),
|
||||||
|
("ix_connector_service_desk_profiles_updated_by", ["updated_by"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_service_desk_profiles", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_service_desk_profiles_tenant_status",
|
||||||
|
"connector_service_desk_profiles",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_service_desk_objects",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("profile_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("object_type", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("external_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("external_ticket_number", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("title", sa.String(length=500), nullable=False),
|
||||||
|
sa.Column("canonical_url", sa.String(length=1500), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("source_revision", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("content_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("visibility", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("acl_tokens", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("mapped_data", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("change_cursor", sa.String(length=4000), nullable=True),
|
||||||
|
sa.Column("source_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["profile_id"],
|
||||||
|
["connector_service_desk_profiles.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_service_desk_objects_profile_id_connector_service_desk_profiles"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_service_desk_objects")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"profile_id",
|
||||||
|
"object_type",
|
||||||
|
"external_id",
|
||||||
|
name="uq_connector_service_desk_object_identity",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_service_desk_objects_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_service_desk_objects_profile_id", ["profile_id"]),
|
||||||
|
("ix_connector_service_desk_objects_object_type", ["object_type"]),
|
||||||
|
("ix_connector_service_desk_objects_external_ticket_number", ["external_ticket_number"]),
|
||||||
|
("ix_connector_service_desk_objects_status", ["status"]),
|
||||||
|
("ix_connector_service_desk_objects_content_hash", ["content_hash"]),
|
||||||
|
("ix_connector_service_desk_objects_change_cursor", ["change_cursor"]),
|
||||||
|
("ix_connector_service_desk_objects_source_updated_at", ["source_updated_at"]),
|
||||||
|
("ix_connector_service_desk_objects_observed_at", ["observed_at"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_service_desk_objects", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_service_desk_objects_tenant_profile_status",
|
||||||
|
"connector_service_desk_objects",
|
||||||
|
["tenant_id", "profile_id", "status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_service_desk_objects_tenant_updated",
|
||||||
|
"connector_service_desk_objects",
|
||||||
|
["tenant_id", "source_updated_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_service_desk_sync_runs",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("profile_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("mode", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("cursor_before", sa.String(length=4000), nullable=True),
|
||||||
|
sa.Column("cursor_after", sa.String(length=4000), nullable=True),
|
||||||
|
sa.Column("high_watermark", sa.String(length=500), nullable=True),
|
||||||
|
sa.Column("counts", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("effects", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("provenance", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["profile_id"],
|
||||||
|
["connector_service_desk_profiles.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_service_desk_sync_runs_profile_id_connector_service_desk_profiles"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_service_desk_sync_runs")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"profile_id",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_connector_service_desk_sync_run_idempotency",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_connector_service_desk_sync_runs_tenant_id", ["tenant_id"]),
|
||||||
|
("ix_connector_service_desk_sync_runs_profile_id", ["profile_id"]),
|
||||||
|
("ix_connector_service_desk_sync_runs_mode", ["mode"]),
|
||||||
|
("ix_connector_service_desk_sync_runs_status", ["status"]),
|
||||||
|
("ix_connector_service_desk_sync_runs_created_by", ["created_by"]),
|
||||||
|
("ix_connector_service_desk_sync_runs_started_at", ["started_at"]),
|
||||||
|
):
|
||||||
|
op.create_index(op.f(name), "connector_service_desk_sync_runs", columns)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_service_desk_runs_profile_started",
|
||||||
|
"connector_service_desk_sync_runs",
|
||||||
|
["tenant_id", "profile_id", "started_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("connector_service_desk_sync_runs")
|
||||||
|
op.drop_table("connector_service_desk_objects")
|
||||||
|
op.drop_table("connector_service_desk_profiles")
|
||||||
+141
@@ -0,0 +1,141 @@
|
|||||||
|
"""v0.1.14 Connectors baseline
|
||||||
|
|
||||||
|
Revision ID: e6b7c8d9f0a1
|
||||||
|
Revises: None
|
||||||
|
Create Date: 2026-07-28 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "e6b7c8d9f0a1"
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_tabular_sources",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("provider", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("source_name", sa.String(length=120), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("description", sa.Text(), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("schema_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("schema", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("rows", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("fingerprint", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("row_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("byte_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("metadata", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_connector_tabular_sources")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"source_name",
|
||||||
|
name="uq_connector_tabular_source_name",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_created_by"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["created_by"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_deleted_at"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["deleted_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_fingerprint"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["fingerprint"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_provider"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["provider"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_status"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["status"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_tenant_id"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["tenant_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_connector_tabular_sources_updated_by"),
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["updated_by"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_status",
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["tenant_id", "status"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_updated",
|
||||||
|
"connector_tabular_sources",
|
||||||
|
["tenant_id", "updated_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_updated",
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_connector_tabular_sources_tenant_status",
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_updated_by"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_tenant_id"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_status"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_provider"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_fingerprint"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_deleted_at"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_connector_tabular_sources_created_by"),
|
||||||
|
table_name="connector_tabular_sources",
|
||||||
|
)
|
||||||
|
op.drop_table("connector_tabular_sources")
|
||||||
+192
@@ -0,0 +1,192 @@
|
|||||||
|
"""Add immutable sanctions source snapshots.
|
||||||
|
|
||||||
|
Revision ID: f7c8d9e0a1b2
|
||||||
|
Revises: e6b7c8d9f0a1
|
||||||
|
Create Date: 2026-07-29
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "f7c8d9e0a1b2"
|
||||||
|
down_revision = "e6b7c8d9f0a1"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("provider_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(length=200), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("request_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("response_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"started_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"finished_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("snapshot_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_sanctions_acquisition_runs"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for column in (
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"source_id",
|
||||||
|
"status",
|
||||||
|
"started_at",
|
||||||
|
"snapshot_id",
|
||||||
|
"created_by",
|
||||||
|
):
|
||||||
|
op.create_index(
|
||||||
|
op.f(
|
||||||
|
"ix_connector_sanctions_acquisition_runs_"
|
||||||
|
f"{column}"
|
||||||
|
),
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
[column],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_sanctions_run_health",
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
["tenant_id", "provider_id", "status", "started_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("provider_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("publisher", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("jurisdiction", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("list_type", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("source_id", sa.String(length=200), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"source_version",
|
||||||
|
sa.String(length=255),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"publication_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"effective_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"acquired_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("source_url", sa.String(length=1500), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"content_type",
|
||||||
|
sa.String(length=200),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("byte_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("signature_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"parser_version",
|
||||||
|
sa.String(length=100),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("licence_notes", sa.Text(), nullable=True),
|
||||||
|
sa.Column("trust_notes", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"connector_run_id",
|
||||||
|
sa.String(length=36),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("transport_evidence", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("raw_content", sa.LargeBinary(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["connector_run_id"],
|
||||||
|
["connector_sanctions_acquisition_runs.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_connector_sanctions_snapshots_connector_run_id_"
|
||||||
|
"connector_sanctions_acquisition_runs"
|
||||||
|
),
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint(
|
||||||
|
"id",
|
||||||
|
name=op.f("pk_connector_sanctions_snapshots"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"connector_run_id",
|
||||||
|
name="uq_connector_sanctions_snapshot_run",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for column in (
|
||||||
|
"tenant_id",
|
||||||
|
"provider_id",
|
||||||
|
"jurisdiction",
|
||||||
|
"list_type",
|
||||||
|
"source_id",
|
||||||
|
"source_version",
|
||||||
|
"acquired_at",
|
||||||
|
"sha256",
|
||||||
|
"connector_run_id",
|
||||||
|
):
|
||||||
|
op.create_index(
|
||||||
|
op.f(f"ix_connector_sanctions_snapshots_{column}"),
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
[column],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_sanctions_snapshot_source",
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
["tenant_id", "provider_id", "acquired_at"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_connector_sanctions_snapshot_version",
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
["provider_id", "source_id", "source_version"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("connector_sanctions_snapshots")
|
||||||
|
op.drop_table("connector_sanctions_acquisition_runs")
|
||||||
@@ -0,0 +1,550 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from hashlib import sha256
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorKnowledgeObject,
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeSyncRun,
|
||||||
|
ConnectorServiceDeskObject,
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskSyncRun,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.provider_governance import (
|
||||||
|
ExternalProviderRuntimeState,
|
||||||
|
ExternalProviderStateContext,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
TABULAR_PROVIDER_ID = "connectors.tabular_snapshot"
|
||||||
|
SANCTIONS_PROVIDER_ID = "connectors.sanctions_snapshot"
|
||||||
|
KNOWLEDGE_PROVIDER_ID = "connectors.mediawiki.pages"
|
||||||
|
SERVICE_DESK_PROVIDER_ID = "connectors.znuny.tickets"
|
||||||
|
|
||||||
|
|
||||||
|
def tabular_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
session = _session(context)
|
||||||
|
statement = select(ConnectorTabularSource).where(
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None)
|
||||||
|
)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
ConnectorTabularSource.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
sources = tuple(
|
||||||
|
session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
ConnectorTabularSource.tenant_id,
|
||||||
|
ConnectorTabularSource.id,
|
||||||
|
).limit(context.max_items + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(_tabular_state(item, observed_at=observed_at) for item in sources)
|
||||||
|
|
||||||
|
|
||||||
|
def sanctions_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
session = _session(context)
|
||||||
|
statement = select(ConnectorSanctionsAcquisitionRun)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
ConnectorSanctionsAcquisitionRun.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
runs = tuple(
|
||||||
|
session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
ConnectorSanctionsAcquisitionRun.tenant_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.provider_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.source_id,
|
||||||
|
ConnectorSanctionsAcquisitionRun.started_at.desc(),
|
||||||
|
).limit(max(context.max_items * 10, context.max_items + 1))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
latest_by_binding: dict[tuple[str, str, str], ConnectorSanctionsAcquisitionRun] = {}
|
||||||
|
for run in runs:
|
||||||
|
key = (run.tenant_id, run.provider_id, run.source_id)
|
||||||
|
latest_by_binding.setdefault(key, run)
|
||||||
|
if len(latest_by_binding) >= context.max_items + 1:
|
||||||
|
break
|
||||||
|
|
||||||
|
snapshot_counts = _snapshot_counts(
|
||||||
|
session,
|
||||||
|
binding_keys=tuple(latest_by_binding),
|
||||||
|
)
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(
|
||||||
|
_sanctions_state(
|
||||||
|
run,
|
||||||
|
observed_at=observed_at,
|
||||||
|
snapshot_count=snapshot_counts.get(key, 0),
|
||||||
|
)
|
||||||
|
for key, run in latest_by_binding.items()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def knowledge_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
session = _session(context)
|
||||||
|
statement = select(ConnectorKnowledgeProfile)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
ConnectorKnowledgeProfile.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
profiles = tuple(
|
||||||
|
session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
ConnectorKnowledgeProfile.tenant_id,
|
||||||
|
ConnectorKnowledgeProfile.id,
|
||||||
|
).limit(context.max_items + 1)
|
||||||
|
)
|
||||||
|
)[: context.max_items]
|
||||||
|
counts = _knowledge_counts(session, profiles)
|
||||||
|
latest_runs = _latest_knowledge_runs(session, profiles)
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(
|
||||||
|
_knowledge_state(
|
||||||
|
profile,
|
||||||
|
observed_at=observed_at,
|
||||||
|
object_count=counts.get(profile.id, 0),
|
||||||
|
latest_run=latest_runs.get(profile.id),
|
||||||
|
)
|
||||||
|
for profile in profiles
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def service_desk_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
session = _session(context)
|
||||||
|
statement = select(ConnectorServiceDeskProfile)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
ConnectorServiceDeskProfile.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
profiles = tuple(
|
||||||
|
session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
ConnectorServiceDeskProfile.tenant_id,
|
||||||
|
ConnectorServiceDeskProfile.id,
|
||||||
|
).limit(context.max_items + 1)
|
||||||
|
)
|
||||||
|
)[: context.max_items]
|
||||||
|
counts = _service_desk_counts(session, profiles)
|
||||||
|
latest_runs = _latest_service_desk_runs(session, profiles)
|
||||||
|
configurations = _service_desk_configurations(session, profiles)
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(
|
||||||
|
_service_desk_state(
|
||||||
|
profile,
|
||||||
|
observed_at=observed_at,
|
||||||
|
object_count=counts.get(profile.id, 0),
|
||||||
|
latest_run=latest_runs.get(profile.id),
|
||||||
|
configuration=configurations.get(profile.configuration_id),
|
||||||
|
)
|
||||||
|
for profile in profiles
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(context: ExternalProviderStateContext) -> Session:
|
||||||
|
if not isinstance(context.session, Session):
|
||||||
|
raise RuntimeError("Connectors provider state requires a database session.")
|
||||||
|
return context.session
|
||||||
|
|
||||||
|
|
||||||
|
def _tabular_state(
|
||||||
|
source: ConnectorTabularSource,
|
||||||
|
*,
|
||||||
|
observed_at: datetime,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
active = source.status == "active"
|
||||||
|
live_origin = source.provider in {"managed_file", "postgresql"}
|
||||||
|
labels = {
|
||||||
|
"managed_file": "Exact managed-file origin",
|
||||||
|
"postgresql": "Live PostgreSQL origin",
|
||||||
|
"snapshot": "Immutable tabular snapshot",
|
||||||
|
}
|
||||||
|
label = labels.get(source.provider, "Tabular source")
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=TABULAR_PROVIDER_ID,
|
||||||
|
binding_ref=f"connectors:tabular-source:{source.id}",
|
||||||
|
authority_mode="external_mirror",
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=active,
|
||||||
|
health=("unknown" if active and live_origin else "healthy" if active else "inactive"),
|
||||||
|
freshness="unknown" if active and live_origin else "not_applicable",
|
||||||
|
conflict="not_applicable",
|
||||||
|
recovery="ready" if active else "not_applicable",
|
||||||
|
last_success_at=_aware(source.updated_at or source.created_at),
|
||||||
|
detail=(
|
||||||
|
f"{label} is configured; live access and drift are checked on preview."
|
||||||
|
if active and live_origin
|
||||||
|
else f"{label} is available."
|
||||||
|
if active
|
||||||
|
else f"{label} is inactive."
|
||||||
|
),
|
||||||
|
metrics={
|
||||||
|
"provider": source.provider,
|
||||||
|
"row_count": int(source.row_count),
|
||||||
|
"byte_count": int(source.byte_count),
|
||||||
|
"schema_version": int(source.schema_version),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_counts(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
binding_keys: tuple[tuple[str, str, str], ...],
|
||||||
|
) -> dict[tuple[str, str, str], int]:
|
||||||
|
if not binding_keys:
|
||||||
|
return {}
|
||||||
|
tenant_ids = {item[0] for item in binding_keys}
|
||||||
|
rows = session.execute(
|
||||||
|
select(
|
||||||
|
ConnectorSanctionsSnapshot.tenant_id,
|
||||||
|
ConnectorSanctionsSnapshot.provider_id,
|
||||||
|
ConnectorSanctionsSnapshot.source_id,
|
||||||
|
func.count(ConnectorSanctionsSnapshot.id),
|
||||||
|
)
|
||||||
|
.where(ConnectorSanctionsSnapshot.tenant_id.in_(tenant_ids))
|
||||||
|
.group_by(
|
||||||
|
ConnectorSanctionsSnapshot.tenant_id,
|
||||||
|
ConnectorSanctionsSnapshot.provider_id,
|
||||||
|
ConnectorSanctionsSnapshot.source_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
(str(tenant_id), str(provider_id), str(source_id)): int(count)
|
||||||
|
for tenant_id, provider_id, source_id, count in rows
|
||||||
|
if (str(tenant_id), str(provider_id), str(source_id)) in binding_keys
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _knowledge_counts(
|
||||||
|
session: Session,
|
||||||
|
profiles: tuple[ConnectorKnowledgeProfile, ...],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
profile_ids = tuple(item.id for item in profiles)
|
||||||
|
if not profile_ids:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
str(profile_id): int(count)
|
||||||
|
for profile_id, count in session.execute(
|
||||||
|
select(
|
||||||
|
ConnectorKnowledgeObject.profile_id,
|
||||||
|
func.count(ConnectorKnowledgeObject.id),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorKnowledgeObject.profile_id.in_(profile_ids),
|
||||||
|
ConnectorKnowledgeObject.status != "deleted",
|
||||||
|
)
|
||||||
|
.group_by(ConnectorKnowledgeObject.profile_id)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_knowledge_runs(
|
||||||
|
session: Session,
|
||||||
|
profiles: tuple[ConnectorKnowledgeProfile, ...],
|
||||||
|
) -> dict[str, ConnectorKnowledgeSyncRun]:
|
||||||
|
profile_ids = tuple(item.id for item in profiles)
|
||||||
|
if not profile_ids:
|
||||||
|
return {}
|
||||||
|
rows = tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(ConnectorKnowledgeSyncRun)
|
||||||
|
.where(
|
||||||
|
ConnectorKnowledgeSyncRun.profile_id.in_(profile_ids),
|
||||||
|
ConnectorKnowledgeSyncRun.mode.in_(("backfill", "delta")),
|
||||||
|
)
|
||||||
|
.order_by(
|
||||||
|
ConnectorKnowledgeSyncRun.profile_id,
|
||||||
|
ConnectorKnowledgeSyncRun.started_at.desc(),
|
||||||
|
ConnectorKnowledgeSyncRun.id.desc(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
latest: dict[str, ConnectorKnowledgeSyncRun] = {}
|
||||||
|
for row in rows:
|
||||||
|
latest.setdefault(row.profile_id, row)
|
||||||
|
return latest
|
||||||
|
|
||||||
|
|
||||||
|
def _service_desk_counts(
|
||||||
|
session: Session,
|
||||||
|
profiles: tuple[ConnectorServiceDeskProfile, ...],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
profile_ids = tuple(item.id for item in profiles)
|
||||||
|
if not profile_ids:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
str(profile_id): int(count)
|
||||||
|
for profile_id, count in session.execute(
|
||||||
|
select(
|
||||||
|
ConnectorServiceDeskObject.profile_id,
|
||||||
|
func.count(ConnectorServiceDeskObject.id),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorServiceDeskObject.profile_id.in_(profile_ids),
|
||||||
|
ConnectorServiceDeskObject.status != "deleted",
|
||||||
|
)
|
||||||
|
.group_by(ConnectorServiceDeskObject.profile_id)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_service_desk_runs(
|
||||||
|
session: Session,
|
||||||
|
profiles: tuple[ConnectorServiceDeskProfile, ...],
|
||||||
|
) -> dict[str, ConnectorServiceDeskSyncRun]:
|
||||||
|
profile_ids = tuple(item.id for item in profiles)
|
||||||
|
if not profile_ids:
|
||||||
|
return {}
|
||||||
|
rows = tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(ConnectorServiceDeskSyncRun)
|
||||||
|
.where(ConnectorServiceDeskSyncRun.profile_id.in_(profile_ids))
|
||||||
|
.order_by(
|
||||||
|
ConnectorServiceDeskSyncRun.profile_id,
|
||||||
|
ConnectorServiceDeskSyncRun.started_at.desc(),
|
||||||
|
ConnectorServiceDeskSyncRun.id.desc(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
latest: dict[str, ConnectorServiceDeskSyncRun] = {}
|
||||||
|
for row in rows:
|
||||||
|
latest.setdefault(row.profile_id, row)
|
||||||
|
return latest
|
||||||
|
|
||||||
|
|
||||||
|
def _service_desk_configurations(
|
||||||
|
session: Session,
|
||||||
|
profiles: tuple[ConnectorServiceDeskProfile, ...],
|
||||||
|
) -> dict[str, ConnectorConfiguration]:
|
||||||
|
configuration_ids = tuple(
|
||||||
|
dict.fromkeys(profile.configuration_id for profile in profiles)
|
||||||
|
)
|
||||||
|
if not configuration_ids:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
row.id: row
|
||||||
|
for row in session.scalars(
|
||||||
|
select(ConnectorConfiguration).where(
|
||||||
|
ConnectorConfiguration.id.in_(configuration_ids)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _knowledge_state(
|
||||||
|
profile: ConnectorKnowledgeProfile,
|
||||||
|
*,
|
||||||
|
observed_at: datetime,
|
||||||
|
object_count: int,
|
||||||
|
latest_run: ConnectorKnowledgeSyncRun | None,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
active = profile.status == "active"
|
||||||
|
health = (
|
||||||
|
"inactive"
|
||||||
|
if not active
|
||||||
|
else "healthy"
|
||||||
|
if profile.health_status == "healthy"
|
||||||
|
else "warning"
|
||||||
|
if profile.health_status in {"unknown", "degraded"}
|
||||||
|
else "error"
|
||||||
|
)
|
||||||
|
last_success = (
|
||||||
|
latest_run.finished_at
|
||||||
|
if latest_run is not None and latest_run.status == "completed"
|
||||||
|
else profile.discovered_at
|
||||||
|
)
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||||
|
binding_ref=f"connectors:knowledge-profile:{profile.id}",
|
||||||
|
authority_mode=profile.source_authority_mode,
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=active,
|
||||||
|
health=health,
|
||||||
|
freshness="unknown" if active else "not_applicable",
|
||||||
|
conflict=(
|
||||||
|
"pending"
|
||||||
|
if latest_run is not None and latest_run.status == "outcome_unknown"
|
||||||
|
else "not_applicable"
|
||||||
|
),
|
||||||
|
recovery=(
|
||||||
|
"attention"
|
||||||
|
if latest_run is not None
|
||||||
|
and latest_run.status in {"failed", "outcome_unknown"}
|
||||||
|
else "ready"
|
||||||
|
if active
|
||||||
|
else "not_applicable"
|
||||||
|
),
|
||||||
|
last_success_at=_aware(last_success),
|
||||||
|
detail=(
|
||||||
|
f"{profile.product} knowledge profile is synchronized and ACL-rechecked."
|
||||||
|
if active and profile.health_status == "healthy"
|
||||||
|
else "Knowledge profile requires discovery, synchronization, or health review."
|
||||||
|
if active
|
||||||
|
else "Knowledge profile is paused."
|
||||||
|
),
|
||||||
|
metrics={
|
||||||
|
"product": profile.product,
|
||||||
|
"product_version": profile.product_version,
|
||||||
|
"desired_maturity": profile.desired_maturity,
|
||||||
|
"discovered_maturity": profile.discovered_maturity,
|
||||||
|
"active_objects": int(object_count),
|
||||||
|
"last_run_status": latest_run.status if latest_run is not None else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _service_desk_state(
|
||||||
|
profile: ConnectorServiceDeskProfile,
|
||||||
|
*,
|
||||||
|
observed_at: datetime,
|
||||||
|
object_count: int,
|
||||||
|
latest_run: ConnectorServiceDeskSyncRun | None,
|
||||||
|
configuration: ConnectorConfiguration | None,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
configured = configuration is not None
|
||||||
|
active = (
|
||||||
|
profile.status == "active"
|
||||||
|
and configuration is not None
|
||||||
|
and configuration.status == "active"
|
||||||
|
)
|
||||||
|
discovery_current = bool(
|
||||||
|
configuration is not None
|
||||||
|
and profile.discovered_configuration_revision
|
||||||
|
== configuration.resource_revision
|
||||||
|
and profile.discovered_configuration_hash == configuration.effective_hash
|
||||||
|
)
|
||||||
|
health = (
|
||||||
|
"inactive"
|
||||||
|
if not active
|
||||||
|
else "warning"
|
||||||
|
if not discovery_current
|
||||||
|
else "healthy"
|
||||||
|
if profile.health_status == "healthy"
|
||||||
|
else "warning"
|
||||||
|
if profile.health_status in {"unknown", "degraded"}
|
||||||
|
else "error"
|
||||||
|
)
|
||||||
|
last_success = (
|
||||||
|
latest_run.finished_at
|
||||||
|
if latest_run is not None and latest_run.status == "completed"
|
||||||
|
else profile.discovered_at
|
||||||
|
)
|
||||||
|
unresolved = latest_run is not None and latest_run.status == "outcome_unknown"
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=SERVICE_DESK_PROVIDER_ID,
|
||||||
|
binding_ref=f"connectors:service-desk-profile:{profile.id}",
|
||||||
|
authority_mode=profile.source_authority_mode,
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=configured,
|
||||||
|
active=active,
|
||||||
|
health=health,
|
||||||
|
freshness="unknown" if active else "not_applicable",
|
||||||
|
conflict="pending" if unresolved else "not_applicable",
|
||||||
|
recovery=(
|
||||||
|
"attention"
|
||||||
|
if latest_run is not None
|
||||||
|
and latest_run.status in {"failed", "outcome_unknown"}
|
||||||
|
else "ready"
|
||||||
|
if active
|
||||||
|
else "not_applicable"
|
||||||
|
),
|
||||||
|
last_success_at=_aware(last_success),
|
||||||
|
detail=(
|
||||||
|
f"{profile.product} service-desk profile is synchronized and ACL-rechecked."
|
||||||
|
if active and discovery_current and profile.health_status == "healthy"
|
||||||
|
else "Service-desk configuration changed; rediscovery is required."
|
||||||
|
if active and not discovery_current
|
||||||
|
else "Service-desk profile requires discovery, synchronization, or recovery review."
|
||||||
|
if active
|
||||||
|
else "Service-desk profile is paused."
|
||||||
|
),
|
||||||
|
metrics={
|
||||||
|
"product": profile.product,
|
||||||
|
"product_version": profile.product_version,
|
||||||
|
"integration_mode": profile.integration_mode,
|
||||||
|
"desired_maturity": profile.desired_maturity,
|
||||||
|
"discovered_maturity": profile.discovered_maturity,
|
||||||
|
"discovery_current": discovery_current,
|
||||||
|
"active_objects": int(object_count),
|
||||||
|
"last_run_status": latest_run.status if latest_run is not None else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sanctions_state(
|
||||||
|
run: ConnectorSanctionsAcquisitionRun,
|
||||||
|
*,
|
||||||
|
observed_at: datetime,
|
||||||
|
snapshot_count: int,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
status = str(run.status)
|
||||||
|
success = status in {"succeeded", "success", "not_modified"}
|
||||||
|
running = status in {"running", "pending", "retry"}
|
||||||
|
has_snapshot = bool(run.snapshot_id) or snapshot_count > 0
|
||||||
|
health = "healthy" if success else "warning" if running else "error"
|
||||||
|
binding_digest = sha256(
|
||||||
|
f"{run.tenant_id}\0{run.provider_id}\0{run.source_id}".encode("utf-8")
|
||||||
|
).hexdigest()[:24]
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=SANCTIONS_PROVIDER_ID,
|
||||||
|
binding_ref=f"connectors:sanctions-source:{binding_digest}",
|
||||||
|
authority_mode="external_mirror",
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=True,
|
||||||
|
health=health,
|
||||||
|
freshness="unknown",
|
||||||
|
conflict="not_applicable",
|
||||||
|
recovery="ready" if success and has_snapshot else "attention",
|
||||||
|
last_success_at=_aware(run.finished_at) if success else None,
|
||||||
|
detail=(
|
||||||
|
"Latest sanctions acquisition completed."
|
||||||
|
if success
|
||||||
|
else "Sanctions acquisition is in progress."
|
||||||
|
if running
|
||||||
|
else "Latest sanctions acquisition failed; prior accepted snapshots remain separate evidence."
|
||||||
|
),
|
||||||
|
metrics={
|
||||||
|
"latest_status": status,
|
||||||
|
"attempt_count": int(run.attempt_count),
|
||||||
|
"accepted_snapshots": int(snapshot_count),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"KNOWLEDGE_PROVIDER_ID",
|
||||||
|
"SANCTIONS_PROVIDER_ID",
|
||||||
|
"SERVICE_DESK_PROVIDER_ID",
|
||||||
|
"TABULAR_PROVIDER_ID",
|
||||||
|
"sanctions_provider_states",
|
||||||
|
"knowledge_provider_states",
|
||||||
|
"service_desk_provider_states",
|
||||||
|
"tabular_provider_states",
|
||||||
|
]
|
||||||
@@ -0,0 +1,395 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
from typing import Any
|
||||||
|
from uuid import NAMESPACE_URL, uuid4, uuid5
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
DurableRecoveryOperation,
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRecoveryError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConnectorRecoveryDeclaration:
|
||||||
|
operation_type: str
|
||||||
|
mode: RecoveryMode
|
||||||
|
provider_mutation: bool
|
||||||
|
idempotency: str
|
||||||
|
verification: tuple[str, ...]
|
||||||
|
recovery: tuple[str, ...]
|
||||||
|
implemented: bool
|
||||||
|
|
||||||
|
|
||||||
|
CONNECTOR_RECOVERY_OPERATIONS = (
|
||||||
|
ConnectorRecoveryDeclaration(
|
||||||
|
operation_type="read-snapshot",
|
||||||
|
mode=RecoveryMode.ATOMIC,
|
||||||
|
provider_mutation=False,
|
||||||
|
idempotency=(
|
||||||
|
"Caller-supplied request keys replay a committed immutable snapshot; "
|
||||||
|
"otherwise each deliberate acquisition receives a generated key."
|
||||||
|
),
|
||||||
|
verification=(
|
||||||
|
"provider revision or conditional cursor is recorded before fetch",
|
||||||
|
"domain snapshot and terminal recovery checkpoint commit together",
|
||||||
|
"stored bytes and provider evidence are checksum verified",
|
||||||
|
),
|
||||||
|
recovery=(
|
||||||
|
"a stale running transaction is failed after its database transaction rolls back",
|
||||||
|
"a new deliberate acquisition may then use a new request key",
|
||||||
|
),
|
||||||
|
implemented=True,
|
||||||
|
),
|
||||||
|
ConnectorRecoveryDeclaration(
|
||||||
|
operation_type="external-mutation",
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
provider_mutation=True,
|
||||||
|
idempotency="A stable caller key and canonical request digest are mandatory.",
|
||||||
|
verification=(
|
||||||
|
"record the remote revision and bounded provider result",
|
||||||
|
"verify the provider state before reporting success",
|
||||||
|
),
|
||||||
|
recovery=(
|
||||||
|
"unknown outcomes remain unresolved until provider-backed reconciliation",
|
||||||
|
"never retry the same remote effect solely to reconstruct local state",
|
||||||
|
),
|
||||||
|
implemented=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def connector_session_factory(session: Session) -> sessionmaker[Session]:
|
||||||
|
bind = session.get_bind()
|
||||||
|
if bind is None:
|
||||||
|
raise ConnectorRecoveryError("Connector recovery requires a bound database session")
|
||||||
|
return sessionmaker(bind=bind, expire_on_commit=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _digest(value: str) -> str:
|
||||||
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_key(value: str | None) -> str:
|
||||||
|
clean = str(value or "").strip()
|
||||||
|
if clean and len(clean) > 500:
|
||||||
|
raise ConnectorRecoveryError("Connector idempotency keys are limited to 500 characters")
|
||||||
|
return clean or str(uuid4())
|
||||||
|
|
||||||
|
|
||||||
|
def _stable_resource_id(
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
provider_id: str,
|
||||||
|
operation_type: str,
|
||||||
|
request_key: str,
|
||||||
|
) -> str:
|
||||||
|
return str(
|
||||||
|
uuid5(
|
||||||
|
NAMESPACE_URL,
|
||||||
|
f"govoplan:{tenant_id}:{provider_id}:{operation_type}:{request_key}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ConnectorReadSnapshotRecovery:
|
||||||
|
operation: DurableRecoveryOperation | None
|
||||||
|
operation_id: str
|
||||||
|
request_key: str
|
||||||
|
resource_id: str
|
||||||
|
replayed: bool
|
||||||
|
|
||||||
|
def commit_success(self, session: Session, *, evidence: dict[str, Any]) -> None:
|
||||||
|
if self.operation is None:
|
||||||
|
raise ConnectorRecoveryError("A replayed connector read cannot be committed again")
|
||||||
|
try:
|
||||||
|
self.operation.commit_atomic_success(session, evidence=evidence)
|
||||||
|
except Exception as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector snapshot and recovery evidence did not commit atomically"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def commit_failure(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
summary: str,
|
||||||
|
evidence: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
if self.operation is None:
|
||||||
|
raise ConnectorRecoveryError("A replayed connector read cannot be failed again")
|
||||||
|
try:
|
||||||
|
self.operation.commit_atomic_failure(
|
||||||
|
session,
|
||||||
|
summary=summary,
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector failure evidence did not commit atomically"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def fail_without_projection(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
summary: str,
|
||||||
|
code: str,
|
||||||
|
) -> None:
|
||||||
|
if self.operation is None:
|
||||||
|
return
|
||||||
|
self.operation.fail(
|
||||||
|
summary=summary,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {
|
||||||
|
"provider_mutation": False,
|
||||||
|
"projection_committed": False,
|
||||||
|
"failure_code": code,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def begin_connector_read_snapshot(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
provider_id: str,
|
||||||
|
idempotency_key: str | None,
|
||||||
|
source_revision: str | None,
|
||||||
|
cursor: str | None,
|
||||||
|
dry_run_evidence: dict[str, Any],
|
||||||
|
request_metadata: dict[str, Any] | None = None,
|
||||||
|
resource_type: str = "connector_sync_run",
|
||||||
|
) -> ConnectorReadSnapshotRecovery:
|
||||||
|
request_key = _clean_key(idempotency_key)
|
||||||
|
resource_id = _stable_resource_id(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
provider_id=provider_id,
|
||||||
|
operation_type="read-snapshot",
|
||||||
|
request_key=request_key,
|
||||||
|
)
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
"request_key_sha256": _digest(request_key),
|
||||||
|
**dict(request_metadata or {}),
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
connector_session_factory(session),
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="connectors",
|
||||||
|
operation_type="read-snapshot",
|
||||||
|
idempotency_key=f"connector-read:{_digest(f'{tenant_id}:{provider_id}:{request_key}')}",
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.ATOMIC,
|
||||||
|
preconditions=(
|
||||||
|
"the actor is authorized for the connector source",
|
||||||
|
"the provider request is read-only",
|
||||||
|
"the source revision, cursor, and dry-run decision are durable",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"validate the bounded provider response and source revision",
|
||||||
|
"commit the immutable snapshot and terminal checkpoint atomically",
|
||||||
|
"compare the stored content digest with the acquired bytes",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"cursor_sha256": _digest(cursor) if cursor else None,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
"provider_mutation": False,
|
||||||
|
},
|
||||||
|
lease_resource_key=f"connectors:read:{tenant_id}:{_digest(provider_id)[:40]}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "external-provider"],
|
||||||
|
"provider_mutation": False,
|
||||||
|
"recovery_declaration": "read-snapshot",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except RecoveryOperationBusy as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"Another runtime is already acquiring this connector source"
|
||||||
|
) from exc
|
||||||
|
except RecoveryOperationStateConflict as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"This connector request is active or unresolved; reconcile it before retrying"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector recovery ledger is unavailable; the provider was not contacted"
|
||||||
|
) from exc
|
||||||
|
return ConnectorReadSnapshotRecovery(
|
||||||
|
operation=started.operation,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
request_key=request_key,
|
||||||
|
resource_id=resource_id,
|
||||||
|
replayed=started.replayed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ConnectorExternalMutationRecovery:
|
||||||
|
operation: DurableRecoveryOperation | None
|
||||||
|
operation_id: str
|
||||||
|
replayed: bool
|
||||||
|
|
||||||
|
def succeed(self, *, provider_evidence: dict[str, Any]) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.succeed(evidence=provider_evidence)
|
||||||
|
|
||||||
|
def commit_success(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
provider_evidence: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.commit_verified_success(
|
||||||
|
session,
|
||||||
|
evidence=provider_evidence,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
def reject(self, *, summary: str, provider_code: str) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.reject(
|
||||||
|
summary=summary,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {"provider_rejection": provider_code},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def outcome_unknown(self, *, summary: str, provider_code: str) -> None:
|
||||||
|
if self.operation is not None:
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary=summary,
|
||||||
|
evidence={"effect_started": True, "provider_code": provider_code},
|
||||||
|
failure_summary="Inspect provider state before any retry",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def begin_connector_external_mutation(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
provider_id: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
request_sha256: str,
|
||||||
|
source_revision: str | None,
|
||||||
|
cursor: str | None,
|
||||||
|
dry_run_evidence: dict[str, Any],
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
) -> ConnectorExternalMutationRecovery:
|
||||||
|
if not str(idempotency_key or "").strip():
|
||||||
|
raise ConnectorRecoveryError("External connector mutations require an idempotency key")
|
||||||
|
request_key = _clean_key(idempotency_key)
|
||||||
|
if len(request_sha256) != 64 or any(
|
||||||
|
character not in "0123456789abcdefABCDEF" for character in request_sha256
|
||||||
|
):
|
||||||
|
raise ConnectorRecoveryError("External connector mutations require a SHA-256 request digest")
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
connector_session_factory(session),
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="connectors",
|
||||||
|
operation_type="external-mutation",
|
||||||
|
idempotency_key=f"connector-write:{_digest(f'{tenant_id}:{provider_id}:{request_key}')}",
|
||||||
|
request={
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"provider_id": provider_id,
|
||||||
|
"request_sha256": request_sha256,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"cursor_sha256": _digest(cursor) if cursor else None,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
},
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the actor and effective connector policy authorize the mutation",
|
||||||
|
"a stable idempotency key and canonical request digest are present",
|
||||||
|
"the dry-run and source revision evidence are durable",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"inspect provider state without repeating the mutation",
|
||||||
|
"record whether the provider accepted the requested revision",
|
||||||
|
"retry only under a new deliberate key when absence is proven",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"compare provider identity and revision with the canonical request",
|
||||||
|
"verify the consuming domain state independently",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"request_sha256": request_sha256,
|
||||||
|
"source_revision": source_revision,
|
||||||
|
"cursor_sha256": _digest(cursor) if cursor else None,
|
||||||
|
"dry_run": dry_run_evidence,
|
||||||
|
"provider_mutation": True,
|
||||||
|
},
|
||||||
|
lease_resource_key=(
|
||||||
|
f"connectors:write:{tenant_id}:{_digest(provider_id)[:24]}:"
|
||||||
|
f"{_digest(resource_id)[:24]}"
|
||||||
|
),
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "queue", "external-provider"],
|
||||||
|
"provider_mutation": True,
|
||||||
|
"recovery_declaration": "external-mutation",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"This external connector effect is active or unresolved"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
raise ConnectorRecoveryError(
|
||||||
|
"The connector recovery ledger is unavailable; no external mutation started"
|
||||||
|
) from exc
|
||||||
|
return ConnectorExternalMutationRecovery(
|
||||||
|
operation=started.operation,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
replayed=started.replayed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CONNECTOR_RECOVERY_OPERATIONS",
|
||||||
|
"ConnectorExternalMutationRecovery",
|
||||||
|
"ConnectorReadSnapshotRecovery",
|
||||||
|
"ConnectorRecoveryDeclaration",
|
||||||
|
"ConnectorRecoveryError",
|
||||||
|
"begin_connector_external_mutation",
|
||||||
|
"begin_connector_read_snapshot",
|
||||||
|
"connector_session_factory",
|
||||||
|
]
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,305 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
|
class FeedAcquireRequest(BaseModel):
|
||||||
|
url: str = Field(min_length=1, max_length=2000)
|
||||||
|
max_entries: int = Field(default=2_000, ge=1, le=10_000)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedImportRequest(FeedAcquireRequest):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
source_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=120,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||||
|
)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedEntryPayload(BaseModel):
|
||||||
|
id: str = Field(min_length=1, max_length=2000)
|
||||||
|
title: str = Field(min_length=1, max_length=1000)
|
||||||
|
url: str | None = Field(default=None, max_length=2000)
|
||||||
|
summary: str | None = None
|
||||||
|
content: str | None = None
|
||||||
|
author: str | None = Field(default=None, max_length=500)
|
||||||
|
published_at: datetime | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
categories: list[str] = Field(default_factory=list, max_length=100)
|
||||||
|
enclosures: list[dict[str, Any]] = Field(default_factory=list, max_length=100)
|
||||||
|
visibility: Literal["public", "tenant", "private"] = "public"
|
||||||
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedPublicationEntryPayload(FeedEntryPayload):
|
||||||
|
source_kind: Literal["event", "publication", "case", "report"]
|
||||||
|
source_module: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=100,
|
||||||
|
pattern=r"^[a-z][a-z0-9_]*$",
|
||||||
|
)
|
||||||
|
source_ref: str = Field(min_length=1, max_length=500)
|
||||||
|
source_revision: str | None = Field(default=None, max_length=200)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedDocumentResponse(BaseModel):
|
||||||
|
format: Literal["rss", "atom"]
|
||||||
|
title: str
|
||||||
|
source_url: str
|
||||||
|
description: str | None = None
|
||||||
|
home_url: str | None = None
|
||||||
|
language: str | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
acquired_at: datetime | None = None
|
||||||
|
fresh_until: datetime | None = None
|
||||||
|
etag: str | None = None
|
||||||
|
last_modified: str | None = None
|
||||||
|
content_type: str | None = None
|
||||||
|
sha256: str
|
||||||
|
entries: list[FeedEntryPayload]
|
||||||
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class FeedRenderPayload(BaseModel):
|
||||||
|
format: Literal["rss", "atom"]
|
||||||
|
title: str = Field(min_length=1, max_length=1000)
|
||||||
|
feed_url: str = Field(min_length=1, max_length=2000)
|
||||||
|
home_url: str = Field(min_length=1, max_length=2000)
|
||||||
|
description: str | None = None
|
||||||
|
language: str | None = Field(default=None, max_length=100)
|
||||||
|
entries: list[FeedPublicationEntryPayload] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
max_length=10_000,
|
||||||
|
)
|
||||||
|
audience: Literal["public", "tenant", "private"] = "public"
|
||||||
|
|
||||||
|
|
||||||
|
class SnapshotCreateRequest(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
source_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=120,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||||
|
)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
format: Literal["json", "csv"] = "json"
|
||||||
|
rows: list[dict[str, Any]] | None = Field(default=None, max_length=10_000)
|
||||||
|
csv_text: str | None = Field(default=None, max_length=5_000_000)
|
||||||
|
delimiter: Literal[",", ";", "\t", "|"] = ","
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_payload(self) -> "SnapshotCreateRequest":
|
||||||
|
if self.format == "json" and self.rows is None:
|
||||||
|
raise ValueError("JSON snapshots require rows.")
|
||||||
|
if self.format == "json" and self.csv_text is not None:
|
||||||
|
raise ValueError("JSON snapshots cannot include CSV text.")
|
||||||
|
if self.format == "csv" and not self.csv_text:
|
||||||
|
raise ValueError("CSV snapshots require CSV text.")
|
||||||
|
if self.format == "csv" and self.rows is not None:
|
||||||
|
raise ValueError("CSV snapshots cannot include JSON rows.")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ManagedFileSourceCreateRequest(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
source_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=120,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||||
|
)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
file_asset_id: str = Field(min_length=1, max_length=36)
|
||||||
|
file_version_id: str | None = Field(default=None, min_length=1, max_length=36)
|
||||||
|
delimiter: Literal[",", ";", "\t", "|"] = ","
|
||||||
|
sheet_name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||||
|
|
||||||
|
|
||||||
|
class SqlSourceCreateRequest(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
source_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=120,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||||
|
)
|
||||||
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
configuration_id: str = Field(min_length=1, max_length=36)
|
||||||
|
schema_name: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_$-]{0,127}$",
|
||||||
|
)
|
||||||
|
table_name: str = Field(
|
||||||
|
min_length=1,
|
||||||
|
max_length=128,
|
||||||
|
pattern=r"^[A-Za-z_][A-Za-z0-9_$-]{0,127}$",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TabularColumnResponse(BaseModel):
|
||||||
|
name: str
|
||||||
|
data_type: str
|
||||||
|
nullable: bool
|
||||||
|
|
||||||
|
|
||||||
|
class TabularPushdownResponse(BaseModel):
|
||||||
|
projections: bool
|
||||||
|
pagination: bool
|
||||||
|
filters: list[str]
|
||||||
|
aggregations: list[str]
|
||||||
|
sorting: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularHealthResponse(BaseModel):
|
||||||
|
status: Literal["healthy", "warning", "error", "unknown"]
|
||||||
|
code: str
|
||||||
|
summary: str
|
||||||
|
checked_at: str | None
|
||||||
|
details: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularPreviewDiagnosticResponse(BaseModel):
|
||||||
|
severity: Literal["info", "warning", "error"]
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
details: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourceResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
provider: str
|
||||||
|
source_name: str
|
||||||
|
name: str
|
||||||
|
description: str | None
|
||||||
|
columns: list[TabularColumnResponse]
|
||||||
|
schema_version: str
|
||||||
|
fingerprint: str
|
||||||
|
row_count: int | None
|
||||||
|
byte_count: int | None
|
||||||
|
updated_at: str | None
|
||||||
|
capabilities: list[str]
|
||||||
|
metadata: dict[str, Any]
|
||||||
|
source_mode: Literal["live", "cached", "file_backed", "static"]
|
||||||
|
pushdown: TabularPushdownResponse
|
||||||
|
health: TabularHealthResponse
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourceListResponse(BaseModel):
|
||||||
|
sources: list[TabularSourceResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourcePreviewResponse(BaseModel):
|
||||||
|
source: TabularSourceResponse
|
||||||
|
rows: list[dict[str, Any]]
|
||||||
|
total_rows: int
|
||||||
|
truncated: bool
|
||||||
|
returned_bytes: int
|
||||||
|
elapsed_ms: int
|
||||||
|
effective_row_limit: int
|
||||||
|
effective_byte_limit: int
|
||||||
|
effective_timeout_ms: int
|
||||||
|
diagnostics: list[TabularPreviewDiagnosticResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class TabularSourceDeleteResponse(BaseModel):
|
||||||
|
deleted: bool
|
||||||
|
source_ref: str
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSourceResponse(BaseModel):
|
||||||
|
provider_id: str
|
||||||
|
publisher: str
|
||||||
|
jurisdiction: str
|
||||||
|
list_type: str
|
||||||
|
source_id: str
|
||||||
|
source_url: str | None
|
||||||
|
parser_version: str
|
||||||
|
licence_notes: str
|
||||||
|
trust_notes: str
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSourceListResponse(BaseModel):
|
||||||
|
sources: list[SanctionsSourceResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSnapshotResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
provider_id: str
|
||||||
|
publisher: str
|
||||||
|
jurisdiction: str
|
||||||
|
list_type: str
|
||||||
|
source_id: str
|
||||||
|
source_version: str
|
||||||
|
publication_at: datetime | None
|
||||||
|
effective_at: datetime | None
|
||||||
|
acquired_at: datetime
|
||||||
|
content_type: str
|
||||||
|
byte_count: int
|
||||||
|
sha256: str
|
||||||
|
parser_version: str
|
||||||
|
raw_evidence_ref: str
|
||||||
|
connector_run_id: str
|
||||||
|
signature_evidence: dict[str, Any]
|
||||||
|
licence_notes: str | None
|
||||||
|
trust_notes: str | None
|
||||||
|
transport_evidence: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSnapshotListResponse(BaseModel):
|
||||||
|
snapshots: list[SanctionsSnapshotResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsAcquisitionRunResponse(BaseModel):
|
||||||
|
id: str
|
||||||
|
provider_id: str
|
||||||
|
source_id: str
|
||||||
|
status: str
|
||||||
|
attempt_count: int
|
||||||
|
request_evidence: dict[str, Any]
|
||||||
|
response_evidence: dict[str, Any]
|
||||||
|
started_at: datetime
|
||||||
|
finished_at: datetime | None
|
||||||
|
snapshot_id: str | None
|
||||||
|
error: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsAcquisitionRunListResponse(BaseModel):
|
||||||
|
runs: list[SanctionsAcquisitionRunResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsRefreshResponse(BaseModel):
|
||||||
|
run_id: str
|
||||||
|
provider_id: str
|
||||||
|
status: str
|
||||||
|
snapshot: SanctionsSnapshotResponse | None
|
||||||
|
error: str | None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FeedAcquireRequest",
|
||||||
|
"FeedDocumentResponse",
|
||||||
|
"FeedEntryPayload",
|
||||||
|
"FeedImportRequest",
|
||||||
|
"FeedRenderPayload",
|
||||||
|
"ManagedFileSourceCreateRequest",
|
||||||
|
"SnapshotCreateRequest",
|
||||||
|
"SqlSourceCreateRequest",
|
||||||
|
"SanctionsAcquisitionRunListResponse",
|
||||||
|
"SanctionsAcquisitionRunResponse",
|
||||||
|
"SanctionsRefreshResponse",
|
||||||
|
"SanctionsSnapshotListResponse",
|
||||||
|
"SanctionsSnapshotResponse",
|
||||||
|
"SanctionsSourceListResponse",
|
||||||
|
"SanctionsSourceResponse",
|
||||||
|
"TabularColumnResponse",
|
||||||
|
"TabularHealthResponse",
|
||||||
|
"TabularPreviewDiagnosticResponse",
|
||||||
|
"TabularPushdownResponse",
|
||||||
|
"TabularSourceDeleteResponse",
|
||||||
|
"TabularSourceListResponse",
|
||||||
|
"TabularSourcePreviewResponse",
|
||||||
|
"TabularSourceResponse",
|
||||||
|
]
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,371 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
import json
|
||||||
|
from typing import Any, Literal
|
||||||
|
from urllib.parse import parse_qsl, urlsplit
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
|
IntegrationMode = Literal["link", "import", "synchronize"]
|
||||||
|
ServiceDeskMaturity = Literal[
|
||||||
|
"discover",
|
||||||
|
"link",
|
||||||
|
"search",
|
||||||
|
"read",
|
||||||
|
"publish",
|
||||||
|
"synchronize",
|
||||||
|
]
|
||||||
|
|
||||||
|
_CREDENTIAL_CONTROL_KEYS = {
|
||||||
|
"authorization",
|
||||||
|
"auth_mode",
|
||||||
|
"sessionid",
|
||||||
|
"userlogin",
|
||||||
|
"customeruserlogin",
|
||||||
|
"password",
|
||||||
|
"x-otrs-header-sessionid",
|
||||||
|
"x-otrs-header-userlogin",
|
||||||
|
"x-otrs-header-customeruserlogin",
|
||||||
|
"x-otrs-header-password",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskRouteMapping(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
search_path: str = Field(default="/Ticket/Search", min_length=1, max_length=500)
|
||||||
|
ticket_path: str = Field(default="/Ticket/{ticket_id}", min_length=1, max_length=500)
|
||||||
|
update_path: str | None = Field(default=None, max_length=500)
|
||||||
|
search_method: Literal["GET", "POST"] = "POST"
|
||||||
|
ticket_method: Literal["GET", "POST"] = "GET"
|
||||||
|
update_method: Literal["PATCH", "POST", "PUT"] = "PATCH"
|
||||||
|
ticket_web_url_template: str | None = Field(default=None, max_length=1500)
|
||||||
|
search_filters: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_templates(self) -> "ServiceDeskRouteMapping":
|
||||||
|
for field_name in ("search_path", "ticket_path", "update_path"):
|
||||||
|
value = getattr(self, field_name)
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
parsed = urlsplit(value)
|
||||||
|
if parsed.scheme or parsed.netloc or parsed.username or parsed.password:
|
||||||
|
raise ValueError(f"{field_name} must be relative to the governed endpoint")
|
||||||
|
if _credential_query_keys(parsed.query):
|
||||||
|
raise ValueError(f"{field_name} cannot contain authentication controls")
|
||||||
|
if any(part == ".." for part in parsed.path.split("/")):
|
||||||
|
raise ValueError(f"{field_name} cannot traverse parent paths")
|
||||||
|
if "{ticket_id}" not in self.ticket_path:
|
||||||
|
raise ValueError("ticket_path must contain {ticket_id}")
|
||||||
|
if self.update_path is not None and "{ticket_id}" not in self.update_path:
|
||||||
|
raise ValueError("update_path must contain {ticket_id}")
|
||||||
|
if (
|
||||||
|
self.ticket_web_url_template is not None
|
||||||
|
and "{ticket_id}" not in self.ticket_web_url_template
|
||||||
|
and "{ticket_number}" not in self.ticket_web_url_template
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"ticket_web_url_template must contain {ticket_id} or {ticket_number}"
|
||||||
|
)
|
||||||
|
if self.ticket_web_url_template is not None:
|
||||||
|
parsed = urlsplit(self.ticket_web_url_template)
|
||||||
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||||
|
raise ValueError("ticket_web_url_template must be an absolute HTTP(S) URL")
|
||||||
|
if parsed.username or parsed.password:
|
||||||
|
raise ValueError("ticket_web_url_template cannot contain credentials")
|
||||||
|
if _credential_query_keys(parsed.query):
|
||||||
|
raise ValueError(
|
||||||
|
"ticket_web_url_template cannot contain authentication controls"
|
||||||
|
)
|
||||||
|
if len(self.search_filters) > 100:
|
||||||
|
raise ValueError("search_filters supports at most 100 governed criteria")
|
||||||
|
reserved = _CREDENTIAL_CONTROL_KEYS | {
|
||||||
|
"limit",
|
||||||
|
"sortby",
|
||||||
|
"orderby",
|
||||||
|
"ticketchangetimenewerdate",
|
||||||
|
}
|
||||||
|
filter_names = {str(value).strip().casefold() for value in self.search_filters}
|
||||||
|
if "" in filter_names:
|
||||||
|
raise ValueError("search_filters keys cannot be empty")
|
||||||
|
if reserved.intersection(filter_names):
|
||||||
|
raise ValueError("search_filters cannot override cursors, bounds, ordering, or authentication")
|
||||||
|
if len(json.dumps(self.search_filters, default=str)) > 20_000:
|
||||||
|
raise ValueError("search_filters exceeds the 20000-character policy limit")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskQueueMapping(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
source_queue: str = Field(min_length=1, max_length=300)
|
||||||
|
target_queue_ref: str | None = Field(default=None, max_length=255)
|
||||||
|
include: bool = True
|
||||||
|
visibility: Literal["tenant", "restricted"] = "restricted"
|
||||||
|
acl_tokens: list[str] = Field(default_factory=list, max_length=200)
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskDynamicFieldMapping(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
source_name: str = Field(min_length=1, max_length=255)
|
||||||
|
target_name: str | None = Field(default=None, max_length=255)
|
||||||
|
include: bool = True
|
||||||
|
value_type: Literal["string", "number", "boolean", "date", "json"] = "string"
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskProfileCreateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
configuration_id: str = Field(min_length=1, max_length=36)
|
||||||
|
integration_mode: IntegrationMode = "synchronize"
|
||||||
|
desired_maturity: ServiceDeskMaturity = "synchronize"
|
||||||
|
source_authority_mode: Literal[
|
||||||
|
"external_authoritative",
|
||||||
|
"external_mirror",
|
||||||
|
"governed_sync",
|
||||||
|
"linked_reference",
|
||||||
|
] = "external_authoritative"
|
||||||
|
default_visibility: Literal["tenant", "restricted"] = "restricted"
|
||||||
|
default_acl_tokens: list[str] = Field(default_factory=list, max_length=200)
|
||||||
|
routes: ServiceDeskRouteMapping = Field(default_factory=ServiceDeskRouteMapping)
|
||||||
|
queue_mappings: list[ServiceDeskQueueMapping] = Field(
|
||||||
|
default_factory=list, max_length=500
|
||||||
|
)
|
||||||
|
dynamic_field_mappings: list[ServiceDeskDynamicFieldMapping] = Field(
|
||||||
|
default_factory=list, max_length=500
|
||||||
|
)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_policy(self) -> "ServiceDeskProfileCreateRequest":
|
||||||
|
_validate_profile_policy(
|
||||||
|
integration_mode=self.integration_mode,
|
||||||
|
authority_mode=self.source_authority_mode,
|
||||||
|
visibility=self.default_visibility,
|
||||||
|
acl_tokens=self.default_acl_tokens,
|
||||||
|
)
|
||||||
|
order = ("discover", "link", "search", "read", "publish", "synchronize")
|
||||||
|
maximum = {"link": "search", "import": "read", "synchronize": "synchronize"}[
|
||||||
|
self.integration_mode
|
||||||
|
]
|
||||||
|
minimum = {"link": "link", "import": "read", "synchronize": "synchronize"}[
|
||||||
|
self.integration_mode
|
||||||
|
]
|
||||||
|
if order.index(self.desired_maturity) < order.index(minimum):
|
||||||
|
raise ValueError(
|
||||||
|
f"{self.integration_mode} mode requires at least {minimum} maturity"
|
||||||
|
)
|
||||||
|
if order.index(self.desired_maturity) > order.index(maximum):
|
||||||
|
raise ValueError(
|
||||||
|
f"{self.integration_mode} mode cannot declare maturity above {maximum}"
|
||||||
|
)
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskProfileUpdateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
expected_resource_revision: int = Field(ge=1)
|
||||||
|
status: Literal["active", "paused"] | None = None
|
||||||
|
integration_mode: IntegrationMode | None = None
|
||||||
|
desired_maturity: ServiceDeskMaturity | None = None
|
||||||
|
source_authority_mode: Literal[
|
||||||
|
"external_authoritative",
|
||||||
|
"external_mirror",
|
||||||
|
"governed_sync",
|
||||||
|
"linked_reference",
|
||||||
|
] | None = None
|
||||||
|
default_visibility: Literal["tenant", "restricted"] | None = None
|
||||||
|
default_acl_tokens: list[str] | None = Field(default=None, max_length=200)
|
||||||
|
routes: ServiceDeskRouteMapping | None = None
|
||||||
|
queue_mappings: list[ServiceDeskQueueMapping] | None = Field(
|
||||||
|
default=None, max_length=500
|
||||||
|
)
|
||||||
|
dynamic_field_mappings: list[ServiceDeskDynamicFieldMapping] | None = Field(
|
||||||
|
default=None, max_length=500
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskDiagnostic(BaseModel):
|
||||||
|
severity: Literal["info", "warning", "error"]
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
object_ref: str | None = None
|
||||||
|
field: str | None = None
|
||||||
|
retryable: bool = False
|
||||||
|
details: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskProfileItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
configuration_id: str
|
||||||
|
status: str
|
||||||
|
integration_mode: str
|
||||||
|
product: str
|
||||||
|
product_version: str | None = None
|
||||||
|
desired_maturity: str
|
||||||
|
discovered_maturity: str
|
||||||
|
source_authority_mode: str
|
||||||
|
default_visibility: str
|
||||||
|
default_acl_tokens: list[str]
|
||||||
|
routes: ServiceDeskRouteMapping
|
||||||
|
queue_mappings: list[ServiceDeskQueueMapping]
|
||||||
|
dynamic_field_mappings: list[ServiceDeskDynamicFieldMapping]
|
||||||
|
capabilities: list[str]
|
||||||
|
discovery_revision: str | None = None
|
||||||
|
discovered_configuration_revision: int | None = None
|
||||||
|
discovered_configuration_hash: str | None = None
|
||||||
|
health_status: str
|
||||||
|
health_details: dict[str, Any]
|
||||||
|
discovered_at: datetime | None = None
|
||||||
|
last_sync_cursor: str | None = None
|
||||||
|
last_high_watermark: str | None = None
|
||||||
|
resource_revision: int
|
||||||
|
credential_reference_present: bool
|
||||||
|
endpoint_configured: bool
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskProfileListResponse(BaseModel):
|
||||||
|
items: list[ServiceDeskProfileItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskDiscoveryResponse(BaseModel):
|
||||||
|
profile: ServiceDeskProfileItem
|
||||||
|
product: str
|
||||||
|
product_version: str | None = None
|
||||||
|
api_family: str
|
||||||
|
capabilities: list[str]
|
||||||
|
maturity: str
|
||||||
|
health_status: str
|
||||||
|
diagnostics: list[ServiceDeskDiagnostic]
|
||||||
|
revision: str
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskSyncRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
mode: Literal["auto", "full", "delta"] = "auto"
|
||||||
|
cursor: str | None = Field(default=None, max_length=4000)
|
||||||
|
limit: int = Field(default=100, ge=1, le=500)
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskObjectItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
profile_id: str
|
||||||
|
object_type: str
|
||||||
|
external_id: str
|
||||||
|
external_ticket_number: str | None = None
|
||||||
|
title: str
|
||||||
|
canonical_url: str | None = None
|
||||||
|
status: str
|
||||||
|
source_revision: str
|
||||||
|
visibility: str
|
||||||
|
acl_tokens: list[str]
|
||||||
|
mapped_data: dict[str, Any]
|
||||||
|
provenance: dict[str, Any]
|
||||||
|
source_updated_at: datetime | None = None
|
||||||
|
observed_at: datetime
|
||||||
|
resource_revision: int
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskObjectListResponse(BaseModel):
|
||||||
|
items: list[ServiceDeskObjectItem]
|
||||||
|
next_cursor: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskSyncRunItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
profile_id: str
|
||||||
|
mode: str
|
||||||
|
idempotency_key: str
|
||||||
|
status: str
|
||||||
|
cursor_before: str | None = None
|
||||||
|
cursor_after: str | None = None
|
||||||
|
high_watermark: str | None = None
|
||||||
|
counts: dict[str, int]
|
||||||
|
effects: list[dict[str, Any]]
|
||||||
|
diagnostics: list[ServiceDeskDiagnostic]
|
||||||
|
provenance: dict[str, Any]
|
||||||
|
started_at: datetime
|
||||||
|
finished_at: datetime | None = None
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskSyncRunListResponse(BaseModel):
|
||||||
|
items: list[ServiceDeskSyncRunItem]
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskTicketUpdateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
||||||
|
expected_external_revision: str = Field(min_length=1, max_length=255)
|
||||||
|
title: str | None = Field(default=None, min_length=1, max_length=500)
|
||||||
|
queue: str | None = Field(default=None, min_length=1, max_length=300)
|
||||||
|
state: str | None = Field(default=None, min_length=1, max_length=200)
|
||||||
|
priority: str | None = Field(default=None, min_length=1, max_length=200)
|
||||||
|
owner: str | None = Field(default=None, min_length=1, max_length=255)
|
||||||
|
responsible: str | None = Field(default=None, min_length=1, max_length=255)
|
||||||
|
dynamic_fields: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def require_change(self) -> "ServiceDeskTicketUpdateRequest":
|
||||||
|
if not any(
|
||||||
|
(
|
||||||
|
self.title,
|
||||||
|
self.queue,
|
||||||
|
self.state,
|
||||||
|
self.priority,
|
||||||
|
self.owner,
|
||||||
|
self.responsible,
|
||||||
|
self.dynamic_fields,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError("At least one supported ticket field must change")
|
||||||
|
if len(self.dynamic_fields) > 100:
|
||||||
|
raise ValueError("At most 100 dynamic fields may be updated")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskTicketUpdateResponse(BaseModel):
|
||||||
|
run: ServiceDeskSyncRunItem
|
||||||
|
object: ServiceDeskObjectItem
|
||||||
|
accepted: bool
|
||||||
|
outcome_unknown: bool
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_profile_policy(
|
||||||
|
*,
|
||||||
|
integration_mode: str,
|
||||||
|
authority_mode: str,
|
||||||
|
visibility: str,
|
||||||
|
acl_tokens: list[str],
|
||||||
|
) -> None:
|
||||||
|
allowed = {
|
||||||
|
"link": {"linked_reference"},
|
||||||
|
"import": {"external_authoritative", "external_mirror"},
|
||||||
|
"synchronize": {"external_authoritative", "governed_sync"},
|
||||||
|
}
|
||||||
|
if authority_mode not in allowed[integration_mode]:
|
||||||
|
raise ValueError(
|
||||||
|
f"{integration_mode} mode does not support {authority_mode} authority"
|
||||||
|
)
|
||||||
|
if visibility == "restricted" and not acl_tokens:
|
||||||
|
raise ValueError("Restricted profiles require at least one ACL token")
|
||||||
|
|
||||||
|
|
||||||
|
def _credential_query_keys(query: str) -> set[str]:
|
||||||
|
return {
|
||||||
|
str(key).strip().casefold()
|
||||||
|
for key, _value in parse_qsl(query, keep_blank_values=True)
|
||||||
|
if str(key).strip().casefold() in _CREDENTIAL_CONTROL_KEYS
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [name for name in globals() if name.startswith("ServiceDesk")]
|
||||||
@@ -0,0 +1,337 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.external_references import ExternalObjectReference
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillPage,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
SearchDocument,
|
||||||
|
SearchResourceType,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorServiceDeskObject,
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.service_desk_connector import (
|
||||||
|
SERVICE_DESK_PROVIDER_ID,
|
||||||
|
SERVICE_DESK_READ_SCOPE,
|
||||||
|
SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_SEARCH_MATURITIES = ("search", "read", "publish", "synchronize", "migrate", "replace")
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalServiceDeskSearchSource:
|
||||||
|
def resource_types(self) -> Sequence[SearchResourceType]:
|
||||||
|
return (
|
||||||
|
SearchResourceType(
|
||||||
|
provider_id=SERVICE_DESK_PROVIDER_ID,
|
||||||
|
module_id="connectors",
|
||||||
|
resource_type=SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
label="External service-desk tickets",
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def backfill(
|
||||||
|
self, session: object, *, request: SearchBackfillRequest
|
||||||
|
) -> SearchBackfillPage:
|
||||||
|
_assert_source(request.provider_id, request.resource_type)
|
||||||
|
db = _session(session)
|
||||||
|
query = (
|
||||||
|
select(ConnectorServiceDeskObject, ConnectorServiceDeskProfile)
|
||||||
|
.join(
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskProfile.id == ConnectorServiceDeskObject.profile_id,
|
||||||
|
)
|
||||||
|
.join(
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorConfiguration.id
|
||||||
|
== ConnectorServiceDeskProfile.configuration_id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorServiceDeskObject.tenant_id == request.tenant_id,
|
||||||
|
ConnectorServiceDeskObject.object_type == "ticket",
|
||||||
|
ConnectorServiceDeskObject.status != "deleted",
|
||||||
|
ConnectorServiceDeskProfile.tenant_id == request.tenant_id,
|
||||||
|
ConnectorServiceDeskProfile.status == "active",
|
||||||
|
ConnectorConfiguration.tenant_id == request.tenant_id,
|
||||||
|
ConnectorConfiguration.status == "active",
|
||||||
|
ConnectorServiceDeskProfile.discovered_configuration_revision
|
||||||
|
== ConnectorConfiguration.resource_revision,
|
||||||
|
ConnectorServiceDeskProfile.discovered_configuration_hash
|
||||||
|
== ConnectorConfiguration.effective_hash,
|
||||||
|
ConnectorServiceDeskProfile.desired_maturity.in_(_SEARCH_MATURITIES),
|
||||||
|
ConnectorServiceDeskProfile.discovered_maturity.in_(_SEARCH_MATURITIES),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if request.cursor:
|
||||||
|
query = query.where(ConnectorServiceDeskObject.id > request.cursor)
|
||||||
|
rows = tuple(
|
||||||
|
db.execute(
|
||||||
|
query.order_by(ConnectorServiceDeskObject.id.asc()).limit(request.limit + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
has_more = len(rows) > request.limit
|
||||||
|
selected = rows[: request.limit]
|
||||||
|
watermark = db.scalar(
|
||||||
|
select(func.max(ConnectorServiceDeskObject.updated_at))
|
||||||
|
.join(
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskProfile.id == ConnectorServiceDeskObject.profile_id,
|
||||||
|
)
|
||||||
|
.join(
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorConfiguration.id
|
||||||
|
== ConnectorServiceDeskProfile.configuration_id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorServiceDeskObject.tenant_id == request.tenant_id,
|
||||||
|
ConnectorServiceDeskObject.object_type == "ticket",
|
||||||
|
ConnectorServiceDeskObject.status != "deleted",
|
||||||
|
ConnectorServiceDeskProfile.tenant_id == request.tenant_id,
|
||||||
|
ConnectorServiceDeskProfile.status == "active",
|
||||||
|
ConnectorConfiguration.tenant_id == request.tenant_id,
|
||||||
|
ConnectorConfiguration.status == "active",
|
||||||
|
ConnectorServiceDeskProfile.discovered_configuration_revision
|
||||||
|
== ConnectorConfiguration.resource_revision,
|
||||||
|
ConnectorServiceDeskProfile.discovered_configuration_hash
|
||||||
|
== ConnectorConfiguration.effective_hash,
|
||||||
|
ConnectorServiceDeskProfile.desired_maturity.in_(_SEARCH_MATURITIES),
|
||||||
|
ConnectorServiceDeskProfile.discovered_maturity.in_(_SEARCH_MATURITIES),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return SearchBackfillPage(
|
||||||
|
documents=tuple(search_document(db, row, profile) for row, profile in selected),
|
||||||
|
next_cursor=selected[-1][0].id if has_more and selected else None,
|
||||||
|
complete=not has_more,
|
||||||
|
high_watermark=watermark.isoformat() if watermark else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def authorize(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
requests: Sequence[SearchAuthorizationRequest],
|
||||||
|
) -> Mapping[str, bool]:
|
||||||
|
db = _session(session)
|
||||||
|
tenant_id = str(getattr(principal, "tenant_id", "") or "")
|
||||||
|
can_read = _has_scope(principal, SERVICE_DESK_READ_SCOPE)
|
||||||
|
tokens = set(_principal_acl_tokens(principal))
|
||||||
|
decisions = {item.reference.key: False for item in requests}
|
||||||
|
if not tenant_id or not can_read:
|
||||||
|
return decisions
|
||||||
|
for request in requests:
|
||||||
|
reference = request.reference
|
||||||
|
if (
|
||||||
|
reference.tenant_id != tenant_id
|
||||||
|
or reference.module_id != "connectors"
|
||||||
|
or reference.resource_type != SERVICE_DESK_RESOURCE_TYPE
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
joined = db.execute(
|
||||||
|
select(ConnectorServiceDeskObject, ConnectorServiceDeskProfile)
|
||||||
|
.join(
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskProfile.id
|
||||||
|
== ConnectorServiceDeskObject.profile_id,
|
||||||
|
)
|
||||||
|
.join(
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorConfiguration.id
|
||||||
|
== ConnectorServiceDeskProfile.configuration_id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ConnectorServiceDeskObject.tenant_id == tenant_id,
|
||||||
|
ConnectorServiceDeskObject.id == reference.resource_id,
|
||||||
|
ConnectorServiceDeskObject.object_type == "ticket",
|
||||||
|
ConnectorServiceDeskObject.status != "deleted",
|
||||||
|
ConnectorServiceDeskProfile.tenant_id == tenant_id,
|
||||||
|
ConnectorServiceDeskProfile.status == "active",
|
||||||
|
ConnectorConfiguration.tenant_id == tenant_id,
|
||||||
|
ConnectorConfiguration.status == "active",
|
||||||
|
ConnectorServiceDeskProfile.discovered_configuration_revision
|
||||||
|
== ConnectorConfiguration.resource_revision,
|
||||||
|
ConnectorServiceDeskProfile.discovered_configuration_hash
|
||||||
|
== ConnectorConfiguration.effective_hash,
|
||||||
|
ConnectorServiceDeskProfile.desired_maturity.in_(_SEARCH_MATURITIES),
|
||||||
|
ConnectorServiceDeskProfile.discovered_maturity.in_(_SEARCH_MATURITIES),
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
if joined is None:
|
||||||
|
continue
|
||||||
|
row, _profile = joined
|
||||||
|
decisions[reference.key] = row.visibility == "tenant" or bool(
|
||||||
|
tokens.intersection(str(value) for value in row.acl_tokens or ())
|
||||||
|
)
|
||||||
|
return decisions
|
||||||
|
|
||||||
|
|
||||||
|
def create_external_service_desk_search_source(
|
||||||
|
_context: ModuleContext,
|
||||||
|
) -> ExternalServiceDeskSearchSource:
|
||||||
|
return ExternalServiceDeskSearchSource()
|
||||||
|
|
||||||
|
|
||||||
|
def search_document(
|
||||||
|
session: Session,
|
||||||
|
row: ConnectorServiceDeskObject,
|
||||||
|
profile: ConnectorServiceDeskProfile | None = None,
|
||||||
|
) -> SearchDocument:
|
||||||
|
if profile is None:
|
||||||
|
profile = session.scalar(
|
||||||
|
select(ConnectorServiceDeskProfile).where(
|
||||||
|
ConnectorServiceDeskProfile.tenant_id == row.tenant_id,
|
||||||
|
ConnectorServiceDeskProfile.id == row.profile_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if profile is None:
|
||||||
|
raise ValueError("External service-desk profile is unavailable.")
|
||||||
|
data = dict(row.mapped_data or {})
|
||||||
|
queue = data.get("queue") if isinstance(data.get("queue"), Mapping) else {}
|
||||||
|
state = data.get("state") if isinstance(data.get("state"), Mapping) else {}
|
||||||
|
priority = data.get("priority") if isinstance(data.get("priority"), Mapping) else {}
|
||||||
|
articles = [value for value in data.get("articles") or () if isinstance(value, Mapping)]
|
||||||
|
article_subjects = tuple(
|
||||||
|
str(value.get("subject") or "")[:500]
|
||||||
|
for value in articles
|
||||||
|
if value.get("subject")
|
||||||
|
)
|
||||||
|
article_body = "\n\n".join(
|
||||||
|
str(value.get("body") or "") for value in articles if value.get("body")
|
||||||
|
)[:200_000]
|
||||||
|
dynamic_fields = data.get("dynamic_fields")
|
||||||
|
dynamic_keywords = tuple(
|
||||||
|
f"{key}:{str(value)[:200]}"
|
||||||
|
for key, value in (
|
||||||
|
dynamic_fields.items() if isinstance(dynamic_fields, Mapping) else ()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
external_reference = ExternalObjectReference(
|
||||||
|
system=profile.product if profile.product != "unknown" else "znuny_otrs",
|
||||||
|
object_type=row.object_type,
|
||||||
|
object_id=row.external_id,
|
||||||
|
maturity=profile.discovered_maturity,
|
||||||
|
authority_mode=profile.source_authority_mode,
|
||||||
|
connector_id=profile.id,
|
||||||
|
canonical_url=row.canonical_url,
|
||||||
|
version=row.source_revision,
|
||||||
|
etag=row.content_hash,
|
||||||
|
observed_at=row.observed_at,
|
||||||
|
metadata={
|
||||||
|
"ticket_number": row.external_ticket_number,
|
||||||
|
"title": row.title,
|
||||||
|
"queue": queue.get("name"),
|
||||||
|
"state": state.get("name"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return SearchDocument(
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
module_id="connectors",
|
||||||
|
provider_id=SERVICE_DESK_PROVIDER_ID,
|
||||||
|
resource_type=SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
resource_id=row.id,
|
||||||
|
title=(
|
||||||
|
f"{row.external_ticket_number}: {row.title}"
|
||||||
|
if row.external_ticket_number
|
||||||
|
else row.title
|
||||||
|
),
|
||||||
|
url=(
|
||||||
|
"/connectors/service-desk?profileId="
|
||||||
|
f"{quote(row.profile_id, safe='')}&objectId={quote(row.id, safe='')}"
|
||||||
|
),
|
||||||
|
summary=(article_subjects[0] if article_subjects else None),
|
||||||
|
body=article_body or None,
|
||||||
|
keywords=tuple(
|
||||||
|
dict.fromkeys(
|
||||||
|
value
|
||||||
|
for value in (
|
||||||
|
str(queue.get("name") or ""),
|
||||||
|
str(state.get("name") or ""),
|
||||||
|
str(priority.get("name") or ""),
|
||||||
|
*article_subjects,
|
||||||
|
*dynamic_keywords,
|
||||||
|
)
|
||||||
|
if value
|
||||||
|
)
|
||||||
|
)[:100],
|
||||||
|
visibility=row.visibility,
|
||||||
|
acl_tokens=(
|
||||||
|
tuple(str(value) for value in row.acl_tokens or ())
|
||||||
|
if row.visibility == "restricted"
|
||||||
|
else ()
|
||||||
|
),
|
||||||
|
external_reference=external_reference,
|
||||||
|
metadata={
|
||||||
|
"profile_id": row.profile_id,
|
||||||
|
"external_ticket_id": row.external_id,
|
||||||
|
"external_ticket_number": row.external_ticket_number,
|
||||||
|
"target_queue_ref": data.get("target_queue_ref"),
|
||||||
|
"integration_mode": profile.integration_mode,
|
||||||
|
"source_authority_mode": profile.source_authority_mode,
|
||||||
|
"status": row.status,
|
||||||
|
},
|
||||||
|
source_revision=row.source_revision,
|
||||||
|
change_cursor=row.change_cursor,
|
||||||
|
source_updated_at=row.source_updated_at or row.observed_at,
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal_acl_tokens(principal: object) -> tuple[str, ...]:
|
||||||
|
values: list[str] = []
|
||||||
|
for prefix, attribute in (
|
||||||
|
("account", "account_id"),
|
||||||
|
("membership", "membership_id"),
|
||||||
|
("identity", "identity_id"),
|
||||||
|
):
|
||||||
|
value = getattr(principal, attribute, None)
|
||||||
|
if value:
|
||||||
|
values.append(f"{prefix}:{value}")
|
||||||
|
for prefix, attribute in (
|
||||||
|
("group", "group_ids"),
|
||||||
|
("role", "role_ids"),
|
||||||
|
("function", "function_assignment_ids"),
|
||||||
|
("scope", "scopes"),
|
||||||
|
):
|
||||||
|
values.extend(
|
||||||
|
f"{prefix}:{value}"
|
||||||
|
for value in getattr(principal, attribute, ())
|
||||||
|
if value
|
||||||
|
)
|
||||||
|
return tuple(dict.fromkeys(values))[:500]
|
||||||
|
|
||||||
|
|
||||||
|
def _has_scope(principal: object, required: str) -> bool:
|
||||||
|
check = getattr(principal, "has", None)
|
||||||
|
if callable(check):
|
||||||
|
return bool(check(required))
|
||||||
|
return required in getattr(principal, "scopes", ())
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_source(provider_id: str, resource_type: str) -> None:
|
||||||
|
if provider_id != SERVICE_DESK_PROVIDER_ID or resource_type != SERVICE_DESK_RESOURCE_TYPE:
|
||||||
|
raise ValueError("Unsupported external service-desk Search source.")
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("External service-desk Search requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ExternalServiceDeskSearchSource",
|
||||||
|
"create_external_service_desk_search_source",
|
||||||
|
"search_document",
|
||||||
|
]
|
||||||
@@ -0,0 +1,896 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import urllib.error
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Any, Protocol
|
||||||
|
from urllib.parse import quote, urlencode, urljoin, urlsplit
|
||||||
|
|
||||||
|
from govoplan_core.security.http_fetch import HttpFetchResponse, fetch_http
|
||||||
|
from govoplan_core.security.outbound_http import OutboundHttpError
|
||||||
|
|
||||||
|
|
||||||
|
MAX_SERVICE_DESK_RESPONSE_BYTES = 10_000_000
|
||||||
|
MAX_SERVICE_DESK_SEARCH_IDS = 10_000
|
||||||
|
MAX_SERVICE_DESK_TICKET_READS = 500
|
||||||
|
SERVICE_DESK_SENSITIVE_HEADERS = (
|
||||||
|
"X-OTRS-Header-UserLogin",
|
||||||
|
"X-OTRS-Header-CustomerUserLogin",
|
||||||
|
"X-OTRS-Header-Password",
|
||||||
|
"X-OTRS-Header-SessionID",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskTransportError(RuntimeError):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
retryable: bool = False,
|
||||||
|
outcome_unknown: bool = False,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
self.retryable = retryable
|
||||||
|
self.outcome_unknown = outcome_unknown
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ServiceDeskChangeBatch:
|
||||||
|
changes: tuple[Mapping[str, Any], ...]
|
||||||
|
next_cursor: str | None
|
||||||
|
complete: bool
|
||||||
|
high_watermark: str | None
|
||||||
|
live_ids: tuple[str, ...] | None
|
||||||
|
evidence: Mapping[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ServiceDeskUpdateResult:
|
||||||
|
ticket: Mapping[str, Any]
|
||||||
|
revision: str
|
||||||
|
evidence: Mapping[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskTransport(Protocol):
|
||||||
|
def discover(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
) -> Mapping[str, Any]: ...
|
||||||
|
|
||||||
|
def changes(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
cursor: str | None,
|
||||||
|
limit: int,
|
||||||
|
force_full: bool,
|
||||||
|
) -> ServiceDeskChangeBatch: ...
|
||||||
|
|
||||||
|
def update_ticket(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
ticket_id: str,
|
||||||
|
expected_revision: str,
|
||||||
|
changes: Mapping[str, Any],
|
||||||
|
) -> ServiceDeskUpdateResult: ...
|
||||||
|
|
||||||
|
|
||||||
|
class HttpServiceDeskTransport:
|
||||||
|
"""Bounded Znuny/OTRS GenericInterface REST transport.
|
||||||
|
|
||||||
|
GenericInterface route names are administrator-defined. The governed profile
|
||||||
|
supplies the paths and methods while this adapter enforces outbound policy,
|
||||||
|
response bounds, credential placement, cursor stability, and write recovery
|
||||||
|
semantics.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def discover(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
) -> Mapping[str, Any]:
|
||||||
|
payload, response = self._search(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
criteria={"Limit": 1, "SortBy": ["Changed"], "OrderBy": ["Up"]},
|
||||||
|
)
|
||||||
|
product, version = _product_version(payload, response.headers, endpoint_url)
|
||||||
|
recognized = product in {"znuny", "otrs"} and _major(version) >= 6
|
||||||
|
capabilities = ["discover", "link", "search", "read"]
|
||||||
|
if recognized:
|
||||||
|
capabilities.append("synchronize")
|
||||||
|
if recognized and routes.get("update_path"):
|
||||||
|
capabilities.append("publish")
|
||||||
|
maturity = "synchronize" if "synchronize" in capabilities else "read"
|
||||||
|
revision = _hash(
|
||||||
|
{
|
||||||
|
"product": product,
|
||||||
|
"version": version,
|
||||||
|
"routes": dict(routes),
|
||||||
|
"capabilities": capabilities,
|
||||||
|
"status": response.status,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"product": product,
|
||||||
|
"product_version": version,
|
||||||
|
"api_family": "generic_interface_rest",
|
||||||
|
"capabilities": capabilities,
|
||||||
|
"maturity": maturity,
|
||||||
|
"health_status": "healthy",
|
||||||
|
"revision": revision,
|
||||||
|
"diagnostics": (
|
||||||
|
[]
|
||||||
|
if recognized
|
||||||
|
else [
|
||||||
|
{
|
||||||
|
"severity": "warning",
|
||||||
|
"code": "product_version_unverified",
|
||||||
|
"message": (
|
||||||
|
"The GenericInterface endpoint is healthy, but its product "
|
||||||
|
"and major version could not be verified; maturity is limited to read."
|
||||||
|
),
|
||||||
|
"retryable": False,
|
||||||
|
"details": {},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
"evidence": {
|
||||||
|
"http_status": response.status,
|
||||||
|
"response_content_type": response.headers.get("Content-Type"),
|
||||||
|
"ticket_search_shape": _search_shape(payload),
|
||||||
|
"credential_present": bool(credential),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def changes(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
cursor: str | None,
|
||||||
|
limit: int,
|
||||||
|
force_full: bool,
|
||||||
|
) -> ServiceDeskChangeBatch:
|
||||||
|
if limit < 1 or limit > MAX_SERVICE_DESK_TICKET_READS:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"read_limit_invalid",
|
||||||
|
"A service-desk synchronization call must request between 1 and 500 tickets.",
|
||||||
|
)
|
||||||
|
if force_full:
|
||||||
|
return self._full_changes(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
cursor=cursor,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
return self._delta_changes(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
cursor=cursor,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
def update_ticket(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
endpoint_url: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
ticket_id: str,
|
||||||
|
expected_revision: str,
|
||||||
|
changes: Mapping[str, Any],
|
||||||
|
) -> ServiceDeskUpdateResult:
|
||||||
|
update_path = _optional_text(routes.get("update_path"))
|
||||||
|
if not update_path:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"update_unsupported",
|
||||||
|
"The configured GenericInterface profile has no ticket update route.",
|
||||||
|
)
|
||||||
|
current, _response = self._ticket(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
ticket_id=ticket_id,
|
||||||
|
)
|
||||||
|
actual_revision = _ticket_revision(current)
|
||||||
|
if actual_revision != expected_revision:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"external_revision_conflict",
|
||||||
|
"The external ticket changed; synchronize it before updating.",
|
||||||
|
)
|
||||||
|
url = _route_url(
|
||||||
|
endpoint_url,
|
||||||
|
update_path.replace("{ticket_id}", quote(ticket_id, safe="")),
|
||||||
|
)
|
||||||
|
method = str(routes.get("update_method") or "PATCH").upper()
|
||||||
|
request_payload = _authenticated_payload(
|
||||||
|
{"Ticket": dict(changes)}, credential
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
_payload, response = self._request(
|
||||||
|
url,
|
||||||
|
method=method,
|
||||||
|
credential=credential,
|
||||||
|
payload=request_payload,
|
||||||
|
mutation=True,
|
||||||
|
)
|
||||||
|
except ServiceDeskTransportError as exc:
|
||||||
|
if exc.outcome_unknown:
|
||||||
|
raise
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
exc.code,
|
||||||
|
str(exc),
|
||||||
|
retryable=exc.retryable,
|
||||||
|
outcome_unknown=False,
|
||||||
|
) from exc
|
||||||
|
refreshed, read_response = self._ticket(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
ticket_id=ticket_id,
|
||||||
|
)
|
||||||
|
if _ticket_id(refreshed) != ticket_id:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"update_verification_identity_mismatch",
|
||||||
|
"The provider verification returned another ticket identity.",
|
||||||
|
outcome_unknown=True,
|
||||||
|
)
|
||||||
|
revision = _ticket_revision(refreshed)
|
||||||
|
if revision == expected_revision:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"update_verification_failed",
|
||||||
|
"The provider accepted the request but the ticket revision did not change.",
|
||||||
|
outcome_unknown=True,
|
||||||
|
)
|
||||||
|
mismatches = _update_mismatches(refreshed, changes)
|
||||||
|
if mismatches:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"update_verification_failed",
|
||||||
|
"The provider revision changed, but the requested ticket fields could not be verified.",
|
||||||
|
outcome_unknown=True,
|
||||||
|
)
|
||||||
|
return ServiceDeskUpdateResult(
|
||||||
|
ticket=refreshed,
|
||||||
|
revision=revision,
|
||||||
|
evidence={
|
||||||
|
"update_http_status": response.status,
|
||||||
|
"verification_http_status": read_response.status,
|
||||||
|
"previous_revision": expected_revision,
|
||||||
|
"accepted_revision": revision,
|
||||||
|
"verified_fields": sorted(changes),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _full_changes(
|
||||||
|
self,
|
||||||
|
endpoint_url: str,
|
||||||
|
*,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
cursor: str | None,
|
||||||
|
limit: int,
|
||||||
|
) -> ServiceDeskChangeBatch:
|
||||||
|
payload, _response = self._search(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
criteria={
|
||||||
|
"Limit": MAX_SERVICE_DESK_SEARCH_IDS + 1,
|
||||||
|
"SortBy": ["TicketID"],
|
||||||
|
"OrderBy": ["Up"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
ids = _ticket_ids(payload)
|
||||||
|
_assert_search_bound(ids)
|
||||||
|
fingerprint = _hash(ids)
|
||||||
|
state = _decode_cursor(cursor, expected_kind="full")
|
||||||
|
offset = int(state.get("offset") or 0)
|
||||||
|
if offset > len(ids):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"full_cursor_stale",
|
||||||
|
"The full synchronization cursor is beyond the current ticket set; restart the backfill.",
|
||||||
|
)
|
||||||
|
if state.get("fingerprint") is not None and state.get("fingerprint") != fingerprint:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"full_cursor_stale",
|
||||||
|
"The external ticket set changed during backfill; restart the full synchronization.",
|
||||||
|
)
|
||||||
|
selected = ids[offset : offset + limit]
|
||||||
|
changes = tuple(
|
||||||
|
self._ticket(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
ticket_id=ticket_id,
|
||||||
|
)[0]
|
||||||
|
for ticket_id in selected
|
||||||
|
)
|
||||||
|
new_offset = offset + len(selected)
|
||||||
|
complete = new_offset >= len(ids)
|
||||||
|
page_high_watermark = _latest_revision(changes)
|
||||||
|
high_watermark = max(
|
||||||
|
value
|
||||||
|
for value in (
|
||||||
|
_optional_text(state.get("high_watermark")),
|
||||||
|
page_high_watermark,
|
||||||
|
)
|
||||||
|
if value is not None
|
||||||
|
) if (_optional_text(state.get("high_watermark")) or page_high_watermark) else None
|
||||||
|
next_cursor = (
|
||||||
|
None
|
||||||
|
if complete
|
||||||
|
else _encode_cursor(
|
||||||
|
{
|
||||||
|
"kind": "full",
|
||||||
|
"offset": new_offset,
|
||||||
|
"fingerprint": fingerprint,
|
||||||
|
"high_watermark": high_watermark,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return ServiceDeskChangeBatch(
|
||||||
|
changes=changes,
|
||||||
|
next_cursor=next_cursor,
|
||||||
|
complete=complete,
|
||||||
|
high_watermark=high_watermark,
|
||||||
|
live_ids=tuple(ids) if complete else None,
|
||||||
|
evidence={
|
||||||
|
"mode": "backfill",
|
||||||
|
"available": len(ids),
|
||||||
|
"offset": offset,
|
||||||
|
"returned": len(changes),
|
||||||
|
"ticket_set_fingerprint": fingerprint,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _delta_changes(
|
||||||
|
self,
|
||||||
|
endpoint_url: str,
|
||||||
|
*,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
cursor: str | None,
|
||||||
|
limit: int,
|
||||||
|
) -> ServiceDeskChangeBatch:
|
||||||
|
state = _decode_cursor(cursor, expected_kind="delta")
|
||||||
|
changed = _optional_text(state.get("changed"))
|
||||||
|
seen = {str(value) for value in state.get("seen") or ()}
|
||||||
|
search_limit = min(
|
||||||
|
MAX_SERVICE_DESK_TICKET_READS,
|
||||||
|
limit + len(seen) + 1,
|
||||||
|
)
|
||||||
|
criteria: dict[str, Any] = {
|
||||||
|
"Limit": search_limit,
|
||||||
|
"SortBy": ["Changed"],
|
||||||
|
"OrderBy": ["Up"],
|
||||||
|
}
|
||||||
|
if changed:
|
||||||
|
criteria["TicketChangeTimeNewerDate"] = _overlap_boundary(changed)
|
||||||
|
payload, _response = self._search(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
criteria=criteria,
|
||||||
|
)
|
||||||
|
ids = _ticket_ids(payload)
|
||||||
|
_assert_search_bound(ids)
|
||||||
|
fetched = tuple(
|
||||||
|
self._ticket(
|
||||||
|
endpoint_url,
|
||||||
|
credential=credential,
|
||||||
|
routes=routes,
|
||||||
|
ticket_id=ticket_id,
|
||||||
|
)[0]
|
||||||
|
for ticket_id in ids
|
||||||
|
)
|
||||||
|
eligible: list[Mapping[str, Any]] = []
|
||||||
|
suppressed = 0
|
||||||
|
for ticket in fetched:
|
||||||
|
revision = _ticket_revision(ticket)
|
||||||
|
ticket_id = _ticket_id(ticket)
|
||||||
|
if changed and (
|
||||||
|
revision < changed or (revision == changed and ticket_id in seen)
|
||||||
|
):
|
||||||
|
suppressed += 1
|
||||||
|
continue
|
||||||
|
eligible.append(ticket)
|
||||||
|
ordered_eligible = sorted(
|
||||||
|
eligible,
|
||||||
|
key=lambda item: (_ticket_revision(item), _ticket_id(item)),
|
||||||
|
)
|
||||||
|
ordered = ordered_eligible[:limit]
|
||||||
|
if not ordered and len(ids) >= search_limit:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"delta_boundary_overflow",
|
||||||
|
"The overlap window contains too many tickets to advance safely; narrow the profile or run a full synchronization.",
|
||||||
|
)
|
||||||
|
latest = _latest_revision(ordered) or changed
|
||||||
|
latest_seen = set()
|
||||||
|
if latest:
|
||||||
|
if latest == changed:
|
||||||
|
latest_seen.update(seen)
|
||||||
|
latest_seen.update(
|
||||||
|
_ticket_id(item)
|
||||||
|
for item in ordered
|
||||||
|
if _ticket_revision(item) == latest
|
||||||
|
)
|
||||||
|
complete = len(ordered_eligible) <= limit and len(ids) < search_limit
|
||||||
|
next_cursor = _encode_cursor(
|
||||||
|
{
|
||||||
|
"kind": "delta",
|
||||||
|
"changed": latest,
|
||||||
|
"seen": sorted(latest_seen),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return ServiceDeskChangeBatch(
|
||||||
|
changes=tuple(ordered),
|
||||||
|
next_cursor=next_cursor,
|
||||||
|
complete=complete,
|
||||||
|
high_watermark=latest,
|
||||||
|
live_ids=None,
|
||||||
|
evidence={
|
||||||
|
"mode": "delta",
|
||||||
|
"searched": len(ids),
|
||||||
|
"returned": len(ordered),
|
||||||
|
"overlap_suppressed": suppressed,
|
||||||
|
"search_limit": search_limit,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _search(
|
||||||
|
self,
|
||||||
|
endpoint_url: str,
|
||||||
|
*,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
criteria: Mapping[str, Any],
|
||||||
|
) -> tuple[Mapping[str, Any], HttpFetchResponse]:
|
||||||
|
url = _route_url(endpoint_url, str(routes.get("search_path") or "/Ticket/Search"))
|
||||||
|
method = str(routes.get("search_method") or "POST").upper()
|
||||||
|
configured_filters = routes.get("search_filters")
|
||||||
|
if configured_filters is None:
|
||||||
|
configured_filters = {}
|
||||||
|
if not isinstance(configured_filters, Mapping):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"search_filters_invalid",
|
||||||
|
"Governed GenericInterface search filters must be a JSON object.",
|
||||||
|
)
|
||||||
|
return self._request(
|
||||||
|
url,
|
||||||
|
method=method,
|
||||||
|
credential=credential,
|
||||||
|
payload=_authenticated_payload(
|
||||||
|
{**dict(configured_filters), **dict(criteria)}, credential
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _ticket(
|
||||||
|
self,
|
||||||
|
endpoint_url: str,
|
||||||
|
*,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
routes: Mapping[str, Any],
|
||||||
|
ticket_id: str,
|
||||||
|
) -> tuple[Mapping[str, Any], HttpFetchResponse]:
|
||||||
|
path = str(routes.get("ticket_path") or "/Ticket/{ticket_id}").replace(
|
||||||
|
"{ticket_id}", quote(ticket_id, safe="")
|
||||||
|
)
|
||||||
|
method = str(routes.get("ticket_method") or "GET").upper()
|
||||||
|
identity_only = bool(routes.get("_identity_only"))
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"TicketID": ticket_id,
|
||||||
|
"DynamicFields": 0 if identity_only else 1,
|
||||||
|
"Extended": 1,
|
||||||
|
"AllArticles": 0 if identity_only else 1,
|
||||||
|
"Attachments": 0 if identity_only else 1,
|
||||||
|
"GetAttachmentContents": 0,
|
||||||
|
}
|
||||||
|
raw, response = self._request(
|
||||||
|
_route_url(endpoint_url, path),
|
||||||
|
method=method,
|
||||||
|
credential=credential,
|
||||||
|
payload=_authenticated_payload(payload, credential),
|
||||||
|
)
|
||||||
|
return _ticket_payload(raw), response
|
||||||
|
|
||||||
|
def _request(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
method: str,
|
||||||
|
credential: Mapping[str, Any] | None,
|
||||||
|
payload: Mapping[str, Any] | None,
|
||||||
|
mutation: bool = False,
|
||||||
|
) -> tuple[Mapping[str, Any], HttpFetchResponse]:
|
||||||
|
headers = {"Accept": "application/json", **_auth_headers(credential)}
|
||||||
|
body: bytes | None = None
|
||||||
|
request_url = url
|
||||||
|
if payload:
|
||||||
|
if method == "GET":
|
||||||
|
secret_keys = {
|
||||||
|
"SessionID",
|
||||||
|
"UserLogin",
|
||||||
|
"CustomerUserLogin",
|
||||||
|
"Password",
|
||||||
|
}
|
||||||
|
if secret_keys.intersection(payload):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"credential_transport_unsafe",
|
||||||
|
"Body authentication cannot be used with a GET route; use header authentication or configure a POST route.",
|
||||||
|
)
|
||||||
|
separator = "&" if urlsplit(request_url).query else "?"
|
||||||
|
request_url = f"{request_url}{separator}{urlencode(payload, doseq=True)}"
|
||||||
|
else:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||||
|
try:
|
||||||
|
response = fetch_http(
|
||||||
|
request_url,
|
||||||
|
timeout=20,
|
||||||
|
label="Service-desk connector URL",
|
||||||
|
method=method,
|
||||||
|
headers=headers,
|
||||||
|
body=body,
|
||||||
|
max_bytes=MAX_SERVICE_DESK_RESPONSE_BYTES,
|
||||||
|
redirect_sensitive_headers=SERVICE_DESK_SENSITIVE_HEADERS,
|
||||||
|
)
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
retryable = exc.code == 429 or exc.code >= 500
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"provider_http_error",
|
||||||
|
f"The service-desk provider returned HTTP {exc.code}.",
|
||||||
|
retryable=retryable,
|
||||||
|
outcome_unknown=mutation and retryable,
|
||||||
|
) from exc
|
||||||
|
except (urllib.error.URLError, socket.timeout, TimeoutError) as exc:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"provider_unavailable",
|
||||||
|
"The service-desk provider did not return a conclusive response.",
|
||||||
|
retryable=True,
|
||||||
|
outcome_unknown=mutation,
|
||||||
|
) from exc
|
||||||
|
except (ValueError, OutboundHttpError) as exc:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"transport_policy_rejected", str(exc), retryable=False
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
decoded = json.loads(response.body.decode("utf-8")) if response.body else {}
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"invalid_provider_response",
|
||||||
|
"The service-desk provider did not return valid JSON.",
|
||||||
|
outcome_unknown=mutation,
|
||||||
|
) from exc
|
||||||
|
if not isinstance(decoded, Mapping):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"invalid_provider_response",
|
||||||
|
"The service-desk provider response must be a JSON object.",
|
||||||
|
outcome_unknown=mutation,
|
||||||
|
)
|
||||||
|
error = decoded.get("Error")
|
||||||
|
if isinstance(error, Mapping):
|
||||||
|
code = _optional_text(error.get("ErrorCode")) or "provider_rejected"
|
||||||
|
message = _optional_text(error.get("ErrorMessage")) or "Provider rejected the request."
|
||||||
|
raise ServiceDeskTransportError(code, message, retryable=False)
|
||||||
|
return decoded, response
|
||||||
|
|
||||||
|
|
||||||
|
def _route_url(endpoint_url: str, route: str) -> str:
|
||||||
|
parsed = urlsplit(route)
|
||||||
|
if parsed.scheme or parsed.netloc or parsed.username or parsed.password:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"route_invalid", "GenericInterface routes must be relative to the governed endpoint."
|
||||||
|
)
|
||||||
|
if any(part == ".." for part in parsed.path.split("/")):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"route_invalid", "GenericInterface routes cannot traverse parent paths."
|
||||||
|
)
|
||||||
|
return urljoin(endpoint_url.rstrip("/") + "/", route.lstrip("/"))
|
||||||
|
|
||||||
|
|
||||||
|
def _auth_headers(credential: Mapping[str, Any] | None) -> dict[str, str]:
|
||||||
|
if not credential:
|
||||||
|
return {}
|
||||||
|
if str(credential.get("auth_mode") or "header").casefold() == "body":
|
||||||
|
return {}
|
||||||
|
headers: dict[str, str] = {}
|
||||||
|
session_id = _credential_value(credential, "session_id", "SessionID")
|
||||||
|
user_login = _credential_value(credential, "user_login", "UserLogin", "username")
|
||||||
|
password = _credential_value(credential, "password", "Password")
|
||||||
|
customer_login = _credential_value(
|
||||||
|
credential, "customer_user_login", "CustomerUserLogin"
|
||||||
|
)
|
||||||
|
if session_id:
|
||||||
|
headers["X-OTRS-Header-SessionID"] = session_id
|
||||||
|
if user_login:
|
||||||
|
headers["X-OTRS-Header-UserLogin"] = user_login
|
||||||
|
if customer_login:
|
||||||
|
headers["X-OTRS-Header-CustomerUserLogin"] = customer_login
|
||||||
|
if password:
|
||||||
|
headers["X-OTRS-Header-Password"] = password
|
||||||
|
return headers
|
||||||
|
|
||||||
|
|
||||||
|
def _authenticated_payload(
|
||||||
|
payload: Mapping[str, Any], credential: Mapping[str, Any] | None
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result = dict(payload)
|
||||||
|
if (
|
||||||
|
not credential
|
||||||
|
or str(credential.get("auth_mode") or "header").casefold() != "body"
|
||||||
|
):
|
||||||
|
return result
|
||||||
|
for target, aliases in (
|
||||||
|
("SessionID", ("session_id", "SessionID")),
|
||||||
|
("UserLogin", ("user_login", "UserLogin", "username")),
|
||||||
|
("CustomerUserLogin", ("customer_user_login", "CustomerUserLogin")),
|
||||||
|
("Password", ("password", "Password")),
|
||||||
|
):
|
||||||
|
value = _credential_value(credential, *aliases)
|
||||||
|
if value:
|
||||||
|
result[target] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _credential_value(credential: Mapping[str, Any], *keys: str) -> str | None:
|
||||||
|
for key in keys:
|
||||||
|
value = _optional_text(credential.get(key))
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ticket_ids(payload: Mapping[str, Any]) -> list[str]:
|
||||||
|
raw = payload.get("TicketID")
|
||||||
|
if raw is None:
|
||||||
|
raw = payload.get("TicketIDs")
|
||||||
|
if raw is None:
|
||||||
|
raw = payload.get("TicketId")
|
||||||
|
if raw is None:
|
||||||
|
return []
|
||||||
|
values: Sequence[Any] = raw if isinstance(raw, Sequence) and not isinstance(raw, str) else [raw]
|
||||||
|
return list(dict.fromkeys(str(value).strip() for value in values if str(value).strip()))
|
||||||
|
|
||||||
|
|
||||||
|
def _ticket_payload(payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||||
|
raw = payload.get("Ticket")
|
||||||
|
if isinstance(raw, Mapping):
|
||||||
|
return raw
|
||||||
|
if isinstance(raw, Sequence) and not isinstance(raw, (str, bytes)):
|
||||||
|
for value in raw:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
return value
|
||||||
|
if any(key in payload for key in ("TicketID", "TicketNumber", "Title", "Changed")):
|
||||||
|
return payload
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"ticket_response_invalid", "The provider response did not contain a ticket object."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _product_version(
|
||||||
|
payload: Mapping[str, Any], headers: Mapping[str, str], endpoint_url: str
|
||||||
|
) -> tuple[str, str | None]:
|
||||||
|
normalized_headers = {key.casefold(): value for key, value in headers.items()}
|
||||||
|
znuny_version = _optional_text(normalized_headers.get("x-znuny-version"))
|
||||||
|
otrs_version = _optional_text(normalized_headers.get("x-otrs-version"))
|
||||||
|
product = _optional_text(payload.get("Product"))
|
||||||
|
version = _optional_text(payload.get("Version"))
|
||||||
|
system_data = payload.get("SystemData")
|
||||||
|
if isinstance(system_data, Mapping):
|
||||||
|
product = product or _optional_text(system_data.get("Product"))
|
||||||
|
version = version or _optional_text(system_data.get("Version"))
|
||||||
|
if znuny_version:
|
||||||
|
return "znuny", znuny_version
|
||||||
|
if otrs_version:
|
||||||
|
return "otrs", otrs_version
|
||||||
|
folded = str(product or "").casefold()
|
||||||
|
if "znuny" in folded:
|
||||||
|
return "znuny", version
|
||||||
|
if "otrs" in folded:
|
||||||
|
return "otrs", version
|
||||||
|
path = urlsplit(endpoint_url).path.casefold()
|
||||||
|
if "/znuny/" in path:
|
||||||
|
return "znuny", version
|
||||||
|
if "/otrs/" in path:
|
||||||
|
return "otrs", version
|
||||||
|
return "znuny_otrs", version
|
||||||
|
|
||||||
|
|
||||||
|
def _major(version: str | None) -> int:
|
||||||
|
if not version:
|
||||||
|
return 0
|
||||||
|
head = version.strip().split(".", 1)[0]
|
||||||
|
return int(head) if head.isdigit() else 0
|
||||||
|
|
||||||
|
|
||||||
|
def _ticket_id(ticket: Mapping[str, Any]) -> str:
|
||||||
|
value = _optional_text(ticket.get("TicketID")) or _optional_text(ticket.get("ID"))
|
||||||
|
if not value:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"ticket_identity_missing", "The provider ticket has no stable TicketID."
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _ticket_revision(ticket: Mapping[str, Any]) -> str:
|
||||||
|
value = (
|
||||||
|
_optional_text(ticket.get("Changed"))
|
||||||
|
or _optional_text(ticket.get("ChangeTime"))
|
||||||
|
or _optional_text(ticket.get("Updated"))
|
||||||
|
)
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
return _hash(ticket)
|
||||||
|
|
||||||
|
|
||||||
|
def _update_mismatches(
|
||||||
|
ticket: Mapping[str, Any],
|
||||||
|
changes: Mapping[str, Any],
|
||||||
|
) -> list[str]:
|
||||||
|
mismatches: list[str] = []
|
||||||
|
for field in ("Title", "Queue", "State", "Priority", "Owner", "Responsible"):
|
||||||
|
if field in changes and ticket.get(field) != changes[field]:
|
||||||
|
mismatches.append(field)
|
||||||
|
expected_dynamic = changes.get("DynamicField")
|
||||||
|
if isinstance(expected_dynamic, Sequence) and not isinstance(
|
||||||
|
expected_dynamic, (str, bytes)
|
||||||
|
):
|
||||||
|
actual_dynamic = _dynamic_field_values(ticket)
|
||||||
|
for item in expected_dynamic:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
mismatches.append("DynamicField")
|
||||||
|
continue
|
||||||
|
name = _optional_text(item.get("Name"))
|
||||||
|
if not name or name not in actual_dynamic or actual_dynamic[name] != item.get("Value"):
|
||||||
|
mismatches.append(f"DynamicField.{name or 'unknown'}")
|
||||||
|
return mismatches
|
||||||
|
|
||||||
|
|
||||||
|
def _dynamic_field_values(ticket: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
raw = (
|
||||||
|
ticket.get("DynamicField")
|
||||||
|
if ticket.get("DynamicField") is not None
|
||||||
|
else ticket.get("DynamicFields")
|
||||||
|
)
|
||||||
|
if isinstance(raw, Mapping):
|
||||||
|
return {str(key): value for key, value in raw.items()}
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
if isinstance(raw, Sequence) and not isinstance(raw, (str, bytes)):
|
||||||
|
for item in raw:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
continue
|
||||||
|
name = _optional_text(item.get("Name"))
|
||||||
|
if name:
|
||||||
|
result[name] = item.get("Value")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_revision(changes: Sequence[Mapping[str, Any]]) -> str | None:
|
||||||
|
values = [_ticket_revision(item) for item in changes]
|
||||||
|
return max(values) if values else None
|
||||||
|
|
||||||
|
|
||||||
|
def _overlap_boundary(value: str) -> str:
|
||||||
|
normalized = value.strip().replace("Z", "+00:00")
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(normalized)
|
||||||
|
except ValueError:
|
||||||
|
return value
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||||
|
return (parsed.astimezone(timezone.utc) - timedelta(seconds=1)).isoformat().replace(
|
||||||
|
"+00:00", "Z"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_search_bound(ids: Sequence[str]) -> None:
|
||||||
|
if len(ids) > MAX_SERVICE_DESK_SEARCH_IDS:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"search_result_unbounded",
|
||||||
|
"The provider returned more than 10000 ticket identities; narrow the profile by queue.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_cursor(cursor: str | None, *, expected_kind: str) -> dict[str, Any]:
|
||||||
|
if not cursor:
|
||||||
|
return {"kind": expected_kind}
|
||||||
|
try:
|
||||||
|
value = json.loads(cursor)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ServiceDeskTransportError("cursor_invalid", "The synchronization cursor is invalid.") from exc
|
||||||
|
if not isinstance(value, Mapping) or value.get("kind") != expected_kind:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"cursor_mode_mismatch", "The synchronization cursor belongs to another mode."
|
||||||
|
)
|
||||||
|
decoded = dict(value)
|
||||||
|
if expected_kind == "full":
|
||||||
|
offset = decoded.get("offset", 0)
|
||||||
|
fingerprint = decoded.get("fingerprint")
|
||||||
|
if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"cursor_invalid", "The full synchronization cursor offset is invalid."
|
||||||
|
)
|
||||||
|
if offset and (
|
||||||
|
not isinstance(fingerprint, str)
|
||||||
|
or len(fingerprint) != 64
|
||||||
|
or any(
|
||||||
|
character not in "0123456789abcdef"
|
||||||
|
for character in fingerprint.casefold()
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"cursor_invalid", "The full synchronization cursor fingerprint is invalid."
|
||||||
|
)
|
||||||
|
elif expected_kind == "delta":
|
||||||
|
changed = decoded.get("changed")
|
||||||
|
seen = decoded.get("seen", [])
|
||||||
|
if changed is not None and not isinstance(changed, str):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"cursor_invalid", "The delta synchronization boundary is invalid."
|
||||||
|
)
|
||||||
|
if not isinstance(seen, list) or any(
|
||||||
|
not isinstance(item, str) or not item for item in seen
|
||||||
|
):
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"cursor_invalid", "The delta synchronization identity set is invalid."
|
||||||
|
)
|
||||||
|
return decoded
|
||||||
|
|
||||||
|
|
||||||
|
def _encode_cursor(value: Mapping[str, Any]) -> str:
|
||||||
|
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"))
|
||||||
|
if len(encoded) > 4000:
|
||||||
|
raise ServiceDeskTransportError(
|
||||||
|
"cursor_boundary_overflow",
|
||||||
|
"Too many tickets share the same change boundary; narrow the profile by queue.",
|
||||||
|
)
|
||||||
|
return encoded
|
||||||
|
|
||||||
|
|
||||||
|
def _search_shape(payload: Mapping[str, Any]) -> str:
|
||||||
|
if "TicketID" in payload:
|
||||||
|
return "TicketID"
|
||||||
|
if "TicketIDs" in payload:
|
||||||
|
return "TicketIDs"
|
||||||
|
return "empty"
|
||||||
|
|
||||||
|
|
||||||
|
def _hash(value: Any) -> str:
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: object) -> str | None:
|
||||||
|
normalized = str(value).strip() if value is not None else ""
|
||||||
|
return normalized or None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"HttpServiceDeskTransport",
|
||||||
|
"ServiceDeskChangeBatch",
|
||||||
|
"ServiceDeskTransport",
|
||||||
|
"ServiceDeskTransportError",
|
||||||
|
"ServiceDeskUpdateResult",
|
||||||
|
]
|
||||||
@@ -0,0 +1,849 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
import zipfile
|
||||||
|
from collections.abc import Callable, Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date, datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from io import BytesIO
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from openpyxl import load_workbook
|
||||||
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
BigInteger,
|
||||||
|
Boolean,
|
||||||
|
Date,
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
Integer,
|
||||||
|
LargeBinary,
|
||||||
|
MetaData,
|
||||||
|
Numeric,
|
||||||
|
String,
|
||||||
|
Table,
|
||||||
|
Text,
|
||||||
|
create_engine,
|
||||||
|
func,
|
||||||
|
select,
|
||||||
|
)
|
||||||
|
from sqlalchemy.engine import Engine, URL, make_url
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.files import (
|
||||||
|
ManagedTabularFileAccessError,
|
||||||
|
ManagedTabularFileError,
|
||||||
|
ManagedTabularFileValidationError,
|
||||||
|
managed_tabular_file_provider,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularColumn,
|
||||||
|
TabularPreviewDiagnostic,
|
||||||
|
TabularPushdown,
|
||||||
|
TabularSourceHealth,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
parse_tabular_csv,
|
||||||
|
)
|
||||||
|
from govoplan_core.security.credential_envelopes import (
|
||||||
|
CredentialAccessContext,
|
||||||
|
CredentialEnvelopeError,
|
||||||
|
resolve_credential_envelope,
|
||||||
|
)
|
||||||
|
from govoplan_core.security.redaction import is_sensitive_key
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorConfiguration
|
||||||
|
|
||||||
|
|
||||||
|
MAX_FILE_BYTES = 5_000_000
|
||||||
|
MAX_FILE_ROWS = 10_000
|
||||||
|
MAX_FILE_COLUMNS = 500
|
||||||
|
MAX_XLSX_ENTRIES = 5_000
|
||||||
|
MAX_XLSX_EXPANDED_BYTES = 50_000_000
|
||||||
|
MAX_XLSX_COMPRESSION_RATIO = 100
|
||||||
|
POSTGRESQL_SCHEMES = frozenset({"postgresql", "postgresql+psycopg"})
|
||||||
|
_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_$-]{0,127}$")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class TabularOriginInspection:
|
||||||
|
provider: str
|
||||||
|
schema: tuple[TabularColumn, ...]
|
||||||
|
fingerprint: str
|
||||||
|
row_count: int
|
||||||
|
byte_count: int
|
||||||
|
metadata: Mapping[str, object]
|
||||||
|
health: TabularSourceHealth
|
||||||
|
pushdown: TabularPushdown
|
||||||
|
rows: tuple[Mapping[str, object], ...] = ()
|
||||||
|
diagnostics: tuple[TabularPreviewDiagnostic, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class TabularOriginRead:
|
||||||
|
inspection: TabularOriginInspection
|
||||||
|
rows: tuple[Mapping[str, object], ...]
|
||||||
|
total_rows: int
|
||||||
|
diagnostics: tuple[TabularPreviewDiagnostic, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
class ManagedFileTabularAdapter:
|
||||||
|
def __init__(self, registry: object | None) -> None:
|
||||||
|
self._registry = registry
|
||||||
|
|
||||||
|
def inspect(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
file_asset_id: str,
|
||||||
|
file_version_id: str | None,
|
||||||
|
delimiter: str = ",",
|
||||||
|
sheet_name: str | None = None,
|
||||||
|
) -> TabularOriginInspection:
|
||||||
|
provider = managed_tabular_file_provider(self._registry)
|
||||||
|
if provider is None:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Managed file sources require the Files module."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
metadata = provider.get_tabular_file(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
file_asset_id=file_asset_id,
|
||||||
|
file_version_id=file_version_id,
|
||||||
|
)
|
||||||
|
if metadata is None:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Managed tabular file or version is unavailable."
|
||||||
|
)
|
||||||
|
content = provider.read_tabular_file(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
file_asset_id=file_asset_id,
|
||||||
|
file_version_id=metadata.file_version_id,
|
||||||
|
max_bytes=MAX_FILE_BYTES,
|
||||||
|
)
|
||||||
|
except ManagedTabularFileAccessError as exc:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Managed tabular file access is no longer authorized."
|
||||||
|
) from exc
|
||||||
|
except ManagedTabularFileValidationError as exc:
|
||||||
|
raise TabularSourceValidationError(str(exc)) from exc
|
||||||
|
except ManagedTabularFileError as exc:
|
||||||
|
raise TabularSourceUnavailableError(str(exc)) from exc
|
||||||
|
rows, resolved_sheet = parse_managed_tabular_content(
|
||||||
|
content.payload,
|
||||||
|
filename=content.file.filename,
|
||||||
|
content_type=content.file.content_type,
|
||||||
|
delimiter=delimiter,
|
||||||
|
sheet_name=sheet_name,
|
||||||
|
)
|
||||||
|
schema = infer_tabular_schema(rows)
|
||||||
|
fingerprint = origin_fingerprint(
|
||||||
|
schema,
|
||||||
|
tokens=(
|
||||||
|
"managed_file",
|
||||||
|
content.file.file_asset_id,
|
||||||
|
content.file.file_version_id,
|
||||||
|
content.file.sha256,
|
||||||
|
resolved_sheet or "",
|
||||||
|
delimiter,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
current = provider.get_tabular_file(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
file_asset_id=file_asset_id,
|
||||||
|
)
|
||||||
|
except ManagedTabularFileAccessError as exc:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Managed tabular file access is no longer authorized."
|
||||||
|
) from exc
|
||||||
|
except ManagedTabularFileError as exc:
|
||||||
|
raise TabularSourceUnavailableError(str(exc)) from exc
|
||||||
|
changed = bool(
|
||||||
|
current is not None
|
||||||
|
and current.file_version_id != content.file.file_version_id
|
||||||
|
)
|
||||||
|
health = TabularSourceHealth(
|
||||||
|
status="warning" if changed else "healthy",
|
||||||
|
code=(
|
||||||
|
"files.newer_version_available"
|
||||||
|
if changed
|
||||||
|
else "files.exact_version_ready"
|
||||||
|
),
|
||||||
|
summary=(
|
||||||
|
"A newer managed file version is available for explicit review."
|
||||||
|
if changed
|
||||||
|
else "The exact managed file version passed access and integrity checks."
|
||||||
|
),
|
||||||
|
checked_at=content.file.updated_at,
|
||||||
|
details={
|
||||||
|
"pinned_version_id": content.file.file_version_id,
|
||||||
|
"current_version_id": (
|
||||||
|
current.file_version_id if current is not None else None
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return TabularOriginInspection(
|
||||||
|
provider="managed_file",
|
||||||
|
schema=schema,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
row_count=len(rows),
|
||||||
|
byte_count=len(content.payload),
|
||||||
|
metadata={
|
||||||
|
"origin_kind": "managed_file",
|
||||||
|
"file_asset_id": content.file.file_asset_id,
|
||||||
|
"file_version_id": content.file.file_version_id,
|
||||||
|
"file_sha256": content.file.sha256,
|
||||||
|
"filename": content.file.filename,
|
||||||
|
"content_type": content.file.content_type,
|
||||||
|
"format": "xlsx" if _is_xlsx(content.file.filename, content.file.content_type) else "csv",
|
||||||
|
"delimiter": delimiter,
|
||||||
|
"sheet_name": resolved_sheet,
|
||||||
|
},
|
||||||
|
health=health,
|
||||||
|
pushdown=TabularPushdown(projections=True, pagination=True),
|
||||||
|
rows=rows,
|
||||||
|
diagnostics=(
|
||||||
|
(
|
||||||
|
TabularPreviewDiagnostic(
|
||||||
|
severity="warning",
|
||||||
|
code="files.newer_version_available",
|
||||||
|
message=(
|
||||||
|
"The preview remains pinned to the reviewed file version; "
|
||||||
|
"refresh the source to adopt the newer version."
|
||||||
|
),
|
||||||
|
details=dict(health.details),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if changed
|
||||||
|
else ()
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def read(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
metadata: Mapping[str, object],
|
||||||
|
) -> TabularOriginRead:
|
||||||
|
inspection = self.inspect(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
file_asset_id=_required_metadata(metadata, "file_asset_id"),
|
||||||
|
file_version_id=_required_metadata(metadata, "file_version_id"),
|
||||||
|
delimiter=str(metadata.get("delimiter") or ","),
|
||||||
|
sheet_name=_optional_text(metadata.get("sheet_name")),
|
||||||
|
)
|
||||||
|
expected_sha256 = _required_metadata(metadata, "file_sha256")
|
||||||
|
if inspection.metadata.get("file_sha256") != expected_sha256:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Managed file content no longer matches its exact-version checksum."
|
||||||
|
)
|
||||||
|
return TabularOriginRead(
|
||||||
|
inspection=inspection,
|
||||||
|
rows=inspection.rows,
|
||||||
|
total_rows=inspection.row_count,
|
||||||
|
diagnostics=inspection.diagnostics,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PostgresqlTabularAdapter:
|
||||||
|
"""Read-only SQLAlchemy adapter with a production PostgreSQL allow-list."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
engine_factory: Callable[[URL], Engine] | None = None,
|
||||||
|
allow_sqlite_for_tests: bool = False,
|
||||||
|
) -> None:
|
||||||
|
self._engine_factory = engine_factory or (
|
||||||
|
lambda url: create_engine(url, pool_pre_ping=True)
|
||||||
|
)
|
||||||
|
self._allow_sqlite_for_tests = allow_sqlite_for_tests
|
||||||
|
|
||||||
|
def inspect(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
table_name: str,
|
||||||
|
schema_name: str | None = None,
|
||||||
|
timeout_ms: int = 2_000,
|
||||||
|
) -> TabularOriginInspection:
|
||||||
|
configuration, url, credential_revision = self._connection(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
table_name = _sql_identifier(table_name, "table")
|
||||||
|
schema_name = (
|
||||||
|
_sql_identifier(schema_name, "schema") if schema_name else None
|
||||||
|
)
|
||||||
|
engine = self._engine_factory(_bounded_connection_url(url, timeout_ms))
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
_configure_read_only(connection, url, timeout_ms)
|
||||||
|
table = Table(
|
||||||
|
table_name,
|
||||||
|
MetaData(),
|
||||||
|
schema=schema_name,
|
||||||
|
autoload_with=connection,
|
||||||
|
)
|
||||||
|
schema = tuple(_sql_column(column) for column in table.columns)
|
||||||
|
if not schema:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"SQL tabular sources require at least one column."
|
||||||
|
)
|
||||||
|
row_count = int(
|
||||||
|
connection.execute(select(func.count()).select_from(table)).scalar_one()
|
||||||
|
)
|
||||||
|
except TabularSourceValidationError:
|
||||||
|
raise
|
||||||
|
except SQLAlchemyError as exc:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL source discovery failed; verify the active configuration, credential, table, and provider health."
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
fingerprint = origin_fingerprint(
|
||||||
|
schema,
|
||||||
|
tokens=(
|
||||||
|
"postgresql",
|
||||||
|
configuration.id,
|
||||||
|
configuration.effective_hash,
|
||||||
|
credential_revision or "",
|
||||||
|
schema_name or "",
|
||||||
|
table_name,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return TabularOriginInspection(
|
||||||
|
provider="postgresql",
|
||||||
|
schema=schema,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
row_count=row_count,
|
||||||
|
byte_count=0,
|
||||||
|
metadata={
|
||||||
|
"origin_kind": "sql",
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
"configuration_hash": configuration.effective_hash,
|
||||||
|
"credential_revision": credential_revision,
|
||||||
|
"schema_name": schema_name,
|
||||||
|
"table_name": table_name,
|
||||||
|
},
|
||||||
|
health=TabularSourceHealth(
|
||||||
|
status="healthy",
|
||||||
|
code="sql.source_ready",
|
||||||
|
summary="The governed PostgreSQL source is reachable and its schema was discovered.",
|
||||||
|
checked_at=configuration.updated_at,
|
||||||
|
details={
|
||||||
|
"configuration_id": configuration.id,
|
||||||
|
"configuration_revision": configuration.resource_revision,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
pushdown=TabularPushdown(projections=True, pagination=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
def read(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
metadata: Mapping[str, object],
|
||||||
|
columns: Sequence[str],
|
||||||
|
offset: int,
|
||||||
|
limit: int,
|
||||||
|
timeout_ms: int,
|
||||||
|
) -> TabularOriginRead:
|
||||||
|
configuration_id = _required_metadata(metadata, "configuration_id")
|
||||||
|
current_configuration, _current_url, current_credential_revision = (
|
||||||
|
self._connection(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if current_configuration.effective_hash != metadata.get(
|
||||||
|
"configuration_hash"
|
||||||
|
):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The SQL connector configuration changed; refresh the source before previewing it."
|
||||||
|
)
|
||||||
|
if current_credential_revision != metadata.get("credential_revision"):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The SQL source credential changed; refresh the source before previewing it."
|
||||||
|
)
|
||||||
|
inspection = self.inspect(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
table_name=_required_metadata(metadata, "table_name"),
|
||||||
|
schema_name=_optional_text(metadata.get("schema_name")),
|
||||||
|
timeout_ms=timeout_ms,
|
||||||
|
)
|
||||||
|
if inspection.fingerprint != metadata.get("discovery_fingerprint"):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The SQL source schema drifted; refresh and review the source before previewing it."
|
||||||
|
)
|
||||||
|
configuration, url, _credential_revision = self._connection(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
)
|
||||||
|
engine = self._engine_factory(_bounded_connection_url(url, timeout_ms))
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
_configure_read_only(connection, url, timeout_ms)
|
||||||
|
table = Table(
|
||||||
|
_required_metadata(metadata, "table_name"),
|
||||||
|
MetaData(),
|
||||||
|
schema=_optional_text(metadata.get("schema_name")),
|
||||||
|
autoload_with=connection,
|
||||||
|
)
|
||||||
|
selected_names = tuple(dict.fromkeys(str(item) for item in columns))
|
||||||
|
selected = (
|
||||||
|
[table.c[name] for name in selected_names]
|
||||||
|
if selected_names
|
||||||
|
else list(table.columns)
|
||||||
|
)
|
||||||
|
statement = select(*selected).offset(max(0, int(offset))).limit(
|
||||||
|
max(1, int(limit))
|
||||||
|
)
|
||||||
|
rows = tuple(
|
||||||
|
_json_row(dict(row._mapping))
|
||||||
|
for row in connection.execute(statement)
|
||||||
|
)
|
||||||
|
except KeyError as exc:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Unknown SQL source column: {exc.args[0]}"
|
||||||
|
) from exc
|
||||||
|
except SQLAlchemyError as exc:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL source preview failed or exceeded its provider budget."
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
return TabularOriginRead(
|
||||||
|
inspection=inspection,
|
||||||
|
rows=rows,
|
||||||
|
total_rows=inspection.row_count,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _connection(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
configuration_id: str,
|
||||||
|
) -> tuple[ConnectorConfiguration, URL, str | None]:
|
||||||
|
configuration = session.scalar(
|
||||||
|
select(ConnectorConfiguration).where(
|
||||||
|
ConnectorConfiguration.id == configuration_id,
|
||||||
|
ConnectorConfiguration.tenant_id == principal.tenant_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if configuration is None:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL connector configuration is unavailable."
|
||||||
|
)
|
||||||
|
if configuration.status != "active":
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL connector configuration is not active."
|
||||||
|
)
|
||||||
|
endpoint = _optional_text(configuration.endpoint_url)
|
||||||
|
if not endpoint:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL connector configuration has no endpoint."
|
||||||
|
)
|
||||||
|
specification = dict(configuration.effective_configuration or {})
|
||||||
|
provider = str(specification.get("provider") or "").strip().casefold()
|
||||||
|
protocol = str(specification.get("protocol") or "").strip().casefold()
|
||||||
|
if provider not in {"postgres", "postgresql", "sql"} or protocol not in {
|
||||||
|
"postgres",
|
||||||
|
"postgresql",
|
||||||
|
"sql",
|
||||||
|
}:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The selected connector configuration is not a PostgreSQL tabular reader."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
url = make_url(endpoint)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"SQL connector endpoint is invalid."
|
||||||
|
) from exc
|
||||||
|
if url.username or url.password:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"SQL connector endpoints must not contain credentials."
|
||||||
|
)
|
||||||
|
if any(is_sensitive_key(key) for key in url.query):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"SQL connector endpoint query parameters must not contain credentials."
|
||||||
|
)
|
||||||
|
allowed_schemes = set(POSTGRESQL_SCHEMES)
|
||||||
|
if self._allow_sqlite_for_tests:
|
||||||
|
allowed_schemes.update({"sqlite", "sqlite+pysqlite"})
|
||||||
|
if url.drivername not in allowed_schemes:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Only the governed PostgreSQL tabular adapter is enabled."
|
||||||
|
)
|
||||||
|
credential_revision: str | None = None
|
||||||
|
credential_ref = _optional_text(configuration.credential_ref)
|
||||||
|
if credential_ref:
|
||||||
|
try:
|
||||||
|
credential = resolve_credential_envelope(
|
||||||
|
session,
|
||||||
|
credential_id=credential_ref,
|
||||||
|
context=CredentialAccessContext(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=_principal_user_id(principal),
|
||||||
|
group_ids=frozenset(principal.principal.group_ids),
|
||||||
|
target_scope_type="tenant",
|
||||||
|
target_scope_id=principal.tenant_id,
|
||||||
|
module_id="connectors",
|
||||||
|
server_ref=endpoint,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except CredentialEnvelopeError as exc:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL source credential is unavailable, inactive, or outside its allowed scope."
|
||||||
|
) from exc
|
||||||
|
public_data = dict(credential.public_data)
|
||||||
|
secret_data = dict(credential.secret_data)
|
||||||
|
username = _optional_text(
|
||||||
|
public_data.get("username")
|
||||||
|
or secret_data.get("username")
|
||||||
|
or secret_data.get("user")
|
||||||
|
)
|
||||||
|
password = _optional_text(secret_data.get("password"))
|
||||||
|
if url.drivername in POSTGRESQL_SCHEMES and (not username or not password):
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"SQL source credential does not provide a username and password."
|
||||||
|
)
|
||||||
|
url = url.set(username=username, password=password)
|
||||||
|
credential_revision = credential.revision
|
||||||
|
elif url.drivername in POSTGRESQL_SCHEMES:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"PostgreSQL tabular sources require a credential envelope reference."
|
||||||
|
)
|
||||||
|
return configuration, url, credential_revision
|
||||||
|
|
||||||
|
|
||||||
|
def parse_managed_tabular_content(
|
||||||
|
payload: bytes,
|
||||||
|
*,
|
||||||
|
filename: str,
|
||||||
|
content_type: str | None,
|
||||||
|
delimiter: str,
|
||||||
|
sheet_name: str | None,
|
||||||
|
) -> tuple[tuple[Mapping[str, object], ...], str | None]:
|
||||||
|
if len(payload) > MAX_FILE_BYTES:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Managed tabular files are limited to {MAX_FILE_BYTES:,} bytes."
|
||||||
|
)
|
||||||
|
if _is_xlsx(filename, content_type):
|
||||||
|
return _parse_xlsx(payload, sheet_name=sheet_name)
|
||||||
|
try:
|
||||||
|
text = payload.decode("utf-8-sig")
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed CSV files must use UTF-8 encoding."
|
||||||
|
) from exc
|
||||||
|
return (
|
||||||
|
tuple(parse_tabular_csv(text, delimiter=delimiter, max_rows=MAX_FILE_ROWS)),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_xlsx(
|
||||||
|
payload: bytes,
|
||||||
|
*,
|
||||||
|
sheet_name: str | None,
|
||||||
|
) -> tuple[tuple[Mapping[str, object], ...], str]:
|
||||||
|
_validate_xlsx_archive(payload)
|
||||||
|
try:
|
||||||
|
workbook = load_workbook(
|
||||||
|
BytesIO(payload),
|
||||||
|
read_only=True,
|
||||||
|
data_only=True,
|
||||||
|
keep_links=False,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content could not be parsed."
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
available = tuple(workbook.sheetnames)
|
||||||
|
if not available:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content requires at least one worksheet."
|
||||||
|
)
|
||||||
|
selected_name = _optional_text(sheet_name) or available[0]
|
||||||
|
if selected_name not in available:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Managed XLSX worksheet {selected_name!r} was not found."
|
||||||
|
)
|
||||||
|
worksheet = workbook[selected_name]
|
||||||
|
iterator = worksheet.iter_rows(values_only=True)
|
||||||
|
try:
|
||||||
|
raw_headers = next(iterator)
|
||||||
|
except StopIteration as exc:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX worksheet requires a header row."
|
||||||
|
) from exc
|
||||||
|
headers = _xlsx_headers(raw_headers)
|
||||||
|
rows: list[Mapping[str, object]] = []
|
||||||
|
for values in iterator:
|
||||||
|
if len(rows) >= MAX_FILE_ROWS:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Managed XLSX worksheets are limited to {MAX_FILE_ROWS:,} data rows."
|
||||||
|
)
|
||||||
|
normalized = tuple(values[: len(headers)])
|
||||||
|
if all(value in (None, "") for value in normalized):
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
header: _json_value(
|
||||||
|
normalized[index] if index < len(normalized) else None
|
||||||
|
)
|
||||||
|
for index, header in enumerate(headers)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return tuple(rows), selected_name
|
||||||
|
finally:
|
||||||
|
workbook.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_xlsx_archive(payload: bytes) -> None:
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(BytesIO(payload)) as archive:
|
||||||
|
entries = archive.infolist()
|
||||||
|
if len(entries) > MAX_XLSX_ENTRIES:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content contains too many archive entries."
|
||||||
|
)
|
||||||
|
expanded = sum(max(0, item.file_size) for item in entries)
|
||||||
|
if expanded > MAX_XLSX_EXPANDED_BYTES:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content exceeds the expanded-size limit."
|
||||||
|
)
|
||||||
|
compressed = sum(max(1, item.compress_size) for item in entries)
|
||||||
|
if expanded > compressed * MAX_XLSX_COMPRESSION_RATIO:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content exceeds the compression-ratio limit."
|
||||||
|
)
|
||||||
|
if any(
|
||||||
|
item.filename.startswith(("/", "\\"))
|
||||||
|
or ".." in item.filename.replace("\\", "/").split("/")
|
||||||
|
for item in entries
|
||||||
|
):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content contains an unsafe archive path."
|
||||||
|
)
|
||||||
|
except zipfile.BadZipFile as exc:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content is not a valid workbook archive."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _xlsx_headers(values: Sequence[object]) -> tuple[str, ...]:
|
||||||
|
if len(values) > MAX_FILE_COLUMNS:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Managed XLSX worksheets are limited to {MAX_FILE_COLUMNS:,} columns."
|
||||||
|
)
|
||||||
|
headers = tuple(str(value or "").strip() for value in values)
|
||||||
|
while headers and not headers[-1]:
|
||||||
|
headers = headers[:-1]
|
||||||
|
if not headers or any(not header for header in headers):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX content requires a non-empty header row."
|
||||||
|
)
|
||||||
|
if len(set(headers)) != len(headers):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Managed XLSX column names must be unique."
|
||||||
|
)
|
||||||
|
return headers
|
||||||
|
|
||||||
|
|
||||||
|
def infer_tabular_schema(
|
||||||
|
rows: Sequence[Mapping[str, object]],
|
||||||
|
) -> tuple[TabularColumn, ...]:
|
||||||
|
names: list[str] = []
|
||||||
|
for row in rows:
|
||||||
|
for name in row:
|
||||||
|
if name not in names:
|
||||||
|
names.append(name)
|
||||||
|
result: list[TabularColumn] = []
|
||||||
|
for name in names:
|
||||||
|
values = [row.get(name) for row in rows]
|
||||||
|
concrete = [value for value in values if value is not None]
|
||||||
|
data_type = _type_name(concrete[0]) if concrete else "unknown"
|
||||||
|
if any(_type_name(value) != data_type for value in concrete[1:]):
|
||||||
|
data_type = "mixed"
|
||||||
|
result.append(
|
||||||
|
TabularColumn(
|
||||||
|
name=name,
|
||||||
|
data_type=data_type,
|
||||||
|
nullable=len(concrete) != len(values),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def origin_fingerprint(
|
||||||
|
schema: Sequence[TabularColumn],
|
||||||
|
*,
|
||||||
|
tokens: Sequence[str],
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"name": column.name,
|
||||||
|
"data_type": column.data_type,
|
||||||
|
"nullable": column.nullable,
|
||||||
|
}
|
||||||
|
for column in schema
|
||||||
|
],
|
||||||
|
"tokens": list(tokens),
|
||||||
|
}
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _sql_column(column: Any) -> TabularColumn:
|
||||||
|
column_type = column.type
|
||||||
|
if isinstance(column_type, Boolean):
|
||||||
|
data_type = "boolean"
|
||||||
|
elif isinstance(column_type, (Integer, BigInteger)):
|
||||||
|
data_type = "integer"
|
||||||
|
elif isinstance(column_type, (Numeric, Float)):
|
||||||
|
data_type = "number"
|
||||||
|
elif isinstance(column_type, DateTime):
|
||||||
|
data_type = "datetime"
|
||||||
|
elif isinstance(column_type, Date):
|
||||||
|
data_type = "date"
|
||||||
|
elif isinstance(column_type, JSON):
|
||||||
|
data_type = "object"
|
||||||
|
elif isinstance(column_type, LargeBinary):
|
||||||
|
data_type = "binary"
|
||||||
|
elif isinstance(column_type, (String, Text)):
|
||||||
|
data_type = "string"
|
||||||
|
else:
|
||||||
|
data_type = str(column_type).casefold()
|
||||||
|
return TabularColumn(
|
||||||
|
name=str(column.name),
|
||||||
|
data_type=data_type,
|
||||||
|
nullable=bool(column.nullable),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configure_read_only(connection: Any, url: URL, timeout_ms: int) -> None:
|
||||||
|
if url.drivername not in POSTGRESQL_SCHEMES:
|
||||||
|
return
|
||||||
|
effective_timeout = max(1, min(int(timeout_ms), 30_000))
|
||||||
|
connection.exec_driver_sql("SET TRANSACTION READ ONLY")
|
||||||
|
connection.exec_driver_sql(f"SET LOCAL statement_timeout = {effective_timeout}")
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_connection_url(url: URL, timeout_ms: int) -> URL:
|
||||||
|
if url.drivername not in POSTGRESQL_SCHEMES:
|
||||||
|
return url
|
||||||
|
connect_timeout = max(1, math.ceil(min(int(timeout_ms), 30_000) / 1_000))
|
||||||
|
return url.update_query_dict({"connect_timeout": str(connect_timeout)})
|
||||||
|
|
||||||
|
|
||||||
|
def _sql_identifier(value: object, label: str) -> str:
|
||||||
|
normalized = str(value or "").strip()
|
||||||
|
if not _IDENTIFIER.fullmatch(normalized):
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"SQL {label} names must use a simple identifier."
|
||||||
|
)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _required_metadata(metadata: Mapping[str, object], key: str) -> str:
|
||||||
|
value = _optional_text(metadata.get(key))
|
||||||
|
if not value:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Tabular source metadata is missing {key}."
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: object | None) -> str | None:
|
||||||
|
normalized = str(value or "").strip()
|
||||||
|
return normalized or None
|
||||||
|
|
||||||
|
|
||||||
|
def _principal_user_id(principal: ApiPrincipal) -> str | None:
|
||||||
|
return _optional_text(getattr(principal.user, "id", None))
|
||||||
|
|
||||||
|
|
||||||
|
def _is_xlsx(filename: str, content_type: str | None) -> bool:
|
||||||
|
normalized_type = str(content_type or "").split(";", 1)[0].strip().casefold()
|
||||||
|
return str(filename or "").strip().casefold().endswith(".xlsx") or normalized_type == (
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _json_row(row: Mapping[str, object]) -> dict[str, object]:
|
||||||
|
return {str(key): _json_value(value) for key, value in row.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def _json_value(value: object) -> object:
|
||||||
|
if value is None or isinstance(value, (str, bool, int)):
|
||||||
|
return value
|
||||||
|
if isinstance(value, float):
|
||||||
|
if not math.isfinite(value):
|
||||||
|
return str(value)
|
||||||
|
return value
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return float(value)
|
||||||
|
if isinstance(value, (datetime, date)):
|
||||||
|
return value.isoformat()
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.hex()
|
||||||
|
if isinstance(value, (list, dict)):
|
||||||
|
return json.loads(json.dumps(value, default=str))
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _type_name(value: object) -> str:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "boolean"
|
||||||
|
if isinstance(value, int):
|
||||||
|
return "integer"
|
||||||
|
if isinstance(value, (float, Decimal)):
|
||||||
|
return "number"
|
||||||
|
if isinstance(value, str):
|
||||||
|
return "string"
|
||||||
|
if isinstance(value, list):
|
||||||
|
return "array"
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return "object"
|
||||||
|
return type(value).__name__.casefold()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"MAX_FILE_BYTES",
|
||||||
|
"MAX_FILE_ROWS",
|
||||||
|
"ManagedFileTabularAdapter",
|
||||||
|
"PostgresqlTabularAdapter",
|
||||||
|
"TabularOriginInspection",
|
||||||
|
"TabularOriginRead",
|
||||||
|
"infer_tabular_schema",
|
||||||
|
"origin_fingerprint",
|
||||||
|
"parse_managed_tabular_content",
|
||||||
|
]
|
||||||
@@ -0,0 +1,770 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
|
from sqlalchemy import or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, has_scope
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularColumn,
|
||||||
|
TabularPreviewDiagnostic,
|
||||||
|
TabularPushdown,
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularReadResult,
|
||||||
|
TabularSnapshotInput,
|
||||||
|
TabularSource,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceNotFoundError,
|
||||||
|
TabularSourceHealth,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
parse_tabular_csv,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime import get_registry
|
||||||
|
from govoplan_core.db.base import utcnow
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.tabular_adapters import (
|
||||||
|
ManagedFileTabularAdapter,
|
||||||
|
PostgresqlTabularAdapter,
|
||||||
|
TabularOriginInspection,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
READ_SCOPE = "connectors:source:read"
|
||||||
|
WRITE_SCOPE = "connectors:source:write"
|
||||||
|
ADMIN_SCOPE = "connectors:source:admin"
|
||||||
|
MAX_SNAPSHOT_ROWS = 10_000
|
||||||
|
MAX_SNAPSHOT_BYTES = 5_000_000
|
||||||
|
MAX_READ_ROWS = 500
|
||||||
|
MAX_READ_BYTES = 1_000_000
|
||||||
|
MAX_READ_TIMEOUT_MS = 2_000
|
||||||
|
|
||||||
|
|
||||||
|
class SqlTabularSourceProvider:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
registry: object | None = None,
|
||||||
|
clock: Callable[[], float] = time.monotonic,
|
||||||
|
sql_adapter: PostgresqlTabularAdapter | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._registry = registry
|
||||||
|
self._clock = clock
|
||||||
|
self._sql_adapter = sql_adapter or PostgresqlTabularAdapter()
|
||||||
|
|
||||||
|
def list_sources(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
query: str = "",
|
||||||
|
limit: int = 100,
|
||||||
|
) -> Sequence[TabularSource]:
|
||||||
|
db, api_principal = _context(session, principal, READ_SCOPE)
|
||||||
|
normalized_query = str(query or "").strip()
|
||||||
|
statement = select(ConnectorTabularSource).where(
|
||||||
|
ConnectorTabularSource.tenant_id == api_principal.tenant_id,
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None),
|
||||||
|
ConnectorTabularSource.status == "active",
|
||||||
|
)
|
||||||
|
if normalized_query:
|
||||||
|
pattern = f"%{_escape_like(normalized_query)}%"
|
||||||
|
statement = statement.where(
|
||||||
|
or_(
|
||||||
|
ConnectorTabularSource.name.ilike(pattern, escape="\\"),
|
||||||
|
ConnectorTabularSource.source_name.ilike(pattern, escape="\\"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
statement = statement.order_by(
|
||||||
|
ConnectorTabularSource.updated_at.desc(),
|
||||||
|
ConnectorTabularSource.name,
|
||||||
|
).limit(max(1, min(int(limit), 100)))
|
||||||
|
return tuple(_source_dto(item) for item in db.scalars(statement))
|
||||||
|
|
||||||
|
def get_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
source_ref: str,
|
||||||
|
) -> TabularSource | None:
|
||||||
|
db, api_principal = _context(session, principal, READ_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=source_ref,
|
||||||
|
)
|
||||||
|
return _source_dto(item) if item is not None else None
|
||||||
|
|
||||||
|
def read_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: TabularReadRequest,
|
||||||
|
) -> TabularReadResult:
|
||||||
|
db, api_principal = _context(session, principal, READ_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=request.source_ref,
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise TabularSourceNotFoundError("Tabular source not found.")
|
||||||
|
if request.expected_fingerprint and request.expected_fingerprint != item.fingerprint:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The source fingerprint changed; refresh the source node before running it."
|
||||||
|
)
|
||||||
|
|
||||||
|
limit = max(1, min(int(request.limit), MAX_READ_ROWS))
|
||||||
|
byte_limit = max(2, min(int(request.max_bytes), MAX_READ_BYTES))
|
||||||
|
timeout_ms = max(1, min(int(request.timeout_ms), MAX_READ_TIMEOUT_MS))
|
||||||
|
offset = max(0, int(request.offset))
|
||||||
|
diagnostics: list[TabularPreviewDiagnostic] = []
|
||||||
|
if limit != request.limit:
|
||||||
|
diagnostics.append(
|
||||||
|
_preview_diagnostic(
|
||||||
|
"preview.row_limit_tightened",
|
||||||
|
"The provider tightened the requested row limit.",
|
||||||
|
requested=request.limit,
|
||||||
|
effective=limit,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if byte_limit != request.max_bytes:
|
||||||
|
diagnostics.append(
|
||||||
|
_preview_diagnostic(
|
||||||
|
"preview.byte_limit_tightened",
|
||||||
|
"The provider tightened the requested byte limit.",
|
||||||
|
requested=request.max_bytes,
|
||||||
|
effective=byte_limit,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if timeout_ms != request.timeout_ms:
|
||||||
|
diagnostics.append(
|
||||||
|
_preview_diagnostic(
|
||||||
|
"preview.timeout_tightened",
|
||||||
|
"The provider tightened the requested time limit.",
|
||||||
|
requested=request.timeout_ms,
|
||||||
|
effective=timeout_ms,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
selected_columns = tuple(dict.fromkeys(request.columns))
|
||||||
|
known_columns = {column["name"] for column in item.schema_}
|
||||||
|
unknown_columns = [column for column in selected_columns if column not in known_columns]
|
||||||
|
if unknown_columns:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Unknown source columns: {', '.join(unknown_columns)}"
|
||||||
|
)
|
||||||
|
started = self._clock()
|
||||||
|
source_rows: Sequence[Mapping[str, object]]
|
||||||
|
total_rows = item.row_count
|
||||||
|
source = _source_dto(item)
|
||||||
|
base_offset = offset
|
||||||
|
if item.provider == "managed_file":
|
||||||
|
read = self._file_adapter().read(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
metadata=item.metadata_,
|
||||||
|
)
|
||||||
|
if read.inspection.fingerprint != item.fingerprint:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"The managed file source changed; refresh and review it before previewing."
|
||||||
|
)
|
||||||
|
source_rows = read.rows[offset:]
|
||||||
|
total_rows = read.total_rows
|
||||||
|
source = _source_dto(item, inspection=read.inspection)
|
||||||
|
diagnostics.extend(read.diagnostics)
|
||||||
|
elif item.provider == "postgresql":
|
||||||
|
read = self._sql_adapter.read(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
metadata=item.metadata_,
|
||||||
|
columns=selected_columns,
|
||||||
|
offset=offset,
|
||||||
|
limit=limit + 1,
|
||||||
|
timeout_ms=timeout_ms,
|
||||||
|
)
|
||||||
|
source_rows = read.rows
|
||||||
|
total_rows = read.total_rows
|
||||||
|
source = _source_dto(item, inspection=read.inspection)
|
||||||
|
diagnostics.extend(read.diagnostics)
|
||||||
|
else:
|
||||||
|
source_rows = item.rows[offset:]
|
||||||
|
|
||||||
|
if int(max(0.0, self._clock() - started) * 1_000) >= timeout_ms:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Tabular source preview exceeded its time budget."
|
||||||
|
)
|
||||||
|
|
||||||
|
rows: list[dict[str, object]] = []
|
||||||
|
returned_bytes = 2
|
||||||
|
stopped_for = ""
|
||||||
|
for row in source_rows:
|
||||||
|
if len(rows) >= limit:
|
||||||
|
stopped_for = "rows"
|
||||||
|
break
|
||||||
|
elapsed_ms = int(max(0.0, self._clock() - started) * 1_000)
|
||||||
|
if elapsed_ms >= timeout_ms:
|
||||||
|
if not rows:
|
||||||
|
raise TabularSourceUnavailableError(
|
||||||
|
"Tabular source preview exceeded its time budget."
|
||||||
|
)
|
||||||
|
stopped_for = "time"
|
||||||
|
break
|
||||||
|
selected = {
|
||||||
|
key: value
|
||||||
|
for key, value in row.items()
|
||||||
|
if not selected_columns or key in selected_columns
|
||||||
|
}
|
||||||
|
row_bytes = len(
|
||||||
|
json.dumps(
|
||||||
|
selected,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
default=str,
|
||||||
|
).encode("utf-8")
|
||||||
|
)
|
||||||
|
additional_bytes = row_bytes + (1 if rows else 0)
|
||||||
|
if returned_bytes + additional_bytes > byte_limit:
|
||||||
|
if not rows:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"A single source row exceeds the preview byte limit."
|
||||||
|
)
|
||||||
|
stopped_for = "bytes"
|
||||||
|
break
|
||||||
|
rows.append(selected)
|
||||||
|
returned_bytes += additional_bytes
|
||||||
|
elapsed_ms = int(max(0.0, self._clock() - started) * 1_000)
|
||||||
|
if stopped_for:
|
||||||
|
labels = {
|
||||||
|
"rows": ("preview.row_limit_reached", "row"),
|
||||||
|
"bytes": ("preview.byte_limit_reached", "byte"),
|
||||||
|
"time": ("preview.timeout_reached", "time"),
|
||||||
|
}
|
||||||
|
code, label = labels[stopped_for]
|
||||||
|
diagnostics.append(
|
||||||
|
TabularPreviewDiagnostic(
|
||||||
|
severity="warning",
|
||||||
|
code=code,
|
||||||
|
message=f"The preview stopped at its effective {label} limit.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return TabularReadResult(
|
||||||
|
source=source,
|
||||||
|
rows=tuple(rows),
|
||||||
|
total_rows=total_rows,
|
||||||
|
truncated=base_offset + len(rows) < total_rows,
|
||||||
|
returned_bytes=returned_bytes,
|
||||||
|
elapsed_ms=elapsed_ms,
|
||||||
|
effective_row_limit=limit,
|
||||||
|
effective_byte_limit=byte_limit,
|
||||||
|
effective_timeout_ms=timeout_ms,
|
||||||
|
diagnostics=tuple(diagnostics),
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_file_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
source_name: str,
|
||||||
|
file_asset_id: str,
|
||||||
|
file_version_id: str | None = None,
|
||||||
|
description: str | None = None,
|
||||||
|
delimiter: str = ",",
|
||||||
|
sheet_name: str | None = None,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
inspection = self._file_adapter().inspect(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
file_asset_id=file_asset_id,
|
||||||
|
file_version_id=file_version_id,
|
||||||
|
delimiter=delimiter,
|
||||||
|
sheet_name=sheet_name,
|
||||||
|
)
|
||||||
|
return self._create_origin(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
name=name,
|
||||||
|
source_name=source_name,
|
||||||
|
description=description,
|
||||||
|
inspection=inspection,
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_sql_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
source_name: str,
|
||||||
|
configuration_id: str,
|
||||||
|
table_name: str,
|
||||||
|
schema_name: str | None = None,
|
||||||
|
description: str | None = None,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
inspection = self._sql_adapter.inspect(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
configuration_id=configuration_id,
|
||||||
|
table_name=table_name,
|
||||||
|
schema_name=schema_name,
|
||||||
|
timeout_ms=MAX_READ_TIMEOUT_MS,
|
||||||
|
)
|
||||||
|
return self._create_origin(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
name=name,
|
||||||
|
source_name=source_name,
|
||||||
|
description=description,
|
||||||
|
inspection=inspection,
|
||||||
|
)
|
||||||
|
|
||||||
|
def refresh_source(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
source_ref: str,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=source_ref,
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise TabularSourceNotFoundError("Tabular source not found.")
|
||||||
|
if item.provider == "managed_file":
|
||||||
|
inspection = self._file_adapter().inspect(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
file_asset_id=str(item.metadata_.get("file_asset_id") or ""),
|
||||||
|
file_version_id=None,
|
||||||
|
delimiter=str(item.metadata_.get("delimiter") or ","),
|
||||||
|
sheet_name=_clean_optional(item.metadata_.get("sheet_name")),
|
||||||
|
)
|
||||||
|
elif item.provider == "postgresql":
|
||||||
|
inspection = self._sql_adapter.inspect(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
configuration_id=str(
|
||||||
|
item.metadata_.get("configuration_id") or ""
|
||||||
|
),
|
||||||
|
table_name=str(item.metadata_.get("table_name") or ""),
|
||||||
|
schema_name=_clean_optional(item.metadata_.get("schema_name")),
|
||||||
|
timeout_ms=MAX_READ_TIMEOUT_MS,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Immutable snapshots cannot be refreshed; import a replacement snapshot."
|
||||||
|
)
|
||||||
|
item.schema_version += 1
|
||||||
|
item.schema_ = [_column_payload(column) for column in inspection.schema]
|
||||||
|
item.fingerprint = inspection.fingerprint
|
||||||
|
item.row_count = inspection.row_count
|
||||||
|
item.byte_count = inspection.byte_count
|
||||||
|
item.metadata_ = {
|
||||||
|
**dict(inspection.metadata),
|
||||||
|
"discovery_fingerprint": inspection.fingerprint,
|
||||||
|
}
|
||||||
|
item.updated_by = _actor_id(api_principal)
|
||||||
|
db.flush()
|
||||||
|
return _source_dto(item, inspection=inspection)
|
||||||
|
|
||||||
|
def _create_origin(
|
||||||
|
self,
|
||||||
|
db: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
source_name: str,
|
||||||
|
description: str | None,
|
||||||
|
inspection: TabularOriginInspection,
|
||||||
|
) -> TabularSource:
|
||||||
|
clean_name = str(name or "").strip()
|
||||||
|
clean_source_name = str(source_name or "").strip()
|
||||||
|
if not clean_name or not clean_source_name:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
"Tabular source name and source identifier are required."
|
||||||
|
)
|
||||||
|
existing = db.scalar(
|
||||||
|
select(ConnectorTabularSource.id).where(
|
||||||
|
ConnectorTabularSource.tenant_id == principal.tenant_id,
|
||||||
|
ConnectorTabularSource.source_name == clean_source_name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"A tabular source named {clean_source_name!r} already exists."
|
||||||
|
)
|
||||||
|
actor_id = _actor_id(principal)
|
||||||
|
item = ConnectorTabularSource(
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
provider=inspection.provider,
|
||||||
|
source_name=clean_source_name,
|
||||||
|
name=clean_name,
|
||||||
|
description=_clean_optional(description),
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[_column_payload(column) for column in inspection.schema],
|
||||||
|
rows=[],
|
||||||
|
fingerprint=inspection.fingerprint,
|
||||||
|
row_count=inspection.row_count,
|
||||||
|
byte_count=inspection.byte_count,
|
||||||
|
metadata_={
|
||||||
|
**dict(inspection.metadata),
|
||||||
|
"discovery_fingerprint": inspection.fingerprint,
|
||||||
|
},
|
||||||
|
created_by=actor_id,
|
||||||
|
updated_by=actor_id,
|
||||||
|
)
|
||||||
|
db.add(item)
|
||||||
|
db.flush()
|
||||||
|
return _source_dto(item, inspection=inspection)
|
||||||
|
|
||||||
|
def _file_adapter(self) -> ManagedFileTabularAdapter:
|
||||||
|
return ManagedFileTabularAdapter(self._registry or get_registry())
|
||||||
|
|
||||||
|
def create_snapshot(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
snapshot: TabularSnapshotInput,
|
||||||
|
source_id: str | None = None,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
name = snapshot.name.strip()
|
||||||
|
source_name = snapshot.source_name.strip()
|
||||||
|
if not name:
|
||||||
|
raise TabularSourceValidationError("Snapshot name is required.")
|
||||||
|
if not source_name:
|
||||||
|
raise TabularSourceValidationError("Snapshot source name is required.")
|
||||||
|
if len(snapshot.rows) > MAX_SNAPSHOT_ROWS:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Snapshots are limited to {MAX_SNAPSHOT_ROWS:,} rows."
|
||||||
|
)
|
||||||
|
rows = [_json_row(row) for row in snapshot.rows]
|
||||||
|
encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
||||||
|
if len(encoded) > MAX_SNAPSHOT_BYTES:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"Snapshots are limited to {MAX_SNAPSHOT_BYTES // 1_000_000} MB."
|
||||||
|
)
|
||||||
|
existing = db.scalar(
|
||||||
|
select(ConnectorTabularSource.id).where(
|
||||||
|
ConnectorTabularSource.tenant_id == api_principal.tenant_id,
|
||||||
|
ConnectorTabularSource.source_name == source_name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
raise TabularSourceValidationError(
|
||||||
|
f"A tabular source named {source_name!r} already exists."
|
||||||
|
)
|
||||||
|
schema = infer_schema(rows)
|
||||||
|
fingerprint = snapshot_fingerprint(rows, schema)
|
||||||
|
actor_id = _actor_id(api_principal)
|
||||||
|
item = ConnectorTabularSource(
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
provider="snapshot",
|
||||||
|
source_name=source_name,
|
||||||
|
name=name,
|
||||||
|
description=_clean_optional(snapshot.description),
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[_column_payload(column) for column in schema],
|
||||||
|
rows=rows,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
row_count=len(rows),
|
||||||
|
byte_count=len(encoded),
|
||||||
|
metadata_=dict(snapshot.metadata),
|
||||||
|
created_by=actor_id,
|
||||||
|
updated_by=actor_id,
|
||||||
|
)
|
||||||
|
if source_id:
|
||||||
|
item.id = source_id
|
||||||
|
db.add(item)
|
||||||
|
db.flush()
|
||||||
|
return _source_dto(item)
|
||||||
|
|
||||||
|
def delete_snapshot(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
source_ref: str,
|
||||||
|
) -> TabularSource:
|
||||||
|
db, api_principal = _context(session, principal, WRITE_SCOPE)
|
||||||
|
item = _source_record(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
source_ref=source_ref,
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise TabularSourceNotFoundError("Tabular source not found.")
|
||||||
|
item.deleted_at = utcnow()
|
||||||
|
item.status = "retired"
|
||||||
|
item.updated_by = _actor_id(api_principal)
|
||||||
|
db.flush()
|
||||||
|
return _source_dto(item)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_csv_snapshot(csv_text: str, *, delimiter: str) -> tuple[Mapping[str, object], ...]:
|
||||||
|
return parse_tabular_csv(
|
||||||
|
csv_text,
|
||||||
|
delimiter=delimiter,
|
||||||
|
max_rows=MAX_SNAPSHOT_ROWS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def infer_schema(rows: Sequence[Mapping[str, object]]) -> tuple[TabularColumn, ...]:
|
||||||
|
names: list[str] = []
|
||||||
|
for row in rows:
|
||||||
|
for name in row:
|
||||||
|
if name not in names:
|
||||||
|
names.append(name)
|
||||||
|
result: list[TabularColumn] = []
|
||||||
|
for name in names:
|
||||||
|
values = [row.get(name) for row in rows]
|
||||||
|
concrete = [value for value in values if value is not None]
|
||||||
|
data_type = _type_name(concrete[0]) if concrete else "unknown"
|
||||||
|
if any(_type_name(value) != data_type for value in concrete[1:]):
|
||||||
|
data_type = "mixed"
|
||||||
|
result.append(
|
||||||
|
TabularColumn(
|
||||||
|
name=name,
|
||||||
|
data_type=data_type,
|
||||||
|
nullable=len(concrete) != len(values),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_fingerprint(
|
||||||
|
rows: Sequence[Mapping[str, object]],
|
||||||
|
schema: Sequence[TabularColumn],
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"schema": [_column_payload(column) for column in schema],
|
||||||
|
"rows": [dict(row) for row in rows],
|
||||||
|
}
|
||||||
|
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
|
||||||
|
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _source_record(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
source_ref: str,
|
||||||
|
) -> ConnectorTabularSource | None:
|
||||||
|
prefix, separator, source_id = str(source_ref or "").partition(":")
|
||||||
|
provider_by_prefix = {
|
||||||
|
"snapshot": "snapshot",
|
||||||
|
"file": "managed_file",
|
||||||
|
"sql": "postgresql",
|
||||||
|
}
|
||||||
|
provider = provider_by_prefix.get(prefix)
|
||||||
|
if not separator or not source_id or provider is None:
|
||||||
|
return None
|
||||||
|
return session.scalar(
|
||||||
|
select(ConnectorTabularSource).where(
|
||||||
|
ConnectorTabularSource.id == source_id,
|
||||||
|
ConnectorTabularSource.tenant_id == tenant_id,
|
||||||
|
ConnectorTabularSource.provider == provider,
|
||||||
|
ConnectorTabularSource.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_dto(
|
||||||
|
item: ConnectorTabularSource,
|
||||||
|
*,
|
||||||
|
inspection: TabularOriginInspection | None = None,
|
||||||
|
) -> TabularSource:
|
||||||
|
prefix_by_provider = {
|
||||||
|
"snapshot": "snapshot",
|
||||||
|
"managed_file": "file",
|
||||||
|
"postgresql": "sql",
|
||||||
|
}
|
||||||
|
source_mode = {
|
||||||
|
"managed_file": "file_backed",
|
||||||
|
"postgresql": "live",
|
||||||
|
}.get(item.provider, "cached")
|
||||||
|
if inspection is not None:
|
||||||
|
pushdown = inspection.pushdown
|
||||||
|
health = inspection.health
|
||||||
|
elif item.provider == "managed_file":
|
||||||
|
pushdown = TabularPushdown(projections=True, pagination=True)
|
||||||
|
health = TabularSourceHealth(
|
||||||
|
status="unknown",
|
||||||
|
code="files.exact_version_not_checked",
|
||||||
|
summary="The exact managed file version will be re-authorized and integrity-checked on preview.",
|
||||||
|
checked_at=item.updated_at,
|
||||||
|
details={
|
||||||
|
"file_version_id": item.metadata_.get("file_version_id"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
elif item.provider == "postgresql":
|
||||||
|
pushdown = TabularPushdown(projections=True, pagination=True)
|
||||||
|
health = TabularSourceHealth(
|
||||||
|
status="unknown",
|
||||||
|
code="sql.health_not_checked",
|
||||||
|
summary="The live SQL source will be checked against its pinned configuration, credential, and schema on preview.",
|
||||||
|
checked_at=item.updated_at,
|
||||||
|
details={
|
||||||
|
"configuration_id": item.metadata_.get("configuration_id"),
|
||||||
|
"configuration_revision": item.metadata_.get(
|
||||||
|
"configuration_revision"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
pushdown = TabularPushdown(
|
||||||
|
projections=True,
|
||||||
|
pagination=True,
|
||||||
|
)
|
||||||
|
health = TabularSourceHealth(
|
||||||
|
status="healthy",
|
||||||
|
code="snapshot.ready",
|
||||||
|
summary="The immutable connector snapshot is ready.",
|
||||||
|
checked_at=item.updated_at,
|
||||||
|
details={"immutable": True},
|
||||||
|
)
|
||||||
|
return TabularSource(
|
||||||
|
ref=f"{prefix_by_provider.get(item.provider, 'snapshot')}:{item.id}",
|
||||||
|
provider=item.provider,
|
||||||
|
source_name=item.source_name,
|
||||||
|
name=item.name,
|
||||||
|
description=item.description,
|
||||||
|
schema=tuple(TabularColumn(**column) for column in item.schema_),
|
||||||
|
schema_version=str(item.schema_version),
|
||||||
|
fingerprint=item.fingerprint,
|
||||||
|
row_count=item.row_count,
|
||||||
|
byte_count=item.byte_count,
|
||||||
|
updated_at=item.updated_at,
|
||||||
|
capabilities=("read", "preview"),
|
||||||
|
metadata=dict(item.metadata_),
|
||||||
|
source_mode=source_mode,
|
||||||
|
pushdown=pushdown,
|
||||||
|
health=health,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _preview_diagnostic(
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
requested: int,
|
||||||
|
effective: int,
|
||||||
|
) -> TabularPreviewDiagnostic:
|
||||||
|
return TabularPreviewDiagnostic(
|
||||||
|
severity="info",
|
||||||
|
code=code,
|
||||||
|
message=message,
|
||||||
|
details={"requested": requested, "effective": effective},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _context(
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
required_scope: str,
|
||||||
|
) -> tuple[Session, ApiPrincipal]:
|
||||||
|
if not isinstance(session, Session):
|
||||||
|
raise TypeError("Tabular source providers require a SQLAlchemy session.")
|
||||||
|
if not isinstance(principal, ApiPrincipal):
|
||||||
|
raise TabularSourceAccessError("A tenant API principal is required.")
|
||||||
|
accepted_scopes = {required_scope, ADMIN_SCOPE}
|
||||||
|
if required_scope == READ_SCOPE:
|
||||||
|
accepted_scopes.update(
|
||||||
|
{
|
||||||
|
WRITE_SCOPE,
|
||||||
|
"datasources:catalogue:read",
|
||||||
|
"datasources:source:write",
|
||||||
|
"datasources:source:admin",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not any(has_scope(principal, scope) for scope in accepted_scopes):
|
||||||
|
raise TabularSourceAccessError(f"Missing scope: {required_scope}")
|
||||||
|
return session, principal
|
||||||
|
|
||||||
|
|
||||||
|
def _json_row(row: Mapping[str, object]) -> dict[str, Any]:
|
||||||
|
normalized = {str(key).strip(): value for key, value in row.items()}
|
||||||
|
if not normalized or any(not key for key in normalized):
|
||||||
|
raise TabularSourceValidationError("Every snapshot row needs named columns.")
|
||||||
|
try:
|
||||||
|
json.dumps(normalized, default=_unsupported_json)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise TabularSourceValidationError(f"Snapshot values must be JSON compatible: {exc}") from exc
|
||||||
|
return json.loads(json.dumps(normalized, default=_unsupported_json))
|
||||||
|
|
||||||
|
|
||||||
|
def _unsupported_json(value: object) -> object:
|
||||||
|
if isinstance(value, (datetime, Decimal)):
|
||||||
|
return str(value)
|
||||||
|
raise TypeError(f"{type(value).__name__} is not JSON serializable")
|
||||||
|
|
||||||
|
|
||||||
|
def _type_name(value: object) -> str:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "boolean"
|
||||||
|
if isinstance(value, int):
|
||||||
|
return "integer"
|
||||||
|
if isinstance(value, (float, Decimal)):
|
||||||
|
return "number"
|
||||||
|
if isinstance(value, str):
|
||||||
|
return "string"
|
||||||
|
if isinstance(value, list):
|
||||||
|
return "array"
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return "object"
|
||||||
|
return type(value).__name__.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _column_payload(column: TabularColumn) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"name": column.name,
|
||||||
|
"data_type": column.data_type,
|
||||||
|
"nullable": column.nullable,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _actor_id(principal: ApiPrincipal) -> str | None:
|
||||||
|
return principal.account_id or principal.membership_id or principal.identity_id
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_optional(value: str | None) -> str | None:
|
||||||
|
cleaned = str(value or "").strip()
|
||||||
|
return cleaned or None
|
||||||
|
|
||||||
|
|
||||||
|
def _escape_like(value: str) -> str:
|
||||||
|
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ADMIN_SCOPE",
|
||||||
|
"MAX_READ_ROWS",
|
||||||
|
"MAX_READ_BYTES",
|
||||||
|
"MAX_READ_TIMEOUT_MS",
|
||||||
|
"MAX_SNAPSHOT_BYTES",
|
||||||
|
"MAX_SNAPSHOT_ROWS",
|
||||||
|
"READ_SCOPE",
|
||||||
|
"SqlTabularSourceProvider",
|
||||||
|
"WRITE_SCOPE",
|
||||||
|
"infer_schema",
|
||||||
|
"parse_csv_snapshot",
|
||||||
|
"snapshot_fingerprint",
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.datasources import DatasourceOriginReadRequest
|
||||||
|
from govoplan_core.core.tabular_sources import TabularSnapshotInput
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.datasource_origins import (
|
||||||
|
ConnectorDatasourceOriginProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.tabular_sources import (
|
||||||
|
WRITE_SCOPE,
|
||||||
|
SqlTabularSourceProvider,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(*, scopes: tuple[str, ...]) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorDatasourceOriginTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[ConnectorTabularSource.__table__],
|
||||||
|
)
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
self.session = self.Session()
|
||||||
|
provider = SqlTabularSourceProvider()
|
||||||
|
self.source = provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=(WRITE_SCOPE,)),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Imported cases",
|
||||||
|
source_name="imported_cases",
|
||||||
|
rows=({"id": 1, "name": "Ada"},),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.origins = ConnectorDatasourceOriginProvider(provider)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
Base.metadata.drop_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[ConnectorTabularSource.__table__],
|
||||||
|
)
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_datasource_reader_can_discover_and_read_connector_origin(self) -> None:
|
||||||
|
datasource_principal = principal(
|
||||||
|
scopes=("datasources:catalogue:read",),
|
||||||
|
)
|
||||||
|
|
||||||
|
origins = self.origins.list_origins(
|
||||||
|
self.session,
|
||||||
|
datasource_principal,
|
||||||
|
)
|
||||||
|
result = self.origins.read_origin(
|
||||||
|
self.session,
|
||||||
|
datasource_principal,
|
||||||
|
request=DatasourceOriginReadRequest(origin_ref=self.source.ref),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((self.source.ref,), tuple(item.ref for item in origins))
|
||||||
|
self.assertEqual(("live", "cached"), origins[0].supported_modes)
|
||||||
|
self.assertEqual(({"id": 1, "name": "Ada"},), result.rows)
|
||||||
|
self.assertEqual("cached", origins[0].source_mode)
|
||||||
|
self.assertTrue(origins[0].pushdown.projections)
|
||||||
|
self.assertEqual("healthy", origins[0].health.status)
|
||||||
|
self.assertGreater(result.returned_bytes, 2)
|
||||||
|
self.assertEqual(1_000_000, result.effective_byte_limit)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,389 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.privacy.dsar_workflow import (
|
||||||
|
create_data_subject_request,
|
||||||
|
search_data_subject_request,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeSyncRun,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.dsar_provider import (
|
||||||
|
CONNECTORS_DSAR_CAPABILITY,
|
||||||
|
ConnectorsDsarProvider,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
NOW = datetime(2026, 8, 22, 9, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, provider: ConnectorsDsarProvider) -> None:
|
||||||
|
self.provider = provider
|
||||||
|
|
||||||
|
def capability_names(self):
|
||||||
|
return (CONNECTORS_DSAR_CAPABILITY,)
|
||||||
|
|
||||||
|
def capability_owner(self, name):
|
||||||
|
if name != CONNECTORS_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
return "connectors"
|
||||||
|
|
||||||
|
def tenant_entitlement_resolver(self):
|
||||||
|
class _Resolver:
|
||||||
|
@staticmethod
|
||||||
|
def resolve(session, tenant_id):
|
||||||
|
del session, tenant_id
|
||||||
|
return type("State", (), {"effective_modules": ("connectors",)})()
|
||||||
|
|
||||||
|
return _Resolver()
|
||||||
|
|
||||||
|
def require_tenant_capability(self, name, session, **kwargs):
|
||||||
|
del session, kwargs
|
||||||
|
if name != CONNECTORS_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
return self.provider
|
||||||
|
|
||||||
|
def manifests(self):
|
||||||
|
return (type("Manifest", (), {"id": "connectors"})(),)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsDsarProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.provider = ConnectorsDsarProvider()
|
||||||
|
self.assertIsInstance(self.provider, DsarProvider)
|
||||||
|
self._seed()
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _seed(self) -> None:
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
ConnectorTabularSource(
|
||||||
|
id="source-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider="snapshot",
|
||||||
|
source_name="people",
|
||||||
|
name="People import",
|
||||||
|
description="Do not export this business description",
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[{"name": "email"}],
|
||||||
|
rows=[{"email": "third-party@example.test"}],
|
||||||
|
fingerprint="source-fingerprint-do-not-export",
|
||||||
|
row_count=1,
|
||||||
|
byte_count=100,
|
||||||
|
metadata_={"secret": "source-metadata-do-not-export"},
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorTabularSource(
|
||||||
|
id="source-other",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
provider="snapshot",
|
||||||
|
source_name="other",
|
||||||
|
name="Other tenant",
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[],
|
||||||
|
rows=[],
|
||||||
|
fingerprint="other-fingerprint",
|
||||||
|
row_count=0,
|
||||||
|
byte_count=0,
|
||||||
|
metadata_={},
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
),
|
||||||
|
ConnectorSanctionsAcquisitionRun(
|
||||||
|
id="acquisition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="un",
|
||||||
|
source_id="consolidated",
|
||||||
|
status="complete",
|
||||||
|
attempt_count=1,
|
||||||
|
request_evidence={"secret": "request-evidence-do-not-export"},
|
||||||
|
response_evidence={"secret": "response-evidence-do-not-export"},
|
||||||
|
started_at=NOW,
|
||||||
|
finished_at=NOW,
|
||||||
|
snapshot_id="snapshot-1",
|
||||||
|
error="transport-detail-do-not-export",
|
||||||
|
created_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorDefinition(
|
||||||
|
id="definition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_key="address-reader",
|
||||||
|
name="Address reader",
|
||||||
|
description="Definition detail",
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
source_package="package-secret-do-not-export",
|
||||||
|
local_definition=True,
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorDefinitionRevision(
|
||||||
|
id="definition-revision-1",
|
||||||
|
definition_id="definition-1",
|
||||||
|
revision=1,
|
||||||
|
specification={"secret": "specification-do-not-export"},
|
||||||
|
definition_hash="definition-hash-do-not-export",
|
||||||
|
origin="local",
|
||||||
|
package_ref="package-ref-do-not-export",
|
||||||
|
created_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorConfiguration(
|
||||||
|
id="configuration-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id="definition-1",
|
||||||
|
name="Production addresses",
|
||||||
|
status="active",
|
||||||
|
endpoint_url="https://secret.example.test/api",
|
||||||
|
credential_ref="vault://secret-do-not-export",
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={"secret": "override-do-not-export"},
|
||||||
|
protected_paths=["secret"],
|
||||||
|
effective_configuration={"secret": "effective-do-not-export"},
|
||||||
|
effective_hash="configuration-hash-do-not-export",
|
||||||
|
resource_revision=2,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
updated_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorSimulationRun(
|
||||||
|
id="simulation-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
configuration_id="configuration-1",
|
||||||
|
mode="dry_run",
|
||||||
|
idempotency_key="simulation-idempotency-do-not-export",
|
||||||
|
request_hash="simulation-request-hash-do-not-export",
|
||||||
|
status="complete",
|
||||||
|
review_state="approved",
|
||||||
|
definition_revision=1,
|
||||||
|
configuration_revision=2,
|
||||||
|
configuration_hash="simulation-config-hash-do-not-export",
|
||||||
|
input_hash="simulation-input-hash-do-not-export",
|
||||||
|
summary={"secret": "summary-do-not-export"},
|
||||||
|
effects=[{"secret": "effect-do-not-export"}],
|
||||||
|
diagnostics=[{"secret": "diagnostic-do-not-export"}],
|
||||||
|
provenance={"secret": "provenance-do-not-export"},
|
||||||
|
created_by="account-1",
|
||||||
|
reviewed_by="account-1",
|
||||||
|
reviewed_at=NOW,
|
||||||
|
review_reason="review-reason-do-not-export",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorKnowledgeProfile(
|
||||||
|
id="knowledge-profile-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
configuration_id="configuration-1",
|
||||||
|
status="active",
|
||||||
|
product="bluespice",
|
||||||
|
desired_maturity="migrate",
|
||||||
|
discovered_maturity="migrate",
|
||||||
|
source_authority_mode="external_mirror",
|
||||||
|
default_visibility="restricted",
|
||||||
|
default_acl_tokens=["group:secret-do-not-export"],
|
||||||
|
namespace_mappings=[{"secret": "mapping-do-not-export"}],
|
||||||
|
capabilities=["read", "synchronize", "migrate"],
|
||||||
|
discovery_evidence={"secret": "discovery-do-not-export"},
|
||||||
|
health_details={"secret": "health-do-not-export"},
|
||||||
|
updated_by="account-1",
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
ConnectorKnowledgeSyncRun(
|
||||||
|
id="knowledge-run-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
profile_id="knowledge-profile-1",
|
||||||
|
mode="delta",
|
||||||
|
idempotency_key="knowledge-idempotency-do-not-export",
|
||||||
|
request_hash="knowledge-request-hash-do-not-export",
|
||||||
|
status="completed",
|
||||||
|
counts={"update": 1},
|
||||||
|
effects=[{"secret": "knowledge-effect-do-not-export"}],
|
||||||
|
diagnostics=[{"secret": "knowledge-diagnostic-do-not-export"}],
|
||||||
|
provenance={"secret": "knowledge-provenance-do-not-export"},
|
||||||
|
created_by="account-1",
|
||||||
|
started_at=NOW,
|
||||||
|
finished_at=NOW,
|
||||||
|
created_at=NOW,
|
||||||
|
updated_at=NOW,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _subject() -> DsarSubjectRef:
|
||||||
|
return DsarSubjectRef(account_id="account-1")
|
||||||
|
|
||||||
|
def test_search_exports_minimized_operator_attribution(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session, tenant_id="tenant-1", subject=self._subject()
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
"source_actor_attribution",
|
||||||
|
"acquisition_actor_attribution",
|
||||||
|
"definition_actor_attribution",
|
||||||
|
"configuration_actor_attribution",
|
||||||
|
"simulation_actor_attribution",
|
||||||
|
"knowledge_profile_actor_attribution",
|
||||||
|
"knowledge_run_actor_attribution",
|
||||||
|
},
|
||||||
|
{record.resource_type for record in records},
|
||||||
|
)
|
||||||
|
exported = json.dumps([record.to_dict() for record in records])
|
||||||
|
for excluded in (
|
||||||
|
"third-party@example.test",
|
||||||
|
"source-fingerprint-do-not-export",
|
||||||
|
"source-metadata-do-not-export",
|
||||||
|
"request-evidence-do-not-export",
|
||||||
|
"response-evidence-do-not-export",
|
||||||
|
"transport-detail-do-not-export",
|
||||||
|
"specification-do-not-export",
|
||||||
|
"definition-hash-do-not-export",
|
||||||
|
"package-ref-do-not-export",
|
||||||
|
"https://secret.example.test/api",
|
||||||
|
"vault://secret-do-not-export",
|
||||||
|
"override-do-not-export",
|
||||||
|
"effective-do-not-export",
|
||||||
|
"configuration-hash-do-not-export",
|
||||||
|
"simulation-idempotency-do-not-export",
|
||||||
|
"simulation-request-hash-do-not-export",
|
||||||
|
"simulation-config-hash-do-not-export",
|
||||||
|
"simulation-input-hash-do-not-export",
|
||||||
|
"summary-do-not-export",
|
||||||
|
"effect-do-not-export",
|
||||||
|
"diagnostic-do-not-export",
|
||||||
|
"provenance-do-not-export",
|
||||||
|
"review-reason-do-not-export",
|
||||||
|
"secret-do-not-export",
|
||||||
|
"mapping-do-not-export",
|
||||||
|
"discovery-do-not-export",
|
||||||
|
"health-do-not-export",
|
||||||
|
"knowledge-idempotency-do-not-export",
|
||||||
|
"knowledge-request-hash-do-not-export",
|
||||||
|
"knowledge-effect-do-not-export",
|
||||||
|
"knowledge-diagnostic-do-not-export",
|
||||||
|
"knowledge-provenance-do-not-export",
|
||||||
|
):
|
||||||
|
self.assertNotIn(excluded, exported)
|
||||||
|
|
||||||
|
def test_requires_exact_account_and_enforces_tenant(self) -> None:
|
||||||
|
email_only = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(email="operator@example.test"),
|
||||||
|
)
|
||||||
|
conflict = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
external_references={"connectors.account": "account-other"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session, tenant_id="tenant-1", subject=self._subject()
|
||||||
|
)
|
||||||
|
self.assertEqual((), email_only)
|
||||||
|
self.assertEqual((), conflict)
|
||||||
|
self.assertNotIn("source-other", {record.resource_id for record in records})
|
||||||
|
|
||||||
|
def test_object_narrowing_does_not_broaden_the_search(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
external_references={"connectors.simulation": "simulation-1"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[("simulation_actor_attribution", "simulation-1")],
|
||||||
|
[(record.resource_type, record.resource_id) for record in records],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_erasure_retains_external_operation_evidence(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session, tenant_id="tenant-1", subject=self._subject()
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self._subject(),
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
self.assertTrue(actions)
|
||||||
|
self.assertTrue(
|
||||||
|
all(action.kind == "retain" and not action.executable for action in actions)
|
||||||
|
)
|
||||||
|
results = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self._subject(),
|
||||||
|
actions=actions,
|
||||||
|
request_id="dsar-connectors-1",
|
||||||
|
)
|
||||||
|
self.assertTrue(all(result.status == "blocked" for result in results))
|
||||||
|
|
||||||
|
def test_manifest_and_core_workflow_discover_provider(self) -> None:
|
||||||
|
self.assertIn(CONNECTORS_DSAR_CAPABILITY, manifest.capability_factories)
|
||||||
|
self.assertIn(
|
||||||
|
"connectors.data-subject-requests",
|
||||||
|
{topic.id for topic in manifest.documentation},
|
||||||
|
)
|
||||||
|
row = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-CONNECTORS-1",
|
||||||
|
request_kind="access",
|
||||||
|
subject=self._subject(),
|
||||||
|
purpose="Connector attribution access request",
|
||||||
|
legal_basis=None,
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="operator-1",
|
||||||
|
)
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=_Registry(self.provider),
|
||||||
|
row=row,
|
||||||
|
expected_revision=row.resource_revision,
|
||||||
|
)
|
||||||
|
self.assertEqual("searched", row.status)
|
||||||
|
self.assertEqual(7, row.search_result["record_count"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from defusedxml import ElementTree as SafeET
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.feeds import (
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
ConnectorFeedProvider,
|
||||||
|
feed_rows,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.router import api_render_feed
|
||||||
|
from govoplan_connectors.backend.schemas import FeedRenderPayload
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.feeds import (
|
||||||
|
FeedCapabilityError,
|
||||||
|
FeedEntry,
|
||||||
|
FeedRenderRequest,
|
||||||
|
)
|
||||||
|
from govoplan_core.security.http_fetch import HttpFetchResponse
|
||||||
|
|
||||||
|
|
||||||
|
RSS = b"""<?xml version="1.0"?>
|
||||||
|
<rss version="2.0"><channel>
|
||||||
|
<title>Decisions</title><link>https://example.test/</link>
|
||||||
|
<description>Published decisions</description>
|
||||||
|
<item><guid>decision-1</guid><title>Decision one</title>
|
||||||
|
<link>https://example.test/1</link>
|
||||||
|
<pubDate>Fri, 31 Jul 2026 10:00:00 GMT</pubDate>
|
||||||
|
<category>planning</category>
|
||||||
|
</item>
|
||||||
|
</channel></rss>"""
|
||||||
|
|
||||||
|
ATOM = b"""<?xml version="1.0"?>
|
||||||
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||||
|
<id>https://example.test/feed</id><title>Updates</title>
|
||||||
|
<updated>2026-07-31T10:00:00Z</updated>
|
||||||
|
<link href="https://example.test/" />
|
||||||
|
<entry><id>update-1</id><title>Update one</title>
|
||||||
|
<updated>2026-07-31T10:00:00Z</updated>
|
||||||
|
<link href="https://example.test/update-1" />
|
||||||
|
</entry>
|
||||||
|
</feed>"""
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorFeedProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.provider = ConnectorFeedProvider()
|
||||||
|
|
||||||
|
def test_rss_and_atom_are_normalized_to_tabular_entries(self) -> None:
|
||||||
|
rss = self.provider.parse(RSS, source_url="https://example.test/rss")
|
||||||
|
atom = self.provider.parse(ATOM, source_url="https://example.test/atom")
|
||||||
|
|
||||||
|
self.assertEqual("rss", rss.format)
|
||||||
|
self.assertEqual("decision-1", rss.entries[0].id)
|
||||||
|
self.assertEqual("atom", atom.format)
|
||||||
|
self.assertEqual("https://example.test/update-1", atom.entries[0].url)
|
||||||
|
self.assertEqual("decision-1", feed_rows(rss)[0]["id"])
|
||||||
|
self.assertEqual(64, len(rss.sha256))
|
||||||
|
|
||||||
|
def test_fetch_records_transport_freshness_and_provenance(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_connectors.backend.feeds.fetch_http",
|
||||||
|
return_value=HttpFetchResponse(
|
||||||
|
status=200,
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/rss+xml",
|
||||||
|
"Cache-Control": "public, max-age=600",
|
||||||
|
"ETag": '"feed-1"',
|
||||||
|
},
|
||||||
|
body=RSS,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
document = self.provider.fetch("https://example.test/rss")
|
||||||
|
|
||||||
|
self.assertEqual('"feed-1"', document.etag)
|
||||||
|
self.assertIsNotNone(document.acquired_at)
|
||||||
|
self.assertEqual(600, int((document.fresh_until - document.acquired_at).total_seconds()))
|
||||||
|
self.assertEqual(len(RSS), document.metadata["byte_count"])
|
||||||
|
|
||||||
|
def test_render_filters_entries_by_explicit_visibility(self) -> None:
|
||||||
|
request = FeedRenderRequest(
|
||||||
|
format="atom",
|
||||||
|
title="Public updates",
|
||||||
|
feed_url="https://example.test/feed.atom",
|
||||||
|
home_url="https://example.test/",
|
||||||
|
entries=(
|
||||||
|
FeedEntry(id="public-1", title="Public", visibility="public"),
|
||||||
|
FeedEntry(id="tenant-1", title="Tenant", visibility="tenant"),
|
||||||
|
),
|
||||||
|
allowed_visibilities=frozenset({"public"}),
|
||||||
|
)
|
||||||
|
result = self.provider.render(request)
|
||||||
|
root = SafeET.fromstring(result.body)
|
||||||
|
|
||||||
|
self.assertEqual(1, result.included_entries)
|
||||||
|
self.assertEqual(1, result.excluded_entries)
|
||||||
|
self.assertIn(b"Public", result.body)
|
||||||
|
self.assertNotIn(b"Tenant", result.body)
|
||||||
|
self.assertTrue(root.tag.endswith("feed"))
|
||||||
|
|
||||||
|
def test_unsafe_xml_is_rejected(self) -> None:
|
||||||
|
payload = b'<!DOCTYPE x [<!ENTITY y SYSTEM "file:///etc/passwd">]><rss>&y;</rss>'
|
||||||
|
with self.assertRaisesRegex(FeedCapabilityError, "not safe or valid"):
|
||||||
|
self.provider.parse(payload, source_url="https://example.test/rss")
|
||||||
|
|
||||||
|
def test_render_api_derives_visibility_from_audience_and_permissions(self) -> None:
|
||||||
|
entries = [
|
||||||
|
{
|
||||||
|
"id": visibility,
|
||||||
|
"title": visibility.title(),
|
||||||
|
"visibility": visibility,
|
||||||
|
"source_kind": source_kind,
|
||||||
|
"source_module": source_module,
|
||||||
|
"source_ref": f"{source_kind}:{visibility}",
|
||||||
|
"source_revision": "7",
|
||||||
|
}
|
||||||
|
for visibility, source_kind, source_module in (
|
||||||
|
("public", "publication", "docs"),
|
||||||
|
("tenant", "case", "cases"),
|
||||||
|
("private", "report", "reporting"),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
public_payload = FeedRenderPayload(
|
||||||
|
format="rss",
|
||||||
|
title="Selected GovOPlaN updates",
|
||||||
|
feed_url="https://example.test/feed.xml",
|
||||||
|
home_url="https://example.test/",
|
||||||
|
audience="public",
|
||||||
|
entries=entries,
|
||||||
|
)
|
||||||
|
public = api_render_feed(
|
||||||
|
public_payload,
|
||||||
|
principal=_principal(FEED_PUBLISH_SCOPE),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("public", public.headers["x-govoplan-feed-audience"])
|
||||||
|
self.assertEqual("1", public.headers["x-govoplan-feed-included"])
|
||||||
|
self.assertEqual("2", public.headers["x-govoplan-feed-excluded"])
|
||||||
|
self.assertIn(b"Public", public.body)
|
||||||
|
self.assertNotIn(b"Tenant", public.body)
|
||||||
|
|
||||||
|
restricted_payload = public_payload.model_copy(update={"audience": "private"})
|
||||||
|
with self.assertRaises(HTTPException) as denied:
|
||||||
|
api_render_feed(
|
||||||
|
restricted_payload,
|
||||||
|
principal=_principal(FEED_PUBLISH_SCOPE),
|
||||||
|
)
|
||||||
|
self.assertEqual(403, denied.exception.status_code)
|
||||||
|
|
||||||
|
restricted = api_render_feed(
|
||||||
|
restricted_payload,
|
||||||
|
principal=_principal(
|
||||||
|
FEED_PUBLISH_SCOPE,
|
||||||
|
FEED_PRIVATE_PUBLISH_SCOPE,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("3", restricted.headers["x-govoplan-feed-included"])
|
||||||
|
self.assertIn(b"Private", restricted.body)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal(*scopes: str) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorSimulationRun,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_runtime import (
|
||||||
|
GovernedConnectorError,
|
||||||
|
create_configuration,
|
||||||
|
execute_run,
|
||||||
|
list_configurations,
|
||||||
|
review_run,
|
||||||
|
update_configuration,
|
||||||
|
upsert_definition,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.governed_schemas import (
|
||||||
|
ConnectorConfigurationCreateRequest,
|
||||||
|
ConnectorConfigurationUpdateRequest,
|
||||||
|
ConnectorDefinitionUpsertRequest,
|
||||||
|
ConnectorReviewRequest,
|
||||||
|
ConnectorRunRequest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(tenant_id: str = "tenant-1") -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(
|
||||||
|
{
|
||||||
|
"connectors:source:read",
|
||||||
|
"connectors:source:write",
|
||||||
|
"connectors:source:admin",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def definition_payload(
|
||||||
|
*,
|
||||||
|
mapping_version: str = "1",
|
||||||
|
package_ref: str = "municipal-addresses@1",
|
||||||
|
timeout_seconds: int = 10,
|
||||||
|
) -> ConnectorDefinitionUpsertRequest:
|
||||||
|
return ConnectorDefinitionUpsertRequest.model_validate(
|
||||||
|
{
|
||||||
|
"definition_key": "municipal.addresses",
|
||||||
|
"name": "Municipal addresses",
|
||||||
|
"description": "A package-managed reference connector.",
|
||||||
|
"origin": "package",
|
||||||
|
"package_ref": package_ref,
|
||||||
|
"specification": {
|
||||||
|
"provider": "municipal-directory",
|
||||||
|
"protocol": "rest",
|
||||||
|
"capabilities": ["discover", "read", "dry_run"],
|
||||||
|
"input_schema": {"type": "object"},
|
||||||
|
"output_schema": {"type": "object"},
|
||||||
|
"mapping": {
|
||||||
|
"version": mapping_version,
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"source": "external_id",
|
||||||
|
"target": "address.external_id",
|
||||||
|
"required": True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "street",
|
||||||
|
"target": "address.street",
|
||||||
|
"required": True,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"validation_rules": [
|
||||||
|
{
|
||||||
|
"kind": "unique",
|
||||||
|
"field": "address.external_id",
|
||||||
|
"severity": "error",
|
||||||
|
"code": "addresses.external_id.ambiguous",
|
||||||
|
"message": "The external identifier is not unique.",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"dry_run": {
|
||||||
|
"supported": True,
|
||||||
|
"simulation_supported": True,
|
||||||
|
"max_items": 50,
|
||||||
|
"redacted_fields": ["address.street"],
|
||||||
|
"sample_rows": [
|
||||||
|
{"external_id": "A-1", "street": "Sample street"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"audit": {
|
||||||
|
"event_prefix": "connectors.municipal_addresses",
|
||||||
|
"expected_events": ["simulation.completed"],
|
||||||
|
"evidence_fields": ["input_hash", "configuration_hash"],
|
||||||
|
},
|
||||||
|
"privacy_classification": "confidential",
|
||||||
|
"retention_class": "connector-preview-30d",
|
||||||
|
"operational_limits": {"timeout_seconds": timeout_seconds},
|
||||||
|
"retry_policy": {"max_attempts": 2},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedConnectorRuntimeTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
self.tables = [
|
||||||
|
ConnectorDefinition.__table__,
|
||||||
|
ConnectorDefinitionRevision.__table__,
|
||||||
|
ConnectorConfiguration.__table__,
|
||||||
|
ConnectorSimulationRun.__table__,
|
||||||
|
]
|
||||||
|
Base.metadata.create_all(self.engine, tables=self.tables)
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
self.session = self.Session()
|
||||||
|
self.audit = patch(
|
||||||
|
"govoplan_connectors.backend.governed_runtime.audit_from_principal"
|
||||||
|
)
|
||||||
|
self.audit_mock = self.audit.start()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.audit.stop()
|
||||||
|
self.session.close()
|
||||||
|
Base.metadata.drop_all(self.engine, tables=reversed(self.tables))
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _configuration(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
ambiguity_policy: str = "manual_review",
|
||||||
|
):
|
||||||
|
definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(),
|
||||||
|
)
|
||||||
|
return create_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
ConnectorConfigurationCreateRequest(
|
||||||
|
definition_id=definition.id,
|
||||||
|
name=f"Address import {ambiguity_policy}",
|
||||||
|
endpoint_url="https://directory.example.invalid/v1",
|
||||||
|
credential_ref="vault://connectors/address-reader",
|
||||||
|
local_overrides={"retry_policy": {"max_attempts": 5}},
|
||||||
|
ambiguity_policy=ambiguity_policy,
|
||||||
|
status="active",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_package_update_is_explicit_and_preserves_local_overrides(self) -> None:
|
||||||
|
configuration = self._configuration()
|
||||||
|
|
||||||
|
updated_definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(
|
||||||
|
mapping_version="2",
|
||||||
|
package_ref="municipal-addresses@2",
|
||||||
|
timeout_seconds=20,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
unchanged = next(
|
||||||
|
item
|
||||||
|
for item in list_configurations(self.session, tenant_id="tenant-1")
|
||||||
|
if item.id == configuration.id
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, updated_definition.current_revision)
|
||||||
|
self.assertTrue(unchanged.update_available)
|
||||||
|
self.assertEqual("1", unchanged.effective_configuration["mapping"]["version"])
|
||||||
|
self.assertEqual(5, unchanged.effective_configuration["retry_policy"]["max_attempts"])
|
||||||
|
self.assertEqual(["retry_policy.max_attempts"], unchanged.protected_paths)
|
||||||
|
|
||||||
|
adopted = update_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
payload=ConnectorConfigurationUpdateRequest(
|
||||||
|
expected_revision=configuration.resource_revision,
|
||||||
|
adopt_latest_definition=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(adopted.update_available)
|
||||||
|
self.assertEqual("2", adopted.effective_configuration["mapping"]["version"])
|
||||||
|
self.assertEqual(
|
||||||
|
20,
|
||||||
|
adopted.effective_configuration["operational_limits"]["timeout_seconds"],
|
||||||
|
)
|
||||||
|
self.assertEqual(5, adopted.effective_configuration["retry_policy"]["max_attempts"])
|
||||||
|
self.assertEqual(["retry_policy.max_attempts"], adopted.protected_paths)
|
||||||
|
|
||||||
|
def test_ambiguous_simulation_requires_review_and_is_idempotent(self) -> None:
|
||||||
|
configuration = self._configuration()
|
||||||
|
payload = ConnectorRunRequest(
|
||||||
|
idempotency_key="simulation-1",
|
||||||
|
external_revision="directory-etag-22",
|
||||||
|
input_rows=[
|
||||||
|
{"external_id": "duplicate", "street": "First"},
|
||||||
|
{"external_id": "duplicate", "street": "Second"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
created = execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
replayed = execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(created.id, replayed.id)
|
||||||
|
self.assertEqual("manual_review", created.status)
|
||||||
|
self.assertEqual("pending", created.review_state)
|
||||||
|
self.assertEqual(2, created.summary["ambiguous"])
|
||||||
|
self.assertEqual("<redacted>", created.effects[0]["sample"]["address"]["street"])
|
||||||
|
self.assertEqual("directory-etag-22", created.provenance["external_revision"])
|
||||||
|
|
||||||
|
reviewed = review_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
run_id=created.id,
|
||||||
|
payload=ConnectorReviewRequest(
|
||||||
|
decision="approved",
|
||||||
|
reason="The duplicate rows represent an approved upstream alias.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("approved", reviewed.review_state)
|
||||||
|
self.assertEqual("review_approved", reviewed.status)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(GovernedConnectorError, "different run inputs"):
|
||||||
|
execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="simulation",
|
||||||
|
payload=ConnectorRunRequest(
|
||||||
|
idempotency_key="simulation-1",
|
||||||
|
input_rows=[{"external_id": "other", "street": "Other"}],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_ambiguity_policy_can_quarantine_or_reject(self) -> None:
|
||||||
|
for policy, expected_status, expected_review in (
|
||||||
|
("quarantine", "quarantined", "quarantined"),
|
||||||
|
("reject", "rejected", "not_required"),
|
||||||
|
):
|
||||||
|
configuration = self._configuration(ambiguity_policy=policy)
|
||||||
|
result = execute_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
mode="dry_run",
|
||||||
|
payload=ConnectorRunRequest(
|
||||||
|
idempotency_key=f"{policy}-1",
|
||||||
|
input_rows=[
|
||||||
|
{"external_id": "same", "street": "First"},
|
||||||
|
{"external_id": "same", "street": "Second"},
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(expected_status, result.status)
|
||||||
|
self.assertEqual(expected_review, result.review_state)
|
||||||
|
|
||||||
|
def test_endpoint_credentials_and_stale_saves_are_rejected(self) -> None:
|
||||||
|
definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(GovernedConnectorError, "must not contain credentials"):
|
||||||
|
create_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
ConnectorConfigurationCreateRequest(
|
||||||
|
definition_id=definition.id,
|
||||||
|
name="Unsafe",
|
||||||
|
endpoint_url="https://user:secret@example.invalid/v1",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
configuration = self._configuration()
|
||||||
|
with self.assertRaisesRegex(GovernedConnectorError, "reload it before saving"):
|
||||||
|
update_configuration(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=configuration.id,
|
||||||
|
payload=ConnectorConfigurationUpdateRequest(
|
||||||
|
expected_revision=configuration.resource_revision + 1,
|
||||||
|
name="Stale",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_definition_ownership_cannot_change_implicitly(self) -> None:
|
||||||
|
package_definition = upsert_definition(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
definition_payload(),
|
||||||
|
)
|
||||||
|
local_payload = definition_payload().model_copy(
|
||||||
|
update={"origin": "local", "package_ref": None}
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
GovernedConnectorError,
|
||||||
|
"configuration overrides",
|
||||||
|
):
|
||||||
|
upsert_definition(self.session, principal(), local_payload)
|
||||||
|
|
||||||
|
self.assertFalse(package_definition.local_definition)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.sanctions import (
|
||||||
|
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.manifest import manifest
|
||||||
|
from govoplan_connectors.backend.knowledge_connector import (
|
||||||
|
KNOWLEDGE_CAPABILITY,
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsManifestTests(unittest.TestCase):
|
||||||
|
def test_manifest_exposes_versioned_tabular_capabilities(self) -> None:
|
||||||
|
self.assertEqual("connectors", manifest.id)
|
||||||
|
self.assertIn("access", manifest.optional_dependencies)
|
||||||
|
self.assertIn(
|
||||||
|
"connectors.tabular_sources",
|
||||||
|
{interface.name for interface in manifest.provides_interfaces},
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||||
|
manifest.capability_factories,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||||
|
manifest.capability_factories,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||||
|
manifest.capability_factories,
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(manifest.migration_spec)
|
||||||
|
self.assertEqual(
|
||||||
|
"@govoplan/connectors-webui",
|
||||||
|
manifest.frontend.package_name,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"connectors.governed-configuration",
|
||||||
|
{topic.id for topic in manifest.documentation},
|
||||||
|
)
|
||||||
|
self.assertIn(KNOWLEDGE_CAPABILITY, manifest.capability_factories)
|
||||||
|
self.assertIn(
|
||||||
|
KNOWLEDGE_CAPABILITY,
|
||||||
|
{interface.name for interface in manifest.provides_interfaces},
|
||||||
|
)
|
||||||
|
self.assertIn("search", manifest.optional_dependencies)
|
||||||
|
self.assertIn("wiki", manifest.optional_dependencies)
|
||||||
|
self.assertIn(
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
{provider.id for provider in manifest.external_providers},
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
{registration.id for registration in manifest.search_sources},
|
||||||
|
)
|
||||||
|
topic = next(
|
||||||
|
item
|
||||||
|
for item in manifest.documentation
|
||||||
|
if item.id == "connectors.mediawiki-bluespice"
|
||||||
|
)
|
||||||
|
self.assertIn("de", topic.translations)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,659 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery import RecoveryOperation, RecoveryStatus
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorKnowledgeObject,
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeSyncRun,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.knowledge_connector import (
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
KnowledgeConnectorError,
|
||||||
|
create_profile,
|
||||||
|
discover_profile,
|
||||||
|
list_objects,
|
||||||
|
migration_dry_run,
|
||||||
|
publish_page,
|
||||||
|
synchronize_profile,
|
||||||
|
update_profile,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.knowledge_schemas import (
|
||||||
|
KnowledgeMigrationDryRunRequest,
|
||||||
|
KnowledgeMigrationTargetState,
|
||||||
|
KnowledgeNamespaceMapping,
|
||||||
|
KnowledgeProfileCreateRequest,
|
||||||
|
KnowledgeProfileUpdateRequest,
|
||||||
|
KnowledgePublishRequest,
|
||||||
|
KnowledgeSyncRequest,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.knowledge_search import (
|
||||||
|
ExternalKnowledgeSearchSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.mediawiki_transport import (
|
||||||
|
MediaWikiChangeBatch,
|
||||||
|
MediaWikiPublishResult,
|
||||||
|
MediaWikiTransportError,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
ALL_SCOPES = frozenset(
|
||||||
|
{
|
||||||
|
"connectors:knowledge:read",
|
||||||
|
"connectors:knowledge:admin",
|
||||||
|
"connectors:knowledge:sync",
|
||||||
|
"connectors:knowledge:publish",
|
||||||
|
"connectors:knowledge:migrate",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
scopes: frozenset[str] = ALL_SCOPES,
|
||||||
|
groups: frozenset[str] = frozenset({"editors"}),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=scopes,
|
||||||
|
group_ids=groups,
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="account-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class StaticTransport:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.batches: list[MediaWikiChangeBatch] = []
|
||||||
|
self.change_calls = 0
|
||||||
|
self.publish_calls = 0
|
||||||
|
self.publish_error: MediaWikiTransportError | None = None
|
||||||
|
|
||||||
|
def discover(self, *, endpoint_url, credential):
|
||||||
|
del endpoint_url, credential
|
||||||
|
return {
|
||||||
|
"curtimestamp": "2026-08-22T10:00:00Z",
|
||||||
|
"query": {
|
||||||
|
"general": {
|
||||||
|
"generator": "MediaWiki 1.43.1",
|
||||||
|
"phpversion": "8.3.8",
|
||||||
|
},
|
||||||
|
"extensions": [
|
||||||
|
{"name": "BlueSpiceFoundation", "version": "4.5.2"},
|
||||||
|
{"name": "BlueSpicePermissionManager", "version": "4.5.2"},
|
||||||
|
],
|
||||||
|
"namespaces": {
|
||||||
|
"0": {"id": 0, "name": "", "content": True},
|
||||||
|
"4": {"id": 4, "name": "GovWiki", "content": True},
|
||||||
|
},
|
||||||
|
"userinfo": {
|
||||||
|
"id": 17,
|
||||||
|
"name": "govoplan",
|
||||||
|
"rights": ["read", "edit"],
|
||||||
|
"groups": ["bot"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def changes(self, **kwargs):
|
||||||
|
del kwargs
|
||||||
|
self.change_calls += 1
|
||||||
|
if not self.batches:
|
||||||
|
raise AssertionError("No deterministic change batch remains")
|
||||||
|
return self.batches.pop(0)
|
||||||
|
|
||||||
|
def publish(self, **kwargs):
|
||||||
|
self.publish_calls += 1
|
||||||
|
if self.publish_error is not None:
|
||||||
|
raise self.publish_error
|
||||||
|
return MediaWikiPublishResult(
|
||||||
|
page_id=str(kwargs.get("expected_page_id") or "99"),
|
||||||
|
revision_id="901",
|
||||||
|
title=str(kwargs["title"]),
|
||||||
|
canonical_url="https://wiki.example.invalid/wiki/Published_Guide",
|
||||||
|
evidence={"result": "Success", "fixture": True},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def page(
|
||||||
|
*,
|
||||||
|
page_id: str = "42",
|
||||||
|
revision_id: str = "501",
|
||||||
|
title: str = "Citizen Guide",
|
||||||
|
acl_tokens: list[str] | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"change_kind": "upsert",
|
||||||
|
"change_cursor": f"rcid:{revision_id}",
|
||||||
|
"pageid": page_id,
|
||||||
|
"ns": 0,
|
||||||
|
"title": title,
|
||||||
|
"fullurl": f"https://wiki.example.invalid/wiki/{title.replace(' ', '_')}",
|
||||||
|
"lastrevid": revision_id,
|
||||||
|
"revisions": [
|
||||||
|
{
|
||||||
|
"revid": revision_id,
|
||||||
|
"timestamp": "2026-08-22T10:00:00Z",
|
||||||
|
"user": "Ada Admin",
|
||||||
|
"comment": "Reviewed guidance",
|
||||||
|
"contentmodel": "wikitext",
|
||||||
|
"sha1": f"sha-{revision_id}",
|
||||||
|
"slots": {
|
||||||
|
"main": {
|
||||||
|
"content": "Welcome {{UnsupportedBox|important}} [[Services]]"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"categories": [{"title": "Category:Citizen service"}],
|
||||||
|
"links": [{"ns": 0, "title": "Services"}],
|
||||||
|
"images": [{"ns": 6, "title": "File:guide.pdf", "pageid": 71}],
|
||||||
|
"discussions": [
|
||||||
|
{
|
||||||
|
"id": "discussion-1",
|
||||||
|
"author": "Ada Admin",
|
||||||
|
"body": "Please verify this section.",
|
||||||
|
"created_at": "2026-08-20T09:00:00Z",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"permissions": {
|
||||||
|
"visibility": "restricted",
|
||||||
|
"acl_tokens": acl_tokens or ["group:editors"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MediaWikiConnectorTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
bind_process_runtime_identity(
|
||||||
|
RuntimeIdentity(
|
||||||
|
installation_id="mediawiki-connector-tests",
|
||||||
|
node_id="node-1",
|
||||||
|
incarnation="incarnation-1",
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="a" * 64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.audit = patch(
|
||||||
|
"govoplan_connectors.backend.knowledge_connector.audit_event"
|
||||||
|
)
|
||||||
|
self.audit.start()
|
||||||
|
self.credential = patch(
|
||||||
|
"govoplan_connectors.backend.knowledge_connector._credential",
|
||||||
|
return_value={"access_token": "fixture-token"},
|
||||||
|
)
|
||||||
|
self.credential.start()
|
||||||
|
self.transport = StaticTransport()
|
||||||
|
self.configuration_id = self._seed_configuration()
|
||||||
|
self.profile_id = self._profile()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
self.credential.stop()
|
||||||
|
self.audit.stop()
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _seed_configuration(self) -> str:
|
||||||
|
definition = ConnectorDefinition(
|
||||||
|
id="definition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_key="knowledge.mediawiki",
|
||||||
|
name="MediaWiki",
|
||||||
|
description="Knowledge transport",
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
local_definition=True,
|
||||||
|
)
|
||||||
|
self.session.add(definition)
|
||||||
|
self.session.add(
|
||||||
|
ConnectorDefinitionRevision(
|
||||||
|
id="definition-revision-1",
|
||||||
|
definition_id=definition.id,
|
||||||
|
revision=1,
|
||||||
|
specification={
|
||||||
|
"provider": "mediawiki",
|
||||||
|
"protocol": "mediawiki_action_api",
|
||||||
|
},
|
||||||
|
definition_hash="definition-hash",
|
||||||
|
origin="local",
|
||||||
|
created_by="account-1",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
configuration = ConnectorConfiguration(
|
||||||
|
id="configuration-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id=definition.id,
|
||||||
|
name="Institutional knowledge",
|
||||||
|
status="active",
|
||||||
|
endpoint_url="https://wiki.example.invalid",
|
||||||
|
credential_ref="credential-envelope-1",
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={},
|
||||||
|
protected_paths=[],
|
||||||
|
effective_configuration={
|
||||||
|
"provider": "mediawiki",
|
||||||
|
"protocol": "mediawiki_action_api",
|
||||||
|
},
|
||||||
|
effective_hash="configuration-hash",
|
||||||
|
resource_revision=1,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add(configuration)
|
||||||
|
self.session.flush()
|
||||||
|
return configuration.id
|
||||||
|
|
||||||
|
def _profile(self) -> str:
|
||||||
|
created = create_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
KnowledgeProfileCreateRequest(
|
||||||
|
configuration_id=self.configuration_id,
|
||||||
|
desired_maturity="migrate",
|
||||||
|
source_authority_mode="external_mirror",
|
||||||
|
default_visibility="restricted",
|
||||||
|
default_acl_tokens=["group:knowledge-managers"],
|
||||||
|
namespace_mappings=[
|
||||||
|
KnowledgeNamespaceMapping(
|
||||||
|
source_namespace_id=0,
|
||||||
|
source_name="",
|
||||||
|
target_space_ref="service-guidance",
|
||||||
|
target_path_prefix="imported",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return created.id
|
||||||
|
|
||||||
|
def _discover(self):
|
||||||
|
return discover_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
transport=self.transport,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _sync(self, raw_pages, *, key="sync-1", force_full=True):
|
||||||
|
self.transport.batches.append(
|
||||||
|
MediaWikiChangeBatch(
|
||||||
|
changes=tuple(raw_pages),
|
||||||
|
next_cursor=None,
|
||||||
|
complete=True,
|
||||||
|
high_watermark="2026-08-22T10:05:00Z",
|
||||||
|
evidence={"fixture": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=KnowledgeSyncRequest(
|
||||||
|
idempotency_key=key,
|
||||||
|
force_full=force_full,
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_discovery_mapping_idempotency_and_migration_loss_diagnostics(self) -> None:
|
||||||
|
discovery = self._discover()
|
||||||
|
self.assertEqual("bluespice", discovery.product)
|
||||||
|
self.assertEqual("4.5.2", discovery.product_version)
|
||||||
|
self.assertEqual("migrate", discovery.maturity)
|
||||||
|
self.assertIn("publish", discovery.capabilities)
|
||||||
|
self.assertIn("permission_metadata", discovery.capabilities)
|
||||||
|
|
||||||
|
run = self._sync([page()])
|
||||||
|
self.assertEqual({"create": 1}, run.counts)
|
||||||
|
self.assertIn(
|
||||||
|
"attachments_reference_only", {item.code for item in run.diagnostics}
|
||||||
|
)
|
||||||
|
objects, _cursor = list_objects(
|
||||||
|
self.session, principal(), profile_id=self.profile_id
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(objects))
|
||||||
|
item = objects[0]
|
||||||
|
self.assertEqual("42", item.external_reference.object_id)
|
||||||
|
self.assertEqual("501", item.external_reference.version)
|
||||||
|
self.assertEqual("imported/Citizen-Guide", item.mapped_data["target_path"])
|
||||||
|
self.assertEqual("Ada Admin", item.mapped_data["revision_author"])
|
||||||
|
self.assertEqual(
|
||||||
|
"user", item.mapped_data["revision_author_reference"]["object_type"]
|
||||||
|
)
|
||||||
|
self.assertEqual("guide.pdf", item.mapped_data["files"][0]["name"])
|
||||||
|
self.assertEqual("discussion-1", item.mapped_data["discussions"][0]["external_id"])
|
||||||
|
|
||||||
|
replay = synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=KnowledgeSyncRequest(
|
||||||
|
idempotency_key="sync-1",
|
||||||
|
force_full=True,
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertEqual(run.id, replay.id)
|
||||||
|
self.assertEqual(1, self.transport.change_calls)
|
||||||
|
with self.assertRaisesRegex(KnowledgeConnectorError, "different request"):
|
||||||
|
synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=KnowledgeSyncRequest(
|
||||||
|
idempotency_key="sync-1",
|
||||||
|
force_full=True,
|
||||||
|
limit=25,
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
preview = migration_dry_run(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=KnowledgeMigrationDryRunRequest(
|
||||||
|
idempotency_key="migration-1",
|
||||||
|
target_space_ref="service-guidance",
|
||||||
|
supported_macros=["SupportedBox"],
|
||||||
|
existing_targets=[
|
||||||
|
KnowledgeMigrationTargetState(
|
||||||
|
path="imported/Citizen-Guide",
|
||||||
|
source_external_id="different-page",
|
||||||
|
attachment_names=["guide.pdf"],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertFalse(preview.can_apply)
|
||||||
|
self.assertEqual("2026-08-22T10:05:00Z", preview.source_revision)
|
||||||
|
self.assertEqual(
|
||||||
|
{"attachment_name_conflict", "target_path_conflict", "unsupported_macro"},
|
||||||
|
{item.code for item in preview.diagnostics},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_acl_changes_moves_deletes_and_search_authorization_are_current(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([page()])
|
||||||
|
source = ExternalKnowledgeSearchSource()
|
||||||
|
backfill = source.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||||
|
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
rebuild_id="rebuild-1",
|
||||||
|
limit=100,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(backfill.documents))
|
||||||
|
document = backfill.documents[0]
|
||||||
|
request = SearchAuthorizationRequest(
|
||||||
|
reference=document.reference,
|
||||||
|
source_revision=document.source_revision,
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
source.authorize(self.session, principal(), requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
denied = principal(groups=frozenset({"other"}))
|
||||||
|
self.assertFalse(
|
||||||
|
source.authorize(self.session, denied, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
self._sync(
|
||||||
|
[
|
||||||
|
page(
|
||||||
|
revision_id="502",
|
||||||
|
title="Resident Guide",
|
||||||
|
acl_tokens=["group:reviewers"],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
key="sync-2",
|
||||||
|
force_full=False,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
source.authorize(self.session, principal(), requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
reviewers = principal(groups=frozenset({"reviewers"}))
|
||||||
|
self.assertTrue(
|
||||||
|
source.authorize(self.session, reviewers, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
stored = self.session.scalar(
|
||||||
|
select(ConnectorKnowledgeObject).where(
|
||||||
|
ConnectorKnowledgeObject.profile_id == self.profile_id,
|
||||||
|
ConnectorKnowledgeObject.external_id == "42",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("Resident Guide", stored.title)
|
||||||
|
self.assertEqual("imported/Resident-Guide", stored.mapped_data["target_path"])
|
||||||
|
|
||||||
|
self._sync(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"change_kind": "delete",
|
||||||
|
"change_cursor": "logid:700",
|
||||||
|
"pageid": 0,
|
||||||
|
"ns": 0,
|
||||||
|
"title": "Resident Guide",
|
||||||
|
"timestamp": "2026-08-22T11:00:00Z",
|
||||||
|
"logid": 700,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
key="sync-3",
|
||||||
|
force_full=False,
|
||||||
|
)
|
||||||
|
self.assertEqual("deleted", stored.status)
|
||||||
|
self.assertFalse(
|
||||||
|
source.authorize(self.session, reviewers, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
after_delete = source.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||||
|
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
rebuild_id="rebuild-2",
|
||||||
|
limit=100,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual((), after_delete.documents)
|
||||||
|
|
||||||
|
def test_publication_replay_and_outcome_unknown_are_evidenced(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
payload = KnowledgePublishRequest(
|
||||||
|
idempotency_key="publish-1",
|
||||||
|
title="Published Guide",
|
||||||
|
body="Reviewed body",
|
||||||
|
summary="Publish approved guidance",
|
||||||
|
expected_external_revision="900",
|
||||||
|
)
|
||||||
|
result = publish_page(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
external_page_id="99",
|
||||||
|
payload=payload,
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
durable_recovery=True,
|
||||||
|
)
|
||||||
|
replay = publish_page(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
external_page_id="99",
|
||||||
|
payload=payload,
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
durable_recovery=False,
|
||||||
|
)
|
||||||
|
self.assertTrue(result.accepted)
|
||||||
|
self.assertEqual(result.run.id, replay.run.id)
|
||||||
|
self.assertEqual("901", result.external_reference.version)
|
||||||
|
self.assertEqual(1, self.transport.publish_calls)
|
||||||
|
recovery = self.session.scalar(
|
||||||
|
select(RecoveryOperation).where(
|
||||||
|
RecoveryOperation.resource_type == "external_knowledge_page",
|
||||||
|
RecoveryOperation.resource_id == "99",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, recovery.status)
|
||||||
|
|
||||||
|
self.transport.publish_error = MediaWikiTransportError(
|
||||||
|
"transport_timeout",
|
||||||
|
"Provider response timed out.",
|
||||||
|
retryable=True,
|
||||||
|
outcome_unknown=True,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(KnowledgeConnectorError, "outcome is unknown"):
|
||||||
|
publish_page(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
external_page_id="100",
|
||||||
|
payload=KnowledgePublishRequest(
|
||||||
|
idempotency_key="publish-unknown",
|
||||||
|
title="Uncertain Guide",
|
||||||
|
body="Body",
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
durable_recovery=False,
|
||||||
|
)
|
||||||
|
unresolved = self.session.scalar(
|
||||||
|
select(ConnectorKnowledgeSyncRun).where(
|
||||||
|
ConnectorKnowledgeSyncRun.profile_id == self.profile_id,
|
||||||
|
ConnectorKnowledgeSyncRun.idempotency_key == "publish-unknown",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("outcome_unknown", unresolved.status)
|
||||||
|
|
||||||
|
def test_fallback_acl_changes_and_profile_pause_fail_closed_immediately(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
fallback_page = page(page_id="43", title="Fallback Guide")
|
||||||
|
fallback_page.pop("permissions")
|
||||||
|
self._sync([fallback_page])
|
||||||
|
source = ExternalKnowledgeSearchSource()
|
||||||
|
document = source.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=KNOWLEDGE_PROVIDER_ID,
|
||||||
|
resource_type=KNOWLEDGE_RESOURCE_TYPE,
|
||||||
|
rebuild_id="fallback-rebuild",
|
||||||
|
),
|
||||||
|
).documents[0]
|
||||||
|
request = SearchAuthorizationRequest(
|
||||||
|
reference=document.reference,
|
||||||
|
source_revision=document.source_revision,
|
||||||
|
)
|
||||||
|
managers = principal(groups=frozenset({"knowledge-managers"}))
|
||||||
|
self.assertTrue(
|
||||||
|
source.authorize(self.session, managers, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
profile = self.session.get(ConnectorKnowledgeProfile, self.profile_id)
|
||||||
|
before_hash = self.session.get(
|
||||||
|
ConnectorKnowledgeObject, document.resource_id
|
||||||
|
).content_hash
|
||||||
|
updated = update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=KnowledgeProfileUpdateRequest(
|
||||||
|
expected_resource_revision=profile.resource_revision,
|
||||||
|
default_acl_tokens=["group:reviewers"],
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
source.authorize(self.session, managers, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
reviewers = principal(groups=frozenset({"reviewers"}))
|
||||||
|
self.assertTrue(
|
||||||
|
source.authorize(self.session, reviewers, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.assertNotEqual(
|
||||||
|
before_hash,
|
||||||
|
self.session.get(ConnectorKnowledgeObject, document.resource_id).content_hash,
|
||||||
|
)
|
||||||
|
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=KnowledgeProfileUpdateRequest(
|
||||||
|
expected_resource_revision=updated.resource_revision,
|
||||||
|
status="paused",
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
source.authorize(self.session, reviewers, requests=[request])[
|
||||||
|
document.reference.key
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_profiles_and_objects_are_tenant_isolated(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([page()])
|
||||||
|
with self.assertRaisesRegex(KnowledgeConnectorError, "not found"):
|
||||||
|
list_objects(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
)
|
||||||
|
profile = self.session.get(ConnectorKnowledgeProfile, self.profile_id)
|
||||||
|
self.assertEqual("tenant-1", profile.tenant_id)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.security.http_fetch import HttpFetchResponse
|
||||||
|
from govoplan_connectors.backend.mediawiki_transport import HttpMediaWikiTransport
|
||||||
|
|
||||||
|
|
||||||
|
def response(payload: dict[str, object]) -> HttpFetchResponse:
|
||||||
|
return HttpFetchResponse(
|
||||||
|
status=200,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
body=json.dumps(payload).encode(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MediaWikiHttpTransportTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.transport = HttpMediaWikiTransport()
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||||
|
def test_discovery_uses_the_action_api_and_sanitized_auth_header(self, fetch) -> None:
|
||||||
|
fetch.return_value = response(
|
||||||
|
{"query": {"general": {"generator": "MediaWiki 1.43"}}}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.transport.discover(
|
||||||
|
endpoint_url="https://wiki.example.test",
|
||||||
|
credential={"access_token": "secret-token"},
|
||||||
|
)
|
||||||
|
|
||||||
|
call = fetch.call_args
|
||||||
|
self.assertEqual("https://wiki.example.test/api.php", urlsplit(call.args[0])._replace(query="").geturl())
|
||||||
|
query = parse_qs(urlsplit(call.args[0]).query)
|
||||||
|
self.assertEqual(["query"], query["action"])
|
||||||
|
self.assertEqual(["siteinfo|userinfo"], query["meta"])
|
||||||
|
self.assertEqual("Bearer secret-token", call.kwargs["headers"]["Authorization"])
|
||||||
|
self.assertNotIn("secret-token", call.args[0])
|
||||||
|
self.assertEqual(10_000_000, call.kwargs["max_bytes"])
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||||
|
def test_full_backfill_resolves_page_details_and_cursor(self, fetch) -> None:
|
||||||
|
fetch.side_effect = (
|
||||||
|
response(
|
||||||
|
{
|
||||||
|
"curtimestamp": "2026-08-22T10:00:00Z",
|
||||||
|
"continue": {"apcontinue": "Next_Page"},
|
||||||
|
"query": {
|
||||||
|
"allpages": [
|
||||||
|
{"pageid": 41, "ns": 0, "title": "First"},
|
||||||
|
{"pageid": 42, "ns": 0, "title": "Second"},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
response(
|
||||||
|
{
|
||||||
|
"query": {
|
||||||
|
"pages": [
|
||||||
|
{"pageid": 41, "ns": 0, "title": "First"},
|
||||||
|
{"pageid": 42, "ns": 0, "title": "Second"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
batch = self.transport.changes(
|
||||||
|
endpoint_url="https://wiki.example.test/api.php",
|
||||||
|
credential=None,
|
||||||
|
cursor=None,
|
||||||
|
limit=2,
|
||||||
|
force_full=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(batch.changes))
|
||||||
|
self.assertEqual("Next_Page", batch.next_cursor)
|
||||||
|
self.assertFalse(batch.complete)
|
||||||
|
details = parse_qs(urlsplit(fetch.call_args_list[1].args[0]).query)
|
||||||
|
self.assertEqual(["41|42"], details["pageids"])
|
||||||
|
self.assertIn("revisions", details["prop"][0])
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||||
|
def test_delta_preserves_deletion_log_tombstone_and_cursor(self, fetch) -> None:
|
||||||
|
fetch.return_value = response(
|
||||||
|
{
|
||||||
|
"curtimestamp": "2026-08-22T11:00:00Z",
|
||||||
|
"query": {
|
||||||
|
"recentchanges": [
|
||||||
|
{
|
||||||
|
"type": "log",
|
||||||
|
"logtype": "delete",
|
||||||
|
"logid": 700,
|
||||||
|
"pageid": 0,
|
||||||
|
"ns": 0,
|
||||||
|
"title": "Deleted page",
|
||||||
|
"timestamp": "2026-08-22T10:59:00Z",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
batch = self.transport.changes(
|
||||||
|
endpoint_url="https://wiki.example.test",
|
||||||
|
credential=None,
|
||||||
|
cursor="rccontinue-token",
|
||||||
|
limit=50,
|
||||||
|
force_full=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("delete", batch.changes[0]["change_kind"])
|
||||||
|
self.assertEqual("logid:700", batch.changes[0]["change_cursor"])
|
||||||
|
self.assertTrue(batch.complete)
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.mediawiki_transport.fetch_http")
|
||||||
|
def test_publish_uses_csrf_body_expected_revision_and_canonical_url(self, fetch) -> None:
|
||||||
|
fetch.return_value = response(
|
||||||
|
{
|
||||||
|
"edit": {
|
||||||
|
"result": "Success",
|
||||||
|
"pageid": 99,
|
||||||
|
"oldrevid": 900,
|
||||||
|
"newrevid": 901,
|
||||||
|
"title": "Published Guide",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.transport.publish(
|
||||||
|
endpoint_url="https://wiki.example.test",
|
||||||
|
credential={"csrf_token": "csrf-secret", "access_token": "token"},
|
||||||
|
title="Published Guide",
|
||||||
|
body="Reviewed body",
|
||||||
|
summary="Approved",
|
||||||
|
expected_revision="900",
|
||||||
|
minor=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
call = fetch.call_args
|
||||||
|
self.assertEqual("POST", call.kwargs["method"])
|
||||||
|
body = parse_qs(call.kwargs["body"].decode())
|
||||||
|
self.assertEqual(["edit"], body["action"])
|
||||||
|
self.assertEqual(["900"], body["baserevid"])
|
||||||
|
self.assertEqual(["1"], body["minor"])
|
||||||
|
self.assertEqual(["csrf-secret"], body["token"])
|
||||||
|
self.assertEqual("99", result.page_id)
|
||||||
|
self.assertEqual(
|
||||||
|
"https://wiki.example.test/wiki/Published_Guide",
|
||||||
|
result.canonical_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from alembic.runtime.migration import MigrationContext
|
||||||
|
from sqlalchemy import create_engine, inspect
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.manifest import get_manifest
|
||||||
|
from govoplan_core.db.migrations import migrate_database
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsMigrationTests(unittest.TestCase):
|
||||||
|
def test_baseline_creates_connector_tables_and_head(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-connectors-migration-") as directory:
|
||||||
|
url = f"sqlite:///{Path(directory) / 'connectors.db'}"
|
||||||
|
migrate_database(
|
||||||
|
database_url=url,
|
||||||
|
enabled_modules=("connectors",),
|
||||||
|
manifest_factories=(get_manifest,),
|
||||||
|
)
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
self.assertIn(
|
||||||
|
"c0f1a2b3c4d5",
|
||||||
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
{
|
||||||
|
"connector_tabular_sources",
|
||||||
|
"connector_sanctions_snapshots",
|
||||||
|
"connector_sanctions_acquisition_runs",
|
||||||
|
"connector_definitions",
|
||||||
|
"connector_definition_revisions",
|
||||||
|
"connector_configurations",
|
||||||
|
"connector_simulation_runs",
|
||||||
|
"connector_knowledge_profiles",
|
||||||
|
"connector_knowledge_objects",
|
||||||
|
"connector_knowledge_sync_runs",
|
||||||
|
"connector_service_desk_profiles",
|
||||||
|
"connector_service_desk_objects",
|
||||||
|
"connector_service_desk_sync_runs",
|
||||||
|
}.issubset(inspect(connection).get_table_names())
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorKnowledgeObject,
|
||||||
|
ConnectorKnowledgeProfile,
|
||||||
|
ConnectorKnowledgeSyncRun,
|
||||||
|
ConnectorServiceDeskObject,
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskSyncRun,
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.manifest import manifest
|
||||||
|
from govoplan_connectors.backend.provider_state import (
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
SANCTIONS_PROVIDER_ID,
|
||||||
|
SERVICE_DESK_PROVIDER_ID,
|
||||||
|
TABULAR_PROVIDER_ID,
|
||||||
|
knowledge_provider_states,
|
||||||
|
sanctions_provider_states,
|
||||||
|
service_desk_provider_states,
|
||||||
|
tabular_provider_states,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.provider_governance import ExternalProviderStateContext
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsProviderStateTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_immutable_tabular_snapshot_reports_ready_state(self) -> None:
|
||||||
|
self.session.add(
|
||||||
|
ConnectorTabularSource(
|
||||||
|
id="tabular-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
source_name="monthly",
|
||||||
|
name="Monthly input",
|
||||||
|
status="active",
|
||||||
|
schema_version=1,
|
||||||
|
schema_=[{"name": "id", "type": "string"}],
|
||||||
|
rows=[{"id": "1"}],
|
||||||
|
fingerprint="a" * 64,
|
||||||
|
row_count=1,
|
||||||
|
byte_count=10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
state = tabular_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("healthy", state.health)
|
||||||
|
self.assertEqual("ready", state.recovery)
|
||||||
|
self.assertEqual("not_applicable", state.freshness)
|
||||||
|
|
||||||
|
def test_sanctions_state_hashes_binding_and_manifest_registers_state(self) -> None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
run = ConnectorSanctionsAcquisitionRun(
|
||||||
|
id="run-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="eu",
|
||||||
|
source_id="secret-source-name",
|
||||||
|
status="succeeded",
|
||||||
|
attempt_count=1,
|
||||||
|
started_at=now,
|
||||||
|
finished_at=now,
|
||||||
|
)
|
||||||
|
snapshot = ConnectorSanctionsSnapshot(
|
||||||
|
id="snapshot-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="eu",
|
||||||
|
publisher="European Union",
|
||||||
|
jurisdiction="EU",
|
||||||
|
list_type="sanctions",
|
||||||
|
source_id="secret-source-name",
|
||||||
|
source_version="2026-08-01",
|
||||||
|
acquired_at=now,
|
||||||
|
source_url="https://source.example.test/list.xml",
|
||||||
|
content_type="application/xml",
|
||||||
|
byte_count=8,
|
||||||
|
sha256="b" * 64,
|
||||||
|
parser_version="1",
|
||||||
|
connector_run_id=run.id,
|
||||||
|
raw_content=b"<list/>",
|
||||||
|
)
|
||||||
|
run.snapshot_id = snapshot.id
|
||||||
|
self.session.add_all((run, snapshot))
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
state = sanctions_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("healthy", state.health)
|
||||||
|
self.assertEqual("ready", state.recovery)
|
||||||
|
rendered = str(state.to_dict())
|
||||||
|
self.assertNotIn("secret-source-name", rendered)
|
||||||
|
self.assertNotIn("source.example.test", rendered)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
TABULAR_PROVIDER_ID,
|
||||||
|
SANCTIONS_PROVIDER_ID,
|
||||||
|
KNOWLEDGE_PROVIDER_ID,
|
||||||
|
SERVICE_DESK_PROVIDER_ID,
|
||||||
|
},
|
||||||
|
{item.provider_id for item in manifest.external_provider_state_providers},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_external_knowledge_state_reports_health_without_acl_or_endpoint_data(self) -> None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
profile = ConnectorKnowledgeProfile(
|
||||||
|
id="knowledge-profile-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
configuration_id="configuration-secret",
|
||||||
|
status="active",
|
||||||
|
product="bluespice",
|
||||||
|
product_version="4.5.2",
|
||||||
|
desired_maturity="migrate",
|
||||||
|
discovered_maturity="migrate",
|
||||||
|
source_authority_mode="external_mirror",
|
||||||
|
default_visibility="restricted",
|
||||||
|
default_acl_tokens=["group:secret-acl"],
|
||||||
|
namespace_mappings=[{"secret": "mapping"}],
|
||||||
|
capabilities=["read", "synchronize", "migrate"],
|
||||||
|
health_status="healthy",
|
||||||
|
discovered_at=now,
|
||||||
|
)
|
||||||
|
page = ConnectorKnowledgeObject(
|
||||||
|
id="knowledge-object-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
profile_id=profile.id,
|
||||||
|
object_type="page",
|
||||||
|
external_id="42",
|
||||||
|
title="Secret page title",
|
||||||
|
status="active",
|
||||||
|
source_revision="501",
|
||||||
|
content_hash="c" * 64,
|
||||||
|
visibility="restricted",
|
||||||
|
acl_tokens=["group:secret-acl"],
|
||||||
|
observed_at=now,
|
||||||
|
)
|
||||||
|
run = ConnectorKnowledgeSyncRun(
|
||||||
|
id="knowledge-run-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
profile_id=profile.id,
|
||||||
|
mode="delta",
|
||||||
|
idempotency_key="secret-key",
|
||||||
|
request_hash="d" * 64,
|
||||||
|
status="completed",
|
||||||
|
started_at=now,
|
||||||
|
finished_at=now,
|
||||||
|
)
|
||||||
|
self.session.add_all((profile, page, run))
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
state = knowledge_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("healthy", state.health)
|
||||||
|
self.assertEqual("ready", state.recovery)
|
||||||
|
self.assertEqual(1, state.metrics["active_objects"])
|
||||||
|
rendered = str(state.to_dict())
|
||||||
|
self.assertNotIn("Secret page title", rendered)
|
||||||
|
self.assertNotIn("group:secret-acl", rendered)
|
||||||
|
self.assertNotIn("configuration-secret", rendered)
|
||||||
|
|
||||||
|
def test_service_desk_state_reports_recovery_without_ticket_or_acl_data(self) -> None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
configuration = ConnectorConfiguration(
|
||||||
|
id="configuration-secret",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id="definition-1",
|
||||||
|
name="Secret service desk",
|
||||||
|
status="active",
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={},
|
||||||
|
protected_paths=[],
|
||||||
|
effective_configuration={},
|
||||||
|
effective_hash="configuration-hash",
|
||||||
|
resource_revision=3,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
)
|
||||||
|
profile = ConnectorServiceDeskProfile(
|
||||||
|
id="service-desk-profile-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
configuration_id="configuration-secret",
|
||||||
|
status="active",
|
||||||
|
integration_mode="synchronize",
|
||||||
|
product="znuny",
|
||||||
|
product_version="7.1.4",
|
||||||
|
desired_maturity="synchronize",
|
||||||
|
discovered_maturity="synchronize",
|
||||||
|
source_authority_mode="governed_sync",
|
||||||
|
default_visibility="restricted",
|
||||||
|
default_acl_tokens=["group:secret-acl"],
|
||||||
|
routes={"secret": "route"},
|
||||||
|
queue_mappings=[{"secret": "queue"}],
|
||||||
|
dynamic_field_mappings=[{"secret": "field"}],
|
||||||
|
capabilities=["read", "search", "synchronize", "publish"],
|
||||||
|
discovered_configuration_revision=3,
|
||||||
|
discovered_configuration_hash="configuration-hash",
|
||||||
|
health_status="healthy",
|
||||||
|
discovered_at=now,
|
||||||
|
)
|
||||||
|
ticket = ConnectorServiceDeskObject(
|
||||||
|
id="service-desk-object-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
profile_id=profile.id,
|
||||||
|
object_type="ticket",
|
||||||
|
external_id="42",
|
||||||
|
external_ticket_number="secret-number",
|
||||||
|
title="Secret ticket title",
|
||||||
|
status="active",
|
||||||
|
source_revision="2026-08-22T10:00:00Z",
|
||||||
|
content_hash="e" * 64,
|
||||||
|
visibility="restricted",
|
||||||
|
acl_tokens=["group:secret-acl"],
|
||||||
|
mapped_data={"secret": "ticket content"},
|
||||||
|
provenance={"secret": "provider evidence"},
|
||||||
|
observed_at=now,
|
||||||
|
)
|
||||||
|
run = ConnectorServiceDeskSyncRun(
|
||||||
|
id="service-desk-run-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
profile_id=profile.id,
|
||||||
|
mode="update",
|
||||||
|
idempotency_key="secret-key",
|
||||||
|
request_hash="f" * 64,
|
||||||
|
status="outcome_unknown",
|
||||||
|
started_at=now,
|
||||||
|
finished_at=now,
|
||||||
|
)
|
||||||
|
self.session.add_all((configuration, profile, ticket, run))
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
state = service_desk_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual(SERVICE_DESK_PROVIDER_ID, state.provider_id)
|
||||||
|
self.assertTrue(state.metrics["discovery_current"])
|
||||||
|
self.assertEqual("pending", state.conflict)
|
||||||
|
self.assertEqual("attention", state.recovery)
|
||||||
|
rendered = str(state.to_dict())
|
||||||
|
for secret in (
|
||||||
|
"configuration-secret",
|
||||||
|
"secret-number",
|
||||||
|
"Secret ticket title",
|
||||||
|
"group:secret-acl",
|
||||||
|
"ticket content",
|
||||||
|
"provider evidence",
|
||||||
|
"secret-key",
|
||||||
|
):
|
||||||
|
self.assertNotIn(secret, rendered)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,263 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.feeds import ConnectorFeedProvider
|
||||||
|
from govoplan_connectors.backend.recovery import (
|
||||||
|
CONNECTOR_RECOVERY_OPERATIONS,
|
||||||
|
ConnectorRecoveryError,
|
||||||
|
begin_connector_external_mutation,
|
||||||
|
begin_connector_read_snapshot,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.router import api_import_feed_snapshot
|
||||||
|
from govoplan_connectors.backend.schemas import FeedImportRequest
|
||||||
|
from govoplan_connectors.backend.tabular_sources import WRITE_SCOPE
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryCheckpoint,
|
||||||
|
RecoveryOperation,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
claim_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import TabularSnapshotInput
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
DistributedLease,
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.tabular_sources import SqlTabularSourceProvider
|
||||||
|
|
||||||
|
|
||||||
|
RSS = b"""<?xml version="1.0"?>
|
||||||
|
<rss version="2.0"><channel><title>Updates</title>
|
||||||
|
<link>https://example.test/</link><description>Updates</description>
|
||||||
|
<item><guid>1</guid><title>One</title></item></channel></rss>"""
|
||||||
|
|
||||||
|
|
||||||
|
def _identity(node: str, incarnation: str) -> RuntimeIdentity:
|
||||||
|
return RuntimeIdentity(
|
||||||
|
installation_id="connector-recovery-tests",
|
||||||
|
node_id=node,
|
||||||
|
incarnation=incarnation,
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="a" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal() -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset({WRITE_SCOPE}),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorRecoveryTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
DistributedLease.__table__,
|
||||||
|
RecoveryOperation.__table__,
|
||||||
|
RecoveryCheckpoint.__table__,
|
||||||
|
ConnectorTabularSource.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine, expire_on_commit=False)
|
||||||
|
bind_process_runtime_identity(_identity("node-1", "incarnation-1"))
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_feed_snapshot_and_recovery_checkpoint_commit_atomically_and_replay(self) -> None:
|
||||||
|
document = ConnectorFeedProvider().parse(
|
||||||
|
RSS,
|
||||||
|
source_url="https://example.test/feed.xml",
|
||||||
|
)
|
||||||
|
payload = FeedImportRequest(
|
||||||
|
url="https://example.test/feed.xml",
|
||||||
|
name="Updates",
|
||||||
|
source_name="updates",
|
||||||
|
)
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_connectors.backend.router.feed_transport.fetch",
|
||||||
|
return_value=document,
|
||||||
|
) as fetch,
|
||||||
|
patch("govoplan_connectors.backend.router.audit_event"),
|
||||||
|
):
|
||||||
|
first = api_import_feed_snapshot(
|
||||||
|
payload,
|
||||||
|
session=self.session,
|
||||||
|
principal=_principal(),
|
||||||
|
idempotency_key="feed-import-1",
|
||||||
|
)
|
||||||
|
replay = api_import_feed_snapshot(
|
||||||
|
payload,
|
||||||
|
session=self.session,
|
||||||
|
principal=_principal(),
|
||||||
|
idempotency_key="feed-import-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(first.ref, replay.ref)
|
||||||
|
fetch.assert_called_once()
|
||||||
|
operation = self.session.scalar(select(RecoveryOperation))
|
||||||
|
assert operation is not None
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||||
|
self.assertEqual(
|
||||||
|
first.ref.removeprefix("snapshot:"),
|
||||||
|
operation.resource_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_recovery_metadata_distinguishes_reads_from_external_mutations(self) -> None:
|
||||||
|
declarations = {
|
||||||
|
item.operation_type: item for item in CONNECTOR_RECOVERY_OPERATIONS
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertFalse(declarations["read-snapshot"].provider_mutation)
|
||||||
|
self.assertTrue(declarations["read-snapshot"].implemented)
|
||||||
|
self.assertTrue(declarations["external-mutation"].provider_mutation)
|
||||||
|
self.assertTrue(declarations["external-mutation"].implemented)
|
||||||
|
|
||||||
|
def test_stale_atomic_connector_fence_fails_without_claiming_an_effect(self) -> None:
|
||||||
|
recovery = begin_connector_read_snapshot(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="provider-1",
|
||||||
|
idempotency_key="read-1",
|
||||||
|
source_revision="revision-1",
|
||||||
|
cursor="cursor-1",
|
||||||
|
dry_run_evidence={"performed": True, "approved": True},
|
||||||
|
)
|
||||||
|
lease = self.session.scalar(select(DistributedLease))
|
||||||
|
assert lease is not None
|
||||||
|
lease.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
|
||||||
|
self.session.commit()
|
||||||
|
bind_process_runtime_identity(_identity("node-2", "incarnation-2"))
|
||||||
|
|
||||||
|
with self.assertRaises(RecoveryOperationStateConflict):
|
||||||
|
claim_durable_recovery_operation(
|
||||||
|
recovery.operation.session_factory,
|
||||||
|
identity=_identity("node-2", "incarnation-2"),
|
||||||
|
operation_id=recovery.operation_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.FAILED.value, operation.status)
|
||||||
|
|
||||||
|
def test_external_mutation_unknown_outcome_blocks_blind_retry(self) -> None:
|
||||||
|
kwargs = {
|
||||||
|
"tenant_id": "tenant-1",
|
||||||
|
"provider_id": "provider-1",
|
||||||
|
"idempotency_key": "publish-1",
|
||||||
|
"request_sha256": "b" * 64,
|
||||||
|
"source_revision": "revision-1",
|
||||||
|
"cursor": None,
|
||||||
|
"dry_run_evidence": {"performed": True, "approved": True},
|
||||||
|
"resource_type": "external_record",
|
||||||
|
"resource_id": "record-1",
|
||||||
|
}
|
||||||
|
recovery = begin_connector_external_mutation(self.session, **kwargs)
|
||||||
|
recovery.outcome_unknown(
|
||||||
|
summary="The provider connection closed after dispatch",
|
||||||
|
provider_code="connection_closed",
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(ConnectorRecoveryError):
|
||||||
|
begin_connector_external_mutation(self.session, **kwargs)
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
|
||||||
|
|
||||||
|
def test_tampered_chain_rolls_back_the_atomic_connector_projection(self) -> None:
|
||||||
|
recovery = begin_connector_read_snapshot(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="provider-1",
|
||||||
|
idempotency_key="tampered-read",
|
||||||
|
source_revision=None,
|
||||||
|
cursor=None,
|
||||||
|
dry_run_evidence={"performed": False, "reason": "read-only"},
|
||||||
|
)
|
||||||
|
checkpoint = self.session.scalar(
|
||||||
|
select(RecoveryCheckpoint)
|
||||||
|
.where(RecoveryCheckpoint.operation_id == recovery.operation_id)
|
||||||
|
.order_by(RecoveryCheckpoint.sequence)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
assert checkpoint is not None
|
||||||
|
checkpoint.summary = "tampered"
|
||||||
|
self.session.commit()
|
||||||
|
source = SqlTabularSourceProvider().create_snapshot(
|
||||||
|
self.session,
|
||||||
|
_principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Tampered",
|
||||||
|
source_name="tampered",
|
||||||
|
rows=({"id": 1},),
|
||||||
|
),
|
||||||
|
source_id=recovery.resource_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(ConnectorRecoveryError):
|
||||||
|
recovery.commit_success(
|
||||||
|
self.session,
|
||||||
|
evidence={
|
||||||
|
"verified": True,
|
||||||
|
"checks": {"snapshot_ref": source.ref},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(
|
||||||
|
self.session.get(ConnectorTabularSource, recovery.resource_id)
|
||||||
|
)
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.RUNNING.value, operation.status)
|
||||||
|
|
||||||
|
def test_definitive_external_rejection_is_terminal(self) -> None:
|
||||||
|
recovery = begin_connector_external_mutation(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id="provider-1",
|
||||||
|
idempotency_key="publish-rejected",
|
||||||
|
request_sha256="c" * 64,
|
||||||
|
source_revision="revision-1",
|
||||||
|
cursor=None,
|
||||||
|
dry_run_evidence={"performed": True, "approved": True},
|
||||||
|
resource_type="external_record",
|
||||||
|
resource_id="record-2",
|
||||||
|
)
|
||||||
|
recovery.reject(
|
||||||
|
summary="The provider rejected the requested revision",
|
||||||
|
provider_code="revision_conflict",
|
||||||
|
)
|
||||||
|
|
||||||
|
operation = self.session.get(RecoveryOperation, recovery.operation_id)
|
||||||
|
self.session.refresh(operation)
|
||||||
|
self.assertEqual(RecoveryStatus.REJECTED.value, operation.status)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
import hashlib
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from urllib.error import URLError
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorSanctionsAcquisitionRun,
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.sanctions_sources import (
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
SOURCE_DEFINITIONS,
|
||||||
|
SYNTHETIC_PROVIDER_ID,
|
||||||
|
SYNTHETIC_UN_XML,
|
||||||
|
SanctionsSourceError,
|
||||||
|
SqlSanctionsSnapshotProvider,
|
||||||
|
TransportResponse,
|
||||||
|
UNSC_PROVIDER_ID,
|
||||||
|
UrllibSanctionsTransport,
|
||||||
|
)
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryCheckpoint,
|
||||||
|
RecoveryOperation,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
DistributedLease,
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base, utcnow
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
scopes: tuple[str, ...] = (
|
||||||
|
SANCTIONS_READ_SCOPE,
|
||||||
|
SANCTIONS_REFRESH_SCOPE,
|
||||||
|
),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Transport:
|
||||||
|
def __init__(self, responses):
|
||||||
|
self.responses = list(responses)
|
||||||
|
self.headers = []
|
||||||
|
|
||||||
|
def fetch(self, definition, *, headers):
|
||||||
|
del definition
|
||||||
|
self.headers.append(dict(headers))
|
||||||
|
response = self.responses.pop(0)
|
||||||
|
if isinstance(response, Exception):
|
||||||
|
raise response
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def response(
|
||||||
|
content: bytes = SYNTHETIC_UN_XML,
|
||||||
|
*,
|
||||||
|
status: int = 200,
|
||||||
|
content_type: str = "application/xml",
|
||||||
|
etag: str = '"fixture-v1"',
|
||||||
|
) -> TransportResponse:
|
||||||
|
return TransportResponse(
|
||||||
|
status=status,
|
||||||
|
final_url="https://scsanctions.un.org/consolidated.xml",
|
||||||
|
headers={
|
||||||
|
"content-type": content_type,
|
||||||
|
"etag": etag,
|
||||||
|
},
|
||||||
|
content=content,
|
||||||
|
attempts=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SanctionsSourcesTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
DistributedLease.__table__,
|
||||||
|
RecoveryOperation.__table__,
|
||||||
|
RecoveryCheckpoint.__table__,
|
||||||
|
ConnectorSanctionsAcquisitionRun.__table__,
|
||||||
|
ConnectorSanctionsSnapshot.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
bind_process_runtime_identity(
|
||||||
|
RuntimeIdentity(
|
||||||
|
installation_id="connectors-tests",
|
||||||
|
node_id="connectors-test-node",
|
||||||
|
incarnation="connectors-test-incarnation",
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="a" * 64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_fixture_refreshes_are_immutable_and_evidence_is_readable(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider()
|
||||||
|
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
second = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertEqual("succeeded", first.status)
|
||||||
|
self.assertEqual("succeeded", second.status)
|
||||||
|
self.assertNotEqual(first.snapshot.ref, second.snapshot.ref)
|
||||||
|
self.assertEqual(
|
||||||
|
hashlib.sha256(SYNTHETIC_UN_XML).hexdigest(),
|
||||||
|
first.snapshot.sha256,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
SYNTHETIC_UN_XML,
|
||||||
|
provider.read_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot_ref=first.snapshot.ref,
|
||||||
|
).content,
|
||||||
|
)
|
||||||
|
runs = provider.list_runs(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
all(
|
||||||
|
run.request_evidence["subject_data_transmitted"]
|
||||||
|
is False
|
||||||
|
for run in runs
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_conditional_fetch_reuses_prior_immutable_snapshot(self) -> None:
|
||||||
|
transport = _Transport(
|
||||||
|
(
|
||||||
|
response(),
|
||||||
|
response(b"", status=304),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
provider = SqlSanctionsSnapshotProvider(transport)
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
second = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("not_modified", second.status)
|
||||||
|
self.assertEqual(first.snapshot.ref, second.snapshot.ref)
|
||||||
|
self.assertEqual(
|
||||||
|
{'If-None-Match': '"fixture-v1"'},
|
||||||
|
transport.headers[1],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
1,
|
||||||
|
self.session.query(ConnectorSanctionsSnapshot).count(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_malformed_and_changed_sources_have_explicit_health(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
cases = (
|
||||||
|
(
|
||||||
|
b"<CONSOLIDATED_LIST>",
|
||||||
|
"application/xml",
|
||||||
|
"malformed",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
b"<DIFFERENT><INDIVIDUALS/><ENTITIES/></DIFFERENT>",
|
||||||
|
"application/xml",
|
||||||
|
"unexpected_change",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
SYNTHETIC_UN_XML,
|
||||||
|
"text/html",
|
||||||
|
"unexpected_change",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for payload, content_type, expected in cases:
|
||||||
|
with self.subTest(expected=expected, content_type=content_type):
|
||||||
|
provider = SqlSanctionsSnapshotProvider(
|
||||||
|
_Transport(
|
||||||
|
(
|
||||||
|
response(
|
||||||
|
payload,
|
||||||
|
content_type=content_type,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
self.assertEqual(expected, result.status)
|
||||||
|
self.assertIsNotNone(result.error)
|
||||||
|
|
||||||
|
def test_unavailable_source_becomes_stale_when_evidence_is_old(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider(
|
||||||
|
_Transport(
|
||||||
|
(
|
||||||
|
response(),
|
||||||
|
SanctionsSourceError("offline"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
record = self.session.get(
|
||||||
|
ConnectorSanctionsSnapshot,
|
||||||
|
first.snapshot.ref.removeprefix("sanctions-snapshot:"),
|
||||||
|
)
|
||||||
|
record.acquired_at = utcnow() - timedelta(days=3)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
result = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("stale", result.status)
|
||||||
|
self.assertEqual(first.snapshot.ref, result.snapshot.ref)
|
||||||
|
|
||||||
|
def test_idempotent_refresh_replays_the_committed_acquisition(self) -> None:
|
||||||
|
transport = _Transport((response(),))
|
||||||
|
provider = SqlSanctionsSnapshotProvider(transport)
|
||||||
|
|
||||||
|
first = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
idempotency_key="scheduled-refresh-1",
|
||||||
|
)
|
||||||
|
replay = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
idempotency_key="scheduled-refresh-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(first.run_id, replay.run_id)
|
||||||
|
self.assertEqual(first.snapshot.ref, replay.snapshot.ref)
|
||||||
|
self.assertEqual([], transport.responses)
|
||||||
|
operation = self.session.query(RecoveryOperation).one()
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||||
|
|
||||||
|
def test_provider_failure_commits_failed_run_and_terminal_recovery(self) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider(
|
||||||
|
_Transport((SanctionsSourceError("offline"),))
|
||||||
|
)
|
||||||
|
|
||||||
|
result = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=UNSC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("unavailable", result.status)
|
||||||
|
operation = self.session.query(RecoveryOperation).one()
|
||||||
|
self.assertEqual(RecoveryStatus.FAILED.value, operation.status)
|
||||||
|
self.assertEqual(
|
||||||
|
result.run_id,
|
||||||
|
self.session.query(ConnectorSanctionsAcquisitionRun).one().id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_snapshot_access_is_tenant_and_scope_isolated(self) -> None:
|
||||||
|
provider = SqlSanctionsSnapshotProvider()
|
||||||
|
created = provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(
|
||||||
|
provider.get_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
snapshot_ref=created.snapshot.ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(Exception, "Missing scope"):
|
||||||
|
provider.list_snapshots(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_transport_retries_transient_network_failures(self) -> None:
|
||||||
|
class _Headers(dict):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class _Response:
|
||||||
|
status = 200
|
||||||
|
headers = _Headers(
|
||||||
|
{
|
||||||
|
"Content-Type": "application/xml",
|
||||||
|
"Content-Length": str(len(SYNTHETIC_UN_XML)),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def geturl(self):
|
||||||
|
return (
|
||||||
|
"https://scsanctions.un.org/"
|
||||||
|
"resources/xml/en/consolidated.xml"
|
||||||
|
)
|
||||||
|
|
||||||
|
def read(self, size):
|
||||||
|
del size
|
||||||
|
if hasattr(self, "_read"):
|
||||||
|
return b""
|
||||||
|
self._read = True
|
||||||
|
return SYNTHETIC_UN_XML
|
||||||
|
|
||||||
|
opener = unittest.mock.Mock()
|
||||||
|
opener.open.side_effect = (
|
||||||
|
URLError("temporary"),
|
||||||
|
URLError("temporary"),
|
||||||
|
_Response(),
|
||||||
|
)
|
||||||
|
sleeps = []
|
||||||
|
transport = UrllibSanctionsTransport(
|
||||||
|
sleeper=sleeps.append
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_connectors.backend.sanctions_sources.build_opener",
|
||||||
|
return_value=opener,
|
||||||
|
):
|
||||||
|
fetched = transport.fetch(
|
||||||
|
SOURCE_DEFINITIONS[UNSC_PROVIDER_ID],
|
||||||
|
headers={},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(3, fetched.attempts)
|
||||||
|
self.assertEqual([1.0, 2.0], sleeps)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,806 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.search import SearchAuthorizationRequest, SearchBackfillRequest
|
||||||
|
from govoplan_core.core.recovery import RecoveryOperation, RecoveryStatus
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorDefinitionRevision,
|
||||||
|
ConnectorServiceDeskObject,
|
||||||
|
ConnectorServiceDeskProfile,
|
||||||
|
ConnectorServiceDeskSyncRun,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.service_desk_connector import (
|
||||||
|
SERVICE_DESK_PROVIDER_ID,
|
||||||
|
SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
ServiceDeskConnectorError,
|
||||||
|
create_profile,
|
||||||
|
discover_profile,
|
||||||
|
list_objects,
|
||||||
|
synchronize_profile,
|
||||||
|
update_profile,
|
||||||
|
update_ticket,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.service_desk_schemas import (
|
||||||
|
ServiceDeskDynamicFieldMapping,
|
||||||
|
ServiceDeskProfileCreateRequest,
|
||||||
|
ServiceDeskProfileUpdateRequest,
|
||||||
|
ServiceDeskQueueMapping,
|
||||||
|
ServiceDeskRouteMapping,
|
||||||
|
ServiceDeskSyncRequest,
|
||||||
|
ServiceDeskTicketUpdateRequest,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.service_desk_search import ExternalServiceDeskSearchSource
|
||||||
|
from govoplan_connectors.backend.service_desk_transport import (
|
||||||
|
ServiceDeskChangeBatch,
|
||||||
|
ServiceDeskTransportError,
|
||||||
|
ServiceDeskUpdateResult,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
ALL_SCOPES = frozenset(
|
||||||
|
{
|
||||||
|
"connectors:service_desk:read",
|
||||||
|
"connectors:service_desk:admin",
|
||||||
|
"connectors:service_desk:sync",
|
||||||
|
"connectors:service_desk:update",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
groups: frozenset[str] = frozenset({"agents"}),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=ALL_SCOPES,
|
||||||
|
group_ids=groups,
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="account-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def ticket(
|
||||||
|
*,
|
||||||
|
ticket_id: str = "42",
|
||||||
|
revision: str = "2026-08-22T10:00:00Z",
|
||||||
|
queue: str = "Residents",
|
||||||
|
acl: list[str] | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"TicketID": ticket_id,
|
||||||
|
"TicketNumber": f"20260822{ticket_id}",
|
||||||
|
"Title": "Resident parking inquiry",
|
||||||
|
"Queue": queue,
|
||||||
|
"QueueID": "3",
|
||||||
|
"State": "open",
|
||||||
|
"StateID": "4",
|
||||||
|
"Priority": "3 normal",
|
||||||
|
"PriorityID": "3",
|
||||||
|
"Owner": "agent.a",
|
||||||
|
"OwnerID": "7",
|
||||||
|
"CustomerUserID": "citizen-17",
|
||||||
|
"CustomerID": "organization-9",
|
||||||
|
"Changed": revision,
|
||||||
|
"GovOPlaNVisibility": "restricted",
|
||||||
|
"GovOPlaNACL": acl or ["group:agents"],
|
||||||
|
"DynamicField": [{"Name": "PermitKind", "Value": "resident"}],
|
||||||
|
"Article": [
|
||||||
|
{
|
||||||
|
"ArticleID": "71",
|
||||||
|
"Subject": "Question",
|
||||||
|
"Body": "Please verify the submitted address.",
|
||||||
|
"Created": "2026-08-22T09:58:00Z",
|
||||||
|
"Attachment": [
|
||||||
|
{
|
||||||
|
"AttachmentID": "91",
|
||||||
|
"Filename": "address.pdf",
|
||||||
|
"Filesize": 1234,
|
||||||
|
"ContentType": "application/pdf",
|
||||||
|
"ContentBase64": "not-retained",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class StaticTransport:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.batches: list[ServiceDeskChangeBatch] = []
|
||||||
|
self.change_calls: list[dict[str, object]] = []
|
||||||
|
self.update_calls = 0
|
||||||
|
self.update_result = ServiceDeskUpdateResult(
|
||||||
|
ticket={
|
||||||
|
**ticket(revision="2026-08-22T11:00:00Z"),
|
||||||
|
"State": "pending reminder",
|
||||||
|
},
|
||||||
|
revision="2026-08-22T11:00:00Z",
|
||||||
|
evidence={"verified": True},
|
||||||
|
)
|
||||||
|
self.update_error: ServiceDeskTransportError | None = None
|
||||||
|
|
||||||
|
def discover(self, **kwargs):
|
||||||
|
del kwargs
|
||||||
|
return {
|
||||||
|
"product": "znuny",
|
||||||
|
"product_version": "7.1.4",
|
||||||
|
"api_family": "generic_interface_rest",
|
||||||
|
"capabilities": [
|
||||||
|
"discover",
|
||||||
|
"link",
|
||||||
|
"search",
|
||||||
|
"read",
|
||||||
|
"synchronize",
|
||||||
|
"publish",
|
||||||
|
],
|
||||||
|
"maturity": "synchronize",
|
||||||
|
"health_status": "healthy",
|
||||||
|
"revision": "discovery-1",
|
||||||
|
"diagnostics": [],
|
||||||
|
"evidence": {"fixture": True},
|
||||||
|
}
|
||||||
|
|
||||||
|
def changes(self, **kwargs):
|
||||||
|
self.change_calls.append(dict(kwargs))
|
||||||
|
if not self.batches:
|
||||||
|
raise AssertionError("No deterministic service-desk batch remains")
|
||||||
|
return self.batches.pop(0)
|
||||||
|
|
||||||
|
def update_ticket(self, **kwargs):
|
||||||
|
del kwargs
|
||||||
|
self.update_calls += 1
|
||||||
|
if self.update_error is not None:
|
||||||
|
raise self.update_error
|
||||||
|
return self.update_result
|
||||||
|
|
||||||
|
|
||||||
|
class RecordingSearchWriter:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.upserts: list[object] = []
|
||||||
|
self.deletes: list[str] = []
|
||||||
|
|
||||||
|
def upsert_document(self, _session, _principal, *, document) -> None:
|
||||||
|
self.upserts.append(document)
|
||||||
|
|
||||||
|
def delete_document(
|
||||||
|
self,
|
||||||
|
_session,
|
||||||
|
_principal,
|
||||||
|
*,
|
||||||
|
tenant_id,
|
||||||
|
module_id,
|
||||||
|
resource_type,
|
||||||
|
resource_id,
|
||||||
|
) -> bool:
|
||||||
|
del tenant_id, module_id, resource_type
|
||||||
|
self.deletes.append(resource_id)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def enqueue_change(self, _session, *, change) -> bool:
|
||||||
|
del change
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class SearchRegistry:
|
||||||
|
def __init__(self, writer: RecordingSearchWriter) -> None:
|
||||||
|
self.writer = writer
|
||||||
|
|
||||||
|
def has_capability(self, name: str) -> bool:
|
||||||
|
return name == "search.index_writer"
|
||||||
|
|
||||||
|
def capability(self, name: str):
|
||||||
|
if not self.has_capability(name):
|
||||||
|
raise KeyError(name)
|
||||||
|
return self.writer
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskConnectorTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
bind_process_runtime_identity(
|
||||||
|
RuntimeIdentity(
|
||||||
|
installation_id="service-desk-connector-tests",
|
||||||
|
node_id="node-1",
|
||||||
|
incarnation="incarnation-1",
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="a" * 64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.audit = patch("govoplan_connectors.backend.service_desk_connector.audit_event")
|
||||||
|
self.audit.start()
|
||||||
|
self.credential = patch(
|
||||||
|
"govoplan_connectors.backend.service_desk_connector._credential",
|
||||||
|
return_value={"user_login": "connector", "password": "secret"},
|
||||||
|
)
|
||||||
|
self.credential.start()
|
||||||
|
self.transport = StaticTransport()
|
||||||
|
self.configuration_id = self._seed_configuration()
|
||||||
|
self.profile_id = self._create_profile()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
self.credential.stop()
|
||||||
|
self.audit.stop()
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _seed_configuration(self) -> str:
|
||||||
|
definition = ConnectorDefinition(
|
||||||
|
id="definition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_key="service-desk.znuny",
|
||||||
|
name="Znuny",
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
local_definition=True,
|
||||||
|
)
|
||||||
|
self.session.add(definition)
|
||||||
|
self.session.add(
|
||||||
|
ConnectorDefinitionRevision(
|
||||||
|
id="definition-revision-1",
|
||||||
|
definition_id=definition.id,
|
||||||
|
revision=1,
|
||||||
|
specification={"provider": "znuny", "protocol": "generic_interface_rest"},
|
||||||
|
definition_hash="definition-hash",
|
||||||
|
origin="local",
|
||||||
|
created_by="account-1",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
configuration = ConnectorConfiguration(
|
||||||
|
id="configuration-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id=definition.id,
|
||||||
|
name="Institutional service desk",
|
||||||
|
status="active",
|
||||||
|
endpoint_url="https://support.example.invalid/znuny/nph-genericinterface.pl/Webservice/GovOPlaN",
|
||||||
|
credential_ref="credential-envelope-1",
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={},
|
||||||
|
protected_paths=[],
|
||||||
|
effective_configuration={
|
||||||
|
"provider": "znuny",
|
||||||
|
"protocol": "generic_interface_rest",
|
||||||
|
},
|
||||||
|
effective_hash="configuration-hash",
|
||||||
|
resource_revision=1,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add(configuration)
|
||||||
|
self.session.flush()
|
||||||
|
return configuration.id
|
||||||
|
|
||||||
|
def _create_profile(self) -> str:
|
||||||
|
item = create_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
ServiceDeskProfileCreateRequest(
|
||||||
|
configuration_id=self.configuration_id,
|
||||||
|
integration_mode="synchronize",
|
||||||
|
desired_maturity="synchronize",
|
||||||
|
source_authority_mode="governed_sync",
|
||||||
|
default_visibility="restricted",
|
||||||
|
default_acl_tokens=["group:service-desk-managers"],
|
||||||
|
routes=ServiceDeskRouteMapping(
|
||||||
|
update_path="/Ticket/{ticket_id}",
|
||||||
|
ticket_web_url_template="https://support.example.invalid/ticket/{ticket_id}",
|
||||||
|
),
|
||||||
|
queue_mappings=[
|
||||||
|
ServiceDeskQueueMapping(
|
||||||
|
source_queue="Residents",
|
||||||
|
target_queue_ref="helpdesk:residents",
|
||||||
|
visibility="restricted",
|
||||||
|
acl_tokens=["group:agents"],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
dynamic_field_mappings=[
|
||||||
|
ServiceDeskDynamicFieldMapping(
|
||||||
|
source_name="PermitKind",
|
||||||
|
target_name="permit_kind",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return item.id
|
||||||
|
|
||||||
|
def _discover(self) -> None:
|
||||||
|
discover_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
transport=self.transport,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _sync(self, raw: list[dict[str, object]], *, key: str, complete: bool = True):
|
||||||
|
self.transport.batches.append(
|
||||||
|
ServiceDeskChangeBatch(
|
||||||
|
changes=tuple(raw),
|
||||||
|
next_cursor=(
|
||||||
|
None
|
||||||
|
if complete
|
||||||
|
else json.dumps(
|
||||||
|
{"kind": "full", "offset": len(raw), "fingerprint": "fixture"}
|
||||||
|
)
|
||||||
|
),
|
||||||
|
complete=complete,
|
||||||
|
high_watermark="2026-08-22T10:00:00Z",
|
||||||
|
live_ids=tuple(str(value["TicketID"]) for value in raw) if complete else None,
|
||||||
|
evidence={"fixture": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(idempotency_key=key),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_mapping_replay_cursor_transition_and_search_acl(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
run = self._sync([ticket()], key="sync-1")
|
||||||
|
self.assertEqual({"created": 1}, run.counts)
|
||||||
|
self.assertEqual("delta", json.loads(run.cursor_after or "{}").get("kind"))
|
||||||
|
self.assertIn("attachment_content_omitted", {item.code for item in run.diagnostics})
|
||||||
|
|
||||||
|
objects, _cursor = list_objects(self.session, principal(), profile_id=self.profile_id)
|
||||||
|
self.assertEqual(1, len(objects))
|
||||||
|
item = objects[0]
|
||||||
|
self.assertEqual("helpdesk:residents", item.mapped_data["target_queue_ref"])
|
||||||
|
self.assertEqual("resident", item.mapped_data["dynamic_fields"]["permit_kind"])
|
||||||
|
self.assertFalse(item.mapped_data["attachments"][0]["content_retained"])
|
||||||
|
self.assertEqual("article", item.mapped_data["articles"][0]["reference"]["object_type"])
|
||||||
|
|
||||||
|
replay = synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(idempotency_key="sync-1"),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertEqual(run.id, replay.id)
|
||||||
|
self.assertEqual(1, len(self.transport.change_calls))
|
||||||
|
|
||||||
|
source = ExternalServiceDeskSearchSource()
|
||||||
|
document = source.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=SERVICE_DESK_PROVIDER_ID,
|
||||||
|
resource_type=SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
rebuild_id="rebuild-1",
|
||||||
|
),
|
||||||
|
).documents[0]
|
||||||
|
authorization = SearchAuthorizationRequest(
|
||||||
|
reference=document.reference,
|
||||||
|
source_revision=document.source_revision,
|
||||||
|
)
|
||||||
|
self.assertTrue(source.authorize(self.session, principal(), requests=[authorization])[document.reference.key])
|
||||||
|
self.assertFalse(
|
||||||
|
source.authorize(
|
||||||
|
self.session,
|
||||||
|
principal(groups=frozenset({"other"})),
|
||||||
|
requests=[authorization],
|
||||||
|
)[document.reference.key]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_paged_full_stays_full_then_switches_to_delta(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
first = self._sync([ticket()], key="page-1", complete=False)
|
||||||
|
self.assertEqual("full", json.loads(first.cursor_after or "{}").get("kind"))
|
||||||
|
second = self._sync([ticket(ticket_id="43")], key="page-2", complete=True)
|
||||||
|
self.assertEqual("delta", json.loads(second.cursor_after or "{}").get("kind"))
|
||||||
|
self.assertTrue(self.transport.change_calls[0]["force_full"])
|
||||||
|
self.assertTrue(self.transport.change_calls[1]["force_full"])
|
||||||
|
|
||||||
|
def test_profile_updates_preserve_provider_acl_until_the_next_sync(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket(acl=["group:provider-agents"])], key="provider-acl")
|
||||||
|
profile = self.session.get(ConnectorServiceDeskProfile, self.profile_id)
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskProfileUpdateRequest(
|
||||||
|
expected_resource_revision=profile.resource_revision,
|
||||||
|
default_visibility="tenant",
|
||||||
|
default_acl_tokens=[],
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
stored = self.session.scalar(
|
||||||
|
select(ConnectorServiceDeskObject).where(
|
||||||
|
ConnectorServiceDeskObject.profile_id == self.profile_id,
|
||||||
|
ConnectorServiceDeskObject.external_id == "42",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("restricted", stored.visibility)
|
||||||
|
self.assertEqual(["group:provider-agents"], stored.acl_tokens)
|
||||||
|
self.assertEqual("provider", stored.mapped_data["permission_source"])
|
||||||
|
|
||||||
|
def test_unchanged_editor_payload_does_not_reset_discovery_or_cursor(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket()], key="before-noop-save")
|
||||||
|
profile = self.session.get(ConnectorServiceDeskProfile, self.profile_id)
|
||||||
|
cursor_before = profile.last_sync_cursor
|
||||||
|
discovered_at = profile.discovered_at
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskProfileUpdateRequest(
|
||||||
|
expected_resource_revision=profile.resource_revision,
|
||||||
|
integration_mode=profile.integration_mode,
|
||||||
|
desired_maturity=profile.desired_maturity,
|
||||||
|
source_authority_mode=profile.source_authority_mode,
|
||||||
|
default_visibility=profile.default_visibility,
|
||||||
|
default_acl_tokens=list(profile.default_acl_tokens),
|
||||||
|
routes=ServiceDeskRouteMapping.model_validate(profile.routes),
|
||||||
|
queue_mappings=[
|
||||||
|
ServiceDeskQueueMapping.model_validate(value)
|
||||||
|
for value in profile.queue_mappings
|
||||||
|
],
|
||||||
|
dynamic_field_mappings=[
|
||||||
|
ServiceDeskDynamicFieldMapping.model_validate(value)
|
||||||
|
for value in profile.dynamic_field_mappings
|
||||||
|
],
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertEqual(cursor_before, profile.last_sync_cursor)
|
||||||
|
self.assertEqual(discovered_at, profile.discovered_at)
|
||||||
|
|
||||||
|
def test_route_or_configuration_changes_require_rediscovery(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket()], key="before-route-change")
|
||||||
|
profile = self.session.get(ConnectorServiceDeskProfile, self.profile_id)
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskProfileUpdateRequest(
|
||||||
|
expected_resource_revision=profile.resource_revision,
|
||||||
|
routes=ServiceDeskRouteMapping(
|
||||||
|
search_path="/GovOPlaN/Ticket/Search",
|
||||||
|
ticket_path="/Ticket/{ticket_id}",
|
||||||
|
update_path="/Ticket/{ticket_id}",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertIsNone(profile.discovered_at)
|
||||||
|
self.assertEqual([], profile.capabilities)
|
||||||
|
stored = self.session.scalar(
|
||||||
|
select(ConnectorServiceDeskObject).where(
|
||||||
|
ConnectorServiceDeskObject.profile_id == self.profile_id,
|
||||||
|
ConnectorServiceDeskObject.external_id == "42",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("deleted", stored.status)
|
||||||
|
with self.assertRaisesRegex(ServiceDeskConnectorError, "Discover the provider"):
|
||||||
|
synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(idempotency_key="route-stale"),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket()], key="after-route-rediscovery")
|
||||||
|
self.assertEqual("active", stored.status)
|
||||||
|
configuration = self.session.get(ConnectorConfiguration, self.configuration_id)
|
||||||
|
configuration.resource_revision += 1
|
||||||
|
configuration.effective_hash = "configuration-hash-changed"
|
||||||
|
self.session.commit()
|
||||||
|
self.assertEqual(
|
||||||
|
(),
|
||||||
|
ExternalServiceDeskSearchSource()
|
||||||
|
.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=SERVICE_DESK_PROVIDER_ID,
|
||||||
|
resource_type=SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
rebuild_id="stale-configuration",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.documents,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(ServiceDeskConnectorError, "configuration changed"):
|
||||||
|
synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(idempotency_key="configuration-stale"),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self._discover()
|
||||||
|
self.assertEqual("deleted", stored.status)
|
||||||
|
self.assertIsNone(profile.last_sync_cursor)
|
||||||
|
|
||||||
|
def test_full_sync_reprojects_unchanged_tickets_into_search(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket()], key="initial-search-projection")
|
||||||
|
writer = RecordingSearchWriter()
|
||||||
|
self.transport.batches.append(
|
||||||
|
ServiceDeskChangeBatch(
|
||||||
|
changes=(ticket(),),
|
||||||
|
next_cursor=None,
|
||||||
|
complete=True,
|
||||||
|
high_watermark="2026-08-22T10:00:00Z",
|
||||||
|
live_ids=("42",),
|
||||||
|
evidence={"fixture": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
run = synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(
|
||||||
|
idempotency_key="full-search-reprojection",
|
||||||
|
mode="full",
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=SearchRegistry(writer),
|
||||||
|
)
|
||||||
|
self.assertEqual({"unchanged": 1}, run.counts)
|
||||||
|
self.assertEqual(1, len(writer.upserts))
|
||||||
|
self.assertIsNone(self.transport.change_calls[-1]["cursor"])
|
||||||
|
|
||||||
|
def test_delta_cannot_bootstrap_without_a_completed_full_sync(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
with self.assertRaisesRegex(ServiceDeskConnectorError, "completed full"):
|
||||||
|
synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(
|
||||||
|
idempotency_key="unsafe-delta-bootstrap",
|
||||||
|
mode="delta",
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertEqual([], self.transport.change_calls)
|
||||||
|
|
||||||
|
def test_link_mode_omits_top_level_attachment_metadata(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
profile = self.session.get(ConnectorServiceDeskProfile, self.profile_id)
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskProfileUpdateRequest(
|
||||||
|
expected_resource_revision=profile.resource_revision,
|
||||||
|
integration_mode="link",
|
||||||
|
desired_maturity="link",
|
||||||
|
source_authority_mode="linked_reference",
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
raw = ticket()
|
||||||
|
raw.pop("Article")
|
||||||
|
raw.pop("DynamicField")
|
||||||
|
raw["Attachment"] = [
|
||||||
|
{
|
||||||
|
"AttachmentID": "top-1",
|
||||||
|
"Filename": "metadata-only.pdf",
|
||||||
|
"Filesize": 42,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
run = self._sync([raw], key="link-refresh")
|
||||||
|
objects, _cursor = list_objects(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
)
|
||||||
|
self.assertEqual([], objects[0].mapped_data["attachments"])
|
||||||
|
self.assertIn("link_mode_content_omitted", {item.code for item in run.diagnostics})
|
||||||
|
self.assertTrue(self.transport.change_calls[-1]["routes"]["_identity_only"])
|
||||||
|
|
||||||
|
def test_excluded_queue_removes_projection_and_profile_policy_fails_closed(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket()], key="sync-active")
|
||||||
|
profile = self.session.get(ConnectorServiceDeskProfile, self.profile_id)
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskProfileUpdateRequest(
|
||||||
|
expected_resource_revision=profile.resource_revision,
|
||||||
|
queue_mappings=[
|
||||||
|
ServiceDeskQueueMapping(
|
||||||
|
source_queue="Residents",
|
||||||
|
include=False,
|
||||||
|
visibility="restricted",
|
||||||
|
acl_tokens=["group:agents"],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
stored = self.session.scalar(
|
||||||
|
select(ConnectorServiceDeskObject).where(
|
||||||
|
ConnectorServiceDeskObject.profile_id == self.profile_id,
|
||||||
|
ConnectorServiceDeskObject.external_id == "42",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("deleted", stored.status)
|
||||||
|
self.assertEqual(
|
||||||
|
(),
|
||||||
|
ExternalServiceDeskSearchSource()
|
||||||
|
.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=SERVICE_DESK_PROVIDER_ID,
|
||||||
|
resource_type=SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
rebuild_id="excluded-rebuild",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.documents,
|
||||||
|
)
|
||||||
|
self.transport.batches.append(
|
||||||
|
ServiceDeskChangeBatch(
|
||||||
|
changes=(ticket(revision="2026-08-22T10:30:00Z"),),
|
||||||
|
next_cursor=json.dumps(
|
||||||
|
{"kind": "delta", "changed": "2026-08-22T10:30:00Z", "seen": ["42"]}
|
||||||
|
),
|
||||||
|
complete=True,
|
||||||
|
high_watermark="2026-08-22T10:30:00Z",
|
||||||
|
live_ids=None,
|
||||||
|
evidence={"fixture": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
run = synchronize_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskSyncRequest(idempotency_key="sync-excluded"),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
self.assertEqual({"ignored": 1}, run.counts)
|
||||||
|
self.assertEqual("deleted", stored.status)
|
||||||
|
|
||||||
|
with self.assertRaises(ServiceDeskConnectorError):
|
||||||
|
update_profile(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
payload=ServiceDeskProfileUpdateRequest(
|
||||||
|
expected_resource_revision=self.session.get(
|
||||||
|
ConnectorServiceDeskProfile, self.profile_id
|
||||||
|
).resource_revision,
|
||||||
|
integration_mode="link",
|
||||||
|
desired_maturity="synchronize",
|
||||||
|
source_authority_mode="linked_reference",
|
||||||
|
),
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_governed_update_replay_and_unknown_outcome(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
self._sync([ticket()], key="sync-before-update")
|
||||||
|
payload = ServiceDeskTicketUpdateRequest(
|
||||||
|
idempotency_key="update-1",
|
||||||
|
expected_external_revision="2026-08-22T10:00:00Z",
|
||||||
|
state="pending reminder",
|
||||||
|
)
|
||||||
|
result = update_ticket(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
external_ticket_id="42",
|
||||||
|
payload=payload,
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
durable_recovery=True,
|
||||||
|
)
|
||||||
|
replay = update_ticket(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
external_ticket_id="42",
|
||||||
|
payload=payload,
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
durable_recovery=False,
|
||||||
|
)
|
||||||
|
self.assertTrue(result.accepted)
|
||||||
|
self.assertEqual(result.run.id, replay.run.id)
|
||||||
|
self.assertEqual(1, self.transport.update_calls)
|
||||||
|
recovery = self.session.scalar(
|
||||||
|
select(RecoveryOperation).where(
|
||||||
|
RecoveryOperation.resource_type == SERVICE_DESK_RESOURCE_TYPE,
|
||||||
|
RecoveryOperation.resource_id == "42",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, recovery.status)
|
||||||
|
|
||||||
|
self.transport.update_error = ServiceDeskTransportError(
|
||||||
|
"provider_unavailable",
|
||||||
|
"No conclusive provider response.",
|
||||||
|
retryable=True,
|
||||||
|
outcome_unknown=True,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(ServiceDeskConnectorError, "outcome is unknown"):
|
||||||
|
update_ticket(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
profile_id=self.profile_id,
|
||||||
|
external_ticket_id="42",
|
||||||
|
payload=ServiceDeskTicketUpdateRequest(
|
||||||
|
idempotency_key="update-unknown",
|
||||||
|
expected_external_revision="2026-08-22T11:00:00Z",
|
||||||
|
priority="4 high",
|
||||||
|
),
|
||||||
|
transport=self.transport,
|
||||||
|
registry=None,
|
||||||
|
durable_recovery=False,
|
||||||
|
)
|
||||||
|
unresolved = self.session.scalar(
|
||||||
|
select(ConnectorServiceDeskSyncRun).where(
|
||||||
|
ConnectorServiceDeskSyncRun.idempotency_key == "update-unknown"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("outcome_unknown", unresolved.status)
|
||||||
|
|
||||||
|
def test_tenant_isolation(self) -> None:
|
||||||
|
with self.assertRaisesRegex(ServiceDeskConnectorError, "not found"):
|
||||||
|
list_objects(self.session, principal("tenant-2"), profile_id=self.profile_id)
|
||||||
|
|
||||||
|
def test_malformed_continuous_batch_is_atomic_and_evidenced(self) -> None:
|
||||||
|
self._discover()
|
||||||
|
malformed = ticket()
|
||||||
|
malformed.pop("Changed")
|
||||||
|
with self.assertRaisesRegex(ServiceDeskConnectorError, "change timestamp"):
|
||||||
|
self._sync([malformed], key="sync-malformed")
|
||||||
|
self.assertIsNone(
|
||||||
|
self.session.scalar(
|
||||||
|
select(ConnectorServiceDeskObject).where(
|
||||||
|
ConnectorServiceDeskObject.profile_id == self.profile_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
failed = self.session.scalar(
|
||||||
|
select(ConnectorServiceDeskSyncRun).where(
|
||||||
|
ConnectorServiceDeskSyncRun.idempotency_key == "sync-malformed"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertEqual("failed", failed.status)
|
||||||
|
self.assertEqual("change_timestamp_missing", failed.diagnostics[0]["code"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.security.http_fetch import HttpFetchResponse
|
||||||
|
from govoplan_connectors.backend.service_desk_transport import (
|
||||||
|
HttpServiceDeskTransport,
|
||||||
|
ServiceDeskTransportError,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.service_desk_schemas import ServiceDeskRouteMapping
|
||||||
|
|
||||||
|
|
||||||
|
def response(payload: dict[str, object], *, headers: dict[str, str] | None = None):
|
||||||
|
return HttpFetchResponse(
|
||||||
|
status=200,
|
||||||
|
headers={"Content-Type": "application/json", **(headers or {})},
|
||||||
|
body=json.dumps(payload).encode(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceDeskTransportTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.transport = HttpServiceDeskTransport()
|
||||||
|
self.endpoint = "https://support.example.test/znuny/nph-genericinterface.pl/Webservice/GovOPlaN"
|
||||||
|
self.routes = {
|
||||||
|
"search_path": "/Ticket/Search",
|
||||||
|
"ticket_path": "/Ticket/{ticket_id}",
|
||||||
|
"search_method": "POST",
|
||||||
|
"ticket_method": "GET",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_governed_routes_reject_embedded_authentication_controls(self) -> None:
|
||||||
|
for field, value in (
|
||||||
|
("search_path", "/Ticket/Search?Password=secret"),
|
||||||
|
(
|
||||||
|
"ticket_web_url_template",
|
||||||
|
"https://desk.example.test/ticket/{ticket_id}?SessionID=secret",
|
||||||
|
),
|
||||||
|
):
|
||||||
|
with self.subTest(field=field), self.assertRaisesRegex(
|
||||||
|
ValueError, "authentication controls"
|
||||||
|
):
|
||||||
|
ServiceDeskRouteMapping(**{field: value})
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_discovery_uses_governed_route_header_auth_and_version(self, fetch) -> None:
|
||||||
|
fetch.return_value = response(
|
||||||
|
{"TicketID": []}, headers={"X-Znuny-Version": "7.1.4"}
|
||||||
|
)
|
||||||
|
discovery = self.transport.discover(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential={"user_login": "connector", "password": "secret"},
|
||||||
|
routes={**self.routes, "search_filters": {"QueueIDs": [3, 7]}},
|
||||||
|
)
|
||||||
|
call = fetch.call_args
|
||||||
|
self.assertEqual("POST", call.kwargs["method"])
|
||||||
|
self.assertTrue(call.args[0].endswith("/Ticket/Search"))
|
||||||
|
self.assertEqual("connector", call.kwargs["headers"]["X-OTRS-Header-UserLogin"])
|
||||||
|
self.assertIn(
|
||||||
|
"X-OTRS-Header-Password",
|
||||||
|
call.kwargs["redirect_sensitive_headers"],
|
||||||
|
)
|
||||||
|
self.assertNotIn("secret", call.args[0])
|
||||||
|
self.assertEqual([3, 7], json.loads(call.kwargs["body"])["QueueIDs"])
|
||||||
|
self.assertEqual("znuny", discovery["product"])
|
||||||
|
self.assertEqual("synchronize", discovery["maturity"])
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_get_ticket_flags_are_query_parameters_without_secrets(self, fetch) -> None:
|
||||||
|
fetch.side_effect = (
|
||||||
|
response({"TicketID": ["42"]}),
|
||||||
|
response({"Ticket": [{"TicketID": "42", "Changed": "2026-08-22T10:00:00Z"}]}),
|
||||||
|
)
|
||||||
|
batch = self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential={"user_login": "connector", "password": "secret"},
|
||||||
|
routes=self.routes,
|
||||||
|
cursor=None,
|
||||||
|
limit=100,
|
||||||
|
force_full=True,
|
||||||
|
)
|
||||||
|
query = parse_qs(urlsplit(fetch.call_args_list[1].args[0]).query)
|
||||||
|
self.assertEqual(["1"], query["AllArticles"])
|
||||||
|
self.assertEqual(["0"], query["GetAttachmentContents"])
|
||||||
|
self.assertNotIn("UserLogin", query)
|
||||||
|
self.assertEqual(1, len(batch.changes))
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_link_identity_reads_do_not_request_articles_attachments_or_dynamic_fields(
|
||||||
|
self, fetch
|
||||||
|
) -> None:
|
||||||
|
fetch.side_effect = (
|
||||||
|
response({"TicketID": ["42"]}),
|
||||||
|
response(
|
||||||
|
{"Ticket": [{"TicketID": "42", "Changed": "2026-08-22T10:00:00Z"}]}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes={**self.routes, "_identity_only": True},
|
||||||
|
cursor=None,
|
||||||
|
limit=100,
|
||||||
|
force_full=True,
|
||||||
|
)
|
||||||
|
query = parse_qs(urlsplit(fetch.call_args_list[1].args[0]).query)
|
||||||
|
self.assertEqual(["0"], query["AllArticles"])
|
||||||
|
self.assertEqual(["0"], query["Attachments"])
|
||||||
|
self.assertEqual(["0"], query["DynamicFields"])
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_paged_full_cursor_preserves_cumulative_high_watermark(self, fetch) -> None:
|
||||||
|
fetch.side_effect = (
|
||||||
|
response({"TicketID": ["41", "42"]}),
|
||||||
|
response({"Ticket": [{"TicketID": "41", "Changed": "2026-08-22T12:00:00Z"}]}),
|
||||||
|
response({"TicketID": ["41", "42"]}),
|
||||||
|
response({"Ticket": [{"TicketID": "42", "Changed": "2026-08-22T10:00:00Z"}]}),
|
||||||
|
)
|
||||||
|
first = self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes=self.routes,
|
||||||
|
cursor=None,
|
||||||
|
limit=1,
|
||||||
|
force_full=True,
|
||||||
|
)
|
||||||
|
second = self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes=self.routes,
|
||||||
|
cursor=first.next_cursor,
|
||||||
|
limit=1,
|
||||||
|
force_full=True,
|
||||||
|
)
|
||||||
|
self.assertFalse(first.complete)
|
||||||
|
self.assertTrue(second.complete)
|
||||||
|
self.assertEqual("2026-08-22T12:00:00Z", second.high_watermark)
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_delta_overlaps_and_pages_all_ids_at_one_timestamp(self, fetch) -> None:
|
||||||
|
shared_revision = "2026-08-22T10:00:00Z"
|
||||||
|
fetch.side_effect = (
|
||||||
|
response({"TicketID": ["41", "42"]}),
|
||||||
|
response({"Ticket": [{"TicketID": "41", "Changed": shared_revision}]}),
|
||||||
|
response({"Ticket": [{"TicketID": "42", "Changed": shared_revision}]}),
|
||||||
|
response({"TicketID": ["41", "42"]}),
|
||||||
|
response({"Ticket": [{"TicketID": "41", "Changed": shared_revision}]}),
|
||||||
|
response({"Ticket": [{"TicketID": "42", "Changed": shared_revision}]}),
|
||||||
|
)
|
||||||
|
first = self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes=self.routes,
|
||||||
|
cursor=json.dumps(
|
||||||
|
{"kind": "delta", "changed": "2026-08-22T09:59:59Z", "seen": []}
|
||||||
|
),
|
||||||
|
limit=1,
|
||||||
|
force_full=False,
|
||||||
|
)
|
||||||
|
second = self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes=self.routes,
|
||||||
|
cursor=first.next_cursor,
|
||||||
|
limit=1,
|
||||||
|
force_full=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(["41"], [item["TicketID"] for item in first.changes])
|
||||||
|
self.assertEqual(["42"], [item["TicketID"] for item in second.changes])
|
||||||
|
second_search = json.loads(fetch.call_args_list[3].kwargs["body"])
|
||||||
|
self.assertEqual(
|
||||||
|
"2026-08-22T09:59:59Z",
|
||||||
|
second_search["TicketChangeTimeNewerDate"],
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_delta_does_not_suppress_a_seen_ticket_that_changed_again(self, fetch) -> None:
|
||||||
|
fetch.side_effect = (
|
||||||
|
response({"TicketID": ["41"]}),
|
||||||
|
response(
|
||||||
|
{
|
||||||
|
"Ticket": [
|
||||||
|
{"TicketID": "41", "Changed": "2026-08-22T10:05:00Z"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
batch = self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes=self.routes,
|
||||||
|
cursor=json.dumps(
|
||||||
|
{
|
||||||
|
"kind": "delta",
|
||||||
|
"changed": "2026-08-22T10:00:00Z",
|
||||||
|
"seen": ["41"],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
limit=10,
|
||||||
|
force_full=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(["41"], [item["TicketID"] for item in batch.changes])
|
||||||
|
self.assertEqual("2026-08-22T10:05:00Z", batch.high_watermark)
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_body_authentication_is_never_put_in_get_url(self, fetch) -> None:
|
||||||
|
fetch.return_value = response({"TicketID": ["42"]})
|
||||||
|
with self.assertRaisesRegex(ServiceDeskTransportError, "cannot be used with a GET"):
|
||||||
|
self.transport.changes(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential={
|
||||||
|
"auth_mode": "body",
|
||||||
|
"user_login": "connector",
|
||||||
|
"password": "secret",
|
||||||
|
},
|
||||||
|
routes={**self.routes, "search_method": "GET"},
|
||||||
|
cursor=None,
|
||||||
|
limit=1,
|
||||||
|
force_full=True,
|
||||||
|
)
|
||||||
|
fetch.assert_not_called()
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_legacy_body_authentication_is_post_only_and_not_duplicated(self, fetch) -> None:
|
||||||
|
fetch.return_value = response({"TicketID": []})
|
||||||
|
self.transport.discover(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential={
|
||||||
|
"auth_mode": "body",
|
||||||
|
"user_login": "connector",
|
||||||
|
"password": "secret",
|
||||||
|
},
|
||||||
|
routes=self.routes,
|
||||||
|
)
|
||||||
|
call = fetch.call_args
|
||||||
|
payload = json.loads(call.kwargs["body"])
|
||||||
|
self.assertEqual("connector", payload["UserLogin"])
|
||||||
|
self.assertEqual("secret", payload["Password"])
|
||||||
|
self.assertNotIn("X-OTRS-Header-UserLogin", call.kwargs["headers"])
|
||||||
|
self.assertNotIn("secret", call.args[0])
|
||||||
|
|
||||||
|
@patch("govoplan_connectors.backend.service_desk_transport.fetch_http")
|
||||||
|
def test_update_requires_revision_and_requested_field_verification(self, fetch) -> None:
|
||||||
|
fetch.side_effect = (
|
||||||
|
response(
|
||||||
|
{
|
||||||
|
"Ticket": [
|
||||||
|
{
|
||||||
|
"TicketID": "42",
|
||||||
|
"State": "open",
|
||||||
|
"Changed": "2026-08-22T10:00:00Z",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
response({"Success": 1}),
|
||||||
|
response(
|
||||||
|
{
|
||||||
|
"Ticket": [
|
||||||
|
{
|
||||||
|
"TicketID": "42",
|
||||||
|
"State": "open",
|
||||||
|
"Changed": "2026-08-22T10:05:00Z",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with self.assertRaises(ServiceDeskTransportError) as raised:
|
||||||
|
self.transport.update_ticket(
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
credential=None,
|
||||||
|
routes={**self.routes, "update_path": "/Ticket/{ticket_id}"},
|
||||||
|
ticket_id="42",
|
||||||
|
expected_revision="2026-08-22T10:00:00Z",
|
||||||
|
changes={"State": "pending reminder"},
|
||||||
|
)
|
||||||
|
self.assertTrue(raised.exception.outcome_unknown)
|
||||||
|
self.assertEqual("update_verification_failed", raised.exception.code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from openpyxl import Workbook
|
||||||
|
from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.files import (
|
||||||
|
CAPABILITY_FILES_TABULAR_CONTENT,
|
||||||
|
ManagedTabularFile,
|
||||||
|
ManagedTabularFileContent,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.tabular_adapters import (
|
||||||
|
ManagedFileTabularAdapter,
|
||||||
|
PostgresqlTabularAdapter,
|
||||||
|
parse_managed_tabular_content,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(tenant_id: str = "tenant-1") -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(
|
||||||
|
{
|
||||||
|
"connectors:source:read",
|
||||||
|
"connectors:source:write",
|
||||||
|
"files:file:read",
|
||||||
|
"files:file:download",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _ManagedFiles:
|
||||||
|
def __init__(self, payload: bytes, *, filename: str = "cases.csv") -> None:
|
||||||
|
self.payload = payload
|
||||||
|
self.filename = filename
|
||||||
|
self.current_version_id = "version-2"
|
||||||
|
|
||||||
|
def _file(self, version_id: str) -> ManagedTabularFile:
|
||||||
|
return ManagedTabularFile(
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
file_version_id=version_id,
|
||||||
|
filename=self.filename,
|
||||||
|
display_path=f"Imports/{self.filename}",
|
||||||
|
content_type=(
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
if self.filename.endswith(".xlsx")
|
||||||
|
else "text/csv"
|
||||||
|
),
|
||||||
|
size_bytes=len(self.payload),
|
||||||
|
sha256=("a" if version_id == "version-1" else "b") * 64,
|
||||||
|
current_version=version_id == self.current_version_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def list_tabular_files(self, session, principal, *, query="", limit=100):
|
||||||
|
del session, principal, query, limit
|
||||||
|
return (self._file(self.current_version_id),)
|
||||||
|
|
||||||
|
def get_tabular_file(
|
||||||
|
self,
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
*,
|
||||||
|
file_asset_id,
|
||||||
|
file_version_id=None,
|
||||||
|
):
|
||||||
|
del session, principal
|
||||||
|
if file_asset_id != "asset-1":
|
||||||
|
return None
|
||||||
|
return self._file(file_version_id or self.current_version_id)
|
||||||
|
|
||||||
|
def read_tabular_file(
|
||||||
|
self,
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
*,
|
||||||
|
file_asset_id,
|
||||||
|
file_version_id,
|
||||||
|
max_bytes,
|
||||||
|
):
|
||||||
|
del session, principal, file_asset_id
|
||||||
|
if len(self.payload) > max_bytes:
|
||||||
|
raise AssertionError("test payload exceeded adapter limit")
|
||||||
|
return ManagedTabularFileContent(
|
||||||
|
file=self._file(file_version_id),
|
||||||
|
payload=self.payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, provider) -> None:
|
||||||
|
self.provider = provider
|
||||||
|
|
||||||
|
def has_capability(self, name):
|
||||||
|
return name == CAPABILITY_FILES_TABULAR_CONTENT
|
||||||
|
|
||||||
|
def require_capability(self, name):
|
||||||
|
if not self.has_capability(name):
|
||||||
|
raise KeyError(name)
|
||||||
|
return self.provider
|
||||||
|
|
||||||
|
|
||||||
|
class ManagedFileTabularAdapterTests(unittest.TestCase):
|
||||||
|
def test_csv_is_exact_version_pinned_and_reports_newer_version(self) -> None:
|
||||||
|
adapter = ManagedFileTabularAdapter(
|
||||||
|
_Registry(_ManagedFiles(b"id,amount\n0012,12.5\n2,7\n"))
|
||||||
|
)
|
||||||
|
|
||||||
|
result = adapter.inspect(
|
||||||
|
object(),
|
||||||
|
principal(),
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
file_version_id="version-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("managed_file", result.provider)
|
||||||
|
self.assertEqual("version-1", result.metadata["file_version_id"])
|
||||||
|
self.assertEqual("warning", result.health.status)
|
||||||
|
self.assertEqual("files.newer_version_available", result.health.code)
|
||||||
|
self.assertEqual("mixed", result.schema[0].data_type)
|
||||||
|
self.assertEqual(2, result.row_count)
|
||||||
|
self.assertTrue(result.pushdown.projections)
|
||||||
|
self.assertEqual("files.newer_version_available", result.diagnostics[0].code)
|
||||||
|
|
||||||
|
def test_xlsx_uses_requested_sheet_and_closed_typed_schema(self) -> None:
|
||||||
|
workbook = Workbook()
|
||||||
|
first = workbook.active
|
||||||
|
first.title = "Ignore"
|
||||||
|
first.append(["ignored"])
|
||||||
|
target = workbook.create_sheet("Monthly")
|
||||||
|
target.append(["case_id", "amount", "active"])
|
||||||
|
target.append(["A-1", 12.5, True])
|
||||||
|
target.append(["A-2", None, False])
|
||||||
|
payload = BytesIO()
|
||||||
|
workbook.save(payload)
|
||||||
|
workbook.close()
|
||||||
|
|
||||||
|
rows, sheet = parse_managed_tabular_content(
|
||||||
|
payload.getvalue(),
|
||||||
|
filename="monthly.xlsx",
|
||||||
|
content_type=(
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
),
|
||||||
|
delimiter=",",
|
||||||
|
sheet_name="Monthly",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("Monthly", sheet)
|
||||||
|
self.assertEqual("A-1", rows[0]["case_id"])
|
||||||
|
self.assertIsNone(rows[1]["amount"])
|
||||||
|
|
||||||
|
def test_missing_files_capability_is_explicitly_unavailable(self) -> None:
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
"require the Files module",
|
||||||
|
):
|
||||||
|
ManagedFileTabularAdapter(None).inspect(
|
||||||
|
object(),
|
||||||
|
principal(),
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
file_version_id=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PostgresqlTabularAdapterTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.directory = tempfile.TemporaryDirectory(
|
||||||
|
prefix="govoplan-connectors-sql-adapter-"
|
||||||
|
)
|
||||||
|
source_path = Path(self.directory.name) / "source.db"
|
||||||
|
self.source_url = f"sqlite+pysqlite:///{source_path}"
|
||||||
|
source_engine = create_engine(self.source_url)
|
||||||
|
metadata = MetaData()
|
||||||
|
self.table = Table(
|
||||||
|
"monthly_cases",
|
||||||
|
metadata,
|
||||||
|
Column("case_id", String, nullable=False),
|
||||||
|
Column("amount", Integer, nullable=True),
|
||||||
|
)
|
||||||
|
metadata.create_all(source_engine)
|
||||||
|
with source_engine.begin() as connection:
|
||||||
|
connection.execute(
|
||||||
|
self.table.insert(),
|
||||||
|
(
|
||||||
|
{"case_id": "A-1", "amount": 12},
|
||||||
|
{"case_id": "A-2", "amount": None},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
source_engine.dispose()
|
||||||
|
|
||||||
|
self.catalog_engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.catalog_engine,
|
||||||
|
tables=(
|
||||||
|
ConnectorDefinition.__table__,
|
||||||
|
ConnectorConfiguration.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.catalog_engine)()
|
||||||
|
definition = ConnectorDefinition(
|
||||||
|
id="definition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_key="postgresql.reader",
|
||||||
|
name="PostgreSQL reader",
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
local_definition=True,
|
||||||
|
)
|
||||||
|
self.configuration = ConnectorConfiguration(
|
||||||
|
id="configuration-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id=definition.id,
|
||||||
|
name="Monthly SQL",
|
||||||
|
status="active",
|
||||||
|
endpoint_url=self.source_url,
|
||||||
|
credential_ref=None,
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={},
|
||||||
|
protected_paths=[],
|
||||||
|
effective_configuration={"provider": "sql", "protocol": "sql"},
|
||||||
|
effective_hash="configuration-hash-1",
|
||||||
|
resource_revision=1,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
)
|
||||||
|
self.session.add_all((definition, self.configuration))
|
||||||
|
self.session.commit()
|
||||||
|
self.adapter = PostgresqlTabularAdapter(allow_sqlite_for_tests=True)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.catalog_engine.dispose()
|
||||||
|
self.directory.cleanup()
|
||||||
|
|
||||||
|
def test_discovers_and_reads_projection_from_governed_sql_configuration(self) -> None:
|
||||||
|
inspection = self.adapter.inspect(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
metadata = {
|
||||||
|
**dict(inspection.metadata),
|
||||||
|
"discovery_fingerprint": inspection.fingerprint,
|
||||||
|
}
|
||||||
|
read = self.adapter.read(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
metadata=metadata,
|
||||||
|
columns=("case_id",),
|
||||||
|
offset=1,
|
||||||
|
limit=10,
|
||||||
|
timeout_ms=2_000,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("live", "live")
|
||||||
|
self.assertEqual(["case_id", "amount"], [item.name for item in inspection.schema])
|
||||||
|
self.assertEqual(2, inspection.row_count)
|
||||||
|
self.assertEqual(({"case_id": "A-2"},), read.rows)
|
||||||
|
self.assertTrue(inspection.pushdown.projections)
|
||||||
|
self.assertFalse(inspection.pushdown.filters)
|
||||||
|
|
||||||
|
def test_schema_drift_and_tenant_isolation_fail_closed(self) -> None:
|
||||||
|
inspection = self.adapter.inspect(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
metadata = {
|
||||||
|
**dict(inspection.metadata),
|
||||||
|
"discovery_fingerprint": inspection.fingerprint,
|
||||||
|
}
|
||||||
|
engine = create_engine(self.source_url)
|
||||||
|
with engine.begin() as connection:
|
||||||
|
connection.exec_driver_sql(
|
||||||
|
"ALTER TABLE monthly_cases ADD COLUMN category TEXT"
|
||||||
|
)
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(TabularSourceValidationError, "schema drifted"):
|
||||||
|
self.adapter.read(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
metadata=metadata,
|
||||||
|
columns=(),
|
||||||
|
offset=0,
|
||||||
|
limit=10,
|
||||||
|
timeout_ms=2_000,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
"configuration is unavailable",
|
||||||
|
):
|
||||||
|
self.adapter.inspect(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_endpoint_query_credentials_are_rejected_before_connection(self) -> None:
|
||||||
|
self.configuration.endpoint_url = f"{self.source_url}?password=not-allowed"
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceValidationError,
|
||||||
|
"query parameters must not contain credentials",
|
||||||
|
):
|
||||||
|
self.adapter.inspect(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.files import (
|
||||||
|
CAPABILITY_FILES_TABULAR_CONTENT,
|
||||||
|
ManagedTabularFile,
|
||||||
|
ManagedTabularFileContent,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.security.credential_envelopes import CredentialEnvelope
|
||||||
|
from govoplan_connectors.backend.db.models import (
|
||||||
|
ConnectorConfiguration,
|
||||||
|
ConnectorDefinition,
|
||||||
|
ConnectorTabularSource,
|
||||||
|
)
|
||||||
|
from govoplan_connectors.backend.tabular_adapters import PostgresqlTabularAdapter
|
||||||
|
from govoplan_connectors.backend.tabular_sources import SqlTabularSourceProvider
|
||||||
|
|
||||||
|
|
||||||
|
def principal(tenant_id: str = "tenant-1") -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(
|
||||||
|
{
|
||||||
|
"connectors:source:read",
|
||||||
|
"connectors:source:write",
|
||||||
|
"files:file:read",
|
||||||
|
"files:file:download",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _ManagedFiles:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.current = "version-1"
|
||||||
|
self.payloads = {
|
||||||
|
"version-1": b"id,name\n1,Ada\n2,Lin\n",
|
||||||
|
"version-2": b"id,name,active\n1,Ada,true\n2,Lin,false\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _metadata(self, version_id: str) -> ManagedTabularFile:
|
||||||
|
payload = self.payloads[version_id]
|
||||||
|
return ManagedTabularFile(
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
file_version_id=version_id,
|
||||||
|
filename="people.csv",
|
||||||
|
display_path="Imports/people.csv",
|
||||||
|
content_type="text/csv",
|
||||||
|
size_bytes=len(payload),
|
||||||
|
sha256=("a" if version_id == "version-1" else "b") * 64,
|
||||||
|
current_version=version_id == self.current,
|
||||||
|
)
|
||||||
|
|
||||||
|
def list_tabular_files(self, session, principal, *, query="", limit=100):
|
||||||
|
del session, principal, query, limit
|
||||||
|
return (self._metadata(self.current),)
|
||||||
|
|
||||||
|
def get_tabular_file(
|
||||||
|
self,
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
*,
|
||||||
|
file_asset_id,
|
||||||
|
file_version_id=None,
|
||||||
|
):
|
||||||
|
del session, principal
|
||||||
|
if file_asset_id != "asset-1":
|
||||||
|
return None
|
||||||
|
return self._metadata(file_version_id or self.current)
|
||||||
|
|
||||||
|
def read_tabular_file(
|
||||||
|
self,
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
*,
|
||||||
|
file_asset_id,
|
||||||
|
file_version_id,
|
||||||
|
max_bytes,
|
||||||
|
):
|
||||||
|
del session, principal, file_asset_id, max_bytes
|
||||||
|
return ManagedTabularFileContent(
|
||||||
|
file=self._metadata(file_version_id),
|
||||||
|
payload=self.payloads[file_version_id],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, files) -> None:
|
||||||
|
self.files = files
|
||||||
|
|
||||||
|
def has_capability(self, name):
|
||||||
|
return name == CAPABILITY_FILES_TABULAR_CONTENT
|
||||||
|
|
||||||
|
def require_capability(self, name):
|
||||||
|
if not self.has_capability(name):
|
||||||
|
raise KeyError(name)
|
||||||
|
return self.files
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorTabularOriginProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.catalog_engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.catalog_engine,
|
||||||
|
tables=(
|
||||||
|
ConnectorTabularSource.__table__,
|
||||||
|
ConnectorDefinition.__table__,
|
||||||
|
ConnectorConfiguration.__table__,
|
||||||
|
CredentialEnvelope.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.catalog_engine)()
|
||||||
|
self.files = _ManagedFiles()
|
||||||
|
self.directory = tempfile.TemporaryDirectory(
|
||||||
|
prefix="govoplan-connectors-origin-provider-"
|
||||||
|
)
|
||||||
|
source_path = Path(self.directory.name) / "source.db"
|
||||||
|
self.sql_url = f"sqlite+pysqlite:///{source_path}"
|
||||||
|
source_engine = create_engine(self.sql_url)
|
||||||
|
metadata = MetaData()
|
||||||
|
source_table = Table(
|
||||||
|
"monthly_cases",
|
||||||
|
metadata,
|
||||||
|
Column("case_id", String, nullable=False),
|
||||||
|
Column("amount", Integer, nullable=True),
|
||||||
|
)
|
||||||
|
metadata.create_all(source_engine)
|
||||||
|
with source_engine.begin() as connection:
|
||||||
|
connection.execute(
|
||||||
|
source_table.insert(),
|
||||||
|
(
|
||||||
|
{"case_id": "A-1", "amount": 12},
|
||||||
|
{"case_id": "A-2", "amount": None},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
source_engine.dispose()
|
||||||
|
definition = ConnectorDefinition(
|
||||||
|
id="definition-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_key="postgresql.reader",
|
||||||
|
name="PostgreSQL reader",
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
local_definition=True,
|
||||||
|
)
|
||||||
|
self.configuration = ConnectorConfiguration(
|
||||||
|
id="configuration-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
definition_id=definition.id,
|
||||||
|
name="Monthly SQL",
|
||||||
|
status="active",
|
||||||
|
endpoint_url=self.sql_url,
|
||||||
|
credential_ref=None,
|
||||||
|
base_definition_revision=1,
|
||||||
|
local_overrides={},
|
||||||
|
protected_paths=[],
|
||||||
|
effective_configuration={"provider": "sql", "protocol": "sql"},
|
||||||
|
effective_hash="configuration-hash-1",
|
||||||
|
resource_revision=1,
|
||||||
|
ambiguity_policy="manual_review",
|
||||||
|
)
|
||||||
|
self.session.add_all((definition, self.configuration))
|
||||||
|
self.session.commit()
|
||||||
|
self.provider = SqlTabularSourceProvider(
|
||||||
|
registry=_Registry(self.files),
|
||||||
|
sql_adapter=PostgresqlTabularAdapter(allow_sqlite_for_tests=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.catalog_engine.dispose()
|
||||||
|
self.directory.cleanup()
|
||||||
|
|
||||||
|
def test_managed_file_source_stays_pinned_until_explicit_refresh(self) -> None:
|
||||||
|
created = self.provider.create_file_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
name="Managed people",
|
||||||
|
source_name="managed_people",
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.files.current = "version-2"
|
||||||
|
|
||||||
|
preview = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(source_ref=created.ref, limit=10),
|
||||||
|
)
|
||||||
|
refreshed = self.provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
source_ref=created.ref,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(created.ref.startswith("file:"))
|
||||||
|
self.assertEqual("file_backed", preview.source.source_mode)
|
||||||
|
self.assertEqual("version-1", preview.source.metadata["file_version_id"])
|
||||||
|
self.assertEqual(
|
||||||
|
"files.newer_version_available",
|
||||||
|
preview.diagnostics[0].code,
|
||||||
|
)
|
||||||
|
self.assertEqual("version-2", refreshed.metadata["file_version_id"])
|
||||||
|
self.assertEqual("2", refreshed.schema_version)
|
||||||
|
self.assertEqual(3, len(refreshed.schema))
|
||||||
|
|
||||||
|
def test_sql_source_projects_and_blocks_changed_configuration_until_refresh(self) -> None:
|
||||||
|
created = self.provider.create_sql_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
name="Monthly cases",
|
||||||
|
source_name="monthly_cases",
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
preview = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
columns=("case_id",),
|
||||||
|
limit=1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(created.ref.startswith("sql:"))
|
||||||
|
self.assertEqual("live", preview.source.source_mode)
|
||||||
|
self.assertEqual(({"case_id": "A-1"},), preview.rows)
|
||||||
|
self.assertEqual("preview.row_limit_reached", preview.diagnostics[-1].code)
|
||||||
|
self.assertIsNone(
|
||||||
|
self.provider.get_source(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
source_ref=created.ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self.configuration.effective_hash = "configuration-hash-2"
|
||||||
|
self.configuration.resource_revision = 2
|
||||||
|
self.session.commit()
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceValidationError,
|
||||||
|
"configuration changed",
|
||||||
|
):
|
||||||
|
self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(source_ref=created.ref),
|
||||||
|
)
|
||||||
|
refreshed = self.provider.refresh_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
source_ref=created.ref,
|
||||||
|
)
|
||||||
|
self.assertEqual("2", refreshed.schema_version)
|
||||||
|
self.assertEqual("configuration-hash-2", refreshed.metadata["configuration_hash"])
|
||||||
|
|
||||||
|
def test_inactive_or_stale_sql_credentials_have_sanitized_diagnostics(self) -> None:
|
||||||
|
self.configuration.endpoint_url = (
|
||||||
|
"postgresql+psycopg://db.example.invalid/govoplan"
|
||||||
|
)
|
||||||
|
self.configuration.credential_ref = "missing-credential"
|
||||||
|
self.session.commit()
|
||||||
|
adapter = PostgresqlTabularAdapter()
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
"credential is unavailable, inactive, or outside its allowed scope",
|
||||||
|
):
|
||||||
|
adapter.inspect(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
self.configuration.status = "disabled"
|
||||||
|
self.session.commit()
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
"configuration is not active",
|
||||||
|
):
|
||||||
|
adapter.inspect(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
configuration_id=self.configuration.id,
|
||||||
|
table_name="monthly_cases",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,268 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.tabular_sources import (
|
||||||
|
TabularReadRequest,
|
||||||
|
TabularSnapshotInput,
|
||||||
|
TabularSourceAccessError,
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
TabularSourceValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
||||||
|
from govoplan_connectors.backend.router import api_create_tabular_snapshot
|
||||||
|
from govoplan_connectors.backend.schemas import SnapshotCreateRequest
|
||||||
|
from govoplan_connectors.backend.tabular_sources import (
|
||||||
|
READ_SCOPE,
|
||||||
|
WRITE_SCOPE,
|
||||||
|
SqlTabularSourceProvider,
|
||||||
|
parse_csv_snapshot,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
scopes: tuple[str, ...] = (READ_SCOPE, WRITE_SCOPE),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=object(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorsTabularSourceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine, tables=[ConnectorTabularSource.__table__])
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
self.session = self.Session()
|
||||||
|
self.provider = SqlTabularSourceProvider()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
Base.metadata.drop_all(self.engine, tables=[ConnectorTabularSource.__table__])
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_snapshot_round_trip_preserves_schema_fingerprint_and_bounds(self) -> None:
|
||||||
|
created = self.provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Monthly cases",
|
||||||
|
source_name="monthly_cases_2026_07",
|
||||||
|
rows=(
|
||||||
|
{"case_id": "A-1", "amount": 12, "active": True},
|
||||||
|
{"case_id": "A-2", "amount": None, "active": False},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
listed = self.provider.list_sources(self.session, principal())
|
||||||
|
preview = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
limit=1,
|
||||||
|
expected_fingerprint=created.fingerprint,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((created.ref,), tuple(source.ref for source in listed))
|
||||||
|
self.assertEqual(
|
||||||
|
["case_id", "amount", "active"],
|
||||||
|
[column.name for column in created.schema],
|
||||||
|
)
|
||||||
|
self.assertEqual(2, preview.total_rows)
|
||||||
|
self.assertEqual(1, len(preview.rows))
|
||||||
|
self.assertTrue(preview.truncated)
|
||||||
|
self.assertEqual(created.fingerprint, preview.source.fingerprint)
|
||||||
|
self.assertEqual("cached", preview.source.source_mode)
|
||||||
|
self.assertTrue(preview.source.pushdown.projections)
|
||||||
|
self.assertTrue(preview.source.pushdown.pagination)
|
||||||
|
self.assertEqual("healthy", preview.source.health.status)
|
||||||
|
self.assertGreater(preview.returned_bytes, 2)
|
||||||
|
self.assertEqual(1, preview.effective_row_limit)
|
||||||
|
self.assertEqual("preview.row_limit_reached", preview.diagnostics[0].code)
|
||||||
|
|
||||||
|
def test_preview_enforces_byte_time_and_provider_ceiling_budgets(self) -> None:
|
||||||
|
created = self.provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Bounded",
|
||||||
|
source_name="bounded",
|
||||||
|
rows=(
|
||||||
|
{"id": 1, "value": "first"},
|
||||||
|
{"id": 2, "value": "second"},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
bounded = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
limit=500,
|
||||||
|
max_bytes=35,
|
||||||
|
timeout_ms=2_000,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(bounded.rows))
|
||||||
|
self.assertTrue(bounded.truncated)
|
||||||
|
self.assertEqual(
|
||||||
|
"preview.byte_limit_reached",
|
||||||
|
bounded.diagnostics[-1].code,
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceValidationError,
|
||||||
|
"single source row exceeds",
|
||||||
|
):
|
||||||
|
self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
max_bytes=2,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
tightened = self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
limit=5_000,
|
||||||
|
max_bytes=5_000_000,
|
||||||
|
timeout_ms=10_000,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(500, tightened.effective_row_limit)
|
||||||
|
self.assertEqual(1_000_000, tightened.effective_byte_limit)
|
||||||
|
self.assertEqual(2_000, tightened.effective_timeout_ms)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
"preview.row_limit_tightened",
|
||||||
|
"preview.byte_limit_tightened",
|
||||||
|
"preview.timeout_tightened",
|
||||||
|
},
|
||||||
|
{item.code for item in tightened.diagnostics},
|
||||||
|
)
|
||||||
|
|
||||||
|
times = iter((0.0, 0.01))
|
||||||
|
timeout_provider = SqlTabularSourceProvider(clock=lambda: next(times))
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
TabularSourceUnavailableError,
|
||||||
|
"time budget",
|
||||||
|
):
|
||||||
|
timeout_provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
timeout_ms=1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_tenant_and_scope_isolation_are_enforced(self) -> None:
|
||||||
|
created = self.provider.create_snapshot(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
snapshot=TabularSnapshotInput(
|
||||||
|
name="Private",
|
||||||
|
source_name="private_source",
|
||||||
|
rows=({"id": 1},),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertEqual((), self.provider.list_sources(self.session, principal("tenant-2")))
|
||||||
|
self.assertIsNone(
|
||||||
|
self.provider.get_source(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
source_ref=created.ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with self.assertRaises(TabularSourceAccessError):
|
||||||
|
self.provider.list_sources(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_duplicate_source_name_and_stale_fingerprint_are_rejected(self) -> None:
|
||||||
|
snapshot = TabularSnapshotInput(
|
||||||
|
name="Cases",
|
||||||
|
source_name="cases",
|
||||||
|
rows=({"id": 1},),
|
||||||
|
)
|
||||||
|
created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
self.provider.read_source(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
request=TabularReadRequest(
|
||||||
|
source_ref=created.ref,
|
||||||
|
expected_fingerprint="stale",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_csv_parser_infers_scalar_values_and_rejects_duplicate_headers(self) -> None:
|
||||||
|
rows = parse_csv_snapshot(
|
||||||
|
"\ufeffid;amount;active;note\n0012;12.5;true;\n2;7;false;ok\n",
|
||||||
|
delimiter=";",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
(
|
||||||
|
{"id": "0012", "amount": 12.5, "active": True, "note": None},
|
||||||
|
{"id": 2, "amount": 7, "active": False, "note": "ok"},
|
||||||
|
),
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
parse_csv_snapshot("id,id\n1,2\n", delimiter=",")
|
||||||
|
with self.assertRaises(TabularSourceValidationError):
|
||||||
|
parse_csv_snapshot("id,name\n1,Ada,extra\n", delimiter=",")
|
||||||
|
|
||||||
|
def test_malformed_csv_api_request_is_reported_as_validation_error(self) -> None:
|
||||||
|
payload = SnapshotCreateRequest(
|
||||||
|
name="Malformed",
|
||||||
|
source_name="malformed",
|
||||||
|
format="csv",
|
||||||
|
csv_text="id,name\n1,Ada,extra\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaises(HTTPException) as raised:
|
||||||
|
api_create_tabular_snapshot(
|
||||||
|
payload,
|
||||||
|
session=self.session,
|
||||||
|
principal=principal(),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(422, raised.exception.status_code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"name": "@govoplan/connectors-webui",
|
||||||
|
"version": "0.1.22",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"main": "src/index.ts",
|
||||||
|
"module": "src/index.ts",
|
||||||
|
"types": "src/index.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"types": "./src/index.ts",
|
||||||
|
"import": "./src/index.ts"
|
||||||
|
},
|
||||||
|
"./styles/connectors.css": "./src/styles/connectors.css"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"@govoplan/core-webui": "^0.1.18",
|
||||||
|
"react": ">=19.2.7 <20",
|
||||||
|
"react-dom": ">=19.2.7 <20"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"@govoplan/core-webui": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"test:connector-governance-ui": "node tests/connector-governance-ui-structure.test.mjs"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
import { apiFetch, apiPath, type ApiSettings } from "@govoplan/core-webui";
|
||||||
|
|
||||||
|
export type KnowledgeDiagnostic = {
|
||||||
|
severity: "info" | "warning" | "error";
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
object_ref?: string | null;
|
||||||
|
field?: string | null;
|
||||||
|
retryable: boolean;
|
||||||
|
details: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgeNamespaceMapping = {
|
||||||
|
source_namespace_id: number;
|
||||||
|
source_name: string;
|
||||||
|
target_space_ref: string;
|
||||||
|
target_path_prefix: string;
|
||||||
|
include: boolean;
|
||||||
|
visibility?: "tenant" | "restricted" | null;
|
||||||
|
acl_tokens: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgeProfile = {
|
||||||
|
id: string;
|
||||||
|
tenant_id: string;
|
||||||
|
configuration_id: string;
|
||||||
|
status: "active" | "paused";
|
||||||
|
product: string;
|
||||||
|
product_version?: string | null;
|
||||||
|
desired_maturity: "discover" | "link" | "search" | "read" | "publish" | "synchronize" | "migrate";
|
||||||
|
discovered_maturity: string;
|
||||||
|
source_authority_mode: "external_authoritative" | "external_mirror" | "governed_sync" | "linked_reference";
|
||||||
|
default_visibility: "tenant" | "restricted";
|
||||||
|
default_acl_tokens: string[];
|
||||||
|
namespace_mappings: KnowledgeNamespaceMapping[];
|
||||||
|
capabilities: string[];
|
||||||
|
discovery_revision?: string | null;
|
||||||
|
health_status: string;
|
||||||
|
health_details: Record<string, unknown>;
|
||||||
|
discovered_at?: string | null;
|
||||||
|
last_sync_cursor?: string | null;
|
||||||
|
last_high_watermark?: string | null;
|
||||||
|
resource_revision: number;
|
||||||
|
credential_reference_present: boolean;
|
||||||
|
endpoint_configured: boolean;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgeObject = {
|
||||||
|
id: string;
|
||||||
|
profile_id: string;
|
||||||
|
object_type: string;
|
||||||
|
external_id: string;
|
||||||
|
external_page_id?: string | null;
|
||||||
|
external_revision_id?: string | null;
|
||||||
|
namespace_id?: number | null;
|
||||||
|
title: string;
|
||||||
|
canonical_url?: string | null;
|
||||||
|
status: string;
|
||||||
|
source_revision: string;
|
||||||
|
visibility: string;
|
||||||
|
acl_tokens: string[];
|
||||||
|
mapped_data: Record<string, unknown>;
|
||||||
|
observed_at: string;
|
||||||
|
resource_revision: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgeRun = {
|
||||||
|
id: string;
|
||||||
|
profile_id: string;
|
||||||
|
mode: string;
|
||||||
|
idempotency_key: string;
|
||||||
|
status: string;
|
||||||
|
cursor_before?: string | null;
|
||||||
|
cursor_after?: string | null;
|
||||||
|
high_watermark?: string | null;
|
||||||
|
counts: Record<string, number>;
|
||||||
|
effects: Array<Record<string, unknown>>;
|
||||||
|
diagnostics: KnowledgeDiagnostic[];
|
||||||
|
provenance: Record<string, unknown>;
|
||||||
|
started_at: string;
|
||||||
|
finished_at?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgeDiscovery = {
|
||||||
|
profile: KnowledgeProfile;
|
||||||
|
product: string;
|
||||||
|
product_version?: string | null;
|
||||||
|
capabilities: string[];
|
||||||
|
namespaces: Array<Record<string, unknown>>;
|
||||||
|
extensions: Array<Record<string, unknown>>;
|
||||||
|
maturity: string;
|
||||||
|
health_status: string;
|
||||||
|
diagnostics: KnowledgeDiagnostic[];
|
||||||
|
revision: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgeMigrationPreview = {
|
||||||
|
run: KnowledgeRun;
|
||||||
|
target_space_ref: string;
|
||||||
|
source_revision: string;
|
||||||
|
source_fingerprint: string;
|
||||||
|
summary: Record<string, number>;
|
||||||
|
effects: Array<Record<string, unknown>>;
|
||||||
|
diagnostics: KnowledgeDiagnostic[];
|
||||||
|
truncated: boolean;
|
||||||
|
can_apply: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type KnowledgePublishResult = {
|
||||||
|
run: KnowledgeRun;
|
||||||
|
external_reference: Record<string, unknown>;
|
||||||
|
accepted: boolean;
|
||||||
|
outcome_unknown: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ROOT = "/api/v1/connectors/knowledge";
|
||||||
|
|
||||||
|
export async function listKnowledgeProfiles(settings: ApiSettings): Promise<KnowledgeProfile[]> {
|
||||||
|
const response = await apiFetch<{ items: KnowledgeProfile[] }>(settings, `${ROOT}/profiles`);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createKnowledgeProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<KnowledgeProfile> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function updateKnowledgeProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<KnowledgeProfile> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}`, {
|
||||||
|
method: "PUT",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function discoverKnowledgeProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string
|
||||||
|
): Promise<KnowledgeDiscovery> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}/discover`, {
|
||||||
|
method: "POST"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function synchronizeKnowledgeProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<KnowledgeRun> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}/sync`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listKnowledgeObjects(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string
|
||||||
|
): Promise<KnowledgeObject[]> {
|
||||||
|
const response = await apiFetch<{ items: KnowledgeObject[] }>(
|
||||||
|
settings,
|
||||||
|
apiPath(`${ROOT}/profiles/${encodeURIComponent(profileId)}/objects`, { limit: 100 })
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listKnowledgeRuns(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId?: string
|
||||||
|
): Promise<KnowledgeRun[]> {
|
||||||
|
const response = await apiFetch<{ items: KnowledgeRun[] }>(
|
||||||
|
settings,
|
||||||
|
apiPath(`${ROOT}/runs`, { profile_id: profileId || undefined, limit: 100 })
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function previewKnowledgeMigration(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<KnowledgeMigrationPreview> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/profiles/${encodeURIComponent(profileId)}/migration-dry-runs`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function publishKnowledgePage(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
externalPageId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<KnowledgePublishResult> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/profiles/${encodeURIComponent(profileId)}/pages/${encodeURIComponent(externalPageId)}/publish`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
import { apiFetch, apiPath, type ApiSettings } from "@govoplan/core-webui";
|
||||||
|
|
||||||
|
export type ServiceDeskDiagnostic = {
|
||||||
|
severity: "info" | "warning" | "error";
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
object_ref?: string | null;
|
||||||
|
field?: string | null;
|
||||||
|
retryable: boolean;
|
||||||
|
details: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskRouteMapping = {
|
||||||
|
search_path: string;
|
||||||
|
ticket_path: string;
|
||||||
|
update_path?: string | null;
|
||||||
|
search_method: "GET" | "POST";
|
||||||
|
ticket_method: "GET" | "POST";
|
||||||
|
update_method: "PATCH" | "POST" | "PUT";
|
||||||
|
ticket_web_url_template?: string | null;
|
||||||
|
search_filters: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskQueueMapping = {
|
||||||
|
source_queue: string;
|
||||||
|
target_queue_ref?: string | null;
|
||||||
|
include: boolean;
|
||||||
|
visibility: "tenant" | "restricted";
|
||||||
|
acl_tokens: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskDynamicFieldMapping = {
|
||||||
|
source_name: string;
|
||||||
|
target_name?: string | null;
|
||||||
|
include: boolean;
|
||||||
|
value_type: "string" | "number" | "boolean" | "date" | "json";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskProfile = {
|
||||||
|
id: string;
|
||||||
|
tenant_id: string;
|
||||||
|
configuration_id: string;
|
||||||
|
status: "active" | "paused";
|
||||||
|
integration_mode: "link" | "import" | "synchronize";
|
||||||
|
product: string;
|
||||||
|
product_version?: string | null;
|
||||||
|
desired_maturity: "discover" | "link" | "search" | "read" | "publish" | "synchronize";
|
||||||
|
discovered_maturity: string;
|
||||||
|
source_authority_mode: "external_authoritative" | "external_mirror" | "governed_sync" | "linked_reference";
|
||||||
|
default_visibility: "tenant" | "restricted";
|
||||||
|
default_acl_tokens: string[];
|
||||||
|
routes: ServiceDeskRouteMapping;
|
||||||
|
queue_mappings: ServiceDeskQueueMapping[];
|
||||||
|
dynamic_field_mappings: ServiceDeskDynamicFieldMapping[];
|
||||||
|
capabilities: string[];
|
||||||
|
discovery_revision?: string | null;
|
||||||
|
discovered_configuration_revision?: number | null;
|
||||||
|
discovered_configuration_hash?: string | null;
|
||||||
|
health_status: string;
|
||||||
|
health_details: Record<string, unknown>;
|
||||||
|
discovered_at?: string | null;
|
||||||
|
last_sync_cursor?: string | null;
|
||||||
|
last_high_watermark?: string | null;
|
||||||
|
resource_revision: number;
|
||||||
|
credential_reference_present: boolean;
|
||||||
|
endpoint_configured: boolean;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskObject = {
|
||||||
|
id: string;
|
||||||
|
profile_id: string;
|
||||||
|
object_type: string;
|
||||||
|
external_id: string;
|
||||||
|
external_ticket_number?: string | null;
|
||||||
|
title: string;
|
||||||
|
canonical_url?: string | null;
|
||||||
|
status: string;
|
||||||
|
source_revision: string;
|
||||||
|
visibility: string;
|
||||||
|
acl_tokens: string[];
|
||||||
|
mapped_data: Record<string, unknown>;
|
||||||
|
provenance: Record<string, unknown>;
|
||||||
|
source_updated_at?: string | null;
|
||||||
|
observed_at: string;
|
||||||
|
resource_revision: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskRun = {
|
||||||
|
id: string;
|
||||||
|
profile_id: string;
|
||||||
|
mode: string;
|
||||||
|
idempotency_key: string;
|
||||||
|
status: string;
|
||||||
|
cursor_before?: string | null;
|
||||||
|
cursor_after?: string | null;
|
||||||
|
high_watermark?: string | null;
|
||||||
|
counts: Record<string, number>;
|
||||||
|
effects: Array<Record<string, unknown>>;
|
||||||
|
diagnostics: ServiceDeskDiagnostic[];
|
||||||
|
provenance: Record<string, unknown>;
|
||||||
|
started_at: string;
|
||||||
|
finished_at?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskDiscovery = {
|
||||||
|
profile: ServiceDeskProfile;
|
||||||
|
product: string;
|
||||||
|
product_version?: string | null;
|
||||||
|
api_family: string;
|
||||||
|
capabilities: string[];
|
||||||
|
maturity: string;
|
||||||
|
health_status: string;
|
||||||
|
diagnostics: ServiceDeskDiagnostic[];
|
||||||
|
revision: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ServiceDeskTicketUpdateResult = {
|
||||||
|
run: ServiceDeskRun;
|
||||||
|
object: ServiceDeskObject;
|
||||||
|
accepted: boolean;
|
||||||
|
outcome_unknown: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ROOT = "/api/v1/connectors/service-desk";
|
||||||
|
|
||||||
|
export async function listServiceDeskProfiles(settings: ApiSettings): Promise<ServiceDeskProfile[]> {
|
||||||
|
const response = await apiFetch<{ items: ServiceDeskProfile[] }>(settings, `${ROOT}/profiles`);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createServiceDeskProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ServiceDeskProfile> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function updateServiceDeskProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ServiceDeskProfile> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}`, {
|
||||||
|
method: "PUT",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function discoverServiceDeskProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string
|
||||||
|
): Promise<ServiceDeskDiscovery> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}/discover`, {
|
||||||
|
method: "POST"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function synchronizeServiceDeskProfile(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ServiceDeskRun> {
|
||||||
|
return apiFetch(settings, `${ROOT}/profiles/${encodeURIComponent(profileId)}/sync`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listServiceDeskObjects(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string
|
||||||
|
): Promise<ServiceDeskObject[]> {
|
||||||
|
const response = await apiFetch<{ items: ServiceDeskObject[] }>(
|
||||||
|
settings,
|
||||||
|
apiPath(`${ROOT}/profiles/${encodeURIComponent(profileId)}/objects`, { limit: 100 })
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listServiceDeskRuns(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId?: string
|
||||||
|
): Promise<ServiceDeskRun[]> {
|
||||||
|
const response = await apiFetch<{ items: ServiceDeskRun[] }>(
|
||||||
|
settings,
|
||||||
|
apiPath(`${ROOT}/runs`, { profile_id: profileId || undefined, limit: 100 })
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function updateServiceDeskTicket(
|
||||||
|
settings: ApiSettings,
|
||||||
|
profileId: string,
|
||||||
|
externalTicketId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ServiceDeskTicketUpdateResult> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/profiles/${encodeURIComponent(profileId)}/tickets/${encodeURIComponent(externalTicketId)}/update`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
import { apiFetch, apiPath, type ApiSettings } from "@govoplan/core-webui";
|
||||||
|
|
||||||
|
export type GovernedConnectorSpecification = {
|
||||||
|
provider: string;
|
||||||
|
protocol: string;
|
||||||
|
capabilities: string[];
|
||||||
|
input_schema: Record<string, unknown>;
|
||||||
|
output_schema: Record<string, unknown>;
|
||||||
|
mapping: {
|
||||||
|
version: string;
|
||||||
|
rules: Array<{
|
||||||
|
source: string;
|
||||||
|
target: string;
|
||||||
|
required?: boolean;
|
||||||
|
default?: unknown;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
validation_rules: Array<{
|
||||||
|
kind: "required" | "one_of" | "unique";
|
||||||
|
field: string;
|
||||||
|
values?: unknown[];
|
||||||
|
severity: "warning" | "error";
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
}>;
|
||||||
|
dry_run: {
|
||||||
|
supported: boolean;
|
||||||
|
simulation_supported: boolean;
|
||||||
|
sample_rows: Array<Record<string, unknown>>;
|
||||||
|
max_items: number;
|
||||||
|
redacted_fields: string[];
|
||||||
|
};
|
||||||
|
audit: {
|
||||||
|
event_prefix: string;
|
||||||
|
expected_events: string[];
|
||||||
|
evidence_fields: string[];
|
||||||
|
};
|
||||||
|
privacy_classification: "public" | "internal" | "confidential" | "restricted";
|
||||||
|
retention_class: string;
|
||||||
|
operational_limits: Record<string, unknown>;
|
||||||
|
retry_policy: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorDefinition = {
|
||||||
|
id: string;
|
||||||
|
definition_key: string;
|
||||||
|
name: string;
|
||||||
|
description?: string | null;
|
||||||
|
status: string;
|
||||||
|
current_revision: number;
|
||||||
|
source_package?: string | null;
|
||||||
|
local_definition: boolean;
|
||||||
|
definition_hash: string;
|
||||||
|
specification: GovernedConnectorSpecification;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorConfiguration = {
|
||||||
|
id: string;
|
||||||
|
definition_id: string;
|
||||||
|
definition_key: string;
|
||||||
|
definition_name: string;
|
||||||
|
name: string;
|
||||||
|
status: "draft" | "active" | "disabled";
|
||||||
|
endpoint_url?: string | null;
|
||||||
|
credential_ref?: string | null;
|
||||||
|
base_definition_revision: number;
|
||||||
|
latest_definition_revision: number;
|
||||||
|
update_available: boolean;
|
||||||
|
local_overrides: Record<string, unknown>;
|
||||||
|
protected_paths: string[];
|
||||||
|
effective_configuration: GovernedConnectorSpecification;
|
||||||
|
effective_hash: string;
|
||||||
|
resource_revision: number;
|
||||||
|
ambiguity_policy: "manual_review" | "quarantine" | "reject";
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorRun = {
|
||||||
|
id: string;
|
||||||
|
configuration_id: string;
|
||||||
|
mode: "dry_run" | "simulation";
|
||||||
|
idempotency_key: string;
|
||||||
|
status: string;
|
||||||
|
review_state: string;
|
||||||
|
definition_revision: number;
|
||||||
|
configuration_revision: number;
|
||||||
|
configuration_hash: string;
|
||||||
|
input_hash: string;
|
||||||
|
summary: Record<string, number | boolean>;
|
||||||
|
effects: Array<Record<string, unknown>>;
|
||||||
|
diagnostics: Array<Record<string, unknown>>;
|
||||||
|
provenance: Record<string, unknown>;
|
||||||
|
reviewed_by?: string | null;
|
||||||
|
reviewed_at?: string | null;
|
||||||
|
review_reason?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ConnectorConfigurationDraft = {
|
||||||
|
name: string;
|
||||||
|
status: ConnectorConfiguration["status"];
|
||||||
|
endpoint_url: string;
|
||||||
|
credential_ref: string;
|
||||||
|
local_overrides: string;
|
||||||
|
ambiguity_policy: ConnectorConfiguration["ambiguity_policy"];
|
||||||
|
};
|
||||||
|
|
||||||
|
const ROOT = "/api/v1/connectors/governed";
|
||||||
|
|
||||||
|
export async function listConnectorDefinitions(
|
||||||
|
settings: ApiSettings
|
||||||
|
): Promise<ConnectorDefinition[]> {
|
||||||
|
const response = await apiFetch<{ items: ConnectorDefinition[] }>(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/definitions`
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function upsertConnectorDefinition(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorDefinition> {
|
||||||
|
return apiFetch(settings, `${ROOT}/definitions`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listConnectorConfigurations(
|
||||||
|
settings: ApiSettings
|
||||||
|
): Promise<ConnectorConfiguration[]> {
|
||||||
|
const response = await apiFetch<{ items: ConnectorConfiguration[] }>(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/configurations`
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createConnectorConfiguration(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorConfiguration> {
|
||||||
|
return apiFetch(settings, `${ROOT}/configurations`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function updateConnectorConfiguration(
|
||||||
|
settings: ApiSettings,
|
||||||
|
configurationId: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorConfiguration> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/configurations/${encodeURIComponent(configurationId)}`,
|
||||||
|
{ method: "PUT", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listConnectorRuns(
|
||||||
|
settings: ApiSettings,
|
||||||
|
configurationId?: string
|
||||||
|
): Promise<ConnectorRun[]> {
|
||||||
|
const response = await apiFetch<{ items: ConnectorRun[] }>(
|
||||||
|
settings,
|
||||||
|
apiPath(`${ROOT}/runs`, {
|
||||||
|
configuration_id: configurationId || undefined,
|
||||||
|
limit: 100
|
||||||
|
})
|
||||||
|
);
|
||||||
|
return response.items;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function executeConnectorRun(
|
||||||
|
settings: ApiSettings,
|
||||||
|
configurationId: string,
|
||||||
|
mode: "dry-runs" | "simulations",
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): Promise<ConnectorRun> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/configurations/${encodeURIComponent(configurationId)}/${mode}`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reviewConnectorRun(
|
||||||
|
settings: ApiSettings,
|
||||||
|
runId: string,
|
||||||
|
decision: "approved" | "rejected",
|
||||||
|
reason: string
|
||||||
|
): Promise<ConnectorRun> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`${ROOT}/runs/${encodeURIComponent(runId)}/review`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ decision, reason })
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,661 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
AdminPageLayout,
|
||||||
|
Button,
|
||||||
|
Card,
|
||||||
|
DataGrid,
|
||||||
|
Dialog,
|
||||||
|
FilterBar,
|
||||||
|
FormField,
|
||||||
|
FormGrid,
|
||||||
|
MetricCard,
|
||||||
|
MetricGrid,
|
||||||
|
PageActionBar,
|
||||||
|
SelectionList,
|
||||||
|
SelectionListItem,
|
||||||
|
SelectionListItemContent,
|
||||||
|
StatePanel,
|
||||||
|
StatusBadge,
|
||||||
|
TableActionGroup,
|
||||||
|
WorkspaceLayout,
|
||||||
|
formatDateTime,
|
||||||
|
hasScope,
|
||||||
|
useUnsavedChanges,
|
||||||
|
useUnsavedDraftGuard,
|
||||||
|
type ApiSettings,
|
||||||
|
type AuthInfo,
|
||||||
|
type DataGridColumn
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import {
|
||||||
|
createConnectorConfiguration,
|
||||||
|
executeConnectorRun,
|
||||||
|
listConnectorConfigurations,
|
||||||
|
listConnectorDefinitions,
|
||||||
|
listConnectorRuns,
|
||||||
|
reviewConnectorRun,
|
||||||
|
updateConnectorConfiguration,
|
||||||
|
upsertConnectorDefinition,
|
||||||
|
type ConnectorConfiguration,
|
||||||
|
type ConnectorConfigurationDraft,
|
||||||
|
type ConnectorDefinition,
|
||||||
|
type ConnectorRun
|
||||||
|
} from "../api/governedConnectors";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
settings: ApiSettings;
|
||||||
|
auth: AuthInfo;
|
||||||
|
};
|
||||||
|
|
||||||
|
const EMPTY_DRAFT: ConnectorConfigurationDraft = {
|
||||||
|
name: "",
|
||||||
|
status: "draft",
|
||||||
|
endpoint_url: "",
|
||||||
|
credential_ref: "",
|
||||||
|
local_overrides: "{}",
|
||||||
|
ambiguity_policy: "manual_review"
|
||||||
|
};
|
||||||
|
|
||||||
|
const EXAMPLE_DEFINITION = JSON.stringify({
|
||||||
|
definition_key: "example.reference-data",
|
||||||
|
name: "Example reference data",
|
||||||
|
description: "Locally governed example connector",
|
||||||
|
origin: "local",
|
||||||
|
specification: {
|
||||||
|
provider: "example-provider",
|
||||||
|
protocol: "rest",
|
||||||
|
capabilities: ["discover", "read", "dry_run"],
|
||||||
|
input_schema: { type: "object" },
|
||||||
|
output_schema: { type: "object" },
|
||||||
|
mapping: {
|
||||||
|
version: "1",
|
||||||
|
rules: [{ source: "id", target: "record.id", required: true }]
|
||||||
|
},
|
||||||
|
validation_rules: [{
|
||||||
|
kind: "unique",
|
||||||
|
field: "record.id",
|
||||||
|
severity: "error",
|
||||||
|
code: "record.id.ambiguous",
|
||||||
|
message: "The record identifier is not unique."
|
||||||
|
}],
|
||||||
|
dry_run: {
|
||||||
|
supported: true,
|
||||||
|
simulation_supported: true,
|
||||||
|
sample_rows: [{ id: "sample-1" }],
|
||||||
|
max_items: 500,
|
||||||
|
redacted_fields: []
|
||||||
|
},
|
||||||
|
audit: {
|
||||||
|
event_prefix: "connectors.example",
|
||||||
|
expected_events: ["simulation.completed"],
|
||||||
|
evidence_fields: ["input_hash", "configuration_hash"]
|
||||||
|
},
|
||||||
|
privacy_classification: "internal",
|
||||||
|
retention_class: "connector-preview-30d",
|
||||||
|
operational_limits: { timeout_seconds: 30 },
|
||||||
|
retry_policy: { max_attempts: 2 }
|
||||||
|
}
|
||||||
|
}, null, 2);
|
||||||
|
|
||||||
|
export default function ConnectorGovernancePage({ settings, auth }: Props) {
|
||||||
|
const [definitions, setDefinitions] = useState<ConnectorDefinition[]>([]);
|
||||||
|
const [configurations, setConfigurations] = useState<ConnectorConfiguration[]>([]);
|
||||||
|
const [runs, setRuns] = useState<ConnectorRun[]>([]);
|
||||||
|
const [selectedId, setSelectedId] = useState("");
|
||||||
|
const [draft, setDraft] = useState<ConnectorConfigurationDraft>(EMPTY_DRAFT);
|
||||||
|
const [savedKey, setSavedKey] = useState("");
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
const [definitionOpen, setDefinitionOpen] = useState(false);
|
||||||
|
const [definitionJson, setDefinitionJson] = useState(EXAMPLE_DEFINITION);
|
||||||
|
const [configurationOpen, setConfigurationOpen] = useState(false);
|
||||||
|
const [newDefinitionId, setNewDefinitionId] = useState("");
|
||||||
|
const [newDraft, setNewDraft] = useState<ConnectorConfigurationDraft>(EMPTY_DRAFT);
|
||||||
|
const [sampleJson, setSampleJson] = useState("[]");
|
||||||
|
const [externalRevision, setExternalRevision] = useState("");
|
||||||
|
const [reviewRun, setReviewRun] = useState<ConnectorRun | null>(null);
|
||||||
|
const [reviewReason, setReviewReason] = useState("");
|
||||||
|
const { requestDiscard } = useUnsavedChanges();
|
||||||
|
|
||||||
|
const selected = configurations.find((item) => item.id === selectedId) ?? null;
|
||||||
|
const canAdmin = hasScope(auth, "connectors:source:admin");
|
||||||
|
const canExecute = canAdmin || hasScope(auth, "connectors:source:write");
|
||||||
|
const dirty = Boolean(selected && draftKey(draft) !== savedKey);
|
||||||
|
|
||||||
|
const applyConfiguration = useCallback((item: ConnectorConfiguration | null) => {
|
||||||
|
const next = item ? draftFromConfiguration(item) : EMPTY_DRAFT;
|
||||||
|
setDraft(next);
|
||||||
|
setSavedKey(item ? draftKey(next) : "");
|
||||||
|
setSampleJson(JSON.stringify(
|
||||||
|
item?.effective_configuration.dry_run.sample_rows ?? [],
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const reload = useCallback(async (preferredId?: string) => {
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const [nextDefinitions, nextConfigurations] = await Promise.all([
|
||||||
|
listConnectorDefinitions(settings),
|
||||||
|
listConnectorConfigurations(settings)
|
||||||
|
]);
|
||||||
|
const nextId = preferredId && nextConfigurations.some((item) => item.id === preferredId)
|
||||||
|
? preferredId
|
||||||
|
: nextConfigurations.some((item) => item.id === selectedId)
|
||||||
|
? selectedId
|
||||||
|
: nextConfigurations[0]?.id ?? "";
|
||||||
|
const nextRuns = await listConnectorRuns(settings, nextId || undefined);
|
||||||
|
setDefinitions(nextDefinitions);
|
||||||
|
setConfigurations(nextConfigurations);
|
||||||
|
setRuns(nextRuns);
|
||||||
|
setSelectedId(nextId);
|
||||||
|
applyConfiguration(nextConfigurations.find((item) => item.id === nextId) ?? null);
|
||||||
|
if (!newDefinitionId && nextDefinitions[0]) setNewDefinitionId(nextDefinitions[0].id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}, [applyConfiguration, newDefinitionId, selectedId, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void reload();
|
||||||
|
}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
||||||
|
|
||||||
|
const visibleConfigurations = useMemo(() => {
|
||||||
|
const needle = search.trim().toLocaleLowerCase();
|
||||||
|
return configurations.filter((item) => !needle ||
|
||||||
|
`${item.name} ${item.definition_name} ${item.status}`
|
||||||
|
.toLocaleLowerCase()
|
||||||
|
.includes(needle));
|
||||||
|
}, [configurations, search]);
|
||||||
|
|
||||||
|
const save = async (): Promise<boolean> => {
|
||||||
|
if (!selected || !canAdmin) return false;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const overrides = parseObject(draft.local_overrides, "Local overrides");
|
||||||
|
const updated = await updateConnectorConfiguration(settings, selected.id, {
|
||||||
|
expected_revision: selected.resource_revision,
|
||||||
|
name: draft.name.trim(),
|
||||||
|
status: draft.status,
|
||||||
|
endpoint_url: draft.endpoint_url.trim() || null,
|
||||||
|
credential_ref: draft.credential_ref.trim() || null,
|
||||||
|
local_overrides: overrides,
|
||||||
|
ambiguity_policy: draft.ambiguity_policy
|
||||||
|
});
|
||||||
|
setSuccess("Connector configuration saved.");
|
||||||
|
await reload(updated.id);
|
||||||
|
return true;
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useUnsavedDraftGuard({
|
||||||
|
dirty,
|
||||||
|
onSave: save,
|
||||||
|
onDiscard: () => applyConfiguration(selected),
|
||||||
|
title: "Unsaved connector changes",
|
||||||
|
message: "Save or discard the current connector changes before continuing."
|
||||||
|
});
|
||||||
|
|
||||||
|
const selectConfiguration = (item: ConnectorConfiguration) => {
|
||||||
|
if (item.id === selectedId) return;
|
||||||
|
requestDiscard(() => {
|
||||||
|
setSelectedId(item.id);
|
||||||
|
applyConfiguration(item);
|
||||||
|
setRuns([]);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
void listConnectorRuns(settings, item.id).then(setRuns).catch((caught) => {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const createDefinition = async () => {
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const payload = parseObject(definitionJson, "Definition");
|
||||||
|
const created = await upsertConnectorDefinition(settings, payload);
|
||||||
|
setDefinitionOpen(false);
|
||||||
|
setSuccess("Connector definition revision saved.");
|
||||||
|
setNewDefinitionId(created.id);
|
||||||
|
await reload(selectedId);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const createConfiguration = async () => {
|
||||||
|
if (!newDefinitionId || !newDraft.name.trim()) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const created = await createConnectorConfiguration(settings, {
|
||||||
|
definition_id: newDefinitionId,
|
||||||
|
name: newDraft.name.trim(),
|
||||||
|
status: newDraft.status,
|
||||||
|
endpoint_url: newDraft.endpoint_url.trim() || null,
|
||||||
|
credential_ref: newDraft.credential_ref.trim() || null,
|
||||||
|
local_overrides: parseObject(newDraft.local_overrides, "Local overrides"),
|
||||||
|
ambiguity_policy: newDraft.ambiguity_policy
|
||||||
|
});
|
||||||
|
setConfigurationOpen(false);
|
||||||
|
setNewDraft(EMPTY_DRAFT);
|
||||||
|
setSuccess("Connector configuration created.");
|
||||||
|
await reload(created.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const adoptUpdate = async () => {
|
||||||
|
if (!selected || dirty || !selected.update_available) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const updated = await updateConnectorConfiguration(settings, selected.id, {
|
||||||
|
expected_revision: selected.resource_revision,
|
||||||
|
adopt_latest_definition: true
|
||||||
|
});
|
||||||
|
setSuccess("Package revision adopted; protected local overrides were reapplied.");
|
||||||
|
await reload(updated.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const run = async (mode: "dry-runs" | "simulations") => {
|
||||||
|
if (!selected || dirty) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const inputRows = parseRows(sampleJson);
|
||||||
|
const created = await executeConnectorRun(settings, selected.id, mode, {
|
||||||
|
idempotency_key: `${mode}-${crypto.randomUUID()}`,
|
||||||
|
input_rows: inputRows,
|
||||||
|
external_revision: externalRevision.trim() || null
|
||||||
|
});
|
||||||
|
setSuccess(`${mode === "dry-runs" ? "Dry-run" : "Simulation"} completed with status ${created.status}.`);
|
||||||
|
setRuns(await listConnectorRuns(settings, selected.id));
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const decideReview = async (decision: "approved" | "rejected") => {
|
||||||
|
if (!reviewRun || reviewReason.trim().length < 5) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
await reviewConnectorRun(settings, reviewRun.id, decision, reviewReason.trim());
|
||||||
|
setReviewRun(null);
|
||||||
|
setReviewReason("");
|
||||||
|
setSuccess(`Simulation ${decision}.`);
|
||||||
|
setRuns(await listConnectorRuns(settings, selectedId));
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const runColumns = useMemo<DataGridColumn<ConnectorRun>[]>(() => [
|
||||||
|
{
|
||||||
|
id: "created",
|
||||||
|
header: "Run",
|
||||||
|
width: 190,
|
||||||
|
sortable: true,
|
||||||
|
value: (row) => row.created_at,
|
||||||
|
render: (row) => <>{row.mode}<br /><span className="muted">{formatDateTime(row.created_at)}</span></>
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "status",
|
||||||
|
header: "Status",
|
||||||
|
width: 150,
|
||||||
|
sortable: true,
|
||||||
|
value: (row) => row.status,
|
||||||
|
render: (row) => <StatusBadge status={row.status} label={row.status.replaceAll("_", " ")} />
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "summary",
|
||||||
|
header: "Effects",
|
||||||
|
width: "1fr",
|
||||||
|
minWidth: 220,
|
||||||
|
render: (row) => `${row.summary.total ?? 0} total · ${row.summary.ambiguous ?? 0} ambiguous · ${row.summary.errors ?? 0} errors`
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "revision",
|
||||||
|
header: "Evidence",
|
||||||
|
width: 170,
|
||||||
|
render: (row) => <code title={row.configuration_hash}>r{row.configuration_revision} · {row.input_hash.slice(0, 8)}</code>
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
header: "Actions",
|
||||||
|
width: 90,
|
||||||
|
sticky: "end",
|
||||||
|
align: "right",
|
||||||
|
render: (row) => <TableActionGroup actions={[{
|
||||||
|
id: "review",
|
||||||
|
label: "Review result",
|
||||||
|
icon: <span>✓</span>,
|
||||||
|
applicable: ["pending", "quarantined"].includes(row.review_state),
|
||||||
|
disabled: !canAdmin || busy,
|
||||||
|
disabledReason: !canAdmin ? "Connector administration permission is required." : undefined,
|
||||||
|
onClick: () => setReviewRun(row)
|
||||||
|
}]} />
|
||||||
|
}
|
||||||
|
], [busy, canAdmin]);
|
||||||
|
|
||||||
|
const actionBar = <PageActionBar
|
||||||
|
variant="editor"
|
||||||
|
state={busy ? "saving" : dirty ? "dirty" : "clean"}
|
||||||
|
refreshable
|
||||||
|
reloadAction={{ onReload: () => void reload(selectedId), loading }}
|
||||||
|
primaryActions={<>
|
||||||
|
<Button onClick={() => setDefinitionOpen(true)} disabled={!canAdmin || busy}>
|
||||||
|
New definition
|
||||||
|
</Button>
|
||||||
|
<Button onClick={() => setConfigurationOpen(true)} disabled={!canAdmin || busy || !definitions.length}>
|
||||||
|
New configuration
|
||||||
|
</Button>
|
||||||
|
{selected?.update_available ? <Button
|
||||||
|
variant="secondary"
|
||||||
|
onClick={() => void adoptUpdate()}
|
||||||
|
disabled={!canAdmin || busy || dirty}
|
||||||
|
disabledReason={dirty ? "Save or discard local edits before adopting a package update." : undefined}
|
||||||
|
>
|
||||||
|
Adopt package revision {selected.latest_definition_revision}
|
||||||
|
</Button> : null}
|
||||||
|
</>}
|
||||||
|
discardAction={{
|
||||||
|
label: "Discard changes",
|
||||||
|
disabled: !selected,
|
||||||
|
onClick: () => applyConfiguration(selected)
|
||||||
|
}}
|
||||||
|
saveAction={{
|
||||||
|
label: "Save",
|
||||||
|
disabled: !selected || !canAdmin || busy,
|
||||||
|
disabledReason: !canAdmin ? "Connector administration permission is required." : undefined,
|
||||||
|
onClick: () => void save()
|
||||||
|
}}
|
||||||
|
/>;
|
||||||
|
|
||||||
|
return <AdminPageLayout
|
||||||
|
archetype="workspace"
|
||||||
|
title="Connector governance"
|
||||||
|
description="Version schemas and mappings, protect local overrides, and review deterministic simulations before provider-specific writes."
|
||||||
|
loading={loading && !configurations.length}
|
||||||
|
error={error}
|
||||||
|
success={success}
|
||||||
|
actions={actionBar}
|
||||||
|
className="connector-governance-page"
|
||||||
|
helpContextId="connectors.admin.governed-configurations"
|
||||||
|
>
|
||||||
|
<MetricGrid columns={4} density="compact" minimum="compact">
|
||||||
|
<MetricCard label="Definitions" value={definitions.length} />
|
||||||
|
<MetricCard label="Configurations" value={configurations.length} />
|
||||||
|
<MetricCard label="Updates available" value={configurations.filter((item) => item.update_available).length} tone="warning" />
|
||||||
|
<MetricCard label="Awaiting review" value={runs.filter((item) => ["pending", "quarantined"].includes(item.review_state)).length} tone="warning" />
|
||||||
|
</MetricGrid>
|
||||||
|
|
||||||
|
<WorkspaceLayout
|
||||||
|
variant="split"
|
||||||
|
primarySize="compact"
|
||||||
|
surface="contained"
|
||||||
|
primaryScrollable={false}
|
||||||
|
contentScrollable={false}
|
||||||
|
primaryLabel="Connector configurations"
|
||||||
|
contentLabel="Configuration details"
|
||||||
|
primary={<div className="connector-governance-list">
|
||||||
|
<FilterBar surface="panel">
|
||||||
|
<input
|
||||||
|
type="search"
|
||||||
|
value={search}
|
||||||
|
onChange={(event) => setSearch(event.target.value)}
|
||||||
|
placeholder="Search configurations"
|
||||||
|
aria-label="Search connector configurations"
|
||||||
|
/>
|
||||||
|
</FilterBar>
|
||||||
|
<SelectionList variant="navigation" label="Connector configurations">
|
||||||
|
{visibleConfigurations.map((item) => <SelectionListItem
|
||||||
|
key={item.id}
|
||||||
|
selected={item.id === selectedId}
|
||||||
|
onClick={() => selectConfiguration(item)}
|
||||||
|
>
|
||||||
|
<SelectionListItemContent
|
||||||
|
title={item.name}
|
||||||
|
description={`${item.definition_name} · revision ${item.base_definition_revision}`}
|
||||||
|
/>
|
||||||
|
<StatusBadge status={item.update_available ? "warning" : item.status} />
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!visibleConfigurations.length
|
||||||
|
? <StatePanel size="compact" description="No matching configurations." />
|
||||||
|
: null}
|
||||||
|
</SelectionList>
|
||||||
|
</div>}
|
||||||
|
>
|
||||||
|
{!selected ? <StatePanel
|
||||||
|
size="fill"
|
||||||
|
title="Connector configurations"
|
||||||
|
description="Create or select a configuration to inspect its pinned definition and simulation evidence."
|
||||||
|
/> : <div className="connector-governance-detail">
|
||||||
|
<Card title={selected.name}>
|
||||||
|
<div className="connector-revision-line">
|
||||||
|
<StatusBadge status={selected.status} />
|
||||||
|
<span>Definition revision {selected.base_definition_revision}</span>
|
||||||
|
{selected.update_available
|
||||||
|
? <StatusBadge status="warning" label={`Revision ${selected.latest_definition_revision} available`} />
|
||||||
|
: null}
|
||||||
|
<code title={selected.effective_hash}>{selected.effective_hash.slice(0, 12)}</code>
|
||||||
|
</div>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Name">
|
||||||
|
<input value={draft.name} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, name: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Status">
|
||||||
|
<select value={draft.status} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, status: event.target.value as ConnectorConfigurationDraft["status"] })}>
|
||||||
|
<option value="draft">Draft</option>
|
||||||
|
<option value="active">Active</option>
|
||||||
|
<option value="disabled">Disabled</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Endpoint URL" hint="Credentials are rejected in URLs.">
|
||||||
|
<input value={draft.endpoint_url} disabled={!canAdmin || busy} placeholder="https://provider.example/api" onChange={(event) => setDraft({ ...draft, endpoint_url: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Credential reference" hint="Reference an approved secret; do not paste a secret.">
|
||||||
|
<input value={draft.credential_ref} disabled={!canAdmin || busy} placeholder="vault://connectors/provider" onChange={(event) => setDraft({ ...draft, credential_ref: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Ambiguous-result policy">
|
||||||
|
<select value={draft.ambiguity_policy} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, ambiguity_policy: event.target.value as ConnectorConfigurationDraft["ambiguity_policy"] })}>
|
||||||
|
<option value="manual_review">Manual review</option>
|
||||||
|
<option value="quarantine">Quarantine</option>
|
||||||
|
<option value="reject">Reject</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Protected local overrides" hint="JSON object leaf paths remain protected when package revisions are adopted.">
|
||||||
|
<textarea rows={8} value={draft.local_overrides} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, local_overrides: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
<p className="muted">
|
||||||
|
Protected paths: {selected.protected_paths.length ? selected.protected_paths.join(", ") : "none"}
|
||||||
|
</p>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Dry-run and simulation">
|
||||||
|
<p className="muted">Runs never apply changes. They retain redacted, revision-pinned evidence for review.</p>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Sample input rows" hint="JSON array, bounded by the definition's maximum.">
|
||||||
|
<textarea rows={9} value={sampleJson} disabled={!canExecute || busy} onChange={(event) => setSampleJson(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="External revision" hint="Optional provider ETag, cursor, or snapshot revision.">
|
||||||
|
<input value={externalRevision} disabled={!canExecute || busy} onChange={(event) => setExternalRevision(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
<div className="connector-run-actions">
|
||||||
|
<Button onClick={() => void run("dry-runs")} disabled={!canExecute || busy || dirty}>Run dry-run</Button>
|
||||||
|
<Button variant="primary" onClick={() => void run("simulations")} disabled={!canExecute || busy || dirty}>Run simulation</Button>
|
||||||
|
{dirty ? <span className="muted">Save or discard configuration changes before running.</span> : null}
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Simulation evidence">
|
||||||
|
<DataGrid
|
||||||
|
id="connector-simulation-runs"
|
||||||
|
rows={runs}
|
||||||
|
columns={runColumns}
|
||||||
|
getRowKey={(row) => row.id}
|
||||||
|
initialFit="container"
|
||||||
|
emptyText="No dry-runs or simulations have been recorded."
|
||||||
|
/>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Effective governed definition">
|
||||||
|
<pre className="connector-json-preview">{JSON.stringify(selected.effective_configuration, null, 2)}</pre>
|
||||||
|
</Card>
|
||||||
|
</div>}
|
||||||
|
</WorkspaceLayout>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={definitionOpen}
|
||||||
|
title="Create or revise connector definition"
|
||||||
|
onClose={() => !busy && setDefinitionOpen(false)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={<>
|
||||||
|
<Button onClick={() => setDefinitionOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createDefinition()} disabled={!canAdmin || busy}>Save definition revision</Button>
|
||||||
|
</>}
|
||||||
|
>
|
||||||
|
<p className="muted">The definition is schema-validated and every changed specification creates an immutable revision. Package definitions must name their package reference.</p>
|
||||||
|
<FormField label="Governed definition JSON">
|
||||||
|
<textarea className="connector-definition-editor" rows={24} value={definitionJson} disabled={busy} onChange={(event) => setDefinitionJson(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={configurationOpen}
|
||||||
|
title="Create connector configuration"
|
||||||
|
onClose={() => !busy && setConfigurationOpen(false)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={<>
|
||||||
|
<Button onClick={() => setConfigurationOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createConfiguration()} disabled={busy || !newDefinitionId || !newDraft.name.trim()}>Create configuration</Button>
|
||||||
|
</>}
|
||||||
|
>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Definition">
|
||||||
|
<select value={newDefinitionId} disabled={busy} onChange={(event) => setNewDefinitionId(event.target.value)}>
|
||||||
|
{definitions.map((item) => <option key={item.id} value={item.id}>{item.name} · revision {item.current_revision}</option>)}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Name">
|
||||||
|
<input value={newDraft.name} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, name: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Endpoint URL">
|
||||||
|
<input value={newDraft.endpoint_url} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, endpoint_url: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Credential reference">
|
||||||
|
<input value={newDraft.credential_ref} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, credential_ref: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Ambiguous-result policy">
|
||||||
|
<select value={newDraft.ambiguity_policy} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, ambiguity_policy: event.target.value as ConnectorConfigurationDraft["ambiguity_policy"] })}>
|
||||||
|
<option value="manual_review">Manual review</option>
|
||||||
|
<option value="quarantine">Quarantine</option>
|
||||||
|
<option value="reject">Reject</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Local overrides">
|
||||||
|
<textarea rows={7} value={newDraft.local_overrides} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, local_overrides: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={Boolean(reviewRun)}
|
||||||
|
title="Review ambiguous connector result"
|
||||||
|
onClose={() => !busy && setReviewRun(null)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={<>
|
||||||
|
<Button onClick={() => setReviewRun(null)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="danger" onClick={() => void decideReview("rejected")} disabled={busy || reviewReason.trim().length < 5}>Reject</Button>
|
||||||
|
<Button variant="primary" onClick={() => void decideReview("approved")} disabled={busy || reviewReason.trim().length < 5}>Approve</Button>
|
||||||
|
</>}
|
||||||
|
>
|
||||||
|
<p>Review {reviewRun?.summary.ambiguous ?? 0} ambiguous effects against the retained input and configuration hashes before deciding.</p>
|
||||||
|
<FormField label="Decision reason">
|
||||||
|
<textarea rows={4} value={reviewReason} disabled={busy} onChange={(event) => setReviewReason(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
<pre className="connector-json-preview">{JSON.stringify(reviewRun?.diagnostics ?? [], null, 2)}</pre>
|
||||||
|
</Dialog>
|
||||||
|
</AdminPageLayout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftFromConfiguration(item: ConnectorConfiguration): ConnectorConfigurationDraft {
|
||||||
|
return {
|
||||||
|
name: item.name,
|
||||||
|
status: item.status,
|
||||||
|
endpoint_url: item.endpoint_url ?? "",
|
||||||
|
credential_ref: item.credential_ref ?? "",
|
||||||
|
local_overrides: JSON.stringify(item.local_overrides, null, 2),
|
||||||
|
ambiguity_policy: item.ambiguity_policy
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftKey(value: ConnectorConfigurationDraft): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
name: value.name.trim(),
|
||||||
|
status: value.status,
|
||||||
|
endpoint_url: value.endpoint_url.trim(),
|
||||||
|
credential_ref: value.credential_ref.trim(),
|
||||||
|
local_overrides: normalizeJson(value.local_overrides),
|
||||||
|
ambiguity_policy: value.ambiguity_policy
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeJson(value: string): unknown {
|
||||||
|
try {
|
||||||
|
return JSON.parse(value);
|
||||||
|
} catch {
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseObject(value: string, label: string): Record<string, unknown> {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") {
|
||||||
|
throw new Error(`${label} must be a JSON object.`);
|
||||||
|
}
|
||||||
|
return parsed as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRows(value: string): Array<Record<string, unknown>> {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!Array.isArray(parsed) || parsed.some((item) => !item || Array.isArray(item) || typeof item !== "object")) {
|
||||||
|
throw new Error("Sample input must be a JSON array of objects.");
|
||||||
|
}
|
||||||
|
return parsed as Array<Record<string, unknown>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
@@ -0,0 +1,561 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
AdminPageLayout,
|
||||||
|
Button,
|
||||||
|
Card,
|
||||||
|
Dialog,
|
||||||
|
FilterBar,
|
||||||
|
FormField,
|
||||||
|
FormGrid,
|
||||||
|
MetricCard,
|
||||||
|
MetricGrid,
|
||||||
|
PageActionBar,
|
||||||
|
SelectionList,
|
||||||
|
SelectionListItem,
|
||||||
|
SelectionListItemContent,
|
||||||
|
StatePanel,
|
||||||
|
StatusBadge,
|
||||||
|
WorkspaceLayout,
|
||||||
|
formatDateTime,
|
||||||
|
hasScope,
|
||||||
|
useUnsavedChanges,
|
||||||
|
useUnsavedDraftGuard,
|
||||||
|
type ApiSettings,
|
||||||
|
type AuthInfo
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import {
|
||||||
|
createKnowledgeProfile,
|
||||||
|
discoverKnowledgeProfile,
|
||||||
|
listKnowledgeObjects,
|
||||||
|
listKnowledgeProfiles,
|
||||||
|
listKnowledgeRuns,
|
||||||
|
previewKnowledgeMigration,
|
||||||
|
publishKnowledgePage,
|
||||||
|
synchronizeKnowledgeProfile,
|
||||||
|
updateKnowledgeProfile,
|
||||||
|
type KnowledgeMigrationPreview,
|
||||||
|
type KnowledgeObject,
|
||||||
|
type KnowledgeProfile,
|
||||||
|
type KnowledgeRun
|
||||||
|
} from "../api/externalKnowledge";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
settings: ApiSettings;
|
||||||
|
auth: AuthInfo;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ProfileDraft = {
|
||||||
|
status: "active" | "paused";
|
||||||
|
desired_maturity: KnowledgeProfile["desired_maturity"];
|
||||||
|
source_authority_mode: KnowledgeProfile["source_authority_mode"];
|
||||||
|
default_visibility: KnowledgeProfile["default_visibility"];
|
||||||
|
default_acl_tokens: string;
|
||||||
|
namespace_mappings: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const EMPTY_DRAFT: ProfileDraft = {
|
||||||
|
status: "active",
|
||||||
|
desired_maturity: "migrate",
|
||||||
|
source_authority_mode: "external_mirror",
|
||||||
|
default_visibility: "restricted",
|
||||||
|
default_acl_tokens: "scope:connectors:knowledge:read",
|
||||||
|
namespace_mappings: JSON.stringify([{
|
||||||
|
source_namespace_id: 0,
|
||||||
|
source_name: "",
|
||||||
|
target_space_ref: "external-knowledge",
|
||||||
|
target_path_prefix: "",
|
||||||
|
include: true,
|
||||||
|
acl_tokens: []
|
||||||
|
}], null, 2)
|
||||||
|
};
|
||||||
|
|
||||||
|
export default function ExternalKnowledgePage({ settings, auth }: Props) {
|
||||||
|
const [profiles, setProfiles] = useState<KnowledgeProfile[]>([]);
|
||||||
|
const [objects, setObjects] = useState<KnowledgeObject[]>([]);
|
||||||
|
const [runs, setRuns] = useState<KnowledgeRun[]>([]);
|
||||||
|
const [selectedId, setSelectedId] = useState("");
|
||||||
|
const [draft, setDraft] = useState<ProfileDraft>(EMPTY_DRAFT);
|
||||||
|
const [savedKey, setSavedKey] = useState("");
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
const [createOpen, setCreateOpen] = useState(false);
|
||||||
|
const [configurationId, setConfigurationId] = useState("");
|
||||||
|
const [newDraft, setNewDraft] = useState<ProfileDraft>(EMPTY_DRAFT);
|
||||||
|
const [migrationOpen, setMigrationOpen] = useState(false);
|
||||||
|
const [targetSpace, setTargetSpace] = useState("external-knowledge");
|
||||||
|
const [supportedMacros, setSupportedMacros] = useState("");
|
||||||
|
const [existingTargets, setExistingTargets] = useState("[]");
|
||||||
|
const [migration, setMigration] = useState<KnowledgeMigrationPreview | null>(null);
|
||||||
|
const [publishOpen, setPublishOpen] = useState(false);
|
||||||
|
const [externalPageId, setExternalPageId] = useState("");
|
||||||
|
const [publishTitle, setPublishTitle] = useState("");
|
||||||
|
const [publishBody, setPublishBody] = useState("");
|
||||||
|
const [publishSummary, setPublishSummary] = useState("");
|
||||||
|
const [publishRevision, setPublishRevision] = useState("");
|
||||||
|
const { requestDiscard } = useUnsavedChanges();
|
||||||
|
|
||||||
|
const selected = profiles.find((item) => item.id === selectedId) ?? null;
|
||||||
|
const canAdmin = hasScope(auth, "connectors:knowledge:admin");
|
||||||
|
const canSync = hasScope(auth, "connectors:knowledge:sync");
|
||||||
|
const canMigrate = hasScope(auth, "connectors:knowledge:migrate");
|
||||||
|
const canPublish = hasScope(auth, "connectors:knowledge:publish");
|
||||||
|
const dirty = Boolean(selected && draftKey(draft) !== savedKey);
|
||||||
|
|
||||||
|
const applyProfile = useCallback((profile: KnowledgeProfile | null) => {
|
||||||
|
const next = profile ? draftFromProfile(profile) : EMPTY_DRAFT;
|
||||||
|
setDraft(next);
|
||||||
|
setSavedKey(profile ? draftKey(next) : "");
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const reload = useCallback(async (preferredId?: string) => {
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const nextProfiles = await listKnowledgeProfiles(settings);
|
||||||
|
const nextId = preferredId && nextProfiles.some((item) => item.id === preferredId)
|
||||||
|
? preferredId
|
||||||
|
: nextProfiles.some((item) => item.id === selectedId)
|
||||||
|
? selectedId
|
||||||
|
: nextProfiles[0]?.id ?? "";
|
||||||
|
const [nextObjects, nextRuns] = nextId
|
||||||
|
? await Promise.all([
|
||||||
|
listKnowledgeObjects(settings, nextId),
|
||||||
|
listKnowledgeRuns(settings, nextId)
|
||||||
|
])
|
||||||
|
: [[], []];
|
||||||
|
setProfiles(nextProfiles);
|
||||||
|
setSelectedId(nextId);
|
||||||
|
setObjects(nextObjects);
|
||||||
|
setRuns(nextRuns);
|
||||||
|
applyProfile(nextProfiles.find((item) => item.id === nextId) ?? null);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}, [applyProfile, selectedId, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void reload();
|
||||||
|
}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
||||||
|
|
||||||
|
const save = async (): Promise<boolean> => {
|
||||||
|
if (!selected || !canAdmin) return false;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const updated = await updateKnowledgeProfile(settings, selected.id, {
|
||||||
|
expected_resource_revision: selected.resource_revision,
|
||||||
|
status: draft.status,
|
||||||
|
desired_maturity: draft.desired_maturity,
|
||||||
|
source_authority_mode: draft.source_authority_mode,
|
||||||
|
default_visibility: draft.default_visibility,
|
||||||
|
default_acl_tokens: lines(draft.default_acl_tokens),
|
||||||
|
namespace_mappings: parseArray(draft.namespace_mappings, "Namespace mappings")
|
||||||
|
});
|
||||||
|
setSuccess("External knowledge profile saved; fallback ACLs and Search projections were refreshed.");
|
||||||
|
await reload(updated.id);
|
||||||
|
return true;
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useUnsavedDraftGuard({
|
||||||
|
dirty,
|
||||||
|
onSave: save,
|
||||||
|
onDiscard: () => applyProfile(selected),
|
||||||
|
title: "Unsaved knowledge profile changes",
|
||||||
|
message: "Save or discard the profile changes before continuing."
|
||||||
|
});
|
||||||
|
|
||||||
|
const selectProfile = (profile: KnowledgeProfile) => {
|
||||||
|
if (profile.id === selectedId) return;
|
||||||
|
requestDiscard(() => {
|
||||||
|
setSelectedId(profile.id);
|
||||||
|
applyProfile(profile);
|
||||||
|
setObjects([]);
|
||||||
|
setRuns([]);
|
||||||
|
setMigration(null);
|
||||||
|
void Promise.all([
|
||||||
|
listKnowledgeObjects(settings, profile.id),
|
||||||
|
listKnowledgeRuns(settings, profile.id)
|
||||||
|
]).then(([nextObjects, nextRuns]) => {
|
||||||
|
setObjects(nextObjects);
|
||||||
|
setRuns(nextRuns);
|
||||||
|
}).catch((caught) => setError(errorMessage(caught)));
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const createProfile = async () => {
|
||||||
|
if (!configurationId.trim()) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const created = await createKnowledgeProfile(settings, {
|
||||||
|
configuration_id: configurationId.trim(),
|
||||||
|
desired_maturity: newDraft.desired_maturity,
|
||||||
|
source_authority_mode: newDraft.source_authority_mode,
|
||||||
|
default_visibility: newDraft.default_visibility,
|
||||||
|
default_acl_tokens: lines(newDraft.default_acl_tokens),
|
||||||
|
namespace_mappings: parseArray(newDraft.namespace_mappings, "Namespace mappings")
|
||||||
|
});
|
||||||
|
setCreateOpen(false);
|
||||||
|
setConfigurationId("");
|
||||||
|
setNewDraft(EMPTY_DRAFT);
|
||||||
|
setSuccess("External knowledge profile created. Run discovery before synchronization.");
|
||||||
|
await reload(created.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const discover = async () => {
|
||||||
|
if (!selected || dirty) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const result = await discoverKnowledgeProfile(settings, selected.id);
|
||||||
|
setSuccess(`Discovered ${result.product} ${result.product_version ?? ""} at ${result.maturity} maturity with ${result.diagnostics.length} diagnostics.`);
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const sync = async (forceFull: boolean) => {
|
||||||
|
if (!selected || dirty) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const run = await synchronizeKnowledgeProfile(settings, selected.id, {
|
||||||
|
idempotency_key: `knowledge-${forceFull ? "backfill" : "delta"}-${crypto.randomUUID()}`,
|
||||||
|
force_full: forceFull,
|
||||||
|
limit: 100
|
||||||
|
});
|
||||||
|
setSuccess(`${forceFull ? "Backfill" : "Delta"} completed with ${effectTotal(run)} effects.`);
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const previewMigration = async () => {
|
||||||
|
if (!selected || !targetSpace.trim()) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const result = await previewKnowledgeMigration(settings, selected.id, {
|
||||||
|
idempotency_key: `knowledge-migration-${crypto.randomUUID()}`,
|
||||||
|
target_space_ref: targetSpace.trim(),
|
||||||
|
max_items: 100,
|
||||||
|
supported_macros: lines(supportedMacros),
|
||||||
|
existing_targets: parseArray(existingTargets, "Existing targets")
|
||||||
|
});
|
||||||
|
setMigration(result);
|
||||||
|
setMigrationOpen(false);
|
||||||
|
setSuccess(result.can_apply
|
||||||
|
? "Migration preview is complete and contains no blocking conflict. It did not write Wiki pages."
|
||||||
|
: "Migration preview found conflicts, errors, or truncation. It did not write Wiki pages.");
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const publish = async () => {
|
||||||
|
if (!selected || !externalPageId.trim() || !publishTitle.trim()) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const result = await publishKnowledgePage(
|
||||||
|
settings,
|
||||||
|
selected.id,
|
||||||
|
externalPageId.trim(),
|
||||||
|
{
|
||||||
|
idempotency_key: `knowledge-publish-${crypto.randomUUID()}`,
|
||||||
|
title: publishTitle.trim(),
|
||||||
|
body: publishBody,
|
||||||
|
summary: publishSummary.trim(),
|
||||||
|
expected_external_revision: publishRevision.trim() || null,
|
||||||
|
minor: false
|
||||||
|
}
|
||||||
|
);
|
||||||
|
setPublishOpen(false);
|
||||||
|
setSuccess(result.outcome_unknown
|
||||||
|
? "Publication outcome is unknown. Reconcile the provider revision before retrying."
|
||||||
|
: "Provider accepted the page revision and durable recovery evidence was recorded.");
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const visibleProfiles = useMemo(() => {
|
||||||
|
const needle = search.trim().toLocaleLowerCase();
|
||||||
|
return profiles.filter((item) => !needle ||
|
||||||
|
`${item.product} ${item.product_version ?? ""} ${item.health_status} ${item.configuration_id}`
|
||||||
|
.toLocaleLowerCase()
|
||||||
|
.includes(needle));
|
||||||
|
}, [profiles, search]);
|
||||||
|
|
||||||
|
const actionBar = <PageActionBar
|
||||||
|
variant="editor"
|
||||||
|
state={busy ? "saving" : dirty ? "dirty" : "clean"}
|
||||||
|
refreshable
|
||||||
|
reloadAction={{ onReload: () => void reload(selectedId), loading }}
|
||||||
|
primaryActions={<>
|
||||||
|
<Button onClick={() => setCreateOpen(true)} disabled={!canAdmin || busy}>New profile</Button>
|
||||||
|
<Button variant="secondary" onClick={() => void discover()} disabled={!selected || !canAdmin || busy || dirty}>Discover</Button>
|
||||||
|
<Button variant="secondary" onClick={() => void sync(false)} disabled={!selected || !canSync || busy || dirty}>Run delta</Button>
|
||||||
|
<Button variant="secondary" onClick={() => void sync(true)} disabled={!selected || !canSync || busy || dirty}>Run full backfill</Button>
|
||||||
|
<Button variant="secondary" onClick={() => setMigrationOpen(true)} disabled={!selected || !canMigrate || busy || dirty}>Preview migration</Button>
|
||||||
|
<Button variant="primary" onClick={() => setPublishOpen(true)} disabled={!selected || !canPublish || busy || dirty}>Publish page</Button>
|
||||||
|
</>}
|
||||||
|
discardAction={{
|
||||||
|
label: "Discard changes",
|
||||||
|
disabled: !selected,
|
||||||
|
onClick: () => applyProfile(selected)
|
||||||
|
}}
|
||||||
|
saveAction={{
|
||||||
|
label: "Save",
|
||||||
|
disabled: !selected || !canAdmin || busy,
|
||||||
|
disabledReason: !canAdmin ? "External knowledge administration permission is required." : undefined,
|
||||||
|
onClick: () => void save()
|
||||||
|
}}
|
||||||
|
/>;
|
||||||
|
|
||||||
|
return <AdminPageLayout
|
||||||
|
archetype="workspace"
|
||||||
|
title="External knowledge"
|
||||||
|
description="Discover and synchronize MediaWiki or BlueSpice, preserve stable identity and permissions, and preview migration into native Wiki."
|
||||||
|
loading={loading && !profiles.length}
|
||||||
|
error={error}
|
||||||
|
success={success}
|
||||||
|
actions={actionBar}
|
||||||
|
className="connector-knowledge-page"
|
||||||
|
helpContextId="connectors.admin.external-knowledge"
|
||||||
|
>
|
||||||
|
<MetricGrid columns={4} density="compact" minimum="compact">
|
||||||
|
<MetricCard label="Profiles" value={profiles.length} />
|
||||||
|
<MetricCard label="Active pages" value={objects.filter((item) => item.status !== "deleted").length} />
|
||||||
|
<MetricCard label="Unhealthy profiles" value={profiles.filter((item) => !["healthy", "unknown"].includes(item.health_status)).length} tone="warning" />
|
||||||
|
<MetricCard label="Unresolved runs" value={runs.filter((item) => ["failed", "outcome_unknown"].includes(item.status)).length} tone="warning" />
|
||||||
|
</MetricGrid>
|
||||||
|
|
||||||
|
<WorkspaceLayout
|
||||||
|
variant="split"
|
||||||
|
primarySize="compact"
|
||||||
|
surface="contained"
|
||||||
|
primaryScrollable={false}
|
||||||
|
contentScrollable={false}
|
||||||
|
primaryLabel="Knowledge profiles"
|
||||||
|
contentLabel="Profile details"
|
||||||
|
primary={<div className="connector-knowledge-list">
|
||||||
|
<FilterBar surface="panel">
|
||||||
|
<input type="search" value={search} onChange={(event) => setSearch(event.target.value)} placeholder="Search profiles" aria-label="Search external knowledge profiles" />
|
||||||
|
</FilterBar>
|
||||||
|
<SelectionList variant="navigation" label="External knowledge profiles">
|
||||||
|
{visibleProfiles.map((profile) => <SelectionListItem key={profile.id} selected={profile.id === selectedId} onClick={() => selectProfile(profile)}>
|
||||||
|
<SelectionListItemContent
|
||||||
|
title={`${profile.product}${profile.product_version ? ` ${profile.product_version}` : ""}`}
|
||||||
|
description={`${profile.discovered_maturity} · ${profile.configuration_id}`}
|
||||||
|
/>
|
||||||
|
<StatusBadge status={profile.status === "paused" ? "inactive" : profile.health_status} />
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!visibleProfiles.length ? <StatePanel size="compact" description="No matching knowledge profiles." /> : null}
|
||||||
|
</SelectionList>
|
||||||
|
</div>}
|
||||||
|
>
|
||||||
|
{!selected ? <StatePanel size="fill" title="External knowledge profiles" description="Create or select a profile to discover provider capabilities and inspect synchronization evidence." /> : <div className="connector-knowledge-detail">
|
||||||
|
<Card title={`${selected.product}${selected.product_version ? ` ${selected.product_version}` : ""}`}>
|
||||||
|
<div className="connector-revision-line">
|
||||||
|
<StatusBadge status={selected.status} />
|
||||||
|
<StatusBadge status={selected.health_status} />
|
||||||
|
<span>Discovered maturity: {selected.discovered_maturity}</span>
|
||||||
|
<code title={selected.discovery_revision ?? undefined}>r{selected.resource_revision}</code>
|
||||||
|
</div>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Status" hint="Pausing immediately makes Search authorization fail closed.">
|
||||||
|
<select value={draft.status} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, status: event.target.value as ProfileDraft["status"] })}>
|
||||||
|
<option value="active">Active</option>
|
||||||
|
<option value="paused">Paused</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Desired maturity" hint="This is the operator ceiling even when the provider offers more.">
|
||||||
|
<select value={draft.desired_maturity} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, desired_maturity: event.target.value as ProfileDraft["desired_maturity"] })}>
|
||||||
|
{['discover', 'link', 'search', 'read', 'publish', 'synchronize', 'migrate'].map((value) => <option key={value} value={value}>{value}</option>)}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Source authority">
|
||||||
|
<select value={draft.source_authority_mode} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, source_authority_mode: event.target.value as ProfileDraft["source_authority_mode"] })}>
|
||||||
|
<option value="external_authoritative">External authoritative</option>
|
||||||
|
<option value="external_mirror">External mirror</option>
|
||||||
|
<option value="governed_sync">Governed sync</option>
|
||||||
|
<option value="linked_reference">Linked reference</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Fallback visibility">
|
||||||
|
<select value={draft.default_visibility} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, default_visibility: event.target.value as ProfileDraft["default_visibility"] })}>
|
||||||
|
<option value="restricted">Restricted</option>
|
||||||
|
<option value="tenant">Tenant</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Fallback ACL tokens" hint="One account, membership, identity, group, role, function, or scope token per line.">
|
||||||
|
<textarea rows={6} value={draft.default_acl_tokens} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, default_acl_tokens: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Namespace mappings" hint="JSON array; maps source namespace ids to target Wiki space references and path prefixes.">
|
||||||
|
<textarea rows={12} value={draft.namespace_mappings} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, namespace_mappings: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
<p className="muted">Capabilities: {selected.capabilities.length ? selected.capabilities.join(", ") : "run discovery"}</p>
|
||||||
|
<p className="muted">Last high-watermark: {selected.last_high_watermark ?? "none"} · credential reference: {selected.credential_reference_present ? "configured" : "not configured"}</p>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Synchronized pages">
|
||||||
|
<SelectionList variant="static" label="Synchronized external knowledge pages">
|
||||||
|
{objects.slice(0, 100).map((item) => <SelectionListItem key={item.id}>
|
||||||
|
<SelectionListItemContent title={item.title} description={`${item.status} · revision ${item.source_revision} · ${item.visibility}`} />
|
||||||
|
{item.canonical_url ? <a href={item.canonical_url} target="_blank" rel="noreferrer">Open source</a> : null}
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!objects.length ? <StatePanel size="compact" description="No synchronized pages. Run a full backfill after discovery." /> : null}
|
||||||
|
</SelectionList>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Synchronization and migration evidence">
|
||||||
|
<SelectionList variant="static" label="Knowledge connector runs">
|
||||||
|
{runs.map((run) => <SelectionListItem key={run.id}>
|
||||||
|
<SelectionListItemContent title={`${run.mode.replaceAll("_", " ")} · ${run.status}`} description={`${formatDateTime(run.started_at)} · ${effectTotal(run)} effects · ${run.diagnostics.length} diagnostics`} />
|
||||||
|
<StatusBadge status={run.status} />
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!runs.length ? <StatePanel size="compact" description="No knowledge connector runs have been recorded." /> : null}
|
||||||
|
</SelectionList>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
{migration ? <Card title="Latest migration preview">
|
||||||
|
<div className="connector-revision-line">
|
||||||
|
<StatusBadge status={migration.can_apply ? "success" : "warning"} label={migration.can_apply ? "No blocking conflict" : "Review required"} />
|
||||||
|
<span>{migration.effects.length} effects</span>
|
||||||
|
<span>{migration.diagnostics.length} diagnostics</span>
|
||||||
|
<code title={migration.source_fingerprint}>{migration.source_fingerprint.slice(0, 12)}</code>
|
||||||
|
</div>
|
||||||
|
<p className="muted">Preview only: no native Wiki page was written.</p>
|
||||||
|
<pre className="connector-json-preview">{JSON.stringify({ summary: migration.summary, diagnostics: migration.diagnostics, truncated: migration.truncated }, null, 2)}</pre>
|
||||||
|
</Card> : null}
|
||||||
|
</div>}
|
||||||
|
</WorkspaceLayout>
|
||||||
|
|
||||||
|
<Dialog open={createOpen} title="Create external knowledge profile" onClose={() => !busy && setCreateOpen(false)} closeDisabled={busy} footer={<>
|
||||||
|
<Button onClick={() => setCreateOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createProfile()} disabled={busy || !configurationId.trim()}>Create profile</Button>
|
||||||
|
</>}>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Governed configuration id" hint="Select an active MediaWiki Action API configuration from Connector governance.">
|
||||||
|
<input value={configurationId} disabled={busy} onChange={(event) => setConfigurationId(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Desired maturity">
|
||||||
|
<select value={newDraft.desired_maturity} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, desired_maturity: event.target.value as ProfileDraft["desired_maturity"] })}>
|
||||||
|
{['discover', 'link', 'search', 'read', 'publish', 'synchronize', 'migrate'].map((value) => <option key={value} value={value}>{value}</option>)}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Fallback visibility">
|
||||||
|
<select value={newDraft.default_visibility} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, default_visibility: event.target.value as ProfileDraft["default_visibility"] })}>
|
||||||
|
<option value="restricted">Restricted</option>
|
||||||
|
<option value="tenant">Tenant</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Fallback ACL tokens">
|
||||||
|
<textarea rows={5} value={newDraft.default_acl_tokens} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, default_acl_tokens: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Namespace mappings">
|
||||||
|
<textarea rows={12} value={newDraft.namespace_mappings} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, namespace_mappings: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog open={migrationOpen} title="Preview migration into native Wiki" onClose={() => !busy && setMigrationOpen(false)} closeDisabled={busy} footer={<>
|
||||||
|
<Button onClick={() => setMigrationOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void previewMigration()} disabled={busy || !targetSpace.trim()}>Run migration preview</Button>
|
||||||
|
</>}>
|
||||||
|
<p className="muted">This dry-run detects path, attachment, macro, and truncation problems. It never writes Wiki pages.</p>
|
||||||
|
<FormField label="Target Wiki space reference"><input value={targetSpace} disabled={busy} onChange={(event) => setTargetSpace(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Supported macros" hint="One macro name per line."><textarea rows={5} value={supportedMacros} disabled={busy} onChange={(event) => setSupportedMacros(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Existing targets" hint="JSON array with path, optional source_external_id, and attachment_names."><textarea rows={10} value={existingTargets} disabled={busy} onChange={(event) => setExistingTargets(event.target.value)} /></FormField>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog open={publishOpen} title="Publish provider page revision" onClose={() => !busy && setPublishOpen(false)} closeDisabled={busy} footer={<>
|
||||||
|
<Button onClick={() => setPublishOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void publish()} disabled={busy || !externalPageId.trim() || !publishTitle.trim()}>Publish revision</Button>
|
||||||
|
</>}>
|
||||||
|
<p className="muted">Publication is an external effect. Supply the current provider revision where possible; an unknown outcome blocks blind retry.</p>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Stable external page id"><input value={externalPageId} disabled={busy} onChange={(event) => setExternalPageId(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Expected provider revision"><input value={publishRevision} disabled={busy} onChange={(event) => setPublishRevision(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Title"><input value={publishTitle} disabled={busy} onChange={(event) => setPublishTitle(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Edit summary"><input value={publishSummary} disabled={busy} onChange={(event) => setPublishSummary(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Wikitext body"><textarea rows={16} value={publishBody} disabled={busy} onChange={(event) => setPublishBody(event.target.value)} /></FormField>
|
||||||
|
</FormGrid>
|
||||||
|
</Dialog>
|
||||||
|
</AdminPageLayout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftFromProfile(profile: KnowledgeProfile): ProfileDraft {
|
||||||
|
return {
|
||||||
|
status: profile.status,
|
||||||
|
desired_maturity: profile.desired_maturity,
|
||||||
|
source_authority_mode: profile.source_authority_mode,
|
||||||
|
default_visibility: profile.default_visibility,
|
||||||
|
default_acl_tokens: profile.default_acl_tokens.join("\n"),
|
||||||
|
namespace_mappings: JSON.stringify(profile.namespace_mappings, null, 2)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftKey(draft: ProfileDraft): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
...draft,
|
||||||
|
default_acl_tokens: lines(draft.default_acl_tokens),
|
||||||
|
namespace_mappings: normalizeJson(draft.namespace_mappings)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function lines(value: string): string[] {
|
||||||
|
return [...new Set(value.split(/\r?\n/).map((item) => item.trim()).filter(Boolean))];
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeJson(value: string): unknown {
|
||||||
|
try {
|
||||||
|
return JSON.parse(value);
|
||||||
|
} catch {
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArray(value: string, label: string): unknown[] {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!Array.isArray(parsed)) throw new Error(`${label} must be a JSON array.`);
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function effectTotal(run: KnowledgeRun): number {
|
||||||
|
return Object.values(run.counts).reduce((total, value) => total + Number(value || 0), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
@@ -0,0 +1,573 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
AdminPageLayout,
|
||||||
|
Button,
|
||||||
|
Card,
|
||||||
|
Dialog,
|
||||||
|
FilterBar,
|
||||||
|
FormField,
|
||||||
|
FormGrid,
|
||||||
|
MetricCard,
|
||||||
|
MetricGrid,
|
||||||
|
PageActionBar,
|
||||||
|
SelectionList,
|
||||||
|
SelectionListItem,
|
||||||
|
SelectionListItemContent,
|
||||||
|
StatePanel,
|
||||||
|
StatusBadge,
|
||||||
|
WorkspaceLayout,
|
||||||
|
formatDateTime,
|
||||||
|
hasScope,
|
||||||
|
useUnsavedChanges,
|
||||||
|
useUnsavedDraftGuard,
|
||||||
|
type ApiSettings,
|
||||||
|
type AuthInfo
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import {
|
||||||
|
createServiceDeskProfile,
|
||||||
|
discoverServiceDeskProfile,
|
||||||
|
listServiceDeskObjects,
|
||||||
|
listServiceDeskProfiles,
|
||||||
|
listServiceDeskRuns,
|
||||||
|
synchronizeServiceDeskProfile,
|
||||||
|
updateServiceDeskProfile,
|
||||||
|
updateServiceDeskTicket,
|
||||||
|
type ServiceDeskObject,
|
||||||
|
type ServiceDeskProfile,
|
||||||
|
type ServiceDeskRun
|
||||||
|
} from "../api/externalServiceDesk";
|
||||||
|
|
||||||
|
type Props = { settings: ApiSettings; auth: AuthInfo };
|
||||||
|
|
||||||
|
type ProfileDraft = {
|
||||||
|
status: "active" | "paused";
|
||||||
|
integration_mode: ServiceDeskProfile["integration_mode"];
|
||||||
|
desired_maturity: ServiceDeskProfile["desired_maturity"];
|
||||||
|
source_authority_mode: ServiceDeskProfile["source_authority_mode"];
|
||||||
|
default_visibility: ServiceDeskProfile["default_visibility"];
|
||||||
|
default_acl_tokens: string;
|
||||||
|
routes: string;
|
||||||
|
queue_mappings: string;
|
||||||
|
dynamic_field_mappings: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_ROUTES = {
|
||||||
|
search_path: "/Ticket/Search",
|
||||||
|
ticket_path: "/Ticket/{ticket_id}",
|
||||||
|
update_path: null,
|
||||||
|
search_method: "POST",
|
||||||
|
ticket_method: "GET",
|
||||||
|
update_method: "PATCH",
|
||||||
|
ticket_web_url_template: null,
|
||||||
|
search_filters: {}
|
||||||
|
};
|
||||||
|
|
||||||
|
const EMPTY_DRAFT: ProfileDraft = {
|
||||||
|
status: "active",
|
||||||
|
integration_mode: "synchronize",
|
||||||
|
desired_maturity: "synchronize",
|
||||||
|
source_authority_mode: "external_authoritative",
|
||||||
|
default_visibility: "restricted",
|
||||||
|
default_acl_tokens: "scope:connectors:service_desk:read",
|
||||||
|
routes: JSON.stringify(DEFAULT_ROUTES, null, 2),
|
||||||
|
queue_mappings: JSON.stringify([], null, 2),
|
||||||
|
dynamic_field_mappings: JSON.stringify([], null, 2)
|
||||||
|
};
|
||||||
|
|
||||||
|
export default function ExternalServiceDeskPage({ settings, auth }: Props) {
|
||||||
|
const [profiles, setProfiles] = useState<ServiceDeskProfile[]>([]);
|
||||||
|
const [objects, setObjects] = useState<ServiceDeskObject[]>([]);
|
||||||
|
const [runs, setRuns] = useState<ServiceDeskRun[]>([]);
|
||||||
|
const [selectedId, setSelectedId] = useState("");
|
||||||
|
const [draft, setDraft] = useState<ProfileDraft>(EMPTY_DRAFT);
|
||||||
|
const [savedKey, setSavedKey] = useState("");
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
const [createOpen, setCreateOpen] = useState(false);
|
||||||
|
const [configurationId, setConfigurationId] = useState("");
|
||||||
|
const [newDraft, setNewDraft] = useState<ProfileDraft>(EMPTY_DRAFT);
|
||||||
|
const [updateOpen, setUpdateOpen] = useState(false);
|
||||||
|
const [updateObjectId, setUpdateObjectId] = useState("");
|
||||||
|
const [updateTitle, setUpdateTitle] = useState("");
|
||||||
|
const [updateQueue, setUpdateQueue] = useState("");
|
||||||
|
const [updateState, setUpdateState] = useState("");
|
||||||
|
const [updatePriority, setUpdatePriority] = useState("");
|
||||||
|
const [updateOwner, setUpdateOwner] = useState("");
|
||||||
|
const [updateResponsible, setUpdateResponsible] = useState("");
|
||||||
|
const [updateDynamicFields, setUpdateDynamicFields] = useState("{}");
|
||||||
|
const { requestDiscard } = useUnsavedChanges();
|
||||||
|
|
||||||
|
const selected = profiles.find((item) => item.id === selectedId) ?? null;
|
||||||
|
const selectedObject = objects.find((item) => item.id === updateObjectId) ?? null;
|
||||||
|
const canAdmin = hasScope(auth, "connectors:service_desk:admin");
|
||||||
|
const canSync = hasScope(auth, "connectors:service_desk:sync");
|
||||||
|
const canUpdate = hasScope(auth, "connectors:service_desk:update");
|
||||||
|
const dirty = Boolean(selected && draftKey(draft) !== savedKey);
|
||||||
|
|
||||||
|
const applyProfile = useCallback((profile: ServiceDeskProfile | null) => {
|
||||||
|
const next = profile ? draftFromProfile(profile) : EMPTY_DRAFT;
|
||||||
|
setDraft(next);
|
||||||
|
setSavedKey(profile ? draftKey(next) : "");
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const reload = useCallback(async (preferredId?: string) => {
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const nextProfiles = await listServiceDeskProfiles(settings);
|
||||||
|
const nextId = preferredId && nextProfiles.some((item) => item.id === preferredId)
|
||||||
|
? preferredId
|
||||||
|
: nextProfiles.some((item) => item.id === selectedId)
|
||||||
|
? selectedId
|
||||||
|
: nextProfiles[0]?.id ?? "";
|
||||||
|
const [nextObjects, nextRuns] = nextId
|
||||||
|
? await Promise.all([
|
||||||
|
listServiceDeskObjects(settings, nextId),
|
||||||
|
listServiceDeskRuns(settings, nextId)
|
||||||
|
])
|
||||||
|
: [[], []];
|
||||||
|
setProfiles(nextProfiles);
|
||||||
|
setSelectedId(nextId);
|
||||||
|
setObjects(nextObjects);
|
||||||
|
setRuns(nextRuns);
|
||||||
|
applyProfile(nextProfiles.find((item) => item.id === nextId) ?? null);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}, [applyProfile, selectedId, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void reload();
|
||||||
|
}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
||||||
|
|
||||||
|
const save = async (): Promise<boolean> => {
|
||||||
|
if (!selected || !canAdmin) return false;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const updated = await updateServiceDeskProfile(settings, selected.id, {
|
||||||
|
expected_resource_revision: selected.resource_revision,
|
||||||
|
status: draft.status,
|
||||||
|
integration_mode: draft.integration_mode,
|
||||||
|
desired_maturity: draft.desired_maturity,
|
||||||
|
source_authority_mode: draft.source_authority_mode,
|
||||||
|
default_visibility: draft.default_visibility,
|
||||||
|
default_acl_tokens: lines(draft.default_acl_tokens),
|
||||||
|
routes: parseObject(draft.routes, "Routes"),
|
||||||
|
queue_mappings: parseArray(draft.queue_mappings, "Queue mappings"),
|
||||||
|
dynamic_field_mappings: parseArray(draft.dynamic_field_mappings, "Dynamic-field mappings")
|
||||||
|
});
|
||||||
|
setSuccess("Service-desk profile saved; queue ACLs and Search projections were refreshed.");
|
||||||
|
await reload(updated.id);
|
||||||
|
return true;
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useUnsavedDraftGuard({
|
||||||
|
dirty,
|
||||||
|
onSave: save,
|
||||||
|
onDiscard: () => applyProfile(selected),
|
||||||
|
title: "Unsaved service-desk profile changes",
|
||||||
|
message: "Save or discard the profile changes before continuing."
|
||||||
|
});
|
||||||
|
|
||||||
|
const selectProfile = (profile: ServiceDeskProfile) => {
|
||||||
|
if (profile.id === selectedId) return;
|
||||||
|
requestDiscard(() => {
|
||||||
|
setSelectedId(profile.id);
|
||||||
|
applyProfile(profile);
|
||||||
|
setObjects([]);
|
||||||
|
setRuns([]);
|
||||||
|
void Promise.all([
|
||||||
|
listServiceDeskObjects(settings, profile.id),
|
||||||
|
listServiceDeskRuns(settings, profile.id)
|
||||||
|
]).then(([nextObjects, nextRuns]) => {
|
||||||
|
setObjects(nextObjects);
|
||||||
|
setRuns(nextRuns);
|
||||||
|
}).catch((caught) => setError(errorMessage(caught)));
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const createProfile = async () => {
|
||||||
|
if (!configurationId.trim()) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const created = await createServiceDeskProfile(settings, {
|
||||||
|
configuration_id: configurationId.trim(),
|
||||||
|
integration_mode: newDraft.integration_mode,
|
||||||
|
desired_maturity: newDraft.desired_maturity,
|
||||||
|
source_authority_mode: newDraft.source_authority_mode,
|
||||||
|
default_visibility: newDraft.default_visibility,
|
||||||
|
default_acl_tokens: lines(newDraft.default_acl_tokens),
|
||||||
|
routes: parseObject(newDraft.routes, "Routes"),
|
||||||
|
queue_mappings: parseArray(newDraft.queue_mappings, "Queue mappings"),
|
||||||
|
dynamic_field_mappings: parseArray(newDraft.dynamic_field_mappings, "Dynamic-field mappings")
|
||||||
|
});
|
||||||
|
setCreateOpen(false);
|
||||||
|
setConfigurationId("");
|
||||||
|
setNewDraft(EMPTY_DRAFT);
|
||||||
|
setSuccess("Service-desk profile created. Run discovery before synchronization.");
|
||||||
|
await reload(created.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const discover = async () => {
|
||||||
|
if (!selected || dirty) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const result = await discoverServiceDeskProfile(settings, selected.id);
|
||||||
|
setSuccess(`Discovered ${result.product} ${result.product_version ?? ""} at ${result.maturity} maturity with ${result.diagnostics.length} diagnostics.`);
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const sync = async (mode: "auto" | "full") => {
|
||||||
|
if (!selected || dirty) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const run = await synchronizeServiceDeskProfile(settings, selected.id, {
|
||||||
|
idempotency_key: `service-desk-${mode}-${crypto.randomUUID()}`,
|
||||||
|
mode,
|
||||||
|
limit: 100
|
||||||
|
});
|
||||||
|
setSuccess(`${mode === "full" ? "Full synchronization" : "Next synchronization page"} completed with ${effectTotal(run)} effects.`);
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const openTicketUpdate = (item: ServiceDeskObject) => {
|
||||||
|
setUpdateObjectId(item.id);
|
||||||
|
setUpdateTitle("");
|
||||||
|
setUpdateQueue("");
|
||||||
|
setUpdateState("");
|
||||||
|
setUpdatePriority("");
|
||||||
|
setUpdateOwner("");
|
||||||
|
setUpdateResponsible("");
|
||||||
|
setUpdateDynamicFields("{}");
|
||||||
|
setUpdateOpen(true);
|
||||||
|
};
|
||||||
|
|
||||||
|
const submitTicketUpdate = async () => {
|
||||||
|
if (!selected || !selectedObject) return;
|
||||||
|
const dynamicFields = parseObject(updateDynamicFields, "Dynamic fields");
|
||||||
|
const changes = compact({
|
||||||
|
title: updateTitle,
|
||||||
|
queue: updateQueue,
|
||||||
|
state: updateState,
|
||||||
|
priority: updatePriority,
|
||||||
|
owner: updateOwner,
|
||||||
|
responsible: updateResponsible
|
||||||
|
});
|
||||||
|
if (!Object.keys(changes).length && !Object.keys(dynamicFields).length) {
|
||||||
|
setError("Enter at least one governed ticket change.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const result = await updateServiceDeskTicket(
|
||||||
|
settings,
|
||||||
|
selected.id,
|
||||||
|
selectedObject.external_id,
|
||||||
|
{
|
||||||
|
idempotency_key: `service-desk-update-${crypto.randomUUID()}`,
|
||||||
|
expected_external_revision: selectedObject.source_revision,
|
||||||
|
...changes,
|
||||||
|
dynamic_fields: dynamicFields
|
||||||
|
}
|
||||||
|
);
|
||||||
|
setUpdateOpen(false);
|
||||||
|
setSuccess(result.outcome_unknown
|
||||||
|
? "Update outcome is unknown. Inspect the provider revision before retrying."
|
||||||
|
: "Provider accepted the revision-checked ticket update and durable evidence was recorded.");
|
||||||
|
await reload(selected.id);
|
||||||
|
} catch (caught) {
|
||||||
|
setError(errorMessage(caught));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const visibleProfiles = useMemo(() => {
|
||||||
|
const needle = search.trim().toLocaleLowerCase();
|
||||||
|
return profiles.filter((item) => !needle ||
|
||||||
|
`${item.product} ${item.product_version ?? ""} ${item.integration_mode} ${item.health_status} ${item.configuration_id}`
|
||||||
|
.toLocaleLowerCase().includes(needle));
|
||||||
|
}, [profiles, search]);
|
||||||
|
|
||||||
|
const actionBar = <PageActionBar
|
||||||
|
variant="editor"
|
||||||
|
state={busy ? "saving" : dirty ? "dirty" : "clean"}
|
||||||
|
refreshable
|
||||||
|
reloadAction={{ onReload: () => void reload(selectedId), loading }}
|
||||||
|
primaryActions={<>
|
||||||
|
<Button onClick={() => setCreateOpen(true)} disabled={!canAdmin || busy}>New profile</Button>
|
||||||
|
<Button variant="secondary" onClick={() => void discover()} disabled={!selected || !canAdmin || busy || dirty}>Discover</Button>
|
||||||
|
<Button variant="secondary" onClick={() => void sync("auto")} disabled={!selected || !canSync || busy || dirty}>Run next page</Button>
|
||||||
|
<Button variant="secondary" onClick={() => void sync("full")} disabled={!selected || !canSync || busy || dirty}>Restart full sync</Button>
|
||||||
|
</>}
|
||||||
|
discardAction={{
|
||||||
|
label: "Discard changes",
|
||||||
|
disabled: !selected,
|
||||||
|
onClick: () => applyProfile(selected)
|
||||||
|
}}
|
||||||
|
saveAction={{
|
||||||
|
label: "Save",
|
||||||
|
disabled: !selected || !canAdmin || busy,
|
||||||
|
disabledReason: !canAdmin ? "Service-desk administration permission is required." : undefined,
|
||||||
|
onClick: () => void save()
|
||||||
|
}}
|
||||||
|
/>;
|
||||||
|
|
||||||
|
return <AdminPageLayout
|
||||||
|
archetype="workspace"
|
||||||
|
title="External service desk"
|
||||||
|
description="Connect Znuny or OTRS-compatible tickets while preserving provider identity, source authority, current ACLs, and domain-module boundaries."
|
||||||
|
loading={loading && !profiles.length}
|
||||||
|
error={error}
|
||||||
|
success={success}
|
||||||
|
actions={actionBar}
|
||||||
|
className="connector-service-desk-page"
|
||||||
|
helpContextId="connectors.admin.external-service-desk"
|
||||||
|
>
|
||||||
|
<MetricGrid columns={4} density="compact" minimum="compact">
|
||||||
|
<MetricCard label="Profiles" value={profiles.length} />
|
||||||
|
<MetricCard label="Active tickets" value={objects.filter((item) => item.status !== "deleted").length} />
|
||||||
|
<MetricCard label="Unhealthy profiles" value={profiles.filter((item) => !["healthy", "unknown"].includes(item.health_status)).length} tone="warning" />
|
||||||
|
<MetricCard label="Unresolved runs" value={runs.filter((item) => ["failed", "outcome_unknown"].includes(item.status)).length} tone="warning" />
|
||||||
|
</MetricGrid>
|
||||||
|
|
||||||
|
<WorkspaceLayout
|
||||||
|
variant="split"
|
||||||
|
primarySize="compact"
|
||||||
|
surface="contained"
|
||||||
|
primaryScrollable={false}
|
||||||
|
contentScrollable={false}
|
||||||
|
primaryLabel="Service-desk profiles"
|
||||||
|
contentLabel="Profile details"
|
||||||
|
primary={<div className="connector-knowledge-list">
|
||||||
|
<FilterBar surface="panel">
|
||||||
|
<input type="search" value={search} onChange={(event) => setSearch(event.target.value)} placeholder="Search profiles" aria-label="Search service-desk profiles" />
|
||||||
|
</FilterBar>
|
||||||
|
<SelectionList variant="navigation" label="External service-desk profiles">
|
||||||
|
{visibleProfiles.map((profile) => <SelectionListItem key={profile.id} selected={profile.id === selectedId} onClick={() => selectProfile(profile)}>
|
||||||
|
<SelectionListItemContent
|
||||||
|
title={`${profile.product}${profile.product_version ? ` ${profile.product_version}` : ""}`}
|
||||||
|
description={`${profile.integration_mode} · ${profile.discovered_maturity} · ${profile.configuration_id}`}
|
||||||
|
/>
|
||||||
|
<StatusBadge status={profile.status === "paused" ? "inactive" : profile.health_status} />
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!visibleProfiles.length ? <StatePanel size="compact" description="No matching service-desk profiles." /> : null}
|
||||||
|
</SelectionList>
|
||||||
|
</div>}
|
||||||
|
>
|
||||||
|
{!selected ? <StatePanel size="fill" title="External service-desk profiles" description="Create or select a profile to discover deployment routes and inspect synchronization evidence." /> : <div className="connector-knowledge-detail">
|
||||||
|
<Card title={`${selected.product}${selected.product_version ? ` ${selected.product_version}` : ""}`}>
|
||||||
|
<div className="connector-revision-line">
|
||||||
|
<StatusBadge status={selected.status} />
|
||||||
|
<StatusBadge status={selected.health_status} />
|
||||||
|
<span>Discovered maturity: {selected.discovered_maturity}</span>
|
||||||
|
<code title={selected.discovery_revision ?? undefined}>r{selected.resource_revision}</code>
|
||||||
|
</div>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Status" hint="Pausing immediately makes Search authorization fail closed.">
|
||||||
|
<select value={draft.status} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, status: event.target.value as ProfileDraft["status"] })}>
|
||||||
|
<option value="active">Active</option><option value="paused">Paused</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Integration mode" hint="Link and import are intentionally not continuous bidirectional synchronization.">
|
||||||
|
<select value={draft.integration_mode} disabled={!canAdmin || busy} onChange={(event) => setDraft(withMode(draft, event.target.value as ProfileDraft["integration_mode"]))}>
|
||||||
|
<option value="link">Link</option><option value="import">Import snapshot</option><option value="synchronize">Synchronize</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Desired maturity" hint="Link permits link/search, import requires read, synchronize requires synchronize.">
|
||||||
|
<select value={draft.desired_maturity} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, desired_maturity: event.target.value as ProfileDraft["desired_maturity"] })}>
|
||||||
|
{maturityOptions(draft.integration_mode).map((value) => <option key={value} value={value}>{value}</option>)}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Source authority">
|
||||||
|
<select value={draft.source_authority_mode} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, source_authority_mode: event.target.value as ProfileDraft["source_authority_mode"] })}>
|
||||||
|
{authorityOptions(draft.integration_mode).map(([value, label]) => <option key={value} value={value}>{label}</option>)}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Fallback visibility">
|
||||||
|
<select value={draft.default_visibility} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, default_visibility: event.target.value as ProfileDraft["default_visibility"] })}>
|
||||||
|
<option value="restricted">Restricted</option><option value="tenant">Tenant</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Fallback ACL tokens" hint="Used only when provider and queue mappings supply no portable ACL.">
|
||||||
|
<textarea rows={6} value={draft.default_acl_tokens} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, default_acl_tokens: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="GenericInterface routes" hint="JSON object; deployment-defined relative paths and methods plus optional absolute browser-link template.">
|
||||||
|
<textarea rows={14} value={draft.routes} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, routes: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Queue mappings" hint="JSON array; inclusion, target queue ref, visibility, and ACL tokens.">
|
||||||
|
<textarea rows={14} value={draft.queue_mappings} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, queue_mappings: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Dynamic-field mappings" hint="JSON array; source name, governed target name, inclusion, and value type.">
|
||||||
|
<textarea rows={14} value={draft.dynamic_field_mappings} disabled={!canAdmin || busy} onChange={(event) => setDraft({ ...draft, dynamic_field_mappings: event.target.value })} />
|
||||||
|
</FormField>
|
||||||
|
</FormGrid>
|
||||||
|
<p className="muted">Capabilities: {selected.capabilities.length ? selected.capabilities.join(", ") : "run discovery"}</p>
|
||||||
|
<p className="muted">Last high-watermark: {selected.last_high_watermark ?? "none"} · credential reference: {selected.credential_reference_present ? "configured" : "not configured"}</p>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Synchronized external tickets">
|
||||||
|
<SelectionList variant="static" label="Synchronized external service-desk tickets">
|
||||||
|
{objects.slice(0, 100).map((item) => <SelectionListItem key={item.id}>
|
||||||
|
<SelectionListItemContent title={`${item.external_ticket_number ? `${item.external_ticket_number}: ` : ""}${item.title}`} description={`${item.status} · revision ${item.source_revision} · ${item.visibility}`} />
|
||||||
|
{item.canonical_url ? <a href={item.canonical_url} target="_blank" rel="noreferrer">Open source</a> : null}
|
||||||
|
<Button variant="secondary" onClick={() => openTicketUpdate(item)} disabled={!canUpdate || busy || dirty || selected.source_authority_mode !== "governed_sync" || !selected.capabilities.includes("publish") || item.status === "deleted"}>Update</Button>
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!objects.length ? <StatePanel size="compact" description="No synchronized tickets. Run discovery and a full synchronization." /> : null}
|
||||||
|
</SelectionList>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card title="Synchronization and mutation evidence">
|
||||||
|
<SelectionList variant="static" label="Service-desk connector runs">
|
||||||
|
{runs.map((run) => <SelectionListItem key={run.id}>
|
||||||
|
<SelectionListItemContent title={`${run.mode.replaceAll("_", " ")} · ${run.status}`} description={`${formatDateTime(run.started_at)} · ${effectTotal(run)} effects · ${run.diagnostics.length} diagnostics`} />
|
||||||
|
<StatusBadge status={run.status} />
|
||||||
|
</SelectionListItem>)}
|
||||||
|
{!runs.length ? <StatePanel size="compact" description="No service-desk connector runs have been recorded." /> : null}
|
||||||
|
</SelectionList>
|
||||||
|
</Card>
|
||||||
|
</div>}
|
||||||
|
</WorkspaceLayout>
|
||||||
|
|
||||||
|
<Dialog open={createOpen} title="Create external service-desk profile" onClose={() => !busy && setCreateOpen(false)} closeDisabled={busy} footer={<>
|
||||||
|
<Button onClick={() => setCreateOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createProfile()} disabled={busy || !configurationId.trim()}>Create profile</Button>
|
||||||
|
</>}>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Governed configuration id" hint="Select an active Znuny/OTRS GenericInterface REST configuration from Connector governance.">
|
||||||
|
<input value={configurationId} disabled={busy} onChange={(event) => setConfigurationId(event.target.value)} />
|
||||||
|
</FormField>
|
||||||
|
<FormField label="Integration mode"><select value={newDraft.integration_mode} disabled={busy} onChange={(event) => setNewDraft(withMode(newDraft, event.target.value as ProfileDraft["integration_mode"]))}><option value="link">Link</option><option value="import">Import snapshot</option><option value="synchronize">Synchronize</option></select></FormField>
|
||||||
|
<FormField label="Desired maturity"><select value={newDraft.desired_maturity} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, desired_maturity: event.target.value as ProfileDraft["desired_maturity"] })}>{maturityOptions(newDraft.integration_mode).map((value) => <option key={value} value={value}>{value}</option>)}</select></FormField>
|
||||||
|
<FormField label="Source authority"><select value={newDraft.source_authority_mode} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, source_authority_mode: event.target.value as ProfileDraft["source_authority_mode"] })}>{authorityOptions(newDraft.integration_mode).map(([value, label]) => <option key={value} value={value}>{label}</option>)}</select></FormField>
|
||||||
|
<FormField label="Fallback visibility"><select value={newDraft.default_visibility} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, default_visibility: event.target.value as ProfileDraft["default_visibility"] })}><option value="restricted">Restricted</option><option value="tenant">Tenant</option></select></FormField>
|
||||||
|
<FormField label="Fallback ACL tokens"><textarea rows={5} value={newDraft.default_acl_tokens} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, default_acl_tokens: event.target.value })} /></FormField>
|
||||||
|
<FormField label="GenericInterface routes"><textarea rows={12} value={newDraft.routes} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, routes: event.target.value })} /></FormField>
|
||||||
|
<FormField label="Queue mappings"><textarea rows={12} value={newDraft.queue_mappings} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, queue_mappings: event.target.value })} /></FormField>
|
||||||
|
<FormField label="Dynamic-field mappings"><textarea rows={12} value={newDraft.dynamic_field_mappings} disabled={busy} onChange={(event) => setNewDraft({ ...newDraft, dynamic_field_mappings: event.target.value })} /></FormField>
|
||||||
|
</FormGrid>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Dialog open={updateOpen} title="Update external ticket" onClose={() => !busy && setUpdateOpen(false)} closeDisabled={busy} footer={<>
|
||||||
|
<Button onClick={() => setUpdateOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void submitTicketUpdate()} disabled={busy || !selectedObject}>Submit revision-checked update</Button>
|
||||||
|
</>}>
|
||||||
|
<p className="muted">This is an external effect. Empty fields remain unchanged; an unknown result blocks blind retry and requires provider reconciliation.</p>
|
||||||
|
<p><strong>{selectedObject?.external_ticket_number}</strong> {selectedObject?.title}<br /><span className="muted">Expected provider revision: {selectedObject?.source_revision}</span></p>
|
||||||
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
|
<FormField label="Title"><input value={updateTitle} disabled={busy} onChange={(event) => setUpdateTitle(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Queue"><input value={updateQueue} disabled={busy} onChange={(event) => setUpdateQueue(event.target.value)} /></FormField>
|
||||||
|
<FormField label="State"><input value={updateState} disabled={busy} onChange={(event) => setUpdateState(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Priority"><input value={updatePriority} disabled={busy} onChange={(event) => setUpdatePriority(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Owner"><input value={updateOwner} disabled={busy} onChange={(event) => setUpdateOwner(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Responsible"><input value={updateResponsible} disabled={busy} onChange={(event) => setUpdateResponsible(event.target.value)} /></FormField>
|
||||||
|
<FormField label="Governed dynamic fields" hint="JSON object keyed by configured target name."><textarea rows={8} value={updateDynamicFields} disabled={busy} onChange={(event) => setUpdateDynamicFields(event.target.value)} /></FormField>
|
||||||
|
</FormGrid>
|
||||||
|
</Dialog>
|
||||||
|
</AdminPageLayout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftFromProfile(profile: ServiceDeskProfile): ProfileDraft {
|
||||||
|
return {
|
||||||
|
status: profile.status,
|
||||||
|
integration_mode: profile.integration_mode,
|
||||||
|
desired_maturity: profile.desired_maturity,
|
||||||
|
source_authority_mode: profile.source_authority_mode,
|
||||||
|
default_visibility: profile.default_visibility,
|
||||||
|
default_acl_tokens: profile.default_acl_tokens.join("\n"),
|
||||||
|
routes: JSON.stringify(profile.routes, null, 2),
|
||||||
|
queue_mappings: JSON.stringify(profile.queue_mappings, null, 2),
|
||||||
|
dynamic_field_mappings: JSON.stringify(profile.dynamic_field_mappings, null, 2)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function withMode(draft: ProfileDraft, mode: ProfileDraft["integration_mode"]): ProfileDraft {
|
||||||
|
if (mode === "link") return { ...draft, integration_mode: mode, desired_maturity: "link", source_authority_mode: "linked_reference" };
|
||||||
|
if (mode === "import") return { ...draft, integration_mode: mode, desired_maturity: "read", source_authority_mode: "external_mirror" };
|
||||||
|
return { ...draft, integration_mode: mode, desired_maturity: "synchronize", source_authority_mode: "external_authoritative" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function maturityOptions(mode: ProfileDraft["integration_mode"]): ProfileDraft["desired_maturity"][] {
|
||||||
|
if (mode === "link") return ["link", "search"];
|
||||||
|
if (mode === "import") return ["read"];
|
||||||
|
return ["synchronize"];
|
||||||
|
}
|
||||||
|
|
||||||
|
function authorityOptions(mode: ProfileDraft["integration_mode"]): Array<[ProfileDraft["source_authority_mode"], string]> {
|
||||||
|
if (mode === "link") return [["linked_reference", "Linked reference"]];
|
||||||
|
if (mode === "import") return [["external_authoritative", "External authoritative"], ["external_mirror", "External mirror"]];
|
||||||
|
return [["external_authoritative", "External authoritative"], ["governed_sync", "Governed sync"]];
|
||||||
|
}
|
||||||
|
|
||||||
|
function draftKey(draft: ProfileDraft): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
...draft,
|
||||||
|
default_acl_tokens: lines(draft.default_acl_tokens),
|
||||||
|
routes: normalizeJson(draft.routes),
|
||||||
|
queue_mappings: normalizeJson(draft.queue_mappings),
|
||||||
|
dynamic_field_mappings: normalizeJson(draft.dynamic_field_mappings)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function lines(value: string): string[] {
|
||||||
|
return [...new Set(value.split(/\r?\n/).map((item) => item.trim()).filter(Boolean))];
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeJson(value: string): unknown {
|
||||||
|
try { return JSON.parse(value); } catch { return value.trim(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArray(value: string, label: string): unknown[] {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!Array.isArray(parsed)) throw new Error(`${label} must be a JSON array.`);
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseObject(value: string, label: string): Record<string, unknown> {
|
||||||
|
const parsed: unknown = JSON.parse(value);
|
||||||
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new Error(`${label} must be a JSON object.`);
|
||||||
|
return parsed as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function compact(values: Record<string, string>): Record<string, string> {
|
||||||
|
return Object.fromEntries(Object.entries(values).map(([key, value]) => [key, value.trim()]).filter(([, value]) => value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function effectTotal(run: ServiceDeskRun): number {
|
||||||
|
return Object.values(run.counts).reduce((total, value) => total + Number(value || 0), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
export { connectorsModule as default, connectorsModule } from "./module";
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import { createElement, lazy } from "react";
|
||||||
|
import type {
|
||||||
|
AdminSectionsUiCapability,
|
||||||
|
PlatformWebModule
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import "./styles/connectors.css";
|
||||||
|
|
||||||
|
const ConnectorGovernancePage = lazy(
|
||||||
|
() => import("./features/ConnectorGovernancePage")
|
||||||
|
);
|
||||||
|
const ExternalKnowledgePage = lazy(
|
||||||
|
() => import("./features/ExternalKnowledgePage")
|
||||||
|
);
|
||||||
|
const ExternalServiceDeskPage = lazy(
|
||||||
|
() => import("./features/ExternalServiceDeskPage")
|
||||||
|
);
|
||||||
|
|
||||||
|
const readScopes = [
|
||||||
|
"connectors:source:read",
|
||||||
|
"connectors:source:admin"
|
||||||
|
];
|
||||||
|
const knowledgeReadScopes = [
|
||||||
|
"connectors:knowledge:read",
|
||||||
|
"connectors:knowledge:admin"
|
||||||
|
];
|
||||||
|
const serviceDeskReadScopes = [
|
||||||
|
"connectors:service_desk:read",
|
||||||
|
"connectors:service_desk:admin"
|
||||||
|
];
|
||||||
|
|
||||||
|
const adminSections: AdminSectionsUiCapability = {
|
||||||
|
sections: [
|
||||||
|
{
|
||||||
|
id: "connector-governance",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "management",
|
||||||
|
surfaceId: "connectors.admin.governed-configurations",
|
||||||
|
label: "Connector governance",
|
||||||
|
group: "SYSTEM",
|
||||||
|
order: 45,
|
||||||
|
anyOf: readScopes,
|
||||||
|
render: ({ settings, auth }) =>
|
||||||
|
createElement(ConnectorGovernancePage, { settings, auth })
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "connector-external-knowledge",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "management",
|
||||||
|
surfaceId: "connectors.admin.external-knowledge",
|
||||||
|
label: "External knowledge",
|
||||||
|
group: "SYSTEM",
|
||||||
|
order: 46,
|
||||||
|
anyOf: knowledgeReadScopes,
|
||||||
|
render: ({ settings, auth }) =>
|
||||||
|
createElement(ExternalKnowledgePage, { settings, auth })
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "connector-external-service-desk",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "management",
|
||||||
|
surfaceId: "connectors.admin.external-service-desk",
|
||||||
|
label: "External service desk",
|
||||||
|
group: "SYSTEM",
|
||||||
|
order: 47,
|
||||||
|
anyOf: serviceDeskReadScopes,
|
||||||
|
render: ({ settings, auth }) =>
|
||||||
|
createElement(ExternalServiceDeskPage, { settings, auth })
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
export const connectorsModule: PlatformWebModule = {
|
||||||
|
id: "connectors",
|
||||||
|
label: "Connectors",
|
||||||
|
version: "0.1.22",
|
||||||
|
dependencies: [],
|
||||||
|
optionalDependencies: ["access", "audit", "policy", "ops", "search", "wiki", "tickets", "helpdesk", "cases"],
|
||||||
|
viewSurfaces: [
|
||||||
|
{
|
||||||
|
id: "connectors.admin.governed-configurations",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "section",
|
||||||
|
label: "Connector governance",
|
||||||
|
order: 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "connectors.admin.simulation-review",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "section",
|
||||||
|
label: "Connector simulation review",
|
||||||
|
parentId: "connectors.admin.governed-configurations",
|
||||||
|
order: 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "connectors.admin.external-knowledge",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "section",
|
||||||
|
label: "External knowledge",
|
||||||
|
parentId: "connectors.admin.governed-configurations",
|
||||||
|
order: 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "connectors.admin.external-service-desk",
|
||||||
|
moduleId: "connectors",
|
||||||
|
kind: "section",
|
||||||
|
label: "External service desk",
|
||||||
|
parentId: "connectors.admin.governed-configurations",
|
||||||
|
order: 40
|
||||||
|
}
|
||||||
|
],
|
||||||
|
uiCapabilities: {
|
||||||
|
"admin.sections": adminSections
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export default connectorsModule;
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
.connector-governance-page .connector-governance-list,
|
||||||
|
.connector-governance-page .connector-governance-detail {
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-knowledge-page .connector-knowledge-list,
|
||||||
|
.connector-knowledge-page .connector-knowledge-detail {
|
||||||
|
display: grid;
|
||||||
|
gap: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page .connector-revision-line,
|
||||||
|
.connector-governance-page .connector-run-actions,
|
||||||
|
.connector-knowledge-page .connector-revision-line {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: var(--space-3, 0.75rem);
|
||||||
|
margin-bottom: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page textarea,
|
||||||
|
.connector-knowledge-page textarea {
|
||||||
|
font-family: var(--font-family-mono, ui-monospace, monospace);
|
||||||
|
resize: vertical;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page .connector-definition-editor,
|
||||||
|
.connector-governance-page .connector-json-preview {
|
||||||
|
max-height: 34rem;
|
||||||
|
overflow: auto;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connector-governance-page .connector-json-preview {
|
||||||
|
background: var(--surface-subtle);
|
||||||
|
border-radius: var(--radius-md, 0.5rem);
|
||||||
|
padding: var(--space-4, 1rem);
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const moduleSource = readFileSync("src/module.ts", "utf8");
|
||||||
|
const page = readFileSync("src/features/ConnectorGovernancePage.tsx", "utf8");
|
||||||
|
const api = readFileSync("src/api/governedConnectors.ts", "utf8");
|
||||||
|
const knowledgePage = readFileSync("src/features/ExternalKnowledgePage.tsx", "utf8");
|
||||||
|
const knowledgeApi = readFileSync("src/api/externalKnowledge.ts", "utf8");
|
||||||
|
const serviceDeskPage = readFileSync("src/features/ExternalServiceDeskPage.tsx", "utf8");
|
||||||
|
const serviceDeskApi = readFileSync("src/api/externalServiceDesk.ts", "utf8");
|
||||||
|
|
||||||
|
assert.match(moduleSource, /"admin.sections": adminSections/);
|
||||||
|
assert.match(moduleSource, /connectors\.admin\.governed-configurations/);
|
||||||
|
assert.match(page, /<AdminPageLayout/);
|
||||||
|
assert.match(page, /<PageActionBar/);
|
||||||
|
assert.match(page, /refreshable/);
|
||||||
|
assert.match(page, /saveAction=/);
|
||||||
|
assert.match(page, /useUnsavedDraftGuard/);
|
||||||
|
assert.match(page, /<WorkspaceLayout/);
|
||||||
|
assert.match(page, /Adopt package revision/);
|
||||||
|
assert.match(page, /Protected paths/);
|
||||||
|
assert.match(page, /manual_review/);
|
||||||
|
assert.match(page, /quarantine/);
|
||||||
|
assert.match(page, /Run simulation/);
|
||||||
|
assert.match(page, /Review ambiguous connector result/);
|
||||||
|
assert.match(api, /idempotency_key/);
|
||||||
|
assert.match(api, /credential_ref/);
|
||||||
|
assert.match(moduleSource, /connectors\.admin\.external-knowledge/);
|
||||||
|
assert.match(knowledgePage, /<AdminPageLayout/);
|
||||||
|
assert.match(knowledgePage, /<PageActionBar/);
|
||||||
|
assert.match(knowledgePage, /refreshable/);
|
||||||
|
assert.match(knowledgePage, /saveAction=/);
|
||||||
|
assert.match(knowledgePage, /useUnsavedDraftGuard/);
|
||||||
|
assert.match(knowledgePage, /<WorkspaceLayout/);
|
||||||
|
assert.match(knowledgePage, /Run full backfill/);
|
||||||
|
assert.match(knowledgePage, /Preview migration/);
|
||||||
|
assert.match(knowledgePage, /outcome is unknown/);
|
||||||
|
assert.match(knowledgeApi, /\/knowledge/);
|
||||||
|
assert.match(knowledgeApi, /migration-dry-runs/);
|
||||||
|
assert.match(knowledgeApi, /\/publish/);
|
||||||
|
assert.match(moduleSource, /connectors\.admin\.external-service-desk/);
|
||||||
|
assert.match(serviceDeskPage, /<AdminPageLayout/);
|
||||||
|
assert.match(serviceDeskPage, /<PageActionBar/);
|
||||||
|
assert.match(serviceDeskPage, /refreshable/);
|
||||||
|
assert.match(serviceDeskPage, /saveAction=/);
|
||||||
|
assert.match(serviceDeskPage, /useUnsavedDraftGuard/);
|
||||||
|
assert.match(serviceDeskPage, /<WorkspaceLayout/);
|
||||||
|
assert.match(serviceDeskPage, /Restart full sync/);
|
||||||
|
assert.match(serviceDeskPage, /Submit revision-checked update/);
|
||||||
|
assert.match(serviceDeskPage, /outcome is unknown/);
|
||||||
|
assert.match(serviceDeskApi, /\/service-desk/);
|
||||||
|
assert.match(serviceDeskApi, /\/discover/);
|
||||||
|
assert.match(serviceDeskApi, /\/sync/);
|
||||||
|
assert.match(serviceDeskApi, /\/update/);
|
||||||
|
|
||||||
|
console.log("Connector governance UI structural contract passed.");
|
||||||
Reference in New Issue
Block a user