Compare commits

...
85 Commits
Author SHA1 Message Date
zemion 1cbf4acaf4 feat(mail): add governed JMAP mailbox sync and search
Module Package Release / publish-packages (push) Successful in 12s
2026-08-22 17:09:05 +02:00
zemion fd808336bc docs(mail): complete German help coverage
Module Package Release / publish-packages (push) Successful in 11s
2026-08-22 05:44:34 +02:00
zemion 218fef11f1 feat(mail): add governed POP3 legacy import
Module Package Release / publish-packages (push) Successful in 11s
2026-08-22 04:52:25 +02:00
zemion 93ecedf607 fix(i18n): complete cached mailbox translations 2026-08-21 17:36:17 +02:00
zemion cfd3847524 feat(mail): return exact messages from Quick Access
Module Package Release / publish-packages (push) Successful in 12s
2026-08-21 16:03:29 +02:00
zemion 2fe56fca00 fix(mail): align package and manifest versions 2026-08-21 00:46:06 +02:00
zemion 34bd5be8d4 feat: add governed Mail DSAR coverage 2026-08-20 23:58:21 +02:00
zemion a044fee379 feat: add reusable SMTP batch sessions 2026-08-20 17:39:00 +02:00
zemion 169b81d9db feat: integrate governed address contacts 2026-08-20 11:10:18 +02:00
zemion 2ad122056e feat(mail): complete Quick Access message launches 2026-08-20 02:39:41 +02:00
zemion 9ac8847559 feat(mail): complete mailbox display states 2026-08-19 23:03:46 +02:00
zemion e0e00d7000 feat(mail): map standard IMAP folders per profile 2026-08-19 21:27:42 +02:00
zemion 393331574a refactor(webui): adopt semantic page actions 2026-08-19 18:47:45 +02:00
zemion bb8c60abb9 Adopt semantic mail action layout 2026-08-19 14:26:26 +02:00
zemion c59c67bd18 style: align mail with shared WebUI foundations 2026-08-18 21:32:50 +02:00
zemion 761dd76b9e Adopt shared WebUI structural primitives 2026-08-18 13:17:31 +02:00
zemion 28f357e151 Adopt shared WebUI layout primitives 2026-08-18 10:42:52 +02:00
zemion cff428b475 feat: bridge observed messages to postboxes 2026-08-07 14:53:56 +02:00
zemion 77f44dfab5 feat: provision SMTP profiles from deployment receipts 2026-08-07 11:15:50 +02:00
zemion 2c5585ba99 Contribute Mail to Quick Access 2026-08-06 19:02:55 +02:00
zemion 62721d204d Release v0.1.18
Module Package Release / publish-packages (push) Successful in 11s
2026-08-05 21:07:45 +02:00
zemion e41d05914a Release v0.1.17
Module Package Release / publish-packages (push) Successful in 12s
2026-08-05 20:34:04 +02:00
zemion a040de4fe5 Release v0.1.16
Module Package Release / publish-packages (push) Successful in 11s
2026-08-05 19:52:11 +02:00
zemion 622f63b4b5 Release v0.1.15
Module Package Release / publish-packages (push) Successful in 11s
2026-08-04 15:22:07 +02:00
zemion b65431f437 Make package publication retries hash-safe 2026-08-04 14:32:19 +02:00
zemion c42a7816fd Define governed secure mail profiles 2026-08-04 14:02:21 +02:00
zemion a63d541f0e Harden module package publication 2026-08-04 14:02:10 +02:00
zemion 7c8cb21144 Partition mail retention by tenant 2026-08-04 09:29:36 +02:00
zemion d7dc4f6b04 Add protected package release workflow 2026-08-04 04:14:05 +02:00
zemion 72d6f8432c Index authorized mailbox cache changes in Search 2026-08-04 03:03:26 +02:00
zemion 7844d9cab4 Migrate Mail surfaces to interface patterns 2026-08-03 10:06:13 +02:00
zemion 5b5077cde7 Fence Mail provider and mailbox recovery effects 2026-08-03 05:00:19 +02:00
zemion ff3aa066ae Reconcile iCalendar replies from mail 2026-08-02 16:38:45 +02:00
zemion 5a3b13e26e feat: declare governed external provider state 2026-08-01 17:48:37 +02:00
zemion fe309dbff5 Add bounce mailbox processing 2026-07-31 22:48:07 +02:00
zemion 2721ee6542 Refine reusable mail profile credential editing 2026-07-31 02:48:56 +02:00
zemion 0cb6719b91 feat: provide governed notification email delivery 2026-07-30 17:42:07 +02:00
zemion b8029c24d6 feat: add durable mail delivery outbox 2026-07-30 14:26:53 +02:00
zemion dec5a4e350 feat: add reusable mail reference selectors 2026-07-29 14:16:29 +02:00
zemion 47c98b3957 Declare mail workspace View surfaces 2026-07-28 21:04:55 +02:00
zemion 3735027b3d Add hierarchical mail servers and credentials 2026-07-28 19:32:56 +02:00
zemion 2813c671c0 docs: update GovOPlaN repository links 2026-07-27 15:46:50 +02:00
zemion 2bccb78960 fix(mail-webui): clarify campaign profile wording 2026-07-22 04:06:25 +02:00
zemion 3e23029090 chore(mail): classify permission identifiers for scanners 2026-07-21 21:17:40 +02:00
zemion 64009471bb chore(mail): bump version to 0.1.10 2026-07-21 20:48:47 +02:00
zemion 2e5921b3ff fix(mail-webui): constrain focused profile updates 2026-07-21 20:48:47 +02:00
zemion 9f49591a98 feat(mail): enforce owned profile self-service 2026-07-21 20:48:46 +02:00
zemion 193898b00d docs: surface custom Mail profile task 2026-07-21 19:15:16 +02:00
zemion e6a3617a94 chore: declare Campaign access interface dependency 2026-07-21 17:51:29 +02:00
zemion 06e0fbd3c3 security: enforce actor-aware Mail profile access 2026-07-21 17:51:25 +02:00
zemion 9d1d9bfb58 docs: expand adaptive Mail guidance 2026-07-21 17:29:08 +02:00
zemion 0f1327638e docs: add multi-perspective Mail handbook 2026-07-21 17:29:01 +02:00
zemion 4a9201aeea security: scrub credentials when Mail profiles are deactivated 2026-07-21 17:12:49 +02:00
zemion 88d5012dea feat: enforce Mail-owned campaign transport boundary 2026-07-21 17:11:48 +02:00
zemion a86a779d5b Use shared settings target layout 2026-07-21 13:47:57 +02:00
zemion 26932d5ca8 refactor(webui): rely on central button layout 2026-07-21 13:32:10 +02:00
zemion b5b5cc8d78 fix(webui): align Mail Core peer lock 2026-07-21 13:27:04 +02:00
zemion 9c0946f8d2 Remove the unused Mail data-grid shim 2026-07-21 13:24:31 +02:00
zemion 260e579701 refactor(webui): use central mailbox icon action 2026-07-21 13:24:06 +02:00
zemion dea1e85bd3 Use the central mailbox work-surface CSS 2026-07-21 13:19:20 +02:00
zemion e4594c355d Reuse shared IMAP message listing path 2026-07-21 12:52:05 +02:00
zemion f3faa4dff9 Refactor mail profile lock validation 2026-07-21 12:52:01 +02:00
zemion ff750ed8bd refactor(webui): use central mailbox pagination 2026-07-21 12:36:49 +02:00
zemion e1f298f300 refactor(webui): use central profile selection 2026-07-21 12:34:39 +02:00
zemion 7cf5a69a85 refactor(webui): centralize mail profile controls 2026-07-21 12:23:51 +02:00
zemion e67da95df6 Harden mail connector network boundaries 2026-07-21 12:10:32 +02:00
zemion b2c013174a Log mailbox refresh failures safely 2026-07-21 03:16:23 +02:00
zemion 658c1e6d5d refactor(mail): focus profile editor on protocols 2026-07-20 20:06:42 +02:00
zemion 764577dc23 fix(mail): quote IMAP mailbox commands 2026-07-20 20:06:36 +02:00
zemion 2e906d9ddd fix(mail): rate-limit direct and fallback sends 2026-07-20 20:06:31 +02:00
zemion 79d3f45068 fix(mail): repair mailbox index schema drift 2026-07-20 20:06:26 +02:00
zemion b0337b2d26 intermittent commit 2026-07-14 13:22:11 +02:00
zemion 4865de5007 Harden mail profile validation 2026-07-11 18:37:51 +02:00
zemion 5371cb3ad4 Release v0.1.8 2026-07-11 16:49:03 +02:00
zemion 286b6cafd9 Release v0.1.7 2026-07-11 02:34:57 +02:00
zemion 47e3f9ec44 Sync GovOPlaN module state 2026-07-10 21:57:27 +02:00
zemion 30fd6e7e5f Use shared policy path help 2026-07-10 18:40:50 +02:00
zemion caaace14f3 Use capability-based module boundaries 2026-07-10 17:33:43 +02:00
zemion d5d5b89603 chore: sync GovOPlaN module split state 2026-07-10 12:51:22 +02:00
zemion b4b0c76455 Release v0.1.6 2026-07-07 15:55:37 +02:00
zemion 5f800a24db Release v0.1.5 2026-07-07 15:49:06 +02:00
zemion 3743b23613 Release v0.1.4 2026-07-02 14:59:52 +02:00
zemion e6e243c14d Release v0.1.3 2026-06-26 01:39:19 +02:00
zemion 3d82d7b32d Release v0.1.2 2026-06-25 19:58:20 +02:00
zemion 3cd8c45c42 Fix WebUI release peer metadata 2026-06-24 20:17:42 +02:00
126 changed files with 34327 additions and 1639 deletions
+35
View File
@@ -0,0 +1,35 @@
---
name: "Bug"
about: "Report a reproducible defect, regression, or incorrect behavior"
title: "[Bug] "
labels:
- type/bug
- status/triage
- module/mail
---
## Scope
- Repository:
- Area/module:
- Affected version or commit:
## Behavior
Expected:
Actual:
## Reproduction
1.
2.
3.
## Evidence
Logs, screenshots, traces, or failing test output:
## Verification Target
Command or workflow that should pass when fixed:
+1
View File
@@ -0,0 +1 @@
blank_issues_enabled: false
+27
View File
@@ -0,0 +1,27 @@
---
name: "Docs / workflow"
about: "Request documentation, process, or developer workflow changes"
title: "[Docs] "
labels:
- type/docs
- status/triage
- module/mail
- area/docs
---
## Scope
- Repository:
- Document or workflow:
## Current State
What is missing, unclear, duplicated, or stale?
## Desired State
What should the docs or workflow make clear?
## Verification Target
How should this be checked?
+32
View File
@@ -0,0 +1,32 @@
---
name: "Feature"
about: "Propose new user-visible behavior or platform capability"
title: "[Feature] "
labels:
- type/feature
- status/triage
- module/mail
---
## Problem
What user, operator, or developer problem should this solve?
## Proposed Capability
What should exist when this is done?
## Ownership
- Owning repository:
- Related module repositories:
- Extension point or integration boundary:
## Acceptance Criteria
- [ ]
- [ ]
## Verification Target
Command, scenario, or UI flow that should prove completion:
+28
View File
@@ -0,0 +1,28 @@
---
name: "Task"
about: "Track implementation, maintenance, or migration work"
title: "[Task] "
labels:
- type/task
- status/triage
- module/mail
---
## Objective
What needs to be completed?
## Scope
- Owning repository:
- In-scope:
- Out-of-scope:
## Checklist
- [ ]
- [ ]
## Verification Target
Command or manual check:
+25
View File
@@ -0,0 +1,25 @@
---
name: "Tech debt"
about: "Track cleanup, refactoring, risk reduction, or deferred engineering work"
title: "[Debt] "
labels:
- type/debt
- status/triage
- module/mail
---
## Current Cost
What does this make harder, riskier, slower, or more fragile?
## Desired Shape
What should the code, tests, or architecture look like afterwards?
## Constraints
What behavior, compatibility, or module boundary must be preserved?
## Verification Target
Focused checks that should pass:
+15
View File
@@ -0,0 +1,15 @@
## Issue
Closes #
## Summary
-
## Verification
-
## Notes
Follow-up issues:
+270
View File
@@ -0,0 +1,270 @@
name: Module Package Release
on:
push:
tags:
- "v*"
workflow_dispatch:
inputs:
release_tag:
description: Existing protected version tag to publish
required: true
type: string
jobs:
publish-packages:
runs-on: ubuntu-latest
env:
GITEA_REPOSITORY: ${{ gitea.repository }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
fetch-depth: 0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version: "22"
- name: Select and validate protected release tag
shell: bash
env:
REQUESTED_TAG: ${{ inputs.release_tag }}
TRIGGER_TAG: ${{ gitea.ref_name }}
run: |
set -euo pipefail
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
case "$tag" in
v[0-9]*.[0-9]*.[0-9]*) ;;
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
esac
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
tag_commit="$(git rev-list -n 1 "$tag")"
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
echo "Release tag is not contained in main" >&2
exit 1
}
git checkout --detach "$tag"
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
- name: Validate package versions
run: |
python - <<'PY'
import json
from pathlib import Path
import os
import re
import tomllib
tag = os.environ["RELEASE_TAG"]
expected = tag.removeprefix("v")
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
if project.get("version") != expected:
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
raise SystemExit("Python distribution name must use the govoplan-* namespace")
webui = Path("webui/package.json")
if webui.is_file():
package = json.loads(webui.read_text(encoding="utf-8"))
if package.get("version") != expected:
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
release = Path("webui/package.release.json")
if release.is_file():
release_package = json.loads(release.read_text(encoding="utf-8"))
if (
release_package.get("name") != package.get("name")
or release_package.get("version") != expected
):
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
PY
- name: Build immutable package artifacts
shell: bash
run: |
set -euo pipefail
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
rm -rf dist .package-webui
python -m build --wheel --outdir dist
python -m twine check dist/*.whl
if [[ -f webui/package.json ]]; then
mkdir .package-webui
cp -a webui/. .package-webui/
rm -rf .package-webui/node_modules .package-webui/dist
if [[ -f .package-webui/package.release.json ]]; then
cp .package-webui/package.release.json .package-webui/package.json
fi
node <<'NODE'
const fs = require("node:fs");
const path = ".package-webui/package.json";
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
for (const group of groups) {
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
if (!name.startsWith("@govoplan/")) continue;
if (typeof specifier !== "string") {
throw new Error(`${group}.${name} must use a string version`);
}
const packageSlug = name.slice("@govoplan/".length);
if (!packageSlug.endsWith("-webui")) {
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
}
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const gitTag = specifier.match(
new RegExp(
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
),
);
if (gitTag) {
packageJson[group][name] = gitTag[1];
continue;
}
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
throw new Error(
`${group}.${name} must resolve to an exact registry version for publication`,
);
}
}
}
delete packageJson.private;
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
NODE
npm pkg delete private --prefix .package-webui
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
fi
python - <<'PY'
import hashlib
import json
from pathlib import Path
import os
import subprocess
artifacts = []
for path in sorted(Path("dist").iterdir()):
if path.suffix not in {".whl", ".tgz"}:
continue
digest = hashlib.sha256(path.read_bytes()).hexdigest()
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
payload = {
"schema_version": "1",
"repository": os.environ["GITEA_REPOSITORY"],
"tag": os.environ["RELEASE_TAG"],
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
"artifacts": artifacts,
}
Path("dist/package-artifacts.json").write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
PY
- name: Retain package hash evidence
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
with:
name: module-packages-${{ gitea.ref_name }}
path: dist/package-artifacts.json
- name: Check immutable registry state
shell: bash
env:
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
run: |
set -euo pipefail
test -n "$PACKAGE_TOKEN"
python - <<'PY'
import hashlib
import json
import os
from pathlib import Path
import tomllib
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request, urlopen
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
token = os.environ["PACKAGE_TOKEN"]
def should_publish(kind, name, version, path):
package_url = "/".join(
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
)
request = Request(
package_url,
headers={"Accept": "application/json", "Authorization": f"token {token}"},
)
try:
with urlopen(request, timeout=30) as response:
files = json.load(response)
except HTTPError as exc:
if exc.code == 404:
print(f"{kind} package {name}=={version} is not published yet")
return True
raise
if not isinstance(files, list) or len(files) != 1:
raise SystemExit(
f"immutable {kind} package {name}=={version} has an unexpected file set"
)
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
if files[0].get("sha256") != expected_sha256:
raise SystemExit(
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
)
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
return False
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
wheels = tuple(Path("dist").glob("*.whl"))
if len(wheels) != 1:
raise SystemExit("release build must contain exactly one wheel")
publish_pypi = should_publish(
"pypi", str(project["name"]), str(project["version"]), wheels[0]
)
tarballs = tuple(Path("dist").glob("*.tgz"))
if len(tarballs) > 1:
raise SystemExit("release build must contain at most one npm package")
publish_npm = False
if tarballs:
webui = json.loads(
Path(".package-webui/package.json").read_text(encoding="utf-8")
)
publish_npm = should_publish(
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
)
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
PY
- name: Publish wheel and WebUI package
shell: bash
env:
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
run: |
set -euo pipefail
test -n "$PACKAGE_USERNAME"
test -n "$PACKAGE_TOKEN"
if [[ "$PUBLISH_PYPI" == 1 ]]; then
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
python -m twine upload --non-interactive \
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
dist/*.whl
else
echo "Exact wheel is already present; skipping immutable retry."
fi
shopt -s nullglob
webui_packages=(dist/*.tgz)
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
npmrc="$(mktemp)"
trap 'rm -f "$npmrc"' EXIT
chmod 600 "$npmrc"
printf '%s\n' \
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
> "$npmrc"
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
--ignore-scripts --access public \
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
elif (( ${#webui_packages[@]} )); then
echo "Exact WebUI package is already present; skipping immutable retry."
fi
+21
View File
@@ -326,3 +326,24 @@ cython_debug/
# Built Visual Studio Code Extensions
*.vsix
# GovOPlaN WebUI test output
webui/.mail-test-build/
# GovOPlaN shared ignore rules from govoplan-core
# Local WebUI test/build scratch directories
.component-test-build/
.module-test-build/
.policy-test-build/
.template-preview-test-build/
.import-test-build/
webui/.component-test-build/
webui/.module-test-build/
webui/.policy-test-build/
webui/.template-preview-test-build/
webui/.import-test-build/
*.db
# GovOPlaN local runtime state
runtime/
webui/.module-test-build/
webui/.component-test-build/
+50
View File
@@ -0,0 +1,50 @@
# GovOPlaN Mail Codex Guide
## Documentation Contract
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
- Keep feature content here; `govoplan-docs` projects it without importing Mail internals.
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
## Scope
This repository owns the `mail` module: SMTP/IMAP profiles, mail profile policy, encrypted mail credentials, SMTP sending, IMAP append and mailbox access, mock mail infrastructure, backend module manifest, and `@govoplan/mail-webui`.
Core owns auth, tenants, RBAC, database/session primitives, CSRF/API helpers, shared components, and shell layout. Campaign/files integrations must remain optional and go through core module metadata or capabilities.
## Local Commands
Install and run through core:
```bash
cd /mnt/DATA/git/govoplan-core
./.venv/bin/python -m pip install -r requirements-dev.txt
```
Focused backend tests:
```bash
cd /mnt/DATA/git/govoplan-core
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
```
Focused WebUI tests:
```bash
cd /mnt/DATA/git/govoplan-mail/webui
PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:mail-ui
```
For combined checks, run:
```bash
cd /mnt/DATA/git/govoplan
tools/checks/check-focused.sh
```
## Working Rules
- Keep mail behavior in this module, not core.
- Do not require campaign or files imports for mail-only startup.
- Shared WebUI components belong in core; mail WebUI should consume them through `@govoplan/core-webui`.
- Prefer mailbox/IMAP tests and targeted API smoke tests before full-suite runs.
+119 -5
View File
@@ -1,20 +1,107 @@
# govoplan-mail
GovOPlaN Mail is the mail transport module. It owns reusable SMTP/IMAP profile management, mail profile policy enforcement, mock mail infrastructure, and the mail WebUI package.
<!-- govoplan-repository-type:start -->
**Repository type:** module (domain).
<!-- govoplan-repository-type:end -->
GovOPlaN Mail is the mail transport module. It owns reusable SMTP/IMAP/JMAP profile management, an explicitly enabled legacy POP3 import path, mail profile policy enforcement, mock mail infrastructure, and the mail WebUI package.
## Ownership
This repository owns:
- backend module manifest `mail`
- mail permissions such as `mail:profile:read`, `mail:profile:write`, `mail:profile:use`, `mail:profile:test`, and `mail:mailbox:read`
- SMTP/IMAP profile models, policy checks, encrypted credential storage, and profile resolution
- mail permissions such as `mail:profile:read`, `mail:profile:write_own`, `mail:profile:write`, `mail:profile:use`, `mail:profile:test`, and `mail:mailbox:read`
- SMTP/IMAP/JMAP profile models, dedicated legacy POP3 sources, policy checks, encrypted credential storage, and profile resolution
- SMTP send and IMAP append adapters, including mock transports for development
- development mock mailbox endpoints used by test-send flows
- WebUI package `@govoplan/mail-webui` with profile management, policy management, and read-only mailbox components
- WebUI package `@govoplan/mail-webui` with profile management, policy management, read-only mailbox components, and governed POP3 import
Core owns auth, tenants, RBAC evaluation, database/session primitives, secret helpers, CSRF/API helpers, and shell layout.
Mail publishes `privacy.dsar.mail` for Core's governed data-subject-request
workflow. It isolates matching mailbox header parties and returns bounded index,
personal-profile, delivery, reconciliation, bounce, and imported-message metadata. SMTP/IMAP/JMAP/POP3
configuration and credentials, encrypted messages and envelopes, source UIDL
and folder/UID
locators, worker and idempotency state, diagnostics, and opaque evidence are
excluded. Delivery and bounce outcomes remain retained evidence; mailbox and
profile changes require coordinated Mail and external-provider review, so the
provider does not perform direct erasure.
## Profile and credential ownership
Mail profiles are separate governed definitions. Mail owns their SMTP/IMAP/JMAP
endpoints, encrypted credentials, tests, scope, and policy. Consumers such as
Campaign store only a stable profile identifier and resolve the authorized,
active profile through `mail.campaign_delivery`; they never copy or override
transport settings or credentials in their own JSON.
The campaign capability returns read-only availability flags and random,
persisted Mail-owned transport revisions without decrypting secrets. Effect calls perform authorization,
revision comparison, credential resolution, policy checks, and SMTP/IMAP
effects inside Mail. Consumer-visible outcomes are sanitized: provider banners,
raw response bytes, hosts, account identities, and credentials are not returned.
Mail also owns a durable delivery-command outbox for effects that do not
already have a consumer-owned job ledger, including Campaign report messages.
It persists the command and attempt before SMTP, binds idempotency keys to
canonical request hashes, distinguishes partial refusal and unknown outcome,
and requires explicit evidence-backed reconciliation before any deliberate
resend. Business readers receive only counts and sanitized state; recipient
refusal details require `mail:delivery:diagnostic`.
Synchronous Campaign batches now preflight DNS, egress, connectivity, TLS, and
authentication before the first message, then reuse the authorized SMTP
connection for the bounded batch. A health check precedes reuse; a stale
connection is reopened before the next message, while a connection loss after
DATA starts remains outcome-unknown and is never replayed automatically.
Systemic authentication, sender, and connectivity failures pause remaining
Campaign jobs instead of producing one failure per recipient. Deployment
operators can disable reuse or bound connection lifetime and reconnects with
`GOVOPLAN_SMTP_BATCH_REUSE`, `GOVOPLAN_SMTP_BATCH_MAX_MESSAGES`,
`GOVOPLAN_SMTP_BATCH_RECONNECT_ATTEMPTS`, and
`GOVOPLAN_SMTP_BATCH_HEALTH_CHECK`.
SMTP effects decrypt only SMTP credentials; Sent-folder effects decrypt only
IMAP credentials. A connection loss after an effect starts is surfaced as an
unknown outcome. Campaign does not automatically retry an unknown IMAP append,
preventing silent duplicate Sent copies while an operator inspects the mailbox.
Every current outbox, Campaign SMTP, and Campaign Sent-folder attempt also
starts a Mail-owned Core recovery operation under a stable per-attempt effect
identifier before contacting the provider. Evidence contains only message,
address, and folder digests plus bounded outcome counts. A completed effect is
never replayed to repair caller state; unknown outcomes require explicit
provider-backed reconciliation.
Read-only mailbox folder/message indexing and bounce/calendar-reply scans use
distributed recovery fences. Cache rows or source cursors commit before an
independent verification closes the operation. A failed read rolls back and is
safe to repeat because these paths never move, delete, flag, or otherwise
mutate provider messages.
The existing SMTP/IMAP credential-inheritance policy remains part of the Mail
policy model for compatibility. Campaign delivery requires effective
inheritance: a policy that requires campaign-local credentials now fails
closed with guidance to store those credentials on a Mail profile and enable
inheritance.
Deleting a profile deactivates its non-secret tombstone metadata and scrubs
its encrypted SMTP, IMAP, JMAP, and POP3 credentials immediately in the same transaction as
a non-secret audit event. Destructive module retirement applies the same rule
to every remaining profile before any Mail table is dropped; a scrub or audit
failure blocks retirement.
Personal profile self-service is a distinct authorization path. An actor with
`mail:profile:write_own` can mutate only a user-scoped profile whose scope id is
their current tenant membership id; `mail:secret:manage_own` applies the same
ownership check to credentials. Neither scope permits profile-policy changes or
management of tenant, group, campaign, system, or another user's profiles.
Changing an SMTP/IMAP endpoint while a stored password remains is a secret
operation and requires the matching credential permission. Deactivating a
credential-free profile needs only profile-write authority; if a password will
be scrubbed, credential authority is required and the deletion is audited.
## Development
Install through the core environment:
@@ -46,9 +133,36 @@ Frontend package:
@govoplan/mail-webui
```
The campaign module depends on this module for sending, append-to-Sent behavior, profile selection, and read-only mailbox inspection.
The campaign module consumes `mail.campaign_delivery` for authorized runtime
profile resolution, sending, append-to-Sent behavior, profile selection, and
policy checks. Mail does not
import campaign internals; campaign-scoped policy and owner context are resolved
through the core `campaigns.mailPolicyContext` capability when the campaign
module is installed.
Development mailbox routes are registered by the mail module only when the
core runtime is in `dev` mode and `dev_mailbox_api_enabled` is enabled. Core
does not contribute these routes directly.
JMAP is available as an opt-in, read-only mailbox transport after the stable
IMAP baseline. It discovers RFC 8620/8621 capabilities, lists folders,
performs server-side message search and pagination, reads bounded message
bodies and attachment metadata, and exposes bounded incremental Email changes.
IMAP behavior is unchanged, and SMTP remains the send transport. JMAP Session
and advertised API origins are governed independently and credentials stay in
Mail's encrypted envelopes. The explicitly enabled POP3 slice remains limited
to bounded, encrypted, duplicate-safe legacy import. The protocol boundary is
documented in [docs/MAIL_PROTOCOL_ROADMAP.md](docs/MAIL_PROTOCOL_ROADMAP.md).
The same roadmap records the
approved S/MIME-first, OpenPGP-additional message-protection profile and its
no-silent-downgrade requirement.
Platform RBAC and governance rules are documented in `govoplan-core/docs/`.
The [Mail handbook](docs/MAIL_HANDBOOK.md) provides the adaptive user,
governance, technical, security, and operations perspectives.
The [Mail interface pattern inventory](docs/INTERFACE_PATTERN_MIGRATION.md)
records the route, administration, state, accessibility, consequence, and
privacy contracts for the Mail WebUI.
## Release packaging
+71
View File
@@ -0,0 +1,71 @@
# Mail Interface Pattern Migration
This inventory records the Mail-owned part of the GovOPlaN interface pattern
language. Core owns the shell and shared components; Mail owns the transport,
mailbox, policy, and delivery-evidence consequences described here.
## Surface inventory
| Surface | Primary task | Archetype | Consequence | Pattern evidence |
| --- | --- | --- | --- | --- |
| `/mail` folder, message, and preview panes | Browse and inspect an authorized mailbox without changing provider state | Directory/explorer | Medium because message metadata and content are private, although navigation is read-only | Full-height three-pane workspace, bounded paging, stable keyboard selection, contextual Help Center link, explicit no-profile blocker |
| `/mail` toolbar, page filter, and pagination | Select a profile, refresh bounded indexes, and find a message on the current page | Explorer actions and local filtering | Low for refresh; medium for provider access | Shared actions expose loading/profile/folder blockers; profile transport summary is non-secret; loading and errors use Core components |
| System/tenant/group/user/campaign profile surfaces | Compare profiles, protocol servers, reusable credentials, status, and scope | Administration/configuration | High because endpoints, credentials, and inheritance control external communication | Shared `ConnectionTree`, stable row actions, textual status, permission/target blockers, and contextual admin help |
| Profile creation and focused profile/server/credential editors | Create a governed transport identity or edit one hierarchy object | Guided setup plus adaptive create/edit | High because saving may enable provider access or replace encrypted credentials | Shared `Dialog` and `StageRail` for multi-object setup; focused edit modes show only the selected hierarchy object; field help, connection tests, unsaved-draft guard, and disabled-save reasons |
| Mail profile policy card | Narrow visible profiles, lower-scope definitions, hosts, senders, and recipients | Effective-policy editor | High because inherited allow/deny rules govern delivery and lower scopes | Shared policy rows, typed selectors, source path, locked/read-only blocker, dirty-save state, and contextual policy help |
| `/mail/bounces` watcher table | Configure bounded IMAP evidence sources and run an explicit scan | Operational administration | High because it accesses a provider mailbox and changes durable evidence cursors | Shared `PageLayout`, `ContentGrid`, `DataGrid`, status, loading/error feedback, field help, actionable no-profile blocker, and stable row actions |
| `/mail/bounces` observation table | Inspect correlated or unmatched delivery-status evidence | Evidence/reporting | Medium because recipient and diagnostic data may be sensitive | Bounded sanitized rows, textual status, filters, correlation state, and no raw bounce body |
| Bounce watcher removal | Stop future scans while retaining evidence | Destructive confirmation | Medium and reversible by recreating the watcher; observations are retained | Shared `ConfirmDialog` states the immediate consequence and retained evidence |
| `mail.profiles` and credential-reference capabilities | Let another module select or validate Mail-owned transport without receiving secrets | Governed capability composition | High because the selected identity can perform external effects | Stable references and Core capability boundaries; no sibling-private WebUI import; authorization and credential resolution remain Mail-owned |
## State and consequence contract
- Loading, success, error, empty results, permission blockers, and destructive
confirmation use Core components. Mail does not reproduce the shell.
- Bounce processing now delegates its page inset, sticky responsive heading,
route actions, alert regions, loading boundary, scrolling, and help audience
to Core `PageLayout`; Mail retains only watcher and evidence semantics.
- A target-dependent profile surface cannot load until a concrete user, group,
or campaign is selected. The blocker identifies the responsible actor and
destination instead of silently hiding the editor.
- Profile, server, credential, policy, mailbox refresh, connection-test, and
bounce actions remain visible when structurally relevant. Missing authority,
required input, or an in-progress operation is exposed through a focusable
disabled-action explanation.
- Connection tests never save the draft and explicitly require the relevant
hostname. Save remains the committing action. Policy save is unavailable
until a local change exists.
- Effective policy keeps inherited source/provenance visible. A locked parent
limit or governed workflow cannot be represented as an editable local value.
- Removing a bounce watcher retains observations and delivery evidence.
Profile/server deactivation and credential unlinking use confirmations whose
copy distinguishes retained reusable credentials from scrubbed owned secrets.
- Mailbox browsing is read-only. Listing or previewing must not mark messages
read, move, delete, reply, or expose unbounded content.
## Accessibility, responsive, and privacy evidence
Shared `Dialog` owns focus entry, Escape handling, focus containment, and focus
return. Toolbar and form DOM order is keyboard order; mailbox rows support
Enter and Space, arrow navigation is bounded to the visible page, and disabled
reasons are keyboard-focusable. Status always has text in addition to color.
Contextual links identify their destination to assistive technology.
Profile/policy grids collapse to one column below 900 px. The mailbox changes
from three panes to two below 1250 px and to a single-column toolbar and message
rows below 760 px while preserving source order and independent scroll regions.
Long identities and transport summaries wrap or ellipsize inside stable bounds.
Profile and mailbox APIs return non-secret transport metadata and bounded
message content only. Passwords are write-only and rendered only as a saved
state marker. Bounce observations contain bounded sanitized diagnostics and a
raw digest, not the raw provider message. Optional Campaign, Addresses, Audit,
Calendar, and Notifications integrations remain capability-driven; Mail WebUI
does not import their private packages.
The focused structural test guards shared components, contextual help,
actionable blockers, confirmation, optional-module boundaries, responsive
rules, and the absence of browser-native confirmation. Mail backend tests,
Core component tests, manifest-shape checks, module permutations, structural
localization audit, theme contract, and bundle budgets provide integration
evidence.
+585
View File
@@ -0,0 +1,585 @@
# Mail Handbook
## Purpose and status
This handbook is the canonical multi-perspective description of the GovOPlaN
Mail module. Mail is a governed transport and mailbox capability. It is not a
campaign definition, contact database, or general records store.
| Perspective | Start here |
| --- | --- |
| User selecting a profile or reading a mailbox | [User tasks](#user-tasks) |
| Mail profile administrator | [Profile administration](#profile-administration) |
| Policy or tenant administrator | [Policy hierarchy](#policy-hierarchy) |
| Delivery operator | [Operations and recovery](#operations-and-recovery) |
| Integrator | [Capability contract](#capability-contract) |
| Security or audit reviewer | [Security, deletion, and audit](#security-deletion-and-audit) |
| Release reviewer | [Acceptance checklist](#acceptance-checklist) |
See also [Mail protocol roadmap](MAIL_PROTOCOL_ROADMAP.md) and the Campaign
[Mail profile boundary](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/src/branch/main/docs/MAIL_PROFILE_BOUNDARY.md).
## Domain ownership
Mail owns:
- reusable SMTP/IMAP/JMAP profile definitions, dedicated legacy POP3 sources, and scope;
- encrypted SMTP/IMAP/JMAP/POP3 credentials and safe credential replacement;
- effective profile policy and visibility/authorization decisions;
- connection tests and protocol adapters;
- SMTP send and IMAP append operations exposed to consumers;
- read-only IMAP/JMAP mailbox folder/message access, bounded indexes, and encrypted
pending-review records imported from legacy POP3 sources; and
- the transport sanitization boundary and throttling behavior. A general
Mail-owned provider-attempt/diagnostic ledger remains planned.
Consumers own their business intent. For example, Campaign owns message
content, recipients, approval, jobs, and delivery evidence, while Mail owns the
profile and performs the provider operation. A consumer stores only a stable
Mail profile reference and non-secret evidence returned by Mail. It never
receives decrypted credentials or a raw resolved transport configuration.
Core owns authentication, tenant context, permission evaluation, database
session primitives, cryptographic secret helpers, audit infrastructure, and the
module registry. Optional consumers provide narrow context through capabilities;
Mail does not import their ORM or service implementations.
## Deployment configuration packages
Mail registers the `mail.configuration` capability for `smtp_profile`
fragments. The provider reads the validated `mail.smtp` entry from the
installer-generated infrastructure capability receipt. Receipt host and port
are authoritative; the generic package workflow asks for missing non-secret
transport fields such as security mode. An external relay may operate without
authentication, or the operator may select an existing credential-envelope id.
Inline usernames, passwords, tokens, and secret values are rejected.
Tenant scope is the default. A system-scoped profile requires system settings
or governance write authority. The fragment's stable slug is its idempotency
identity: an absent profile is created, an exact profile is skipped, and a
conflicting profile is preserved unless the reviewed fragment explicitly sets
`on_conflict` to `update`. Credential bindings are added idempotently and are
never removed merely because a package omits a credential reference.
Preflight does not prove SMTP reachability. After apply, use the normal Mail
profile test and Ops health surfaces. If the receipt says SMTP is unavailable,
is invalid, or is not mounted, import is blocked with an operator-facing
resolution instead of creating a partial profile.
Configuration packages remain SMTP-focused. A POP3 legacy source is a deliberate
operational migration action and is not silently exported, cloned, or enabled by
an SMTP profile package.
## Interface patterns and unavailable actions
Mail uses the platform's shared explorer, connection tree, adaptive form,
effective-policy, dialog, status, alert, and confirmation components. The
module-owned surface inventory and consequence classification are recorded in
[Mail interface pattern migration](INTERFACE_PATTERN_MIGRATION.md).
An unavailable action remains visible when it belongs to the current task. Its
hover/focus explanation identifies the missing field, active operation, or
permission. Larger blockers state what must change, who can change it, and
where to go. Contextual Help Center links resolve to the configured Docs module
when installed and otherwise use the public documentation fallback.
Mailbox reads, profile reloads, connection tests, policy saves, bounce scans,
and destructive actions each expose their current loading or blocked state.
Removing a bounce watcher and deactivating profile hierarchy items use shared
confirmation dialogs. Removing a watcher retains existing observations;
deactivating a profile may scrub Mail-owned credentials as described below.
## Concepts
### Profile
A profile is a reusable, named delivery identity with optional SMTP and IMAP
configuration and one or more opt-in JMAP mailbox endpoints. It can additionally own a dedicated POP3 endpoint for an
explicit legacy-import workflow. It has a stable id, lifecycle state, scope,
owner context, and non-secret connection metadata. Passwords are write-only
encrypted values and are never returned through list/read/capability responses.
An IMAP server may map the standard Inbox, Sent, Drafts, Trash, Archive, and
Junk roles to exact provider folder names. These mappings belong to the reusable
profile/server. Empty roles retain automatic behavior. The historical
`imap.sent_folder` value is read as the Sent mapping and remains synchronized
for compatibility; a Campaign-specific Sent override still wins for that
Campaign.
Profiles may be scoped to system, tenant, user, group, or campaign context.
Scope controls where a profile can be discovered; effective policy can narrow
that further. A visible profile is not automatically authorized for every
operation: using, testing, managing, and managing secrets are separate rights.
### Policy
Mail policy controls approved profile ids, which lower scopes may define
profiles, allowed/denied SMTP/IMAP/JMAP hosts and addressing patterns, credential inheritance,
and lower-level limits. The effective result is assembled from applicable
system, tenant, user/group, and campaign context. Denials and locked parent
limits cannot be relaxed by a lower scope.
### Transport identity and revision
Mail owns opaque random revisions for relevant SMTP/IMAP/JMAP identity and
configuration. They are concurrency tokens, not deterministic hashes that a
consumer could use to guess a host or account. Credentials are excluded. A
consumer can freeze these revisions at build time and ask Mail to require the
same identity when the effect is later performed. Credential rotation alone
therefore need not invalidate prepared work, while a host/account/protocol
identity change does.
The revision is evidence and a concurrency guard, not a substitute for
authorization. Mail re-evaluates profile activity, visibility, policy, and
revision immediately before it resolves credentials and performs the
effect.
POP3 imports pin the endpoint/credential transport revision at preview time. A
changed revision or a missing provider UIDL stops import and requires a fresh
preview. POP3 does not participate in the ordinary mailbox folder/message
projection.
### Provider outcomes
SMTP acceptance, partial or complete recipient refusal, temporary/permanent
failure, connection loss, and unknown outcome are distinct. IMAP append is a
separate operation and outcome. Ordinary Campaign recipient delivery retains
its Campaign-owned job ledger. Mail owns an encrypted durable command and
attempt ledger for report delivery and other effects that do not have such a
consumer ledger.
The Mail worker commits an attempt before it starts SMTP and then commits an
effect-start marker before opening the provider effect. Worker redelivery may
recover a stale pre-effect claim, but an accepted, partially accepted,
in-progress-after-effect, or unknown command is never sent automatically.
Unknown outcomes require a separately authorized reconciliation with an
external evidence reference. A deliberate resend creates a new command with a
new idempotency key and links it to the prior command.
## User tasks
### Choose a profile
1. Open the task that requires mail, such as Campaign **Mail settings**.
2. Choose from the profiles visible and authorized for the current tenant,
owner, group, and task context. Never enter or copy a profile id manually
when the UI can present a picker.
3. Review the safe summary: name, scope, active state, SMTP/IMAP/JMAP availability,
and policy-relevant sender identity. Credentials and raw provider internals
are not visible.
4. Save the reference in the consuming module. If ownership or policy changes,
select and validate again.
The user needs `mail:profile:use`; reading profile summaries and testing may
require their own scopes. The consuming module still requires its own action
permission.
### Test a profile
An authorized profile test verifies connection and authentication for the
selected active/visible SMTP, IMAP, or JMAP profile using Mail-owned credentials. The
consumer-use path evaluates effective Mail policy separately. Use a
non-production provider and mailbox first. A successful connection test does
not prove policy authorization for a later Campaign context, deliverability,
recipient acceptance, SPF/DKIM/DMARC alignment, or future availability.
Testing a saved profile requires both `mail:profile:test` and
`mail:profile:use`, and the profile must be active. Profile creation or test
authority alone is not enough.
Raw settings test endpoints accept new settings only for actors who may both
test profiles and manage secrets. They are an administration aid, not a way for
ordinary consumers to bypass reusable profiles.
### Read a mailbox
The current mailbox UI and API are read-only. An authorized user can list IMAP
or JMAP folders, page through messages, and inspect a bounded full message.
IMAP folder names are parsed and quoted defensively; Sent-folder discovery uses
provider flags and common names. JMAP discovers the Session and Mail account,
uses `Mailbox/get` hierarchy and roles, runs text search with `Email/query`, and
uses `Email/get` for bounded summaries/details. `Email/changes` exposes a
bounded incremental cursor; an expired state tells the caller to perform a full
refresh. The list exposes the provider's `Seen`/`$seen` state as a
read/unread indicator without changing it. It also labels whether the current
page came directly from the provider, from the bounded mailbox index, or from
an index while a refresh is in progress, including the index timestamp when
available.
Message HTML is displayed only in the shared sandboxed message component.
Remote URLs and active markup are removed, embedded `data:`/`cid:` image
references remain isolated, and plain text is always available when supplied.
Attachments and provider/content failures remain explicit rather than being
silently interpreted as an empty message.
Mailbox access requires both `mail:mailbox:read` and `mail:profile:use`. It must
not mutate read/unread, delete, move, or reply state. Message responses are
bounded by the endpoint response/body policy; ordinary UI should avoid loading a
whole large mailbox or attachment merely to show a list.
JMAP is opt-in per server endpoint. Its authenticated Session URL is governed
by `jmap_hosts`; an advertised API URL on another origin fails closed unless
that origin is explicitly listed for the endpoint. Bearer tokens or Basic
credentials are stored only in Mail credential envelopes. This slice does not
send through JMAP, mutate provider state, download attachment binaries, or add
calendar/contact/thread features.
### Import a legacy POP3 mailbox
POP3 is available only for bounded migration from a legacy server that cannot
provide IMAP or JMAP. It is disabled by default and is not a replacement for
the read-only mailbox UI.
1. An actor with profile-write, secret-management, and `mail:pop3:manage`
authority opens **Legacy POP3 import**, selects a Mail profile, and creates a
dedicated source. The UI stages the endpoint disabled, stores its encrypted
username/password credential, and enables it only after both operations
succeed.
2. The administrator explicitly enables legacy import, sets TLS mode, timeout,
maximum message and batch sizes, and preview body lines, and tests connection,
authentication, TLS, and provider message count. Plain transport remains
subject to deployment egress/security policy and should not be used across
an untrusted network.
3. An operator with `mail:profile:use` and `mail:pop3:import` refreshes a live
preview of at most 100 messages. Preview sends no `DELE`, changes no flags,
and exposes only bounded headers/body text. A server without stable UIDL
identifiers is rejected. If `TOP` is unavailable, Mail uses `RETR` only
inside the configured size bound and suppresses an oversized preview.
4. The operator selects messages. Mail downloads within the size gate and
creates encrypted local `pending_review` records. Tenant, profile, endpoint,
and UIDL form the duplicate boundary. Raw content never appears in list,
provider-state, audit, or DSAR output.
5. Source messages remain untouched by default. Delete-after-import requires
the endpoint's separate `allow_delete_after_import` policy, the operator's
`mail:pop3:delete` permission, an explicit per-batch choice, and destructive
confirmation. Mail commits the local import plus `mail.pop3.imported` audit
evidence before sending `DELE`. It separately records
`mail.pop3.source_deletion`; disconnect during `QUIT` is outcome-unknown and
must be reconciled before another destructive attempt.
The supplied **Mail legacy import operator** role can test, preview, and import
without deleting. The **Mail profile administrator** role also contains source
management and destructive-delete permissions; deployments should remove or
split `mail:pop3:delete` when operators must never delete provider messages.
Imported records follow configured Mail/records retention and require manual
review for a data-subject request or deletion decision.
Contextual help is available from the page and from each policy-sensitive
source setting, credential field, size limit, import action, and destructive
confirmation. Press F1 while a control has focus to open the Mail-owned German
reference for the exact POP3 context; the same topic remains available through
the page help action.
## Profile administration
### Roles
The supplied templates are:
- **Mail profile user:** read/use/test approved profiles and read permitted
mailboxes without reading secrets.
- **Mail profile self-service user:** additionally create, edit, deactivate,
and manage credentials only for the current account's own user-scoped
profiles, subject to the effective Mail policy.
- **Mail profile administrator:** additionally create/update profiles and
create/replace encrypted credentials across tenant-owned scopes, configure
legacy POP3 imports, and—unless the template is narrowed—request source
deletion after import.
- **Mail legacy import operator:** test approved POP3 sources and preview/import
messages without permission to delete them at the provider.
The specific permissions are `mail:profile:read`, `mail:profile:use`,
`mail:profile:test`, `mail:mailbox:read`, `mail:profile:write_own`,
`mail:secret:manage_own`, `mail:profile:write`, `mail:secret:manage`,
`mail:pop3:manage`, `mail:pop3:import`, and `mail:pop3:delete`.
The `_own` permissions are enforced against the authenticated membership id and
never authorize a tenant, group, campaign, system, or another user's profile.
They also do not authorize profile-policy changes. System-scoped definitions
use the corresponding system settings authority. Keep secret management
separate when an institution wants profile metadata administrators not to know
or replace credentials.
That separation is fail-closed for transport rebinding: changing an SMTP or
IMAP host, port, or security mode while the profile retains a stored password
requires the matching secret-management permission. A credential-free profile
can be deactivated with profile-write authority alone; deactivation that
scrubs a stored password also requires secret-management authority and records
the deletion in the audit log.
### Create or change a profile
The configured Help Center exposes **Create a custom Mail profile** only when
the current actor has broad or self-service profile-write authority and the
effective user-scope policy permits user profiles. It states the active SMTP/IMAP/JMAP hostname
allow-list groups and deny rules, plus the actor's separate credential, test,
use, and approval requirements. The Settings task creates in the current
account's user scope. Grant `mail:profile:write_own` for self-service;
`mail:profile:write` remains broad profile administration authority.
1. Choose the narrowest suitable scope and a stable, descriptive name/slug.
2. Configure SMTP and optional IMAP, or add an optional JMAP Session endpoint after creating the profile. Set TLS mode, account identity, standard folder
mappings, and timeouts. Folder discovery proposes provider-visible Inbox,
Sent, Drafts, Trash, Archive, and Junk names without mutating the mailbox.
Sender/envelope/recipient constraints belong to
effective Mail policy; Campaign rate limits remain delivery configuration.
3. Enter credentials only in the dedicated credential fields. Returned profile
data indicates whether credentials are configured without returning them.
4. Save and run SMTP/IMAP/JMAP tests against a non-production target.
5. Verify the effective policy for every intended consumer context.
6. Communicate changes that alter the non-secret transport identity; prepared
consumer snapshots will deliberately stop until rebuilt.
An update that omits a password preserves the current encrypted password. A
credential replacement never depends on reading the old cleartext value back.
### Delete a profile
Profile deletion is immediate for Mail-owned secrets and audit evidence:
1. Mail clears encrypted SMTP, IMAP, JMAP, and POP3 credential envelopes in the same transaction.
2. It deactivates the remaining non-secret tombstone state so historical stable
references can fail safely rather than resolve to another profile.
3. When owned encrypted secrets existed, it emits
`mail.profile_credentials_deleted` with profile id, scope, reason, and the
kinds of secret removed—never values, usernames/passwords, or raw
configuration.
4. If scrubbing or audit insertion fails, the transaction rolls back and the
deletion fails.
5. Repeating deletion after the secrets are gone is an idempotent no-op for the
secret-deletion effect.
Consumers that still reference the profile fail closed at validation or
execution. Deletion does not silently rebind them to a default profile.
## Policy hierarchy
Effective policy is contextual. Administrators should document:
- which profile ids are approved globally or for a tenant;
- whether tenant, user, group, or campaign-scoped profiles may be created;
- allowed and denied SMTP/IMAP/JMAP hosts;
- permitted From, envelope sender (including bounce address), and envelope
recipient-domain patterns;
- whether SMTP/IMAP credentials inherit from the reusable profile; and
- which lower-level settings are locked by a parent policy.
Campaign delivery requires reusable profile credentials. A legacy policy that
requires campaign-local credentials fails closed with guidance to store them on
the Mail profile and enable effective inheritance. This preserves compatibility
of the policy model without reopening a consumer-owned secret store.
Policy reads are available through system/tenant/context routes to suitably
authorized actors. Adaptive Docs exposes a safe explanation of the effective
tenant posture; it does not expose credential material.
## Operations and recovery
### Deployment egress policy
Private-network connector access is controlled deployment-wide by
`GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS`. Whether private or public targets
are allowed, SMTP, IMAP, and legacy POP3 resolve, validate, and connect to the
exact approved address records at connection time while retaining the original
hostname for TLS SNI and certificate verification. A DNS change cannot redirect
the socket after validation.
Transports that cannot pin every connection peer or revalidate protocol-managed
redirects/referrals must fail before client construction. Do not weaken this
rule to make a target test pass; change the connector implementation or the
target topology.
### Response and time bounds
The shared defaults are 16 MiB for structured connector responses and 512 MiB
for file transfers. Mailbox full-message fetch additionally uses bounded IMAP
ranges and the applicable deployment limit. Operators may lower limits for
their environment. A declared or streamed over-limit response fails visibly
rather than being retained partially as if complete.
Configure connection/read timeouts and provider-specific message/attachment
limits. A timeout after the SMTP effect may be an unknown outcome; it must not
be flattened into a safe-to-retry connection failure.
### Throttling
Mail rate limiting uses a Redis lock and next-allowed timestamp across worker
processes when worker mode is enabled. Direct development uses a process-local
limiter. If Redis fails in worker mode, the current implementation silently
falls back to a limiter that protects only that process; it does not provide
cluster-wide coordination or emit its own health signal. Production operators
must therefore monitor Redis independently and should stop or restrict
multi-worker delivery during an outage until explicit degradation telemetry is
implemented.
### SMTP/IMAP incidents
Current SMTP delivery and Sent-folder APPEND paths start a Mail-owned Core
recovery operation before the network effect. Outbox attempts use the durable
command and attempt number; Campaign jobs and single-message actions pass their
own stable attempt identifiers through the versioned capability. A matching
completed identifier is never sent or appended again merely to reconstruct
caller state. Provider acceptance, definitive rejection, and outcome-unknown
states are recorded independently of the consuming transaction.
Mailbox folder/message indexing and configured bounce/calendar-reply scans are
read-only provider operations. They acquire distributed per-profile/folder or
per-source fences, commit bounded projection state, and verify that state in an
independent session. A failed read is rolled back and can be repeated; it is not
treated as an unknown provider mutation.
1. Stop new consumer work if duplicate effects or credential compromise are
possible.
2. Preserve safe Mail, consumer-job, worker, and provider evidence.
3. Rotate credentials in Mail. Rebuild consumers only if the revisioned
non-secret identity also changed.
4. Treat post-command connection loss as potentially unknown until provider
evidence establishes whether SMTP accepted the message.
5. Handle IMAP append failure independently; do not resend an accepted message
merely to recreate a Sent copy.
6. Record the incident/reconciliation reference in the consuming domain's audit
trail without copying raw provider secrets or message content unnecessarily.
7. For an outbox `outcome_unknown`, reconcile the Mail command with provider
evidence. Confirmed acceptance closes the provider operation as succeeded;
confirmed absence records verified recovery and permits only a new,
deliberate attempt identifier.
### Delivery-status and calendar-reply sources
An authorized Mail bounce source scans a bounded IMAP UID range without
changing mailbox flags. It correlates DSN reports with durable Mail commands
and, when the optional Calendar invitation capability is active, forwards
`text/calendar` or `.ics` `METHOD:REPLY` parts to Calendar. Calendar remains
owner of attendee state; Mail records only a raw digest, mailbox coordinates,
Message-ID, and audit linkage. Ordinary or malformed calendar messages do not
block DSN progress. A repeated UID/message digest produces no second Calendar
state transition or outbound synchronization effect.
### Backup, restore, and retirement
Backups contain encrypted credentials and encrypted raw POP3 import records and
therefore need the same protection as the live database and key material.
Restoring a Mail database without the matching encryption key makes those
records unusable; restoring it with keys can reactivate sensitive historical
state and must be controlled.
Destructive module retirement first applies the same immediate credential
scrub/audit rule to every remaining profile, then drops Mail-owned tables after
the installer snapshot/confirmation gate. Any scrub or audit failure blocks
retirement. Historical backups are separate retained copies and require the
deployment's approved backup-destruction policy.
## Capability contract
Mail provides `mail.campaign_delivery` version `0.2.x`. This version is a
breaking ownership boundary: it exposes reference-oriented operations, not raw
resolved configuration or credentials.
The Mail REST API supplies the profile list used by pickers. Given a stable
profile reference, the capability supports these kinds of operation:
- resolve a profile reference and evaluate its Mail context/policy to produce a
non-secret delivery summary and
opaque Mail-owned revisions;
- send Campaign-owned RFC message bytes using the selected profile;
- optionally append an accepted message to Sent; and
- return sanitized outcome/revision evidence.
For a real effect, one Mail call must:
1. resolve current campaign/tenant/owner context through the optional narrow
context capability;
2. load the selected active profile;
3. evaluate profile-scope eligibility and effective policy;
4. compare the consumer's expected opaque transport revision;
5. decrypt credentials in Mail memory;
6. perform the SMTP or IMAP operation; and
7. return only the safe result.
Keeping comparison, credential resolution, and transport inside one operation
avoids a time-of-check/time-of-use gap and prevents the consumer from becoming
a credential-processing boundary. The capability does not receive the acting
principal: the Campaign route must first enforce `mail:profile:use`, while Mail
still re-evaluates active profile scope and contextual policy inside the call.
Mail optionally consumes `campaigns.mail_policy_context` and
`addresses.lookup`. Their physical absence must leave Mail installable and its
unrelated profile/mailbox behavior usable. Interface-version compatibility is a
mandatory release gate.
## Security, deletion, and audit
Security invariants:
- Decrypted SMTP/IMAP passwords and JMAP/POP3 credentials never cross the Mail capability/API boundary.
- Password fields are write-only and encrypted at rest; safe responses expose
configuration state, not values.
- Consumers persist stable profile references, not transport copies.
- Host policy is deployment-wide and every supported connection is DNS/IP
pinned at socket creation.
- Redirect/referral-capable transports fail closed unless every peer can be
revalidated and pinned.
- Profile deletion and destructive retirement scrub every Mail-owned encrypted
password and emit non-secret audit in the same lifecycle action.
- Mailbox reads are separately authorized, read-only, paginated, and bounded.
- Logs/audit/results minimize message content, addresses, provider responses,
and secret-like values.
Credential replacement/deletion emits canonical non-secret Mail audit events.
Profile/policy create/update/deactivate currently emits change-feed evidence,
but connection tests and the complete administrative lifecycle do not yet have
equivalent canonical audit events. Mail delivery commands emit request,
terminal-outcome, reconciliation, and deliberate-resend events without
addresses or raw provider responses. MIME, envelope addresses, detailed
refusals, and reconciliation notes are encrypted and minimized after their
retention deadline while hashes, counts, attempts, and decisions remain.
## Acceptance checklist
Before claiming a Mail composition is production-ready:
1. Install Mail alone with Core and with each optional consumer absent.
2. Prove profile scope, visibility, use, test, write, and secret permissions for
representative tenant/user/group contexts.
3. Prove policy deny precedence, locked lower limits, approved-profile mode, and
the campaign-local-credential fail-closed path.
4. Verify all read/list/capability/audit/log responses contain no plaintext or
encrypted password, raw credential token, or consumer-owned inline config.
5. Test DNS rebinding resistance and private/public deployment policy for SMTP
and IMAP.
6. Test SMTP acceptance, refusal, temporary/permanent failure, timeout/unknown
outcome, and IMAP append success/failure against the target provider.
7. Test Redis worker throttling and prove that independent infrastructure
monitoring detects Redis loss; restrict multi-worker delivery during the
current silent local-only fallback.
8. Delete tenant and system profiles, inject audit failure, repeat deletion, and
prove transactional scrub/rollback/idempotency.
9. Run destructive-retirement preflight on a snapshot and prove credentials are
scrubbed/audited before table drop.
10. Pass module permutations, interface/version alignment, WebUI/i18n, full
security audit, backup/restore, and target-environment release checks.
## Explicitly planned, not yet claimed
- Canonical audit events for profile tests and the remaining profile/policy
administration lifecycle, plus an operator-visible Redis-throttling
degradation signal.
- Reusable SMTP batch sessions and their final connection/error/isolation
semantics (`govoplan-mail#16`).
- Final Campaign **test / single send / single resend** semantics; those are a
Campaign business-action contract built on Mail transport operations.
- JMAP provider-side mutation, submission, push, thread, calendar, contact,
attachment-binary, and automatic background-sync support; read-only mailbox
synchronization/search is implemented on the stable IMAP mailbox contract.
- Expanding POP3 beyond the implemented explicit legacy download/import
workflow; it has no folder, flag, search, or synchronization contract.
- A full mail client with compose/reply/move/delete/read-state mutation.
- Quick Access may launch the operating environment's configured composer via
`mailto:`. That explicit handoff is not a GovOPlaN Mail delivery: it selects
no Mail profile or credential, bypasses no Mail policy, and reports no
GovOPlaN delivery result. Recent-message and Drafts links remain read-only
deep links into the authorized Mail profile and preserve their Quick Access
return context.
- Recovery-ledger adoption for future provider-side move, delete, and flag
mutations; no such production path exists in the current read-only mailbox.
- Proof that process-local throttling coordinates multiple workers when Redis
is unavailable; it deliberately does not.
+106
View File
@@ -0,0 +1,106 @@
# Mail Protocol Roadmap
GovOPlaN Mail uses SMTP for sending and supports read-only mailbox access over
IMAP or JMAP. It also provides an explicitly enabled, bounded POP3
legacy-import path. The first JMAP slice is implemented on the stable,
protocol-neutral mailbox contract.
## Current Baseline
- SMTP is the send protocol.
- IMAP is the established read/append protocol and remains unchanged.
- JMAP is an opt-in read-only sync/search protocol; it is not used for sending
or append-to-Sent.
- POP3 is an optional legacy migration source, never a general mailbox
protocol or default profile endpoint.
- Mail profile policy, encrypted credentials, connection diagnostics, and the
read-only mailbox UI cover SMTP/IMAP/JMAP endpoints as applicable.
This baseline matches the first production use case: send campaign mail, append
sent copies when configured, and inspect mailboxes read-only.
## Message Protection Profiles
The product and security profile approved on 2026-08-04 makes S/MIME the first
institutional signing/encryption profile and OpenPGP an additional explicit
profile. Neither profile is implemented by treating protection as a local Mail
toggle:
- private-key custody belongs to an Encryption/KMS provider and usable private
keys are never persisted by Mail;
- recipient certificates and keys initially come from administered directory
or LDAP sources; opportunistic Internet discovery is deferred;
- required signing or encryption fails closed when material is missing,
expired, revoked, unverifiable, or its provider is unavailable;
- plaintext fallback is permitted only by an explicit, audited policy and is
never inferred from provider failure; and
- delivery evidence pins the signing identity, trust/revocation evidence,
algorithm suite, key version, and any explicit downgrade decision.
Provider-neutral S/MIME custody and interoperability fixtures are the first
implementation slice. OpenPGP uses the same no-silent-downgrade boundary after
the S/MIME profile is stable.
## JMAP
JMAP is the preferred sync/search protocol where target mail servers support
RFC 8620 Core and RFC 8621 Mail capabilities.
Reasons:
- HTTP/JSON transport fits the platform API style better than stateful IMAP
- efficient mailbox state sync and changes endpoints
- modern search and thread models
- better fit for browser-facing mailbox UX through a server proxy
The implemented first slice provides:
- authenticated Session discovery with Bearer or Basic credentials;
- explicit account selection or primary Mail-account selection;
- folder hierarchy projection through `Mailbox/get`;
- server-side text search and pagination through `Email/query` plus bounded
summaries and details through `Email/get`;
- incremental state through bounded `Email/changes`, with an explicit full
refresh when the provider can no longer calculate changes;
- per-endpoint response and body-value bounds;
- a dedicated JMAP hostname policy and fail-closed cross-origin API discovery;
and
- protocol-neutral folder/message DTOs and mailbox UI selection while keeping
the IMAP path unchanged.
The current boundary is read-only. JMAP submission, mailbox/message mutation,
threads, calendars, contacts, push subscriptions, binary attachment download,
and automatic background synchronization are deferred until a separately
governed slice needs them.
## POP3
POP3 remains legacy-only. The bounded import slice is available when a concrete
deployment must retire a mailbox that cannot offer IMAP or JMAP.
It is disabled until an administrator creates a dedicated POP3 endpoint and
sets `legacy_import_enabled`. The endpoint has its own encrypted credential,
connection/TLS/authentication diagnostics, maximum message and batch sizes, preview body
limit, and a separate `allow_delete_after_import` policy. Stable UIDL support is
mandatory; Mail refuses import when a provider cannot supply it.
Preview and ordinary import are non-destructive. Selected messages become
encrypted `pending_review` records with a content digest, pinned transport
revision, source UIDL, and audit evidence. Repeating a UIDL reports a duplicate.
Provider deletion requires both endpoint policy and `mail:pop3:delete`, is
chosen separately per batch, and runs only after the local import and its audit
event commit. A disconnect while POP3 `QUIT` commits deletions becomes
`outcome_unknown` and is never retried blindly.
POP3 does not supply folder, flag, thread, search, or synchronization semantics.
It is therefore excluded from the normal mailbox UI and from the recommended
ongoing Mail profile. Configuration-package export/import remains SMTP-focused;
legacy source rollout is an explicit operational action.
## Decision
Keep the implemented POP3 surface limited to governed legacy import. Do not
expand it into mailbox browsing. Keep the protocol-neutral mailbox DTOs and
use the implemented JMAP path for modern synchronization/search support when
an administrator explicitly configures it. Keep SMTP for sending and POP3
limited to governed legacy import.
+93
View File
@@ -0,0 +1,93 @@
{
"name": "@govoplan/mail-webui",
"version": "0.1.22",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@govoplan/mail-webui",
"version": "0.1.22",
"peerDependencies": {
"@govoplan/core-webui": "^0.1.18",
"lucide-react": "^1.23.0",
"react": ">=19.2.7 <20",
"react-dom": ">=19.2.7 <20",
"react-router": ">=8.3.0 <9"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
"optional": true
}
}
},
"node_modules/cookie-es": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-3.1.1.tgz",
"integrity": "sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg==",
"license": "MIT",
"peer": true
},
"node_modules/lucide-react": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.28.0.tgz",
"integrity": "sha512-fARAFJULsGuDDydjp6+6blekG/sBIM29TerzLjc9bQUKAcEfrSc4ZQKb25KRz4OMKd87cZTb5dgq0w/T6KufVg==",
"license": "ISC",
"peer": true,
"peerDependencies": {
"react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/react": {
"version": "19.2.8",
"resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz",
"integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/react-dom": {
"version": "19.2.8",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz",
"integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"scheduler": "^0.27.0"
},
"peerDependencies": {
"react": "^19.2.8"
}
},
"node_modules/react-router": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/react-router/-/react-router-8.3.0.tgz",
"integrity": "sha512-qyPMvW83jGIct3yiieisxdk9M745anqhpIMKN5m1t6yBMfgVPpt77aHOqs5fUlEJRMCGffg9BaQLH9oPVOL7xQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"cookie-es": "^3.1.1"
},
"engines": {
"node": ">=22.22.0"
},
"peerDependencies": {
"react": ">=19.2.7",
"react-dom": ">=19.2.7"
},
"peerDependenciesMeta": {
"react-dom": {
"optional": true
}
}
},
"node_modules/scheduler": {
"version": "0.27.0",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz",
"integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==",
"license": "MIT",
"peer": true
}
}
}
+11 -6
View File
@@ -1,6 +1,6 @@
{
"name": "@govoplan/mail-webui",
"version": "0.1.0",
"version": "0.1.22",
"private": true,
"type": "module",
"main": "webui/src/index.ts",
@@ -19,10 +19,15 @@
"LICENSE"
],
"peerDependencies": {
"@govoplan/core-webui": "^0.1.0",
"lucide-react": "^0.555.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.1"
"@govoplan/core-webui": "^0.1.18",
"lucide-react": "^1.23.0",
"react": ">=19.2.7 <20",
"react-dom": ">=19.2.7 <20",
"react-router": ">=8.3.0 <9"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
"optional": true
}
}
}
+2 -2
View File
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
[project]
name = "govoplan-mail"
version = "0.1.0"
version = "0.1.22"
description = "GovOPlaN mail module with backend and WebUI integration."
readme = "README.md"
requires-python = ">=3.12"
license = { file = "LICENSE" }
authors = [{ name = "GovOPlaN" }]
dependencies = [
"govoplan-core>=0.1.0",
"govoplan-core>=0.1.18",
"pydantic>=2,<3",
"redis>=5,<6",
"SQLAlchemy>=2,<3",
@@ -0,0 +1,761 @@
from __future__ import annotations
import hashlib
from collections import defaultdict
from collections.abc import Mapping
from datetime import datetime, timezone
from email import policy
from email.message import EmailMessage
from email.parser import BytesParser
from email.utils import parsedate_to_datetime
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.core.calendar import (
CalendarCapabilityError,
calendar_invitation_provider,
)
from govoplan_core.core.mail import (
MailBounceObservationRef,
MailBounceProcessingProvider,
)
from govoplan_mail.backend.db.models import (
MailBounceObservation,
MailBounceSource,
MailDeliveryCommand,
MailServerProfile,
)
from govoplan_mail.backend.sending.imap import (
get_imap_raw_message,
list_imap_uids_since,
)
from govoplan_mail.backend.server_hierarchy import (
MailServerHierarchyError,
hierarchy_context_for_profile,
resolve_mail_transport,
)
from govoplan_mail.backend.runtime import get_registry
from govoplan_mail.backend.recovery import (
MailRecoveryError,
begin_bounce_scan_recovery,
)
class MailBounceError(RuntimeError):
pass
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def normalize_message_id(value: object | None) -> str | None:
normalized = " ".join(str(value or "").split())
return normalized[:998] or None
def _calendar_reply_parts(raw_message: bytes) -> tuple[tuple[str, str | None], ...]:
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
except Exception as exc:
raise MailBounceError("Mail message could not be parsed.") from exc
parts: list[tuple[str, str | None]] = []
for part in message.walk():
filename = part.get_filename()
if part.get_content_type() != "text/calendar" and not str(
filename or ""
).casefold().endswith(".ics"):
continue
try:
content = part.get_content()
except Exception:
payload = part.get_payload(decode=True) or b""
content = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
if isinstance(content, bytes):
content = content.decode(
part.get_content_charset() or "utf-8",
errors="replace",
)
text = str(content).strip()
if text:
parts.append((text, filename))
return tuple(parts)
def _reconcile_calendar_replies(
session: Session,
*,
tenant_id: str,
profile_id: str,
folder: str,
uid: str,
raw_message: bytes,
) -> int:
provider = calendar_invitation_provider(get_registry())
if provider is None:
return 0
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
message_id = normalize_message_id(message.get("Message-ID"))
except Exception:
message_id = None
recorded_count = 0
for icalendar, filename in _calendar_reply_parts(raw_message):
try:
recorded = provider.record_icalendar_reply(
session,
tenant_id=tenant_id,
icalendar=icalendar,
received_at=utcnow(),
evidence={
"transport": "mail-imap",
"profile_id": profile_id,
"folder": folder,
"mailbox_uid": uid,
"message_id": message_id,
"filename": filename,
"raw_sha256": raw_sha256,
},
)
except CalendarCapabilityError:
# Mailboxes routinely contain unrelated or malformed invitations.
# They are not allowed to block DSN progress for the entire source.
continue
for invitation in recorded:
audit_event(
session,
tenant_id=tenant_id,
user_id=None,
action="mail.calendar_reply.reconciled",
object_type="calendar_invitation",
object_id=invitation.event_id,
details={
"profile_id": profile_id,
"folder": folder,
"mailbox_uid": uid,
"message_id": message_id,
"calendar_uid": invitation.uid,
"correlation_id": invitation.correlation_id,
"raw_sha256": raw_sha256,
},
)
recorded_count += 1
return recorded_count
def configure_bounce_source(
session: Session,
*,
tenant_id: str,
profile_id: str,
folder: str = "INBOX",
imap_server_id: str | None = None,
imap_credential_id: str | None = None,
is_active: bool = True,
created_by_user_id: str | None = None,
) -> MailBounceSource:
profile = _profile(session, tenant_id=tenant_id, profile_id=profile_id)
clean_folder = folder.strip() or "INBOX"
resolved = _resolve_imap(
session,
profile=profile,
server_id=imap_server_id,
credential_id=imap_credential_id,
)
source = session.scalar(
select(MailBounceSource).where(
MailBounceSource.profile_id == profile.id,
MailBounceSource.folder == clean_folder,
)
)
if source is None:
source = MailBounceSource(
tenant_id=tenant_id,
profile_id=profile.id,
folder=clean_folder,
expected_imap_transport_revision=resolved.transport_revision,
created_by_user_id=created_by_user_id,
)
session.add(source)
source.imap_server_id = resolved.server.id if resolved.server else None
source.imap_credential_id = (
resolved.credential.id if resolved.credential else None
)
source.expected_imap_transport_revision = resolved.transport_revision
source.is_active = is_active
source.last_error = None
session.flush()
return source
def list_bounce_sources(
session: Session,
*,
tenant_id: str,
) -> tuple[MailBounceSource, ...]:
return tuple(
session.scalars(
select(MailBounceSource)
.where(MailBounceSource.tenant_id == tenant_id)
.order_by(MailBounceSource.created_at, MailBounceSource.id)
)
)
def delete_bounce_source(
session: Session,
*,
tenant_id: str,
source_id: str,
) -> None:
source = session.get(MailBounceSource, source_id)
if source is None or source.tenant_id != tenant_id:
raise MailBounceError("Bounce source not found.")
session.delete(source)
session.flush()
def list_bounce_observations(
session: Session,
*,
tenant_id: str,
command_id: str | None = None,
limit: int = 100,
) -> tuple[MailBounceObservationRef, ...]:
statement = select(MailBounceObservation).where(
MailBounceObservation.tenant_id == tenant_id
)
if command_id:
statement = statement.where(MailBounceObservation.command_id == command_id)
rows = session.scalars(
statement.order_by(
MailBounceObservation.observed_at.desc(),
MailBounceObservation.id,
).limit(max(1, min(int(limit), 500)))
)
return tuple(_observation_ref(item) for item in rows)
class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
def scan_source(
self,
session: object,
*,
tenant_id: str,
source_id: str,
limit: int = 100,
) -> Mapping[str, object]:
db = _session(session)
source = db.get(MailBounceSource, source_id)
if source is None or source.tenant_id != tenant_id:
raise MailBounceError("Bounce source not found.")
processed, observations, calendar_replies = self._scan_source(
db,
source,
limit=max(1, min(int(limit), 1_000)),
)
return {
"sources": 1,
"processed_messages": processed,
"observations": observations,
"calendar_replies": calendar_replies,
"failures": [],
}
def process_raw_message(
self,
session: object,
*,
tenant_id: str,
profile_id: str,
folder: str,
uid: str,
raw_message: bytes,
reconcile_calendar: bool = True,
) -> tuple[MailBounceObservationRef, ...]:
db = _session(session)
if reconcile_calendar:
_reconcile_calendar_replies(
db,
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
uid=uid,
raw_message=raw_message,
)
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
reports = parse_delivery_status(raw_message)
observations: list[MailBounceObservationRef] = []
for report in reports:
original_message_id = normalize_message_id(
report.get("original_message_id")
)
command = _correlated_command(
db,
tenant_id=tenant_id,
original_message_id=original_message_id,
command_id=str(report.get("command_id") or "") or None,
)
fingerprint = _observation_fingerprint(
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
uid=uid,
raw_sha256=raw_sha256,
recipient=report.get("recipient"),
action=report.get("action"),
status_code=report.get("status_code"),
)
existing = db.scalar(
select(MailBounceObservation).where(
MailBounceObservation.tenant_id == tenant_id,
MailBounceObservation.fingerprint == fingerprint,
)
)
if existing is not None:
observations.append(_observation_ref(existing))
continue
action = str(report.get("action") or "unknown").casefold()[:40]
status_code = _bounded(report.get("status_code"), 80)
item = MailBounceObservation(
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder[:255],
uid=uid[:255],
fingerprint=fingerprint,
raw_sha256=raw_sha256,
original_message_id=original_message_id,
command_id=command.id if command else None,
recipient=_bounded(report.get("recipient"), 998),
action=action,
status_code=status_code,
diagnostic=_bounded(report.get("diagnostic"), 500),
permanent=action == "failed" or bool(status_code and status_code.startswith("5")),
observed_at=report.get("observed_at") or utcnow(),
matched=command is not None,
evidence={
"reporting_mta": report.get("reporting_mta"),
"remote_mta": report.get("remote_mta"),
"diagnostic_type": report.get("diagnostic_type"),
},
)
try:
with db.begin_nested():
db.add(item)
db.flush()
except IntegrityError:
item = db.scalar(
select(MailBounceObservation).where(
MailBounceObservation.tenant_id == tenant_id,
MailBounceObservation.fingerprint == fingerprint,
)
)
if item is None:
raise
audit_event(
db,
tenant_id=tenant_id,
user_id=None,
action="mail.bounce.observed",
object_type="mail_bounce_observation",
object_id=item.id,
details={
"profile_id": profile_id,
"folder": folder,
"uid": uid,
"command_id": item.command_id,
"action": item.action,
"status_code": item.status_code,
"matched": item.matched,
"raw_sha256": raw_sha256,
},
)
observations.append(_observation_ref(item))
return tuple(observations)
def scan_due(
self,
session: object,
*,
tenant_id: str | None = None,
limit: int = 100,
) -> Mapping[str, object]:
db = _session(session)
remaining = max(1, min(int(limit), 1_000))
statement = select(MailBounceSource).where(
MailBounceSource.is_active.is_(True)
)
if tenant_id:
statement = statement.where(MailBounceSource.tenant_id == tenant_id)
sources = tuple(
db.scalars(statement.order_by(MailBounceSource.last_scanned_at, MailBounceSource.id))
)
processed = 0
observations = 0
calendar_replies = 0
failures: list[dict[str, str]] = []
for source in sources:
if remaining <= 0:
break
try:
count, found, replies = self._scan_source(
db,
source,
limit=remaining,
)
processed += count
observations += found
calendar_replies += replies
remaining -= count
except Exception as exc:
source.last_scanned_at = utcnow()
source.last_error = _bounded(exc, 500)
db.flush()
failures.append({"source_id": source.id, "error": source.last_error or "Scan failed"})
return {
"sources": len(sources),
"processed_messages": processed,
"observations": observations,
"calendar_replies": calendar_replies,
"failures": failures,
}
def observations_for_commands(
self,
session: object,
*,
tenant_id: str,
command_ids: tuple[str, ...],
) -> Mapping[str, tuple[MailBounceObservationRef, ...]]:
if not command_ids:
return {}
rows = _session(session).scalars(
select(MailBounceObservation)
.where(
MailBounceObservation.tenant_id == tenant_id,
MailBounceObservation.command_id.in_(tuple(set(command_ids))),
)
.order_by(MailBounceObservation.observed_at, MailBounceObservation.id)
)
grouped: defaultdict[str, list[MailBounceObservationRef]] = defaultdict(list)
for row in rows:
if row.command_id:
grouped[row.command_id].append(_observation_ref(row))
return {key: tuple(value) for key, value in grouped.items()}
def _scan_source(
self,
session: Session,
source: MailBounceSource,
*,
limit: int,
) -> tuple[int, int, int]:
profile = _profile(
session,
tenant_id=source.tenant_id,
profile_id=source.profile_id,
)
resolved = _resolve_imap(
session,
profile=profile,
server_id=source.imap_server_id,
credential_id=source.imap_credential_id,
)
if resolved.transport_revision != source.expected_imap_transport_revision:
raise MailBounceError(
"Bounce-source IMAP settings changed; review and save the source before scanning."
)
try:
recovery = begin_bounce_scan_recovery(
tenant_id=source.tenant_id,
profile_id=source.profile_id,
source_id=source.id,
folder=source.folder,
)
except MailRecoveryError as exc:
raise MailBounceError(str(exc)) from exc
try:
page = list_imap_uids_since(
imap_config=resolved.config,
folder=source.folder,
highest_uid=source.highest_processed_uid,
expected_uidvalidity=source.uidvalidity,
limit=limit,
)
found = 0
calendar_replies = 0
highest = 0 if page.cursor_reset else source.highest_processed_uid
for uid in page.uids:
raw = get_imap_raw_message(
imap_config=resolved.config,
folder=source.folder,
uid=uid,
)
calendar_replies += _reconcile_calendar_replies(
session,
tenant_id=source.tenant_id,
profile_id=source.profile_id,
folder=source.folder,
uid=uid,
raw_message=raw.raw,
)
found += len(
self.process_raw_message(
session,
tenant_id=source.tenant_id,
profile_id=source.profile_id,
folder=source.folder,
uid=uid,
raw_message=raw.raw,
reconcile_calendar=False,
)
)
highest = max(highest, int(uid))
now = utcnow()
source.uidvalidity = page.uidvalidity
source.highest_processed_uid = highest
source.last_scanned_at = now
source.last_success_at = now
source.last_error = None
session.flush()
session.commit()
except Exception as exc:
session.rollback()
if not recovery.operation.closed:
recovery.reject(code=exc.__class__.__name__)
raise
recovery.complete(highest_uid=highest, uidvalidity=page.uidvalidity)
return len(page.uids), found, calendar_replies
def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]:
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
except Exception as exc:
raise MailBounceError("Bounce message could not be parsed.") from exc
original_message_id = normalize_message_id(message.get("Original-Message-ID"))
command_id = _bounded(message.get("X-GovOPlaN-Delivery-ID"), 36)
reporting_mta = None
reports: list[dict[str, object]] = []
is_report = message.get_content_type() == "multipart/report"
for part in message.walk():
content_type = part.get_content_type()
if content_type == "message/delivery-status":
payload = part.get_payload()
blocks = payload if isinstance(payload, list) else []
for position, block in enumerate(blocks):
if not isinstance(block, EmailMessage):
continue
if position == 0:
reporting_mta = block.get("Reporting-MTA") or reporting_mta
original_message_id = normalize_message_id(
block.get("Original-Message-ID") or original_message_id
)
continue
reports.append(
_delivery_status_block(
block,
original_message_id=original_message_id,
command_id=command_id,
reporting_mta=reporting_mta,
fallback_date=message.get("Date"),
)
)
elif content_type == "message/rfc822":
payload = part.get_payload()
if isinstance(payload, list) and payload and isinstance(payload[0], EmailMessage):
original_message_id = normalize_message_id(
payload[0].get("Message-ID") or original_message_id
)
command_id = _bounded(
payload[0].get("X-GovOPlaN-Delivery-ID") or command_id,
36,
)
elif content_type == "text/rfc822-headers":
try:
headers = BytesParser(policy=policy.default).parsebytes(
part.get_payload(decode=True) or b"",
headersonly=True,
)
except Exception:
continue
original_message_id = normalize_message_id(
headers.get("Message-ID") or original_message_id
)
command_id = _bounded(
headers.get("X-GovOPlaN-Delivery-ID") or command_id,
36,
)
if reports:
for report in reports:
report["original_message_id"] = (
report.get("original_message_id") or original_message_id
)
report["command_id"] = report.get("command_id") or command_id
return tuple(reports)
failed = message.get_all("X-Failed-Recipients", [])
recipients = [value.strip() for header in failed for value in str(header).split(",") if value.strip()]
if not recipients and not is_report:
return ()
return tuple(
{
"original_message_id": original_message_id,
"command_id": command_id,
"recipient": recipient or None,
"action": "failed" if recipient else "unknown",
"status_code": None,
"diagnostic": "Unstructured delivery-status report",
"observed_at": _parsed_date(message.get("Date")) or utcnow(),
"reporting_mta": reporting_mta,
}
for recipient in (recipients or [""])
)
def _delivery_status_block(
block: EmailMessage,
*,
original_message_id: str | None,
command_id: str | None,
reporting_mta: object | None,
fallback_date: object | None,
) -> dict[str, object]:
diagnostic = str(block.get("Diagnostic-Code") or "")
diagnostic_type, _, diagnostic_text = diagnostic.partition(";")
return {
"original_message_id": normalize_message_id(
block.get("Original-Message-ID") or original_message_id
),
"command_id": command_id,
"recipient": _dsn_address(
block.get("Final-Recipient") or block.get("Original-Recipient")
),
"action": str(block.get("Action") or "unknown").casefold(),
"status_code": _bounded(block.get("Status"), 80),
"diagnostic": diagnostic_text.strip() or diagnostic_type.strip() or None,
"diagnostic_type": diagnostic_type.strip() or None,
"reporting_mta": str(reporting_mta or "") or None,
"remote_mta": _dsn_address(block.get("Remote-MTA")),
"observed_at": _parsed_date(
block.get("Last-Attempt-Date") or fallback_date
) or utcnow(),
}
def _correlated_command(
session: Session,
*,
tenant_id: str,
original_message_id: str | None,
command_id: str | None,
) -> MailDeliveryCommand | None:
if command_id:
command = session.get(MailDeliveryCommand, command_id)
if command is not None and command.tenant_id == tenant_id:
return command
if not original_message_id:
return None
return session.scalar(
select(MailDeliveryCommand).where(
MailDeliveryCommand.tenant_id == tenant_id,
MailDeliveryCommand.rfc_message_id == original_message_id,
)
)
def _observation_fingerprint(**values: object) -> str:
canonical = "\x1f".join(str(values[key] or "") for key in sorted(values))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _observation_ref(item: MailBounceObservation) -> MailBounceObservationRef:
return MailBounceObservationRef(
id=item.id,
tenant_id=item.tenant_id,
profile_id=item.profile_id,
folder=item.folder,
uid=item.uid,
original_message_id=item.original_message_id,
command_id=item.command_id,
recipient=item.recipient,
action=item.action,
status_code=item.status_code,
diagnostic=item.diagnostic,
permanent=item.permanent,
observed_at=item.observed_at,
matched=item.matched,
evidence=dict(item.evidence or {}),
)
def _profile(session: Session, *, tenant_id: str, profile_id: str) -> MailServerProfile:
profile = session.get(MailServerProfile, profile_id)
if profile is None or profile.tenant_id not in {None, tenant_id} or not profile.is_active:
raise MailBounceError("Active Mail profile not found.")
return profile
def _resolve_imap(
session: Session,
*,
profile: MailServerProfile,
server_id: str | None,
credential_id: str | None,
):
try:
resolved = resolve_mail_transport(
session,
profile=profile,
protocol="imap",
context=hierarchy_context_for_profile(profile, administrative=True),
server_id=server_id,
credential_id=credential_id,
)
except MailServerHierarchyError as exc:
raise MailBounceError(str(exc)) from exc
return resolved
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise MailBounceError("Bounce processing requires a SQLAlchemy Session.")
return value
def _bounded(value: object | None, limit: int) -> str | None:
text = " ".join(str(value or "").split())
return text[:limit] or None
def _dsn_address(value: object | None) -> str | None:
text = str(value or "")
_, separator, address = text.partition(";")
return _bounded(address if separator else text, 998)
def _parsed_date(value: object | None) -> datetime | None:
if not value:
return None
try:
parsed = parsedate_to_datetime(str(value))
except (TypeError, ValueError, OverflowError):
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
__all__ = [
"MailBounceError",
"SqlMailBounceProcessingProvider",
"configure_bounce_source",
"delete_bounce_source",
"list_bounce_sources",
"list_bounce_observations",
"normalize_message_id",
"parse_delivery_status",
]
+893
View File
@@ -0,0 +1,893 @@
from __future__ import annotations
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
from typing import Any, Iterator
from sqlalchemy.orm import Session
from govoplan_core.core.mail import NotificationMailDeliveryRequest
from govoplan_core.core.modules import ModuleContext
from govoplan_mail.backend.mail_profiles import (
MailProfileError,
_assert_campaign_inherits_profile_credentials,
assert_campaign_mail_policy_allows_json,
assert_mail_policy_allows_send,
campaign_mail_owner_context,
campaign_profile_transport_revisions,
effective_mail_profile_policy,
ensure_mail_profile_allowed_for_campaign,
get_mail_server_profile,
imap_config_from_profile,
mail_profile_id_from_campaign_json,
smtp_config_from_profile,
)
from govoplan_mail.backend.server_hierarchy import (
MailHierarchyContext,
MailServerHierarchyError,
resolve_mail_transport,
select_mail_transport,
)
from govoplan_mail.backend.runtime import configure_runtime
from govoplan_mail.backend.recovery import (
MailRecoveryError,
begin_provider_effect_recovery,
)
from govoplan_mail.backend.sending.imap import (
ImapAppendError,
ImapConfigurationError,
append_message_to_sent,
)
from govoplan_mail.backend.sending.rate_limit import wait_for_rate_limit
from govoplan_mail.backend.sending.smtp import (
SmtpBatchSession,
SmtpConfigurationError,
SmtpSendError,
send_email_bytes,
)
_ACTIVE_SMTP_BATCH: ContextVar[SmtpBatchSession | None] = ContextVar(
"govoplan_mail_active_smtp_batch",
default=None,
)
@dataclass(frozen=True, slots=True)
class CampaignSmtpDeliveryResult:
envelope_recipients: list[str]
refused_recipients: dict[str, dict[str, int | str]]
connection_sequence: int = 1
session_reused: bool = False
reconnect_count: int = 0
@property
def accepted_count(self) -> int:
return len(self.envelope_recipients) - len(self.refused_recipients)
@dataclass(frozen=True, slots=True)
class CampaignImapAppendResult:
folder: str
@dataclass(frozen=True, slots=True)
class CampaignSmtpBatchState:
session: SmtpBatchSession
@property
def status(self) -> str:
return "ready"
@property
def connection_count(self) -> int:
return self.session.connection_count
@property
def reconnect_count(self) -> int:
return self.session.reconnect_count
def _sanitized_refusals(
refused_recipients: dict[str, tuple[int, bytes | str]],
) -> dict[str, dict[str, int | str]]:
sanitized: dict[str, dict[str, int | str]] = {}
for recipient, (raw_code, _provider_message) in refused_recipients.items():
try:
code = int(raw_code)
except (TypeError, ValueError):
code = 0
if 400 <= code < 500:
classification = "temporary"
message = "Temporary recipient rejection"
elif 500 <= code < 600:
classification = "permanent"
message = "Permanent recipient rejection"
else:
classification = "unknown"
message = "Recipient rejected"
sanitized[str(recipient)] = {
"status_code": code,
"classification": classification,
"message": message,
}
return sanitized
def _sanitized_smtp_error(exc: SmtpSendError) -> SmtpSendError:
if exc.outcome_unknown:
message = "Mail delivery outcome is unknown after transmission started."
elif exc.temporary:
message = "Mail delivery failed temporarily."
else:
message = "Mail delivery was rejected."
return SmtpSendError(
message,
temporary=exc.temporary,
outcome_unknown=exc.outcome_unknown,
systemic=exc.systemic,
reason_code=exc.reason_code,
phase=exc.phase,
)
def _sanitized_imap_error(exc: ImapAppendError) -> ImapAppendError:
if exc.outcome_unknown:
message = "The Sent-folder append outcome is unknown; inspect the mailbox before retrying."
elif exc.temporary:
message = "Appending the sent message failed temporarily."
else:
message = "Appending the sent message was rejected."
return ImapAppendError(
message,
temporary=exc.temporary,
outcome_unknown=exc.outcome_unknown,
)
def _authorized_campaign_profile(
session: Session,
*,
tenant_id: str,
campaign_id: str,
profile_id: str,
selection: dict[str, str | None] | None = None,
):
profile = ensure_mail_profile_allowed_for_campaign(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
profile_id=profile_id,
require_active=True,
)
policy = effective_mail_profile_policy(session, tenant_id=tenant_id, campaign_id=campaign_id)
_assert_campaign_inherits_profile_credentials(profile, policy, selection)
return profile
def _campaign_hierarchy_context(
session: Session,
*,
tenant_id: str,
campaign_id: str,
) -> MailHierarchyContext:
campaign = campaign_mail_owner_context(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
)
return MailHierarchyContext(
tenant_id=tenant_id,
user_id=campaign.owner_user_id,
group_ids=(
frozenset({campaign.owner_group_id})
if campaign.owner_group_id
else frozenset()
),
target_scope_type="campaign",
target_scope_id=campaign.id,
)
def _selection_payload(
*,
profile_id: str,
smtp_server_id: str | None = None,
smtp_credential_id: str | None = None,
imap_server_id: str | None = None,
imap_credential_id: str | None = None,
) -> dict[str, str | None]:
return {
"mail_profile_id": profile_id,
"smtp_server_id": smtp_server_id,
"smtp_credential_id": smtp_credential_id,
"imap_server_id": imap_server_id,
"imap_credential_id": imap_credential_id,
}
def _supports_hierarchy(session: object) -> bool:
return callable(getattr(session, "execute", None))
def campaign_profile_delivery_summary(
session: Session,
*,
tenant_id: str,
profile_id: str,
campaign_id: str | None = None,
owner_user_id: str | None = None,
owner_group_id: str | None = None,
smtp_server_id: str | None = None,
smtp_credential_id: str | None = None,
imap_server_id: str | None = None,
imap_credential_id: str | None = None,
) -> dict[str, Any]:
"""Return only non-secret capabilities and opaque drift evidence."""
selection = _selection_payload(
profile_id=profile_id,
smtp_server_id=smtp_server_id,
smtp_credential_id=smtp_credential_id,
imap_server_id=imap_server_id,
imap_credential_id=imap_credential_id,
)
if campaign_id:
profile = _authorized_campaign_profile(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
profile_id=profile_id,
selection=selection,
)
else:
assert_campaign_mail_policy_allows_json(
session,
tenant_id=tenant_id,
raw_json={"server": {key: value for key, value in selection.items() if value}},
owner_user_id=owner_user_id,
owner_group_id=owner_group_id,
)
profile = get_mail_server_profile(
session,
tenant_id=tenant_id,
profile_id=profile_id,
require_active=True,
)
if campaign_id and _supports_hierarchy(session):
context = _campaign_hierarchy_context(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
)
try:
smtp = select_mail_transport(
session,
profile=profile,
protocol="smtp",
context=context,
server_id=smtp_server_id,
credential_id=smtp_credential_id,
)
imap = select_mail_transport(
session,
profile=profile,
protocol="imap",
context=context,
server_id=imap_server_id,
credential_id=imap_credential_id,
)
except MailServerHierarchyError as exc:
raise MailProfileError(str(exc)) from exc
smtp_available = smtp.available
imap_available = imap.available
smtp_revision = smtp.transport_revision
imap_revision = imap.transport_revision if imap.available else None
resolved_smtp_server_id = smtp.server.id if smtp.server else None
resolved_smtp_credential_id = smtp.credential.id if smtp.credential else None
resolved_imap_server_id = imap.server.id if imap.server else None
resolved_imap_credential_id = imap.credential.id if imap.credential else None
else:
revisions = campaign_profile_transport_revisions(profile)
smtp_config = profile.smtp_config or {}
imap_config = profile.imap_config or {}
smtp_available = bool(smtp_config.get("host") and smtp_config.get("port"))
imap_available = bool(imap_config.get("host") and imap_config.get("port"))
smtp_revision = revisions["smtp"]
imap_revision = revisions["imap"]
resolved_smtp_server_id = smtp_server_id
resolved_smtp_credential_id = smtp_credential_id
resolved_imap_server_id = imap_server_id
resolved_imap_credential_id = imap_credential_id
return {
"mail_profile_id": profile_id,
"smtp_server_id": resolved_smtp_server_id,
"smtp_credential_id": resolved_smtp_credential_id,
"imap_server_id": resolved_imap_server_id,
"imap_credential_id": resolved_imap_credential_id,
"smtp_available": smtp_available,
"imap_available": imap_available,
"smtp_transport_revision": smtp_revision,
"imap_transport_revision": imap_revision,
}
@contextmanager
def campaign_smtp_batch(
session: Session,
*,
tenant_id: str,
campaign_id: str,
profile_id: str,
envelope_from: str,
envelope_recipients: list[str],
from_header: str | None,
expected_smtp_transport_revision: str,
smtp_server_id: str | None = None,
smtp_credential_id: str | None = None,
) -> Iterator[CampaignSmtpBatchState]:
"""Preflight and retain one authorized SMTP connection for a batch."""
selection = _selection_payload(
profile_id=profile_id,
smtp_server_id=smtp_server_id,
smtp_credential_id=smtp_credential_id,
)
try:
profile = _authorized_campaign_profile(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
profile_id=profile_id,
selection=selection,
)
except MailProfileError:
raise
except Exception:
raise SmtpConfigurationError("The selected Mail profile's SMTP configuration is unusable.") from None
if _supports_hierarchy(session):
context = _campaign_hierarchy_context(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
)
try:
selected_smtp = select_mail_transport(
session,
profile=profile,
protocol="smtp",
context=context,
server_id=smtp_server_id,
credential_id=smtp_credential_id,
)
except MailServerHierarchyError as exc:
raise MailProfileError(str(exc)) from exc
current_revision = selected_smtp.transport_revision
else:
context = None
current_revision = campaign_profile_transport_revisions(profile)["smtp"]
if current_revision != expected_smtp_transport_revision:
raise MailProfileError(
"The selected Mail profile's SMTP settings changed after this campaign was built. "
"Revalidate and rebuild the campaign before delivery."
)
try:
smtp = (
resolve_mail_transport(
session,
profile=profile,
protocol="smtp",
context=context,
server_id=smtp_server_id,
credential_id=smtp_credential_id,
).config
if context is not None
else smtp_config_from_profile(profile)
)
except MailProfileError:
raise
except Exception:
raise SmtpConfigurationError("The selected Mail profile's SMTP configuration is unusable.") from None
try:
assert_mail_policy_allows_send(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
smtp=smtp,
imap=None,
envelope_sender=envelope_from,
from_header=from_header,
recipients=envelope_recipients,
)
except MailProfileError:
raise MailProfileError("Mail delivery is blocked by the effective Mail policy.") from None
smtp_session = SmtpBatchSession(smtp)
smtp_session.preflight()
token = _ACTIVE_SMTP_BATCH.set(smtp_session)
try:
yield CampaignSmtpBatchState(session=smtp_session)
finally:
_ACTIVE_SMTP_BATCH.reset(token)
smtp_session.close()
def send_campaign_email_bytes(
session: Session,
*,
tenant_id: str,
campaign_id: str,
profile_id: str,
message_bytes: bytes,
envelope_from: str,
envelope_recipients: list[str],
from_header: str | None,
expected_smtp_transport_revision: str,
smtp_server_id: str | None = None,
smtp_credential_id: str | None = None,
recovery_effect_id: str | None = None,
recovery_resource_type: str | None = None,
recovery_resource_id: str | None = None,
) -> CampaignSmtpDeliveryResult:
selection = _selection_payload(
profile_id=profile_id,
smtp_server_id=smtp_server_id,
smtp_credential_id=smtp_credential_id,
)
try:
profile = _authorized_campaign_profile(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
profile_id=profile_id,
selection=selection,
)
except MailProfileError:
raise
except Exception:
raise SmtpConfigurationError("The selected Mail profile's SMTP configuration is unusable.") from None
resolved_smtp = None
if _supports_hierarchy(session):
context = _campaign_hierarchy_context(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
)
try:
selected_smtp = select_mail_transport(
session,
profile=profile,
protocol="smtp",
context=context,
server_id=smtp_server_id,
credential_id=smtp_credential_id,
)
except MailServerHierarchyError as exc:
raise MailProfileError(str(exc)) from exc
current_smtp_revision = selected_smtp.transport_revision
else:
current_smtp_revision = campaign_profile_transport_revisions(profile)["smtp"]
if current_smtp_revision != expected_smtp_transport_revision:
raise MailProfileError(
"The selected Mail profile's SMTP settings changed after this campaign was built. "
"Revalidate and rebuild the campaign before delivery."
)
try:
if _supports_hierarchy(session):
resolved_smtp = resolve_mail_transport(
session,
profile=profile,
protocol="smtp",
context=context,
server_id=smtp_server_id,
credential_id=smtp_credential_id,
)
smtp = resolved_smtp.config
else:
smtp = smtp_config_from_profile(profile)
except MailProfileError:
raise
except Exception:
raise SmtpConfigurationError("The selected Mail profile's SMTP configuration is unusable.") from None
try:
assert_mail_policy_allows_send(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
smtp=smtp,
imap=None,
envelope_sender=envelope_from,
from_header=from_header,
recipients=envelope_recipients,
)
except MailProfileError:
raise MailProfileError("Mail delivery is blocked by the effective Mail policy.") from None
try:
recovery = begin_provider_effect_recovery(
kind="smtp-delivery",
effect_id=recovery_effect_id,
tenant_id=tenant_id,
profile_id=profile_id,
message_bytes=message_bytes,
expected_transport_revision=expected_smtp_transport_revision,
recipient_count=len(envelope_recipients),
resource_type=recovery_resource_type,
resource_id=recovery_resource_id,
)
except MailRecoveryError as exc:
raise SmtpConfigurationError(str(exc)) from None
if recovery is not None and recovery.replayed:
raise SmtpSendError(
"The matching SMTP effect already succeeded; reconcile caller state without resending.",
outcome_unknown=True,
)
try:
result = send_email_bytes(
message_bytes,
smtp_config=smtp,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
batch_session=_ACTIVE_SMTP_BATCH.get(),
)
except SmtpSendError as exc:
sanitized = _sanitized_smtp_error(exc)
if recovery is not None:
if sanitized.outcome_unknown:
recovery.unknown(code="smtp_outcome_unknown", summary=str(sanitized))
else:
recovery.reject(code="smtp_rejected", summary=str(sanitized))
raise sanitized from None
except SmtpConfigurationError as exc:
if recovery is not None:
recovery.reject(code="smtp_configuration", summary=str(exc))
raise SmtpConfigurationError("The selected Mail profile's SMTP configuration is unusable.") from None
except Exception:
if recovery is not None:
recovery.unknown(
code="unexpected_provider_error",
summary="SMTP outcome is unknown after an unexpected provider failure",
)
raise SmtpSendError(
"Mail delivery outcome is unknown after the provider effect started.",
outcome_unknown=True,
) from None
sanitized_result = CampaignSmtpDeliveryResult(
envelope_recipients=list(result.envelope_recipients),
refused_recipients=_sanitized_refusals(result.refused_recipients),
connection_sequence=getattr(result, "connection_sequence", 0),
session_reused=getattr(result, "session_reused", False),
reconnect_count=getattr(result, "reconnect_count", 0),
)
if recovery is not None:
try:
recovery.succeed_smtp(
accepted_count=sanitized_result.accepted_count,
refused_recipients=sanitized_result.refused_recipients,
)
except Exception:
raise SmtpSendError(
"SMTP returned an outcome, but durable recovery evidence could not be finalized.",
outcome_unknown=True,
) from None
return sanitized_result
def append_campaign_message_to_sent(
session: Session,
*,
tenant_id: str,
campaign_id: str,
profile_id: str,
message_bytes: bytes,
folder: str | None,
expected_smtp_transport_revision: str,
expected_imap_transport_revision: str | None,
smtp_server_id: str | None = None,
smtp_credential_id: str | None = None,
imap_server_id: str | None = None,
imap_credential_id: str | None = None,
recovery_effect_id: str | None = None,
recovery_resource_type: str | None = None,
recovery_resource_id: str | None = None,
) -> CampaignImapAppendResult:
selection = _selection_payload(
profile_id=profile_id,
smtp_server_id=smtp_server_id,
smtp_credential_id=smtp_credential_id,
imap_server_id=imap_server_id,
imap_credential_id=imap_credential_id,
)
try:
profile = _authorized_campaign_profile(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
profile_id=profile_id,
selection=selection,
)
except MailProfileError:
raise
except Exception:
raise ImapConfigurationError("The selected Mail profile's IMAP configuration is unusable.") from None
if _supports_hierarchy(session):
context = _campaign_hierarchy_context(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
)
try:
selected_smtp = select_mail_transport(
session,
profile=profile,
protocol="smtp",
context=context,
server_id=smtp_server_id,
credential_id=smtp_credential_id,
)
selected_imap = select_mail_transport(
session,
profile=profile,
protocol="imap",
context=context,
server_id=imap_server_id,
credential_id=imap_credential_id,
)
except MailServerHierarchyError as exc:
raise MailProfileError(str(exc)) from exc
smtp_revision = selected_smtp.transport_revision
imap_revision = selected_imap.transport_revision
else:
revisions = campaign_profile_transport_revisions(profile)
smtp_revision = revisions["smtp"]
imap_revision = revisions["imap"]
if smtp_revision != expected_smtp_transport_revision:
raise MailProfileError(
"The selected Mail profile's SMTP settings changed after this campaign was built. "
"Revalidate and rebuild the campaign before append-to-Sent delivery."
)
if imap_revision != expected_imap_transport_revision:
raise MailProfileError(
"The selected Mail profile's IMAP settings changed after this campaign was built. "
"Revalidate and rebuild the campaign before append-to-Sent delivery."
)
try:
if _supports_hierarchy(session):
imap = resolve_mail_transport(
session,
profile=profile,
protocol="imap",
context=context,
server_id=imap_server_id,
credential_id=imap_credential_id,
).config
else:
imap = imap_config_from_profile(profile)
except MailProfileError:
raise
except Exception:
raise ImapConfigurationError("The selected Mail profile's IMAP configuration is unusable.") from None
if imap is None:
raise ImapConfigurationError("The selected Mail profile has no IMAP configuration")
try:
assert_mail_policy_allows_send(
session,
tenant_id=tenant_id,
campaign_id=campaign_id,
smtp=None,
imap=imap,
)
except MailProfileError:
raise MailProfileError("Appending to Sent is blocked by the effective Mail policy.") from None
try:
recovery = begin_provider_effect_recovery(
kind="imap-append",
effect_id=recovery_effect_id,
tenant_id=tenant_id,
profile_id=profile_id,
message_bytes=message_bytes,
expected_transport_revision=expected_imap_transport_revision,
folder=folder,
resource_type=recovery_resource_type,
resource_id=recovery_resource_id,
)
except MailRecoveryError as exc:
raise ImapConfigurationError(str(exc)) from None
if recovery is not None and recovery.replayed:
raise ImapAppendError(
"The matching IMAP append already succeeded; reconcile caller state without appending again.",
outcome_unknown=True,
)
try:
result = append_message_to_sent(message_bytes, imap_config=imap, folder=folder)
except ImapAppendError as exc:
sanitized = _sanitized_imap_error(exc)
if recovery is not None:
if sanitized.outcome_unknown:
recovery.unknown(code="imap_outcome_unknown", summary=str(sanitized))
else:
recovery.reject(code="imap_rejected", summary=str(sanitized))
raise sanitized from None
except ImapConfigurationError as exc:
if recovery is not None:
recovery.reject(code="imap_configuration", summary=str(exc))
raise ImapConfigurationError("The selected Mail profile's IMAP configuration is unusable.") from None
except Exception:
if recovery is not None:
recovery.unknown(
code="unexpected_provider_error",
summary="IMAP APPEND outcome is unknown after an unexpected provider failure",
)
raise ImapAppendError(
"The Sent-folder append outcome is unknown; inspect the mailbox before retrying.",
outcome_unknown=True,
) from None
if recovery is not None:
try:
recovery.succeed_imap(folder=result.folder)
except Exception:
raise ImapAppendError(
"IMAP APPEND returned success, but durable recovery evidence could not be finalized.",
outcome_unknown=True,
) from None
return CampaignImapAppendResult(folder=result.folder)
class MailCampaignCapability:
MailProfileError = MailProfileError
SmtpConfigurationError = SmtpConfigurationError
SmtpSendError = SmtpSendError
ImapConfigurationError = ImapConfigurationError
ImapAppendError = ImapAppendError
assert_campaign_mail_policy_allows_json = staticmethod(assert_campaign_mail_policy_allows_json)
mail_profile_id_from_campaign_json = staticmethod(mail_profile_id_from_campaign_json)
campaign_profile_delivery_summary = staticmethod(campaign_profile_delivery_summary)
campaign_smtp_batch = staticmethod(campaign_smtp_batch)
send_campaign_email_bytes = staticmethod(send_campaign_email_bytes)
append_campaign_message_to_sent = staticmethod(append_campaign_message_to_sent)
wait_for_rate_limit = staticmethod(wait_for_rate_limit)
@staticmethod
def submit_delivery_command(session: Session, **kwargs: Any) -> dict[str, object]:
from govoplan_mail.backend.delivery_outbox import submit_delivery_command
return submit_delivery_command(session, **kwargs)
@staticmethod
def delivery_command_summary(
session: Session,
*,
tenant_id: str,
command_id: str,
) -> dict[str, object]:
from govoplan_mail.backend.delivery_outbox import (
delivery_command_summary,
get_delivery_command,
)
return delivery_command_summary(
get_delivery_command(
session,
tenant_id=tenant_id,
command_id=command_id,
)
)
@staticmethod
def mock_mailbox():
from govoplan_mail.backend.dev import mock_mailbox
return mock_mailbox
def campaign_capability(context: ModuleContext) -> MailCampaignCapability:
configure_runtime(settings=context.settings)
return MailCampaignCapability()
def delivery_outbox_capability(context: ModuleContext):
from govoplan_mail.backend.delivery_outbox import MailDeliveryOutboxCapability
configure_runtime(settings=context.settings)
return MailDeliveryOutboxCapability()
class MailNotificationDeliveryCapability:
"""Submit notification email to the Mail-owned durable outbox."""
def submit_notification_mail(
self,
session: Session,
request: NotificationMailDeliveryRequest,
) -> dict[str, object]:
profile_id = str(request.mail_profile_id or "").strip()
from_address = str(request.from_address or "").strip()
if not profile_id or not from_address:
return {
"status": "paused",
"provider": "mail.notificationDelivery",
"error": (
"Notification email requires a Mail profile and sender "
"selected by tenant policy."
),
}
try:
transport = campaign_profile_delivery_summary(
session,
tenant_id=request.tenant_id,
profile_id=profile_id,
smtp_server_id=request.smtp_server_id,
smtp_credential_id=request.smtp_credential_id,
)
except MailProfileError:
return {
"status": "paused",
"provider": "mail.notificationDelivery",
"error": (
"The selected notification Mail profile is unavailable or "
"blocked by effective policy."
),
}
if not transport.get("smtp_available"):
return {
"status": "paused",
"provider": "mail.notificationDelivery",
"error": "The selected notification Mail profile has no usable SMTP transport.",
}
message = EmailMessage()
message["Date"] = formatdate(localtime=True)
message["Message-ID"] = make_msgid()
message["Subject"] = request.subject
message["From"] = from_address
message["To"] = request.recipient
message.set_content(request.body_text)
if request.body_html:
message.add_alternative(request.body_html, subtype="html")
from govoplan_mail.backend.delivery_outbox import submit_delivery_command
command = submit_delivery_command(
session,
tenant_id=request.tenant_id,
command_type="notification",
source_module="notifications",
source_resource_type="notification",
source_resource_id=request.notification_id,
source_version_id=None,
idempotency_key=f"notification:{request.notification_id}",
profile_id=profile_id,
smtp_server_id=(
str(transport.get("smtp_server_id"))
if transport.get("smtp_server_id")
else request.smtp_server_id
),
smtp_credential_id=(
str(transport.get("smtp_credential_id"))
if transport.get("smtp_credential_id")
else request.smtp_credential_id
),
message_bytes=bytes(message),
envelope_from=from_address,
envelope_recipients=[request.recipient],
from_header=from_address,
expected_smtp_transport_revision=str(
transport["smtp_transport_revision"]
),
)
return {
"status": "accepted",
"provider": "mail.delivery_outbox",
"external_message_id": command["id"],
"delivery_status": command["status"],
"duplicate": bool(command.get("duplicate")),
}
def notification_delivery_capability(
context: ModuleContext,
) -> MailNotificationDeliveryCapability:
configure_runtime(settings=context.settings)
return MailNotificationDeliveryCapability()
+130 -26
View File
@@ -1,35 +1,139 @@
from __future__ import annotations
from enum import StrEnum
import urllib.parse
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from pydantic import Field, model_validator
from govoplan_core.mail.config import (
ImapConfig,
ImapFolderMappings,
ImapServerConfig,
SmtpConfig,
SmtpServerConfig,
StrictModel,
TransportCredentials,
TransportSecurity,
normalize_split_transport_credentials,
)
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", populate_by_name=True)
class Pop3ServerConfig(StrictModel):
"""Server-only settings for an explicitly enabled legacy POP3 source."""
class TransportSecurity(StrEnum):
PLAIN = "plain"
TLS = "tls"
STARTTLS = "starttls"
class SmtpConfig(StrictModel):
host: str | None = None
port: int | None = Field(default=None, ge=1, le=65535)
username: str | None = None
password: str | None = None
security: TransportSecurity = TransportSecurity.STARTTLS
timeout_seconds: int = Field(default=30, ge=1)
class ImapConfig(StrictModel):
enabled: bool = False
host: str | None = None
port: int | None = Field(default=None, ge=1, le=65535)
username: str | None = None
password: str | None = None
security: TransportSecurity = TransportSecurity.TLS
sent_folder: str = "auto"
timeout_seconds: int = Field(default=30, ge=1)
timeout_seconds: int = Field(default=30, ge=1, le=300)
max_message_bytes: int = Field(default=25 * 1024 * 1024, ge=1_024, le=50 * 1024 * 1024)
max_batch_bytes: int = Field(default=100 * 1024 * 1024, ge=1_048_576, le=500 * 1024 * 1024)
preview_body_lines: int = Field(default=20, ge=0, le=100)
legacy_import_enabled: bool = False
allow_delete_after_import: bool = False
@model_validator(mode="after")
def apply_default_port(self) -> "Pop3ServerConfig":
if self.port is None:
self.port = 995 if self.security == TransportSecurity.TLS else 110
if self.legacy_import_enabled and not str(self.host or "").strip():
raise ValueError(
"POP3 host is required when legacy import is enabled"
)
if self.max_batch_bytes < self.max_message_bytes:
raise ValueError(
"POP3 batch size limit cannot be lower than the per-message limit"
)
if self.allow_delete_after_import and not self.legacy_import_enabled:
raise ValueError(
"POP3 delete-after-import cannot be enabled while legacy import is disabled"
)
return self
class Pop3Config(Pop3ServerConfig):
username: str | None = None
password: str | None = None
class JmapServerConfig(StrictModel):
"""Server-only settings for an RFC 8620/8621 mailbox endpoint."""
session_url: str
account_id: str | None = Field(default=None, max_length=255)
auth_scheme: Literal["bearer", "basic"] = "bearer"
timeout_seconds: int = Field(default=20, ge=1, le=120)
max_response_bytes: int = Field(
default=5 * 1024 * 1024,
ge=64 * 1024,
le=25 * 1024 * 1024,
)
max_body_value_bytes: int = Field(
default=1 * 1024 * 1024,
ge=1_024,
le=5 * 1024 * 1024,
)
allowed_api_origins: list[str] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def validate_urls(self) -> "JmapServerConfig":
self.session_url = _absolute_http_url(self.session_url, label="JMAP session URL")
session_origin = _http_origin(self.session_url)
origins: list[str] = []
for value in self.allowed_api_origins:
normalized = _http_origin(
_absolute_http_url(value, label="JMAP allowed API origin")
)
if normalized != session_origin and normalized not in origins:
origins.append(normalized)
self.allowed_api_origins = origins
if self.account_id is not None:
self.account_id = self.account_id.strip() or None
return self
class JmapConfig(JmapServerConfig):
username: str | None = Field(default=None, max_length=320)
password: str | None = None
@model_validator(mode="after")
def validate_credentials(self) -> "JmapConfig":
if self.auth_scheme == "basic" and not (self.username and self.password):
raise ValueError("JMAP Basic authentication requires username and password")
if self.auth_scheme == "bearer" and not self.password:
raise ValueError("JMAP Bearer authentication requires an access token")
return self
def _absolute_http_url(value: str, *, label: str) -> str:
parsed = urllib.parse.urlsplit(str(value or "").strip())
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError(f"{label} must be an absolute HTTP(S) URL")
if parsed.username or parsed.password:
raise ValueError(f"{label} must not include embedded credentials")
if parsed.fragment:
raise ValueError(f"{label} must not include a fragment")
return urllib.parse.urlunsplit(parsed)
def _http_origin(value: str) -> str:
parsed = urllib.parse.urlsplit(value)
port = parsed.port or (443 if parsed.scheme == "https" else 80)
default_port = 443 if parsed.scheme == "https" else 80
suffix = "" if port == default_port else f":{port}"
return f"{parsed.scheme.lower()}://{(parsed.hostname or '').lower()}{suffix}"
__all__ = [
"ImapConfig",
"ImapFolderMappings",
"ImapServerConfig",
"JmapConfig",
"JmapServerConfig",
"Pop3Config",
"Pop3ServerConfig",
"SmtpConfig",
"SmtpServerConfig",
"StrictModel",
"TransportCredentials",
"TransportSecurity",
"normalize_split_transport_credentials",
]
File diff suppressed because it is too large Load Diff
+425 -7
View File
@@ -3,10 +3,13 @@ from __future__ import annotations
import uuid
from typing import Any
from sqlalchemy import Boolean, ForeignKey, Index, JSON, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from datetime import datetime
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from govoplan_core.db.base import Base, TimestampMixin
from govoplan_core.security import credential_envelopes as core_credential_models # noqa: F401
def new_uuid() -> str:
@@ -21,20 +24,435 @@ class MailServerProfile(Base, TimestampMixin):
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=True, index=True)
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
slug: Mapped[str] = mapped_column(String(100), nullable=False)
description: Mapped[str | None] = mapped_column(Text)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
inherit_to_lower_scopes: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
smtp_config: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
smtp_username: Mapped[str | None] = mapped_column(String(320))
smtp_password_encrypted: Mapped[str | None] = mapped_column(Text)
smtp_transport_revision: Mapped[str] = mapped_column(String(36), default=new_uuid, nullable=False)
imap_config: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
imap_username: Mapped[str | None] = mapped_column(String(320))
imap_password_encrypted: Mapped[str | None] = mapped_column(Text)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
tenant: Mapped["Tenant"] = relationship()
imap_transport_revision: Mapped[str] = mapped_column(String(36), default=new_uuid, nullable=False)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class MailServerEndpoint(Base, TimestampMixin):
__tablename__ = "mail_server_endpoints"
__table_args__ = (
UniqueConstraint("profile_id", "protocol", "name", name="uq_mail_server_endpoints_profile_protocol_name"),
Index("ix_mail_server_endpoints_profile_protocol", "profile_id", "protocol", "is_active"),
Index("ix_mail_server_endpoints_scope", "tenant_id", "scope_type", "scope_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
profile_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_profiles.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
protocol: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
config: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
inherit_to_lower_scopes: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
is_default: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False, index=True)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
transport_revision: Mapped[str] = mapped_column(String(36), default=new_uuid, nullable=False)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class MailServerCredentialBinding(Base, TimestampMixin):
__tablename__ = "mail_server_credential_bindings"
__table_args__ = (
UniqueConstraint("server_id", "credential_id", name="uq_mail_server_credential_bindings_server_credential"),
Index("ix_mail_server_credential_bindings_default", "server_id", "is_default"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
server_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_endpoints.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
credential_id: Mapped[str] = mapped_column(
ForeignKey("core_credential_envelopes.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
is_default: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False, index=True)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class MailProfilePolicy(Base, TimestampMixin):
__tablename__ = "mail_profile_policies"
__table_args__ = (
UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_mail_profile_policies_scope"),
Index("ix_mail_profile_policies_scope", "scope_type", "scope_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
class MailMailboxFolderIndex(Base, TimestampMixin):
__tablename__ = "mail_mailbox_folder_index"
__table_args__ = (
UniqueConstraint("profile_id", "folder", name="uq_mail_mailbox_folder_index_profile_folder"),
Index("ix_mail_mailbox_folder_index_tenant_profile", "tenant_id", "profile_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(ForeignKey("mail_server_profiles.id", ondelete="CASCADE"), nullable=False, index=True)
folder: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
flags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
message_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
unseen_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
uidvalidity: Mapped[str | None] = mapped_column(String(255), nullable=True)
indexed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
message_indexed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
class MailMailboxMessageIndex(Base, TimestampMixin):
__tablename__ = "mail_mailbox_message_index"
__table_args__ = (
UniqueConstraint("profile_id", "folder", "uid", name="uq_mail_mailbox_message_index_profile_folder_uid"),
Index("ix_mail_mailbox_message_index_page", "tenant_id", "profile_id", "folder", "sort_position"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(ForeignKey("mail_server_profiles.id", ondelete="CASCADE"), nullable=False, index=True)
folder: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
uid: Mapped[str] = mapped_column(String(255), nullable=False)
uid_int: Mapped[int] = mapped_column(BigInteger, default=0, nullable=False, index=True)
sort_position: Mapped[int] = mapped_column(BigInteger, default=0, nullable=False, index=True)
subject: Mapped[str | None] = mapped_column(Text, nullable=True)
from_header: Mapped[str | None] = mapped_column(Text, nullable=True)
to_header: Mapped[str | None] = mapped_column(Text, nullable=True)
cc_header: Mapped[str | None] = mapped_column(Text, nullable=True)
date: Mapped[str | None] = mapped_column(String(255), nullable=True)
message_id: Mapped[str | None] = mapped_column(Text, nullable=True)
flags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
body_preview: Mapped[str | None] = mapped_column(Text, nullable=True)
attachment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
indexed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
class MailDeliveryCommand(Base, TimestampMixin):
__tablename__ = "mail_delivery_commands"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"command_type",
"idempotency_key",
name="uq_mail_delivery_commands_idempotency",
),
Index(
"ix_mail_delivery_commands_dispatch",
"status",
"next_attempt_at",
"created_at",
),
Index(
"ix_mail_delivery_commands_source",
"tenant_id",
"source_module",
"source_resource_type",
"source_resource_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
command_type: Mapped[str] = mapped_column(String(60), nullable=False, index=True)
source_module: Mapped[str] = mapped_column(String(80), nullable=False, index=True)
source_resource_type: Mapped[str] = mapped_column(String(80), nullable=False)
source_resource_id: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_version_id: Mapped[str | None] = mapped_column(String(120), nullable=True)
idempotency_key: Mapped[str] = mapped_column(String(200), nullable=False)
canonical_request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
profile_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_profiles.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
smtp_server_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
smtp_credential_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
expected_smtp_transport_revision: Mapped[str] = mapped_column(String(120), nullable=False)
envelope_from_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
envelope_recipients_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
from_header_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
message_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
message_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
rfc_message_id: Mapped[str | None] = mapped_column(
String(998), nullable=True, index=True
)
message_size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
recipient_count: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(40), nullable=False, default="pending", index=True)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
next_attempt_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True),
nullable=True,
index=True,
)
claimed_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
effect_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
accepted_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
refused_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
refusal_summary: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict)
refusal_details_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
failure_code: Mapped[str | None] = mapped_column(String(80), nullable=True)
failure_summary: Mapped[str | None] = mapped_column(String(500), nullable=True)
created_by_user_id: Mapped[str | None] = mapped_column(
ForeignKey("access_users.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
supersedes_command_id: Mapped[str | None] = mapped_column(
ForeignKey("mail_delivery_commands.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
payload_purged_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class MailDeliveryAttempt(Base, TimestampMixin):
__tablename__ = "mail_delivery_attempts"
__table_args__ = (
UniqueConstraint(
"command_id",
"attempt_number",
name="uq_mail_delivery_attempts_number",
),
Index("ix_mail_delivery_attempts_command_started", "command_id", "started_at"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
command_id: Mapped[str] = mapped_column(
ForeignKey("mail_delivery_commands.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
attempt_number: Mapped[int] = mapped_column(Integer, nullable=False)
worker_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
status: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
effect_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
accepted_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
refused_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
outcome_code: Mapped[str | None] = mapped_column(String(80), nullable=True)
diagnostic_summary: Mapped[str | None] = mapped_column(String(500), nullable=True)
class MailDeliveryReconciliation(Base, TimestampMixin):
__tablename__ = "mail_delivery_reconciliations"
__table_args__ = (
Index(
"ix_mail_delivery_reconciliations_command_created",
"command_id",
"created_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
command_id: Mapped[str] = mapped_column(
ForeignKey("mail_delivery_commands.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
decision: Mapped[str] = mapped_column(String(40), nullable=False)
evidence_reference: Mapped[str] = mapped_column(String(500), nullable=False)
note_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
created_by_user_id: Mapped[str | None] = mapped_column(
ForeignKey("access_users.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
class MailBounceSource(Base, TimestampMixin):
__tablename__ = "mail_bounce_sources"
__table_args__ = (
UniqueConstraint(
"profile_id",
"folder",
name="uq_mail_bounce_sources_profile_folder",
),
Index(
"ix_mail_bounce_sources_scan",
"is_active",
"last_scanned_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_profiles.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
folder: Mapped[str] = mapped_column(
String(255), default="INBOX", nullable=False
)
imap_server_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
imap_credential_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
expected_imap_transport_revision: Mapped[str] = mapped_column(
String(120), nullable=False
)
is_active: Mapped[bool] = mapped_column(
Boolean, default=True, nullable=False, index=True
)
uidvalidity: Mapped[str | None] = mapped_column(String(255), nullable=True)
highest_processed_uid: Mapped[int] = mapped_column(
BigInteger, default=0, nullable=False
)
last_scanned_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, index=True
)
last_success_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
last_error: Mapped[str | None] = mapped_column(String(500), nullable=True)
created_by_user_id: Mapped[str | None] = mapped_column(
ForeignKey("access_users.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
class MailBounceObservation(Base, TimestampMixin):
__tablename__ = "mail_bounce_observations"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"fingerprint",
name="uq_mail_bounce_observations_fingerprint",
),
Index(
"ix_mail_bounce_observations_command",
"tenant_id",
"command_id",
"observed_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_profiles.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
folder: Mapped[str] = mapped_column(String(255), nullable=False)
uid: Mapped[str] = mapped_column(String(255), nullable=False)
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
raw_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
original_message_id: Mapped[str | None] = mapped_column(
String(998), nullable=True, index=True
)
command_id: Mapped[str | None] = mapped_column(
ForeignKey("mail_delivery_commands.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
recipient: Mapped[str | None] = mapped_column(String(998), nullable=True)
action: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
status_code: Mapped[str | None] = mapped_column(String(80), nullable=True)
diagnostic: Mapped[str | None] = mapped_column(String(500), nullable=True)
permanent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
observed_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
matched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
class MailPop3Import(Base, TimestampMixin):
"""Governed local review record created from a legacy POP3 mailbox."""
__tablename__ = "mail_pop3_imports"
__table_args__ = (
UniqueConstraint(
"tenant_id",
"profile_id",
"pop3_server_id",
"provider_uidl",
name="uq_mail_pop3_imports_source_uidl",
),
Index(
"ix_mail_pop3_imports_review",
"tenant_id",
"status",
"imported_at",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_profiles.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
pop3_server_id: Mapped[str] = mapped_column(
ForeignKey("mail_server_endpoints.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
pop3_credential_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
transport_revision: Mapped[str] = mapped_column(String(120), nullable=False)
provider_uidl: Mapped[str] = mapped_column(String(500), nullable=False)
provider_message_number: Mapped[int | None] = mapped_column(Integer, nullable=True)
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
raw_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
raw_message_encrypted: Mapped[str] = mapped_column(Text, nullable=False)
message_id: Mapped[str | None] = mapped_column(String(998), nullable=True, index=True)
subject: Mapped[str | None] = mapped_column(Text, nullable=True)
from_header: Mapped[str | None] = mapped_column(Text, nullable=True)
to_header: Mapped[str | None] = mapped_column(Text, nullable=True)
date: Mapped[str | None] = mapped_column(String(255), nullable=True)
body_preview: Mapped[str | None] = mapped_column(Text, nullable=True)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
status: Mapped[str] = mapped_column(
String(40), default="pending_review", nullable=False, index=True
)
imported_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True
)
imported_by_user_id: Mapped[str | None] = mapped_column(
ForeignKey("access_users.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
deletion_requested: Mapped[bool] = mapped_column(
Boolean, default=False, nullable=False
)
deletion_status: Mapped[str] = mapped_column(
String(40), default="not_requested", nullable=False, index=True
)
deletion_attempted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
deletion_error: Mapped[str | None] = mapped_column(String(500), nullable=True)
@@ -0,0 +1,919 @@
from __future__ import annotations
import base64
import hashlib
import json
from collections import Counter
from datetime import datetime, timedelta, timezone
from email import policy
from email.parser import BytesParser
from typing import Any
from sqlalchemy import or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_mail.backend.capabilities import send_campaign_email_bytes
from govoplan_mail.backend.db.models import (
MailDeliveryAttempt,
MailDeliveryCommand,
MailDeliveryReconciliation,
)
from govoplan_mail.backend.mail_profiles import MailProfileError
from govoplan_mail.backend.recovery import (
MailRecoveryError,
reconcile_outbox_provider_effect,
)
from govoplan_mail.backend.sending.smtp import SmtpConfigurationError, SmtpSendError
DISPATCHABLE_STATUSES = frozenset({"pending", "temporary_failure", "reconciled_not_accepted"})
TERMINAL_STATUSES = frozenset(
{
"accepted",
"partially_refused",
"permanent_failure",
"outcome_unknown",
"reconciled_accepted",
"cancelled",
}
)
NON_RETRYABLE_STATUSES = TERMINAL_STATUSES | frozenset({"claimed", "in_progress"})
DEFAULT_PAYLOAD_RETENTION_DAYS = 30
STALE_CLAIM_AFTER = timedelta(minutes=10)
class MailDeliveryError(RuntimeError):
pass
class MailDeliveryIdempotencyConflict(MailDeliveryError):
pass
class MailDeliveryNotFound(MailDeliveryError):
pass
class MailDeliveryStateError(MailDeliveryError):
pass
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def _bounded_text(value: object | None, *, limit: int = 500) -> str | None:
if value is None:
return None
candidate = " ".join(str(value).split())
return candidate[:limit] or None
def _canonical_hash(payload: dict[str, object]) -> str:
encoded = json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _encrypt_json(value: object) -> str:
encrypted = encrypt_secret(
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
)
if not encrypted:
raise MailDeliveryError("Mail delivery evidence could not be encrypted")
return encrypted
def _decrypt_json(value: str | None) -> Any:
plaintext = decrypt_secret(value)
if plaintext is None:
return None
return json.loads(plaintext)
def _message_bytes(command: MailDeliveryCommand) -> bytes:
encoded = decrypt_secret(command.message_encrypted)
if encoded is None:
raise MailDeliveryStateError("Mail delivery payload is no longer available")
try:
message = base64.b64decode(encoded.encode("ascii"), validate=True)
except (ValueError, TypeError) as exc:
raise MailDeliveryStateError("Mail delivery payload is invalid") from exc
if hashlib.sha256(message).hexdigest() != command.message_sha256:
raise MailDeliveryStateError("Mail delivery payload integrity check failed")
return message
def _rfc_message_id(message: bytes) -> str | None:
try:
value = BytesParser(policy=policy.default).parsebytes(
message,
headersonly=True,
).get("Message-ID")
except Exception:
return None
normalized = " ".join(str(value or "").split())
return normalized[:998] or None
def _delivery_payload(
*,
command_type: str,
source_module: str,
source_resource_type: str,
source_resource_id: str | None,
source_version_id: str | None,
profile_id: str,
smtp_server_id: str | None,
smtp_credential_id: str | None,
expected_smtp_transport_revision: str,
envelope_from: str,
envelope_recipients: list[str],
from_header: str | None,
message_sha256: str,
) -> dict[str, object]:
return {
"command_type": command_type,
"source_module": source_module,
"source_resource_type": source_resource_type,
"source_resource_id": source_resource_id,
"source_version_id": source_version_id,
"profile_id": profile_id,
"smtp_server_id": smtp_server_id,
"smtp_credential_id": smtp_credential_id,
"expected_smtp_transport_revision": expected_smtp_transport_revision,
"envelope_from": envelope_from,
"envelope_recipients": envelope_recipients,
"from_header": from_header,
"message_sha256": message_sha256,
}
def submit_delivery_command(
session: Session,
*,
tenant_id: str,
command_type: str,
source_module: str,
source_resource_type: str,
source_resource_id: str | None,
source_version_id: str | None,
idempotency_key: str,
profile_id: str,
message_bytes: bytes,
envelope_from: str,
envelope_recipients: list[str],
from_header: str | None,
expected_smtp_transport_revision: str,
smtp_server_id: str | None = None,
smtp_credential_id: str | None = None,
created_by_user_id: str | None = None,
retention_days: int = DEFAULT_PAYLOAD_RETENTION_DAYS,
supersedes_command_id: str | None = None,
) -> dict[str, object]:
clean_key = idempotency_key.strip()
clean_recipients = [str(value).strip() for value in envelope_recipients if str(value).strip()]
if not clean_key or len(clean_key) > 200:
raise MailDeliveryError("A bounded idempotency key is required")
if not clean_recipients:
raise MailDeliveryError("At least one envelope recipient is required")
if retention_days < 1:
raise MailDeliveryError("Mail payload retention must be at least one day")
digest = hashlib.sha256(message_bytes).hexdigest()
rfc_message_id = _rfc_message_id(message_bytes)
request_hash = _canonical_hash(
_delivery_payload(
command_type=command_type,
source_module=source_module,
source_resource_type=source_resource_type,
source_resource_id=source_resource_id,
source_version_id=source_version_id,
profile_id=profile_id,
smtp_server_id=smtp_server_id,
smtp_credential_id=smtp_credential_id,
expected_smtp_transport_revision=expected_smtp_transport_revision,
envelope_from=envelope_from,
envelope_recipients=clean_recipients,
from_header=from_header,
message_sha256=digest,
)
)
existing = session.scalar(
select(MailDeliveryCommand).where(
MailDeliveryCommand.tenant_id == tenant_id,
MailDeliveryCommand.command_type == command_type,
MailDeliveryCommand.idempotency_key == clean_key,
)
)
if existing is not None:
if existing.canonical_request_hash != request_hash:
raise MailDeliveryIdempotencyConflict(
"The idempotency key is already bound to a different mail command"
)
return delivery_command_summary(existing, duplicate=True)
now = utcnow()
command = MailDeliveryCommand(
tenant_id=tenant_id,
command_type=command_type,
source_module=source_module,
source_resource_type=source_resource_type,
source_resource_id=source_resource_id,
source_version_id=source_version_id,
idempotency_key=clean_key,
canonical_request_hash=request_hash,
profile_id=profile_id,
smtp_server_id=smtp_server_id,
smtp_credential_id=smtp_credential_id,
expected_smtp_transport_revision=expected_smtp_transport_revision,
envelope_from_encrypted=encrypt_secret(envelope_from),
envelope_recipients_encrypted=_encrypt_json(clean_recipients),
from_header_encrypted=encrypt_secret(from_header),
message_encrypted=encrypt_secret(base64.b64encode(message_bytes).decode("ascii")),
message_sha256=digest,
rfc_message_id=rfc_message_id,
message_size_bytes=len(message_bytes),
recipient_count=len(clean_recipients),
status="pending",
next_attempt_at=now,
created_by_user_id=created_by_user_id,
supersedes_command_id=supersedes_command_id,
expires_at=now + timedelta(days=retention_days),
)
try:
with session.begin_nested():
session.add(command)
session.flush()
except IntegrityError:
existing = session.scalar(
select(MailDeliveryCommand).where(
MailDeliveryCommand.tenant_id == tenant_id,
MailDeliveryCommand.command_type == command_type,
MailDeliveryCommand.idempotency_key == clean_key,
)
)
if existing is None or existing.canonical_request_hash != request_hash:
raise MailDeliveryIdempotencyConflict(
"The idempotency key is already bound to a different mail command"
) from None
return delivery_command_summary(existing, duplicate=True)
audit_event(
session,
tenant_id=tenant_id,
user_id=created_by_user_id,
action="mail.delivery_requested",
object_type="mail_delivery_command",
object_id=command.id,
details={
"command_type": command_type,
"source_module": source_module,
"source_resource_type": source_resource_type,
"source_resource_id": source_resource_id,
"recipient_count": len(clean_recipients),
"message_sha256": digest,
"supersedes_command_id": supersedes_command_id,
},
)
return delivery_command_summary(command)
def get_delivery_command(
session: Session,
*,
tenant_id: str,
command_id: str,
) -> MailDeliveryCommand:
command = session.get(MailDeliveryCommand, command_id)
if command is None or command.tenant_id != tenant_id:
raise MailDeliveryNotFound("Mail delivery command not found")
return command
def delivery_command_summary(
command: MailDeliveryCommand,
*,
duplicate: bool = False,
) -> dict[str, object]:
return {
"id": command.id,
"tenant_id": command.tenant_id,
"command_type": command.command_type,
"source_module": command.source_module,
"source_resource_type": command.source_resource_type,
"source_resource_id": command.source_resource_id,
"source_version_id": command.source_version_id,
"status": command.status,
"recipient_count": command.recipient_count,
"accepted_count": command.accepted_count,
"refused_count": command.refused_count,
"refusal_summary": dict(command.refusal_summary or {}),
"attempt_count": command.attempt_count,
"failure_code": command.failure_code,
"failure_summary": command.failure_summary,
"message_sha256": command.message_sha256,
"rfc_message_id": command.rfc_message_id,
"message_size_bytes": command.message_size_bytes,
"created_at": command.created_at,
"completed_at": command.completed_at,
"expires_at": command.expires_at,
"payload_purged_at": command.payload_purged_at,
"safe_to_retry": command.status in DISPATCHABLE_STATUSES,
"outcome_known": command.status
not in {"claimed", "in_progress", "outcome_unknown"},
"duplicate": duplicate,
}
def delivery_command_diagnostics(
session: Session,
*,
tenant_id: str,
command_id: str,
) -> dict[str, object]:
command = get_delivery_command(session, tenant_id=tenant_id, command_id=command_id)
attempts = session.scalars(
select(MailDeliveryAttempt)
.where(MailDeliveryAttempt.command_id == command.id)
.order_by(MailDeliveryAttempt.attempt_number)
).all()
reconciliations = session.scalars(
select(MailDeliveryReconciliation)
.where(MailDeliveryReconciliation.command_id == command.id)
.order_by(MailDeliveryReconciliation.created_at)
).all()
refusals = _decrypt_json(command.refusal_details_encrypted) or {}
return {
**delivery_command_summary(command),
"refused_recipients": refusals,
"attempts": [
{
"id": attempt.id,
"attempt_number": attempt.attempt_number,
"worker_id": attempt.worker_id,
"status": attempt.status,
"started_at": attempt.started_at,
"effect_started_at": attempt.effect_started_at,
"completed_at": attempt.completed_at,
"accepted_count": attempt.accepted_count,
"refused_count": attempt.refused_count,
"outcome_code": attempt.outcome_code,
"diagnostic_summary": attempt.diagnostic_summary,
}
for attempt in attempts
],
"reconciliations": [
{
"id": item.id,
"decision": item.decision,
"evidence_reference": item.evidence_reference,
"note": decrypt_secret(item.note_encrypted),
"created_by_user_id": item.created_by_user_id,
"created_at": item.created_at,
}
for item in reconciliations
],
}
def _current_attempt(
session: Session,
command: MailDeliveryCommand,
) -> MailDeliveryAttempt | None:
return session.scalar(
select(MailDeliveryAttempt)
.where(
MailDeliveryAttempt.command_id == command.id,
MailDeliveryAttempt.attempt_number == command.attempt_count,
)
.limit(1)
)
def _recover_stale_commands(
session: Session,
*,
now: datetime,
tenant_id: str | None,
) -> tuple[int, int]:
cutoff = now - STALE_CLAIM_AFTER
clauses = [
MailDeliveryCommand.status.in_(("claimed", "in_progress")),
MailDeliveryCommand.claimed_at <= cutoff,
]
if tenant_id:
clauses.append(MailDeliveryCommand.tenant_id == tenant_id)
commands = session.scalars(
select(MailDeliveryCommand).where(*clauses).order_by(MailDeliveryCommand.claimed_at)
).all()
recovered = 0
unknown = 0
for command in commands:
attempt = _current_attempt(session, command)
effect_started = command.effect_started_at or (
attempt.effect_started_at if attempt else None
)
if effect_started is None:
command.status = "pending"
command.next_attempt_at = now
command.claimed_at = None
command.claimed_by = None
if attempt is not None:
attempt.status = "claim_abandoned"
attempt.completed_at = now
attempt.outcome_code = "worker_lost_before_effect"
recovered += 1
continue
command.status = "outcome_unknown"
command.completed_at = now
command.next_attempt_at = None
command.failure_code = "worker_lost_after_effect_start"
command.failure_summary = (
"Delivery outcome is unknown because the worker stopped after transmission began."
)
if attempt is not None:
attempt.status = "outcome_unknown"
attempt.completed_at = now
attempt.outcome_code = command.failure_code
attempt.diagnostic_summary = command.failure_summary
unknown += 1
if commands:
session.commit()
return recovered, unknown
def _claim_command(
session: Session,
*,
command_id: str,
now: datetime,
worker_id: str | None,
) -> tuple[MailDeliveryCommand, MailDeliveryAttempt] | None:
command = session.scalar(
select(MailDeliveryCommand)
.where(
MailDeliveryCommand.id == command_id,
MailDeliveryCommand.status.in_(DISPATCHABLE_STATUSES),
or_(
MailDeliveryCommand.next_attempt_at.is_(None),
MailDeliveryCommand.next_attempt_at <= now,
),
MailDeliveryCommand.payload_purged_at.is_(None),
)
.with_for_update(skip_locked=True)
)
if command is None:
session.rollback()
return None
command.attempt_count += 1
command.status = "claimed"
command.claimed_by = _bounded_text(worker_id, limit=255)
command.claimed_at = now
command.effect_started_at = None
command.next_attempt_at = None
command.failure_code = None
command.failure_summary = None
attempt = MailDeliveryAttempt(
command_id=command.id,
attempt_number=command.attempt_count,
worker_id=command.claimed_by,
status="claimed",
started_at=now,
)
session.add(attempt)
session.commit()
return command, attempt
def _mark_effect_started(
session: Session,
command: MailDeliveryCommand,
attempt: MailDeliveryAttempt,
) -> None:
now = utcnow()
command.status = "in_progress"
command.effect_started_at = now
attempt.status = "in_progress"
attempt.effect_started_at = now
session.commit()
def _refusal_summary(refusals: dict[str, dict[str, int | str]]) -> dict[str, int]:
classifications = Counter(
str(item.get("classification") or "unknown")
for item in refusals.values()
)
return dict(sorted(classifications.items()))
def _record_outcome(
session: Session,
*,
command: MailDeliveryCommand,
attempt: MailDeliveryAttempt,
status: str,
accepted_count: int = 0,
refusals: dict[str, dict[str, int | str]] | None = None,
failure_code: str | None = None,
failure_summary: str | None = None,
) -> None:
now = utcnow()
refusal_map = refusals or {}
command.status = status
command.accepted_count = accepted_count
command.refused_count = len(refusal_map)
command.refusal_summary = _refusal_summary(refusal_map)
command.refusal_details_encrypted = (
_encrypt_json(refusal_map) if refusal_map else None
)
command.failure_code = failure_code
command.failure_summary = _bounded_text(failure_summary)
command.claimed_by = None
command.claimed_at = None
command.next_attempt_at = (
now + timedelta(minutes=min(60, 2 ** min(command.attempt_count, 5)))
if status == "temporary_failure"
else None
)
if status != "temporary_failure":
command.completed_at = now
attempt.status = status
attempt.completed_at = now
attempt.accepted_count = accepted_count
attempt.refused_count = len(refusal_map)
attempt.outcome_code = failure_code or status
attempt.diagnostic_summary = _bounded_text(failure_summary)
session.commit()
audit_event(
session,
tenant_id=command.tenant_id,
user_id=command.created_by_user_id,
action="mail.delivery_completed",
object_type="mail_delivery_command",
object_id=command.id,
details={
"status": status,
"attempt_number": attempt.attempt_number,
"accepted_count": accepted_count,
"refused_count": len(refusal_map),
"refusal_summary": command.refusal_summary,
"failure_code": failure_code,
"source_module": command.source_module,
"source_resource_id": command.source_resource_id,
},
)
session.commit()
def _process_claimed(
session: Session,
command: MailDeliveryCommand,
attempt: MailDeliveryAttempt,
) -> str:
try:
message = _message_bytes(command)
envelope_from = decrypt_secret(command.envelope_from_encrypted)
recipients = _decrypt_json(command.envelope_recipients_encrypted)
from_header = decrypt_secret(command.from_header_encrypted)
if not envelope_from or not isinstance(recipients, list) or not recipients:
raise MailDeliveryStateError("Mail delivery envelope is unavailable")
except MailDeliveryStateError as exc:
_record_outcome(
session,
command=command,
attempt=attempt,
status="permanent_failure",
failure_code="payload_unavailable",
failure_summary=str(exc),
)
return "permanent_failure"
_mark_effect_started(session, command, attempt)
try:
result = send_campaign_email_bytes(
session,
tenant_id=command.tenant_id,
campaign_id=str(command.source_resource_id or ""),
profile_id=command.profile_id,
message_bytes=message,
envelope_from=envelope_from,
envelope_recipients=[str(item) for item in recipients],
from_header=from_header,
expected_smtp_transport_revision=command.expected_smtp_transport_revision,
smtp_server_id=command.smtp_server_id,
smtp_credential_id=command.smtp_credential_id,
recovery_effect_id=(
f"outbox:{command.id}:smtp-attempt:{attempt.attempt_number}"
),
recovery_resource_type="mail_delivery_command",
recovery_resource_id=command.id,
)
except SmtpSendError as exc:
if exc.outcome_unknown:
status = "outcome_unknown"
code = "smtp_outcome_unknown"
elif exc.temporary:
status = "temporary_failure"
code = "smtp_temporary_failure"
else:
status = "permanent_failure"
code = "smtp_permanent_failure"
_record_outcome(
session,
command=command,
attempt=attempt,
status=status,
failure_code=code,
failure_summary=str(exc),
)
return status
except (MailProfileError, SmtpConfigurationError) as exc:
_record_outcome(
session,
command=command,
attempt=attempt,
status="permanent_failure",
failure_code="authorization_or_configuration_changed",
failure_summary=str(exc),
)
return "permanent_failure"
except Exception:
_record_outcome(
session,
command=command,
attempt=attempt,
status="outcome_unknown",
failure_code="unexpected_error_after_effect_start",
failure_summary="Mail delivery outcome is unknown after transmission began.",
)
return "outcome_unknown"
refusals = dict(result.refused_recipients)
accepted_count = result.accepted_count
if not refusals:
status = "accepted"
elif accepted_count > 0:
status = "partially_refused"
elif all(
item.get("classification") == "temporary" for item in refusals.values()
):
status = "temporary_failure"
elif any(
item.get("classification") == "unknown" for item in refusals.values()
):
status = "outcome_unknown"
else:
status = "permanent_failure"
_record_outcome(
session,
command=command,
attempt=attempt,
status=status,
accepted_count=accepted_count,
refusals=refusals,
failure_code=None if status == "accepted" else f"smtp_{status}",
failure_summary=None if status == "accepted" else "One or more recipients were refused.",
)
return status
def dispatch_due(
session: Session,
*,
tenant_id: str | None = None,
limit: int = 25,
worker_id: str | None = None,
) -> dict[str, object]:
bounded_limit = max(1, min(int(limit), 100))
now = utcnow()
recovered, recovered_unknown = _recover_stale_commands(
session,
now=now,
tenant_id=tenant_id,
)
clauses = [
MailDeliveryCommand.status.in_(DISPATCHABLE_STATUSES),
MailDeliveryCommand.payload_purged_at.is_(None),
or_(
MailDeliveryCommand.next_attempt_at.is_(None),
MailDeliveryCommand.next_attempt_at <= now,
),
]
if tenant_id:
clauses.append(MailDeliveryCommand.tenant_id == tenant_id)
command_ids = list(
session.scalars(
select(MailDeliveryCommand.id)
.where(*clauses)
.order_by(MailDeliveryCommand.next_attempt_at, MailDeliveryCommand.created_at)
.limit(bounded_limit)
).all()
)
counters: Counter[str] = Counter()
processed_ids: list[str] = []
for command_id in command_ids:
claimed = _claim_command(
session,
command_id=command_id,
now=utcnow(),
worker_id=worker_id,
)
if claimed is None:
continue
command, attempt = claimed
outcome = _process_claimed(session, command, attempt)
counters[outcome] += 1
processed_ids.append(command.id)
return {
"selected": len(processed_ids),
"accepted": counters["accepted"],
"partially_refused": counters["partially_refused"],
"retrying": counters["temporary_failure"],
"failed": counters["permanent_failure"],
"outcome_unknown": counters["outcome_unknown"] + recovered_unknown,
"recovered_before_effect": recovered,
"command_ids": processed_ids,
}
def reconcile_delivery_command(
session: Session,
*,
tenant_id: str,
command_id: str,
decision: str,
evidence_reference: str,
note: str | None,
user_id: str,
) -> dict[str, object]:
command = get_delivery_command(session, tenant_id=tenant_id, command_id=command_id)
if command.status not in {"outcome_unknown", "in_progress"}:
raise MailDeliveryStateError(
"Only a delivery with an unknown outcome can be reconciled"
)
clean_decision = decision.strip().casefold()
if clean_decision not in {"accepted", "not_accepted"}:
raise MailDeliveryStateError(
"Reconciliation decision must be accepted or not_accepted"
)
clean_evidence = _bounded_text(evidence_reference)
if not clean_evidence:
raise MailDeliveryStateError("An evidence reference is required")
try:
reconcile_outbox_provider_effect(
command_id=command.id,
effect_occurred=clean_decision == "accepted",
evidence_reference=clean_evidence,
user_id=user_id,
)
except MailRecoveryError as exc:
raise MailDeliveryStateError(str(exc)) from exc
item = MailDeliveryReconciliation(
command_id=command.id,
decision=clean_decision,
evidence_reference=clean_evidence,
note_encrypted=encrypt_secret(note),
created_by_user_id=user_id,
)
session.add(item)
command.status = (
"reconciled_accepted"
if clean_decision == "accepted"
else "reconciled_not_accepted"
)
command.completed_at = utcnow() if clean_decision == "accepted" else None
command.next_attempt_at = None
command.failure_code = f"reconciled_{clean_decision}"
command.failure_summary = "Delivery outcome was reconciled from external evidence."
audit_event(
session,
tenant_id=tenant_id,
user_id=user_id,
action="mail.delivery_reconciled",
object_type="mail_delivery_command",
object_id=command.id,
details={
"decision": clean_decision,
"evidence_reference": clean_evidence,
},
)
session.commit()
return delivery_command_summary(command)
def resend_delivery_command(
session: Session,
*,
tenant_id: str,
command_id: str,
idempotency_key: str,
user_id: str,
) -> dict[str, object]:
command = get_delivery_command(session, tenant_id=tenant_id, command_id=command_id)
if command.status not in {
"outcome_unknown",
"reconciled_not_accepted",
"permanent_failure",
"partially_refused",
}:
raise MailDeliveryStateError(
"A deliberate resend is only available after a terminal or reconciled failure"
)
if command.payload_purged_at is not None:
raise MailDeliveryStateError("The retained delivery payload is no longer available")
message = _message_bytes(command)
recipients = _decrypt_json(command.envelope_recipients_encrypted)
envelope_from = decrypt_secret(command.envelope_from_encrypted)
if not envelope_from or not isinstance(recipients, list):
raise MailDeliveryStateError("The retained delivery envelope is unavailable")
result = submit_delivery_command(
session,
tenant_id=tenant_id,
command_type=command.command_type,
source_module=command.source_module,
source_resource_type=command.source_resource_type,
source_resource_id=command.source_resource_id,
source_version_id=command.source_version_id,
idempotency_key=idempotency_key,
profile_id=command.profile_id,
message_bytes=message,
envelope_from=envelope_from,
envelope_recipients=[str(item) for item in recipients],
from_header=decrypt_secret(command.from_header_encrypted),
expected_smtp_transport_revision=command.expected_smtp_transport_revision,
smtp_server_id=command.smtp_server_id,
smtp_credential_id=command.smtp_credential_id,
created_by_user_id=user_id,
retention_days=max(
1,
(command.expires_at - utcnow()).days,
),
supersedes_command_id=command.id,
)
audit_event(
session,
tenant_id=tenant_id,
user_id=user_id,
action="mail.delivery_resend_requested",
object_type="mail_delivery_command",
object_id=str(result["id"]),
details={"supersedes_command_id": command.id},
)
session.commit()
return result
def purge_expired(
session: Session,
*,
tenant_id: str | None = None,
limit: int = 250,
) -> dict[str, object]:
now = utcnow()
clauses = [
MailDeliveryCommand.expires_at <= now,
MailDeliveryCommand.payload_purged_at.is_(None),
]
if tenant_id:
clauses.append(MailDeliveryCommand.tenant_id == tenant_id)
commands = session.scalars(
select(MailDeliveryCommand)
.where(*clauses)
.order_by(MailDeliveryCommand.expires_at)
.limit(max(1, min(int(limit), 1000)))
).all()
for command in commands:
command.envelope_from_encrypted = None
command.envelope_recipients_encrypted = None
command.from_header_encrypted = None
command.message_encrypted = None
command.refusal_details_encrypted = None
command.payload_purged_at = now
reconciliations = session.scalars(
select(MailDeliveryReconciliation)
.join(
MailDeliveryCommand,
MailDeliveryCommand.id == MailDeliveryReconciliation.command_id,
)
.where(
MailDeliveryCommand.payload_purged_at == now,
MailDeliveryReconciliation.note_encrypted.is_not(None),
)
).all()
for reconciliation in reconciliations:
reconciliation.note_encrypted = None
session.commit()
return {"purged": len(commands)}
class MailDeliveryOutboxCapability:
dispatch_due = staticmethod(dispatch_due)
purge_expired = staticmethod(purge_expired)
+1 -1
View File
@@ -5,7 +5,7 @@ from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, ConfigDict, Field
from govoplan_core.auth.dependencies import ApiPrincipal, require_scope
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_mail.backend.dev.mock_mailbox import (
clear_records,
get_failures,
+600
View File
@@ -0,0 +1,600 @@
from __future__ import annotations
from typing import Any
from sqlalchemy.orm import Session
from govoplan_core.core.modules import (
DocumentationCondition,
DocumentationConfigurationDecision,
DocumentationContext,
DocumentationLink,
DocumentationTopic,
)
from govoplan_mail.backend.mail_profiles import (
EffectiveMailProfilePolicy,
MailProfileError,
effective_mail_profile_policy_for_scope,
)
MAIL_POLICY_DOC_SCOPES = ("mail:profile:read", "admin:policies:read", "system:settings:read")
MAIL_PROFILE_READ_SCOPE = "mail:profile:read"
MAIL_PROFILE_WRITE_SCOPE = "mail:profile:write"
MAIL_PROFILE_WRITE_OWN_SCOPE = "mail:profile:write_own"
# RBAC permission identifiers; neither value is a stored credential.
MAIL_SECRET_MANAGE_SCOPE = "mail:secret:manage" # noqa: S105 # nosec B105
MAIL_SECRET_MANAGE_OWN_SCOPE = "mail:secret:manage_own" # noqa: S105 # nosec B105
MAIL_PROFILE_TEST_SCOPE = "mail:profile:test"
MAIL_PROFILE_USE_SCOPE = "mail:profile:use"
_HOST_POLICY_FIELDS = (("SMTP", "smtp_hosts"), ("IMAP", "imap_hosts"), ("JMAP", "jmap_hosts"))
def documentation_topics(context: DocumentationContext) -> tuple[DocumentationTopic, ...]:
topics = [
topic
for topic in (
_tenant_mail_policy_topic(context),
_custom_mail_profile_topic(context),
)
if topic is not None
]
return tuple(topics)
def documentation_configuration_states(
context: DocumentationContext,
keys: tuple[str, ...],
) -> dict[str, DocumentationConfigurationDecision]:
if "mail_profile_policy" not in keys:
return {}
principal = context.principal
tenant_id = str(getattr(principal, "tenant_id", "") or "")
session = context.session
if not tenant_id or not isinstance(session, Session):
return {
"mail_profile_policy": DocumentationConfigurationDecision(
key="mail_profile_policy",
state="unavailable",
reason="The effective tenant policy cannot be evaluated in this context.",
)
}
try:
policy = effective_mail_profile_policy_for_scope(
session,
tenant_id=tenant_id,
scope_type="tenant",
)
except MailProfileError:
return {
"mail_profile_policy": DocumentationConfigurationDecision(
key="mail_profile_policy",
state="unavailable",
reason="The effective tenant policy could not be evaluated.",
)
}
tenant_sources = [
source
for source in policy.source_policies
if source.get("scope_type") == "tenant"
]
explicitly_configured = any(
bool(source.get("applied_fields"))
for source in tenant_sources
)
return {
"mail_profile_policy": DocumentationConfigurationDecision(
key="mail_profile_policy",
state="enabled" if explicitly_configured else "inherited",
source="tenant" if explicitly_configured else "system default",
reason=(
"A tenant policy is configured."
if explicitly_configured
else "The tenant uses the inherited system policy."
),
)
}
def _tenant_mail_policy_topic(context: DocumentationContext) -> DocumentationTopic | None:
principal = context.principal
tenant_id = str(getattr(principal, "tenant_id", "") or "")
if not tenant_id:
return None
if context.documentation_type == "admin" and not _has_any_scope(principal, MAIL_POLICY_DOC_SCOPES):
return None
session = context.session
if not isinstance(session, Session):
return None
try:
policy = effective_mail_profile_policy_for_scope(session, tenant_id=tenant_id, scope_type="tenant")
except MailProfileError as exc:
user_documentation = context.documentation_type == "user"
return DocumentationTopic(
id="mail.tenant-profile-policy-unavailable",
title="Mail server policy could not be evaluated",
summary="Mail profile documentation is installed, but the tenant-level effective policy could not be read for this request.",
body=(
"The current Mail policy could not be loaded. Try again or ask a Mail administrator for help."
if user_documentation
else str(exc)
),
layer="available",
documentation_types=(context.documentation_type,),
source_module_id="mail",
order=41,
links=(
(DocumentationLink(label="Public mail help", href="https://govplan.add-ideas.de/modules/mail", kind="public"),)
if user_documentation
else (_mail_policy_api_link(),)
),
metadata={} if user_documentation else {"error_type": type(exc).__name__},
)
effective = policy.as_dict()
if context.documentation_type == "user":
summary, body = _mail_policy_user_text(effective)
return DocumentationTopic(
id="mail.tenant-profile-policy-user",
title="Choosing a mail server",
summary=summary,
body=body,
layer="configured",
documentation_types=("user",),
audience=("mail_user", "campaign_user"),
order=40,
links=(
DocumentationLink(label="Public mail help", href="https://govplan.add-ideas.de/modules/mail", kind="public"),
),
related_modules=("campaigns",),
unlocks=("Campaigns can use these mail choices when the campaign module is available.",),
configuration_keys=("mail_profile_policy",),
i18n_key="mail.topic.tenant_profile_policy_user",
translations=_user_mail_policy_translations(effective),
source_module_id="mail",
metadata=_mail_policy_user_metadata(effective),
)
summary, body = _mail_policy_admin_text(effective, source_count=len(policy.source_policies))
return DocumentationTopic(
id="mail.tenant-profile-policy-admin",
title="Mail server choices for this tenant",
summary=summary,
body=body,
layer="configured",
documentation_types=("admin",),
audience=("tenant_admin", "mail_admin", "campaign_admin"),
order=40,
links=(
DocumentationLink(label="Mail profiles API", href="/api/v1/mail/profiles", kind="api"),
_mail_policy_api_link(),
DocumentationLink(label="Public mail module documentation", href="https://govplan.add-ideas.de/modules/mail", kind="public"),
),
related_modules=("campaigns",),
unlocks=("Campaign delivery can use reusable mail profiles when govoplan-campaign is installed.",),
configuration_keys=("mail_profile_policy",),
i18n_key="mail.topic.tenant_profile_policy_admin",
source_module_id="mail",
metadata=_mail_policy_metadata(effective, source_count=len(policy.source_policies)),
)
def _custom_mail_profile_topic(context: DocumentationContext) -> DocumentationTopic | None:
if context.documentation_type != "user":
return None
principal = context.principal
if not _has_any_scope(principal, (MAIL_PROFILE_WRITE_SCOPE, MAIL_PROFILE_WRITE_OWN_SCOPE)):
return None
if not _has_all_scopes(principal, (MAIL_PROFILE_READ_SCOPE,)):
return None
tenant_id = str(getattr(principal, "tenant_id", "") or "")
user_id = str(getattr(getattr(principal, "user", None), "id", "") or "")
session = context.session
if not tenant_id or not user_id or not isinstance(session, Session):
return None
try:
policy = effective_mail_profile_policy_for_scope(
session,
tenant_id=tenant_id,
scope_type="user",
scope_id=user_id,
)
except MailProfileError:
# A user-facing runtime topic must not turn internal policy resolution
# details into documentation output. The task simply remains absent
# until its effective policy can be proven.
return None
if not policy.allow_user_profiles:
return None
can_manage_credentials = _has_any_scope(
principal,
(MAIL_SECRET_MANAGE_SCOPE, MAIL_SECRET_MANAGE_OWN_SCOPE),
)
can_test_profile = _has_all_scopes(principal, (MAIL_PROFILE_TEST_SCOPE, MAIL_PROFILE_USE_SCOPE))
can_use_profile = _has_any_scope(principal, (MAIL_PROFILE_USE_SCOPE,))
approval_required = bool(policy.allowed_profile_id_sets)
constraints = _host_policy_constraint_records(policy)
authority_lines = _custom_profile_authority_lines(
can_manage_credentials=can_manage_credentials,
can_test_profile=can_test_profile,
can_use_profile=can_use_profile,
approval_required=approval_required,
)
steps = _custom_profile_steps(
can_manage_credentials=can_manage_credentials,
can_test_profile=can_test_profile,
can_use_profile=can_use_profile,
approval_required=approval_required,
)
return DocumentationTopic(
id="mail.workflow.create-custom-profile",
title="Create a custom Mail profile",
summary=(
"Create a reusable profile in the current account's user-scoped Settings view, "
"within the active SMTP, IMAP, and JMAP hostname policy."
),
body="\n".join(authority_lines),
layer="configured",
documentation_types=("user",),
audience=("mail_profile_author", "campaign_manager"),
order=41,
conditions=(
DocumentationCondition(
required_modules=("mail",),
required_scopes=(MAIL_PROFILE_READ_SCOPE,),
any_scopes=(MAIL_PROFILE_WRITE_SCOPE, MAIL_PROFILE_WRITE_OWN_SCOPE),
configuration_keys=("mail_profile_policy",),
),
),
links=(
DocumentationLink(label="My Mail profiles", href="/settings?section=mail-profiles", kind="runtime"),
DocumentationLink(label="Public mail help", href="https://govplan.add-ideas.de/modules/mail", kind="public"),
),
related_modules=("campaigns",),
unlocks=("A custom Mail-owned transport definition that authorized tasks can reference after all policy checks pass.",),
configuration_keys=("mail_profile_policy",),
i18n_key="mail.topic.create_custom_profile",
source_module_id="mail",
metadata={
"kind": "workflow",
"route": "/settings?section=mail-profiles",
"screen": "My Mail profiles",
"help_contexts": ["mail.profiles", "app.settings"],
"prerequisites": [
"The Mail profile editor is available in Settings and opens in the current account's user scope.",
"The effective Mail policy permits user-scoped profiles.",
],
"steps": list(steps),
"outcome": "A user-scoped custom Mail profile is saved without copying its credentials into a consuming module.",
"current_configuration": list(authority_lines),
"constraints": list(constraints),
"verification": _custom_profile_verification(
can_test_profile=can_test_profile,
can_use_profile=can_use_profile,
approval_required=approval_required,
),
"can_manage_credentials": can_manage_credentials,
"can_test_profile": can_test_profile,
"can_use_profile": can_use_profile,
"approval_required_before_use": approval_required,
"related_topic_ids": [
"mail.workflow.choose-and-test-profile",
"mail.profile-ownership-and-consumers",
"mail.profiles-and-policy",
],
},
)
def _host_policy_constraint_records(policy: EffectiveMailProfilePolicy) -> tuple[dict[str, Any], ...]:
constraints: list[dict[str, Any]] = []
for label, key in _HOST_POLICY_FIELDS:
prefix = label.casefold()
denied = _display_patterns(policy.blacklist_patterns.get(key, []))
constraints.append({
"id": f"{prefix}-host-deny",
"label": f"{label} denied hosts",
"description": (
"Deny rules are checked first. The hostname must not match any listed pattern."
if denied
else "Deny rules are checked first. No hostname deny pattern is active."
),
**({"values": list(denied)} if denied else {}),
})
allowed_groups = tuple(
group
for group in (_display_patterns(items) for items in policy.whitelist_groups.get(key, []))
if group
)
if not allowed_groups:
constraints.append({
"id": f"{prefix}-host-allow",
"label": f"{label} allowed hosts",
"description": "After deny checks, no hostname allow-list group is active.",
})
continue
for index, group in enumerate(allowed_groups, start=1):
constraints.append({
"id": f"{prefix}-host-allow-{index}",
"label": f"{label} allowed hosts — group {index}",
"description": (
"After deny checks, the hostname must match at least one pattern in this group. "
"It must satisfy every active allow-list group shown for this protocol."
),
"values": list(group),
})
return tuple(constraints)
def _display_patterns(patterns: list[str]) -> tuple[str, ...]:
result: list[str] = []
for pattern in patterns:
value = str(pattern).strip()
if value and value not in result:
result.append(value)
return tuple(result)
def _custom_profile_authority_lines(
*,
can_manage_credentials: bool,
can_test_profile: bool,
can_use_profile: bool,
approval_required: bool,
) -> tuple[str, ...]:
credentials = (
"Credential authority: you may save or replace Mail-owned SMTP/IMAP passwords and JMAP tokens or Basic credentials."
if can_manage_credentials
else "Credential authority: you may define the profile, but you cannot save or replace passwords; an actor with both profile-write and secret-management authority must do that when authentication requires one."
)
testing = (
"Test authority: you may run the profile's SMTP/IMAP/JMAP connection tests after saving it as active."
if can_test_profile
else "Test authority: creating the profile does not let you run connection tests; ask an actor with both profile-test and profile-use authority to verify an active profile."
)
use = (
"Use authority: you may select the profile in an authorized task after its contextual policy checks pass."
if can_use_profile
else "Use authority: creating the profile does not let you select or use it; separate Mail profile use authority is required."
)
approval = (
"Approval: an approved-profile list is active. A newly generated profile reference must be approved before the profile can be selected or used."
if approval_required
else "Approval: no approved-profile list currently blocks a newly created profile, but each consuming task still rechecks its contextual policy."
)
return credentials, testing, use, approval
def _custom_profile_steps(
*,
can_manage_credentials: bool,
can_test_profile: bool,
can_use_profile: bool,
approval_required: bool,
) -> tuple[str, ...]:
steps = [
"Open Settings, choose Mail profiles, and select Add profile in the current account's user-scoped view.",
"Enter a stable name and configure SMTP plus optional IMAP settings; add an optional JMAP server after the profile exists. Every endpoint must satisfy the host-policy statements shown above.",
]
steps.append(
"Enter the required SMTP/IMAP credentials, or link an encrypted bearer token or Basic credential to the JMAP server."
if can_manage_credentials
else "Save the non-secret profile definition, then ask an actor with both profile-write and secret-management authority to add credentials if the server requires authentication."
)
steps.append("Save the profile; Mail validates the effective user-scope host policy again on the server.")
steps.append(
"Save the profile as active, then run the available SMTP, IMAP, and JMAP connection tests."
if can_test_profile
else "Ask an actor with both profile-test and profile-use authority to run the SMTP, IMAP, and JMAP connection tests after the profile is active."
)
if approval_required:
steps.append("Ask a Mail administrator to add the new profile to the active approved-profile list.")
steps.append(
"Select the profile from the consuming task's picker and complete that task's contextual validation."
if can_use_profile
else "Ask an actor with Mail profile use authority to select it in the consuming task after approval and testing."
)
return tuple(steps)
def _custom_profile_verification(*, can_test_profile: bool, can_use_profile: bool, approval_required: bool) -> str:
checks = ["Reopen My Mail profiles and confirm the saved profile remains in the current account's user-scoped view."]
checks.append(
"Confirm the authorized SMTP/IMAP/JMAP tests succeed."
if can_test_profile
else "Have an authorized tester confirm the SMTP/IMAP/JMAP tests succeed."
)
if approval_required:
checks.append("Confirm an administrator approved the generated profile reference before expecting it in a picker.")
checks.append(
"Confirm the intended task can select it and passes its own policy validation."
if can_use_profile
else "Confirm a separately authorized user can select it and passes the consuming task's policy validation."
)
return " ".join(checks)
def _mail_policy_admin_text(policy: dict[str, Any], *, source_count: int) -> tuple[str, str]:
allowed_profile_ids = policy.get("allowed_profile_ids")
lower_scopes = _allowed_lower_scopes(policy)
approved_profile_limit = isinstance(allowed_profile_ids, list)
pattern_counts = _pattern_counts(policy)
locked_limit_count = sum(1 for value in _lower_limit_values(policy) if value is False)
if approved_profile_limit and not lower_scopes:
summary = "This tenant is in approved-profile mode: users can choose configured mail profiles, but lower scopes cannot bring arbitrary SMTP, IMAP, or JMAP servers."
elif approved_profile_limit:
summary = "This tenant limits mail sending to approved profile ids, while selected lower scopes can still define profiles within policy limits."
elif lower_scopes:
summary = "This tenant permits lower-scope mail profiles subject to the effective host, sender, recipient, and credential policy."
else:
summary = "This tenant uses centrally managed mail profiles; lower-scope profile creation is disabled by the effective policy."
approved_line = _approved_profile_line(allowed_profile_ids)
lower_scope_line = _lower_scope_line(lower_scopes)
credential_line = _credential_line(policy)
pattern_line = f"Allow-list pattern groups: {pattern_counts['whitelist']}. Deny-list patterns: {pattern_counts['blacklist']}."
lower_limit_line = f"Locked lower-level limits: {locked_limit_count}. Policy sources applied: {source_count}."
body = "\n".join([approved_line, lower_scope_line, credential_line, pattern_line, lower_limit_line])
return summary, body
def _mail_policy_user_text(policy: dict[str, Any]) -> tuple[str, str]:
allowed_profile_ids = policy.get("allowed_profile_ids")
lower_scopes = _allowed_lower_scopes(policy)
approved_profile_limit = isinstance(allowed_profile_ids, list)
if approved_profile_limit and not lower_scopes:
summary = "You can choose from approved mail servers, but you cannot add your own mail server for this tenant."
body = "This is set by tenant policy. If the mail server you need is not offered, ask an administrator to add it as an approved mail profile."
elif approved_profile_limit:
summary = "You can use approved Mail profiles. Some scopes may also define additional reusable profiles."
body = "The available profiles are limited by tenant policy. Campaigns select one profile by reference; SMTP/IMAP/JMAP settings and credentials remain managed in Mail."
elif lower_scopes:
summary = "You may be able to define reusable Mail profiles in selected scopes, subject to tenant rules."
body = "GovOPlaN checks each profile before use. Campaigns reference an available profile and never store its SMTP/IMAP/JMAP settings or credentials."
else:
summary = "Mail servers are managed centrally for this tenant."
body = "You cannot add a personal, group, or campaign mail server here. Choose one of the configured options or ask an administrator to add another approved profile."
return summary, body
def _user_mail_policy_translations(policy: dict[str, Any]) -> dict[str, dict[str, str]]:
allowed_profile_ids = policy.get("allowed_profile_ids")
lower_scopes = _allowed_lower_scopes(policy)
approved_profile_limit = isinstance(allowed_profile_ids, list)
if approved_profile_limit and not lower_scopes:
return {
"de": {
"title": "Mailserver auswaehlen",
"summary": "Sie koennen aus freigegebenen Mailservern waehlen, aber keinen eigenen Mailserver fuer diesen Tenant eintragen.",
"body": "Das wird durch die Tenant-Regeln festgelegt. Wenn der benoetigte Mailserver nicht angeboten wird, bitten Sie eine Administratorin oder einen Administrator, ihn als freigegebenes Mailprofil anzulegen.",
},
}
if approved_profile_limit:
return {
"de": {
"title": "Mailserver auswaehlen",
"summary": "Sie koennen freigegebene Mailprofile nutzen. In manchen Bereichen koennen weitere wiederverwendbare Profile angelegt werden.",
"body": "Die Auswahl wird durch Tenant-Regeln begrenzt. Kampagnen speichern nur die Profilreferenz; SMTP-/IMAP-Einstellungen und Zugangsdaten verbleiben im Mail-Modul.",
},
}
if lower_scopes:
return {
"de": {
"title": "Mailserver auswaehlen",
"summary": "In bestimmten Bereichen koennen Sie wiederverwendbare Mailprofile anlegen, solange die aktiven Tenant-Regeln eingehalten werden.",
"body": "GovOPlaN prueft jedes Profil vor der Verwendung. Kampagnen referenzieren ein freigegebenes Profil und speichern keine SMTP-/IMAP-Einstellungen oder Zugangsdaten.",
},
}
return {
"de": {
"title": "Mailserver auswaehlen",
"summary": "Mailserver werden fuer diesen Tenant zentral verwaltet.",
"body": "Sie koennen hier keinen persoenlichen, Gruppen- oder Kampagnen-Mailserver eintragen. Waehlen Sie eine konfigurierte Option oder bitten Sie eine Administratorin oder einen Administrator um ein weiteres freigegebenes Profil.",
},
}
def _approved_profile_line(allowed_profile_ids: Any) -> str:
if isinstance(allowed_profile_ids, list):
count = len(allowed_profile_ids)
if count == 0:
return "Approved profiles: the active policy currently allows no reusable profile ids."
return f"Approved profiles: users are limited to {count} reusable profile id(s) selected by policy."
return "Approved profiles: no explicit approved-profile list is active."
def _lower_scope_line(lower_scopes: tuple[str, ...]) -> str:
if not lower_scopes:
return "Lower scopes: user, group, and campaign-local profile creation are disabled."
return "Lower scopes allowed to define profiles: " + ", ".join(lower_scopes) + "."
def _credential_line(policy: dict[str, Any]) -> str:
smtp_inherit = bool((policy.get("smtp_credentials") or {}).get("inherit", True))
imap_inherit = bool((policy.get("imap_credentials") or {}).get("inherit", True))
return (
f"Credential inheritance: SMTP {'inherits' if smtp_inherit else 'requires local credentials'}; "
f"IMAP {'inherits' if imap_inherit else 'requires local credentials'}. "
"Campaign delivery is available only for protocols that inherit credentials from the selected Mail profile."
)
def _allowed_lower_scopes(policy: dict[str, Any]) -> tuple[str, ...]:
scopes: list[str] = []
if policy.get("allow_user_profiles") is not False:
scopes.append("user")
if policy.get("allow_group_profiles") is not False:
scopes.append("group")
if policy.get("allow_campaign_profiles") is not False:
scopes.append("campaign")
return tuple(scopes)
def _pattern_counts(policy: dict[str, Any]) -> dict[str, int]:
return {
"whitelist": _pattern_count(policy.get("whitelist")),
"blacklist": _pattern_count(policy.get("blacklist")),
}
def _pattern_count(payload: Any) -> int:
if not isinstance(payload, dict):
return 0
return sum(len(value) for value in payload.values() if isinstance(value, list))
def _lower_limit_values(policy: dict[str, Any]) -> tuple[bool, ...]:
lower_limits = policy.get("allow_lower_level_limits")
if not isinstance(lower_limits, dict):
return ()
return tuple(value for value in lower_limits.values() if isinstance(value, bool))
def _mail_policy_metadata(policy: dict[str, Any], *, source_count: int) -> dict[str, Any]:
allowed_profile_ids = policy.get("allowed_profile_ids")
pattern_counts = _pattern_counts(policy)
return {
"approved_profile_limit_active": isinstance(allowed_profile_ids, list),
"approved_profile_count": len(allowed_profile_ids) if isinstance(allowed_profile_ids, list) else None,
"lower_scopes_allowed": list(_allowed_lower_scopes(policy)),
"whitelist_pattern_count": pattern_counts["whitelist"],
"blacklist_pattern_count": pattern_counts["blacklist"],
"locked_lower_level_limit_count": sum(1 for value in _lower_limit_values(policy) if value is False),
"policy_source_count": source_count,
}
def _mail_policy_user_metadata(policy: dict[str, Any]) -> dict[str, Any]:
allowed_profile_ids = policy.get("allowed_profile_ids")
return {
"approved_profile_limit_active": isinstance(allowed_profile_ids, list),
"can_add_local_mail_server": bool(_allowed_lower_scopes(policy)),
"lower_scopes_allowed": list(_allowed_lower_scopes(policy)),
}
def _has_any_scope(principal: object | None, scopes: tuple[str, ...]) -> bool:
has = getattr(principal, "has", None)
if callable(has):
return any(bool(has(scope)) for scope in scopes)
principal_scopes = set(getattr(principal, "scopes", ()) or ())
return "*" in principal_scopes or any(scope in principal_scopes for scope in scopes)
def _has_all_scopes(principal: object | None, scopes: tuple[str, ...]) -> bool:
has = getattr(principal, "has", None)
if callable(has):
try:
return all(bool(has(scope)) for scope in scopes)
except Exception:
return False
principal_scopes = set(getattr(principal, "scopes", ()) or ())
return "*" in principal_scopes or all(scope in principal_scopes for scope in scopes)
def _mail_policy_api_link() -> DocumentationLink:
return DocumentationLink(label="Tenant mail policy API", href="/api/v1/mail/policies/tenant", kind="api")
+639
View File
@@ -0,0 +1,639 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from email.utils import getaddresses
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from govoplan_core.core.dsar import (
DsarErasureActionRef,
DsarExecutionResultRef,
DsarRecordRef,
DsarSubjectRef,
dsar_capability_name,
)
from govoplan_mail.backend.db.models import (
MailBounceObservation,
MailDeliveryAttempt,
MailDeliveryCommand,
MailDeliveryReconciliation,
MailMailboxMessageIndex,
MailPop3Import,
MailServerProfile,
)
MAIL_DSAR_CAPABILITY = dsar_capability_name("mail")
_MAX_RECORDS = 5_000
class MailDsarProvider:
provider_id = "mail"
module_id = "mail"
def search_subject(
self,
session: object,
*,
tenant_id: str,
subject: DsarSubjectRef,
) -> Sequence[DsarRecordRef]:
db = _session(session)
email = _subject_email(subject)
membership_ids = _membership_ids(subject)
references = _mail_references(subject)
if email is None and not membership_ids and not references:
return ()
records: list[DsarRecordRef] = []
def append(record: DsarRecordRef) -> None:
if len(records) >= _MAX_RECORDS:
raise ValueError(
"Mail DSAR match limit exceeded; narrow the subject selectors."
)
records.append(record)
profiles = _matching_profiles(
db,
tenant_id=tenant_id,
membership_ids=membership_ids,
profile_id=references.get("profile"),
)
for profile in profiles:
append(
_record(
"mail_server_profile",
profile.id,
"mail_profile",
profile.name,
{
"match_fields": _profile_matching_fields(
profile, membership_ids
),
"name": profile.name,
"slug": profile.slug,
"description": profile.description,
"scope_type": profile.scope_type,
"is_active": profile.is_active,
"inherit_to_lower_scopes": profile.inherit_to_lower_scopes,
},
observed_at=profile.updated_at,
source_path="/settings?section=mail-profiles",
)
)
messages = _matching_messages(
db,
tenant_id=tenant_id,
email=email,
message_id=references.get("message_index"),
)
for message in messages:
matching_headers = _matching_headers(message, email)
append(
_record(
"mailbox_message_index",
message.id,
"mailbox_message",
message.subject or "Mailbox message",
{
"match_fields": list(matching_headers),
"subject": _bounded_text(message.subject),
"matching_headers": matching_headers,
"date": message.date,
"flags": tuple(
str(flag)[:100] for flag in (message.flags or ())[:32]
),
"size_bytes": message.size_bytes,
"body_preview": _bounded_text(message.body_preview),
"attachment_count": message.attachment_count,
"indexed_at": _iso(message.indexed_at),
},
observed_at=message.updated_at,
source_path="/mail",
)
)
pop3_imports = _matching_pop3_imports(
db,
tenant_id=tenant_id,
email=email,
import_id=references.get("pop3_import"),
)
for imported in pop3_imports:
matching_headers = _matching_pop3_headers(imported, email)
append(
_record(
"mail_pop3_import",
imported.id,
"mail_imported_message",
imported.subject or "Imported legacy message",
{
"match_fields": list(matching_headers),
"subject": _bounded_text(imported.subject),
"matching_headers": matching_headers,
"date": imported.date,
"message_id": _bounded_text(imported.message_id),
"body_preview": _bounded_text(imported.body_preview),
"size_bytes": imported.size_bytes,
"status": imported.status,
"imported_at": _iso(imported.imported_at),
"deletion_requested": imported.deletion_requested,
"deletion_status": imported.deletion_status,
},
observed_at=imported.updated_at,
source_path="/mail/legacy-import",
)
)
bounces = _matching_bounces(
db,
tenant_id=tenant_id,
email=email,
bounce_id=references.get("bounce"),
)
bounce_command_ids = {row.command_id for row in bounces if row.command_id}
commands = _matching_commands(
db,
tenant_id=tenant_id,
membership_ids=membership_ids,
command_id=references.get("command"),
related_command_ids=bounce_command_ids,
)
command_ids = {row.id for row in commands}
for command in commands:
append(
_record(
"mail_delivery_command",
command.id,
"mail_delivery_evidence",
f"Mail {command.command_type} command",
{
"match_fields": (
["created_by_user_id"]
if command.created_by_user_id in membership_ids
else (
["reference"]
if command.id == references.get("command")
else ["bounce"]
)
),
"command_type": command.command_type,
"source_module": command.source_module,
"source_resource_type": command.source_resource_type,
"message_sha256": command.message_sha256,
"rfc_message_id": command.rfc_message_id,
"message_size_bytes": command.message_size_bytes,
"recipient_count": command.recipient_count,
"status": command.status,
"attempt_count": command.attempt_count,
"effect_started_at": _iso(command.effect_started_at),
"completed_at": _iso(command.completed_at),
"accepted_count": command.accepted_count,
"refused_count": command.refused_count,
"failure_code": command.failure_code,
"payload_purged_at": _iso(command.payload_purged_at),
},
observed_at=command.updated_at,
immutable=True,
retention_reason="Mail delivery commands are immutable transport, retry, and outcome evidence; encrypted payload retention is governed separately.",
)
)
for attempt in _command_attempts(db, command_ids):
append(
_record(
"mail_delivery_attempt",
attempt.id,
"mail_delivery_evidence",
f"Mail delivery attempt {attempt.attempt_number}",
{
"command_id": attempt.command_id,
"attempt_number": attempt.attempt_number,
"status": attempt.status,
"started_at": _iso(attempt.started_at),
"effect_started_at": _iso(attempt.effect_started_at),
"completed_at": _iso(attempt.completed_at),
"accepted_count": attempt.accepted_count,
"refused_count": attempt.refused_count,
"outcome_code": attempt.outcome_code,
},
observed_at=attempt.updated_at,
immutable=True,
retention_reason="Per-attempt Mail outcome state is immutable delivery and recovery evidence.",
)
)
for reconciliation in _command_reconciliations(db, command_ids):
append(
_record(
"mail_delivery_reconciliation",
reconciliation.id,
"mail_delivery_evidence",
"Mail delivery reconciliation",
{
"command_id": reconciliation.command_id,
"decision": reconciliation.decision,
},
observed_at=reconciliation.updated_at,
immutable=True,
retention_reason="Mail reconciliation decisions are immutable authorization and recovery evidence.",
)
)
for bounce in bounces:
append(
_record(
"mail_bounce_observation",
bounce.id,
"mail_bounce_evidence",
"Mail bounce observation",
{
"match_fields": (
["recipient"]
if email and _normalized_email(bounce.recipient) == email
else ["reference"]
),
"command_id": bounce.command_id
if bounce.command_id in command_ids
else None,
"recipient": email
if email and _normalized_email(bounce.recipient) == email
else None,
"action": bounce.action,
"status_code": bounce.status_code,
"permanent": bounce.permanent,
"observed_at": _iso(bounce.observed_at),
"matched": bounce.matched,
},
observed_at=bounce.updated_at,
immutable=True,
retention_reason="Bounce observations are immutable delivery-status and suppression evidence.",
source_path="/mail/bounces",
)
)
return tuple(records)
def plan_erasure(
self,
session: object,
*,
tenant_id: str,
subject: DsarSubjectRef,
records: Sequence[DsarRecordRef],
) -> Sequence[DsarErasureActionRef]:
del session, tenant_id, subject
actions = []
for record in records:
if (
record.provider_id != self.provider_id
or record.module_id != self.module_id
):
raise ValueError("Mail DSAR received a foreign provider record.")
actions.append(
DsarErasureActionRef(
action_id=f"mail:{'retain' if record.immutable_evidence else 'review'}:{record.resource_type}:{record.resource_id}",
provider_id="mail",
module_id="mail",
kind="retain" if record.immutable_evidence else "manual_review",
resource_type=record.resource_type,
resource_id=record.resource_id,
title=f"{'Retain' if record.immutable_evidence else 'Review'} {record.title}",
rationale=record.retention_reason
or "Mailbox indexes and personal profiles must be reviewed through Mail and the authoritative external mailbox lifecycle.",
executable=False,
)
)
return tuple(actions)
def execute_erasure(
self,
session: object,
*,
tenant_id: str,
subject: DsarSubjectRef,
actions: Sequence[DsarErasureActionRef],
request_id: str,
) -> Sequence[DsarExecutionResultRef]:
del session, tenant_id, subject, request_id
return tuple(
DsarExecutionResultRef(
action_id=action.action_id,
status="blocked",
summary="Mail erasure requires an authorized Mail/external-mailbox lifecycle action; the DSAR provider does not mutate it directly.",
)
for action in actions
)
def _matching_profiles(
session: Session,
*,
tenant_id: str,
membership_ids: set[str],
profile_id: str | None,
) -> list[MailServerProfile]:
conditions = []
if profile_id:
conditions.append(MailServerProfile.id == profile_id)
if membership_ids:
conditions.extend(
(
MailServerProfile.created_by_user_id.in_(membership_ids),
MailServerProfile.updated_by_user_id.in_(membership_ids),
(MailServerProfile.scope_type == "user")
& MailServerProfile.scope_id.in_(membership_ids),
)
)
if not conditions:
return []
return _bounded_rows(
session.query(MailServerProfile)
.filter(MailServerProfile.tenant_id == tenant_id, or_(*conditions))
.order_by(MailServerProfile.id)
)
def _profile_matching_fields(
row: MailServerProfile, membership_ids: set[str]
) -> list[str]:
fields = []
if row.scope_type == "user" and row.scope_id in membership_ids:
fields.append("scope_id")
for field in ("created_by_user_id", "updated_by_user_id"):
if getattr(row, field) in membership_ids:
fields.append(field)
return fields
def _matching_messages(
session: Session, *, tenant_id: str, email: str | None, message_id: str | None
) -> list[MailMailboxMessageIndex]:
conditions = []
if message_id:
conditions.append(MailMailboxMessageIndex.id == message_id)
if email:
pattern = f"%{_escape_like(email)}%"
conditions.extend(
func.lower(field).like(pattern, escape="\\")
for field in (
MailMailboxMessageIndex.from_header,
MailMailboxMessageIndex.to_header,
MailMailboxMessageIndex.cc_header,
)
)
if not conditions:
return []
candidates = _bounded_rows(
session.query(MailMailboxMessageIndex)
.filter(MailMailboxMessageIndex.tenant_id == tenant_id, or_(*conditions))
.order_by(MailMailboxMessageIndex.id)
)
return [
row
for row in candidates
if row.id == message_id or _matching_headers(row, email)
]
def _matching_headers(
row: MailMailboxMessageIndex, email: str | None
) -> dict[str, list[dict[str, str | None]]]:
if email is None:
return {}
result = {}
for role, value in (
("from", row.from_header),
("to", row.to_header),
("cc", row.cc_header),
):
matches = [
{"email": address.casefold(), "name": name or None}
for name, address in getaddresses([value or ""])
if address.casefold() == email
]
if matches:
result[role] = matches[:64]
return result
def _matching_bounces(
session: Session, *, tenant_id: str, email: str | None, bounce_id: str | None
) -> list[MailBounceObservation]:
conditions = []
if bounce_id:
conditions.append(MailBounceObservation.id == bounce_id)
if email:
conditions.append(func.lower(MailBounceObservation.recipient) == email)
if not conditions:
return []
return _bounded_rows(
session.query(MailBounceObservation)
.filter(MailBounceObservation.tenant_id == tenant_id, or_(*conditions))
.order_by(MailBounceObservation.id)
)
def _matching_pop3_imports(
session: Session,
*,
tenant_id: str,
email: str | None,
import_id: str | None,
) -> list[MailPop3Import]:
conditions = []
if import_id:
conditions.append(MailPop3Import.id == import_id)
if email:
pattern = f"%{_escape_like(email)}%"
conditions.extend(
func.lower(field).like(pattern, escape="\\")
for field in (
MailPop3Import.from_header,
MailPop3Import.to_header,
)
)
if not conditions:
return []
candidates = _bounded_rows(
session.query(MailPop3Import)
.filter(MailPop3Import.tenant_id == tenant_id, or_(*conditions))
.order_by(MailPop3Import.id)
)
return [
row
for row in candidates
if row.id == import_id or _matching_pop3_headers(row, email)
]
def _matching_pop3_headers(
row: MailPop3Import,
email: str | None,
) -> dict[str, list[dict[str, str | None]]]:
if email is None:
return {}
result = {}
for role, value in (("from", row.from_header), ("to", row.to_header)):
matches = [
{"email": address.casefold(), "name": name or None}
for name, address in getaddresses([value or ""])
if address.casefold() == email
]
if matches:
result[role] = matches[:64]
return result
def _matching_commands(
session: Session,
*,
tenant_id: str,
membership_ids: set[str],
command_id: str | None,
related_command_ids: set[str],
) -> list[MailDeliveryCommand]:
conditions = []
if command_id:
conditions.append(MailDeliveryCommand.id == command_id)
if related_command_ids:
conditions.append(MailDeliveryCommand.id.in_(related_command_ids))
if membership_ids:
conditions.append(MailDeliveryCommand.created_by_user_id.in_(membership_ids))
if not conditions:
return []
return _bounded_rows(
session.query(MailDeliveryCommand)
.filter(MailDeliveryCommand.tenant_id == tenant_id, or_(*conditions))
.order_by(MailDeliveryCommand.id)
)
def _command_attempts(
session: Session, command_ids: set[str]
) -> list[MailDeliveryAttempt]:
if not command_ids:
return []
return _bounded_rows(
session.query(MailDeliveryAttempt)
.filter(MailDeliveryAttempt.command_id.in_(command_ids))
.order_by(MailDeliveryAttempt.id)
)
def _command_reconciliations(
session: Session, command_ids: set[str]
) -> list[MailDeliveryReconciliation]:
if not command_ids:
return []
return _bounded_rows(
session.query(MailDeliveryReconciliation)
.filter(MailDeliveryReconciliation.command_id.in_(command_ids))
.order_by(MailDeliveryReconciliation.id)
)
def _mail_references(subject: DsarSubjectRef) -> dict[str, str]:
aliases = {
"mail.profile": "profile",
"mail.message_index": "message_index",
"mail.delivery_command": "command",
"mail.bounce_observation": "bounce",
"mail.pop3_import": "pop3_import",
}
return {
target: value
for key, target in aliases.items()
if (value := str(subject.external_references.get(key) or "").strip())
}
def _membership_ids(subject: DsarSubjectRef) -> set[str]:
values = [subject.membership_id]
values.extend(
subject.external_references.get(key)
for key in (
"mail.user",
"mail.membership",
"access.membership",
"membership_id",
)
)
return {value for item in values if (value := str(item or "").strip())}
def _subject_email(subject: DsarSubjectRef) -> str | None:
values = [subject.email, subject.external_references.get("mail.email")]
normalized = {email for value in values if (email := _normalized_email(value))}
return normalized.pop() if len(normalized) == 1 else None
def _normalized_email(value: object) -> str | None:
if not isinstance(value, str):
return None
value = value.strip().casefold()
return value or None
def _record(
resource_type: str,
resource_id: str,
category: str,
title: str,
data: Mapping[str, object],
*,
observed_at: datetime | None = None,
immutable: bool = False,
retention_reason: str | None = None,
source_path: str | None = None,
) -> DsarRecordRef:
return DsarRecordRef(
provider_id="mail",
module_id="mail",
resource_type=resource_type,
resource_id=resource_id,
category=category,
title=title,
data=data,
observed_at=observed_at,
immutable_evidence=immutable,
retention_reason=retention_reason,
source_path=source_path,
)
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("Mail DSAR provider requires a SQLAlchemy session.")
return value
def _bounded_rows(query: object) -> list[object]:
rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined]
if len(rows) > _MAX_RECORDS:
raise ValueError(
"Mail DSAR match limit exceeded; narrow the subject selectors."
)
return rows
def _bounded_text(value: str | None) -> str | None:
return value[:2_000] if value else None
def _escape_like(value: str) -> str:
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
def _iso(value: datetime | None) -> str | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.isoformat()
__all__ = ["MAIL_DSAR_CAPABILITY", "MailDsarProvider"]
File diff suppressed because it is too large Load Diff
+409
View File
@@ -0,0 +1,409 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
from sqlalchemy import func
from sqlalchemy.orm import Session
from govoplan_core.core.change_sequence import record_change
from govoplan_core.security.time import ensure_aware_utc, utc_now
from govoplan_mail.backend.db.models import (
MailMailboxFolderIndex,
MailMailboxMessageIndex,
new_uuid,
)
from govoplan_mail.backend.sending.imap import ImapFolderListResult, ImapMailboxInfo, ImapMailboxMessageListResult, ImapMailboxMessageSummary
MAILBOX_INDEX_TTL_SECONDS = 30
MAILBOX_MESSAGES_COLLECTION = "mail.mailbox_messages"
_refresh_lock = Lock()
_refreshing_keys: set[tuple[str, str, str]] = set()
@dataclass(frozen=True, slots=True)
class CachedFolderList:
folders: list[ImapMailboxInfo]
indexed_at: datetime | None
stale: bool
@dataclass(frozen=True, slots=True)
class CachedMessagePage:
folder: str
messages: list[ImapMailboxMessageSummary]
total_count: int
offset: int
limit: int
uidvalidity: str | None
indexed_at: datetime | None
stale: bool
def clear_mailbox_index(session: Session, *, profile_id: str) -> tuple[int, int]:
"""Remove every cached mailbox row for a profile in the caller's transaction.
System profiles can be used by more than one tenant, so invalidation is
intentionally profile-wide rather than limited to the tenant making the
transport change. Message rows are removed before their folder metadata.
"""
messages = (
session.query(MailMailboxMessageIndex)
.filter(MailMailboxMessageIndex.profile_id == profile_id)
.all()
)
for message in messages:
_record_message_change(session, message, operation="deleted")
deleted_messages = (
session.query(MailMailboxMessageIndex)
.filter(MailMailboxMessageIndex.profile_id == profile_id)
.delete(synchronize_session=False)
)
deleted_folders = (
session.query(MailMailboxFolderIndex)
.filter(MailMailboxFolderIndex.profile_id == profile_id)
.delete(synchronize_session=False)
)
return int(deleted_folders or 0), int(deleted_messages or 0)
def begin_mailbox_refresh(tenant_id: str, profile_id: str, folder: str) -> bool:
key = (tenant_id, profile_id, folder)
with _refresh_lock:
if key in _refreshing_keys:
return False
_refreshing_keys.add(key)
return True
def finish_mailbox_refresh(tenant_id: str, profile_id: str, folder: str) -> None:
with _refresh_lock:
_refreshing_keys.discard((tenant_id, profile_id, folder))
def cache_mailbox_folders(
session: Session,
*,
tenant_id: str,
profile_id: str,
result: ImapFolderListResult,
indexed_at: datetime | None = None,
) -> None:
indexed_at = indexed_at or utc_now()
existing = {
row.folder: row
for row in session.query(MailMailboxFolderIndex)
.filter(MailMailboxFolderIndex.tenant_id == tenant_id, MailMailboxFolderIndex.profile_id == profile_id)
.all()
}
expected_names = {folder.name for folder in result.folders}
removed_names = set(existing) - expected_names
if removed_names:
removed_messages = (
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder.in_(removed_names),
)
.all()
)
for message in removed_messages:
_record_message_change(session, message, operation="deleted")
(
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder.in_(removed_names),
)
.delete(synchronize_session=False)
)
for name in removed_names:
session.delete(existing[name])
for folder in result.folders:
row = existing.get(folder.name)
if row is None:
row = MailMailboxFolderIndex(tenant_id=tenant_id, profile_id=profile_id, folder=folder.name)
row.flags = list(folder.flags or [])
row.message_count = folder.message_count
row.unseen_count = folder.unseen_count
row.indexed_at = indexed_at
session.add(row)
def cache_mailbox_messages(
session: Session,
*,
tenant_id: str,
profile_id: str,
result: ImapMailboxMessageListResult,
indexed_at: datetime | None = None,
) -> None:
indexed_at = indexed_at or utc_now()
session.flush()
folder_row = (
session.query(MailMailboxFolderIndex)
.filter(
MailMailboxFolderIndex.tenant_id == tenant_id,
MailMailboxFolderIndex.profile_id == profile_id,
MailMailboxFolderIndex.folder == result.folder,
)
.one_or_none()
)
if folder_row is None:
folder_row = MailMailboxFolderIndex(tenant_id=tenant_id, profile_id=profile_id, folder=result.folder)
folder_row.message_count = result.total_count
folder_row.uidvalidity = result.uidvalidity
folder_row.message_indexed_at = indexed_at
session.add(folder_row)
uids = [message.uid for message in result.messages]
if result.total_count <= 0:
removed_messages = (
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
)
.all()
)
for message in removed_messages:
_record_message_change(session, message, operation="deleted")
(
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
)
.delete(synchronize_session=False)
)
return
window_count = max(0, min(result.limit, result.total_count - result.offset))
if window_count:
stale_query = session.query(MailMailboxMessageIndex).filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
MailMailboxMessageIndex.sort_position >= result.offset,
MailMailboxMessageIndex.sort_position < result.offset + window_count,
)
if uids:
stale_query = stale_query.filter(MailMailboxMessageIndex.uid.notin_(uids))
for row in stale_query.all():
_record_message_change(session, row, operation="deleted")
session.delete(row)
existing = {}
if uids:
existing = {
row.uid: row
for row in session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
MailMailboxMessageIndex.uid.in_(uids),
)
.all()
}
for index, message in enumerate(result.messages):
row = existing.get(message.uid)
created = row is None
if row is None:
row = MailMailboxMessageIndex(
id=new_uuid(),
tenant_id=tenant_id,
profile_id=profile_id,
folder=result.folder,
uid=message.uid,
)
previous = None if created else _message_state(row)
row.uid_int = _uid_int(message.uid)
row.sort_position = result.offset + index
row.subject = message.subject
row.from_header = message.from_header
row.to_header = message.to_header
row.cc_header = message.cc_header
row.date = message.date
row.message_id = message.message_id
row.flags = list(message.flags or [])
row.size_bytes = message.size_bytes
row.body_preview = message.body_preview
row.attachment_count = message.attachment_count
row.indexed_at = indexed_at
session.add(row)
if created or previous != _message_state(row):
_record_message_change(
session,
row,
operation="created" if created else "updated",
)
def cached_mailbox_folders(
session: Session,
*,
tenant_id: str,
profile_id: str,
max_age_seconds: int = MAILBOX_INDEX_TTL_SECONDS,
) -> CachedFolderList | None:
rows = (
session.query(MailMailboxFolderIndex)
.filter(
MailMailboxFolderIndex.tenant_id == tenant_id,
MailMailboxFolderIndex.profile_id == profile_id,
MailMailboxFolderIndex.indexed_at.isnot(None),
)
.order_by(MailMailboxFolderIndex.folder.asc())
.all()
)
if not rows:
return None
indexed_at = max((row.indexed_at for row in rows if row.indexed_at), default=None)
return CachedFolderList(
folders=[
ImapMailboxInfo(name=row.folder, flags=list(row.flags or []), message_count=row.message_count, unseen_count=row.unseen_count)
for row in rows
],
indexed_at=indexed_at,
stale=_is_stale(indexed_at, max_age_seconds=max_age_seconds),
)
def cached_mailbox_message_page(
session: Session,
*,
tenant_id: str,
profile_id: str,
folder: str,
limit: int,
offset: int,
max_age_seconds: int = MAILBOX_INDEX_TTL_SECONDS,
) -> CachedMessagePage | None:
folder_row = (
session.query(MailMailboxFolderIndex)
.filter(
MailMailboxFolderIndex.tenant_id == tenant_id,
MailMailboxFolderIndex.profile_id == profile_id,
MailMailboxFolderIndex.folder == folder,
)
.one_or_none()
)
if folder_row is None or folder_row.message_indexed_at is None:
return None
total_count = folder_row.message_count
if total_count is None:
total_count = int(
session.query(func.count(MailMailboxMessageIndex.id))
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == folder,
)
.scalar()
or 0
)
rows = (
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == folder,
)
.order_by(MailMailboxMessageIndex.sort_position.asc(), MailMailboxMessageIndex.uid_int.desc(), MailMailboxMessageIndex.uid.desc())
.offset(offset)
.limit(limit)
.all()
)
expected_count = max(0, min(limit, total_count - offset))
if len(rows) < expected_count:
return None
indexed_at = min((row.indexed_at for row in rows if row.indexed_at), default=folder_row.message_indexed_at)
return CachedMessagePage(
folder=folder,
messages=[_message_from_index(row) for row in rows],
total_count=total_count,
offset=offset,
limit=limit,
uidvalidity=folder_row.uidvalidity,
indexed_at=indexed_at,
stale=_is_stale(indexed_at, max_age_seconds=max_age_seconds),
)
def _message_from_index(row: MailMailboxMessageIndex) -> ImapMailboxMessageSummary:
return ImapMailboxMessageSummary(
uid=row.uid,
folder=row.folder,
subject=row.subject,
from_header=row.from_header,
to_header=row.to_header,
cc_header=row.cc_header,
date=row.date,
message_id=row.message_id,
flags=list(row.flags or []),
size_bytes=row.size_bytes,
body_preview=row.body_preview,
attachment_count=row.attachment_count,
)
def _message_state(row: MailMailboxMessageIndex) -> tuple[object, ...]:
return (
row.folder,
row.uid,
row.sort_position,
row.subject,
row.from_header,
row.to_header,
row.cc_header,
row.date,
row.message_id,
tuple(row.flags or ()),
row.size_bytes,
row.body_preview,
row.attachment_count,
)
def _record_message_change(
session: Session,
row: MailMailboxMessageIndex,
*,
operation: str,
) -> None:
record_change(
session,
module_id="mail",
collection=MAILBOX_MESSAGES_COLLECTION,
resource_type="mailbox_message",
resource_id=row.id,
operation=operation,
tenant_id=row.tenant_id,
actor_type="system",
payload={
"profile_id": row.profile_id,
"folder": row.folder,
"uid": row.uid,
"message_id": row.message_id,
},
)
def _uid_int(uid: str) -> int:
try:
return int(str(uid))
except ValueError:
return 0
def _is_stale(indexed_at: datetime | None, *, max_age_seconds: int) -> bool:
aware = ensure_aware_utc(indexed_at)
if aware is None:
return True
return (utc_now() - aware).total_seconds() > max_age_seconds
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,56 @@
"""mail profile policies
Revision ID: 3d4e5f708192
Revises: 2e3f4a5b6c7d
Create Date: 2026-07-06 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "3d4e5f708192"
down_revision = "2e3f4a5b6c7d"
branch_labels = None
depends_on = None
def _scope_fk_target(inspector) -> str:
tables = set(inspector.get_table_names())
return "core_scopes.id" if "core_scopes" in tables else "tenancy_tenants.id"
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "mail_profile_policies" in inspector.get_table_names():
return
scope_fk_target = _scope_fk_target(inspector)
op.create_table(
"mail_profile_policies",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=True),
sa.Column("scope_type", sa.String(length=20), nullable=False),
sa.Column("scope_id", sa.String(length=36), nullable=True),
sa.Column("policy", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["tenant_id"], [scope_fk_target], name=op.f("fk_mail_profile_policies_tenant_id_scopes"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_profile_policies")),
sa.UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_mail_profile_policies_scope"),
)
op.create_index(op.f("ix_mail_profile_policies_tenant_id"), "mail_profile_policies", ["tenant_id"], unique=False)
op.create_index(op.f("ix_mail_profile_policies_scope_type"), "mail_profile_policies", ["scope_type"], unique=False)
op.create_index(op.f("ix_mail_profile_policies_scope_id"), "mail_profile_policies", ["scope_id"], unique=False)
op.create_index("ix_mail_profile_policies_scope", "mail_profile_policies", ["scope_type", "scope_id"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "mail_profile_policies" not in inspector.get_table_names():
return
op.drop_index("ix_mail_profile_policies_scope", table_name="mail_profile_policies")
op.drop_index(op.f("ix_mail_profile_policies_scope_id"), table_name="mail_profile_policies")
op.drop_index(op.f("ix_mail_profile_policies_scope_type"), table_name="mail_profile_policies")
op.drop_index(op.f("ix_mail_profile_policies_tenant_id"), table_name="mail_profile_policies")
op.drop_table("mail_profile_policies")
@@ -0,0 +1,112 @@
"""mail mailbox index
Revision ID: 4e5f708192ab
Revises: 3d4e5f708192
Create Date: 2026-07-14 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "4e5f708192ab"
down_revision = "3d4e5f708192"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_folder_index" not in tables:
op.create_table(
"mail_mailbox_folder_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("message_count", sa.Integer(), nullable=True),
sa.Column("unseen_count", sa.Integer(), nullable=True),
sa.Column("uidvalidity", sa.String(length=255), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_folder_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_folder_index")),
sa.UniqueConstraint("profile_id", "folder", name="uq_mail_mailbox_folder_index_profile_folder"),
)
op.create_index(op.f("ix_mail_mailbox_folder_index_folder"), "mail_mailbox_folder_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), "mail_mailbox_folder_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_profile_id"), "mail_mailbox_folder_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), "mail_mailbox_folder_index", ["tenant_id"], unique=False)
op.create_index("ix_mail_mailbox_folder_index_tenant_profile", "mail_mailbox_folder_index", ["tenant_id", "profile_id"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_folder_index")}
if "message_indexed_at" not in columns:
op.add_column("mail_mailbox_folder_index", sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True))
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
if "mail_mailbox_message_index" not in tables:
op.create_table(
"mail_mailbox_message_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("uid", sa.String(length=255), nullable=False),
sa.Column("uid_int", sa.BigInteger(), nullable=False),
sa.Column("sort_position", sa.BigInteger(), nullable=False),
sa.Column("subject", sa.Text(), nullable=True),
sa.Column("from_header", sa.Text(), nullable=True),
sa.Column("to_header", sa.Text(), nullable=True),
sa.Column("cc_header", sa.Text(), nullable=True),
sa.Column("date", sa.String(length=255), nullable=True),
sa.Column("message_id", sa.Text(), nullable=True),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("body_preview", sa.Text(), nullable=True),
sa.Column("attachment_count", sa.Integer(), nullable=False),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_message_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_message_index")),
sa.UniqueConstraint("profile_id", "folder", "uid", name="uq_mail_mailbox_message_index_profile_folder_uid"),
)
op.create_index(op.f("ix_mail_mailbox_message_index_folder"), "mail_mailbox_message_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_indexed_at"), "mail_mailbox_message_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_profile_id"), "mail_mailbox_message_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_tenant_id"), "mail_mailbox_message_index", ["tenant_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_uid_int"), "mail_mailbox_message_index", ["uid_int"], unique=False)
op.create_index("ix_mail_mailbox_message_index_page", "mail_mailbox_message_index", ["tenant_id", "profile_id", "folder", "sort_position"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_message_index")}
if "sort_position" not in columns:
op.add_column("mail_mailbox_message_index", sa.Column("sort_position", sa.BigInteger(), nullable=False, server_default="0"))
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_message_index" in tables:
op.drop_index("ix_mail_mailbox_message_index_page", table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_uid_int"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_tenant_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_profile_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_indexed_at"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_folder"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_sort_position"), table_name="mail_mailbox_message_index")
op.drop_table("mail_mailbox_message_index")
if "mail_mailbox_folder_index" in tables:
op.drop_index("ix_mail_mailbox_folder_index_tenant_profile", table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_profile_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_folder"), table_name="mail_mailbox_folder_index")
op.drop_table("mail_mailbox_folder_index")
@@ -0,0 +1,67 @@
"""repair mail mailbox index columns
Revision ID: 5f708192abcd
Revises: 4e5f708192ab
Create Date: 2026-07-14 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "5f708192abcd"
down_revision = "4e5f708192ab"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_folder_index" in tables:
columns = _columns(inspector, "mail_mailbox_folder_index")
indexes = _indexes(inspector, "mail_mailbox_folder_index")
if "message_indexed_at" not in columns:
op.add_column("mail_mailbox_folder_index", sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True))
if "ix_mail_mailbox_folder_index_message_indexed_at" not in indexes:
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
if "mail_mailbox_message_index" in tables:
columns = _columns(inspector, "mail_mailbox_message_index")
indexes = _indexes(inspector, "mail_mailbox_message_index")
if "sort_position" not in columns:
op.add_column("mail_mailbox_message_index", sa.Column("sort_position", sa.BigInteger(), nullable=False, server_default="0"))
if op.get_bind().dialect.name != "sqlite":
op.alter_column("mail_mailbox_message_index", "sort_position", server_default=None)
if "ix_mail_mailbox_message_index_sort_position" not in indexes:
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
if "ix_mail_mailbox_message_index_page" not in indexes:
op.create_index("ix_mail_mailbox_message_index_page", "mail_mailbox_message_index", ["tenant_id", "profile_id", "folder", "sort_position"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_message_index" in tables:
indexes = _indexes(inspector, "mail_mailbox_message_index")
if "ix_mail_mailbox_message_index_page" in indexes:
op.drop_index("ix_mail_mailbox_message_index_page", table_name="mail_mailbox_message_index")
if "ix_mail_mailbox_message_index_sort_position" in indexes:
op.drop_index(op.f("ix_mail_mailbox_message_index_sort_position"), table_name="mail_mailbox_message_index")
if "sort_position" in _columns(inspector, "mail_mailbox_message_index"):
op.drop_column("mail_mailbox_message_index", "sort_position")
if "mail_mailbox_folder_index" in tables:
indexes = _indexes(inspector, "mail_mailbox_folder_index")
if "ix_mail_mailbox_folder_index_message_indexed_at" in indexes:
op.drop_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), table_name="mail_mailbox_folder_index")
if "message_indexed_at" in _columns(inspector, "mail_mailbox_folder_index"):
op.drop_column("mail_mailbox_folder_index", "message_indexed_at")
def _columns(inspector, table_name: str) -> set[str]:
return {column["name"] for column in inspector.get_columns(table_name)}
def _indexes(inspector, table_name: str) -> set[str]:
return {index["name"] for index in inspector.get_indexes(table_name)}
@@ -0,0 +1,78 @@
"""add opaque Mail-owned transport revisions
Revision ID: 608192abcdef
Revises: 5f708192abcd
Create Date: 2026-07-21 00:00:00.000000
"""
from __future__ import annotations
import uuid
from alembic import op
import sqlalchemy as sa
revision = "608192abcdef"
down_revision = "5f708192abcd"
branch_labels = None
depends_on = None
_SMTP_COLUMN = "smtp_transport_revision"
_IMAP_COLUMN = "imap_transport_revision"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if "mail_server_profiles" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")}
with op.batch_alter_table("mail_server_profiles") as batch:
if _SMTP_COLUMN not in columns:
batch.add_column(sa.Column(_SMTP_COLUMN, sa.String(length=36), nullable=True))
if _IMAP_COLUMN not in columns:
batch.add_column(sa.Column(_IMAP_COLUMN, sa.String(length=36), nullable=True))
rows = list(bind.execute(
sa.text(
"SELECT id, smtp_transport_revision, imap_transport_revision "
"FROM mail_server_profiles"
)
).mappings())
for row in rows:
values: dict[str, str] = {}
if not row[_SMTP_COLUMN]:
values[_SMTP_COLUMN] = str(uuid.uuid4())
if not row[_IMAP_COLUMN]:
values[_IMAP_COLUMN] = str(uuid.uuid4())
if values:
bind.execute(
sa.text(
"UPDATE mail_server_profiles "
"SET smtp_transport_revision = COALESCE(:smtp_revision, smtp_transport_revision), "
"imap_transport_revision = COALESCE(:imap_revision, imap_transport_revision) "
"WHERE id = :profile_id"
),
{
"profile_id": row["id"],
"smtp_revision": values.get(_SMTP_COLUMN),
"imap_revision": values.get(_IMAP_COLUMN),
},
)
with op.batch_alter_table("mail_server_profiles") as batch:
batch.alter_column(_SMTP_COLUMN, existing_type=sa.String(length=36), nullable=False)
batch.alter_column(_IMAP_COLUMN, existing_type=sa.String(length=36), nullable=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "mail_server_profiles" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")}
with op.batch_alter_table("mail_server_profiles") as batch:
if _IMAP_COLUMN in columns:
batch.drop_column(_IMAP_COLUMN)
if _SMTP_COLUMN in columns:
batch.drop_column(_SMTP_COLUMN)
@@ -0,0 +1,22 @@
"""split mail envelopes into servers and reusable credential bindings
Revision ID: 7192a3bcdef0
Revises: 608192abcdef
Create Date: 2026-07-23 00:00:00.000000
"""
from __future__ import annotations
from importlib import import_module
hierarchy = import_module(
"govoplan_mail.backend.migrations.versions.7192a3bcdef0_mail_server_hierarchy"
)
revision = hierarchy.revision
down_revision = hierarchy.down_revision
branch_labels = hierarchy.branch_labels
depends_on = hierarchy.depends_on
upgrade = hierarchy.upgrade
downgrade = hierarchy.downgrade
@@ -0,0 +1,22 @@
"""add durable mail delivery outbox
Revision ID: 82a3b4c5d6e7
Revises: 7192a3bcdef0
Create Date: 2026-07-30 00:00:00.000000
"""
from __future__ import annotations
from importlib import import_module
delivery_outbox = import_module(
"govoplan_mail.backend.migrations.versions.82a3b4c5d6e7_mail_delivery_outbox"
)
revision = delivery_outbox.revision
down_revision = delivery_outbox.down_revision
branch_labels = delivery_outbox.branch_labels
depends_on = delivery_outbox.depends_on
upgrade = delivery_outbox.upgrade
downgrade = delivery_outbox.downgrade
@@ -0,0 +1,39 @@
"""v0.1.7 mail baseline
Revision ID: 3d4e5f708192
Revises: None
Create Date: 2026-07-11 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = '3d4e5f708192'
down_revision = None
branch_labels = None
depends_on = '4f2a9c8e7b6d'
def upgrade() -> None:
op.create_table('mail_profile_policies',
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36), nullable=True),
sa.Column('scope_type', sa.String(length=20), nullable=False),
sa.Column('scope_id', sa.String(length=36), nullable=True),
sa.Column('policy', sa.JSON(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['tenant_id'], ['core_scopes.id'], name=op.f('fk_mail_profile_policies_tenant_id_scopes'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_mail_profile_policies')),
sa.UniqueConstraint('tenant_id', 'scope_type', 'scope_id', name='uq_mail_profile_policies_scope')
)
op.create_index('ix_mail_profile_policies_scope', 'mail_profile_policies', ['scope_type', 'scope_id'], unique=False)
op.create_index(op.f('ix_mail_profile_policies_scope_id'), 'mail_profile_policies', ['scope_id'], unique=False)
op.create_index(op.f('ix_mail_profile_policies_scope_type'), 'mail_profile_policies', ['scope_type'], unique=False)
op.create_index(op.f('ix_mail_profile_policies_tenant_id'), 'mail_profile_policies', ['tenant_id'], unique=False)
def downgrade() -> None:
op.drop_table('mail_profile_policies')
@@ -0,0 +1,112 @@
"""v0.1.8 mail mailbox index
Revision ID: 4e5f708192ab
Revises: 3d4e5f708192
Create Date: 2026-07-14 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "4e5f708192ab"
down_revision = "3d4e5f708192"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_folder_index" not in tables:
op.create_table(
"mail_mailbox_folder_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("message_count", sa.Integer(), nullable=True),
sa.Column("unseen_count", sa.Integer(), nullable=True),
sa.Column("uidvalidity", sa.String(length=255), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_folder_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_folder_index")),
sa.UniqueConstraint("profile_id", "folder", name="uq_mail_mailbox_folder_index_profile_folder"),
)
op.create_index(op.f("ix_mail_mailbox_folder_index_folder"), "mail_mailbox_folder_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), "mail_mailbox_folder_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_profile_id"), "mail_mailbox_folder_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), "mail_mailbox_folder_index", ["tenant_id"], unique=False)
op.create_index("ix_mail_mailbox_folder_index_tenant_profile", "mail_mailbox_folder_index", ["tenant_id", "profile_id"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_folder_index")}
if "message_indexed_at" not in columns:
op.add_column("mail_mailbox_folder_index", sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True))
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
if "mail_mailbox_message_index" not in tables:
op.create_table(
"mail_mailbox_message_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("uid", sa.String(length=255), nullable=False),
sa.Column("uid_int", sa.BigInteger(), nullable=False),
sa.Column("sort_position", sa.BigInteger(), nullable=False),
sa.Column("subject", sa.Text(), nullable=True),
sa.Column("from_header", sa.Text(), nullable=True),
sa.Column("to_header", sa.Text(), nullable=True),
sa.Column("cc_header", sa.Text(), nullable=True),
sa.Column("date", sa.String(length=255), nullable=True),
sa.Column("message_id", sa.Text(), nullable=True),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("body_preview", sa.Text(), nullable=True),
sa.Column("attachment_count", sa.Integer(), nullable=False),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_message_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_message_index")),
sa.UniqueConstraint("profile_id", "folder", "uid", name="uq_mail_mailbox_message_index_profile_folder_uid"),
)
op.create_index(op.f("ix_mail_mailbox_message_index_folder"), "mail_mailbox_message_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_indexed_at"), "mail_mailbox_message_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_profile_id"), "mail_mailbox_message_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_tenant_id"), "mail_mailbox_message_index", ["tenant_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_uid_int"), "mail_mailbox_message_index", ["uid_int"], unique=False)
op.create_index("ix_mail_mailbox_message_index_page", "mail_mailbox_message_index", ["tenant_id", "profile_id", "folder", "sort_position"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_message_index")}
if "sort_position" not in columns:
op.add_column("mail_mailbox_message_index", sa.Column("sort_position", sa.BigInteger(), nullable=False, server_default="0"))
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_message_index" in tables:
op.drop_index("ix_mail_mailbox_message_index_page", table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_uid_int"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_tenant_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_profile_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_indexed_at"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_folder"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_sort_position"), table_name="mail_mailbox_message_index")
op.drop_table("mail_mailbox_message_index")
if "mail_mailbox_folder_index" in tables:
op.drop_index("ix_mail_mailbox_folder_index_tenant_profile", table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_profile_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_folder"), table_name="mail_mailbox_folder_index")
op.drop_table("mail_mailbox_folder_index")
@@ -0,0 +1,67 @@
"""repair mail mailbox index columns
Revision ID: 5f708192abcd
Revises: 4e5f708192ab
Create Date: 2026-07-14 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "5f708192abcd"
down_revision = "4e5f708192ab"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_folder_index" in tables:
columns = _columns(inspector, "mail_mailbox_folder_index")
indexes = _indexes(inspector, "mail_mailbox_folder_index")
if "message_indexed_at" not in columns:
op.add_column("mail_mailbox_folder_index", sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True))
if "ix_mail_mailbox_folder_index_message_indexed_at" not in indexes:
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
if "mail_mailbox_message_index" in tables:
columns = _columns(inspector, "mail_mailbox_message_index")
indexes = _indexes(inspector, "mail_mailbox_message_index")
if "sort_position" not in columns:
op.add_column("mail_mailbox_message_index", sa.Column("sort_position", sa.BigInteger(), nullable=False, server_default="0"))
if op.get_bind().dialect.name != "sqlite":
op.alter_column("mail_mailbox_message_index", "sort_position", server_default=None)
if "ix_mail_mailbox_message_index_sort_position" not in indexes:
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
if "ix_mail_mailbox_message_index_page" not in indexes:
op.create_index("ix_mail_mailbox_message_index_page", "mail_mailbox_message_index", ["tenant_id", "profile_id", "folder", "sort_position"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_message_index" in tables:
indexes = _indexes(inspector, "mail_mailbox_message_index")
if "ix_mail_mailbox_message_index_page" in indexes:
op.drop_index("ix_mail_mailbox_message_index_page", table_name="mail_mailbox_message_index")
if "ix_mail_mailbox_message_index_sort_position" in indexes:
op.drop_index(op.f("ix_mail_mailbox_message_index_sort_position"), table_name="mail_mailbox_message_index")
if "sort_position" in _columns(inspector, "mail_mailbox_message_index"):
op.drop_column("mail_mailbox_message_index", "sort_position")
if "mail_mailbox_folder_index" in tables:
indexes = _indexes(inspector, "mail_mailbox_folder_index")
if "ix_mail_mailbox_folder_index_message_indexed_at" in indexes:
op.drop_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), table_name="mail_mailbox_folder_index")
if "message_indexed_at" in _columns(inspector, "mail_mailbox_folder_index"):
op.drop_column("mail_mailbox_folder_index", "message_indexed_at")
def _columns(inspector, table_name: str) -> set[str]:
return {column["name"] for column in inspector.get_columns(table_name)}
def _indexes(inspector, table_name: str) -> set[str]:
return {index["name"] for index in inspector.get_indexes(table_name)}
@@ -0,0 +1,78 @@
"""add opaque Mail-owned transport revisions
Revision ID: 608192abcdef
Revises: 5f708192abcd
Create Date: 2026-07-21 00:00:00.000000
"""
from __future__ import annotations
import uuid
from alembic import op
import sqlalchemy as sa
revision = "608192abcdef"
down_revision = "5f708192abcd"
branch_labels = None
depends_on = None
_SMTP_COLUMN = "smtp_transport_revision"
_IMAP_COLUMN = "imap_transport_revision"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if "mail_server_profiles" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")}
with op.batch_alter_table("mail_server_profiles") as batch:
if _SMTP_COLUMN not in columns:
batch.add_column(sa.Column(_SMTP_COLUMN, sa.String(length=36), nullable=True))
if _IMAP_COLUMN not in columns:
batch.add_column(sa.Column(_IMAP_COLUMN, sa.String(length=36), nullable=True))
rows = list(bind.execute(
sa.text(
"SELECT id, smtp_transport_revision, imap_transport_revision "
"FROM mail_server_profiles"
)
).mappings())
for row in rows:
values: dict[str, str] = {}
if not row[_SMTP_COLUMN]:
values[_SMTP_COLUMN] = str(uuid.uuid4())
if not row[_IMAP_COLUMN]:
values[_IMAP_COLUMN] = str(uuid.uuid4())
if values:
bind.execute(
sa.text(
"UPDATE mail_server_profiles "
"SET smtp_transport_revision = COALESCE(:smtp_revision, smtp_transport_revision), "
"imap_transport_revision = COALESCE(:imap_revision, imap_transport_revision) "
"WHERE id = :profile_id"
),
{
"profile_id": row["id"],
"smtp_revision": values.get(_SMTP_COLUMN),
"imap_revision": values.get(_IMAP_COLUMN),
},
)
with op.batch_alter_table("mail_server_profiles") as batch:
batch.alter_column(_SMTP_COLUMN, existing_type=sa.String(length=36), nullable=False)
batch.alter_column(_IMAP_COLUMN, existing_type=sa.String(length=36), nullable=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if "mail_server_profiles" not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")}
with op.batch_alter_table("mail_server_profiles") as batch:
if _IMAP_COLUMN in columns:
batch.drop_column(_IMAP_COLUMN)
if _SMTP_COLUMN in columns:
batch.drop_column(_SMTP_COLUMN)
@@ -0,0 +1,265 @@
"""split mail envelopes into servers and reusable credential bindings
Revision ID: 7192a3bcdef0
Revises: 608192abcdef
Create Date: 2026-07-23 00:00:00.000000
"""
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from alembic import op
import sqlalchemy as sa
revision = "7192a3bcdef0"
down_revision = "608192abcdef"
branch_labels = None
depends_on = "c91f0a72be34"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
tables = set(inspector.get_table_names())
if "mail_server_profiles" not in tables:
return
profile_columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")}
if "inherit_to_lower_scopes" not in profile_columns:
with op.batch_alter_table("mail_server_profiles") as batch:
batch.add_column(
sa.Column(
"inherit_to_lower_scopes",
sa.Boolean(),
nullable=False,
server_default=sa.true(),
)
)
if "mail_server_endpoints" not in tables:
op.create_table(
"mail_server_endpoints",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=True),
sa.Column("protocol", sa.String(length=20), nullable=False),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("config", sa.JSON(), nullable=False),
sa.Column("scope_type", sa.String(length=20), nullable=False),
sa.Column("scope_id", sa.String(length=36), nullable=True),
sa.Column("inherit_to_lower_scopes", sa.Boolean(), nullable=False),
sa.Column("is_default", sa.Boolean(), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False),
sa.Column("transport_revision", sa.String(length=36), nullable=False),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"],
["mail_server_profiles.id"],
name=op.f("fk_mail_server_endpoints_profile_id_mail_server_profiles"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["tenant_id"],
["core_scopes.id"],
name=op.f("fk_mail_server_endpoints_tenant_id_core_scopes"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["created_by_user_id"],
["access_users.id"],
name=op.f("fk_mail_server_endpoints_created_by_user_id_access_users"),
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["updated_by_user_id"],
["access_users.id"],
name=op.f("fk_mail_server_endpoints_updated_by_user_id_access_users"),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_server_endpoints")),
sa.UniqueConstraint(
"profile_id",
"protocol",
"name",
name="uq_mail_server_endpoints_profile_protocol_name",
),
)
_create_endpoint_indexes()
inspector = sa.inspect(bind)
if "mail_server_credential_bindings" not in inspector.get_table_names():
op.create_table(
"mail_server_credential_bindings",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("server_id", sa.String(length=36), nullable=False),
sa.Column("credential_id", sa.String(length=36), nullable=False),
sa.Column("is_default", sa.Boolean(), nullable=False),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["server_id"],
["mail_server_endpoints.id"],
name=op.f("fk_mail_server_credential_bindings_server_id_mail_server_endpoints"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["credential_id"],
["core_credential_envelopes.id"],
name=op.f("fk_mail_server_credential_bindings_credential_id_core_credential_envelopes"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["created_by_user_id"],
["access_users.id"],
name=op.f("fk_mail_server_credential_bindings_created_by_user_id_access_users"),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_server_credential_bindings")),
sa.UniqueConstraint(
"server_id",
"credential_id",
name="uq_mail_server_credential_bindings_server_credential",
),
)
op.create_index(
"ix_mail_server_credential_bindings_default",
"mail_server_credential_bindings",
["server_id", "is_default"],
unique=False,
)
for column in ("server_id", "credential_id", "is_default", "created_by_user_id"):
op.create_index(
op.f(f"ix_mail_server_credential_bindings_{column}"),
"mail_server_credential_bindings",
[column],
unique=False,
)
_seed_legacy_endpoints(bind)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_server_credential_bindings" in tables:
op.drop_table("mail_server_credential_bindings")
if "mail_server_endpoints" in tables:
op.drop_table("mail_server_endpoints")
if "mail_server_profiles" in tables:
columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")}
if "inherit_to_lower_scopes" in columns:
with op.batch_alter_table("mail_server_profiles") as batch:
batch.drop_column("inherit_to_lower_scopes")
def _create_endpoint_indexes() -> None:
op.create_index(
"ix_mail_server_endpoints_profile_protocol",
"mail_server_endpoints",
["profile_id", "protocol", "is_active"],
unique=False,
)
op.create_index(
"ix_mail_server_endpoints_scope",
"mail_server_endpoints",
["tenant_id", "scope_type", "scope_id"],
unique=False,
)
for column in (
"profile_id",
"tenant_id",
"protocol",
"scope_type",
"scope_id",
"is_default",
"is_active",
"created_by_user_id",
"updated_by_user_id",
):
op.create_index(
op.f(f"ix_mail_server_endpoints_{column}"),
"mail_server_endpoints",
[column],
unique=False,
)
def _seed_legacy_endpoints(bind) -> None:
existing = {
(row.profile_id, row.protocol)
for row in bind.execute(
sa.text("SELECT profile_id, protocol FROM mail_server_endpoints WHERE is_default = :is_default"),
{"is_default": True},
)
}
rows = bind.execute(
sa.text(
"SELECT id, tenant_id, scope_type, scope_id, smtp_config, imap_config, "
"smtp_transport_revision, imap_transport_revision, created_by_user_id, updated_by_user_id "
"FROM mail_server_profiles"
)
).mappings()
now = datetime.now(timezone.utc)
table = sa.table(
"mail_server_endpoints",
sa.column("id", sa.String),
sa.column("profile_id", sa.String),
sa.column("tenant_id", sa.String),
sa.column("protocol", sa.String),
sa.column("name", sa.String),
sa.column("config", sa.JSON),
sa.column("scope_type", sa.String),
sa.column("scope_id", sa.String),
sa.column("inherit_to_lower_scopes", sa.Boolean),
sa.column("is_default", sa.Boolean),
sa.column("is_active", sa.Boolean),
sa.column("transport_revision", sa.String),
sa.column("created_by_user_id", sa.String),
sa.column("updated_by_user_id", sa.String),
sa.column("created_at", sa.DateTime(timezone=True)),
sa.column("updated_at", sa.DateTime(timezone=True)),
)
for row in rows:
for protocol, config_key, revision_key in (
("smtp", "smtp_config", "smtp_transport_revision"),
("imap", "imap_config", "imap_transport_revision"),
):
config = _json_object(row[config_key])
if not config or (row["id"], protocol) in existing:
continue
bind.execute(
table.insert().values(
id=str(uuid.uuid4()),
profile_id=row["id"],
tenant_id=row["tenant_id"],
protocol=protocol,
name=protocol.upper(),
config=config,
scope_type=row["scope_type"] or "tenant",
scope_id=row["scope_id"],
inherit_to_lower_scopes=True,
is_default=True,
is_active=True,
transport_revision=row[revision_key] or str(uuid.uuid4()),
created_by_user_id=row["created_by_user_id"],
updated_by_user_id=row["updated_by_user_id"],
created_at=now,
updated_at=now,
)
)
def _json_object(value):
if isinstance(value, dict):
return value
if isinstance(value, str) and value.strip():
parsed = json.loads(value)
return parsed if isinstance(parsed, dict) else {}
return {}
@@ -0,0 +1,243 @@
"""add durable mail delivery outbox
Revision ID: 82a3b4c5d6e7
Revises: 7192a3bcdef0
Create Date: 2026-07-30 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "82a3b4c5d6e7"
down_revision = "7192a3bcdef0"
branch_labels = None
depends_on = "c91f0a72be34"
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_delivery_commands" not in tables:
op.create_table(
"mail_delivery_commands",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("command_type", sa.String(length=60), nullable=False),
sa.Column("source_module", sa.String(length=80), nullable=False),
sa.Column("source_resource_type", sa.String(length=80), nullable=False),
sa.Column("source_resource_id", sa.String(length=120), nullable=True),
sa.Column("source_version_id", sa.String(length=120), nullable=True),
sa.Column("idempotency_key", sa.String(length=200), nullable=False),
sa.Column("canonical_request_hash", sa.String(length=64), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("smtp_server_id", sa.String(length=36), nullable=True),
sa.Column("smtp_credential_id", sa.String(length=36), nullable=True),
sa.Column(
"expected_smtp_transport_revision",
sa.String(length=120),
nullable=False,
),
sa.Column("envelope_from_encrypted", sa.Text(), nullable=True),
sa.Column("envelope_recipients_encrypted", sa.Text(), nullable=True),
sa.Column("from_header_encrypted", sa.Text(), nullable=True),
sa.Column("message_encrypted", sa.Text(), nullable=True),
sa.Column("message_sha256", sa.String(length=64), nullable=False),
sa.Column("message_size_bytes", sa.BigInteger(), nullable=False),
sa.Column("recipient_count", sa.Integer(), nullable=False),
sa.Column("status", sa.String(length=40), nullable=False),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("claimed_by", sa.String(length=255), nullable=True),
sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("effect_started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("accepted_count", sa.Integer(), nullable=False),
sa.Column("refused_count", sa.Integer(), nullable=False),
sa.Column("refusal_summary", sa.JSON(), nullable=False),
sa.Column("refusal_details_encrypted", sa.Text(), nullable=True),
sa.Column("failure_code", sa.String(length=80), nullable=True),
sa.Column("failure_summary", sa.String(length=500), nullable=True),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("supersedes_command_id", sa.String(length=36), nullable=True),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("payload_purged_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"],
["mail_server_profiles.id"],
name=op.f(
"fk_mail_delivery_commands_profile_id_mail_server_profiles"
),
ondelete="RESTRICT",
),
sa.ForeignKeyConstraint(
["created_by_user_id"],
["access_users.id"],
name=op.f(
"fk_mail_delivery_commands_created_by_user_id_access_users"
),
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["supersedes_command_id"],
["mail_delivery_commands.id"],
name=op.f(
"fk_mail_delivery_commands_supersedes_command_id_mail_delivery_commands"
),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint(
"id",
name=op.f("pk_mail_delivery_commands"),
),
sa.UniqueConstraint(
"tenant_id",
"command_type",
"idempotency_key",
name="uq_mail_delivery_commands_idempotency",
),
)
op.create_index(
"ix_mail_delivery_commands_dispatch",
"mail_delivery_commands",
["status", "next_attempt_at", "created_at"],
unique=False,
)
op.create_index(
"ix_mail_delivery_commands_source",
"mail_delivery_commands",
[
"tenant_id",
"source_module",
"source_resource_type",
"source_resource_id",
],
unique=False,
)
for column in (
"tenant_id",
"command_type",
"source_module",
"profile_id",
"status",
"next_attempt_at",
"created_by_user_id",
"supersedes_command_id",
"expires_at",
):
op.create_index(
op.f(f"ix_mail_delivery_commands_{column}"),
"mail_delivery_commands",
[column],
unique=False,
)
inspector = sa.inspect(op.get_bind())
if "mail_delivery_attempts" not in inspector.get_table_names():
op.create_table(
"mail_delivery_attempts",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("command_id", sa.String(length=36), nullable=False),
sa.Column("attempt_number", sa.Integer(), nullable=False),
sa.Column("worker_id", sa.String(length=255), nullable=True),
sa.Column("status", sa.String(length=40), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("effect_started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("accepted_count", sa.Integer(), nullable=False),
sa.Column("refused_count", sa.Integer(), nullable=False),
sa.Column("outcome_code", sa.String(length=80), nullable=True),
sa.Column("diagnostic_summary", sa.String(length=500), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["command_id"],
["mail_delivery_commands.id"],
name=op.f(
"fk_mail_delivery_attempts_command_id_mail_delivery_commands"
),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint(
"id",
name=op.f("pk_mail_delivery_attempts"),
),
sa.UniqueConstraint(
"command_id",
"attempt_number",
name="uq_mail_delivery_attempts_number",
),
)
op.create_index(
"ix_mail_delivery_attempts_command_started",
"mail_delivery_attempts",
["command_id", "started_at"],
unique=False,
)
for column in ("command_id", "status"):
op.create_index(
op.f(f"ix_mail_delivery_attempts_{column}"),
"mail_delivery_attempts",
[column],
unique=False,
)
inspector = sa.inspect(op.get_bind())
if "mail_delivery_reconciliations" not in inspector.get_table_names():
op.create_table(
"mail_delivery_reconciliations",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("command_id", sa.String(length=36), nullable=False),
sa.Column("decision", sa.String(length=40), nullable=False),
sa.Column("evidence_reference", sa.String(length=500), nullable=False),
sa.Column("note_encrypted", sa.Text(), nullable=True),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["command_id"],
["mail_delivery_commands.id"],
name=op.f(
"fk_mail_delivery_reconciliations_command_id_mail_delivery_commands"
),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["created_by_user_id"],
["access_users.id"],
name=op.f(
"fk_mail_delivery_reconciliations_created_by_user_id_access_users"
),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint(
"id",
name=op.f("pk_mail_delivery_reconciliations"),
),
)
op.create_index(
"ix_mail_delivery_reconciliations_command_created",
"mail_delivery_reconciliations",
["command_id", "created_at"],
unique=False,
)
for column in ("command_id", "created_by_user_id"):
op.create_index(
op.f(f"ix_mail_delivery_reconciliations_{column}"),
"mail_delivery_reconciliations",
[column],
unique=False,
)
def downgrade() -> None:
tables = set(sa.inspect(op.get_bind()).get_table_names())
if "mail_delivery_reconciliations" in tables:
op.drop_table("mail_delivery_reconciliations")
if "mail_delivery_attempts" in tables:
op.drop_table("mail_delivery_attempts")
if "mail_delivery_commands" in tables:
op.drop_table("mail_delivery_commands")
@@ -0,0 +1,133 @@
"""add mail bounce processing
Revision ID: 93b4c5d6e7f8
Revises: 82a3b4c5d6e7
Create Date: 2026-07-31 19:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "93b4c5d6e7f8"
down_revision = "82a3b4c5d6e7"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("mail_delivery_commands") as batch_op:
batch_op.add_column(
sa.Column("rfc_message_id", sa.String(length=998), nullable=True)
)
batch_op.create_index(
"ix_mail_delivery_commands_rfc_message_id",
("rfc_message_id",),
)
op.create_table(
"mail_bounce_sources",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("imap_server_id", sa.String(length=36), nullable=True),
sa.Column("imap_credential_id", sa.String(length=36), nullable=True),
sa.Column(
"expected_imap_transport_revision",
sa.String(length=120),
nullable=False,
),
sa.Column("is_active", sa.Boolean(), nullable=False),
sa.Column("uidvalidity", sa.String(length=255), nullable=True),
sa.Column("highest_processed_uid", sa.BigInteger(), nullable=False),
sa.Column("last_scanned_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.String(length=500), nullable=True),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"], ["mail_server_profiles.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"profile_id", "folder", name="uq_mail_bounce_sources_profile_folder"
),
)
op.create_index("ix_mail_bounce_sources_tenant_id", "mail_bounce_sources", ["tenant_id"])
op.create_index("ix_mail_bounce_sources_profile_id", "mail_bounce_sources", ["profile_id"])
op.create_index("ix_mail_bounce_sources_is_active", "mail_bounce_sources", ["is_active"])
op.create_index("ix_mail_bounce_sources_last_scanned_at", "mail_bounce_sources", ["last_scanned_at"])
op.create_index("ix_mail_bounce_sources_created_by_user_id", "mail_bounce_sources", ["created_by_user_id"])
op.create_index(
"ix_mail_bounce_sources_scan",
"mail_bounce_sources",
["is_active", "last_scanned_at"],
)
op.create_table(
"mail_bounce_observations",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("uid", sa.String(length=255), nullable=False),
sa.Column("fingerprint", sa.String(length=64), nullable=False),
sa.Column("raw_sha256", sa.String(length=64), nullable=False),
sa.Column("original_message_id", sa.String(length=998), nullable=True),
sa.Column("command_id", sa.String(length=36), nullable=True),
sa.Column("recipient", sa.String(length=998), nullable=True),
sa.Column("action", sa.String(length=40), nullable=False),
sa.Column("status_code", sa.String(length=80), nullable=True),
sa.Column("diagnostic", sa.String(length=500), nullable=True),
sa.Column("permanent", sa.Boolean(), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("matched", sa.Boolean(), nullable=False),
sa.Column("evidence", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"], ["mail_server_profiles.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["command_id"], ["mail_delivery_commands.id"], ondelete="SET NULL"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"tenant_id",
"fingerprint",
name="uq_mail_bounce_observations_fingerprint",
),
)
for column in (
"tenant_id",
"profile_id",
"original_message_id",
"command_id",
"action",
"observed_at",
"matched",
):
op.create_index(
f"ix_mail_bounce_observations_{column}",
"mail_bounce_observations",
[column],
)
op.create_index(
"ix_mail_bounce_observations_command",
"mail_bounce_observations",
["tenant_id", "command_id", "observed_at"],
)
def downgrade() -> None:
op.drop_table("mail_bounce_observations")
op.drop_table("mail_bounce_sources")
with op.batch_alter_table("mail_delivery_commands") as batch_op:
batch_op.drop_index("ix_mail_delivery_commands_rfc_message_id")
batch_op.drop_column("rfc_message_id")
@@ -0,0 +1,91 @@
"""add governed POP3 legacy imports
Revision ID: a4c5d6e7f809
Revises: 93b4c5d6e7f8
Create Date: 2026-08-22 12:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "a4c5d6e7f809"
down_revision = "93b4c5d6e7f8"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"mail_pop3_imports",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("pop3_server_id", sa.String(length=36), nullable=False),
sa.Column("pop3_credential_id", sa.String(length=36), nullable=True),
sa.Column("transport_revision", sa.String(length=120), nullable=False),
sa.Column("provider_uidl", sa.String(length=500), nullable=False),
sa.Column("provider_message_number", sa.Integer(), nullable=True),
sa.Column("fingerprint", sa.String(length=64), nullable=False),
sa.Column("raw_sha256", sa.String(length=64), nullable=False),
sa.Column("raw_message_encrypted", sa.Text(), nullable=False),
sa.Column("message_id", sa.String(length=998), nullable=True),
sa.Column("subject", sa.Text(), nullable=True),
sa.Column("from_header", sa.Text(), nullable=True),
sa.Column("to_header", sa.Text(), nullable=True),
sa.Column("date", sa.String(length=255), nullable=True),
sa.Column("body_preview", sa.Text(), nullable=True),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("status", sa.String(length=40), nullable=False),
sa.Column("imported_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("imported_by_user_id", sa.String(length=36), nullable=True),
sa.Column("deletion_requested", sa.Boolean(), nullable=False),
sa.Column("deletion_status", sa.String(length=40), nullable=False),
sa.Column("deletion_attempted_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("deletion_error", sa.String(length=500), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["profile_id"], ["mail_server_profiles.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["pop3_server_id"], ["mail_server_endpoints.id"], ondelete="RESTRICT"
),
sa.ForeignKeyConstraint(
["imported_by_user_id"], ["access_users.id"], ondelete="SET NULL"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"tenant_id",
"profile_id",
"pop3_server_id",
"provider_uidl",
name="uq_mail_pop3_imports_source_uidl",
),
)
for column in (
"tenant_id",
"profile_id",
"pop3_server_id",
"fingerprint",
"message_id",
"status",
"imported_at",
"imported_by_user_id",
"deletion_status",
):
op.create_index(
f"ix_mail_pop3_imports_{column}",
"mail_pop3_imports",
[column],
)
op.create_index(
"ix_mail_pop3_imports_review",
"mail_pop3_imports",
["tenant_id", "status", "imported_at"],
)
def downgrade() -> None:
op.drop_table("mail_pop3_imports")
+246
View File
@@ -0,0 +1,246 @@
from __future__ import annotations
import base64
import hashlib
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Iterable
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from govoplan_core.security.secrets import encrypt_secret
from govoplan_mail.backend.db.models import MailPop3Import, MailServerEndpoint
from govoplan_mail.backend.sending.pop3 import Pop3DownloadedMessage
class Pop3ImportError(RuntimeError):
pass
@dataclass(frozen=True, slots=True)
class Pop3ImportResult:
imported: tuple[MailPop3Import, ...]
duplicates: tuple[MailPop3Import, ...]
def create_pop3_imports(
session: Session,
*,
tenant_id: str,
profile_id: str,
pop3_server_id: str,
pop3_credential_id: str | None,
transport_revision: str,
messages: Iterable[Pop3DownloadedMessage],
user_id: str | None,
deletion_requested: bool,
) -> Pop3ImportResult:
downloaded = tuple(messages)
if not downloaded:
raise Pop3ImportError("No POP3 messages were downloaded for import")
uidls = [item.uidl for item in downloaded]
if len(uidls) != len(set(uidls)):
raise Pop3ImportError("The POP3 download contained duplicate UIDL identifiers")
# Serialize imports per source before checking UIDLs. The database unique
# constraint remains the last line of defense, while this lock lets a
# concurrent request observe the first request's committed rows and report
# them as duplicates instead of surfacing an integrity error.
source = session.scalar(
select(MailServerEndpoint)
.where(
MailServerEndpoint.id == pop3_server_id,
MailServerEndpoint.profile_id == profile_id,
or_(
MailServerEndpoint.tenant_id == tenant_id,
MailServerEndpoint.tenant_id.is_(None),
),
MailServerEndpoint.protocol == "pop3",
)
.with_for_update()
)
if source is None:
raise Pop3ImportError("The selected POP3 source is unavailable")
existing = {
row.provider_uidl: row
for row in session.scalars(
select(MailPop3Import).where(
MailPop3Import.tenant_id == tenant_id,
MailPop3Import.profile_id == profile_id,
MailPop3Import.pop3_server_id == pop3_server_id,
MailPop3Import.provider_uidl.in_(uidls),
)
)
}
imported: list[MailPop3Import] = []
duplicates: list[MailPop3Import] = []
now = datetime.now(timezone.utc)
for message in downloaded:
duplicate = existing.get(message.uidl)
if duplicate is not None:
duplicates.append(duplicate)
continue
encrypted = encrypt_secret(base64.b64encode(message.raw).decode("ascii"))
if not encrypted:
raise Pop3ImportError("The downloaded POP3 message could not be encrypted")
summary = message.summary
row = MailPop3Import(
tenant_id=tenant_id,
profile_id=profile_id,
pop3_server_id=pop3_server_id,
pop3_credential_id=pop3_credential_id,
transport_revision=_required_revision(transport_revision),
provider_uidl=message.uidl,
provider_message_number=message.message_number,
fingerprint=_fingerprint(
tenant_id=tenant_id,
profile_id=profile_id,
pop3_server_id=pop3_server_id,
uidl=message.uidl,
raw_sha256=message.raw_sha256,
),
raw_sha256=message.raw_sha256,
raw_message_encrypted=encrypted,
message_id=summary.message_id,
subject=summary.subject,
from_header=summary.from_header,
to_header=summary.to_header,
date=summary.date,
body_preview=summary.body_preview,
size_bytes=len(message.raw),
status="pending_review",
imported_at=now,
imported_by_user_id=user_id,
deletion_requested=bool(deletion_requested),
deletion_status=("pending" if deletion_requested else "not_requested"),
)
session.add(row)
imported.append(row)
session.flush()
return Pop3ImportResult(imported=tuple(imported), duplicates=tuple(duplicates))
def list_pop3_imports(
session: Session,
*,
tenant_id: str,
profile_id: str | None = None,
profile_ids: Iterable[str] | None = None,
limit: int = 100,
) -> tuple[MailPop3Import, ...]:
statement = select(MailPop3Import).where(
MailPop3Import.tenant_id == tenant_id
)
if profile_id:
statement = statement.where(MailPop3Import.profile_id == profile_id)
elif profile_ids is not None:
allowed = tuple(dict.fromkeys(str(value) for value in profile_ids if value))
if not allowed:
return ()
statement = statement.where(MailPop3Import.profile_id.in_(allowed))
rows = session.scalars(
statement.order_by(
MailPop3Import.imported_at.desc(),
MailPop3Import.id.desc(),
).limit(max(1, min(int(limit), 500)))
)
return tuple(rows)
def mark_pop3_deletion_result(
session: Session,
*,
tenant_id: str,
import_ids: Iterable[str],
status: str,
error: str | None = None,
) -> tuple[MailPop3Import, ...]:
clean_status = str(status or "").strip().casefold()
if clean_status not in {"succeeded", "failed", "outcome_unknown"}:
raise Pop3ImportError("Unsupported POP3 deletion result")
ids = tuple(dict.fromkeys(str(value).strip() for value in import_ids if str(value).strip()))
if not ids:
return ()
rows = tuple(
session.scalars(
select(MailPop3Import)
.where(
MailPop3Import.tenant_id == tenant_id,
MailPop3Import.id.in_(ids),
MailPop3Import.deletion_requested.is_(True),
)
.with_for_update()
)
)
if len(rows) != len(ids):
raise Pop3ImportError("One or more POP3 import records are unavailable")
now = datetime.now(timezone.utc)
safe_error = _bounded_error(error)
for row in rows:
row.deletion_status = clean_status
row.deletion_attempted_at = now
row.deletion_error = safe_error
session.flush()
return rows
def pop3_import_payload(row: MailPop3Import) -> dict[str, object]:
return {
"id": row.id,
"profile_id": row.profile_id,
"pop3_server_id": row.pop3_server_id,
"transport_revision": row.transport_revision,
"provider_uidl": row.provider_uidl,
"message_id": row.message_id,
"subject": row.subject,
"from_header": row.from_header,
"to_header": row.to_header,
"date": row.date,
"body_preview": row.body_preview,
"size_bytes": row.size_bytes,
"raw_sha256": row.raw_sha256,
"status": row.status,
"imported_at": row.imported_at,
"deletion_requested": row.deletion_requested,
"deletion_status": row.deletion_status,
"deletion_attempted_at": row.deletion_attempted_at,
"deletion_error": row.deletion_error,
}
def _fingerprint(
*,
tenant_id: str,
profile_id: str,
pop3_server_id: str,
uidl: str,
raw_sha256: str,
) -> str:
material = "\x1f".join(
(tenant_id, profile_id, pop3_server_id, uidl, raw_sha256)
).encode("utf-8")
return hashlib.sha256(material).hexdigest()
def _required_revision(value: object) -> str:
clean = str(value or "").strip()
if not clean or len(clean) > 120:
raise Pop3ImportError("A valid POP3 transport revision is required")
return clean
def _bounded_error(value: str | None) -> str | None:
clean = " ".join(str(value or "").split())
return clean[:500] or None
__all__ = [
"Pop3ImportError",
"Pop3ImportResult",
"create_pop3_imports",
"list_pop3_imports",
"mark_pop3_deletion_result",
"pop3_import_payload",
]
+211
View File
@@ -0,0 +1,211 @@
from __future__ import annotations
import hashlib
from email import policy
from email.message import Message
from email.parser import BytesParser
from email.utils import getaddresses
from sqlalchemy.orm import Session
from govoplan_core.core.mail import (
MailPostboxBridgeProvider,
MailPostboxBridgeRequest,
MailPostboxBridgeResult,
)
from govoplan_core.core.modules import ModuleContext
from govoplan_core.core.postbox import (
PostboxAttachmentRef,
PostboxDeliveryRequest,
PostboxParticipantRef,
PostboxTargetRef,
postbox_delivery_provider,
)
from govoplan_mail.backend.db.models import MailServerProfile
from govoplan_mail.backend.runtime import configure_runtime, get_registry
MAX_BRIDGE_MESSAGE_BYTES = 50 * 1024 * 1024
MAX_POSTBOX_BODY_CHARS = 500_000
class MailPostboxBridgeError(RuntimeError):
pass
class MailPostboxBridge(MailPostboxBridgeProvider):
"""Translate immutable IMAP observations into native Postbox delivery."""
def bridge_message(
self,
session: object,
request: MailPostboxBridgeRequest,
) -> MailPostboxBridgeResult:
if not isinstance(session, Session):
raise TypeError("Mail Postbox bridging requires a SQLAlchemy session.")
if not isinstance(request.target, PostboxTargetRef):
raise MailPostboxBridgeError("A typed Postbox target is required.")
if len(request.raw_message) > MAX_BRIDGE_MESSAGE_BYTES:
raise MailPostboxBridgeError("Mail message exceeds the Postbox bridge limit.")
profile = session.get(MailServerProfile, request.profile_id)
if profile is None or profile.tenant_id != request.tenant_id:
raise MailPostboxBridgeError("Mail profile not found.")
folder = request.folder.strip()
uid = request.uid.strip()
uidvalidity = request.uidvalidity.strip()
if not folder or not uid or not uidvalidity:
raise MailPostboxBridgeError(
"Mail folder, UIDVALIDITY, and immutable UID are required."
)
try:
message = BytesParser(policy=policy.default).parsebytes(request.raw_message)
except Exception as exc:
raise MailPostboxBridgeError("Mail message could not be parsed.") from exc
provider = postbox_delivery_provider(get_registry())
if provider is None:
raise MailPostboxBridgeError("Postbox delivery is not available.")
source_digest = hashlib.sha256(request.raw_message).hexdigest()
source_key = hashlib.sha256(
f"{request.tenant_id}\0{request.profile_id}\0{folder}\0{uidvalidity}\0{uid}".encode("utf-8")
).hexdigest()
result = provider.deliver(
session,
PostboxDeliveryRequest(
tenant_id=request.tenant_id,
target=request.target,
producer_module="mail",
producer_resource_type="imap_message",
producer_resource_id=source_key,
idempotency_key=f"mail-postbox:{source_key}:{source_digest}",
subject=_header(message, "Subject") or "(No subject)",
body_text=_plain_text_body(message),
sender_label=_header(message, "From"),
classification=request.classification,
participants=_participants(message),
attachments=_attachments(
message,
profile_id=request.profile_id,
folder=folder,
uidvalidity=uidvalidity,
uid=uid,
),
metadata={
**dict(request.metadata),
"transport": "mail-imap",
"mail_profile_id": request.profile_id,
"mailbox_folder": folder,
"mailbox_uidvalidity": uidvalidity,
"mailbox_uid": uid,
"rfc_message_id": _header(message, "Message-ID"),
"raw_sha256": source_digest,
},
),
)
return MailPostboxBridgeResult(
postbox_id=result.postbox_id,
message_id=result.message_id,
delivery_id=result.delivery_id,
duplicate=result.duplicate,
source_digest=source_digest,
)
def _header(message: Message, name: str) -> str | None:
value = " ".join(str(message.get(name) or "").split())
return value[:1000] or None
def _plain_text_body(message: Message) -> str | None:
candidates = message.walk() if message.is_multipart() else (message,)
for part in candidates:
if part.get_content_type() != "text/plain":
continue
if part.get_content_disposition() == "attachment":
continue
try:
value = part.get_content()
except Exception:
payload = part.get_payload(decode=True) or b""
value = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
if isinstance(value, bytes):
value = value.decode(part.get_content_charset() or "utf-8", errors="replace")
normalized = str(value).strip()
if normalized:
return normalized[:MAX_POSTBOX_BODY_CHARS]
return None
def _participants(message: Message) -> tuple[PostboxParticipantRef, ...]:
result: list[PostboxParticipantRef] = []
for kind, headers in (
("sender", ("From",)),
("to", ("To",)),
("cc", ("Cc",)),
("bcc", ("Bcc",)),
):
for name, address in getaddresses(
[str(value) for header in headers for value in message.get_all(header, [])]
):
clean_address = address.strip()[:500]
if not clean_address:
continue
result.append(
PostboxParticipantRef(
kind=kind,
reference_type="external_email",
label=name.strip()[:500] or None,
address=clean_address,
)
)
return tuple(result)
def _attachments(
message: Message,
*,
profile_id: str,
folder: str,
uidvalidity: str,
uid: str,
) -> tuple[PostboxAttachmentRef, ...]:
result: list[PostboxAttachmentRef] = []
for index, part in enumerate(message.walk()):
filename = part.get_filename()
if part.get_content_disposition() != "attachment" and not filename:
continue
payload = part.get_payload(decode=True) or b""
digest = hashlib.sha256(payload).hexdigest()
reference_id = hashlib.sha256(
f"{profile_id}\0{folder}\0{uidvalidity}\0{uid}\0{index}\0{digest}".encode("utf-8")
).hexdigest()
result.append(
PostboxAttachmentRef(
reference_type="mail_attachment",
reference_id=reference_id,
name=str(filename or f"attachment-{index + 1}")[:1000],
media_type=part.get_content_type(),
size_bytes=len(payload),
digest=digest,
metadata={
"mail_profile_id": profile_id,
"mailbox_folder": folder,
"mailbox_uidvalidity": uidvalidity,
"mailbox_uid": uid,
"mime_part_index": index,
},
)
)
return tuple(result)
def create_postbox_bridge(context: ModuleContext) -> MailPostboxBridge:
configure_runtime(registry=context.registry, settings=context.settings)
return MailPostboxBridge()
__all__ = [
"MailPostboxBridge",
"MailPostboxBridgeError",
"create_postbox_bridge",
]
+502
View File
@@ -0,0 +1,502 @@
from __future__ import annotations
from collections import defaultdict
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import case, func, or_, select
from sqlalchemy.orm import Session
from govoplan_core.core.provider_governance import (
ExternalProviderRuntimeState,
ExternalProviderStateContext,
)
from govoplan_mail.backend.db.models import (
MailBounceSource,
MailDeliveryCommand,
MailMailboxFolderIndex,
MailMailboxMessageIndex,
MailPop3Import,
MailServerEndpoint,
MailServerProfile,
)
SMTP_PROVIDER_ID = "mail.smtp_delivery"
IMAP_PROVIDER_ID = "mail.imap_mailbox"
JMAP_PROVIDER_ID = "mail.jmap_mailbox"
POP3_PROVIDER_ID = "mail.pop3_legacy_import"
_CURRENT_INDEX_WINDOW = timedelta(minutes=30)
def smtp_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
return _mail_provider_states(context, protocol="smtp")
def imap_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
return _mail_provider_states(context, protocol="imap")
def jmap_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
return _mail_provider_states(context, protocol="jmap")
def pop3_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
if not isinstance(context.session, Session):
raise RuntimeError("Mail provider state requires a database session.")
profiles = _profiles(context)
if not profiles:
return ()
profile_ids = tuple(item.id for item in profiles)
endpoints = _endpoints(
context.session,
profile_ids=profile_ids,
protocol="pop3",
)
endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list)
for endpoint in endpoints:
endpoints_by_profile[endpoint.profile_id].append(endpoint)
metrics = _pop3_metrics(
context.session,
profile_ids=profile_ids,
tenant_id=context.tenant_id,
)
observed_at = datetime.now(UTC)
return tuple(
_pop3_state(
profile,
endpoints=endpoints_by_profile.get(profile.id, []),
metrics=metrics.get(profile.id, {}),
observed_at=observed_at,
)
for profile in profiles
if endpoints_by_profile.get(profile.id)
)
def _mail_provider_states(
context: ExternalProviderStateContext,
*,
protocol: str,
) -> tuple[ExternalProviderRuntimeState, ...]:
if not isinstance(context.session, Session):
raise RuntimeError("Mail provider state requires a database session.")
profiles = _profiles(context)
if not profiles:
return ()
profile_ids = tuple(item.id for item in profiles)
endpoints = _endpoints(context.session, profile_ids=profile_ids, protocol=protocol)
endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list)
for endpoint in endpoints:
endpoints_by_profile[endpoint.profile_id].append(endpoint)
observed_at = datetime.now(UTC)
if protocol == "smtp":
metrics = _smtp_metrics(context.session, profile_ids=profile_ids)
return tuple(
_smtp_state(
profile,
endpoints=endpoints_by_profile.get(profile.id, []),
metrics=metrics.get(profile.id, {}),
observed_at=observed_at,
)
for profile in profiles
if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "smtp")
)
metrics = _imap_metrics(context.session, profile_ids=profile_ids)
if protocol == "jmap":
metrics = {
profile_id: {
key: value
for key, value in values.items()
if key in {"indexed_folders", "indexed_messages", "last_indexed_at"}
}
for profile_id, values in metrics.items()
}
provider_id = JMAP_PROVIDER_ID if protocol == "jmap" else IMAP_PROVIDER_ID
return tuple(
_imap_state(
profile,
endpoints=endpoints_by_profile.get(profile.id, []),
metrics=metrics.get(profile.id, {}),
observed_at=observed_at,
protocol=protocol,
provider_id=provider_id,
)
for profile in profiles
if endpoints_by_profile.get(profile.id)
or (protocol == "imap" and _legacy_configured(profile, "imap"))
)
def _profiles(context: ExternalProviderStateContext) -> tuple[MailServerProfile, ...]:
statement = select(MailServerProfile)
if context.tenant_id is not None:
statement = statement.where(
or_(
MailServerProfile.tenant_id.is_(None),
MailServerProfile.tenant_id == context.tenant_id,
)
)
return tuple(
context.session.scalars(
statement.order_by(
MailServerProfile.tenant_id,
MailServerProfile.id,
).limit(context.max_items + 1)
)
)
def _endpoints(
session: Session,
*,
profile_ids: tuple[str, ...],
protocol: str,
) -> tuple[MailServerEndpoint, ...]:
return tuple(
session.scalars(
select(MailServerEndpoint).where(
MailServerEndpoint.profile_id.in_(profile_ids),
MailServerEndpoint.protocol == protocol,
)
)
)
def _smtp_metrics(
session: Session,
*,
profile_ids: tuple[str, ...],
) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = defaultdict(dict)
rows = session.execute(
select(
MailDeliveryCommand.profile_id,
MailDeliveryCommand.status,
func.count(MailDeliveryCommand.id),
func.max(MailDeliveryCommand.completed_at),
)
.where(MailDeliveryCommand.profile_id.in_(profile_ids))
.group_by(MailDeliveryCommand.profile_id, MailDeliveryCommand.status)
)
for profile_id, status, count, last_completed_at in rows:
item = result[str(profile_id)]
item[str(status)] = int(count)
if status in {"accepted", "reconciled_accepted", "partially_refused"}:
current = _aware(item.get("last_success_at"))
candidate = _aware(last_completed_at)
if candidate is not None and (current is None or candidate > current):
item["last_success_at"] = candidate
return result
def _imap_metrics(
session: Session,
*,
profile_ids: tuple[str, ...],
) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = defaultdict(dict)
folder_rows = session.execute(
select(
MailMailboxFolderIndex.profile_id,
func.count(MailMailboxFolderIndex.id),
func.max(MailMailboxFolderIndex.indexed_at),
)
.where(MailMailboxFolderIndex.profile_id.in_(profile_ids))
.group_by(MailMailboxFolderIndex.profile_id)
)
for profile_id, count, indexed_at in folder_rows:
result[str(profile_id)]["indexed_folders"] = int(count)
result[str(profile_id)]["last_indexed_at"] = _aware(indexed_at)
message_rows = session.execute(
select(
MailMailboxMessageIndex.profile_id,
func.count(MailMailboxMessageIndex.id),
)
.where(MailMailboxMessageIndex.profile_id.in_(profile_ids))
.group_by(MailMailboxMessageIndex.profile_id)
)
for profile_id, count in message_rows:
result[str(profile_id)]["indexed_messages"] = int(count)
bounce_rows = session.execute(
select(
MailBounceSource.profile_id,
func.count(MailBounceSource.id),
func.sum(
case((MailBounceSource.last_error.is_not(None), 1), else_=0)
),
func.max(MailBounceSource.last_success_at),
)
.where(
MailBounceSource.profile_id.in_(profile_ids),
MailBounceSource.is_active.is_(True),
)
.group_by(MailBounceSource.profile_id)
)
for profile_id, count, errors, last_success_at in bounce_rows:
item = result[str(profile_id)]
item["active_bounce_sources"] = int(count)
item["bounce_source_errors"] = int(errors or 0)
item["last_bounce_success_at"] = _aware(last_success_at)
return result
def _pop3_metrics(
session: Session,
*,
profile_ids: tuple[str, ...],
tenant_id: str | None,
) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = defaultdict(dict)
statement = select(
MailPop3Import.profile_id,
MailPop3Import.deletion_status,
func.count(MailPop3Import.id),
func.max(MailPop3Import.imported_at),
).where(MailPop3Import.profile_id.in_(profile_ids))
if tenant_id is not None:
statement = statement.where(MailPop3Import.tenant_id == tenant_id)
rows = session.execute(
statement.group_by(
MailPop3Import.profile_id,
MailPop3Import.deletion_status,
)
)
for profile_id, deletion_status, count, last_imported_at in rows:
item = result[str(profile_id)]
item[f"deletion_{deletion_status}"] = int(count)
current = _aware(item.get("last_imported_at"))
candidate = _aware(last_imported_at)
if candidate is not None and (current is None or candidate > current):
item["last_imported_at"] = candidate
return result
def _smtp_state(
profile: MailServerProfile,
*,
endpoints: list[MailServerEndpoint],
metrics: dict[str, Any],
observed_at: datetime,
) -> ExternalProviderRuntimeState:
active = bool(profile.is_active) and (
any(item.is_active for item in endpoints)
or (not endpoints and _legacy_configured(profile, "smtp"))
)
outcome_unknown = int(metrics.get("outcome_unknown", 0))
last_success = _aware(metrics.get("last_success_at"))
health = (
"inactive"
if not active
else "warning"
if outcome_unknown
else "healthy"
if last_success is not None
else "unknown"
)
return ExternalProviderRuntimeState(
provider_id=SMTP_PROVIDER_ID,
binding_ref=f"mail:profile:{profile.id}:smtp",
authority_mode="governance_overlay",
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness="not_applicable",
conflict="pending" if outcome_unknown else "clear",
recovery=(
"not_applicable"
if not active
else "attention"
if outcome_unknown or last_success is None
else "ready"
),
last_success_at=last_success,
detail=(
"SMTP delivery is disabled."
if not active
else "SMTP outcomes require reconciliation."
if outcome_unknown
else "SMTP delivery has no retained successful observation yet."
if last_success is None
else "SMTP delivery evidence is available."
),
metrics={
"pending_commands": int(metrics.get("pending", 0)),
"temporary_failures": int(metrics.get("temporary_failure", 0)),
"permanent_failures": int(metrics.get("permanent_failure", 0)),
"outcome_unknown_commands": outcome_unknown,
"active_endpoints": sum(1 for item in endpoints if item.is_active),
},
)
def _imap_state(
profile: MailServerProfile,
*,
endpoints: list[MailServerEndpoint],
metrics: dict[str, Any],
observed_at: datetime,
protocol: str = "imap",
provider_id: str = IMAP_PROVIDER_ID,
) -> ExternalProviderRuntimeState:
protocol_label = protocol.upper()
active = bool(profile.is_active) and (
any(item.is_active for item in endpoints)
or (protocol == "imap" and not endpoints and _legacy_configured(profile, "imap"))
)
indexed_at = _aware(metrics.get("last_indexed_at"))
errors = int(metrics.get("bounce_source_errors", 0))
freshness = (
"not_applicable"
if not active
else "unknown"
if indexed_at is None
else "current"
if observed_at - indexed_at <= _CURRENT_INDEX_WINDOW
else "stale"
)
health = (
"inactive"
if not active
else "error"
if errors
else "healthy"
if indexed_at is not None
else "unknown"
)
return ExternalProviderRuntimeState(
provider_id=provider_id,
binding_ref=f"mail:profile:{profile.id}:{protocol}",
authority_mode="external_mirror",
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness=freshness,
conflict="not_applicable",
recovery=(
"not_applicable"
if not active
else "ready"
if health == "healthy" and freshness == "current"
else "attention"
),
last_success_at=indexed_at or _aware(metrics.get("last_bounce_success_at")),
detail=(
f"{protocol_label} mailbox access is disabled."
if not active
else f"{protocol_label} mailbox or bounce-source errors require attention."
if errors
else f"{protocol_label} mailbox state has not been indexed yet."
if indexed_at is None
else f"{protocol_label} mailbox index state is available."
),
metrics={
"indexed_folders": int(metrics.get("indexed_folders", 0)),
"indexed_messages": int(metrics.get("indexed_messages", 0)),
"active_bounce_sources": int(metrics.get("active_bounce_sources", 0)),
"bounce_source_errors": errors,
"active_endpoints": sum(1 for item in endpoints if item.is_active),
},
)
def _pop3_state(
profile: MailServerProfile,
*,
endpoints: list[MailServerEndpoint],
metrics: dict[str, Any],
observed_at: datetime,
) -> ExternalProviderRuntimeState:
enabled_endpoints = [
item
for item in endpoints
if item.is_active and bool((item.config or {}).get("legacy_import_enabled"))
]
active = bool(profile.is_active) and bool(enabled_endpoints)
failed_deletions = int(metrics.get("deletion_failed", 0))
unknown_deletions = int(metrics.get("deletion_outcome_unknown", 0))
last_imported_at = _aware(metrics.get("last_imported_at"))
health = (
"inactive"
if not active
else "warning"
if failed_deletions or unknown_deletions
else "healthy"
if last_imported_at is not None
else "unknown"
)
return ExternalProviderRuntimeState(
provider_id=POP3_PROVIDER_ID,
binding_ref=f"mail:profile:{profile.id}:pop3",
authority_mode="governance_overlay",
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness="not_applicable",
conflict="pending" if unknown_deletions else "clear",
recovery=(
"not_applicable"
if not active
else "attention"
if failed_deletions or unknown_deletions
else "ready"
),
last_success_at=last_imported_at,
detail=(
"POP3 legacy import is disabled."
if not active
else "POP3 source deletion evidence requires attention."
if failed_deletions or unknown_deletions
else "POP3 legacy import is enabled but has no retained import yet."
if last_imported_at is None
else "POP3 governed import evidence is available."
),
metrics={
"active_endpoints": len(enabled_endpoints),
"imports": sum(
int(value)
for key, value in metrics.items()
if key.startswith("deletion_")
),
"failed_deletions": failed_deletions,
"outcome_unknown_deletions": unknown_deletions,
},
)
def _legacy_configured(profile: MailServerProfile, protocol: str) -> bool:
value = profile.smtp_config if protocol == "smtp" else profile.imap_config
return isinstance(value, dict) and bool(value)
def _aware(value: object | None) -> datetime | None:
if not isinstance(value, datetime):
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
__all__ = [
"IMAP_PROVIDER_ID",
"JMAP_PROVIDER_ID",
"POP3_PROVIDER_ID",
"SMTP_PROVIDER_ID",
"imap_provider_states",
"jmap_provider_states",
"pop3_provider_states",
"smtp_provider_states",
]
+679
View File
@@ -0,0 +1,679 @@
from __future__ import annotations
from dataclasses import dataclass
import hashlib
from threading import Lock
from uuid import uuid4
from sqlalchemy import select
from govoplan_core.core.recovery import (
RecoveryGuaranteeError,
RecoveryMode,
RecoveryOperation,
RecoveryPlan,
RecoveryStatus,
)
from govoplan_core.core.recovery_runtime import (
DurableRecoveryOperation,
RecoveryOperationBusy,
RecoveryOperationStateConflict,
begin_durable_recovery_operation,
claim_durable_recovery_operation,
)
from govoplan_core.core.runtime_coordination import process_runtime_identity
from govoplan_core.db.session import get_database
from govoplan_mail.backend.db.models import (
MailBounceSource,
MailMailboxFolderIndex,
MailMailboxMessageIndex,
)
from govoplan_mail.backend.sending.imap import (
ImapFolderListResult,
ImapMailboxMessageListResult,
)
class MailRecoveryError(RuntimeError):
pass
class MailboxRefreshBusy(MailRecoveryError):
pass
_local_refresh_lock = Lock()
_local_refreshes: set[str] = set()
def _claim_local_refresh(key: str) -> bool:
with _local_refresh_lock:
if key in _local_refreshes:
return False
_local_refreshes.add(key)
return True
def _release_local_refresh(key: str) -> None:
with _local_refresh_lock:
_local_refreshes.discard(key)
def _digest(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _bounded_idempotency(kind: str, effect_id: str) -> str:
return f"mail-{kind}:{_digest(effect_id)}"
@dataclass(slots=True)
class ProviderEffectRecovery:
operation: DurableRecoveryOperation | None
operation_id: str
replayed: bool
kind: str
def reject(self, *, code: str, summary: str) -> None:
if self.operation is None:
return
self.operation.reject(
summary=summary,
evidence={
"verified": True,
"checks": {"provider_rejection": code},
"effect_kind": self.kind,
},
)
def unknown(self, *, code: str, summary: str) -> None:
if self.operation is None:
return
self.operation.unresolved(
status=RecoveryStatus.OUTCOME_UNKNOWN,
summary=summary,
evidence={"effect_started": True, "failure_code": code},
failure_summary=(
"Inspect the provider and reconcile the outcome before any retry"
),
)
def succeed_smtp(
self,
*,
accepted_count: int,
refused_recipients: dict[str, dict[str, int | str]],
) -> None:
if self.operation is None:
return
refused = [
{
"recipient_sha256": _digest(address.casefold()),
"classification": str(item.get("classification") or "unknown"),
"status_code": int(item.get("status_code") or 0),
}
for address, item in sorted(refused_recipients.items())
]
evidence = {
"verified": True,
"checks": {
"provider_returned": True,
"accepted_count": accepted_count,
"refused_count": len(refused),
},
"accepted_count": accepted_count,
"refused": refused,
}
if accepted_count > 0:
self.operation.succeed(evidence=evidence)
else:
self.operation.reject(
summary="SMTP definitively accepted no recipients",
evidence=evidence,
)
def succeed_imap(self, *, folder: str) -> None:
if self.operation is None:
return
self.operation.succeed(
evidence={
"verified": True,
"checks": {"provider_append_returned": True},
"folder_sha256": _digest(folder),
}
)
def begin_provider_effect_recovery(
*,
kind: str,
effect_id: str | None,
tenant_id: str,
profile_id: str,
message_bytes: bytes,
expected_transport_revision: str | None,
recipient_count: int | None = None,
folder: str | None = None,
resource_type: str | None = None,
resource_id: str | None = None,
) -> ProviderEffectRecovery | None:
"""Fence a Mail-owned provider mutation before any network effect.
``effect_id`` is optional only for compatibility with callers predating the
recovery contract. Current Mail and Campaign paths always supply one.
"""
clean_effect_id = str(effect_id or "").strip()
if not clean_effect_id:
return None
if kind not in {"smtp-delivery", "imap-append"}:
raise ValueError("Unsupported Mail provider recovery kind")
message_sha256 = hashlib.sha256(message_bytes).hexdigest()
request = {
"tenant_id": tenant_id,
"profile_id": profile_id,
"message_sha256": message_sha256,
"message_size_bytes": len(message_bytes),
"transport_revision": expected_transport_revision,
"recipient_count": recipient_count,
"folder_sha256": _digest(folder) if folder else None,
"effect_id_sha256": _digest(clean_effect_id),
}
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="mail",
operation_type=kind,
idempotency_key=_bounded_idempotency(kind, clean_effect_id),
request=request,
recovery_plan=RecoveryPlan(
mode=RecoveryMode.FORWARD_RECOVERY,
preconditions=(
"Mail authorized the profile and effective transport policy",
"the caller supplied a stable effect identifier",
"the request records message and address digests, never content or credentials",
),
forward_recovery_steps=(
"inspect provider evidence without repeating the effect",
"record whether the provider accepted the effect",
"retry only under a new deliberate attempt identifier when absence is proven",
),
verification_steps=(
"compare the provider outcome with the message digest and effect identifier",
"verify the caller's durable attempt state independently",
),
),
precondition_evidence=request,
lease_resource_key=f"mail:{kind}:{tenant_id}:{_digest(clean_effect_id)[:40]}",
lease_ttl_seconds=15 * 60,
resource_type=resource_type,
resource_id=resource_id,
metadata={"resources": ["postgresql", kind.split("-", 1)[0]]},
)
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
raise MailRecoveryError(
"This Mail provider effect already has an active or unresolved recovery record"
) from exc
except (RecoveryGuaranteeError, RuntimeError) as exc:
raise MailRecoveryError(
"The Mail recovery ledger is unavailable; no provider effect was started"
) from exc
return ProviderEffectRecovery(
operation=started.operation,
operation_id=started.operation_id,
replayed=started.replayed,
kind=kind,
)
def reconcile_outbox_provider_effect(
*,
command_id: str,
effect_occurred: bool,
evidence_reference: str,
user_id: str,
) -> bool:
"""Resolve the newest unknown SMTP effect for a durable outbox command."""
factory = get_database().SessionLocal
with factory() as evidence_session:
operation = evidence_session.scalar(
select(RecoveryOperation)
.where(
RecoveryOperation.module_id == "mail",
RecoveryOperation.operation_type == "smtp-delivery",
RecoveryOperation.resource_type == "mail_delivery_command",
RecoveryOperation.resource_id == command_id,
)
.order_by(RecoveryOperation.created_at.desc(), RecoveryOperation.id.desc())
.limit(1)
)
if operation is None:
return False
if effect_occurred and operation.status == RecoveryStatus.SUCCEEDED.value:
return True
if not effect_occurred and operation.status == RecoveryStatus.RECOVERED.value:
return True
if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value:
raise MailRecoveryError(
f"The Mail recovery record is already {operation.status}"
)
operation_id = operation.id
try:
recovery = claim_durable_recovery_operation(
factory,
identity=process_runtime_identity(),
operation_id=operation_id,
lease_ttl_seconds=15 * 60,
)
recovery.resolve_unknown(
effect_occurred=effect_occurred,
summary="An operator reconciled the SMTP provider outcome",
evidence={
"verified": True,
"checks": {
"provider_evidence_sha256": _digest(evidence_reference),
"reconciled_by_user_id": user_id,
},
"effect_occurred": effect_occurred,
},
)
except (RecoveryGuaranteeError, RecoveryOperationBusy) as exc:
raise MailRecoveryError(
"The Mail provider recovery record could not be reconciled"
) from exc
return True
@dataclass(slots=True)
class MailboxRefreshRecovery:
operation: DurableRecoveryOperation
tenant_id: str
profile_id: str
folder: str
local_key: str
def complete_folders(self, result: ImapFolderListResult) -> None:
expected = {item.name for item in result.folders}
with get_database().SessionLocal() as session:
rows = session.scalars(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
)
).all()
present = {row.folder for row in rows}
verified = present == expected
evidence = {
"verified": verified,
"checks": {
"expected_folder_count": len(expected),
"indexed_folder_count": len(present),
"expected_folders_sha256": _digest("\n".join(sorted(expected))),
"indexed_folders_sha256": _digest("\n".join(sorted(present))),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The IMAP folder read committed an incomplete mailbox index",
evidence=evidence,
failure_summary="Repeat the read-only index refresh under a new fence",
)
finally:
_release_local_refresh(self.local_key)
def complete_messages(self, result: ImapMailboxMessageListResult) -> None:
expected = {item.uid for item in result.messages}
with get_database().SessionLocal() as session:
rows = session.scalars(
select(MailMailboxMessageIndex).where(
MailMailboxMessageIndex.tenant_id == self.tenant_id,
MailMailboxMessageIndex.profile_id == self.profile_id,
MailMailboxMessageIndex.folder == result.folder,
MailMailboxMessageIndex.uid.in_(expected or {""}),
)
).all()
folder = session.scalar(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
MailMailboxFolderIndex.folder == result.folder,
)
)
present = {row.uid for row in rows}
verified = (
present == expected
and folder is not None
and folder.uidvalidity == result.uidvalidity
and folder.message_count == result.total_count
)
evidence = {
"verified": verified,
"checks": {
"expected_message_count": len(expected),
"indexed_message_count": len(present),
"uids_sha256": _digest("\n".join(sorted(expected))),
"uidvalidity_matches": bool(
folder is not None and folder.uidvalidity == result.uidvalidity
),
"total_count_matches": bool(
folder is not None and folder.message_count == result.total_count
),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The IMAP message read committed an incomplete mailbox index",
evidence=evidence,
failure_summary="Repeat the read-only index refresh under a new fence",
)
finally:
_release_local_refresh(self.local_key)
def complete_bootstrap(
self,
folders: ImapFolderListResult,
messages: ImapMailboxMessageListResult,
) -> None:
expected_folders = {item.name for item in folders.folders}
expected_uids = {item.uid for item in messages.messages}
with get_database().SessionLocal() as session:
folder_rows = session.scalars(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
)
).all()
message_rows = session.scalars(
select(MailMailboxMessageIndex).where(
MailMailboxMessageIndex.tenant_id == self.tenant_id,
MailMailboxMessageIndex.profile_id == self.profile_id,
MailMailboxMessageIndex.folder == messages.folder,
MailMailboxMessageIndex.uid.in_(expected_uids or {""}),
)
).all()
selected_folder = session.scalar(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
MailMailboxFolderIndex.folder == messages.folder,
)
)
present_folders = {row.folder for row in folder_rows}
present_uids = {row.uid for row in message_rows}
verified = (
present_folders == expected_folders
and present_uids == expected_uids
and selected_folder is not None
and selected_folder.uidvalidity == messages.uidvalidity
and selected_folder.message_count == messages.total_count
)
evidence = {
"verified": verified,
"checks": {
"expected_folder_count": len(expected_folders),
"indexed_folder_count": len(present_folders),
"expected_message_count": len(expected_uids),
"indexed_message_count": len(present_uids),
"folder_set_matches": present_folders == expected_folders,
"message_window_matches": present_uids == expected_uids,
"uidvalidity_matches": bool(
selected_folder is not None
and selected_folder.uidvalidity == messages.uidvalidity
),
"total_count_matches": bool(
selected_folder is not None
and selected_folder.message_count == messages.total_count
),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The IMAP bootstrap committed an incomplete mailbox index",
evidence=evidence,
failure_summary="Repeat the read-only index refresh under a new fence",
)
finally:
_release_local_refresh(self.local_key)
def reject(self, *, summary: str, code: str) -> None:
try:
self.operation.reject(
summary=summary,
evidence={
"verified": True,
"checks": {
"provider_mutation": False,
"business_transaction_rolled_back": True,
"failure_code": code,
},
},
)
finally:
_release_local_refresh(self.local_key)
@dataclass(slots=True)
class BounceScanRecovery:
operation: DurableRecoveryOperation
source_id: str
local_key: str
def complete(self, *, highest_uid: int, uidvalidity: str | None) -> None:
with get_database().SessionLocal() as session:
source = session.get(MailBounceSource, self.source_id)
verified = bool(
source is not None
and source.highest_processed_uid == highest_uid
and source.uidvalidity == uidvalidity
and source.last_success_at is not None
)
evidence = {
"verified": verified,
"checks": {
"source_present": source is not None,
"cursor_matches": bool(
source is not None
and source.highest_processed_uid == highest_uid
),
"uidvalidity_matches": bool(
source is not None and source.uidvalidity == uidvalidity
),
"success_recorded": bool(
source is not None and source.last_success_at is not None
),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The read-only bounce scan did not persist its verified cursor",
evidence=evidence,
failure_summary="Repeat the source scan under a new distributed fence",
)
finally:
_release_local_refresh(self.local_key)
def reject(self, *, code: str) -> None:
try:
self.operation.reject(
summary="The read-only bounce scan failed before its cursor committed",
evidence={
"verified": True,
"checks": {
"provider_mutation": False,
"business_transaction_rolled_back": True,
"failure_code": code,
},
},
)
finally:
_release_local_refresh(self.local_key)
def begin_mailbox_refresh_recovery(
*,
tenant_id: str,
profile_id: str,
folder: str,
purpose: str,
) -> MailboxRefreshRecovery:
folder_sha256 = _digest(folder)
refresh_id = str(uuid4())
local_key = f"mailbox:{tenant_id}:{profile_id}:{folder_sha256}"
if not _claim_local_refresh(local_key):
raise MailboxRefreshBusy(
"Another request in this runtime is already refreshing this mailbox index"
)
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="mail",
operation_type="mailbox-index-refresh",
idempotency_key=f"mailbox-refresh:{refresh_id}",
request={
"tenant_id": tenant_id,
"profile_id": profile_id,
"folder_sha256": folder_sha256,
"purpose": purpose,
},
recovery_plan=RecoveryPlan(
mode=RecoveryMode.ATOMIC,
preconditions=(
"the actor is authorized to read the selected Mail profile",
"the IMAP operation is read-only",
),
verification_steps=(
"reload the bounded mailbox index through an independent session",
"compare provider UID and folder metadata with committed cache rows",
),
),
precondition_evidence={
"profile_id": profile_id,
"folder_sha256": folder_sha256,
"provider_mutation": False,
},
lease_resource_key=f"mail:mailbox-refresh:{tenant_id}:{profile_id}:{folder_sha256[:24]}",
lease_ttl_seconds=5 * 60,
resource_type="mail_profile",
resource_id=profile_id,
metadata={"resources": ["postgresql", "imap"], "purpose": purpose},
)
except RecoveryOperationBusy as exc:
_release_local_refresh(local_key)
raise MailboxRefreshBusy(
"Another runtime is already refreshing this mailbox index"
) from exc
except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc:
_release_local_refresh(local_key)
raise MailRecoveryError(
"The mailbox recovery fence is unavailable; IMAP was not read"
) from exc
if started.operation is None:
_release_local_refresh(local_key)
raise MailRecoveryError("A mailbox refresh cannot replay a completed read")
return MailboxRefreshRecovery(
operation=started.operation,
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
local_key=local_key,
)
def begin_bounce_scan_recovery(
*,
tenant_id: str,
profile_id: str,
source_id: str,
folder: str,
) -> BounceScanRecovery:
folder_sha256 = _digest(folder)
local_key = f"bounce:{tenant_id}:{source_id}"
if not _claim_local_refresh(local_key):
raise MailboxRefreshBusy(
"Another request in this runtime is already scanning this bounce source"
)
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="mail",
operation_type="bounce-source-scan",
idempotency_key=f"bounce-scan:{uuid4()}",
request={
"tenant_id": tenant_id,
"profile_id": profile_id,
"source_id": source_id,
"folder_sha256": folder_sha256,
},
recovery_plan=RecoveryPlan(
mode=RecoveryMode.ATOMIC,
preconditions=(
"the configured source transport revision still matches",
"the IMAP scan does not alter provider mailbox state",
),
verification_steps=(
"reload the bounce source cursor independently",
"verify UIDVALIDITY, highest UID, and success timestamp",
),
),
precondition_evidence={
"source_id": source_id,
"folder_sha256": folder_sha256,
"provider_mutation": False,
},
lease_resource_key=f"mail:bounce-scan:{tenant_id}:{source_id}",
lease_ttl_seconds=15 * 60,
resource_type="mail_bounce_source",
resource_id=source_id,
metadata={"resources": ["postgresql", "imap"]},
)
except RecoveryOperationBusy as exc:
_release_local_refresh(local_key)
raise MailboxRefreshBusy(
"Another runtime is already scanning this bounce source"
) from exc
except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc:
_release_local_refresh(local_key)
raise MailRecoveryError(
"The bounce-source recovery fence is unavailable; IMAP was not read"
) from exc
if started.operation is None:
_release_local_refresh(local_key)
raise MailRecoveryError("A bounce-source scan cannot replay a completed read")
return BounceScanRecovery(
operation=started.operation,
source_id=source_id,
local_key=local_key,
)
__all__ = [
"MailRecoveryError",
"BounceScanRecovery",
"MailboxRefreshBusy",
"MailboxRefreshRecovery",
"ProviderEffectRecovery",
"begin_mailbox_refresh_recovery",
"begin_bounce_scan_recovery",
"begin_provider_effect_recovery",
"reconcile_outbox_provider_effect",
]
File diff suppressed because it is too large Load Diff
+6 -25
View File
@@ -1,29 +1,10 @@
from __future__ import annotations
from typing import Any
from govoplan_core.core.runtime import ModuleRuntimeState
_runtime_settings: object | None = None
_runtime = ModuleRuntimeState("Mail")
def configure_runtime(*, settings: object | None = None) -> None:
global _runtime_settings
if settings is not None:
_runtime_settings = settings
def get_settings() -> object:
if _runtime_settings is not None:
return _runtime_settings
try:
from govoplan_core.settings import settings as legacy_settings
except ModuleNotFoundError as exc:
raise RuntimeError("GovOPlaN Mail runtime settings are not configured") from exc
return legacy_settings
class SettingsProxy:
def __getattr__(self, name: str) -> Any:
return getattr(get_settings(), name)
settings = SettingsProxy()
configure_runtime = _runtime.configure_runtime
get_registry = _runtime.get_registry
get_settings = _runtime.get_settings
settings = _runtime.settings
+467 -10
View File
@@ -3,9 +3,17 @@ from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
from govoplan_mail.backend.config import ImapConfig, SmtpConfig
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_mail.backend.config import (
ImapConfig,
ImapServerConfig,
SmtpConfig,
SmtpServerConfig,
TransportCredentials,
normalize_split_transport_credentials,
)
class MailSmtpTestRequest(SmtpConfig):
@@ -15,7 +23,29 @@ class MailSmtpTestRequest(SmtpConfig):
class MailImapTestRequest(ImapConfig):
"""IMAP settings supplied directly from the WebUI mail settings form."""
enabled: bool = True
class MailTransportCredentialsPayload(TransportCredentials):
"""Credentials supplied separately from saved server settings."""
class MailServerProfileCredentialsPayload(BaseModel):
model_config = ConfigDict(extra="forbid")
smtp: MailTransportCredentialsPayload = Field(default_factory=MailTransportCredentialsPayload)
imap: MailTransportCredentialsPayload = Field(default_factory=MailTransportCredentialsPayload)
def _normalize_profile_transport_payload(value: object) -> object:
return normalize_split_transport_credentials(value)
def _merge_transport_credentials(server: SmtpServerConfig | ImapServerConfig, credentials: MailTransportCredentialsPayload) -> dict[str, object]:
payload = server.model_dump(mode="json")
if "username" in credentials.model_fields_set:
payload["username"] = credentials.username
if "password" in credentials.model_fields_set:
payload["password"] = credentials.password
return payload
MailProfileScope = Literal["system", "tenant", "user", "group", "campaign"]
@@ -25,7 +55,6 @@ class MailCredentialPolicyPayload(BaseModel):
model_config = ConfigDict(extra="forbid")
inherit: bool | None = None
allow_override: bool | None = None
class MailProfilePolicyPayload(BaseModel):
@@ -39,6 +68,7 @@ class MailProfilePolicyPayload(BaseModel):
imap_credentials: MailCredentialPolicyPayload = Field(default_factory=MailCredentialPolicyPayload)
whitelist: dict[str, list[str]] = Field(default_factory=dict)
blacklist: dict[str, list[str]] = Field(default_factory=dict)
allow_lower_level_limits: dict[str, bool] = Field(default_factory=dict)
class MailProfilePolicyUpdateRequest(BaseModel):
@@ -50,8 +80,10 @@ class MailProfilePolicyUpdateRequest(BaseModel):
class PolicySourceStepResponse(BaseModel):
scope_type: str
scope_id: str | None = None
path: str
label: str
applied_fields: list[str] = Field(default_factory=list)
policy: dict[str, Any] = Field(default_factory=dict)
class MailProfilePolicyResponse(BaseModel):
@@ -71,10 +103,32 @@ class MailServerProfileCreateRequest(BaseModel):
slug: str | None = Field(default=None, max_length=100)
description: str | None = None
is_active: bool = True
inherit_to_lower_scopes: bool = True
scope_type: MailProfileScope = "tenant"
scope_id: str | None = None
smtp: SmtpConfig
imap: ImapConfig | None = None
smtp: SmtpServerConfig
imap: ImapServerConfig | None = None
credentials: MailServerProfileCredentialsPayload = Field(default_factory=MailServerProfileCredentialsPayload)
@model_validator(mode="before")
@classmethod
def normalize_legacy_transport_payload(cls, value: object) -> object:
return _normalize_profile_transport_payload(value)
def smtp_config(self) -> SmtpConfig:
return SmtpConfig.model_validate(_merge_transport_credentials(self.smtp, self.credentials.smtp))
def imap_config(self) -> ImapConfig | None:
if self.imap is None:
return None
return ImapConfig.model_validate(_merge_transport_credentials(self.imap, self.credentials.imap))
def credentials_supplied(self) -> bool:
return any(
field in credentials.model_fields_set
for credentials in (self.credentials.smtp, self.credentials.imap)
for field in ("username", "password")
)
class MailServerProfileUpdateRequest(BaseModel):
@@ -84,10 +138,157 @@ class MailServerProfileUpdateRequest(BaseModel):
slug: str | None = Field(default=None, max_length=100)
description: str | None = None
is_active: bool | None = None
smtp: SmtpConfig | None = None
imap: ImapConfig | None = None
inherit_to_lower_scopes: bool | None = None
smtp: SmtpServerConfig | None = None
imap: ImapServerConfig | None = None
credentials: MailServerProfileCredentialsPayload = Field(default_factory=MailServerProfileCredentialsPayload)
clear_imap: bool = False
@model_validator(mode="before")
@classmethod
def normalize_legacy_transport_payload(cls, value: object) -> object:
return _normalize_profile_transport_payload(value)
def smtp_config(self) -> SmtpConfig | None:
if self.smtp is None and "smtp" not in self.credentials.model_fields_set:
return None
server = self.smtp or SmtpServerConfig()
return SmtpConfig.model_validate(_merge_transport_credentials(server, self.credentials.smtp))
def imap_config(self) -> ImapConfig | None:
if self.imap is None and "imap" not in self.credentials.model_fields_set:
return None
server = self.imap or ImapServerConfig()
return ImapConfig.model_validate(_merge_transport_credentials(server, self.credentials.imap))
def credentials_supplied(self) -> bool:
return any(
field in credentials.model_fields_set
for credentials in (self.credentials.smtp, self.credentials.imap)
for field in ("username", "password")
)
class MailCredentialEnvelopeResponse(BaseModel):
id: str
binding_id: str | None = None
server_id: str | None = None
tenant_id: str | None = None
scope_type: MailProfileScope
scope_id: str | None = None
name: str
description: str | None = None
credential_kind: str
public_data: dict[str, Any] = Field(default_factory=dict)
secret_keys: list[str] = Field(default_factory=list)
secret_configured: bool = False
allowed_modules: list[str] = Field(default_factory=list)
allowed_server_refs: list[str] = Field(default_factory=list)
inherit_to_lower_scopes: bool = False
is_default: bool = False
is_active: bool = True
revision: str
created_at: datetime
updated_at: datetime
deleted_at: datetime | None = None
class MailServerEndpointResponse(BaseModel):
id: str
profile_id: str
tenant_id: str | None = None
protocol: Literal["smtp", "imap", "jmap", "pop3"]
name: str
config: dict[str, Any] = Field(default_factory=dict)
scope_type: MailProfileScope
scope_id: str | None = None
inherit_to_lower_scopes: bool = True
is_default: bool = False
is_active: bool = True
transport_revision: str
credentials: list[MailCredentialEnvelopeResponse] = Field(default_factory=list)
created_at: datetime
updated_at: datetime
class MailServerEndpointCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
protocol: Literal["smtp", "imap", "jmap", "pop3"]
name: str = Field(min_length=1, max_length=255)
config: dict[str, Any] = Field(default_factory=dict)
inherit_to_lower_scopes: bool | None = None
is_default: bool = False
is_active: bool = True
class MailServerEndpointUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str | None = Field(default=None, max_length=255)
config: dict[str, Any] | None = None
inherit_to_lower_scopes: bool | None = None
is_default: bool | None = None
is_active: bool | None = None
class MailCredentialCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=255)
description: str | None = None
credential_kind: str = "username_password"
username: str | None = None
password: str | None = None
public_data: dict[str, Any] = Field(default_factory=dict)
secret_data: dict[str, Any] = Field(default_factory=dict)
inherit_to_lower_scopes: bool | None = None
allowed_modules: list[str] = Field(default_factory=lambda: ["mail"])
allowed_server_refs: list[str] = Field(default_factory=list)
is_default: bool = False
class MailCampaignCredentialCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=255)
username: str = Field(min_length=1, max_length=320)
password: SecretStr
server_ids: list[str] = Field(min_length=1)
class MailCredentialBindRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
credential_id: str = Field(min_length=1)
is_default: bool = False
class MailCredentialUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str | None = Field(default=None, max_length=255)
description: str | None = None
username: str | None = None
password: str | None = None
public_data: dict[str, Any] | None = None
secret_data: dict[str, Any] | None = None
inherit_to_lower_scopes: bool | None = None
allowed_modules: list[str] | None = None
allowed_server_refs: list[str] | None = None
is_default: bool | None = None
is_active: bool | None = None
class MailCredentialUnlinkRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
retire_if_unused: bool = False
class MailCredentialListResponse(BaseModel):
credentials: list[MailCredentialEnvelopeResponse] = Field(default_factory=list)
class MailServerProfileResponse(BaseModel):
id: str
@@ -98,10 +299,13 @@ class MailServerProfileResponse(BaseModel):
slug: str
description: str | None = None
is_active: bool
inherit_to_lower_scopes: bool = True
smtp: dict[str, Any]
imap: dict[str, Any] | None = None
credentials: dict[str, Any] = Field(default_factory=dict)
smtp_password_configured: bool = False
imap_password_configured: bool = False
servers: list[MailServerEndpointResponse] = Field(default_factory=list)
created_at: datetime
updated_at: datetime
@@ -110,9 +314,76 @@ class MailServerProfileListResponse(BaseModel):
profiles: list[MailServerProfileResponse] = Field(default_factory=list)
class MailSettingsDeltaResponse(BaseModel):
profiles: list[MailServerProfileResponse] = Field(default_factory=list)
policy: MailProfilePolicyResponse | None = None
changed_sections: list[str] = Field(default_factory=list)
deleted: list[DeltaDeletedItem] = Field(default_factory=list)
watermark: str | None = None
has_more: bool = False
full: bool = False
class MailAddressLookupCandidate(BaseModel):
contact_id: str
address_book_id: str
display_name: str
email: str | None = None
email_label: str | None = None
organization: str | None = None
role_title: str | None = None
tags: list[str] = Field(default_factory=list)
source_kind: str = "local"
source_ref: str | None = None
source_revision: str | None = None
provenance: dict[str, Any] = Field(default_factory=dict)
class MailAddressLookupResponse(BaseModel):
available: bool = False
candidates: list[MailAddressLookupCandidate] = Field(default_factory=list)
class MailAddressWriteTarget(BaseModel):
address_book_id: str
address_book_label: str | None = None
operation: str = "create_contact"
allowed: bool = False
reason: str
message: str
scope_type: str | None = None
scope_id: str | None = None
source_kind: str | None = None
read_only: bool = False
required_scopes: list[str] = Field(default_factory=list)
provenance: dict[str, Any] = Field(default_factory=dict)
class MailAddressWriteTargetResponse(BaseModel):
available: bool = False
targets: list[MailAddressWriteTarget] = Field(default_factory=list)
class MailContactCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
address_book_id: str = Field(min_length=1, max_length=36)
display_name: str | None = Field(default=None, max_length=255)
email: str = Field(min_length=3, max_length=320, pattern=r"^[^\s@]+@[^\s@]+\.[^\s@]+$")
class MailContactCreateResponse(BaseModel):
contact_id: str
address_book_id: str
display_name: str
email: str | None = None
source_kind: str = "local"
provenance: dict[str, Any] = Field(default_factory=dict)
class MailConnectionTestResponse(BaseModel):
ok: bool
protocol: Literal["smtp", "imap"]
protocol: Literal["smtp", "imap", "jmap", "pop3"]
host: str | None = None
port: int | None = None
security: str | None = None
@@ -120,20 +391,102 @@ class MailConnectionTestResponse(BaseModel):
details: dict[str, Any] = Field(default_factory=dict)
class MailPop3PreviewRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
server_id: str = Field(min_length=1, max_length=36)
credential_id: str | None = Field(default=None, max_length=36)
limit: int = Field(default=50, ge=1, le=100)
class MailPop3MessagePreviewResponse(BaseModel):
message_number: int
uidl: str
subject: str | None = None
from_header: str | None = None
to_header: str | None = None
date: str | None = None
message_id: str | None = None
size_bytes: int = 0
body_preview: str | None = None
already_imported: bool = False
class MailPop3PreviewResponse(BaseModel):
profile_id: str
server_id: str
transport_revision: str
host: str
port: int
security: str
message_count: int
mailbox_size_bytes: int
delete_after_import_allowed: bool = False
messages: list[MailPop3MessagePreviewResponse] = Field(default_factory=list)
class MailPop3ImportRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
server_id: str = Field(min_length=1, max_length=36)
credential_id: str | None = Field(default=None, max_length=36)
expected_transport_revision: str = Field(min_length=1, max_length=120)
uidls: list[str] = Field(min_length=1, max_length=100)
delete_after_import: bool = False
class MailPop3ImportRecordResponse(BaseModel):
id: str
profile_id: str
pop3_server_id: str
transport_revision: str
provider_uidl: str
message_id: str | None = None
subject: str | None = None
from_header: str | None = None
to_header: str | None = None
date: str | None = None
body_preview: str | None = None
size_bytes: int
raw_sha256: str
status: str
imported_at: datetime
deletion_requested: bool = False
deletion_status: str
deletion_attempted_at: datetime | None = None
deletion_error: str | None = None
class MailPop3ImportResponse(BaseModel):
imports: list[MailPop3ImportRecordResponse] = Field(default_factory=list)
duplicate_uidls: list[str] = Field(default_factory=list)
deletion_status: str = "not_requested"
class MailPop3ImportListResponse(BaseModel):
imports: list[MailPop3ImportRecordResponse] = Field(default_factory=list)
class MailImapFolderResponse(BaseModel):
name: str
flags: list[str] = Field(default_factory=list)
message_count: int | None = None
unseen_count: int | None = None
class MailImapFolderListResponse(BaseModel):
ok: bool
protocol: Literal["imap"] = "imap"
protocol: Literal["imap", "jmap"] = "imap"
host: str | None = None
port: int | None = None
security: str | None = None
message: str
folders: list[MailImapFolderResponse] = Field(default_factory=list)
detected_sent_folder: str | None = None
detected_folder_mappings: dict[str, str] = Field(default_factory=dict)
from_cache: bool = False
refreshing: bool = False
indexed_at: datetime | None = None
details: dict[str, Any] = Field(default_factory=dict)
class MailMailboxAttachmentResponse(BaseModel):
@@ -171,9 +524,37 @@ class MailMailboxMessageListResponse(BaseModel):
port: int | None = None
security: str | None = None
total_count: int = 0
offset: int = 0
limit: int = 50
cursor: str | None = None
next_cursor: str | None = None
cursor_stable: bool = False
full: bool = False
from_cache: bool = False
refreshing: bool = False
indexed_at: datetime | None = None
messages: list[MailMailboxMessageSummaryResponse] = Field(default_factory=list)
class MailMailboxChangesResponse(BaseModel):
profile_id: str
protocol: Literal["jmap"] = "jmap"
account_id: str
old_state: str
new_state: str
has_more_changes: bool = False
created: list[str] = Field(default_factory=list)
updated: list[str] = Field(default_factory=list)
destroyed: list[str] = Field(default_factory=list)
class MailMailboxBootstrapResponse(BaseModel):
profile_id: str
folder: str
folders: MailImapFolderListResponse
messages: MailMailboxMessageListResponse
class MailMailboxMessageResponse(BaseModel):
profile_id: str
folder: str
@@ -182,3 +563,79 @@ class MailMailboxMessageResponse(BaseModel):
security: str | None = None
message: MailMailboxMessageDetailResponse
class MailDeliveryCommandResponse(BaseModel):
result: dict[str, Any]
class MailDeliveryReconcileRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
decision: Literal["accepted", "not_accepted"]
evidence_reference: str = Field(min_length=1, max_length=500)
note: str | None = Field(default=None, max_length=4000)
class MailDeliveryResendRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
idempotency_key: str = Field(min_length=1, max_length=200)
class MailBounceSourceRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
profile_id: str = Field(min_length=1, max_length=36)
folder: str = Field(default="INBOX", min_length=1, max_length=255)
imap_server_id: str | None = Field(default=None, max_length=36)
imap_credential_id: str | None = Field(default=None, max_length=36)
is_active: bool = True
class MailBounceSourceResponse(BaseModel):
id: str
tenant_id: str
profile_id: str
folder: str
imap_server_id: str | None = None
imap_credential_id: str | None = None
expected_imap_transport_revision: str
is_active: bool
uidvalidity: str | None = None
highest_processed_uid: int
last_scanned_at: datetime | None = None
last_success_at: datetime | None = None
last_error: str | None = None
class MailBounceSourceListResponse(BaseModel):
sources: list[MailBounceSourceResponse]
class MailBounceObservationResponse(BaseModel):
id: str
tenant_id: str
profile_id: str
folder: str
uid: str
original_message_id: str | None = None
command_id: str | None = None
recipient: str | None = None
action: str
status_code: str | None = None
diagnostic: str | None = None
permanent: bool
observed_at: datetime
matched: bool
evidence: dict[str, Any] = Field(default_factory=dict)
class MailBounceObservationListResponse(BaseModel):
observations: list[MailBounceObservationResponse]
class MailBounceScanResponse(BaseModel):
sources: int
processed_messages: int
observations: int
failures: list[dict[str, str]] = Field(default_factory=list)
+292
View File
@@ -0,0 +1,292 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from urllib.parse import quote
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.events import PlatformEvent
from govoplan_core.core.modules import ModuleContext
from govoplan_core.core.search import (
SearchAuthorizationRequest,
SearchBackfillPage,
SearchBackfillRequest,
SearchDocument,
SearchIndexChange,
SearchResourceReference,
SearchResourceType,
)
from govoplan_mail.backend.db.models import (
MailMailboxMessageIndex,
MailServerProfile,
)
from govoplan_mail.backend.mail_profiles import mail_profile_visible_to_actor
PROVIDER_ID = "mail.mailbox_messages"
RESOURCE_TYPE = "mailbox_message"
READ_SCOPE = "mail:mailbox:read"
USE_SCOPE = "mail:profile:use"
class MailSearchSource:
def resource_types(self) -> Sequence[SearchResourceType]:
return (
SearchResourceType(
provider_id=PROVIDER_ID,
module_id="mail",
resource_type=RESOURCE_TYPE,
label="Mailbox messages",
requires_authorization_recheck=True,
),
)
def backfill(
self,
session: object,
*,
request: SearchBackfillRequest,
) -> SearchBackfillPage:
_assert_source(request.provider_id, request.resource_type)
db = _session(session)
statement = (
select(MailMailboxMessageIndex, MailServerProfile)
.join(
MailServerProfile,
MailServerProfile.id == MailMailboxMessageIndex.profile_id,
)
.where(MailMailboxMessageIndex.tenant_id == request.tenant_id)
)
if request.cursor:
statement = statement.where(
MailMailboxMessageIndex.id > request.cursor
)
rows = list(
db.execute(
statement.order_by(MailMailboxMessageIndex.id).limit(
request.limit + 1
)
).all()
)
has_more = len(rows) > request.limit
selected = rows[: request.limit]
high_watermark = db.scalar(
select(func.max(MailMailboxMessageIndex.indexed_at)).where(
MailMailboxMessageIndex.tenant_id == request.tenant_id
)
)
return SearchBackfillPage(
documents=tuple(
_document(message, profile=profile)
for message, profile in selected
),
next_cursor=(
selected[-1][0].id if has_more and selected else None
),
complete=not has_more,
high_watermark=(
high_watermark.isoformat()
if high_watermark is not None
else None
),
)
def authorize(
self,
session: object,
principal: object,
*,
requests: Sequence[SearchAuthorizationRequest],
) -> Mapping[str, bool]:
decisions = {item.reference.key: False for item in requests}
if not isinstance(principal, ApiPrincipal) or not (
principal.has(READ_SCOPE) and principal.has(USE_SCOPE)
):
return decisions
valid = tuple(
item
for item in requests
if item.reference.tenant_id == principal.tenant_id
and item.reference.module_id == "mail"
and item.reference.resource_type == RESOURCE_TYPE
)
if not valid:
return decisions
db = _session(session)
ids = {item.reference.resource_id for item in valid}
messages = {
message.id: (message, profile)
for message, profile in db.execute(
select(MailMailboxMessageIndex, MailServerProfile)
.join(
MailServerProfile,
MailServerProfile.id
== MailMailboxMessageIndex.profile_id,
)
.where(
MailMailboxMessageIndex.id.in_(ids),
MailMailboxMessageIndex.tenant_id == principal.tenant_id,
)
).all()
}
user_id = str(
getattr(principal.user, "id", "") or principal.membership_id or ""
)
for item in valid:
match = messages.get(item.reference.resource_id)
if match is None:
continue
_message, profile = match
try:
allowed = mail_profile_visible_to_actor(
db,
profile=profile,
tenant_id=principal.tenant_id,
user_id=user_id,
group_ids=principal.group_ids,
tenant_admin=principal.has("tenant:*"),
require_active=True,
)
except (RuntimeError, ValueError):
allowed = False
decisions[item.reference.key] = allowed
return decisions
def index_changes_for_event(
self,
session: object,
*,
event: PlatformEvent,
delivery_key: str,
) -> Sequence[SearchIndexChange]:
if (
event.module_id != "mail"
or event.tenant is None
or event.resource is None
or event.resource.type != RESOURCE_TYPE
or event.resource.id is None
):
return ()
db = _session(session)
row = db.get(MailMailboxMessageIndex, event.resource.id)
profile = (
db.get(MailServerProfile, row.profile_id)
if row is not None and row.tenant_id == event.tenant.id
else None
)
deleted = (
row is None
or row.tenant_id != event.tenant.id
or profile is None
)
cursor = event.event_id
document = (
None
if deleted
else _document(row, profile=profile, change_cursor=cursor)
)
reference = SearchResourceReference(
tenant_id=event.tenant.id,
module_id="mail",
resource_type=RESOURCE_TYPE,
resource_id=event.resource.id,
)
return (
SearchIndexChange(
change_id=f"{delivery_key}:{PROVIDER_ID}",
provider_id=PROVIDER_ID,
kind="delete" if deleted else "upsert",
reference=reference,
source_revision=(
document.source_revision if document is not None else cursor
),
cursor=cursor,
document=document,
occurred_at=event.occurred_at,
),
)
def create_mail_search_source(_context: ModuleContext) -> MailSearchSource:
return MailSearchSource()
def _document(
message: MailMailboxMessageIndex,
*,
profile: MailServerProfile,
change_cursor: str | None = None,
) -> SearchDocument:
tokens = [f"scope:{READ_SCOPE}", f"scope:{USE_SCOPE}"]
scope_type = str(profile.scope_type or "tenant")
if scope_type == "user" and profile.scope_id:
tokens.append(f"membership:{profile.scope_id}")
elif scope_type == "group" and profile.scope_id:
tokens.append(f"group:{profile.scope_id}")
title = (message.subject or "(No subject)")[:500]
summary = " | ".join(
value for value in (message.from_header, message.date) if value
)[:4000]
body = " ".join(
value
for value in (
message.from_header,
message.to_header,
message.cc_header,
message.body_preview,
)
if value
)[:200_000]
updated_at = message.updated_at or message.indexed_at
return SearchDocument(
tenant_id=message.tenant_id,
module_id="mail",
provider_id=PROVIDER_ID,
resource_type=RESOURCE_TYPE,
resource_id=message.id,
title=title,
url=(
"/mail?"
f"profileId={quote(message.profile_id, safe='')}"
f"&folder={quote(message.folder, safe='')}"
f"&uid={quote(message.uid, safe='')}"
),
summary=summary or None,
body=body or None,
keywords=(message.folder[:200], profile.name[:200]),
visibility="restricted",
acl_tokens=tuple(dict.fromkeys(tokens)),
metadata={
"profile_id": message.profile_id,
"profile_name": profile.name,
"folder": message.folder,
"uid": message.uid,
"date": message.date,
"attachment_count": message.attachment_count,
},
source_revision=updated_at.isoformat(),
change_cursor=change_cursor,
source_updated_at=updated_at,
requires_authorization_recheck=True,
)
def _assert_source(provider_id: str, resource_type: str) -> None:
if provider_id != PROVIDER_ID or resource_type != RESOURCE_TYPE:
raise ValueError("Unsupported Mail search source.")
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("Mail search requires a SQLAlchemy session.")
return value
__all__ = [
"MailSearchSource",
"PROVIDER_ID",
"RESOURCE_TYPE",
"create_mail_search_source",
]
File diff suppressed because it is too large Load Diff
+847
View File
@@ -0,0 +1,847 @@
from __future__ import annotations
import base64
import json
import urllib.error
import urllib.parse
from dataclasses import dataclass
from typing import Any, Iterable, Mapping
from govoplan_core.security.http_fetch import fetch_http
from govoplan_mail.backend.config import JmapConfig
from govoplan_mail.backend.sending.imap import (
ImapMailboxAttachmentInfo,
ImapMailboxInfo,
ImapMailboxMessageDetail,
ImapMailboxMessageSummary,
)
JMAP_CORE_CAPABILITY = "urn:ietf:params:jmap:core"
JMAP_MAIL_CAPABILITY = "urn:ietf:params:jmap:mail"
_SUMMARY_PROPERTIES = [
"id",
"threadId",
"mailboxIds",
"keywords",
"size",
"receivedAt",
"sentAt",
"messageId",
"from",
"to",
"cc",
"subject",
"hasAttachment",
"preview",
]
_DETAIL_PROPERTIES = _SUMMARY_PROPERTIES + [
"replyTo",
"bcc",
"textBody",
"htmlBody",
"bodyValues",
"attachments",
]
class JmapConfigurationError(ValueError):
pass
class JmapAuthenticationError(RuntimeError):
pass
class JmapPermissionError(RuntimeError):
pass
class JmapCapabilityError(RuntimeError):
pass
class JmapProviderError(RuntimeError):
pass
@dataclass(frozen=True, slots=True)
class JmapSession:
api_url: str
account_id: str
session_state: str
capabilities: tuple[str, ...]
account_capabilities: tuple[str, ...]
username: str | None
@dataclass(frozen=True, slots=True)
class JmapConnectionTestResult:
host: str
port: int
security: str
authenticated: bool
account_id: str
session_state: str
capabilities: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class JmapFolderListResult:
host: str
port: int
security: str
folders: list[ImapMailboxInfo]
detected_sent_folder: str | None
detected_folder_mappings: dict[str, str]
protocol: str = "jmap"
@dataclass(frozen=True, slots=True)
class JmapMailboxMessageListResult:
host: str
port: int
security: str
folder: str
messages: list[ImapMailboxMessageSummary]
total_count: int
offset: int
limit: int
uidvalidity: str
cursor_reset: bool = False
@dataclass(frozen=True, slots=True)
class JmapMailboxBootstrapResult:
folders: JmapFolderListResult
messages: JmapMailboxMessageListResult
@dataclass(frozen=True, slots=True)
class JmapMailboxMessageResult:
host: str
port: int
security: str
folder: str
message: ImapMailboxMessageDetail
@dataclass(frozen=True, slots=True)
class JmapEmailChangesResult:
account_id: str
old_state: str
new_state: str
has_more_changes: bool
created: tuple[str, ...]
updated: tuple[str, ...]
destroyed: tuple[str, ...]
def discover_jmap(config: JmapConfig) -> JmapSession:
payload = _fetch_json(config.session_url, config=config, method="GET")
capabilities = _string_keys(payload.get("capabilities"), "JMAP capabilities")
if JMAP_CORE_CAPABILITY not in capabilities:
raise JmapCapabilityError("The server does not advertise the JMAP Core capability")
if JMAP_MAIL_CAPABILITY not in capabilities:
raise JmapCapabilityError("The server does not advertise the JMAP Mail capability")
accounts = _object(payload.get("accounts"), "JMAP accounts")
account_id = _select_account_id(payload, accounts, config.account_id)
account = _object(accounts.get(account_id), "JMAP account")
account_capabilities = _string_keys(
account.get("accountCapabilities"),
"JMAP account capabilities",
)
if JMAP_MAIL_CAPABILITY not in account_capabilities:
raise JmapCapabilityError("The selected account does not support JMAP Mail")
api_url = _resolve_session_url(
config,
_required_text(payload.get("apiUrl"), "JMAP Session is missing apiUrl"),
label="JMAP apiUrl",
)
return JmapSession(
api_url=api_url,
account_id=account_id,
session_state=_required_text(
payload.get("state"),
"JMAP Session is missing state",
),
capabilities=tuple(sorted(capabilities)),
account_capabilities=tuple(sorted(account_capabilities)),
username=_optional_text(payload.get("username")),
)
def test_jmap_connection(*, jmap_config: JmapConfig) -> JmapConnectionTestResult:
session = discover_jmap(jmap_config)
_jmap_call(
jmap_config,
session,
[("Mailbox/get", {"accountId": session.account_id, "ids": []}, "mailboxes")],
)
host, port, security = _transport_coordinates(jmap_config.session_url)
return JmapConnectionTestResult(
host=host,
port=port,
security=security,
authenticated=True,
account_id=session.account_id,
session_state=session.session_state,
capabilities=session.capabilities,
)
def list_jmap_folders(*, jmap_config: JmapConfig) -> JmapFolderListResult:
session = discover_jmap(jmap_config)
mailboxes = _get_mailboxes(jmap_config, session)
paths = _mailbox_paths(mailboxes)
folders: list[ImapMailboxInfo] = []
mappings: dict[str, str] = {}
for mailbox in sorted(mailboxes, key=lambda item: paths[str(item["id"])].casefold()):
mailbox_id = str(mailbox["id"])
path = paths[mailbox_id]
role = _optional_text(mailbox.get("role"))
flags = [_jmap_role_flag(role)] if role else []
folders.append(
ImapMailboxInfo(
name=path,
flags=[flag for flag in flags if flag],
message_count=_optional_nonnegative_int(mailbox.get("totalEmails")),
unseen_count=_optional_nonnegative_int(mailbox.get("unreadEmails")),
)
)
if role in {"inbox", "sent", "drafts", "trash", "archive", "junk"}:
mappings[role] = path
host, port, security = _transport_coordinates(jmap_config.session_url)
return JmapFolderListResult(
host=host,
port=port,
security=security,
folders=folders,
detected_sent_folder=mappings.get("sent"),
detected_folder_mappings=mappings,
)
def list_jmap_messages(
*,
jmap_config: JmapConfig,
folder: str,
limit: int = 50,
offset: int = 0,
expected_query_state: str | None = None,
query: str | None = None,
) -> JmapMailboxMessageListResult:
clean_limit = max(1, min(int(limit), 100))
clean_offset = max(0, min(int(offset), 100_000))
clean_query = str(query or "").strip()
if len(clean_query) > 500:
raise JmapConfigurationError("JMAP mailbox search is limited to 500 characters")
session = discover_jmap(jmap_config)
mailboxes = _get_mailboxes(jmap_config, session)
mailbox, paths = _resolve_mailbox(mailboxes, folder)
query_payload: dict[str, Any] = {
"accountId": session.account_id,
"filter": {"inMailbox": mailbox["id"]},
"sort": [{"property": "receivedAt", "isAscending": False}],
"position": clean_offset,
"limit": clean_limit,
"calculateTotal": True,
}
if clean_query:
query_payload["filter"] = {
"operator": "AND",
"conditions": [
{"inMailbox": mailbox["id"]},
{"text": clean_query},
],
}
query_result = _method_result(
_jmap_call(
jmap_config,
session,
[("Email/query", query_payload, "query")],
),
name="Email/query",
call_id="query",
)
query_state = _required_text(
query_result.get("queryState"),
"JMAP Email/query response is missing queryState",
)
cursor_reset = bool(expected_query_state and expected_query_state != query_state)
if cursor_reset and clean_offset:
clean_offset = 0
query_payload["position"] = 0
query_result = _method_result(
_jmap_call(
jmap_config,
session,
[("Email/query", query_payload, "query-reset")],
),
name="Email/query",
call_id="query-reset",
)
query_state = _required_text(
query_result.get("queryState"),
"JMAP Email/query response is missing queryState",
)
ids = _string_list(query_result.get("ids"), "JMAP Email/query ids", maximum=100)
emails: list[dict[str, Any]] = []
if ids:
get_result = _method_result(
_jmap_call(
jmap_config,
session,
[(
"Email/get",
{
"accountId": session.account_id,
"ids": ids,
"properties": _SUMMARY_PROPERTIES,
},
"emails",
)],
),
name="Email/get",
call_id="emails",
)
emails = _object_list(get_result.get("list"), "JMAP Email/get list", maximum=100)
by_id = {str(item.get("id")): item for item in emails if item.get("id") is not None}
folder_path = paths[str(mailbox["id"])]
messages = [
_email_summary(by_id[email_id], folder=folder_path)
for email_id in ids
if email_id in by_id
]
host, port, security = _transport_coordinates(jmap_config.session_url)
total = query_result.get("total")
total_count = int(total) if isinstance(total, int) and total >= 0 else clean_offset + len(messages)
return JmapMailboxMessageListResult(
host=host,
port=port,
security=security,
folder=folder_path,
messages=messages,
total_count=total_count,
offset=clean_offset,
limit=clean_limit,
uidvalidity=query_state,
cursor_reset=cursor_reset,
)
def load_jmap_mailbox_bootstrap(
*,
jmap_config: JmapConfig,
folder: str = "INBOX",
limit: int = 50,
offset: int = 0,
) -> JmapMailboxBootstrapResult:
folders = list_jmap_folders(jmap_config=jmap_config)
selected = folder
names = {item.name for item in folders.folders}
if selected not in names:
selected = (
folders.detected_folder_mappings.get("inbox")
or (folders.folders[0].name if folders.folders else folder)
)
messages = list_jmap_messages(
jmap_config=jmap_config,
folder=selected,
limit=limit,
offset=offset,
)
return JmapMailboxBootstrapResult(folders=folders, messages=messages)
def get_jmap_message(
*,
jmap_config: JmapConfig,
folder: str,
email_id: str,
) -> JmapMailboxMessageResult:
clean_id = _required_text(email_id, "JMAP Email id is required")
if len(clean_id) > 255:
raise JmapConfigurationError("JMAP Email id is too long")
session = discover_jmap(jmap_config)
mailbox, paths = _resolve_mailbox(_get_mailboxes(jmap_config, session), folder)
mailbox_id = str(mailbox["id"])
canonical_folder = paths[mailbox_id]
result = _method_result(
_jmap_call(
jmap_config,
session,
[(
"Email/get",
{
"accountId": session.account_id,
"ids": [clean_id],
"properties": _DETAIL_PROPERTIES,
"bodyProperties": [
"partId",
"blobId",
"size",
"name",
"type",
"charset",
"disposition",
"cid",
],
"fetchTextBodyValues": True,
"fetchHTMLBodyValues": True,
"maxBodyValueBytes": jmap_config.max_body_value_bytes,
},
"email",
)],
),
name="Email/get",
call_id="email",
)
values = _object_list(result.get("list"), "JMAP Email/get list", maximum=1)
if not values:
raise JmapProviderError("JMAP message not found")
email = values[0]
mailbox_ids = email.get("mailboxIds")
if not isinstance(mailbox_ids, dict) or mailbox_ids.get(mailbox_id) is not True:
raise JmapProviderError("JMAP message is not available in the requested mailbox")
summary = _email_summary(email, folder=canonical_folder)
body_values = _object(email.get("bodyValues") or {}, "JMAP Email bodyValues")
body_text = _body_value(email.get("textBody"), body_values)
body_html = _body_value(email.get("htmlBody"), body_values)
attachments = [
ImapMailboxAttachmentInfo(
filename=_optional_text(item.get("name")),
content_type=_optional_text(item.get("type")) or "application/octet-stream",
size_bytes=_optional_nonnegative_int(item.get("size")) or 0,
)
for item in _object_list(
email.get("attachments") or [],
"JMAP Email attachments",
maximum=1_000,
)
]
headers = {
key: value
for key, value in {
"From": summary.from_header,
"To": summary.to_header,
"Cc": summary.cc_header,
"Bcc": _format_addresses(email.get("bcc")),
"Reply-To": _format_addresses(email.get("replyTo")),
"Message-ID": summary.message_id,
"Date": summary.date,
"Subject": summary.subject,
}.items()
if value
}
detail = ImapMailboxMessageDetail(
uid=summary.uid,
folder=summary.folder,
subject=summary.subject,
from_header=summary.from_header,
to_header=summary.to_header,
cc_header=summary.cc_header,
date=summary.date,
message_id=summary.message_id,
flags=summary.flags,
size_bytes=summary.size_bytes,
body_preview=summary.body_preview,
body_text=body_text,
body_html=body_html,
headers=headers,
attachments=attachments,
)
host, port, security = _transport_coordinates(jmap_config.session_url)
return JmapMailboxMessageResult(
host=host,
port=port,
security=security,
folder=canonical_folder,
message=detail,
)
def get_jmap_email_changes(
*,
jmap_config: JmapConfig,
since_state: str,
max_changes: int = 500,
) -> JmapEmailChangesResult:
clean_state = _required_text(since_state, "JMAP Email change state is required")
if len(clean_state) > 1_000:
raise JmapConfigurationError("JMAP Email change state is too long")
clean_max = max(1, min(int(max_changes), 1_000))
session = discover_jmap(jmap_config)
result = _method_result(
_jmap_call(
jmap_config,
session,
[(
"Email/changes",
{
"accountId": session.account_id,
"sinceState": clean_state,
"maxChanges": clean_max,
},
"changes",
)],
),
name="Email/changes",
call_id="changes",
)
return JmapEmailChangesResult(
account_id=session.account_id,
old_state=_required_text(result.get("oldState"), "JMAP changes is missing oldState"),
new_state=_required_text(result.get("newState"), "JMAP changes is missing newState"),
has_more_changes=bool(result.get("hasMoreChanges")),
created=tuple(_string_list(result.get("created"), "JMAP created ids", maximum=clean_max)),
updated=tuple(_string_list(result.get("updated"), "JMAP updated ids", maximum=clean_max)),
destroyed=tuple(_string_list(result.get("destroyed"), "JMAP destroyed ids", maximum=clean_max)),
)
def _get_mailboxes(config: JmapConfig, session: JmapSession) -> list[dict[str, Any]]:
result = _method_result(
_jmap_call(
config,
session,
[(
"Mailbox/get",
{
"accountId": session.account_id,
"properties": [
"id",
"name",
"parentId",
"role",
"sortOrder",
"isSubscribed",
"totalEmails",
"unreadEmails",
],
},
"mailboxes",
)],
),
name="Mailbox/get",
call_id="mailboxes",
)
rows = _object_list(result.get("list"), "JMAP Mailbox/get list", maximum=10_000)
for row in rows:
_required_text(row.get("id"), "JMAP Mailbox is missing id")
_required_text(row.get("name"), "JMAP Mailbox is missing name")
return rows
def _jmap_call(
config: JmapConfig,
session: JmapSession,
calls: Iterable[tuple[str, Mapping[str, Any], str]],
) -> dict[str, Any]:
method_calls = [[name, dict(arguments), call_id] for name, arguments, call_id in calls]
if not method_calls or len(method_calls) > 32:
raise JmapConfigurationError("A JMAP request must contain between 1 and 32 method calls")
body = json.dumps(
{
"using": [JMAP_CORE_CAPABILITY, JMAP_MAIL_CAPABILITY],
"methodCalls": method_calls,
},
separators=(",", ":"),
).encode("utf-8")
return _fetch_json(session.api_url, config=config, method="POST", body=body)
def _fetch_json(
url: str,
*,
config: JmapConfig,
method: str,
body: bytes | None = None,
) -> dict[str, Any]:
try:
response = fetch_http(
url,
timeout=config.timeout_seconds,
label="JMAP endpoint",
method=method,
headers={
"Accept": "application/json",
"Authorization": _authorization_header(config),
**({"Content-Type": "application/json"} if body is not None else {}),
},
body=body,
max_bytes=config.max_response_bytes,
)
except urllib.error.HTTPError as exc:
if exc.code == 401:
raise JmapAuthenticationError("JMAP authentication failed") from exc
if exc.code == 403:
raise JmapPermissionError("JMAP access is forbidden for this credential") from exc
raise JmapProviderError(f"JMAP provider returned HTTP {exc.code}") from exc
except (JmapAuthenticationError, JmapPermissionError, JmapProviderError):
raise
except Exception as exc:
raise JmapProviderError("JMAP provider is unavailable") from exc
try:
payload = json.loads(response.body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise JmapProviderError("JMAP provider returned invalid JSON") from exc
return _object(payload, "JMAP response")
def _method_result(payload: Mapping[str, Any], *, name: str, call_id: str) -> dict[str, Any]:
responses = payload.get("methodResponses")
if not isinstance(responses, list):
raise JmapProviderError("JMAP response is missing methodResponses")
for item in responses:
if not isinstance(item, list) or len(item) != 3:
continue
response_name, arguments, response_id = item
if response_id != call_id:
continue
if response_name == "error":
error = _object(arguments, "JMAP method error")
error_type = _optional_text(error.get("type")) or "unknown"
if error_type in {"accountNotFound", "forbidden"}:
raise JmapPermissionError(f"JMAP {name} was denied ({error_type})")
if error_type in {"unknownMethod", "unknownCapability"}:
raise JmapCapabilityError(f"JMAP {name} is unsupported ({error_type})")
if error_type == "cannotCalculateChanges":
raise JmapCapabilityError("JMAP incremental state expired; perform a full refresh")
raise JmapProviderError(f"JMAP {name} failed ({error_type})")
if response_name != name:
raise JmapProviderError(f"JMAP returned {response_name!r} for {name}")
return _object(arguments, f"JMAP {name} response")
raise JmapProviderError(f"JMAP response did not include call {call_id!r}")
def _select_account_id(
session_payload: Mapping[str, Any],
accounts: Mapping[str, Any],
configured: str | None,
) -> str:
if configured:
if configured not in accounts:
raise JmapPermissionError("The configured JMAP account is not available")
return configured
primary = session_payload.get("primaryAccounts")
if isinstance(primary, dict) and primary.get(JMAP_MAIL_CAPABILITY):
account_id = str(primary[JMAP_MAIL_CAPABILITY])
if account_id in accounts:
return account_id
capable = [
str(account_id)
for account_id, value in accounts.items()
if isinstance(value, dict)
and JMAP_MAIL_CAPABILITY
in _string_keys(value.get("accountCapabilities"), "JMAP account capabilities")
]
if len(capable) == 1:
return capable[0]
if not capable:
raise JmapCapabilityError("No accessible account supports JMAP Mail")
raise JmapConfigurationError("Configure a JMAP account id because multiple mail accounts are available")
def _resolve_session_url(config: JmapConfig, value: str, *, label: str) -> str:
candidate = urllib.parse.urljoin(config.session_url, value)
candidate_origin = _origin(candidate)
allowed = {_origin(config.session_url), *config.allowed_api_origins}
if candidate_origin not in allowed:
raise JmapConfigurationError(
f"{label} uses unapproved origin {candidate_origin}; add it to allowed_api_origins"
)
return candidate
def _authorization_header(config: JmapConfig) -> str:
if config.auth_scheme == "bearer":
return f"Bearer {config.password}"
raw = f"{config.username}:{config.password}".encode("utf-8")
return f"Basic {base64.b64encode(raw).decode('ascii')}"
def _transport_coordinates(url: str) -> tuple[str, int, str]:
parsed = urllib.parse.urlsplit(url)
return (
parsed.hostname or "",
parsed.port or (443 if parsed.scheme == "https" else 80),
parsed.scheme,
)
def _origin(value: str) -> str:
parsed = urllib.parse.urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise JmapConfigurationError("JMAP Session advertised an invalid HTTP(S) URL")
if parsed.username or parsed.password or parsed.fragment:
raise JmapConfigurationError("JMAP Session advertised an unsafe URL")
port = parsed.port or (443 if parsed.scheme == "https" else 80)
default = 443 if parsed.scheme == "https" else 80
suffix = "" if port == default else f":{port}"
return f"{parsed.scheme.lower()}://{parsed.hostname.lower()}{suffix}"
def _mailbox_paths(mailboxes: list[dict[str, Any]]) -> dict[str, str]:
by_id = {str(item["id"]): item for item in mailboxes}
paths: dict[str, str] = {}
def path_for(mailbox_id: str, stack: tuple[str, ...] = ()) -> str:
if mailbox_id in paths:
return paths[mailbox_id]
if mailbox_id in stack:
raise JmapProviderError("JMAP mailbox hierarchy contains a cycle")
mailbox = by_id[mailbox_id]
name = _required_text(mailbox.get("name"), "JMAP Mailbox is missing name")
parent_id = _optional_text(mailbox.get("parentId"))
if parent_id and parent_id in by_id:
value = f"{path_for(parent_id, (*stack, mailbox_id))}/{name}"
else:
value = name
paths[mailbox_id] = value
return value
for mailbox_id in by_id:
path_for(mailbox_id)
return paths
def _resolve_mailbox(
mailboxes: list[dict[str, Any]],
folder: str,
) -> tuple[dict[str, Any], dict[str, str]]:
paths = _mailbox_paths(mailboxes)
clean = str(folder or "INBOX").strip()
for item in mailboxes:
mailbox_id = str(item["id"])
role = _optional_text(item.get("role"))
if mailbox_id == clean or paths[mailbox_id] == clean:
return item, paths
if clean.casefold() == "inbox" and role == "inbox":
return item, paths
raise JmapConfigurationError(f"JMAP mailbox {clean!r} is not available")
def _jmap_role_flag(role: str | None) -> str:
return {
"inbox": "\\Inbox",
"sent": "\\Sent",
"drafts": "\\Drafts",
"trash": "\\Trash",
"archive": "\\Archive",
"junk": "\\Junk",
}.get(role or "", "")
def _email_summary(email: Mapping[str, Any], *, folder: str) -> ImapMailboxMessageSummary:
email_id = _required_text(email.get("id"), "JMAP Email is missing id")
message_ids = email.get("messageId")
message_id = None
if isinstance(message_ids, list) and message_ids:
message_id = _optional_text(message_ids[0])
elif isinstance(message_ids, str):
message_id = _optional_text(message_ids)
keywords = email.get("keywords") if isinstance(email.get("keywords"), dict) else {}
flags = [
flag
for keyword, flag in (
("$seen", "\\Seen"),
("$flagged", "\\Flagged"),
("$answered", "\\Answered"),
("$draft", "\\Draft"),
)
if keywords.get(keyword) is True
]
return ImapMailboxMessageSummary(
uid=email_id,
folder=folder,
subject=_optional_text(email.get("subject")),
from_header=_format_addresses(email.get("from")),
to_header=_format_addresses(email.get("to")),
cc_header=_format_addresses(email.get("cc")),
date=_optional_text(email.get("receivedAt")) or _optional_text(email.get("sentAt")),
message_id=message_id,
flags=flags,
size_bytes=_optional_nonnegative_int(email.get("size")),
body_preview=_optional_text(email.get("preview")),
attachment_count=(1 if email.get("hasAttachment") is True else 0),
)
def _format_addresses(value: object) -> str | None:
if not isinstance(value, list):
return None
parts: list[str] = []
for item in value[:1_000]:
if not isinstance(item, dict):
continue
name = _optional_text(item.get("name"))
email = _optional_text(item.get("email"))
if name and email:
parts.append(f"{name} <{email}>")
elif email or name:
parts.append(email or name or "")
return ", ".join(parts) or None
def _body_value(parts: object, values: Mapping[str, Any]) -> str | None:
if not isinstance(parts, list):
return None
result: list[str] = []
for part in parts[:1_000]:
if not isinstance(part, dict):
continue
part_id = _optional_text(part.get("partId"))
body = values.get(part_id) if part_id else None
if isinstance(body, dict) and isinstance(body.get("value"), str):
result.append(body["value"])
return "\n".join(result) or None
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise JmapProviderError(f"{label} must be an object")
return dict(value)
def _object_list(value: object, label: str, *, maximum: int) -> list[dict[str, Any]]:
if not isinstance(value, list) or len(value) > maximum:
raise JmapProviderError(f"{label} must be an array with at most {maximum} items")
if not all(isinstance(item, dict) for item in value):
raise JmapProviderError(f"{label} contains an invalid item")
return [dict(item) for item in value]
def _string_keys(value: object, label: str) -> set[str]:
if not isinstance(value, dict):
raise JmapProviderError(f"{label} must be an object")
return {str(key) for key in value}
def _string_list(value: object, label: str, *, maximum: int) -> list[str]:
if not isinstance(value, list) or len(value) > maximum:
raise JmapProviderError(f"{label} must be an array with at most {maximum} items")
if not all(isinstance(item, str) and item for item in value):
raise JmapProviderError(f"{label} contains an invalid id")
return list(value)
def _required_text(value: object, message: str) -> str:
text = str(value).strip() if isinstance(value, str) else ""
if not text:
raise JmapProviderError(message)
return text
def _optional_text(value: object) -> str | None:
if not isinstance(value, str):
return None
return value.strip() or None
def _optional_nonnegative_int(value: object) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
+492
View File
@@ -0,0 +1,492 @@
from __future__ import annotations
import hashlib
import poplib
import socket
import ssl
from dataclasses import dataclass
from email import policy
from email.message import Message
from email.parser import BytesParser
from typing import Iterable
from govoplan_core.security.outbound_http import (
OutboundHttpError,
create_outbound_connection,
validate_outbound_host,
)
from govoplan_mail.backend.config import Pop3Config, TransportSecurity
class _OutboundPolicyPOP3(poplib.POP3):
def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def]
return create_outbound_connection(
self.host,
self.port,
timeout=timeout,
label="POP3 legacy import",
)
class _OutboundPolicyPOP3SSL(poplib.POP3_SSL):
def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def]
sock = create_outbound_connection(
self.host,
self.port,
timeout=timeout,
label="POP3 legacy import",
)
try:
return self.context.wrap_socket(sock, server_hostname=self.host)
except Exception:
sock.close()
raise
class Pop3ConfigurationError(ValueError):
pass
class Pop3ProviderError(RuntimeError):
def __init__(self, message: str, *, outcome_unknown: bool = False):
super().__init__(message)
self.outcome_unknown = outcome_unknown
@dataclass(frozen=True, slots=True)
class Pop3LoginTestResult:
host: str
port: int
security: str
authenticated: bool
message_count: int
mailbox_size_bytes: int
@dataclass(frozen=True, slots=True)
class Pop3MessageSummary:
message_number: int
uidl: str
subject: str | None
from_header: str | None
to_header: str | None
date: str | None
message_id: str | None
size_bytes: int
body_preview: str | None
@dataclass(frozen=True, slots=True)
class Pop3PreviewResult:
host: str
port: int
security: str
message_count: int
mailbox_size_bytes: int
messages: tuple[Pop3MessageSummary, ...]
@dataclass(frozen=True, slots=True)
class Pop3DownloadedMessage:
message_number: int
uidl: str
raw: bytes
raw_sha256: str
summary: Pop3MessageSummary
@dataclass(frozen=True, slots=True)
class Pop3DeletionResult:
deleted_uidls: tuple[str, ...]
def _require_pop3_config(config: Pop3Config) -> tuple[str, int]:
if not config.legacy_import_enabled:
raise Pop3ConfigurationError(
"POP3 legacy import is disabled for the selected server"
)
if not config.host:
raise Pop3ConfigurationError("POP3 host is required")
if not config.port:
raise Pop3ConfigurationError("POP3 port is required")
if not config.username or not config.password:
raise Pop3ConfigurationError("POP3 username and password are required")
return config.host, config.port
def _open_pop3(config: Pop3Config) -> poplib.POP3:
host, port = _require_pop3_config(config)
try:
validate_outbound_host(host, port=port, label="POP3 legacy import")
except OutboundHttpError as exc:
raise Pop3ConfigurationError(str(exc)) from exc
context = ssl.create_default_context()
client: poplib.POP3 | None = None
try:
if config.security == TransportSecurity.TLS:
client = _OutboundPolicyPOP3SSL(
host=host,
port=port,
timeout=config.timeout_seconds,
context=context,
)
else:
client = _OutboundPolicyPOP3(
host=host,
port=port,
timeout=config.timeout_seconds,
)
if config.security == TransportSecurity.STARTTLS:
client.stls(context=context)
client.user(config.username)
client.pass_(config.password)
return client
except ssl.SSLError as exc:
_close_without_commit(client)
raise Pop3ProviderError("POP3 TLS negotiation failed") from exc
except poplib.error_proto as exc:
_close_without_commit(client)
raise Pop3ProviderError("POP3 authentication failed") from exc
except (OSError, socket.error) as exc:
_close_without_commit(client)
raise Pop3ProviderError("POP3 connection failed") from exc
except Exception:
_close_without_commit(client)
raise
def test_pop3_login(*, pop3_config: Pop3Config) -> Pop3LoginTestResult:
client = _open_pop3(pop3_config)
try:
message_count, mailbox_size = client.stat()
return Pop3LoginTestResult(
host=str(pop3_config.host),
port=int(pop3_config.port or 0),
security=pop3_config.security.value,
authenticated=True,
message_count=int(message_count),
mailbox_size_bytes=int(mailbox_size),
)
except poplib.error_proto as exc:
raise Pop3ProviderError("POP3 mailbox statistics are unavailable") from exc
finally:
_quit_without_deletions(client)
def preview_pop3_messages(
*,
pop3_config: Pop3Config,
limit: int = 50,
) -> Pop3PreviewResult:
clean_limit = max(1, min(int(limit), 100))
client = _open_pop3(pop3_config)
try:
message_count, mailbox_size = client.stat()
uidls = _uidl_map(client)
sizes = _size_map(client)
selected_numbers = sorted(uidls, reverse=True)[:clean_limit]
messages = tuple(
_preview_message(
client,
message_number=number,
uidl=uidls[number],
size_bytes=sizes.get(number, 0),
body_lines=pop3_config.preview_body_lines,
max_message_bytes=pop3_config.max_message_bytes,
)
for number in selected_numbers
)
return Pop3PreviewResult(
host=str(pop3_config.host),
port=int(pop3_config.port or 0),
security=pop3_config.security.value,
message_count=int(message_count),
mailbox_size_bytes=int(mailbox_size),
messages=messages,
)
except poplib.error_proto as exc:
raise Pop3ProviderError("POP3 message preview failed") from exc
finally:
_quit_without_deletions(client)
def download_pop3_messages(
*,
pop3_config: Pop3Config,
uidls: Iterable[str],
) -> tuple[Pop3DownloadedMessage, ...]:
selected_uidls = tuple(dict.fromkeys(_required_uidl(value) for value in uidls))
if not selected_uidls:
raise Pop3ConfigurationError("Select at least one POP3 message to import")
if len(selected_uidls) > 100:
raise Pop3ConfigurationError("At most 100 POP3 messages can be imported at once")
client = _open_pop3(pop3_config)
try:
uidl_by_number = _uidl_map(client)
number_by_uidl = {uidl: number for number, uidl in uidl_by_number.items()}
missing = [uidl for uidl in selected_uidls if uidl not in number_by_uidl]
if missing:
raise Pop3ProviderError(
"One or more previewed POP3 messages are no longer available; refresh the preview"
)
sizes = _size_map(client)
advertised_batch_size = sum(
max(0, int(sizes.get(number_by_uidl[uidl], 0)))
for uidl in selected_uidls
)
if advertised_batch_size > pop3_config.max_batch_bytes:
raise Pop3ProviderError(
"The selected POP3 messages exceed the configured batch size limit"
)
downloaded: list[Pop3DownloadedMessage] = []
downloaded_bytes = 0
for uidl in selected_uidls:
number = number_by_uidl[uidl]
advertised_size = sizes.get(number, 0)
if advertised_size > pop3_config.max_message_bytes:
raise Pop3ProviderError(
f"POP3 message {uidl} exceeds the configured import size limit"
)
_response, lines, _octets = client.retr(number)
raw = _message_bytes(lines)
if len(raw) > pop3_config.max_message_bytes:
raise Pop3ProviderError(
f"POP3 message {uidl} exceeds the configured import size limit"
)
downloaded_bytes += len(raw)
if downloaded_bytes > pop3_config.max_batch_bytes:
raise Pop3ProviderError(
"The selected POP3 messages exceed the configured batch size limit"
)
summary = _message_summary(
raw,
message_number=number,
uidl=uidl,
size_bytes=len(raw),
)
downloaded.append(
Pop3DownloadedMessage(
message_number=number,
uidl=uidl,
raw=raw,
raw_sha256=hashlib.sha256(raw).hexdigest(),
summary=summary,
)
)
return tuple(downloaded)
except poplib.error_proto as exc:
raise Pop3ProviderError("POP3 message download failed") from exc
finally:
_quit_without_deletions(client)
def delete_pop3_messages(
*,
pop3_config: Pop3Config,
uidls: Iterable[str],
) -> Pop3DeletionResult:
selected_uidls = tuple(dict.fromkeys(_required_uidl(value) for value in uidls))
if not selected_uidls:
return Pop3DeletionResult(deleted_uidls=())
if not pop3_config.allow_delete_after_import:
raise Pop3ConfigurationError(
"POP3 delete-after-import is disabled for the selected server"
)
client = _open_pop3(pop3_config)
quit_started = False
try:
number_by_uidl = {
uidl: number for number, uidl in _uidl_map(client).items()
}
missing = [uidl for uidl in selected_uidls if uidl not in number_by_uidl]
if missing:
raise Pop3ProviderError(
"One or more imported POP3 messages are no longer available for deletion"
)
for uidl in selected_uidls:
client.dele(number_by_uidl[uidl])
quit_started = True
client.quit()
return Pop3DeletionResult(deleted_uidls=selected_uidls)
except Pop3ProviderError:
_close_without_commit(client)
raise
except poplib.error_proto as exc:
_close_without_commit(client)
raise Pop3ProviderError(
"POP3 deletion outcome is unknown" if quit_started else "POP3 deletion was rejected",
outcome_unknown=quit_started,
) from exc
except Exception as exc:
_close_without_commit(client)
raise Pop3ProviderError(
"POP3 deletion outcome is unknown" if quit_started else "POP3 deletion failed",
outcome_unknown=quit_started,
) from exc
def _uidl_map(client: poplib.POP3) -> dict[int, str]:
_response, lines, _octets = client.uidl()
result: dict[int, str] = {}
for raw_line in lines:
parts = bytes(raw_line).decode("utf-8", errors="replace").split(maxsplit=1)
if len(parts) != 2 or not parts[0].isdigit():
continue
uidl = _required_uidl(parts[1])
result[int(parts[0])] = uidl
if not result:
raise Pop3ProviderError(
"The POP3 server does not provide stable UIDL identifiers; safe import is unavailable"
)
return result
def _size_map(client: poplib.POP3) -> dict[int, int]:
_response, lines, _octets = client.list()
result: dict[int, int] = {}
for raw_line in lines:
parts = bytes(raw_line).decode("ascii", errors="ignore").split(maxsplit=1)
if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
result[int(parts[0])] = int(parts[1])
return result
def _preview_message(
client: poplib.POP3,
*,
message_number: int,
uidl: str,
size_bytes: int,
body_lines: int,
max_message_bytes: int,
) -> Pop3MessageSummary:
raw: bytes | None = None
try:
_response, lines, _octets = client.top(message_number, body_lines)
raw = _message_bytes(lines)
except (poplib.error_proto, AttributeError):
# TOP is optional. Never use RETR as a preview fallback when the
# advertised message already exceeds the configured download bound.
if size_bytes > max_message_bytes:
return Pop3MessageSummary(
message_number=message_number,
uidl=uidl,
subject=None,
from_header=None,
to_header=None,
date=None,
message_id=None,
size_bytes=size_bytes,
body_preview=None,
)
_response, lines, _octets = client.retr(message_number)
raw = _message_bytes(lines)
if len(raw) > min(max_message_bytes, 256 * 1024):
return Pop3MessageSummary(
message_number=message_number,
uidl=uidl,
subject=None,
from_header=None,
to_header=None,
date=None,
message_id=None,
size_bytes=size_bytes,
body_preview=None,
)
return _message_summary(
raw,
message_number=message_number,
uidl=uidl,
size_bytes=size_bytes,
)
def _message_summary(
raw: bytes,
*,
message_number: int,
uidl: str,
size_bytes: int,
) -> Pop3MessageSummary:
message = BytesParser(policy=policy.default).parsebytes(raw)
return Pop3MessageSummary(
message_number=message_number,
uidl=uidl,
subject=_header(message, "Subject"),
from_header=_header(message, "From"),
to_header=_header(message, "To"),
date=_header(message, "Date"),
message_id=_header(message, "Message-ID"),
size_bytes=max(0, int(size_bytes)),
body_preview=_body_preview(message),
)
def _header(message: Message, name: str) -> str | None:
value = message.get(name)
if value is None:
return None
text = str(value).strip()
return text[:2_000] or None
def _body_preview(message: Message) -> str | None:
body = message.get_body(preferencelist=("plain",)) if message.is_multipart() else message
if body is None:
return None
try:
text = body.get_content()
except Exception:
payload = body.get_payload(decode=True)
text = payload.decode("utf-8", errors="replace") if isinstance(payload, bytes) else str(payload or "")
normalized = " ".join(str(text).split())
return normalized[:500] or None
def _message_bytes(lines: Iterable[bytes]) -> bytes:
return b"\r\n".join(bytes(line) for line in lines) + b"\r\n"
def _required_uidl(value: object) -> str:
clean = str(value or "").strip()
if not clean or len(clean) > 500 or any(char.isspace() for char in clean):
raise Pop3ConfigurationError("POP3 UIDL must be a non-empty token")
return clean
def _quit_without_deletions(client: poplib.POP3) -> None:
try:
client.quit()
except Exception:
_close_without_commit(client)
def _close_without_commit(client: poplib.POP3 | None) -> None:
if client is None:
return
try:
client.rset()
except Exception:
pass
try:
client.close()
except Exception:
pass
__all__ = [
"Pop3ConfigurationError",
"Pop3DeletionResult",
"Pop3DownloadedMessage",
"Pop3LoginTestResult",
"Pop3MessageSummary",
"Pop3PreviewResult",
"Pop3ProviderError",
"delete_pop3_messages",
"download_pop3_messages",
"preview_pop3_messages",
"test_pop3_login",
]
@@ -1,6 +1,7 @@
from __future__ import annotations
import time
import threading
from dataclasses import dataclass
from redis import Redis
@@ -17,6 +18,10 @@ class RateLimitDecision:
waited_seconds: float
_local_rate_limit_lock = threading.Lock()
_local_next_allowed: dict[str, float] = {}
def _redis_client() -> Redis:
return Redis.from_url(settings.redis_url, decode_responses=True)
@@ -30,18 +35,21 @@ def wait_for_rate_limit(*, key: str, messages_per_minute: int, enabled: bool = T
The implementation stores the next allowed send timestamp per key. A Redis
lock keeps multiple Celery processes from reading/updating the timestamp at
the same time. Direct local development runs do not have a broker, so Redis
is only used when Celery/worker mode is explicitly enabled. If Redis is
unavailable, it falls back to no distributed wait.
the same time. Direct local development runs do not have a broker, so they
use a process-local limiter. If Redis is unavailable in worker mode, the
process-local fallback still protects single-process sends.
"""
messages_per_minute = max(1, int(messages_per_minute or 1))
gap = 60.0 / messages_per_minute
if not enabled or not _distributed_rate_limit_enabled():
if not enabled:
return RateLimitDecision(key=key, messages_per_minute=messages_per_minute, gap_seconds=gap, waited_seconds=0.0)
if not _distributed_rate_limit_enabled():
waited = _wait_for_local_rate_limit(key, gap)
return RateLimitDecision(key=key, messages_per_minute=messages_per_minute, gap_seconds=gap, waited_seconds=waited)
redis_key = f"multimailer:ratelimit:{key}:next_allowed"
lock_key = f"multimailer:ratelimit:{key}:lock"
redis_key = f"govoplan:ratelimit:{key}:next_allowed"
lock_key = f"govoplan:ratelimit:{key}:lock"
waited = 0.0
try:
@@ -56,7 +64,17 @@ def wait_for_rate_limit(*, key: str, messages_per_minute: int, enabled: bool = T
now = time.time()
client.set(redis_key, now + gap, ex=max(60, int(gap * 10)))
except (RedisError, TimeoutError, ValueError):
# Development fallback: do not fail sending because Redis is absent.
waited = 0.0
waited = _wait_for_local_rate_limit(key, gap)
return RateLimitDecision(key=key, messages_per_minute=messages_per_minute, gap_seconds=gap, waited_seconds=waited)
def _wait_for_local_rate_limit(key: str, gap: float) -> float:
with _local_rate_limit_lock:
now = time.time()
next_allowed = _local_next_allowed.get(key, now)
waited = max(0.0, next_allowed - now)
_local_next_allowed[key] = max(now, next_allowed) + gap
if waited > 0:
time.sleep(waited)
return waited
+486 -26
View File
@@ -1,12 +1,20 @@
from __future__ import annotations
import copy
import logging
import os
import smtplib
import ssl
from dataclasses import dataclass
from email.message import EmailMessage
from email.utils import formataddr
from govoplan_core.security.outbound_http import (
OutboundHttpError,
create_outbound_connection,
validate_outbound_host,
)
from govoplan_mail.backend.config import SmtpConfig, TransportSecurity
from govoplan_mail.backend.dev.mock_mailbox import (
consume_fail_next_smtp,
@@ -15,6 +23,35 @@ from govoplan_mail.backend.dev.mock_mailbox import (
record_smtp_delivery,
)
logger = logging.getLogger(__name__)
class _OutboundPolicySMTP(smtplib.SMTP):
def _get_socket(self, host: str, port: int, timeout: float | None): # type: ignore[no-untyped-def]
return create_outbound_connection(
host,
port,
timeout=timeout,
source_address=self.source_address,
label="SMTP connector",
)
class _OutboundPolicySMTPSSL(smtplib.SMTP_SSL):
def _get_socket(self, host: str, port: int, timeout: float | None): # type: ignore[no-untyped-def]
sock = create_outbound_connection(
host,
port,
timeout=timeout,
source_address=self.source_address,
label="SMTP connector",
)
try:
return self.context.wrap_socket(sock, server_hostname=self._host)
except Exception:
sock.close()
raise
class SmtpConfigurationError(ValueError):
"""Raised when SMTP settings are incomplete or inconsistent."""
@@ -28,10 +65,22 @@ class SmtpSendError(RuntimeError):
started, so automatic retry is intentionally forbidden.
"""
def __init__(self, message: str, *, temporary: bool = False, outcome_unknown: bool = False):
def __init__(
self,
message: str,
*,
temporary: bool = False,
outcome_unknown: bool = False,
systemic: bool = False,
reason_code: str | None = None,
phase: str = "send",
):
super().__init__(message)
self.temporary = temporary
self.outcome_unknown = outcome_unknown
self.systemic = systemic
self.reason_code = reason_code
self.phase = phase
@dataclass(frozen=True, slots=True)
@@ -50,12 +99,319 @@ class SmtpSendResult:
envelope_from: str
envelope_recipients: list[str]
refused_recipients: dict[str, tuple[int, bytes | str]]
connection_sequence: int = 1
session_reused: bool = False
reconnect_count: int = 0
@property
def accepted_count(self) -> int:
return len(self.envelope_recipients) - len(self.refused_recipients)
@dataclass(frozen=True, slots=True)
class SmtpBatchPolicy:
reuse_connections: bool = True
max_messages_per_connection: int = 100
reconnect_attempts: int = 1
health_check_before_reuse: bool = True
@classmethod
def from_environment(cls) -> "SmtpBatchPolicy":
return cls(
reuse_connections=_environment_bool("GOVOPLAN_SMTP_BATCH_REUSE", True),
max_messages_per_connection=_environment_int(
"GOVOPLAN_SMTP_BATCH_MAX_MESSAGES",
default=100,
minimum=1,
maximum=10_000,
),
reconnect_attempts=_environment_int(
"GOVOPLAN_SMTP_BATCH_RECONNECT_ATTEMPTS",
default=1,
minimum=0,
maximum=5,
),
health_check_before_reuse=_environment_bool(
"GOVOPLAN_SMTP_BATCH_HEALTH_CHECK",
True,
),
)
@dataclass(frozen=True, slots=True)
class SmtpBatchPreflightResult:
ready: bool
authenticated: bool
connection_sequence: int
reconnect_count: int
class SmtpBatchSession:
"""Bounded reusable SMTP connection for one already-authorized batch."""
def __init__(
self,
smtp_config: SmtpConfig,
*,
policy: SmtpBatchPolicy | None = None,
) -> None:
self.smtp_config = smtp_config
self.policy = policy or SmtpBatchPolicy.from_environment()
self._smtp: smtplib.SMTP | None = None
self._connection_sequence = 0
self._reconnect_count = 0
self._messages_on_connection = 0
self._closed = False
@property
def connection_count(self) -> int:
return self._connection_sequence
@property
def reconnect_count(self) -> int:
return self._reconnect_count
def preflight(self) -> SmtpBatchPreflightResult:
"""Validate DNS/egress/connectivity/TLS/auth before a provider effect."""
if self._closed:
raise SmtpSendError(
"SMTP batch session is closed.",
systemic=True,
reason_code="batch_session_closed",
phase="preflight",
)
_require_smtp_config(self.smtp_config)
if is_mock_smtp_host(self.smtp_config.host):
if self._connection_sequence == 0:
self._connection_sequence = 1
return self._preflight_result()
if self._smtp is None:
self._connect_with_retries()
return self._preflight_result()
def send(
self,
message: EmailMessage | bytes,
*,
envelope_from: str,
envelope_recipients: list[str],
) -> SmtpSendResult:
host, port, recipients = _prepare_smtp_send(
smtp_config=self.smtp_config,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
)
if is_mock_smtp_host(self.smtp_config.host):
preflight = self.preflight()
_accepted, refused = _send_mock_smtp_payload(
message,
smtp_config=self.smtp_config,
envelope_from=envelope_from,
envelope_recipients=recipients,
)
self._messages_on_connection += 1
return _smtp_send_result(
smtp_config=self.smtp_config,
host=host,
port=port,
envelope_from=envelope_from,
envelope_recipients=recipients,
refused=refused,
connection_sequence=preflight.connection_sequence,
session_reused=self._messages_on_connection > 1,
reconnect_count=preflight.reconnect_count,
)
reused = self._prepare_connection_for_send()
smtp = self._smtp
if smtp is None: # Defensive: preflight either opens or raises.
raise SmtpSendError(
"SMTP preflight did not establish a connection.",
temporary=True,
systemic=True,
reason_code="smtp_connectivity_unavailable",
phase="preflight",
)
try:
if isinstance(message, bytes):
refused = smtp.sendmail(envelope_from, recipients, message)
else:
refused = smtp.send_message(
message,
from_addr=envelope_from,
to_addrs=recipients,
)
except smtplib.SMTPRecipientsRefused as exc:
raise SmtpSendError(
f"all SMTP recipients were refused: {_decode_refused(exc.recipients)}",
temporary=False,
reason_code="smtp_recipients_refused",
) from exc
except smtplib.SMTPSenderRefused as exc:
self._discard_connection()
raise SmtpSendError(
f"SMTP sender was refused: {exc.smtp_code} {exc.smtp_error!r}",
temporary=400 <= int(exc.smtp_code) < 500,
systemic=True,
reason_code="smtp_sender_refused",
) from exc
except smtplib.SMTPResponseException as exc:
disconnected = int(exc.smtp_code) == 421
if disconnected:
self._discard_connection()
raise SmtpSendError(
f"SMTP error: {exc.smtp_code} {exc.smtp_error!r}",
temporary=400 <= int(exc.smtp_code) < 500,
systemic=disconnected,
reason_code="smtp_connection_closed" if disconnected else "smtp_message_rejected",
) from exc
except (OSError, smtplib.SMTPServerDisconnected, smtplib.SMTPException) as exc:
self._discard_connection()
raise SmtpSendError(
f"SMTP outcome is unknown after transmission started: {exc}",
outcome_unknown=True,
systemic=True,
reason_code="smtp_connection_lost_after_transmission",
) from exc
self._messages_on_connection += 1
result = _smtp_send_result(
smtp_config=self.smtp_config,
host=host,
port=port,
envelope_from=envelope_from,
envelope_recipients=recipients,
refused=refused,
connection_sequence=self._connection_sequence,
session_reused=reused,
reconnect_count=self._reconnect_count,
)
if not self.policy.reuse_connections:
self._discard_connection()
return result
def close(self) -> None:
self._closed = True
self._discard_connection()
def __enter__(self) -> "SmtpBatchSession":
self.preflight()
return self
def __exit__(self, _exc_type, _exc, _traceback) -> None:
self.close()
def _preflight_result(self) -> SmtpBatchPreflightResult:
return SmtpBatchPreflightResult(
ready=True,
authenticated=bool(self.smtp_config.username and self.smtp_config.password),
connection_sequence=self._connection_sequence,
reconnect_count=self._reconnect_count,
)
def _prepare_connection_for_send(self) -> bool:
reused = self._smtp is not None and self._messages_on_connection > 0
if self._smtp is not None and self._messages_on_connection >= self.policy.max_messages_per_connection:
self._discard_connection()
reused = False
elif reused and self.policy.health_check_before_reuse:
try:
code, _message = self._smtp.noop()
if int(code) >= 400:
raise smtplib.SMTPServerDisconnected(f"SMTP NOOP returned {code}")
except (OSError, smtplib.SMTPException):
self._discard_connection()
reused = False
self.preflight()
return reused and self._smtp is not None
def _connect_with_retries(self) -> None:
last_error: BaseException | None = None
for attempt in range(self.policy.reconnect_attempts + 1):
try:
smtp = _open_smtp(self.smtp_config)
except smtplib.SMTPAuthenticationError as exc:
raise SmtpSendError(
"SMTP authentication failed during batch preflight.",
systemic=True,
reason_code="smtp_authentication_failed",
phase="preflight",
) from exc
except SmtpConfigurationError:
raise
except smtplib.SMTPResponseException as exc:
temporary = 400 <= int(exc.smtp_code) < 500
last_error = exc
if not temporary or attempt >= self.policy.reconnect_attempts:
raise SmtpSendError(
"SMTP server rejected batch preflight.",
temporary=temporary,
systemic=True,
reason_code="smtp_preflight_rejected",
phase="preflight",
) from exc
continue
except (OSError, smtplib.SMTPException) as exc:
last_error = exc
if attempt >= self.policy.reconnect_attempts:
raise SmtpSendError(
"SMTP connectivity is unavailable during batch preflight.",
temporary=True,
systemic=True,
reason_code="smtp_connectivity_unavailable",
phase="preflight",
) from exc
continue
self._smtp = smtp
if attempt > 0 or self._connection_sequence > 0:
self._reconnect_count += 1
self._connection_sequence += 1
self._messages_on_connection = 0
return
raise SmtpSendError(
f"SMTP batch preflight failed: {last_error}",
temporary=True,
systemic=True,
reason_code="smtp_connectivity_unavailable",
phase="preflight",
)
def _discard_connection(self) -> None:
smtp, self._smtp = self._smtp, None
self._messages_on_connection = 0
if smtp is None:
return
try:
smtp.quit()
except Exception as quit_exc:
_log_smtp_cleanup_failure("closing batch connection", quit_exc)
try:
smtp.close()
except Exception as close_exc:
_log_smtp_cleanup_failure("closing batch socket", close_exc)
def _environment_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().casefold() in {"1", "true", "yes", "on"}
def _environment_int(name: str, *, default: int, minimum: int, maximum: int) -> int:
value = os.getenv(name)
try:
parsed = int(value) if value is not None else default
except ValueError:
parsed = default
return max(minimum, min(maximum, parsed))
def _log_smtp_cleanup_failure(action: str, exc: BaseException) -> None:
logger.debug("SMTP cleanup failed while %s: %s", action, exc, exc_info=True)
def _require_smtp_config(config: SmtpConfig) -> tuple[str, int]:
if not config.host:
raise SmtpConfigurationError("SMTP host is required")
@@ -68,14 +424,23 @@ def _require_smtp_config(config: SmtpConfig) -> tuple[str, int]:
def _open_smtp(config: SmtpConfig) -> smtplib.SMTP:
host, port = _require_smtp_config(config)
try:
validate_outbound_host(host, port=port, label="SMTP connector")
except OutboundHttpError as exc:
raise SmtpConfigurationError(str(exc)) from exc
context = ssl.create_default_context()
try:
if config.security == TransportSecurity.TLS:
smtp: smtplib.SMTP = smtplib.SMTP_SSL(host=host, port=port, timeout=config.timeout_seconds, context=context)
smtp: smtplib.SMTP = _OutboundPolicySMTPSSL(
host=host,
port=port,
timeout=config.timeout_seconds,
context=context,
)
smtp.ehlo()
else:
smtp = smtplib.SMTP(host=host, port=port, timeout=config.timeout_seconds)
smtp = _OutboundPolicySMTP(host=host, port=port, timeout=config.timeout_seconds)
smtp.ehlo()
if config.security == TransportSecurity.STARTTLS:
smtp.starttls(context=context)
@@ -89,8 +454,8 @@ def _open_smtp(config: SmtpConfig) -> smtplib.SMTP:
# on GC, but explicit cleanup is safer when the variable exists.
try:
smtp.quit() # type: ignore[possibly-undefined]
except Exception:
pass
except Exception as cleanup_exc:
_log_smtp_cleanup_failure("opening connection", cleanup_exc)
raise
@@ -134,11 +499,12 @@ def test_smtp_login(*, smtp_config: SmtpConfig) -> SmtpLoginTestResult:
finally:
try:
smtp.quit()
except Exception:
except Exception as quit_exc:
_log_smtp_cleanup_failure("testing login quit", quit_exc)
try:
smtp.close()
except Exception:
pass
except Exception as close_exc:
_log_smtp_cleanup_failure("testing login close", close_exc)
def prepare_test_message(
@@ -161,12 +527,12 @@ def prepare_test_message(
del test_message[header]
# Replace potential previous marker headers if the user test-sends an EML twice.
for header in ["X-MultiMailer-Test-Send"]:
for header in ["X-GovOPlaN-Test-Send"]:
if header in test_message:
del test_message[header]
test_message["To"] = formataddr((test_recipient_name or test_recipient, test_recipient))
test_message["X-MultiMailer-Test-Send"] = "true"
test_message["X-GovOPlaN-Test-Send"] = "true"
return test_message
@@ -177,13 +543,66 @@ def _send_smtp_payload(
envelope_from: str,
envelope_recipients: list[str],
) -> SmtpSendResult:
host, port, recipients = _prepare_smtp_send(
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
)
if is_mock_smtp_host(smtp_config.host):
_accepted, refused = _send_mock_smtp_payload(
message,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=recipients,
)
return _smtp_send_result(
smtp_config=smtp_config,
host=host,
port=port,
envelope_from=envelope_from,
envelope_recipients=recipients,
refused=refused,
)
refused = _send_network_smtp_payload(
message,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=recipients,
)
return _smtp_send_result(
smtp_config=smtp_config,
host=host,
port=port,
envelope_from=envelope_from,
envelope_recipients=recipients,
refused=refused,
)
def _prepare_smtp_send(
*,
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
) -> tuple[str, int, list[str]]:
host, port = _require_smtp_config(smtp_config)
if not envelope_from:
raise SmtpConfigurationError("SMTP envelope sender is required")
if not envelope_recipients:
recipients = [recipient for recipient in envelope_recipients if recipient]
if not recipients:
raise SmtpConfigurationError("at least one SMTP envelope recipient is required")
return host, port, recipients
if is_mock_smtp_host(smtp_config.host):
def _send_mock_smtp_payload(
message: EmailMessage | bytes,
*,
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
) -> tuple[list[str], dict[str, tuple[int, bytes]]]:
if consume_fail_next_smtp():
raise SmtpSendError("Mock SMTP configured to fail the next send")
failures = get_failures()
@@ -205,30 +624,43 @@ def _send_smtp_payload(
envelope_recipients=accepted,
smtp_host=smtp_config.host,
)
return SmtpSendResult(
host=host,
port=port,
security=smtp_config.security.value,
envelope_from=envelope_from,
envelope_recipients=list(envelope_recipients),
refused_recipients=_decode_refused(refused),
)
return accepted, refused
def _send_network_smtp_payload(
message: EmailMessage | bytes,
*,
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
) -> dict[str, tuple[int, bytes]]:
try:
smtp = _open_smtp(smtp_config)
except smtplib.SMTPAuthenticationError as exc:
raise SmtpSendError(
f"SMTP authentication failed: {exc.smtp_code} {exc.smtp_error!r}",
temporary=False,
systemic=True,
reason_code="smtp_authentication_failed",
phase="preflight",
) from exc
except smtplib.SMTPResponseException as exc:
raise SmtpSendError(
f"SMTP connection error: {exc.smtp_code} {exc.smtp_error!r}",
temporary=400 <= int(exc.smtp_code) < 500,
systemic=True,
reason_code="smtp_preflight_rejected",
phase="preflight",
) from exc
except (OSError, smtplib.SMTPException) as exc:
# No message transmission has begun yet; a later explicit retry is safe.
raise SmtpSendError(f"SMTP connection failed: {exc}", temporary=True) from exc
raise SmtpSendError(
f"SMTP connection failed: {exc}",
temporary=True,
systemic=True,
reason_code="smtp_connectivity_unavailable",
phase="preflight",
) from exc
try:
if isinstance(message, bytes):
@@ -248,6 +680,8 @@ def _send_smtp_payload(
raise SmtpSendError(
f"SMTP sender was refused: {exc.smtp_code} {exc.smtp_error!r}",
temporary=400 <= int(exc.smtp_code) < 500,
systemic=True,
reason_code="smtp_sender_refused",
) from exc
except smtplib.SMTPResponseException as exc:
# An explicit SMTP response means the server rejected the transaction;
@@ -255,6 +689,8 @@ def _send_smtp_payload(
raise SmtpSendError(
f"SMTP error: {exc.smtp_code} {exc.smtp_error!r}",
temporary=400 <= int(exc.smtp_code) < 500,
systemic=int(exc.smtp_code) == 421,
reason_code="smtp_connection_closed" if int(exc.smtp_code) == 421 else "smtp_message_rejected",
) from exc
except (OSError, smtplib.SMTPServerDisconnected, smtplib.SMTPException) as exc:
# A connection loss after DATA began can happen after the server accepted
@@ -262,16 +698,33 @@ def _send_smtp_payload(
raise SmtpSendError(
f"SMTP outcome is unknown after transmission started: {exc}",
outcome_unknown=True,
systemic=True,
reason_code="smtp_connection_lost_after_transmission",
) from exc
finally:
try:
smtp.quit()
except Exception:
except Exception as quit_exc:
_log_smtp_cleanup_failure("sending message quit", quit_exc)
try:
smtp.close()
except Exception:
pass
except Exception as close_exc:
_log_smtp_cleanup_failure("sending message close", close_exc)
return refused
def _smtp_send_result(
*,
smtp_config: SmtpConfig,
host: str,
port: int,
envelope_from: str,
envelope_recipients: list[str],
refused: dict[str, tuple[int, bytes]],
connection_sequence: int = 1,
session_reused: bool = False,
reconnect_count: int = 0,
) -> SmtpSendResult:
return SmtpSendResult(
host=host,
port=port,
@@ -279,6 +732,9 @@ def _send_smtp_payload(
envelope_from=envelope_from,
envelope_recipients=list(envelope_recipients),
refused_recipients=_decode_refused(refused),
connection_sequence=connection_sequence,
session_reused=session_reused,
reconnect_count=reconnect_count,
)
@@ -288,15 +744,19 @@ def send_email_bytes(
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
batch_session: SmtpBatchSession | None = None,
) -> SmtpSendResult:
"""Send exact RFC 5322 bytes through SMTP without reserializing the message."""
return _send_smtp_payload(
if batch_session is not None:
if batch_session.smtp_config != smtp_config:
raise SmtpConfigurationError("SMTP batch session does not match the resolved transport.")
return batch_session.send(
message_bytes,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
)
return _send_smtp_payload(message_bytes, smtp_config=smtp_config, envelope_from=envelope_from, envelope_recipients=envelope_recipients)
def send_email_message(
File diff suppressed because it is too large Load Diff
+141
View File
@@ -0,0 +1,141 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from fastapi import HTTPException
from pydantic import ValidationError
from govoplan_mail.backend.router import (
create_mail_address_contact,
list_mail_address_write_targets,
lookup_mail_addresses,
)
from govoplan_mail.backend.schemas import MailContactCreateRequest
class _Session:
def __init__(self) -> None:
self.commits = 0
self.rollbacks = 0
def commit(self) -> None:
self.commits += 1
def rollback(self) -> None:
self.rollbacks += 1
class _Writer:
def __init__(self, *, allowed: bool = True, read_only: bool = False) -> None:
self.allowed = allowed
self.read_only = read_only
self.created_payload = None
self.created_provenance = None
def list_write_targets(self, _session, _principal, *, operation):
return (
SimpleNamespace(
address_book_id="book-1",
address_book_label="Personal contacts",
operation=operation,
allowed=self.allowed,
reason="allowed" if self.allowed else "read_only_source",
message="Contact can be added." if self.allowed else "This source is read-only.",
scope_type="user",
scope_id="user-1",
source_kind="local" if self.allowed else "ldap",
read_only=self.read_only,
required_scopes=("addresses:contacts:write",),
provenance={"policy": "addresses"},
),
)
def can_write_to_address_book(self, _session, _principal, *, address_book_id, operation):
return self.list_write_targets(_session, _principal, operation=operation)[0]
def create_contact(self, _session, _principal, *, address_book_id, payload, provenance):
self.created_payload = payload
self.created_provenance = provenance
return SimpleNamespace(
contact_id="contact-1",
address_book_id=address_book_id,
display_name=payload["display_name"],
email=payload["emails"][0]["email"],
source_kind="local",
provenance=provenance,
)
def _principal():
return SimpleNamespace(has=lambda scope: scope == "mail:profile:use")
class MailAddressIntegrationTests(unittest.TestCase):
def test_optional_capabilities_fail_open_for_mail(self) -> None:
with patch("govoplan_mail.backend.router._registry_capability", return_value=None):
lookup = lookup_mail_addresses(query="ada", limit=25, session=_Session(), principal=_principal())
targets = list_mail_address_write_targets(session=_Session(), principal=_principal())
self.assertFalse(lookup.available)
self.assertEqual(lookup.candidates, [])
self.assertFalse(targets.available)
self.assertEqual(targets.targets, [])
def test_write_target_preserves_read_only_decision(self) -> None:
writer = _Writer(allowed=False, read_only=True)
with patch("govoplan_mail.backend.router._registry_capability", return_value=writer):
response = list_mail_address_write_targets(session=_Session(), principal=_principal())
self.assertTrue(response.available)
self.assertFalse(response.targets[0].allowed)
self.assertTrue(response.targets[0].read_only)
self.assertEqual(response.targets[0].reason, "read_only_source")
def test_blocked_target_cannot_be_bypassed_by_create(self) -> None:
session = _Session()
writer = _Writer(allowed=False, read_only=True)
with (
patch("govoplan_mail.backend.router._registry_capability", return_value=writer),
self.assertRaises(HTTPException) as raised,
):
create_mail_address_contact(
MailContactCreateRequest(
address_book_id="book-1",
display_name="Ada Lovelace",
email="ada@example.test",
),
session=session,
principal=_principal(),
)
self.assertEqual(raised.exception.status_code, 422)
self.assertEqual(session.commits, 0)
def test_allowed_create_uses_writer_and_records_consumer_provenance(self) -> None:
session = _Session()
writer = _Writer()
with patch("govoplan_mail.backend.router._registry_capability", return_value=writer):
result = create_mail_address_contact(
MailContactCreateRequest(
address_book_id="book-1",
display_name="Ada Lovelace",
email="ada@example.test",
),
session=session,
principal=_principal(),
)
self.assertEqual(result.contact_id, "contact-1")
self.assertEqual(writer.created_payload["emails"][0]["email"], "ada@example.test")
self.assertEqual(writer.created_provenance["consumer_module"], "mail")
self.assertEqual(session.commits, 1)
def test_proxy_rejects_invalid_email_before_calling_writer(self) -> None:
with self.assertRaises(ValidationError):
MailContactCreateRequest(address_book_id="book-1", email="not-an-email")
if __name__ == "__main__":
unittest.main()
+248
View File
@@ -0,0 +1,248 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from sqlalchemy import Column, String, Table, create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from govoplan_core.db.base import Base
from govoplan_core.core.calendar import CalendarInvitationRef
from govoplan_mail.backend.bounce_processing import (
SqlMailBounceProcessingProvider,
parse_delivery_status,
)
from govoplan_mail.backend.db.models import (
MailBounceObservation,
MailBounceSource,
MailDeliveryCommand,
MailServerProfile,
)
from govoplan_mail.backend.delivery_outbox import submit_delivery_command
DSN = b"""From: Mail Delivery Subsystem <mailer-daemon@example.test>
To: sender@example.test
Date: Fri, 31 Jul 2026 12:00:00 +0000
Subject: Delivery Status Notification (Failure)
MIME-Version: 1.0
Content-Type: multipart/report; report-type=delivery-status; boundary="dsn"
--dsn
Content-Type: text/plain; charset=utf-8
Delivery failed.
--dsn
Content-Type: message/delivery-status
Reporting-MTA: dns; mx.example.test
Original-Message-ID: <outgoing-1@example.test>
Final-Recipient: rfc822; recipient@example.test
Action: failed
Status: 5.1.1
Remote-MTA: dns; destination.example.test
Diagnostic-Code: smtp; 550 mailbox unavailable
Last-Attempt-Date: Fri, 31 Jul 2026 11:59:00 +0000
--dsn
Content-Type: message/rfc822
Message-ID: <outgoing-1@example.test>
From: sender@example.test
To: recipient@example.test
Subject: Original
Body
--dsn--
"""
CALENDAR_REPLY = b"""From: Ada <ada@example.test>
To: organizer@example.test
Message-ID: <calendar-reply-1@example.test>
Subject: Accepted: Planning
MIME-Version: 1.0
Content-Type: text/calendar; method=REPLY; charset=utf-8
BEGIN:VCALENDAR
VERSION:2.0
METHOD:REPLY
BEGIN:VEVENT
UID:invitation-1@govoplan.local
DTSTART:20260805T090000Z
ATTENDEE;PARTSTAT=ACCEPTED:mailto:ada@example.test
END:VEVENT
END:VCALENDAR
"""
class MailBounceProcessingTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine(
"sqlite+pysqlite://",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
access_users = Base.metadata.tables.get("access_users")
if access_users is None:
access_users = Table(
"access_users",
Base.metadata,
Column("id", String(36), primary_key=True),
)
Base.metadata.create_all(
self.engine,
tables=[
access_users,
MailServerProfile.__table__,
MailDeliveryCommand.__table__,
MailBounceSource.__table__,
MailBounceObservation.__table__,
],
)
self.SessionLocal = sessionmaker(
bind=self.engine,
class_=Session,
expire_on_commit=False,
)
with self.SessionLocal() as session:
session.add(
MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Delivery",
slug="delivery",
smtp_config={"host": "smtp.example.test", "port": 25},
)
)
session.commit()
self.audit_delivery = patch(
"govoplan_mail.backend.delivery_outbox.audit_event"
)
self.audit_bounce = patch(
"govoplan_mail.backend.bounce_processing.audit_event"
)
self.audit_delivery.start()
self.audit_bounce.start()
self.addCleanup(self.audit_delivery.stop)
self.addCleanup(self.audit_bounce.stop)
self.addCleanup(self.engine.dispose)
def test_parser_extracts_structured_recipient_outcome(self) -> None:
reports = parse_delivery_status(DSN)
self.assertEqual(1, len(reports))
self.assertEqual("recipient@example.test", reports[0]["recipient"])
self.assertEqual("failed", reports[0]["action"])
self.assertEqual("5.1.1", reports[0]["status_code"])
self.assertEqual(
"<outgoing-1@example.test>",
reports[0]["original_message_id"],
)
def test_processing_correlates_and_is_idempotent(self) -> None:
provider = SqlMailBounceProcessingProvider()
with self.SessionLocal() as session:
command = submit_delivery_command(
session,
tenant_id="tenant-1",
command_type="campaign_report",
source_module="campaigns",
source_resource_type="campaign",
source_resource_id="campaign-1",
source_version_id="version-1",
idempotency_key="delivery-1",
profile_id="profile-1",
message_bytes=(
b"Message-ID: <outgoing-1@example.test>\r\n"
b"Subject: Original\r\n\r\nBody"
),
envelope_from="sender@example.test",
envelope_recipients=["recipient@example.test"],
from_header="sender@example.test",
expected_smtp_transport_revision="revision-1",
)
first = provider.process_raw_message(
session,
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
uid="42",
raw_message=DSN,
)
second = provider.process_raw_message(
session,
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
uid="42",
raw_message=DSN,
)
self.assertEqual(command["id"], first[0].command_id)
self.assertTrue(first[0].matched)
self.assertTrue(first[0].permanent)
self.assertEqual(first[0].id, second[0].id)
self.assertEqual(1, session.query(MailBounceObservation).count())
self.assertEqual(
"pending",
session.get(MailDeliveryCommand, command["id"]).status,
)
def test_ordinary_mail_is_not_misclassified(self) -> None:
self.assertEqual(
(),
parse_delivery_status(
b"From: person@example.test\r\nSubject: Hello\r\n\r\nNot a DSN"
),
)
def test_calendar_reply_is_forwarded_without_becoming_a_bounce(self) -> None:
class CalendarProvider:
def __init__(self) -> None:
self.calls = []
def record_icalendar_reply(self, session, **kwargs):
self.calls.append((session, kwargs))
return (
CalendarInvitationRef(
event_id="event-1",
calendar_id="calendar-1",
uid="invitation-1@govoplan.local",
correlation_id="campaign:version-1:entry-1",
source_module="campaigns",
source_resource_type="campaign_version",
source_resource_id="version-1",
),
)
calendar = CalendarProvider()
provider = SqlMailBounceProcessingProvider()
with (
self.SessionLocal() as session,
patch(
"govoplan_mail.backend.bounce_processing.calendar_invitation_provider",
return_value=calendar,
),
):
observations = provider.process_raw_message(
session,
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
uid="43",
raw_message=CALENDAR_REPLY,
)
self.assertEqual((), observations)
self.assertEqual(1, len(calendar.calls))
forwarded = calendar.calls[0][1]
self.assertIn("METHOD:REPLY", forwarded["icalendar"])
self.assertEqual("43", forwarded["evidence"]["mailbox_uid"])
if __name__ == "__main__":
unittest.main()
+322
View File
@@ -0,0 +1,322 @@
from __future__ import annotations
import json
import unittest
from dataclasses import asdict
from types import SimpleNamespace
from unittest.mock import patch
from govoplan_mail.backend.capabilities import (
MailCampaignCapability,
append_campaign_message_to_sent,
campaign_profile_delivery_summary,
send_campaign_email_bytes,
)
from govoplan_mail.backend.config import ImapConfig, SmtpConfig
from govoplan_mail.backend.mail_profiles import MailProfileError
from govoplan_mail.backend.sending.imap import ImapAppendError
from govoplan_mail.backend.sending.smtp import SmtpSendError
def _profile() -> SimpleNamespace:
return SimpleNamespace(
smtp_config={
"host": "smtp.internal.example",
"port": 587,
"security": "starttls",
},
smtp_username="service-account",
smtp_password_encrypted="encrypted-secret",
smtp_transport_revision="smtp-revision",
imap_config=None,
imap_username=None,
imap_password_encrypted=None,
imap_transport_revision="imap-revision",
)
class CampaignMailCapabilityTests(unittest.TestCase):
def test_summary_does_not_materialize_or_decrypt_credentials(self) -> None:
profile = _profile()
with (
patch(
"govoplan_mail.backend.capabilities.ensure_mail_profile_allowed_for_campaign",
return_value=profile,
),
patch("govoplan_mail.backend.capabilities.effective_mail_profile_policy", return_value=object()),
patch("govoplan_mail.backend.capabilities._assert_campaign_inherits_profile_credentials"),
patch(
"govoplan_mail.backend.capabilities.smtp_config_from_profile",
side_effect=AssertionError("summary must not decrypt SMTP credentials"),
),
patch(
"govoplan_mail.backend.capabilities.imap_config_from_profile",
side_effect=AssertionError("summary must not decrypt IMAP credentials"),
),
):
summary = campaign_profile_delivery_summary(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
)
self.assertTrue(summary["smtp_available"])
self.assertFalse(summary["imap_available"])
self.assertNotIn("service-account", repr(summary))
self.assertNotIn("internal.example", repr(summary))
self.assertNotIn("secret", repr(summary))
def test_send_compares_revision_before_provider_effect(self) -> None:
profile = _profile()
with (
patch(
"govoplan_mail.backend.capabilities._authorized_campaign_profile",
return_value=profile,
),
patch(
"govoplan_mail.backend.capabilities.smtp_config_from_profile",
side_effect=AssertionError("stale revisions must be rejected before SMTP credential decryption"),
),
patch(
"govoplan_mail.backend.capabilities.campaign_profile_transport_revisions",
return_value={"smtp": "current", "imap": None},
),
patch("govoplan_mail.backend.capabilities.send_email_bytes") as provider_send,
):
with self.assertRaisesRegex(MailProfileError, "changed after this campaign was built"):
send_campaign_email_bytes(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
message_bytes=b"message",
envelope_from="sender@example.test",
envelope_recipients=["recipient@example.test"],
from_header="sender@example.test",
expected_smtp_transport_revision="stale",
)
provider_send.assert_not_called()
def test_stale_imap_revision_is_rejected_before_credential_decryption(self) -> None:
profile = _profile()
profile.imap_config = {
"host": "imap.internal.example",
"port": 993,
"security": "tls",
}
with (
patch(
"govoplan_mail.backend.capabilities._authorized_campaign_profile",
return_value=profile,
),
patch(
"govoplan_mail.backend.capabilities.campaign_profile_transport_revisions",
return_value={"smtp": "smtp-current", "imap": "imap-current"},
),
patch(
"govoplan_mail.backend.capabilities.imap_config_from_profile",
side_effect=AssertionError("stale revisions must be rejected before IMAP credential decryption"),
),
patch("govoplan_mail.backend.capabilities.append_message_to_sent") as provider_append,
):
with self.assertRaisesRegex(MailProfileError, "changed after this campaign was built"):
append_campaign_message_to_sent(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
message_bytes=b"message",
folder="Sent",
expected_smtp_transport_revision="smtp-current",
expected_imap_transport_revision="imap-stale",
)
provider_append.assert_not_called()
def test_provider_response_is_sanitized_before_campaign_evidence(self) -> None:
profile = _profile()
smtp = SmtpConfig(host="smtp.internal.example", port=587)
provider_secret = b"smtp.internal.example says credential=provider-secret"
provider_result = SimpleNamespace(
envelope_recipients=["ok@example.test", "blocked@example.test"],
refused_recipients={"blocked@example.test": (550, provider_secret)},
)
with (
patch(
"govoplan_mail.backend.capabilities._authorized_campaign_profile",
return_value=profile,
),
patch("govoplan_mail.backend.capabilities.smtp_config_from_profile", return_value=smtp),
patch(
"govoplan_mail.backend.capabilities.imap_config_from_profile",
side_effect=AssertionError("SMTP delivery must not decrypt IMAP credentials"),
),
patch(
"govoplan_mail.backend.capabilities.campaign_profile_transport_revisions",
return_value={"smtp": "current", "imap": None},
),
patch("govoplan_mail.backend.capabilities.assert_mail_policy_allows_send"),
patch("govoplan_mail.backend.capabilities.send_email_bytes", return_value=provider_result),
):
result = send_campaign_email_bytes(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
message_bytes=b"message",
envelope_from="sender@example.test",
envelope_recipients=provider_result.envelope_recipients,
from_header="sender@example.test",
expected_smtp_transport_revision="current",
)
evidence = json.dumps(asdict(result), sort_keys=True)
self.assertEqual(result.accepted_count, 1)
self.assertIn('"status_code": 550', evidence)
self.assertIn('"classification": "permanent"', evidence)
self.assertNotIn("smtp.internal.example", evidence)
self.assertNotIn("provider-secret", evidence)
def test_append_materializes_only_imap_credentials_and_preserves_unknown_outcome(self) -> None:
profile = _profile()
profile.imap_config = {
"host": "imap.internal.example",
"port": 993,
"security": "tls",
}
imap = ImapConfig(host="imap.internal.example", port=993)
with (
patch(
"govoplan_mail.backend.capabilities._authorized_campaign_profile",
return_value=profile,
),
patch("govoplan_mail.backend.capabilities.imap_config_from_profile", return_value=imap),
patch(
"govoplan_mail.backend.capabilities.smtp_config_from_profile",
side_effect=AssertionError("IMAP append must not decrypt SMTP credentials"),
),
patch(
"govoplan_mail.backend.capabilities.campaign_profile_transport_revisions",
return_value={"smtp": "smtp-current", "imap": "imap-current"},
),
patch("govoplan_mail.backend.capabilities.assert_mail_policy_allows_send"),
patch(
"govoplan_mail.backend.capabilities.append_message_to_sent",
side_effect=ImapAppendError(
"imap.internal.example provider-secret",
outcome_unknown=True,
),
),
):
with self.assertRaises(ImapAppendError) as captured:
append_campaign_message_to_sent(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
message_bytes=b"message",
folder="Sent",
expected_smtp_transport_revision="smtp-current",
expected_imap_transport_revision="imap-current",
)
self.assertTrue(captured.exception.outcome_unknown)
self.assertFalse(captured.exception.temporary)
self.assertIn("inspect the mailbox", str(captured.exception))
self.assertNotIn("internal.example", str(captured.exception))
self.assertNotIn("provider-secret", str(captured.exception))
def test_provider_exception_is_sanitized_and_bounded(self) -> None:
profile = _profile()
smtp = SmtpConfig(host="smtp.internal.example", port=587)
with (
patch(
"govoplan_mail.backend.capabilities._authorized_campaign_profile",
return_value=profile,
),
patch("govoplan_mail.backend.capabilities.smtp_config_from_profile", return_value=smtp),
patch(
"govoplan_mail.backend.capabilities.campaign_profile_transport_revisions",
return_value={"smtp": "current", "imap": None},
),
patch("govoplan_mail.backend.capabilities.assert_mail_policy_allows_send"),
patch(
"govoplan_mail.backend.capabilities.send_email_bytes",
side_effect=SmtpSendError(
"smtp.internal.example provider-secret " + ("x" * 1_000),
temporary=True,
),
),
):
with self.assertRaises(SmtpSendError) as captured:
send_campaign_email_bytes(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
message_bytes=b"message",
envelope_from="sender@example.test",
envelope_recipients=["recipient@example.test"],
from_header="sender@example.test",
expected_smtp_transport_revision="current",
)
message = str(captured.exception)
self.assertEqual(message, "Mail delivery failed temporarily.")
self.assertNotIn("internal.example", message)
self.assertLessEqual(len(message), 80)
self.assertIsNone(captured.exception.__cause__)
def test_unclassified_provider_exception_is_outcome_unknown(self) -> None:
profile = _profile()
smtp = SmtpConfig(host="smtp.internal.example", port=587)
with (
patch(
"govoplan_mail.backend.capabilities._authorized_campaign_profile",
return_value=profile,
),
patch("govoplan_mail.backend.capabilities.smtp_config_from_profile", return_value=smtp),
patch(
"govoplan_mail.backend.capabilities.campaign_profile_transport_revisions",
return_value={"smtp": "current", "imap": None},
),
patch("govoplan_mail.backend.capabilities.assert_mail_policy_allows_send"),
patch(
"govoplan_mail.backend.capabilities.send_email_bytes",
side_effect=RuntimeError("smtp.internal.example provider-secret"),
),
):
with self.assertRaises(SmtpSendError) as captured:
send_campaign_email_bytes(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
profile_id="profile-1",
message_bytes=b"message",
envelope_from="sender@example.test",
envelope_recipients=["recipient@example.test"],
from_header="sender@example.test",
expected_smtp_transport_revision="current",
)
self.assertTrue(captured.exception.outcome_unknown)
self.assertNotIn("internal.example", str(captured.exception))
self.assertIsNone(captured.exception.__cause__)
def test_capability_does_not_export_raw_profile_or_transport_helpers(self) -> None:
capability = MailCampaignCapability()
for name in (
"smtp_config_from_profile",
"imap_config_from_profile",
"send_email_bytes",
"send_email_message",
"materialize_campaign_mail_profile_config",
):
self.assertFalse(hasattr(capability, name), name)
if __name__ == "__main__":
unittest.main()
+388
View File
@@ -0,0 +1,388 @@
from __future__ import annotations
from pathlib import Path
from dataclasses import replace
import tempfile
import unittest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from govoplan_access.backend.db import models as access_models # noqa: F401
from govoplan_core.core.configuration_packages import (
ConfigurationPackageFragment,
ConfigurationPreflightContext,
)
from govoplan_core.admin.models import SystemSettings
from govoplan_core.core.infrastructure_capabilities import (
infrastructure_capability_receipt_from_mapping,
)
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, reset_database
from govoplan_core.security.credential_envelopes import CredentialEnvelope
from govoplan_mail.backend.configuration_provider import (
MAIL_CONFIGURATION_CAPABILITY,
SqlMailConfigurationProvider,
)
from govoplan_mail.backend.db.models import (
MailProfilePolicy,
MailServerCredentialBinding,
MailServerEndpoint,
MailServerProfile,
)
from govoplan_mail.backend.manifest import manifest
def _receipt():
return infrastructure_capability_receipt_from_mapping(
{
"schema_version": 1,
"installation_id": "mail-provider-test",
"profile": "evaluation",
"capabilities": [
{
"id": "mail.smtp",
"label": "SMTP delivery",
"state": "available_unconfigured",
"source": "installer-managed-test",
"detail": "GreenMail is available for profile binding.",
"endpoint": {
"scheme": "smtp",
"host": "test-mail",
"port": 3025,
},
"secret_refs": [],
"dependent_modules": ["mail"],
}
],
"post_install_tasks": [
{
"id": "mail.smtp-profile",
"resume_key": "mail-provider-test:mail.smtp-profile:v1",
"capability_id": "mail.smtp",
"state": "pending",
"owner_module": "mail",
"summary": "Create a Mail SMTP profile.",
"required_inputs": ["credential envelope reference when required"],
"secret_boundary": "credential-envelope-reference-only",
}
],
}
)
def _external_receipt(*, credential_required: bool = False):
return infrastructure_capability_receipt_from_mapping(
{
"schema_version": 1,
"installation_id": "mail-provider-external",
"profile": "production",
"capabilities": [
{
"id": "mail.smtp",
"label": "SMTP delivery",
"state": "available_unconfigured",
"source": "operator-supplied",
"detail": "An external relay needs reviewed Mail configuration.",
"endpoint": {},
"secret_refs": (
["env:SMTP_CREDENTIAL_ENVELOPE_REF"]
if credential_required
else []
),
"dependent_modules": ["mail"],
}
],
"post_install_tasks": [],
}
)
class MailConfigurationProviderTests(unittest.TestCase):
def setUp(self) -> None:
self.tempdir = tempfile.TemporaryDirectory(prefix="govoplan-mail-config-")
self.addCleanup(self.tempdir.cleanup)
database_path = Path(self.tempdir.name) / "mail.sqlite3"
self.engine = create_engine(f"sqlite:///{database_path}")
Base.metadata.create_all(
self.engine,
tables=(
access_models.User.__table__,
SystemSettings.__table__,
CredentialEnvelope.__table__,
MailServerProfile.__table__,
MailServerEndpoint.__table__,
MailServerCredentialBinding.__table__,
MailProfilePolicy.__table__,
),
)
configure_database(
f"sqlite:///{database_path}",
engine=self.engine,
dispose_previous=True,
)
self.SessionLocal = sessionmaker(
bind=self.engine,
class_=Session,
expire_on_commit=False,
)
with self.SessionLocal() as session:
session.add(
MailProfilePolicy(
id="tenant-policy",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
policy={},
)
)
session.add(
CredentialEnvelope(
id="credential-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="SMTP credential",
credential_kind="username_password",
public_data={"username": "mailer"},
secret_data_encrypted="encrypted-outside-package",
secret_keys=["password"],
allowed_modules=["mail"],
allowed_server_refs=[],
inherit_to_lower_scopes=True,
is_active=True,
)
)
session.commit()
self.addCleanup(self._cleanup_database)
self.provider = SqlMailConfigurationProvider()
self.context = ConfigurationPreflightContext(
tenant_id="tenant-1",
operator_user_id=None,
infrastructure_receipt=_receipt(),
)
def _cleanup_database(self) -> None:
reset_database()
self.engine.dispose()
def test_provider_is_registered_and_describes_receipt_bound_fragment(self) -> None:
self.assertIn(MAIL_CONFIGURATION_CAPABILITY, manifest.capability_factories)
description = self.provider.describe()
self.assertEqual(("smtp_profile",), description.fragment_types)
def test_apply_is_idempotent_and_binds_existing_credential_envelope(self) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
fragment_id="test-smtp",
payload={"credential_envelope_id": "credential-1"},
)
first_plan = self.provider.preflight(fragment, self.context)
self.assertEqual("create", first_plan.plan[0].action)
self.assertFalse(
any(item.severity == "blocker" for item in first_plan.diagnostics)
)
first_apply = self.provider.apply(fragment, {}, self.context)
self.assertIn("test-smtp", first_apply.created_refs)
second_plan = self.provider.preflight(fragment, self.context)
second_apply = self.provider.apply(fragment, {}, self.context)
self.assertEqual("skip", second_plan.plan[0].action)
self.assertEqual({}, second_apply.created_refs)
self.assertEqual({}, second_apply.updated_refs)
with self.SessionLocal() as session:
profiles = session.scalars(select(MailServerProfile)).all()
servers = session.scalars(select(MailServerEndpoint)).all()
bindings = session.scalars(select(MailServerCredentialBinding)).all()
self.assertEqual(1, len(profiles))
self.assertEqual("test-mail", profiles[0].smtp_config["host"])
self.assertEqual("plain", profiles[0].smtp_config["security"])
self.assertEqual(1, len(servers))
self.assertEqual("test-mail", servers[0].config["host"])
self.assertEqual(1, len(bindings))
self.assertEqual("credential-1", bindings[0].credential_id)
def test_conflicting_existing_profile_is_preserved_by_default(self) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
fragment_id="test-smtp",
payload={},
)
self.provider.apply(fragment, {}, self.context)
with self.SessionLocal() as session:
server = session.scalar(select(MailServerEndpoint))
assert server is not None
server.config = {**server.config, "host": "manually-changed.example.test"}
session.commit()
plan = self.provider.preflight(fragment, self.context)
result = self.provider.apply(fragment, {}, self.context)
self.assertEqual("blocked", plan.plan[0].action)
self.assertIn(
"mail_configuration_conflict",
{item.code for item in result.diagnostics},
)
with self.SessionLocal() as session:
server = session.scalar(select(MailServerEndpoint))
assert server is not None
self.assertEqual("manually-changed.example.test", server.config["host"])
def test_explicit_conflict_update_reconciles_then_becomes_noop(self) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
fragment_id="test-smtp",
payload={},
)
self.provider.apply(fragment, {}, self.context)
with self.SessionLocal() as session:
server = session.scalar(select(MailServerEndpoint))
assert server is not None
server.config = {**server.config, "host": "manually-changed.example.test"}
session.commit()
update_fragment = replace(
fragment,
payload={"on_conflict": "update"},
)
plan = self.provider.preflight(update_fragment, self.context)
result = self.provider.apply(update_fragment, {}, self.context)
settled = self.provider.preflight(update_fragment, self.context)
self.assertEqual("update", plan.plan[0].action)
self.assertIn("test-smtp", result.updated_refs)
self.assertEqual("skip", settled.plan[0].action)
with self.SessionLocal() as session:
server = session.scalar(select(MailServerEndpoint))
assert server is not None
self.assertEqual("test-mail", server.config["host"])
def test_inline_credentials_are_rejected(self) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
payload={
"smtp": {
"host": "test-mail",
"port": 3025,
"security": "plain",
"password": "must-not-cross-boundary",
}
},
)
plan = self.provider.preflight(fragment, self.context)
self.assertEqual("blocked", plan.plan[0].action)
self.assertIn(
"mail_configuration_secret_forbidden",
{item.code for item in plan.diagnostics},
)
def test_external_relay_collects_missing_non_secret_inputs(self) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
fragment_id="external-smtp",
payload={},
)
missing = self.provider.preflight(
fragment,
replace(self.context, infrastructure_receipt=_external_receipt()),
)
ready = self.provider.preflight(
fragment,
replace(
self.context,
infrastructure_receipt=_external_receipt(),
supplied_data={
"mail.smtp.external-smtp.host": "smtp.example.test",
"mail.smtp.external-smtp.port": 587,
"mail.smtp.external-smtp.security": "starttls",
},
),
)
self.assertEqual("blocked", missing.plan[0].action)
self.assertEqual(
{
"mail.smtp.external-smtp.host",
"mail.smtp.external-smtp.port",
"mail.smtp.external-smtp.security",
},
{item.key for item in missing.required_data if item.required},
)
self.assertEqual("create", ready.plan[0].action)
def test_receipt_secret_reference_requires_credential_envelope_reference(
self,
) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
fragment_id="authenticated-smtp",
payload={
"smtp": {
"host": "smtp.example.test",
"port": 587,
"security": "starttls",
}
},
)
context = replace(
self.context,
infrastructure_receipt=_external_receipt(credential_required=True),
)
missing = self.provider.preflight(fragment, context)
ready = self.provider.preflight(
ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
fragment_id="authenticated-smtp",
payload={
**fragment.payload,
"credential_envelope_id": "credential-1",
},
),
context,
)
self.assertEqual("blocked", missing.plan[0].action)
self.assertIn(
"mail.smtp.authenticated-smtp.credential_envelope_id",
{item.key for item in missing.required_data if item.required},
)
self.assertEqual("create", ready.plan[0].action)
def test_system_profile_requires_system_configuration_authority(self) -> None:
fragment = ConfigurationPackageFragment(
module_id="mail",
fragment_type="smtp_profile",
payload={"profile": {"scope_type": "system"}},
)
blocked = self.provider.preflight(fragment, self.context)
ready = self.provider.preflight(
fragment,
replace(
self.context,
operator_scopes=frozenset({"system:settings:write"}),
),
)
self.assertEqual("blocked", blocked.plan[0].action)
self.assertIn(
"system_configuration_authority_required",
{item.code for item in blocked.diagnostics},
)
self.assertEqual("create", ready.plan[0].action)
if __name__ == "__main__":
unittest.main()
+294
View File
@@ -0,0 +1,294 @@
from __future__ import annotations
import unittest
from datetime import timedelta
from types import SimpleNamespace
from unittest.mock import patch
from sqlalchemy import Column, String, Table, create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from govoplan_core.db.base import Base
from govoplan_mail.backend.db.models import (
MailDeliveryAttempt,
MailDeliveryCommand,
MailDeliveryReconciliation,
MailServerProfile,
)
from govoplan_mail.backend.delivery_outbox import (
MailDeliveryIdempotencyConflict,
delivery_command_diagnostics,
dispatch_due,
purge_expired,
submit_delivery_command,
utcnow,
)
from govoplan_mail.backend.sending.smtp import SmtpSendError
class MailDeliveryOutboxTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine(
"sqlite+pysqlite://",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
access_users = Base.metadata.tables.get("access_users")
if access_users is None:
access_users = Table(
"access_users",
Base.metadata,
Column("id", String(36), primary_key=True),
)
Base.metadata.create_all(
self.engine,
tables=[
access_users,
MailServerProfile.__table__,
MailDeliveryCommand.__table__,
MailDeliveryAttempt.__table__,
MailDeliveryReconciliation.__table__,
],
)
self.SessionLocal = sessionmaker(
bind=self.engine,
class_=Session,
expire_on_commit=False,
)
with self.SessionLocal() as session:
session.add(
MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Delivery",
slug="delivery",
smtp_config={"host": "smtp.example.test", "port": 25},
)
)
session.commit()
self.audit = patch(
"govoplan_mail.backend.delivery_outbox.audit_event"
)
self.audit.start()
self.addCleanup(self.audit.stop)
self.addCleanup(self.engine.dispose)
def _submit(
self,
session: Session,
*,
key: str = "request-1",
recipients: list[str] | None = None,
) -> dict[str, object]:
return submit_delivery_command(
session,
tenant_id="tenant-1",
command_type="campaign_report",
source_module="campaigns",
source_resource_type="campaign",
source_resource_id="campaign-1",
source_version_id="version-1",
idempotency_key=key,
profile_id="profile-1",
message_bytes=b"Subject: report\r\n\r\ncontent",
envelope_from="sender@example.test",
envelope_recipients=recipients or ["recipient@example.test"],
from_header="Sender <sender@example.test>",
expected_smtp_transport_revision="revision-1",
created_by_user_id=None,
)
def test_submission_is_idempotent_and_conflicts_on_changed_intent(self) -> None:
with self.SessionLocal() as session:
first = self._submit(session)
session.commit()
repeated = self._submit(session)
self.assertEqual(first["id"], repeated["id"])
self.assertTrue(repeated["duplicate"])
with self.assertRaises(MailDeliveryIdempotencyConflict):
self._submit(
session,
recipients=["different@example.test"],
)
def test_attempt_is_committed_before_effect_and_accepted_is_not_resent(self) -> None:
observed: dict[str, object] = {}
with self.SessionLocal() as session:
command_id = str(self._submit(session)["id"])
session.commit()
def provider_effect(session: Session, **_kwargs):
command = session.get(MailDeliveryCommand, command_id)
attempt = session.query(MailDeliveryAttempt).one()
observed.update(
command_status=command.status,
effect_started=command.effect_started_at is not None,
attempt_status=attempt.status,
)
return SimpleNamespace(
accepted_count=1,
refused_recipients={},
)
with (
self.SessionLocal() as session,
patch(
"govoplan_mail.backend.delivery_outbox.send_campaign_email_bytes",
side_effect=provider_effect,
) as send,
):
result = dispatch_due(session, worker_id="worker-1")
repeated = dispatch_due(session, worker_id="worker-2")
self.assertEqual(result["accepted"], 1)
self.assertEqual(repeated["selected"], 0)
self.assertEqual(send.call_count, 1)
self.assertEqual(
observed,
{
"command_status": "in_progress",
"effect_started": True,
"attempt_status": "in_progress",
},
)
def test_partial_refusal_is_terminal_and_diagnostics_are_separate(self) -> None:
with self.SessionLocal() as session:
command_id = str(
self._submit(
session,
recipients=["accepted@example.test", "blocked@example.test"],
)["id"]
)
session.commit()
with patch(
"govoplan_mail.backend.delivery_outbox.send_campaign_email_bytes",
return_value=SimpleNamespace(
accepted_count=1,
refused_recipients={
"blocked@example.test": {
"status_code": 550,
"classification": "permanent",
"message": "Permanent recipient rejection",
}
},
),
):
result = dispatch_due(session)
command = session.get(MailDeliveryCommand, command_id)
assert command is not None
self.assertEqual(result["partially_refused"], 1)
self.assertNotIn("blocked@example.test", repr({
"status": command.status,
"summary": command.refusal_summary,
}))
diagnostics = delivery_command_diagnostics(
session,
tenant_id="tenant-1",
command_id=command_id,
)
self.assertIn(
"blocked@example.test",
diagnostics["refused_recipients"],
)
def test_unknown_outcome_is_never_automatically_retried(self) -> None:
with self.SessionLocal() as session:
command_id = str(self._submit(session)["id"])
session.commit()
with patch(
"govoplan_mail.backend.delivery_outbox.send_campaign_email_bytes",
side_effect=SmtpSendError(
"unknown",
outcome_unknown=True,
),
) as send:
first = dispatch_due(session)
second = dispatch_due(session)
command = session.get(MailDeliveryCommand, command_id)
assert command is not None
self.assertEqual(first["outcome_unknown"], 1)
self.assertEqual(second["selected"], 0)
self.assertEqual(command.status, "outcome_unknown")
self.assertEqual(send.call_count, 1)
def test_audit_failure_after_acceptance_cannot_make_command_retryable(self) -> None:
with self.SessionLocal() as session:
command_id = str(self._submit(session)["id"])
session.commit()
with (
self.SessionLocal() as session,
patch(
"govoplan_mail.backend.delivery_outbox.send_campaign_email_bytes",
return_value=SimpleNamespace(
accepted_count=1,
refused_recipients={},
),
) as send,
patch(
"govoplan_mail.backend.delivery_outbox.audit_event",
side_effect=RuntimeError("audit unavailable"),
),
):
with self.assertRaisesRegex(RuntimeError, "audit unavailable"):
dispatch_due(session)
with self.SessionLocal() as session:
command = session.get(MailDeliveryCommand, command_id)
assert command is not None
self.assertEqual(command.status, "accepted")
self.assertEqual(dispatch_due(session)["selected"], 0)
self.assertEqual(send.call_count, 1)
def test_stale_effect_started_command_becomes_unknown_without_send(self) -> None:
with self.SessionLocal() as session:
command_id = str(self._submit(session)["id"])
session.commit()
command = session.get(MailDeliveryCommand, command_id)
assert command is not None
command.status = "in_progress"
command.attempt_count = 1
command.claimed_at = utcnow() - timedelta(hours=1)
command.effect_started_at = utcnow() - timedelta(hours=1)
session.add(
MailDeliveryAttempt(
command_id=command.id,
attempt_number=1,
status="in_progress",
started_at=utcnow() - timedelta(hours=1),
effect_started_at=utcnow() - timedelta(hours=1),
)
)
session.commit()
with patch(
"govoplan_mail.backend.delivery_outbox.send_campaign_email_bytes"
) as send:
result = dispatch_due(session)
self.assertEqual(result["outcome_unknown"], 1)
self.assertEqual(command.status, "outcome_unknown")
send.assert_not_called()
def test_expired_payload_is_minimized_but_evidence_remains(self) -> None:
with self.SessionLocal() as session:
command_id = str(self._submit(session)["id"])
session.commit()
command = session.get(MailDeliveryCommand, command_id)
assert command is not None
command.expires_at = utcnow() - timedelta(seconds=1)
session.commit()
self.assertEqual(purge_expired(session), {"purged": 1})
session.refresh(command)
self.assertIsNone(command.message_encrypted)
self.assertIsNone(command.envelope_recipients_encrypted)
self.assertIsNotNone(command.payload_purged_at)
self.assertEqual(command.message_sha256, command.message_sha256)
if __name__ == "__main__":
unittest.main()
+233
View File
@@ -0,0 +1,233 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from sqlalchemy.orm import Session
from govoplan_core.core.modules import DocumentationContext
from govoplan_mail.backend.documentation import (
documentation_configuration_states,
documentation_topics,
)
from govoplan_mail.backend.mail_profiles import EffectiveMailProfilePolicy, MailProfileError
class _Principal:
tenant_id = "tenant-1"
user = SimpleNamespace(id="user-1")
def __init__(self, scopes: set[str]) -> None:
self.scopes = frozenset(scopes)
def has(self, scope: str) -> bool:
return scope in self.scopes
class MailRuntimeDocumentationTests(unittest.TestCase):
def setUp(self) -> None:
self.session = Session()
def tearDown(self) -> None:
self.session.close()
def context(self, scopes: set[str], *, documentation_type: str = "user") -> DocumentationContext:
return DocumentationContext(
registry=object(),
principal=_Principal(scopes),
settings=None,
session=self.session,
documentation_type=documentation_type, # type: ignore[arg-type]
)
def topics(self, context: DocumentationContext, user_policy: EffectiveMailProfilePolicy) -> dict[str, object]:
tenant_policy = EffectiveMailProfilePolicy()
def effective_policy(_session, *, scope_type, **_kwargs):
return user_policy if scope_type == "user" else tenant_policy
with patch(
"govoplan_mail.backend.documentation.effective_mail_profile_policy_for_scope",
side_effect=effective_policy,
):
return {topic.id: topic for topic in documentation_topics(context)}
def test_custom_profile_task_requires_write_authority_and_enabled_user_profiles(self) -> None:
enabled = EffectiveMailProfilePolicy(allow_user_profiles=True)
without_write = self.topics(self.context({"mail:profile:read"}), enabled)
self.assertNotIn("mail.workflow.create-custom-profile", without_write)
without_read = self.topics(self.context({"mail:profile:write"}), enabled)
self.assertNotIn("mail.workflow.create-custom-profile", without_read)
disabled = EffectiveMailProfilePolicy(allow_user_profiles=False)
blocked = self.topics(self.context({"mail:profile:read", "mail:profile:write"}), disabled)
self.assertNotIn("mail.workflow.create-custom-profile", blocked)
available = self.topics(self.context({"mail:profile:read", "mail:profile:write"}), enabled)
self.assertIn("mail.workflow.create-custom-profile", available)
task = available["mail.workflow.create-custom-profile"]
self.assertEqual(task.title, "Create a custom Mail profile")
self.assertEqual(task.metadata["route"], "/settings?section=mail-profiles")
self.assertEqual(task.metadata["help_contexts"], ["mail.profiles", "app.settings"])
self.assertIn("current account's user scope", task.metadata["prerequisites"][0])
self_service = self.topics(
self.context(
{"mail:profile:read", "mail:profile:write_own"}
),
enabled,
)
self.assertIn("mail.workflow.create-custom-profile", self_service)
condition = self_service["mail.workflow.create-custom-profile"].conditions[0]
self.assertEqual(condition.required_scopes, ("mail:profile:read",))
self.assertEqual(
condition.any_scopes,
("mail:profile:write", "mail:profile:write_own"),
)
def test_custom_profile_task_distinguishes_secret_test_and_use_authority(self) -> None:
policy = EffectiveMailProfilePolicy()
cases = (
(set(), False, False, False),
({"mail:secret:manage"}, True, False, False),
({"mail:profile:test"}, False, False, False),
({"mail:profile:use"}, False, False, True),
({"mail:profile:test", "mail:profile:use"}, False, True, True),
({"mail:secret:manage", "mail:profile:test", "mail:profile:use"}, True, True, True),
({"mail:secret:manage_own"}, True, False, False),
)
for extra_scopes, credentials, testing, use in cases:
with self.subTest(extra_scopes=extra_scopes):
topics = self.topics(self.context({"mail:profile:read", "mail:profile:write", *extra_scopes}), policy)
task = topics["mail.workflow.create-custom-profile"]
self.assertEqual(task.metadata["can_manage_credentials"], credentials)
self.assertEqual(task.metadata["can_test_profile"], testing)
self.assertEqual(task.metadata["can_use_profile"], use)
self.assertIn(
"you may save or replace" if credentials else "you cannot save or replace passwords",
task.body,
)
self.assertIn(
"you may run" if testing else "does not let you run connection tests",
task.body,
)
self.assertIn(
"you may select" if use else "does not let you select or use it",
task.body,
)
def test_custom_profile_task_preserves_host_allow_group_and_deny_precedence(self) -> None:
policy = EffectiveMailProfilePolicy(
whitelist_groups={
"smtp_hosts": [
["*.example.edu", "smtp.shared.test"],
["smtp-*.example.edu"],
],
"imap_hosts": [["imap.example.edu"]],
},
blacklist_patterns={
"smtp_hosts": ["smtp-blocked.example.edu"],
"imap_hosts": ["imap-legacy.example.edu"],
},
allowed_profile_id_sets=[{"private-profile-id"}],
)
topics = self.topics(
self.context({"mail:profile:read", "mail:profile:write", "mail:secret:manage", "mail:profile:test", "mail:profile:use"}),
policy,
)
task = topics["mail.workflow.create-custom-profile"]
constraints = {item["id"]: item for item in task.metadata["constraints"]}
constraints_text = " ".join(item["description"] for item in task.metadata["constraints"])
self.assertIn("Deny rules are checked first", constraints_text)
self.assertEqual(constraints["smtp-host-deny"]["values"], ["smtp-blocked.example.edu"])
self.assertEqual(constraints["smtp-host-allow-1"]["values"], ["*.example.edu", "smtp.shared.test"])
self.assertEqual(constraints["smtp-host-allow-2"]["values"], ["smtp-*.example.edu"])
self.assertIn("every active allow-list group", constraints_text)
self.assertEqual(constraints["imap-host-allow-1"]["values"], ["imap.example.edu"])
self.assertTrue(task.metadata["approval_required_before_use"])
self.assertIn("must be approved before the profile can be selected or used", task.body)
self.assertNotIn("private-profile-id", repr(task))
self.assertIn("every active allow-list group", constraints["smtp-host-allow-1"]["description"])
def test_custom_profile_task_explains_when_no_approval_list_is_active(self) -> None:
topics = self.topics(
self.context({"mail:profile:read", "mail:profile:write", "mail:profile:use"}),
EffectiveMailProfilePolicy(),
)
task = topics["mail.workflow.create-custom-profile"]
self.assertFalse(task.metadata["approval_required_before_use"])
self.assertIn("no approved-profile list currently blocks", task.body)
def test_user_policy_errors_are_generic_and_never_create_a_task(self) -> None:
context = self.context({"mail:profile:read", "mail:profile:write"})
with patch(
"govoplan_mail.backend.documentation.effective_mail_profile_policy_for_scope",
side_effect=MailProfileError("sensitive source-id profile-id username secret"),
):
topics = {topic.id: topic for topic in documentation_topics(context)}
self.assertNotIn("mail.workflow.create-custom-profile", topics)
unavailable = topics["mail.tenant-profile-policy-unavailable"]
self.assertEqual(
unavailable.body,
"The current Mail policy could not be loaded. Try again or ask a Mail administrator for help.",
)
self.assertNotIn("sensitive", repr(unavailable))
def test_manifest_workflows_declare_contextual_help_targets(self) -> None:
from govoplan_mail.backend.manifest import get_manifest
topics = {topic.id: topic for topic in get_manifest().documentation}
self.assertEqual(topics["mail.workflow.choose-and-test-profile"].metadata["help_contexts"], ["mail.profiles", "app.settings"])
self.assertEqual(topics["mail.workflow.read-mailbox"].metadata["help_contexts"], ["mail.list", "mail.mailbox"])
self.assertIn("mail.admin.profiles", topics["mail.profiles-and-policy"].metadata["help_contexts"])
self.assertIn("mail.bounce-processing", topics["mail.bounce-processing"].metadata["help_contexts"])
def test_configuration_provider_exposes_only_explicit_or_inherited_state(self) -> None:
inherited = EffectiveMailProfilePolicy(
source_policies=[
{"scope_type": "system", "applied_fields": ["defaults"]},
{"scope_type": "tenant", "applied_fields": []},
]
)
explicit = EffectiveMailProfilePolicy(
source_policies=[
{"scope_type": "system", "applied_fields": ["defaults"]},
{"scope_type": "tenant", "applied_fields": ["smtp_hosts"]},
]
)
context = self.context({"mail:profile:read"}, documentation_type="admin")
with patch(
"govoplan_mail.backend.documentation.effective_mail_profile_policy_for_scope",
return_value=inherited,
):
inherited_state = documentation_configuration_states(
context,
("mail_profile_policy",),
)
with patch(
"govoplan_mail.backend.documentation.effective_mail_profile_policy_for_scope",
return_value=explicit,
):
explicit_state = documentation_configuration_states(
context,
("mail_profile_policy",),
)
self.assertEqual(
inherited_state["mail_profile_policy"].state,
"inherited",
)
self.assertEqual(
explicit_state["mail_profile_policy"].state,
"enabled",
)
self.assertNotIn("smtp_hosts", repr(explicit_state))
if __name__ == "__main__":
unittest.main()
+419
View File
@@ -0,0 +1,419 @@
from __future__ import annotations
import unittest
from datetime import datetime, timedelta, timezone
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef
from govoplan_core.db.base import Base
from govoplan_core.privacy.dsar_workflow import (
DataSubjectRequest,
create_data_subject_request,
plan_data_subject_erasure,
search_data_subject_request,
)
from govoplan_mail.backend.db.models import (
MailBounceObservation,
MailDeliveryAttempt,
MailDeliveryCommand,
MailDeliveryReconciliation,
MailMailboxMessageIndex,
MailPop3Import,
MailServerEndpoint,
MailServerProfile,
)
from govoplan_mail.backend.dsar_provider import MAIL_DSAR_CAPABILITY, MailDsarProvider
from govoplan_mail.backend.manifest import manifest
class _Registry:
def __init__(self, provider, active=True):
self.provider = provider
self.active = active
def capability_names(self):
return (MAIL_DSAR_CAPABILITY,)
def capability_owner(self, name):
assert name == MAIL_DSAR_CAPABILITY
return "mail"
def tenant_entitlement_resolver(self):
active = self.active
class Resolver:
@staticmethod
def resolve(session, tenant_id):
del session, tenant_id
return type(
"State", (), {"effective_modules": ("mail",) if active else ()}
)()
return Resolver()
def require_tenant_capability(self, name, session, **kwargs):
del session, kwargs
assert name == MAIL_DSAR_CAPABILITY
return self.provider
class MailDsarProviderTests(unittest.TestCase):
def setUp(self):
self.engine = create_engine("sqlite:///:memory:", future=True)
Base.metadata.create_all(
self.engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
ChangeSequenceEntry.__table__,
DataSubjectRequest.__table__,
MailServerProfile.__table__,
MailServerEndpoint.__table__,
MailMailboxMessageIndex.__table__,
MailPop3Import.__table__,
MailDeliveryCommand.__table__,
MailDeliveryAttempt.__table__,
MailDeliveryReconciliation.__table__,
MailBounceObservation.__table__,
],
)
self.session = sessionmaker(bind=self.engine, future=True)()
now = datetime.now(timezone.utc)
account = Account(
id="account-1",
email="subject@example.test",
normalized_email="subject@example.test",
display_name="Subject",
)
user = User(
id="membership-1",
tenant_id="tenant-1",
account_id=account.id,
email="subject@example.test",
display_name="Subject",
)
profile = MailServerProfile(
id="profile-subject",
tenant_id="tenant-1",
scope_type="user",
scope_id=user.id,
name="Personal mail",
slug="personal",
smtp_config={"host": "smtp-secret-do-not-export"},
smtp_username="smtp-user-do-not-export",
smtp_password_encrypted="smtp-cipher-do-not-export",
imap_config={"host": "imap-secret-do-not-export"},
imap_username="imap-user-do-not-export",
imap_password_encrypted="imap-cipher-do-not-export",
created_by_user_id=user.id,
)
message = MailMailboxMessageIndex(
id="message-subject",
tenant_id="tenant-1",
profile_id=profile.id,
folder="INBOX-secret-do-not-export",
uid="uid-secret-do-not-export",
uid_int=1,
sort_position=1,
subject="Subject notice",
from_header="Office <office@example.test>",
to_header="Subject Person <Subject@Example.Test>",
cc_header="Unrelated Person <other@example.test>",
date="2026-08-20",
message_id="message-locator-do-not-export",
flags=["\\Seen"],
size_bytes=42,
body_preview="Message preview for the subject",
attachment_count=1,
indexed_at=now,
)
unrelated = MailMailboxMessageIndex(
id="message-other",
tenant_id="tenant-1",
profile_id=profile.id,
folder="INBOX",
uid="2",
uid_int=2,
sort_position=2,
subject="Unrelated message do not export",
from_header="other@example.test",
to_header="someone@example.test",
indexed_at=now,
)
tenant_two_profile = MailServerProfile(
id="profile-tenant-2",
tenant_id="tenant-2",
name="Tenant two",
slug="tenant-two",
smtp_config={},
)
tenant_two = MailMailboxMessageIndex(
id="message-tenant-2",
tenant_id="tenant-2",
profile_id=tenant_two_profile.id,
folder="INBOX",
uid="1",
uid_int=1,
sort_position=1,
subject="Tenant two message do not export",
to_header="subject@example.test",
indexed_at=now,
)
pop3_server = MailServerEndpoint(
id="pop3-server-subject",
profile_id=profile.id,
tenant_id="tenant-1",
protocol="pop3",
name="Legacy POP3",
config={"host": "pop3-secret-do-not-export"},
scope_type="tenant",
scope_id="tenant-1",
)
pop3_import = MailPop3Import(
id="pop3-import-subject",
tenant_id="tenant-1",
profile_id=profile.id,
pop3_server_id=pop3_server.id,
pop3_credential_id="credential-secret-do-not-export",
transport_revision="pop3-revision-secret-do-not-export",
provider_uidl="provider-uidl-secret-do-not-export",
fingerprint="e" * 64,
raw_sha256="f" * 64,
raw_message_encrypted="pop3-message-cipher-do-not-export",
message_id="pop3-message-id",
subject="Imported subject notice",
from_header="Legacy office <legacy@example.test>",
to_header="Subject Person <subject@example.test>",
date="2026-08-19",
body_preview="Imported message preview for the subject",
size_bytes=84,
status="pending_review",
imported_at=now,
deletion_requested=False,
deletion_status="not_requested",
)
command = MailDeliveryCommand(
id="command-subject",
tenant_id="tenant-1",
command_type="send",
source_module="notifications",
source_resource_type="notification",
idempotency_key="idempotency-do-not-export",
canonical_request_hash="a" * 64,
profile_id=profile.id,
expected_smtp_transport_revision="revision-secret",
envelope_recipients_encrypted="recipient-cipher-do-not-export",
message_encrypted="message-cipher-do-not-export",
message_sha256="b" * 64,
rfc_message_id="rfc-message-id",
message_size_bytes=100,
recipient_count=1,
status="succeeded",
attempt_count=1,
accepted_count=1,
created_by_user_id=user.id,
completed_at=now,
expires_at=now + timedelta(days=30),
)
attempt = MailDeliveryAttempt(
id="attempt-subject",
command_id=command.id,
attempt_number=1,
worker_id="worker-do-not-export",
status="succeeded",
started_at=now,
completed_at=now,
accepted_count=1,
diagnostic_summary="diagnostic-do-not-export",
)
reconciliation = MailDeliveryReconciliation(
id="reconciliation-subject",
command_id=command.id,
decision="confirmed_sent",
evidence_reference="private-reference-do-not-export",
note_encrypted="private-note-do-not-export",
created_by_user_id=user.id,
)
bounce = MailBounceObservation(
id="bounce-subject",
tenant_id="tenant-1",
profile_id=profile.id,
folder="bounce-folder-do-not-export",
uid="bounce-uid-do-not-export",
fingerprint="c" * 64,
raw_sha256="d" * 64,
original_message_id="original-id-do-not-export",
command_id=command.id,
recipient="subject@example.test",
action="failed",
status_code="5.1.1",
diagnostic="bounce-diagnostic-do-not-export",
permanent=True,
observed_at=now,
matched=True,
evidence={"secret": "bounce-evidence-do-not-export"},
)
self.session.add_all(
[
account,
user,
profile,
message,
unrelated,
tenant_two_profile,
tenant_two,
pop3_server,
pop3_import,
command,
attempt,
reconciliation,
bounce,
]
)
self.session.commit()
self.provider = MailDsarProvider()
self.subject = DsarSubjectRef(
membership_id=user.id, email="subject@example.test"
)
def tearDown(self):
self.session.close()
self.engine.dispose()
def test_manifest_and_minimized_tenant_scoped_search(self):
self.assertIn(
MAIL_DSAR_CAPABILITY, {item.name for item in manifest.provides_interfaces}
)
self.assertIsInstance(
manifest.capability_factories[MAIL_DSAR_CAPABILITY](None), DsarProvider
)
records = self.provider.search_subject(
self.session, tenant_id="tenant-1", subject=self.subject
)
self.assertTrue(
{
"mail_server_profile",
"mailbox_message_index",
"mail_delivery_command",
"mail_delivery_attempt",
"mail_delivery_reconciliation",
"mail_bounce_observation",
"mail_pop3_import",
}.issubset({r.resource_type for r in records})
)
serialized = repr([record.to_dict() for record in records])
for hidden in (
"other@example.test",
"Unrelated Person",
"message-other",
"Unrelated message do not export",
"message-tenant-2",
"Tenant two message do not export",
"smtp-secret-do-not-export",
"smtp-user-do-not-export",
"smtp-cipher-do-not-export",
"imap-secret-do-not-export",
"imap-user-do-not-export",
"imap-cipher-do-not-export",
"INBOX-secret-do-not-export",
"uid-secret-do-not-export",
"message-locator-do-not-export",
"idempotency-do-not-export",
"recipient-cipher-do-not-export",
"message-cipher-do-not-export",
"worker-do-not-export",
"diagnostic-do-not-export",
"private-reference-do-not-export",
"private-note-do-not-export",
"bounce-folder-do-not-export",
"bounce-uid-do-not-export",
"original-id-do-not-export",
"bounce-diagnostic-do-not-export",
"bounce-evidence-do-not-export",
"pop3-secret-do-not-export",
"credential-secret-do-not-export",
"pop3-revision-secret-do-not-export",
"provider-uidl-secret-do-not-export",
"pop3-message-cipher-do-not-export",
):
self.assertNotIn(hidden, serialized)
def test_conflicting_email_fails_closed(self):
records = self.provider.search_subject(
self.session,
tenant_id="tenant-1",
subject=DsarSubjectRef(
email="subject@example.test",
external_references={"mail.email": "other@example.test"},
),
)
self.assertEqual((), records)
def test_plan_preserves_evidence_and_executes_nothing(self):
records = self.provider.search_subject(
self.session, tenant_id="tenant-1", subject=self.subject
)
actions = self.provider.plan_erasure(
self.session, tenant_id="tenant-1", subject=self.subject, records=records
)
self.assertTrue({"retain", "manual_review"}.issubset({a.kind for a in actions}))
self.assertFalse(any(action.executable for action in actions))
def test_core_workflow_discovers_active_and_skips_disabled_provider(self):
request = create_data_subject_request(
self.session,
tenant_id="tenant-1",
reference="DSAR-MAIL-1",
request_kind="access_and_erasure",
subject=self.subject,
purpose="Authorized request",
legal_basis="GDPR",
due_at=None,
requested_by_account_id="officer",
)
self.session.commit()
search_data_subject_request(
self.session,
registry=_Registry(self.provider),
row=request,
expected_revision=1,
)
self.assertEqual(["mail"], request.coverage["covered_modules"])
plan_data_subject_erasure(
self.session,
registry=_Registry(self.provider),
row=request,
expected_revision=2,
)
self.assertFalse(
any(action["executable"] for action in request.erasure_plan["actions"])
)
disabled = create_data_subject_request(
self.session,
tenant_id="tenant-1",
reference="DSAR-MAIL-OFF",
request_kind="access",
subject=self.subject,
purpose="Coverage",
legal_basis=None,
due_at=None,
requested_by_account_id="officer",
)
search_data_subject_request(
self.session,
registry=_Registry(self.provider, active=False),
row=disabled,
expected_revision=1,
)
self.assertEqual(
[MAIL_DSAR_CAPABILITY], disabled.coverage["inactive_provider_capabilities"]
)
if __name__ == "__main__":
unittest.main()
+343
View File
@@ -0,0 +1,343 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from govoplan_core.security.outbound_http import OutboundHttpBlocked
from govoplan_mail.backend.config import ImapConfig
from govoplan_mail.backend.sending.imap import (
ImapAppendError,
ImapConfigurationError,
_detect_standard_folder_mappings,
_detect_sent_folder,
_extract_mailbox_name,
_fetch_message_by_uid,
_normalize_mailbox_page,
_open_imap,
_paged_descending_sequences,
_parse_fetch_sequence,
_select_readonly,
_sequence_set,
append_message_to_sent,
list_imap_folders,
list_imap_messages,
list_imap_uids_since,
)
class ImapFolderParserTests(unittest.TestCase):
def test_real_imap_connections_honor_deployment_egress_policy(self):
config = ImapConfig(host="imap.internal", port=993, security="tls")
with patch(
"govoplan_mail.backend.sending.imap.validate_outbound_host",
side_effect=OutboundHttpBlocked("private network blocked"),
), self.assertRaisesRegex(ImapConfigurationError, "private network blocked"):
_open_imap(config)
def test_imap_revalidates_and_pins_at_connection_time(self):
config = ImapConfig(host="imap.example.test", port=993, security="tls")
public = [(2, 1, 6, "", ("93.184.216.34", 993))]
private = [(2, 1, 6, "", ("127.0.0.1", 993))]
with patch.dict(
"os.environ",
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
side_effect=(public, private),
), patch("govoplan_core.security.outbound_http.socket.socket") as socket_factory, self.assertRaisesRegex(
OutboundHttpBlocked,
"non-public network",
):
_open_imap(config)
socket_factory.assert_not_called()
def test_extracts_quoted_mailbox_after_quoted_delimiter(self):
self.assertEqual(
_extract_mailbox_name(b'(\\HasNoChildren \\Sent) "/" "Sent Items"'),
("Sent Items", {"\\hasnochildren", "\\sent"}),
)
def test_extracts_atom_mailbox(self):
self.assertEqual(
_extract_mailbox_name('(\\HasNoChildren) "/" INBOX'),
("INBOX", {"\\hasnochildren"}),
)
def test_extracts_nil_delimiter_mailbox(self):
self.assertEqual(
_extract_mailbox_name('(\\HasNoChildren) NIL "INBOX.Sent"'),
("INBOX.Sent", {"\\hasnochildren"}),
)
def test_detects_sent_by_flag_before_name(self):
self.assertEqual(
_detect_sent_folder([("Archive", {"\\sent"}), ("Sent", set())]),
"Archive",
)
def test_detects_common_sent_name(self):
self.assertEqual(
_detect_sent_folder([("INBOX", set()), ("Gesendet", set())]),
"Gesendet",
)
def test_detects_all_standard_folder_roles_by_flag_then_name(self):
self.assertEqual(
{
"inbox": "INBOX",
"sent": "Gesendete Elemente",
"drafts": "Entwürfe",
"trash": "Deleted",
"archive": "All Mail",
"junk": "Spam",
},
_detect_standard_folder_mappings(
[
("INBOX", set()),
("Gesendete Elemente", set()),
("Entwürfe", set()),
("Deleted", {"\\trash"}),
("All Mail", {"\\all"}),
("Spam", {"\\junk"}),
]
),
)
def test_mock_folder_listing_exposes_detected_standard_mappings(self):
result = list_imap_folders(
imap_config=ImapConfig(host="mock.imap.local"), include_status=False
)
self.assertEqual(
{
"inbox": "INBOX",
"sent": "Sent",
"drafts": "Drafts",
"trash": "Trash",
"archive": "Archive",
},
result.detected_folder_mappings,
)
self.assertEqual("Sent", result.detected_sent_folder)
class ImapMessagePaginationTests(unittest.TestCase):
def test_mock_message_cursor_preserves_order_and_resets_when_stale(self):
records = [
{
"id": "3",
"kind": "smtp",
"raw_eml": "Subject: Three\r\n\r\nBody",
"size_bytes": 26,
},
{
"id": "2",
"kind": "smtp",
"raw_eml": "Subject: Two\r\n\r\nBody",
"size_bytes": 24,
},
]
config = ImapConfig(host="mock.imap.local")
with patch(
"govoplan_mail.backend.sending.imap.list_records",
return_value=records,
):
page = list_imap_messages(
imap_config=config,
after_uid="3",
expected_uidvalidity="mock-v1",
limit=1,
)
reset_page = list_imap_messages(
imap_config=config,
after_uid="3",
expected_uidvalidity="stale",
limit=1,
)
self.assertEqual([message.uid for message in page.messages], ["2"])
self.assertEqual(page.offset, 1)
self.assertFalse(page.cursor_reset)
self.assertEqual([message.uid for message in reset_page.messages], ["3"])
self.assertEqual(reset_page.offset, 0)
self.assertTrue(reset_page.cursor_reset)
def test_real_message_listing_delegates_to_shared_client_path(self):
class Client:
logged_out = False
def logout(self):
self.logged_out = True
client = Client()
expected = object()
config = ImapConfig(host="imap.example.org")
with (
patch(
"govoplan_mail.backend.sending.imap._open_imap",
return_value=client,
),
patch(
"govoplan_mail.backend.sending.imap._list_imap_messages_on_client",
return_value=expected,
) as list_on_client,
):
result = list_imap_messages(
imap_config=config,
folder=" Archive ",
limit=25,
offset=5,
after_uid="42",
expected_uidvalidity="7",
)
self.assertIs(result, expected)
self.assertTrue(client.logged_out)
list_on_client.assert_called_once_with(
client,
host="imap.example.org",
port=993,
security="tls",
folder="Archive",
limit=25,
offset=5,
after_uid="42",
expected_uidvalidity="7",
)
def test_full_message_fetch_uses_partial_range_and_deployment_limit(self):
class Client:
command = ""
def uid(self, command, uid, fetch_spec):
del command, uid
self.command = fetch_spec
return "OK", [(b"1 (UID 1 RFC822.SIZE 11)", b"12345678901")]
client = Client()
with patch.dict("os.environ", {"GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES": "10"}), self.assertRaisesRegex(
ImapAppendError,
"deployment limit",
):
_fetch_message_by_uid(client, "1")
self.assertIn("BODY.PEEK[]<0.11>", client.command)
def test_paginates_sequence_numbers_newest_first(self):
self.assertEqual(
_paged_descending_sequences(120, offset=0, limit=5),
["120", "119", "118", "117", "116"],
)
self.assertEqual(_paged_descending_sequences(120, offset=118, limit=5), ["2", "1"])
self.assertEqual(_paged_descending_sequences(120, offset=120, limit=5), [])
def test_compacts_contiguous_sequence_sets(self):
self.assertEqual(_sequence_set(["5", "4", "3"]), "3:5")
self.assertEqual(_sequence_set(["9", "7", "5"]), "5,7,9")
def test_extracts_fetch_sequence_number(self):
self.assertEqual(
_parse_fetch_sequence("42 (UID 123 FLAGS (\\Seen) RFC822.SIZE 100)"),
"42",
)
self.assertIsNone(_parse_fetch_sequence("UID 123 FLAGS ()"))
def test_normalizes_mailbox_pagination_request(self):
self.assertEqual(_normalize_mailbox_page(folder="", limit=0, offset=-5), ("INBOX", 1, 0))
self.assertEqual(_normalize_mailbox_page(folder=" Sent ", limit=500, offset=3), ("Sent", 100, 3))
class ImapMailboxCommandTests(unittest.TestCase):
def test_watcher_lists_new_uids_oldest_first(self):
class Client:
def uid(self, command, _charset, criterion):
self.search = (command, criterion)
return "OK", [b"5 9 7"]
def logout(self):
return "BYE", [b"logged out"]
client = Client()
config = ImapConfig(
host="imap.example.org",
username="user",
password="secret",
)
with (
patch("govoplan_mail.backend.sending.imap._open_imap", return_value=client),
patch(
"govoplan_mail.backend.sending.imap._select_readonly",
return_value=(3, "42"),
),
):
result = list_imap_uids_since(
imap_config=config,
folder="INBOX",
highest_uid=6,
expected_uidvalidity="42",
limit=2,
)
self.assertEqual(["7", "9"], result.uids)
self.assertEqual(("search", "ALL"), client.search)
def test_select_quotes_mailbox_name_with_spaces(self):
class Client:
untagged_responses = {"EXISTS": [b"0"], "UIDVALIDITY": [b"1"]}
def select(self, mailbox, readonly=False):
self.mailbox = mailbox
self.readonly = readonly
return "OK", [b"0"]
def response(self, code):
return "OK", [b"1"] if code == "UIDVALIDITY" else []
client = Client()
self.assertEqual(_select_readonly(client, "Gesendete Elemente"), (0, "1"))
self.assertEqual(client.mailbox, '"Gesendete Elemente"')
self.assertTrue(client.readonly)
def test_append_quotes_mailbox_name_with_spaces(self):
class Client:
def append(self, mailbox, flags, date_time, message):
self.mailbox = mailbox
self.flags = flags
self.date_time = date_time
self.message = message
return "OK", [b"APPEND completed"]
def logout(self):
return "BYE", [b"logged out"]
client = Client()
config = ImapConfig(host="imap.example.org", username="user", password="secret", sent_folder="auto")
with patch("govoplan_mail.backend.sending.imap._open_imap", return_value=client):
result = append_message_to_sent(b"Subject: test\r\n\r\nBody", imap_config=config, folder="Gesendete Elemente")
self.assertEqual(client.mailbox, '"Gesendete Elemente"')
self.assertEqual(result.folder, "Gesendete Elemente")
self.assertEqual(result.bytes_appended, len(b"Subject: test\r\n\r\nBody"))
def test_connection_loss_after_append_starts_has_unknown_outcome(self):
class Client:
def append(self, _mailbox, _flags, _date_time, _message):
raise OSError("provider detail")
def logout(self):
return "BYE", [b"logged out"]
config = ImapConfig(host="imap.example.org", username="user", password="secret", sent_folder="Sent")
with patch("govoplan_mail.backend.sending.imap._open_imap", return_value=Client()):
with self.assertRaises(ImapAppendError) as captured:
append_message_to_sent(b"Subject: test\r\n\r\nBody", imap_config=config, folder="Sent")
self.assertTrue(captured.exception.outcome_unknown)
self.assertFalse(captured.exception.temporary)
if __name__ == "__main__":
unittest.main()
+344
View File
@@ -0,0 +1,344 @@
from __future__ import annotations
import base64
import json
import unittest
import urllib.error
from types import SimpleNamespace
from unittest.mock import patch
from govoplan_mail.backend.config import JmapConfig, JmapServerConfig
from govoplan_mail.backend.sending.jmap import (
JMAP_CORE_CAPABILITY,
JMAP_MAIL_CAPABILITY,
JmapAuthenticationError,
JmapCapabilityError,
JmapConfigurationError,
discover_jmap,
get_jmap_email_changes,
get_jmap_message,
list_jmap_folders,
list_jmap_messages,
test_jmap_connection,
)
def _response(payload: dict) -> SimpleNamespace:
return SimpleNamespace(
status=200,
headers={"Content-Type": "application/json"},
body=json.dumps(payload).encode("utf-8"),
)
def _session(*, api_url: str = "https://jmap.example.test/api") -> dict:
return {
"capabilities": {
JMAP_CORE_CAPABILITY: {"maxCallsInRequest": 32},
JMAP_MAIL_CAPABILITY: {},
},
"accounts": {
"account-1": {
"name": "Example",
"isPersonal": True,
"isReadOnly": False,
"accountCapabilities": {JMAP_MAIL_CAPABILITY: {}},
}
},
"primaryAccounts": {JMAP_MAIL_CAPABILITY: "account-1"},
"username": "reader@example.test",
"apiUrl": api_url,
"downloadUrl": "https://jmap.example.test/download/{accountId}/{blobId}/{name}",
"uploadUrl": "https://jmap.example.test/upload/{accountId}",
"eventSourceUrl": "https://jmap.example.test/events",
"state": "session-state-1",
}
def _mailboxes() -> list[dict]:
return [
{
"id": "mb-inbox",
"name": "Inbox",
"parentId": None,
"role": "inbox",
"sortOrder": 10,
"isSubscribed": True,
"totalEmails": 2,
"unreadEmails": 1,
},
{
"id": "mb-projects",
"name": "Projects",
"parentId": None,
"role": None,
"sortOrder": 20,
"isSubscribed": True,
"totalEmails": 1,
"unreadEmails": 0,
},
{
"id": "mb-project-2026",
"name": "2026",
"parentId": "mb-projects",
"role": None,
"sortOrder": 1,
"isSubscribed": True,
"totalEmails": 1,
"unreadEmails": 0,
},
{
"id": "mb-sent",
"name": "Sent",
"parentId": None,
"role": "sent",
"sortOrder": 30,
"isSubscribed": True,
"totalEmails": 4,
"unreadEmails": 0,
},
]
def _email(email_id: str = "email-1", *, detail: bool = False) -> dict:
value = {
"id": email_id,
"threadId": "thread-1",
"mailboxIds": {"mb-inbox": True},
"keywords": {"$seen": False, "$flagged": True},
"size": 1234,
"receivedAt": "2026-08-22T10:30:00Z",
"sentAt": "2026-08-22T10:29:00Z",
"messageId": ["message-1@example.test"],
"from": [{"name": "Sender", "email": "sender@example.test"}],
"to": [{"name": "Reader", "email": "reader@example.test"}],
"cc": [],
"subject": "A governed message",
"hasAttachment": True,
"preview": "A bounded preview",
}
if detail:
value.update(
{
"replyTo": [{"email": "reply@example.test"}],
"bcc": [],
"textBody": [{"partId": "text", "type": "text/plain"}],
"htmlBody": [{"partId": "html", "type": "text/html"}],
"bodyValues": {
"text": {"value": "Plain body", "isTruncated": False},
"html": {"value": "<p>HTML body</p>", "isTruncated": False},
},
"attachments": [
{
"partId": "attachment",
"blobId": "blob-1",
"name": "evidence.pdf",
"type": "application/pdf",
"size": 44,
}
],
}
)
return value
class _JmapProvider:
def __init__(self, *, query_states: list[str] | None = None) -> None:
self.requests: list[tuple[str, str, dict[str, str], dict | None]] = []
self.query_states = list(query_states or ["query-state-1"])
def __call__(self, url: str, **kwargs):
body = json.loads(kwargs["body"].decode("utf-8")) if kwargs.get("body") else None
self.requests.append((url, kwargs.get("method", "GET"), kwargs.get("headers", {}), body))
if kwargs.get("method") == "GET":
return _response(_session())
method, arguments, call_id = body["methodCalls"][0]
if method == "Mailbox/get":
payload = {"accountId": "account-1", "state": "mailbox-state-1", "list": _mailboxes(), "notFound": []}
elif method == "Email/query":
state = self.query_states.pop(0) if self.query_states else "query-state-1"
payload = {
"accountId": "account-1",
"queryState": state,
"canCalculateChanges": True,
"position": arguments["position"],
"ids": ["email-1", "email-2"][arguments["position"] : arguments["position"] + arguments["limit"]],
"total": 2,
"limit": arguments["limit"],
}
elif method == "Email/get":
detail = bool(arguments.get("fetchTextBodyValues"))
payload = {
"accountId": "account-1",
"state": "email-state-1",
"list": [_email(email_id, detail=detail) for email_id in arguments["ids"]],
"notFound": [],
}
elif method == "Email/changes":
payload = {
"accountId": "account-1",
"oldState": arguments["sinceState"],
"newState": "email-state-2",
"hasMoreChanges": False,
"created": ["email-new"],
"updated": ["email-1"],
"destroyed": ["email-old"],
}
else: # pragma: no cover - fixture guard
raise AssertionError(method)
return _response({"methodResponses": [[method, payload, call_id]], "sessionState": "session-state-1"})
class JmapTransportTests(unittest.TestCase):
def config(self, **overrides) -> JmapConfig:
return JmapConfig(
session_url="https://jmap.example.test/.well-known/jmap",
password="access-token",
**overrides,
)
def test_server_configuration_rejects_embedded_credentials_and_normalizes_origins(self) -> None:
with self.assertRaisesRegex(ValueError, "embedded credentials"):
JmapServerConfig(session_url="https://user:secret@example.test/jmap")
config = JmapServerConfig(
session_url="https://jmap.example.test/.well-known/jmap",
allowed_api_origins=["https://api.example.test/path", "https://api.example.test"],
)
self.assertEqual(config.allowed_api_origins, ["https://api.example.test"])
def test_discovery_selects_primary_mail_account_and_bearer_auth(self) -> None:
provider = _JmapProvider()
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
result = discover_jmap(self.config())
self.assertEqual(result.account_id, "account-1")
self.assertIn(JMAP_MAIL_CAPABILITY, result.account_capabilities)
self.assertEqual(provider.requests[0][2]["Authorization"], "Bearer access-token")
self.assertNotIn("access-token", repr(result))
def test_basic_authentication_is_supported_without_exposing_credentials(self) -> None:
provider = _JmapProvider()
config = self.config(auth_scheme="basic", username="reader")
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
test_jmap_connection(jmap_config=config)
expected = base64.b64encode(b"reader:access-token").decode("ascii")
self.assertEqual(provider.requests[0][2]["Authorization"], f"Basic {expected}")
def test_cross_origin_api_url_is_fail_closed_unless_allowlisted(self) -> None:
provider = _JmapProvider()
provider.requests = []
with patch(
"govoplan_mail.backend.sending.jmap.fetch_http",
return_value=_response(_session(api_url="https://api.example.test/jmap")),
):
with self.assertRaisesRegex(JmapConfigurationError, "unapproved origin"):
discover_jmap(self.config())
def test_lists_hierarchical_mailboxes_with_roles_and_counts(self) -> None:
provider = _JmapProvider()
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
result = list_jmap_folders(jmap_config=self.config())
self.assertEqual(result.protocol, "jmap")
self.assertEqual(result.detected_folder_mappings["inbox"], "Inbox")
self.assertEqual(result.detected_sent_folder, "Sent")
self.assertIn("Projects/2026", [item.name for item in result.folders])
inbox = next(item for item in result.folders if item.name == "Inbox")
self.assertEqual((inbox.message_count, inbox.unseen_count), (2, 1))
def test_query_search_and_get_share_protocol_neutral_message_shape(self) -> None:
provider = _JmapProvider()
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
result = list_jmap_messages(
jmap_config=self.config(),
folder="INBOX",
limit=1,
query="governed",
)
self.assertEqual(result.folder, "Inbox")
self.assertEqual(result.total_count, 2)
self.assertEqual(result.uidvalidity, "query-state-1")
self.assertEqual(result.messages[0].uid, "email-1")
self.assertEqual(result.messages[0].from_header, "Sender <sender@example.test>")
query_call = next(request[3]["methodCalls"][0] for request in provider.requests if request[3] and request[3]["methodCalls"][0][0] == "Email/query")
self.assertEqual(query_call[1]["filter"]["conditions"][1], {"text": "governed"})
def test_changed_query_state_restarts_the_page_without_skipping(self) -> None:
provider = _JmapProvider(query_states=["new-state", "newer-state"])
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
result = list_jmap_messages(
jmap_config=self.config(),
folder="Inbox",
limit=1,
offset=1,
expected_query_state="old-state",
)
self.assertTrue(result.cursor_reset)
self.assertEqual(result.offset, 0)
query_positions = [
request[3]["methodCalls"][0][1]["position"]
for request in provider.requests
if request[3] and request[3]["methodCalls"][0][0] == "Email/query"
]
self.assertEqual(query_positions, [1, 0])
def test_detail_returns_bounded_body_values_and_attachment_metadata(self) -> None:
provider = _JmapProvider()
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
result = get_jmap_message(
jmap_config=self.config(),
folder="Inbox",
email_id="email-1",
)
self.assertEqual(result.message.body_text, "Plain body")
self.assertEqual(result.message.body_html, "<p>HTML body</p>")
self.assertEqual(result.message.attachments[0].filename, "evidence.pdf")
detail_call = provider.requests[-1][3]["methodCalls"][0][1]
self.assertEqual(detail_call["maxBodyValueBytes"], 1024 * 1024)
def test_incremental_changes_are_bounded_and_preserve_server_state(self) -> None:
provider = _JmapProvider()
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=provider):
result = get_jmap_email_changes(
jmap_config=self.config(),
since_state="email-state-1",
max_changes=25,
)
self.assertEqual(result.new_state, "email-state-2")
self.assertEqual(result.created, ("email-new",))
change_call = provider.requests[-1][3]["methodCalls"][0][1]
self.assertEqual(change_call["maxChanges"], 25)
def test_authentication_and_expired_change_state_have_distinct_diagnostics(self) -> None:
http_error = urllib.error.HTTPError(
"https://jmap.example.test/.well-known/jmap",
401,
"Unauthorized",
{},
None,
)
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=http_error):
with self.assertRaisesRegex(JmapAuthenticationError, "authentication failed"):
discover_jmap(self.config())
def expired(url: str, **kwargs):
if kwargs.get("method") == "GET":
return _response(_session())
body = json.loads(kwargs["body"])
call_id = body["methodCalls"][0][2]
return _response(
{
"methodResponses": [
["error", {"type": "cannotCalculateChanges"}, call_id]
]
}
)
with patch("govoplan_mail.backend.sending.jmap.fetch_http", side_effect=expired):
with self.assertRaisesRegex(JmapCapabilityError, "full refresh"):
get_jmap_email_changes(
jmap_config=self.config(),
since_state="expired-state",
)
if __name__ == "__main__":
unittest.main()
+643
View File
@@ -0,0 +1,643 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_mail.backend.config import ImapConfig, SmtpConfig
from govoplan_mail.backend.mail_profiles import (
EffectiveCredentialPolicy,
EffectiveMailProfilePolicy,
MailProfileError,
_assert_campaign_inherits_profile_credentials,
_campaign_mail_profile_reference_id,
_apply_profile_transport_update,
campaign_mail_selection_from_json,
campaign_profile_transport_revisions,
create_mail_server_profile,
_merge_policy,
_next_profile_transport_state,
_policy_parent_lock_message,
_policy_parent_lock_violations,
delete_mail_profile_credentials,
update_mail_server_profile,
)
class MailProfileTransportHelperTests(unittest.TestCase):
def test_inactive_profile_creation_rejects_dormant_credentials(self):
session = SimpleNamespace()
with (
patch(
"govoplan_mail.backend.mail_profiles._scope_tuple_for_create",
return_value=("tenant-1", "tenant", "tenant-1"),
),
patch("govoplan_mail.backend.mail_profiles._ensure_scope_allows_profile_creation"),
patch("govoplan_mail.backend.mail_profiles.assert_mail_policy_allows_transport"),
patch("govoplan_mail.backend.mail_profiles._ensure_unique_slug"),
self.assertRaisesRegex(MailProfileError, "cannot retain credentials"),
):
create_mail_server_profile(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
user_id="user-1",
name="Dormant secret",
slug=None,
description=None,
smtp=SmtpConfig(host="smtp.example.test", password="secret"),
imap=None,
is_active=False,
)
def test_inactive_profile_cannot_retain_or_recreate_dormant_credentials(self):
session = SimpleNamespace()
legacy = SimpleNamespace(
is_active=False,
smtp_password_encrypted="legacy-ciphertext",
imap_password_encrypted=None,
)
with self.assertRaisesRegex(MailProfileError, "use DELETE to scrub"):
update_mail_server_profile(
session, # type: ignore[arg-type]
legacy, # type: ignore[arg-type]
tenant_id="tenant-1",
user_id="user-1",
is_active=True,
)
self.assertFalse(legacy.is_active)
self.assertEqual(legacy.smtp_password_encrypted, "legacy-ciphertext")
scrubbed = SimpleNamespace(
is_active=False,
smtp_password_encrypted=None,
imap_password_encrypted=None,
)
with self.assertRaisesRegex(MailProfileError, "activated in the same update"):
update_mail_server_profile(
session, # type: ignore[arg-type]
scrubbed, # type: ignore[arg-type]
tenant_id="tenant-1",
user_id="user-1",
smtp=SmtpConfig(host="smtp.example.test", password="new-secret"),
)
self.assertFalse(scrubbed.is_active)
self.assertIsNone(scrubbed.smtp_password_encrypted)
def test_campaign_transport_revisions_are_random_opaque_values(self):
profile = SimpleNamespace(
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="first-user",
smtp_password_encrypted="first-secret-ciphertext",
smtp_transport_revision="032837ce-fd97-401a-8a9f-8df67f3ab41d",
imap_config=None,
imap_username=None,
imap_password_encrypted=None,
imap_transport_revision="9ca09521-c543-42ed-a8f2-f06783f1ebd3",
)
original = campaign_profile_transport_revisions(profile)
profile.smtp_password_encrypted = "second-secret-ciphertext"
self.assertEqual(campaign_profile_transport_revisions(profile), original)
self.assertNotIn("smtp.example.org", repr(original))
self.assertEqual(original["smtp"], profile.smtp_transport_revision)
def test_profile_deletion_scrubs_both_passwords_and_writes_non_secret_audit(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_password_encrypted="smtp-ciphertext",
imap_password_encrypted="imap-ciphertext",
)
session = SimpleNamespace(add=lambda _value: None, flush=lambda: None)
with patch("govoplan_mail.backend.mail_profiles.audit_event") as audit:
deleted = delete_mail_profile_credentials(
session, # type: ignore[arg-type]
profile=profile, # type: ignore[arg-type]
deletion_reason="profile_deactivated",
user_id="user-1",
)
self.assertEqual(deleted, ("smtp", "imap"))
self.assertIsNone(profile.smtp_password_encrypted)
self.assertIsNone(profile.imap_password_encrypted)
self.assertEqual(audit.call_count, 1)
self.assertEqual(audit.call_args.kwargs["action"], "mail.profile_credentials_deleted")
self.assertEqual(audit.call_args.kwargs["scope"], "tenant")
self.assertEqual(audit.call_args.kwargs["details"]["deletion_reason"], "profile_deactivated")
self.assertNotIn("ciphertext", repr(audit.call_args.kwargs))
def test_system_profile_secret_deletion_uses_system_audit_scope(self):
profile = SimpleNamespace(
id="profile-system",
tenant_id=None,
scope_type="system",
scope_id=None,
smtp_password_encrypted="smtp-ciphertext",
imap_password_encrypted=None,
)
session = SimpleNamespace(add=lambda _value: None, flush=lambda: None)
with patch("govoplan_mail.backend.mail_profiles.audit_event") as audit:
delete_mail_profile_credentials(
session, # type: ignore[arg-type]
profile=profile, # type: ignore[arg-type]
deletion_reason="module_data_retired",
)
self.assertEqual(audit.call_args.kwargs["scope"], "system")
self.assertIsNone(audit.call_args.kwargs["tenant_id"])
self.assertNotIn("ciphertext", repr(audit.call_args.kwargs))
def test_repeated_profile_secret_deletion_is_an_idempotent_no_op(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_password_encrypted=None,
imap_password_encrypted=None,
)
session = SimpleNamespace(add=lambda _value: self.fail("no-op must not add"), flush=lambda: self.fail("no-op must not flush"))
with patch("govoplan_mail.backend.mail_profiles.audit_event") as audit:
deleted = delete_mail_profile_credentials(
session, # type: ignore[arg-type]
profile=profile, # type: ignore[arg-type]
deletion_reason="profile_deactivated",
)
self.assertEqual(deleted, ())
audit.assert_not_called()
def test_profile_secret_deletion_does_not_mutate_ciphertext_when_audit_fails(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_password_encrypted="smtp-ciphertext",
imap_password_encrypted="imap-ciphertext",
)
session = SimpleNamespace(add=lambda _value: None, flush=lambda: None)
with (
patch(
"govoplan_mail.backend.mail_profiles.audit_event",
side_effect=RuntimeError("audit unavailable"),
),
self.assertRaisesRegex(RuntimeError, "audit unavailable"),
):
delete_mail_profile_credentials(
session, # type: ignore[arg-type]
profile=profile, # type: ignore[arg-type]
deletion_reason="profile_deactivated",
)
self.assertEqual(profile.smtp_password_encrypted, "smtp-ciphertext")
self.assertEqual(profile.imap_password_encrypted, "imap-ciphertext")
def test_next_transport_state_uses_only_saved_non_secret_metadata(self):
profile = SimpleNamespace(
tenant_id="tenant-1",
scope_type="tenant",
scope_id=None,
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="saved-smtp",
smtp_password_encrypted=encrypt_secret("smtp-secret"),
imap_config={"host": "imap.example.org", "port": 993, "security": "tls", "sent_folder": "Sent", "timeout_seconds": 30},
imap_username="saved-imap",
imap_password_encrypted=encrypt_secret("imap-secret"),
)
with patch(
"govoplan_mail.backend.mail_profiles.decrypt_secret",
side_effect=AssertionError("policy validation must not decrypt credentials"),
):
smtp, imap = _next_profile_transport_state(profile, smtp=None, imap=None, clear_imap=False)
self.assertEqual(smtp["host"], "smtp.example.org")
self.assertNotIn("username", smtp)
self.assertNotIn("password", smtp)
self.assertIsNotNone(imap)
assert imap is not None
self.assertEqual(imap["host"], "imap.example.org")
self.assertNotIn("username", imap)
self.assertNotIn("password", imap)
_, cleared_imap = _next_profile_transport_state(profile, smtp=None, imap=None, clear_imap=True)
self.assertIsNone(cleared_imap)
def test_apply_transport_update_preserves_unsupplied_passwords(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="saved-smtp",
smtp_password_encrypted=encrypt_secret("smtp-secret"),
smtp_transport_revision="smtp-before",
imap_config={"host": "imap.example.org", "port": 993, "security": "tls", "sent_folder": "Sent", "timeout_seconds": 30},
imap_username="saved-imap",
imap_password_encrypted=encrypt_secret("imap-secret"),
imap_transport_revision="imap-before",
)
session = SimpleNamespace(flush=lambda: None)
with patch("govoplan_mail.backend.mail_profiles.clear_mailbox_index") as clear_index:
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id=None,
smtp=SmtpConfig(host="smtp2.example.org", username="new-smtp"),
imap=ImapConfig(host="imap2.example.org", username="new-imap"),
clear_imap=False,
)
clear_index.assert_called_once_with(session, profile_id="profile-1")
self.assertEqual(profile.smtp_config["host"], "smtp2.example.org")
self.assertEqual(profile.smtp_username, "new-smtp")
self.assertEqual(decrypt_secret(profile.smtp_password_encrypted), "smtp-secret")
self.assertEqual(profile.imap_config["host"], "imap2.example.org")
self.assertEqual(profile.imap_username, "new-imap")
self.assertEqual(decrypt_secret(profile.imap_password_encrypted), "imap-secret")
self.assertNotEqual(profile.smtp_transport_revision, "smtp-before")
self.assertNotEqual(profile.imap_transport_revision, "imap-before")
with (
patch("govoplan_mail.backend.mail_profiles.audit_event") as audit,
patch("govoplan_mail.backend.mail_profiles.clear_mailbox_index") as clear_index,
):
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id=None,
smtp=None,
imap=None,
clear_imap=True,
)
clear_index.assert_called_once_with(session, profile_id="profile-1")
self.assertIsNone(profile.imap_config)
self.assertIsNone(profile.imap_username)
self.assertIsNone(profile.imap_password_encrypted)
self.assertEqual(audit.call_args.kwargs["action"], "mail.profile_credentials_deleted")
self.assertEqual(audit.call_args.kwargs["details"]["protocol"], "imap")
self.assertNotIn("imap-secret", repr(audit.call_args.kwargs))
def test_apply_transport_update_persists_standard_folder_mappings(self):
profile = SimpleNamespace(
id="profile-folders",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org"},
smtp_username=None,
smtp_password_encrypted=None,
smtp_transport_revision="smtp-before",
imap_config={"host": "imap.example.org", "sent_folder": "Legacy Sent"},
imap_username=None,
imap_password_encrypted=None,
imap_transport_revision="imap-before",
)
session = SimpleNamespace(flush=lambda: None)
with patch("govoplan_mail.backend.mail_profiles.clear_mailbox_index"):
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id=None,
smtp=None,
imap=ImapConfig(
host="imap.example.org",
folder_mappings={
"inbox": "INBOX",
"sent": "Sent Items",
"drafts": "Drafts",
},
),
clear_imap=False,
)
self.assertEqual(profile.imap_config["sent_folder"], "Sent Items")
self.assertEqual(
profile.imap_config["folder_mappings"],
{
"inbox": "INBOX",
"sent": "Sent Items",
"drafts": "Drafts",
},
)
def test_imap_password_replacement_clears_cache_without_rotating_identity_revision(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org"},
smtp_username=None,
smtp_password_encrypted=None,
smtp_transport_revision="smtp-before",
imap_config={
"host": "imap.example.org",
"port": 993,
"security": "tls",
"sent_folder": "auto",
"timeout_seconds": 30,
},
imap_username="saved-imap",
imap_password_encrypted=encrypt_secret("old-secret"),
imap_transport_revision="imap-before",
)
session = SimpleNamespace(flush=lambda: None)
with (
patch("govoplan_mail.backend.mail_profiles.audit_event"),
patch("govoplan_mail.backend.mail_profiles.clear_mailbox_index") as clear_index,
):
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id=None,
smtp=None,
imap=ImapConfig(
host="imap.example.org",
port=993,
security="tls",
timeout_seconds=30,
password="new-secret",
),
clear_imap=False,
)
clear_index.assert_called_once_with(session, profile_id="profile-1")
self.assertEqual(profile.imap_transport_revision, "imap-before")
self.assertEqual(decrypt_secret(profile.imap_password_encrypted), "new-secret")
def test_password_replacement_preserves_revision_and_is_audited_without_secret(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="saved-smtp",
smtp_password_encrypted=encrypt_secret("old-secret"),
smtp_transport_revision="smtp-before",
imap_config=None,
imap_username=None,
imap_password_encrypted=None,
imap_transport_revision="imap-before",
)
session = SimpleNamespace(flush=lambda: None)
with patch("govoplan_mail.backend.mail_profiles.audit_event") as audit:
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id="key-1",
smtp=SmtpConfig(
host="smtp.example.org",
port=587,
security="starttls",
timeout_seconds=30,
password="new-secret",
),
imap=None,
clear_imap=False,
)
self.assertEqual(decrypt_secret(profile.smtp_password_encrypted), "new-secret")
self.assertEqual(profile.smtp_transport_revision, "smtp-before")
self.assertEqual(audit.call_args.kwargs["action"], "mail.profile_credentials_replaced")
self.assertEqual(audit.call_args.kwargs["details"]["protocol"], "smtp")
self.assertNotIn("old-secret", repr(audit.call_args.kwargs))
self.assertNotIn("new-secret", repr(audit.call_args.kwargs))
def test_audit_failure_leaves_existing_ciphertext_and_revision_unchanged(self):
encrypted = encrypt_secret("old-secret")
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="saved-smtp",
smtp_password_encrypted=encrypted,
smtp_transport_revision="smtp-before",
imap_config=None,
imap_username=None,
imap_password_encrypted=None,
imap_transport_revision="imap-before",
)
session = SimpleNamespace(flush=lambda: None)
with (
patch(
"govoplan_mail.backend.mail_profiles.audit_event",
side_effect=RuntimeError("audit unavailable"),
),
self.assertRaisesRegex(RuntimeError, "audit unavailable"),
):
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id=None,
smtp=SmtpConfig(host="smtp.example.org", password="new-secret"),
imap=None,
clear_imap=False,
)
self.assertEqual(profile.smtp_password_encrypted, encrypted)
self.assertEqual(profile.smtp_transport_revision, "smtp-before")
def test_supplied_password_is_replaced_without_decrypting_old_ciphertext(self):
encrypted = encrypt_secret("same-secret")
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="saved-smtp",
smtp_password_encrypted=encrypted,
smtp_transport_revision="smtp-before",
imap_config=None,
imap_username=None,
imap_password_encrypted=None,
imap_transport_revision="imap-before",
)
session = SimpleNamespace(flush=lambda: None)
with (
patch("govoplan_mail.backend.mail_profiles.audit_event") as audit,
patch(
"govoplan_mail.backend.mail_profiles.decrypt_secret",
side_effect=AssertionError("replacement must not decrypt old ciphertext"),
),
):
_apply_profile_transport_update(
session, # type: ignore[arg-type]
profile,
user_id="user-1",
api_key_id=None,
smtp=SmtpConfig(host="smtp.example.org", password="same-secret"),
imap=None,
clear_imap=False,
)
self.assertNotEqual(profile.smtp_password_encrypted, encrypted)
self.assertEqual(decrypt_secret(profile.smtp_password_encrypted), "same-secret")
self.assertEqual(profile.smtp_transport_revision, "smtp-before")
self.assertEqual(audit.call_args.kwargs["action"], "mail.profile_credentials_replaced")
def test_metadata_update_does_not_decrypt_unrelated_transport_credentials(self):
profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
smtp_config={"host": "smtp.example.org", "port": 587, "security": "starttls", "timeout_seconds": 30},
smtp_username="saved-smtp",
smtp_password_encrypted="corrupt-smtp-ciphertext",
smtp_transport_revision="smtp-before",
imap_config={"host": "imap.example.org", "port": 993, "security": "tls", "sent_folder": "Sent", "timeout_seconds": 30},
imap_username="saved-imap",
imap_password_encrypted="corrupt-imap-ciphertext",
imap_transport_revision="imap-before",
name="Profile",
slug="profile",
description=None,
is_active=True,
updated_by_user_id=None,
)
session = SimpleNamespace(add=lambda _value: None, flush=lambda: None)
with (
patch("govoplan_mail.backend.mail_profiles.decrypt_secret", side_effect=AssertionError("must not decrypt")),
patch("govoplan_mail.backend.mail_profiles._assert_profile_transport_allowed") as policy_check,
):
update_mail_server_profile(
session, # type: ignore[arg-type]
profile,
tenant_id="tenant-1",
user_id="user-1",
description="Updated",
)
policy_check.assert_called_once()
self.assertEqual(profile.description, "Updated")
self.assertEqual(profile.smtp_password_encrypted, "corrupt-smtp-ciphertext")
self.assertEqual(profile.imap_password_encrypted, "corrupt-imap-ciphertext")
class MailProfilePolicyHelperTests(unittest.TestCase):
def test_campaign_contract_accepts_only_mail_owned_references(self):
self.assertEqual(
_campaign_mail_profile_reference_id({"mail_profile_id": " profile-1 "}),
"profile-1",
)
for legacy in ("smtp", "imap", "credentials", "inherit_smtp_credentials"):
with self.subTest(legacy=legacy), self.assertRaisesRegex(MailProfileError, "select Mail resources"):
_campaign_mail_profile_reference_id({"mail_profile_id": "profile-1", legacy: {}})
self.assertEqual(
campaign_mail_selection_from_json(
{
"server": {
"mail_profile_id": "profile-1",
"smtp_server_id": "smtp-1",
"smtp_credential_id": "credential-1",
}
}
)["smtp_credential_id"],
"credential-1",
)
def test_campaign_delivery_fails_when_policy_requires_local_credentials(self):
profile = SimpleNamespace(imap_config=None)
policy = EffectiveMailProfilePolicy(
smtp_credentials=EffectiveCredentialPolicy(inherit=False),
)
with self.assertRaisesRegex(MailProfileError, "explicit credential selection"):
_assert_campaign_inherits_profile_credentials(profile, policy)
def test_merge_policy_respects_locked_lower_level_limits(self):
policy = EffectiveMailProfilePolicy()
_merge_policy(
policy,
{
"blacklist": {"smtp_hosts": ["*.blocked.example"]},
"allow_lower_level_limits": {"blacklist.smtp_hosts": False},
},
source="system",
)
_merge_policy(
policy,
{"blacklist": {"smtp_hosts": ["*.tenant.example"]}},
source="tenant",
source_id="tenant-1",
)
self.assertEqual(policy.blacklist_patterns["smtp_hosts"], ["*.blocked.example"])
self.assertFalse(policy.allow_lower_level_limits["blacklist.smtp_hosts"])
def test_jmap_hostname_policy_has_independent_deny_and_lower_scope_lock(self):
policy = EffectiveMailProfilePolicy()
_merge_policy(
policy,
{
"blacklist": {"jmap_hosts": ["*.blocked.example"]},
"allow_lower_level_limits": {"blacklist.jmap_hosts": False},
},
source="system",
)
allowed, reason = policy.value_allowed("jmap_hosts", "mail.blocked.example")
self.assertFalse(allowed)
self.assertIn("jmap_hosts", reason or "")
self.assertFalse(policy.allow_lower_level_limits["blacklist.jmap_hosts"])
def test_parent_lock_violations_are_reported_per_field(self):
violations = _policy_parent_lock_violations(
{
"allow_user_profiles": False,
"smtp_credentials.inherit": False,
"blacklist.smtp_hosts": False,
},
{
"allow_user_profiles": True,
"smtp_credentials": {"inherit": False},
"blacklist": {"smtp_hosts": ["*.blocked.example"]},
"allow_lower_level_limits": {"allow_user_profiles": True},
},
)
self.assertEqual(
violations,
[
"allow_user_profiles",
"blacklist.smtp_hosts",
"smtp_credentials.inherit",
"allow_lower_level_limits.allow_user_profiles",
],
)
self.assertEqual(
_policy_parent_lock_message("smtp_credentials.inherit"),
"SMTP credential inheritance is locked by an ancestor policy",
)
if __name__ == "__main__":
unittest.main()
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
import unittest
from govoplan_mail.backend.db.models import MailMailboxFolderIndex, MailMailboxMessageIndex
from govoplan_mail.backend.mailbox_index import clear_mailbox_index
class _DeleteQuery:
def __init__(self, model: type, deleted_models: list[type]) -> None:
self.model = model
self.deleted_models = deleted_models
def filter(self, *_criteria):
return self
def all(self):
return []
def delete(self, *, synchronize_session: bool) -> int:
if synchronize_session is not False:
raise AssertionError("bulk invalidation must not synchronize loaded cache rows")
self.deleted_models.append(self.model)
return 2 if self.model is MailMailboxMessageIndex else 1
class _Session:
def __init__(self) -> None:
self.deleted_models: list[type] = []
def query(self, model: type) -> _DeleteQuery:
return _DeleteQuery(model, self.deleted_models)
class MailboxIndexInvalidationTests(unittest.TestCase):
def test_profile_wide_invalidation_deletes_messages_before_folders(self) -> None:
session = _Session()
deleted_folders, deleted_messages = clear_mailbox_index(
session, # type: ignore[arg-type]
profile_id="profile-1",
)
self.assertEqual(
session.deleted_models,
[MailMailboxMessageIndex, MailMailboxFolderIndex],
)
self.assertEqual((deleted_folders, deleted_messages), (1, 2))
if __name__ == "__main__":
unittest.main()
+207
View File
@@ -0,0 +1,207 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from govoplan_core.core.modules import MigrationRetirementPlan, ModuleManifest
from govoplan_mail.backend.manifest import _mail_retirement_provider, get_manifest
class MailManifestTests(unittest.TestCase):
def test_mail_quick_access_can_return_exact_message_references(self) -> None:
frontend = get_manifest().frontend
self.assertIsNotNone(frontend)
tool = next(
item for item in frontend.quick_access_tools if item.id == "mail.messages" # type: ignore[union-attr]
)
self.assertIn("select", tool.modes)
self.assertEqual(("mail.message",), tool.returned_reference_kinds)
def test_manifest_declares_optional_addresses_lookup(self) -> None:
manifest = get_manifest()
self.assertIsInstance(manifest, ModuleManifest)
self.assertEqual(manifest.id, "mail")
self.assertIn("addresses", manifest.optional_dependencies)
self.assertIn("addresses.lookup", {interface.name for interface in manifest.requires_interfaces})
self.assertIn("addresses.contact_writer", {interface.name for interface in manifest.requires_interfaces})
self.assertIn(
{
"name": "campaigns.access",
"version_min": "0.1.0",
"version_max_exclusive": "0.2.0",
"optional": True,
},
[
{
"name": interface.name,
"version_min": interface.version_min,
"version_max_exclusive": interface.version_max_exclusive,
"optional": interface.optional,
}
for interface in manifest.requires_interfaces
],
)
self.assertIn(
{"name": "mail.campaign_delivery", "version": "0.2.0"},
[
{"name": interface.name, "version": interface.version}
for interface in manifest.provides_interfaces
],
)
permissions = {permission.scope for permission in manifest.permissions}
self.assertIn("mail:profile:write_own", permissions)
self.assertIn("mail:secret:manage_own", permissions)
self.assertTrue(
{"mail:pop3:manage", "mail:pop3:import", "mail:pop3:delete"}.issubset(
permissions
)
)
roles = {template.slug: template for template in manifest.role_templates}
self.assertIn("mail:pop3:delete", roles["mail_profile_admin"].permissions)
self.assertEqual(
set(roles["mail_legacy_import_operator"].permissions),
{
"mail:profile:read",
"mail:profile:use",
"mail:profile:test",
"mail:pop3:import",
},
)
self.assertEqual(
set(roles["mail_profile_self_service"].permissions),
{
"mail:profile:read",
"mail:profile:use",
"mail:profile:test",
"mail:mailbox:read",
"mail:profile:write_own",
"mail:secret:manage_own",
},
)
topics = {topic.id: topic for topic in manifest.documentation}
self.assertTrue(
{
"mail.workflow.choose-and-test-profile",
"mail.workflow.read-mailbox",
"mail.reference.credentials-egress-retirement",
"mail.reference.campaign-delivery-contract",
"mail.address-book-integration",
"mail.workflow.legacy-pop3-import",
}.issubset(topics)
)
pop3_topic = topics["mail.workflow.legacy-pop3-import"]
self.assertEqual(("user", "admin"), pop3_topic.documentation_types)
self.assertIn("mail:pop3:import", pop3_topic.conditions[0].any_scopes)
self.assertTrue(
{
"mail.pop3",
"mail.pop3.source-editor",
"mail.pop3.action.reload",
"mail.pop3.action.save-source",
"mail.pop3.action.import",
"mail.pop3.field.transport-security",
"mail.pop3.field.max-message-size",
"mail.pop3.field.password",
"mail.pop3.field.delete-after-import",
"mail.pop3.confirm-delete-source",
}.issubset(pop3_topic.metadata["help_contexts"])
)
self.assertGreaterEqual(len(pop3_topic.metadata["fields"]), 4)
self.assertGreaterEqual(
len(pop3_topic.metadata["operational_consequences"]),
3,
)
for topic in manifest.documentation:
with self.subTest(topic_id=topic.id):
german = topic.translations.get("de", {})
for attribute in ("title", "summary", "body"):
self.assertTrue(
str(german.get(attribute, "")).strip(),
f"{topic.id} is missing its German {attribute}",
)
pop3_provider = next(
item
for item in manifest.external_providers
if item.id == "mail.pop3_legacy_import"
)
self.assertIn("delete", pop3_provider.operations)
self.assertTrue(pop3_provider.behavior.outcome_unknown_supported)
self.assertIn("mail.pop3.source_deletion", pop3_provider.behavior.audit_event_types)
self.assertTrue(
any(
route.path == "/mail/legacy-import"
for route in manifest.frontend.routes # type: ignore[union-attr]
)
)
ownership = topics["mail.profile-ownership-and-consumers"]
self.assertEqual(ownership.metadata["kind"], "reference")
self.assertEqual(ownership.metadata["route"], "/settings?section=mail-profiles")
self.assertIn("campaigns.mail-profile-user-journey", ownership.metadata["related_topic_ids"])
mailbox = topics["mail.workflow.read-mailbox"]
self.assertEqual(mailbox.metadata["route"], "/mail")
self.assertIn("mail.mailbox", mailbox.metadata["help_contexts"])
self.assertEqual(
mailbox.conditions[0].required_scopes,
("mail:mailbox:read", "mail:profile:use"),
)
campaign_contract = topics["mail.reference.campaign-delivery-contract"]
self.assertEqual(campaign_contract.conditions[0].required_modules, ("mail", "campaigns"))
profile_policy = topics["mail.profiles-and-policy"]
self.assertIn("mail.admin.profiles", profile_policy.metadata["help_contexts"])
bounce_processing = topics["mail.bounce-processing"]
self.assertIn("mail.bounce-processing", bounce_processing.metadata["help_contexts"])
def test_retirement_scrubs_credentials_before_dropping_mail_tables(self) -> None:
events: list[str] = []
def table_provider(_session, _module_id):
return MigrationRetirementPlan(
supported=True,
summary="Mail tables",
destroy_data_supported=True,
destroy_data_executor=lambda _execute_session, _execute_module_id: events.append("drop"),
)
session = SimpleNamespace(get_bind=lambda: object(), query=lambda *_args: None)
inspector = SimpleNamespace(has_table=lambda _name: True)
with (
patch("govoplan_mail.backend.manifest._mail_table_retirement_provider", table_provider),
patch("govoplan_mail.backend.manifest.inspect", return_value=inspector),
patch(
"govoplan_mail.backend.mail_profiles.delete_mail_profile_credentials_for_retirement",
side_effect=lambda _session: events.append("scrub"),
),
):
plan = _mail_retirement_provider(session, "mail")
assert plan.destroy_data_executor is not None
plan.destroy_data_executor(session, "mail")
self.assertEqual(events, ["scrub", "drop"])
self.assertTrue(any("audit" in warning for warning in plan.destroy_data_warnings))
events.clear()
with (
patch("govoplan_mail.backend.manifest._mail_table_retirement_provider", table_provider),
patch("govoplan_mail.backend.manifest.inspect", return_value=inspector),
patch(
"govoplan_mail.backend.mail_profiles.delete_mail_profile_credentials_for_retirement",
side_effect=RuntimeError("audit unavailable"),
),
):
blocked_plan = _mail_retirement_provider(session, "mail")
assert blocked_plan.destroy_data_executor is not None
with self.assertRaisesRegex(RuntimeError, "audit unavailable"):
blocked_plan.destroy_data_executor(session, "mail")
self.assertEqual(events, [])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,91 @@
from __future__ import annotations
import unittest
from email import message_from_bytes
from unittest.mock import ANY, patch
from govoplan_core.core.mail import NotificationMailDeliveryRequest
from govoplan_mail.backend.capabilities import MailNotificationDeliveryCapability
class MailNotificationDeliveryCapabilityTests(unittest.TestCase):
def test_missing_policy_selection_pauses_without_transport_effect(self) -> None:
capability = MailNotificationDeliveryCapability()
with patch(
"govoplan_mail.backend.delivery_outbox.submit_delivery_command"
) as submit:
result = capability.submit_notification_mail(
object(),
NotificationMailDeliveryRequest(
tenant_id="tenant-1",
notification_id="notification-1",
recipient="person@example.test",
subject="Subject",
body_text="Body",
),
)
self.assertEqual(result["status"], "paused")
submit.assert_not_called()
def test_notification_is_submitted_to_durable_mail_outbox(self) -> None:
capability = MailNotificationDeliveryCapability()
transport = {
"smtp_available": True,
"smtp_transport_revision": "revision-1",
"smtp_server_id": "server-1",
"smtp_credential_id": "credential-1",
}
command = {
"id": "command-1",
"status": "pending",
"duplicate": False,
}
with (
patch(
"govoplan_mail.backend.capabilities.campaign_profile_delivery_summary",
return_value=transport,
) as summary,
patch(
"govoplan_mail.backend.delivery_outbox.submit_delivery_command",
return_value=command,
) as submit,
):
result = capability.submit_notification_mail(
object(),
NotificationMailDeliveryRequest(
tenant_id="tenant-1",
notification_id="notification-1",
recipient="person@example.test",
subject="Subject",
body_text="Body",
body_html="<p>Body</p>",
mail_profile_id="profile-1",
from_address="notifications@example.test",
),
)
self.assertEqual(result["status"], "accepted")
self.assertEqual(result["external_message_id"], "command-1")
summary.assert_called_once_with(
ANY,
tenant_id="tenant-1",
profile_id="profile-1",
smtp_server_id=None,
smtp_credential_id=None,
)
payload = submit.call_args.kwargs
self.assertEqual(payload["idempotency_key"], "notification:notification-1")
self.assertEqual(payload["profile_id"], "profile-1")
self.assertEqual(payload["smtp_server_id"], "server-1")
self.assertEqual(payload["smtp_credential_id"], "credential-1")
message = message_from_bytes(payload["message_bytes"])
self.assertEqual(message["From"], "notifications@example.test")
self.assertEqual(message["To"], "person@example.test")
self.assertEqual(message["Subject"], "Subject")
if __name__ == "__main__":
unittest.main()
+440
View File
@@ -0,0 +1,440 @@
from __future__ import annotations
import base64
from datetime import UTC, datetime
import poplib
import ssl
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, User
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.db.base import Base
from govoplan_core.security.secrets import decrypt_secret
from govoplan_mail.backend.config import Pop3Config, TransportSecurity
from govoplan_mail.backend.db.models import (
MailPop3Import,
MailServerEndpoint,
MailServerProfile,
)
from govoplan_mail.backend.pop3_imports import (
Pop3ImportResult,
create_pop3_imports,
list_pop3_imports,
)
from govoplan_mail.backend.router import import_profile_pop3_messages
from govoplan_mail.backend.schemas import MailPop3ImportRequest
from govoplan_mail.backend.sending.pop3 import (
Pop3ConfigurationError,
Pop3DownloadedMessage,
Pop3MessageSummary,
Pop3ProviderError,
_open_pop3,
delete_pop3_messages,
download_pop3_messages,
preview_pop3_messages,
)
_RAW = (
b"Subject: Legacy notice\r\n"
b"From: Office <office@example.test>\r\n"
b"To: Subject <subject@example.test>\r\n"
b"Message-ID: <legacy-1@example.test>\r\n"
b"\r\n"
b"A bounded legacy message.\r\n"
)
class _Pop3Client:
def __init__(self, *, quit_error: Exception | None = None) -> None:
self.deletions: list[int] = []
self.quit_calls = 0
self.rset_calls = 0
self.close_calls = 0
self.quit_error = quit_error
def stat(self):
return 1, len(_RAW)
def uidl(self):
return b"+OK", [b"1 uid-1"], 1
def list(self):
return b"+OK", [f"1 {len(_RAW)}".encode("ascii")], 1
def top(self, _number, _lines):
return b"+OK", _RAW.rstrip(b"\r\n").split(b"\r\n"), len(_RAW)
def retr(self, _number):
return b"+OK", _RAW.rstrip(b"\r\n").split(b"\r\n"), len(_RAW)
def dele(self, number):
self.deletions.append(number)
def quit(self):
self.quit_calls += 1
if self.quit_error is not None:
raise self.quit_error
return b"+OK"
def rset(self):
self.rset_calls += 1
def close(self):
self.close_calls += 1
def _config(**changes) -> Pop3Config:
values = {
"host": "pop3.example.test",
"security": "tls",
"username": "legacy-user",
"password": "legacy-password",
"legacy_import_enabled": True,
}
values.update(changes)
return Pop3Config.model_validate(values)
def _download(uidl: str = "uid-1") -> Pop3DownloadedMessage:
summary = Pop3MessageSummary(
message_number=1,
uidl=uidl,
subject="Legacy notice",
from_header="Office <office@example.test>",
to_header="Subject <subject@example.test>",
date="Sat, 22 Aug 2026 10:00:00 +0200",
message_id="<legacy-1@example.test>",
size_bytes=len(_RAW),
body_preview="A bounded legacy message.",
)
import hashlib
return Pop3DownloadedMessage(
message_number=1,
uidl=uidl,
raw=_RAW,
raw_sha256=hashlib.sha256(_RAW).hexdigest(),
summary=summary,
)
class Pop3TransportTests(unittest.TestCase):
def test_legacy_import_is_disabled_until_explicitly_enabled(self) -> None:
with self.assertRaisesRegex(Pop3ConfigurationError, "disabled"):
preview_pop3_messages(
pop3_config=_config(legacy_import_enabled=False),
limit=10,
)
with self.assertRaisesRegex(ValueError, "batch size limit"):
_config(max_message_bytes=2 * 1024 * 1024, max_batch_bytes=1024 * 1024)
def test_preview_and_download_are_non_destructive(self) -> None:
preview_client = _Pop3Client()
with patch(
"govoplan_mail.backend.sending.pop3._open_pop3",
return_value=preview_client,
):
preview = preview_pop3_messages(pop3_config=_config(), limit=10)
self.assertEqual(["uid-1"], [item.uidl for item in preview.messages])
self.assertEqual([], preview_client.deletions)
self.assertEqual(1, preview_client.quit_calls)
download_client = _Pop3Client()
with patch(
"govoplan_mail.backend.sending.pop3._open_pop3",
return_value=download_client,
):
downloaded = download_pop3_messages(
pop3_config=_config(), uidls=("uid-1",)
)
self.assertEqual(_RAW, downloaded[0].raw)
self.assertEqual([], download_client.deletions)
self.assertEqual(1, download_client.quit_calls)
def test_source_deletion_needs_policy_and_commits_with_quit(self) -> None:
with self.assertRaisesRegex(Pop3ConfigurationError, "disabled"):
delete_pop3_messages(pop3_config=_config(), uidls=("uid-1",))
client = _Pop3Client()
with patch(
"govoplan_mail.backend.sending.pop3._open_pop3", return_value=client
):
result = delete_pop3_messages(
pop3_config=_config(allow_delete_after_import=True),
uidls=("uid-1",),
)
self.assertEqual(("uid-1",), result.deleted_uidls)
self.assertEqual([1], client.deletions)
self.assertEqual(1, client.quit_calls)
def test_quit_failure_marks_deletion_outcome_unknown(self) -> None:
client = _Pop3Client(quit_error=poplib.error_proto("connection lost"))
with (
patch(
"govoplan_mail.backend.sending.pop3._open_pop3",
return_value=client,
),
self.assertRaises(Pop3ProviderError) as captured,
):
delete_pop3_messages(
pop3_config=_config(allow_delete_after_import=True),
uidls=("uid-1",),
)
self.assertTrue(captured.exception.outcome_unknown)
self.assertEqual([1], client.deletions)
def test_tls_and_authentication_failures_are_sanitized(self) -> None:
with (
patch(
"govoplan_mail.backend.sending.pop3.validate_outbound_host"
),
patch(
"govoplan_mail.backend.sending.pop3._OutboundPolicyPOP3SSL",
side_effect=ssl.SSLError("private TLS detail"),
),
self.assertRaisesRegex(Pop3ProviderError, "TLS negotiation failed"),
):
_open_pop3(_config())
auth_client = _Pop3Client()
auth_client.user = lambda _value: None # type: ignore[attr-defined]
auth_client.pass_ = lambda _value: (_ for _ in ()).throw( # type: ignore[attr-defined]
poplib.error_proto("private auth detail")
)
with (
patch(
"govoplan_mail.backend.sending.pop3.validate_outbound_host"
),
patch(
"govoplan_mail.backend.sending.pop3._OutboundPolicyPOP3",
return_value=auth_client,
),
self.assertRaisesRegex(Pop3ProviderError, "authentication failed"),
):
_open_pop3(_config(security=TransportSecurity.PLAIN))
class Pop3PersistenceTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
Base.metadata.create_all(
self.engine,
tables=(
Account.__table__,
User.__table__,
MailServerProfile.__table__,
MailServerEndpoint.__table__,
MailPop3Import.__table__,
),
)
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
self.profile = MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Legacy source",
slug="legacy-source",
smtp_config={},
)
self.server = MailServerEndpoint(
id="server-1",
profile_id=self.profile.id,
tenant_id="tenant-1",
protocol="pop3",
name="Legacy POP3",
config={"legacy_import_enabled": True},
scope_type="tenant",
scope_id="tenant-1",
transport_revision="revision-1",
)
self.session.add_all((self.profile, self.server))
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_import_is_encrypted_and_duplicate_uidl_is_reused(self) -> None:
first = create_pop3_imports(
self.session,
tenant_id="tenant-1",
profile_id=self.profile.id,
pop3_server_id=self.server.id,
pop3_credential_id=None,
transport_revision="revision-1",
messages=(_download(),),
user_id=None,
deletion_requested=False,
)
self.session.commit()
self.assertEqual(1, len(first.imported))
encrypted = first.imported[0].raw_message_encrypted
self.assertNotIn("Legacy notice", encrypted)
self.assertEqual(
_RAW,
base64.b64decode(decrypt_secret(encrypted) or ""),
)
self.assertEqual("not_requested", first.imported[0].deletion_status)
repeated = create_pop3_imports(
self.session,
tenant_id="tenant-1",
profile_id=self.profile.id,
pop3_server_id=self.server.id,
pop3_credential_id=None,
transport_revision="revision-1",
messages=(_download(),),
user_id=None,
deletion_requested=False,
)
self.assertEqual((), repeated.imported)
self.assertEqual((first.imported[0].id,), tuple(row.id for row in repeated.duplicates))
self.assertEqual(
(first.imported[0].id,),
tuple(
row.id
for row in list_pop3_imports(
self.session,
tenant_id="tenant-1",
profile_ids=(self.profile.id,),
)
),
)
self.assertEqual(
(),
list_pop3_imports(
self.session,
tenant_id="tenant-1",
profile_ids=("unrelated-profile",),
),
)
class _RouteSession:
def __init__(self, events: list[str]) -> None:
self.events = events
def commit(self) -> None:
self.events.append("commit")
def rollback(self) -> None:
self.events.append("rollback")
class Pop3ImportRouteTests(unittest.TestCase):
def test_local_import_and_audit_commit_before_source_deletion(self) -> None:
events: list[str] = []
now = datetime.now(UTC)
row = SimpleNamespace(
id="import-1",
profile_id="profile-1",
pop3_server_id="server-1",
transport_revision="revision-1",
provider_uidl="uid-1",
message_id="<legacy-1@example.test>",
subject="Legacy notice",
from_header="office@example.test",
to_header="subject@example.test",
date="2026-08-22",
body_preview="A bounded legacy message.",
size_bytes=len(_RAW),
raw_sha256=_download().raw_sha256,
status="pending_review",
imported_at=now,
deletion_requested=True,
deletion_status="pending",
deletion_attempted_at=None,
deletion_error=None,
)
resolved = SimpleNamespace(
config=_config(allow_delete_after_import=True),
server=SimpleNamespace(id="server-1"),
credential=None,
transport_revision="revision-1",
)
principal = ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="user-1",
tenant_id="tenant-1",
scopes=frozenset(
{
"mail:profile:use",
"mail:pop3:import",
"mail:pop3:delete",
}
),
),
account=SimpleNamespace(id="account-1"),
user=SimpleNamespace(id="user-1"),
)
payload = MailPop3ImportRequest(
server_id="server-1",
expected_transport_revision="revision-1",
uidls=["uid-1"],
delete_after_import=True,
)
def audit(*_args, **_kwargs) -> None:
events.append("audit")
def delete(**_kwargs):
self.assertEqual(["audit", "commit"], events)
events.append("delete")
def mark(*_args, **_kwargs):
events.append("mark")
row.deletion_status = "succeeded"
row.deletion_attempted_at = now
return (row,)
with (
patch(
"govoplan_mail.backend.router._resolve_profile_pop3_transport",
return_value=(SimpleNamespace(id="profile-1"), resolved),
),
patch(
"govoplan_mail.backend.router.download_pop3_messages",
return_value=(_download(),),
),
patch(
"govoplan_mail.backend.router.create_pop3_imports",
return_value=Pop3ImportResult(imported=(row,), duplicates=()),
),
patch("govoplan_mail.backend.router.audit_event", side_effect=audit),
patch("govoplan_mail.backend.router.delete_pop3_messages", side_effect=delete),
patch(
"govoplan_mail.backend.router.mark_pop3_deletion_result",
side_effect=mark,
),
):
result = import_profile_pop3_messages(
"profile-1",
payload,
principal=principal,
session=_RouteSession(events), # type: ignore[arg-type]
)
self.assertEqual("succeeded", result.deletion_status)
self.assertEqual(
["audit", "commit", "delete", "mark", "commit", "audit", "commit"],
events,
)
if __name__ == "__main__":
unittest.main()
+138
View File
@@ -0,0 +1,138 @@
from __future__ import annotations
import unittest
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import patch
from sqlalchemy import Column, String, Table, create_engine
from sqlalchemy.orm import Session, sessionmaker
from govoplan_core.core.mail import MailPostboxBridgeRequest
from govoplan_core.core.postbox import PostboxTargetRef
from govoplan_core.db.base import Base
from govoplan_mail.backend.db.models import MailServerProfile
from govoplan_mail.backend.postbox_bridge import MailPostboxBridge
RAW_MESSAGE = b"""From: Ada Example <ada@example.test>
To: Clerk <clerk@example.test>
Cc: Archive <archive@example.test>
Message-ID: <bridge-1@example.test>
Subject: Submitted evidence
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary=bridge
--bridge
Content-Type: text/plain; charset=utf-8
Please process the attached evidence.
--bridge
Content-Type: application/pdf
Content-Disposition: attachment; filename=evidence.pdf
Content-Transfer-Encoding: base64
UERG
--bridge--
"""
class _DeliveryProvider:
def __init__(self) -> None:
self.requests = []
def deliver(self, session, request):
del session
self.requests.append(request)
return SimpleNamespace(
postbox_id="postbox-1",
message_id="message-1",
delivery_id="delivery-1",
duplicate=len(self.requests) > 1,
)
class MailPostboxBridgeTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite:///:memory:")
access_users = Base.metadata.tables.get("access_users")
if access_users is None:
access_users = Table(
"access_users",
Base.metadata,
Column("id", String(36), primary_key=True),
)
Base.metadata.create_all(
self.engine,
tables=[access_users, MailServerProfile.__table__],
)
self.Session = sessionmaker(
bind=self.engine,
class_=Session,
expire_on_commit=False,
)
with self.Session() as session:
session.add(
MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Inbound",
slug="inbound",
smtp_config={},
)
)
session.commit()
self.addCleanup(self.engine.dispose)
def test_bridge_parses_bounded_content_and_delegates_idempotently(self) -> None:
delivery = _DeliveryProvider()
request = MailPostboxBridgeRequest(
tenant_id="tenant-1",
target=PostboxTargetRef(postbox_id="postbox-1"),
profile_id="profile-1",
folder="INBOX",
uid="42",
uidvalidity="20260807",
raw_message=RAW_MESSAGE,
)
with self.Session() as session, patch(
"govoplan_mail.backend.postbox_bridge.postbox_delivery_provider",
return_value=delivery,
):
first = MailPostboxBridge().bridge_message(session, request)
second = MailPostboxBridge().bridge_message(session, request)
MailPostboxBridge().bridge_message(
session,
replace(request, uidvalidity="20260808"),
)
self.assertFalse(first.duplicate)
self.assertTrue(second.duplicate)
self.assertEqual(first.source_digest, second.source_digest)
self.assertEqual(
delivery.requests[0].idempotency_key,
delivery.requests[1].idempotency_key,
)
self.assertNotEqual(
delivery.requests[0].idempotency_key,
delivery.requests[2].idempotency_key,
)
bridged = delivery.requests[0]
self.assertEqual("Submitted evidence", bridged.subject)
self.assertIn("Please process", bridged.body_text)
self.assertEqual(
["sender", "to", "cc"],
[participant.kind for participant in bridged.participants],
)
self.assertEqual(1, len(bridged.attachments))
self.assertEqual("evidence.pdf", bridged.attachments[0].name)
self.assertEqual("mail_attachment", bridged.attachments[0].reference_type)
self.assertEqual("<bridge-1@example.test>", bridged.metadata["rfc_message_id"])
self.assertEqual("20260807", bridged.metadata["mailbox_uidvalidity"])
self.assertNotIn("raw_message", bridged.metadata)
if __name__ == "__main__":
unittest.main()
+943
View File
@@ -0,0 +1,943 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import ANY, Mock, patch
from fastapi import HTTPException
from sqlalchemy.dialects import postgresql
from govoplan_mail.backend import router
from govoplan_mail.backend import mail_profiles
from govoplan_mail.backend.mail_profiles import (
EffectiveMailProfilePolicy,
MailProfileError,
get_mail_server_profile_for_actor,
list_mail_server_profiles,
mail_profile_visible_to_actor,
)
from govoplan_mail.backend.schemas import (
MailServerProfileCreateRequest,
MailServerProfileUpdateRequest,
)
def _profile(
profile_id: str,
*,
scope_type: str,
scope_id: str | None,
tenant_id: str | None = "tenant-1",
):
return SimpleNamespace(
id=profile_id,
tenant_id=tenant_id,
scope_type=scope_type,
scope_id=scope_id,
is_active=True,
name=profile_id,
smtp_config={"host": "smtp.example.test"},
imap_config={"host": "imap.example.test"},
smtp_password_encrypted=None,
imap_password_encrypted=None,
)
class _Query:
def __init__(self, profiles):
self._profiles = profiles
def filter(self, *_args):
return self
def all(self):
return list(self._profiles)
class _Session:
def __init__(self, profiles=()):
self.profiles = list(profiles)
self.commits = 0
self.rollbacks = 0
def query(self, _model):
return _Query(self.profiles)
def get(self, _model, profile_id):
return next((profile for profile in self.profiles if profile.id == profile_id), None)
def commit(self):
self.commits += 1
def rollback(self):
self.rollbacks += 1
def refresh(self, _value):
return None
class _Principal:
tenant_id = "tenant-1"
user = SimpleNamespace(id="user-1")
group_ids = frozenset({"group-1"})
api_key_id = None
def __init__(self, scopes):
self._scopes = frozenset(scopes)
def has(self, scope: str) -> bool:
return scope in self._scopes
class ProfileActorAuthorizationTests(unittest.TestCase):
def test_profile_mutation_selector_is_owner_bounded_and_row_locked(self) -> None:
statement = mail_profiles._mail_server_profile_mutation_statement( # noqa: SLF001
tenant_id="tenant-1",
profile_id="profile-1",
owner_user_id="user-1",
can_manage_tenant_profiles=False,
can_manage_system_profiles=False,
)
sql = str(statement.compile(dialect=postgresql.dialect()))
self.assertIn("FOR UPDATE", sql)
self.assertIn("mail_server_profiles.tenant_id =", sql)
self.assertIn("mail_server_profiles.scope_type =", sql)
self.assertIn("mail_server_profiles.scope_id =", sql)
self.assertTrue(statement.get_execution_options()["populate_existing"])
def test_profile_mutation_selector_without_authority_cannot_lock_a_row(self) -> None:
statement = mail_profiles._mail_server_profile_mutation_statement( # noqa: SLF001
tenant_id="tenant-1",
profile_id="profile-1",
owner_user_id=None,
can_manage_tenant_profiles=False,
can_manage_system_profiles=False,
)
sql = str(statement.compile(dialect=postgresql.dialect()))
self.assertIn("mail_server_profiles.id IS NULL", sql)
self.assertIn("FOR UPDATE", sql)
def test_self_service_profile_permissions_are_limited_to_own_user_scope(self) -> None:
principal = _Principal(
{"mail:profile:write_own", "mail:secret:manage_own"}
)
router._require_profile_write_scope( # noqa: SLF001 - authorization seam
principal, # type: ignore[arg-type]
"user",
"user-1",
)
router._require_profile_credentials_scope( # noqa: SLF001 - authorization seam
principal, # type: ignore[arg-type]
"user",
"user-1",
)
for scope_type, scope_id in (
("user", "user-2"),
("tenant", "tenant-1"),
("group", "group-1"),
("campaign", "campaign-1"),
("system", None),
):
with self.subTest(scope_type=scope_type, scope_id=scope_id):
with self.assertRaises(HTTPException) as write_denied:
router._require_profile_write_scope( # noqa: SLF001
principal, # type: ignore[arg-type]
scope_type,
scope_id,
)
self.assertEqual(write_denied.exception.status_code, 403)
with self.assertRaises(HTTPException) as secret_denied:
router._require_profile_credentials_scope( # noqa: SLF001
principal, # type: ignore[arg-type]
scope_type,
scope_id,
)
self.assertEqual(secret_denied.exception.status_code, 403)
def test_self_service_create_rejects_another_user_before_persistence(self) -> None:
principal = _Principal(
{
"mail:profile:read",
"mail:profile:write_own",
"mail:secret:manage_own",
}
)
payload = MailServerProfileCreateRequest.model_validate(
{
"name": "Other user's profile",
"scope_type": "user",
"scope_id": "user-2",
"smtp": {
"host": "smtp.example.test",
"password": "not-persisted",
},
}
)
with (
patch("govoplan_mail.backend.router.create_mail_server_profile") as create,
self.assertRaises(HTTPException) as denied,
):
router.create_profile(
payload,
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(denied.exception.status_code, 403)
create.assert_not_called()
def test_self_service_update_hides_another_user_before_mutation(self) -> None:
principal = _Principal(
{"mail:profile:write_own", "mail:secret:manage_own"}
)
profile = _profile(
"other-user-profile",
scope_type="user",
scope_id="user-2",
)
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
),
patch("govoplan_mail.backend.router.update_mail_server_profile") as update,
self.assertRaises(HTTPException) as denied,
):
router.update_profile(
profile.id,
MailServerProfileUpdateRequest(name="Must not change"),
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(denied.exception.status_code, 404)
update.assert_not_called()
def test_self_service_delete_hides_another_user_before_mutation(self) -> None:
principal = _Principal(
{"mail:profile:write_own", "mail:secret:manage_own"}
)
profile = _profile(
"other-user-profile",
scope_type="user",
scope_id="user-2",
)
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
),
patch("govoplan_mail.backend.router.delete_mail_profile_credentials") as delete,
self.assertRaises(HTTPException) as denied,
):
router.deactivate_profile(
profile.id,
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(denied.exception.status_code, 404)
delete.assert_not_called()
def test_mutation_lookup_returns_the_same_not_found_for_missing_and_non_owned(self) -> None:
principal = _Principal({"mail:profile:write_own"})
other_profile = _profile(
"other-user-profile",
scope_type="user",
scope_id="user-2",
)
cases = (
(other_profile, None),
(None, MailProfileError("Mail-server profile not found")),
)
for returned, failure in cases:
with self.subTest(failure=failure is not None):
kwargs = {"side_effect": failure} if failure else {"return_value": returned}
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
**kwargs,
),
self.assertRaises(HTTPException) as denied,
):
router._profile_for_mutation( # noqa: SLF001 - authorization seam
_Session(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
profile_id="candidate-id",
)
self.assertEqual(denied.exception.status_code, 404)
self.assertEqual(denied.exception.detail, "Mail-server profile not found")
def test_broad_profile_admin_retains_cross_owner_mutation_access(self) -> None:
principal = _Principal({"mail:profile:write"})
profile = _profile(
"other-user-profile",
scope_type="user",
scope_id="user-2",
)
with patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
) as get_profile:
resolved = router._profile_for_mutation( # noqa: SLF001 - authorization seam
_Session(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
profile_id=profile.id,
)
self.assertIs(resolved, profile)
get_profile.assert_called_once_with(
ANY,
tenant_id="tenant-1",
profile_id=profile.id,
for_update=True,
mutation_owner_user_id=None,
can_manage_tenant_profiles=True,
can_manage_system_profiles=False,
)
def test_self_service_transport_rebind_requires_own_secret_authority(self) -> None:
principal = _Principal({"mail:profile:write_own"})
profile = _profile(
"own-user-profile",
scope_type="user",
scope_id="user-1",
)
profile.smtp_config = {
"host": "smtp.example.test",
"port": 587,
"security": "starttls",
"timeout_seconds": 30,
}
profile.smtp_password_encrypted = "existing-ciphertext"
payload = MailServerProfileUpdateRequest.model_validate(
{"smtp": {"host": "smtp.attacker.test"}}
)
session = _Session()
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
),
patch("govoplan_mail.backend.router.update_mail_server_profile") as update,
self.assertRaises(HTTPException) as denied,
):
router.update_profile(
profile.id,
payload,
principal=principal, # type: ignore[arg-type]
session=session, # type: ignore[arg-type]
)
self.assertEqual(denied.exception.status_code, 403)
self.assertEqual(session.rollbacks, 1)
update.assert_not_called()
def test_self_service_imap_rebind_requires_own_secret_authority(self) -> None:
principal = _Principal({"mail:profile:write_own"})
profile = _profile(
"own-user-profile",
scope_type="user",
scope_id="user-1",
)
profile.imap_config = {
"host": "imap.example.test",
"port": 993,
"security": "tls",
"sent_folder": "auto",
"timeout_seconds": 30,
}
profile.imap_password_encrypted = "existing-ciphertext"
payload = MailServerProfileUpdateRequest.model_validate(
{"imap": {"host": "imap.attacker.test"}}
)
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
),
patch("govoplan_mail.backend.router.update_mail_server_profile") as update,
self.assertRaises(HTTPException) as denied,
):
router.update_profile(
profile.id,
payload,
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(denied.exception.status_code, 403)
update.assert_not_called()
def test_self_service_transport_rebind_is_allowed_with_own_secret_authority(self) -> None:
principal = _Principal(
{"mail:profile:write_own", "mail:secret:manage_own"}
)
profile = _profile(
"own-user-profile",
scope_type="user",
scope_id="user-1",
)
profile.smtp_config = {
"host": "smtp.example.test",
"port": 587,
"security": "starttls",
"timeout_seconds": 30,
}
profile.smtp_password_encrypted = "existing-ciphertext"
payload = MailServerProfileUpdateRequest.model_validate(
{"smtp": {"host": "smtp.allowed.test"}}
)
session = _Session()
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
),
patch(
"govoplan_mail.backend.router.update_mail_server_profile",
return_value=profile,
) as update,
patch("govoplan_mail.backend.router._record_mail_change"),
patch(
"govoplan_mail.backend.router._profile_response",
return_value=profile,
),
patch("govoplan_mail.backend.router.sync_default_profile_server"),
):
result = router.update_profile(
profile.id,
payload,
principal=principal, # type: ignore[arg-type]
session=session, # type: ignore[arg-type]
)
self.assertIs(result, profile)
self.assertEqual(session.commits, 1)
update.assert_called_once()
def test_transport_endpoint_change_without_stored_secret_needs_only_write_authority(self) -> None:
principal = _Principal({"mail:profile:write_own"})
profile = _profile(
"own-user-profile",
scope_type="user",
scope_id="user-1",
)
payload = MailServerProfileUpdateRequest.model_validate(
{"smtp": {"host": "smtp.allowed.test"}}
)
session = _Session()
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile",
return_value=profile,
),
patch(
"govoplan_mail.backend.router.update_mail_server_profile",
return_value=profile,
) as update,
patch("govoplan_mail.backend.router._record_mail_change"),
patch(
"govoplan_mail.backend.router._profile_response",
return_value=profile,
),
patch("govoplan_mail.backend.router.sync_default_profile_server"),
):
router.update_profile(
profile.id,
payload,
principal=principal, # type: ignore[arg-type]
session=session, # type: ignore[arg-type]
)
self.assertEqual(session.commits, 1)
update.assert_called_once()
def test_general_profile_list_hides_other_owner_contexts(self) -> None:
profiles = [
_profile("system", scope_type="system", scope_id=None, tenant_id=None),
_profile("tenant", scope_type="tenant", scope_id="tenant-1"),
_profile("own-user", scope_type="user", scope_id="user-1"),
_profile("other-user", scope_type="user", scope_id="user-2"),
_profile("own-group", scope_type="group", scope_id="group-1"),
_profile("other-group", scope_type="group", scope_id="group-2"),
]
session = _Session(profiles)
with patch(
"govoplan_mail.backend.mail_profiles.effective_mail_profile_policy",
return_value=EffectiveMailProfilePolicy(),
):
visible = list_mail_server_profiles(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
actor_user_id="user-1",
actor_group_ids=(group_id for group_id in ("group-1",)),
)
self.assertEqual(
{profile.id for profile in visible},
{"system", "tenant", "own-user", "own-group"},
)
def test_general_access_denies_shared_profile_excluded_by_effective_policy(self) -> None:
denied = EffectiveMailProfilePolicy(allowed_profile_id_sets=[{"approved"}])
cases = (("system", None), ("tenant", "tenant-1"))
for scope_type, tenant_id in cases:
with self.subTest(scope_type=scope_type):
profile = _profile("denied", scope_type=scope_type, scope_id=tenant_id, tenant_id=tenant_id)
with patch("govoplan_mail.backend.mail_profiles.effective_mail_profile_policy", return_value=denied):
visible = mail_profile_visible_to_actor(
_Session(), # type: ignore[arg-type]
profile=profile,
tenant_id="tenant-1",
user_id="user-1",
group_ids={"group-1"},
)
self.assertFalse(visible)
def test_administrative_list_can_include_inactive_visible_profiles(self) -> None:
profile = _profile("inactive", scope_type="tenant", scope_id="tenant-1")
profile.is_active = False
session = _Session([profile])
with patch(
"govoplan_mail.backend.mail_profiles.effective_mail_profile_policy",
return_value=EffectiveMailProfilePolicy(),
):
visible = list_mail_server_profiles(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
include_inactive=True,
actor_user_id="user-1",
actor_can_manage_tenant_profiles=True,
actor_administrative_visibility=True,
)
self.assertEqual([item.id for item in visible], ["inactive"])
def test_self_service_repair_visibility_is_limited_to_the_exact_owner(self) -> None:
own_profile = _profile(
"own-policy-invalid",
scope_type="user",
scope_id="user-1",
)
other_profile = _profile(
"other-policy-invalid",
scope_type="user",
scope_id="user-2",
)
denied = EffectiveMailProfilePolicy(
allowed_profile_id_sets=[{"different-profile"}]
)
session = _Session([own_profile, other_profile])
with patch(
"govoplan_mail.backend.mail_profiles.effective_mail_profile_policy",
return_value=denied,
):
ordinary_visible = list_mail_server_profiles(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
include_inactive=True,
actor_user_id="user-1",
actor_administrative_visibility=True,
)
repair_visible = list_mail_server_profiles(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
include_inactive=True,
actor_user_id="user-1",
actor_can_manage_own_profiles=True,
actor_administrative_visibility=True,
)
resolved = get_mail_server_profile_for_actor(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
profile_id=own_profile.id,
user_id="user-1",
can_manage_own_profiles=True,
administrative_visibility=True,
)
with self.assertRaises(MailProfileError):
get_mail_server_profile_for_actor(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
profile_id=other_profile.id,
user_id="user-1",
can_manage_own_profiles=True,
administrative_visibility=True,
)
self.assertEqual(ordinary_visible, [])
self.assertEqual([item.id for item in repair_visible], [own_profile.id])
self.assertIs(resolved, own_profile)
def test_profile_admin_can_list_but_not_use_a_policy_denied_profile(self) -> None:
profile = _profile("denied", scope_type="tenant", scope_id="tenant-1")
session = _Session([profile])
denied = EffectiveMailProfilePolicy(allowed_profile_id_sets=[{"approved"}])
with patch("govoplan_mail.backend.mail_profiles.effective_mail_profile_policy", return_value=denied):
listed = list_mail_server_profiles(
session, # type: ignore[arg-type]
tenant_id="tenant-1",
actor_user_id="user-1",
actor_group_ids={"group-1"},
actor_can_manage_tenant_profiles=True,
actor_administrative_visibility=True,
)
self.assertEqual([item.id for item in listed], ["denied"])
principal = _Principal(
{"mail:profile:write", "mail:profile:test", "mail:profile:use"}
)
with (
patch("govoplan_mail.backend.router.smtp_config_from_profile") as materialize,
patch("govoplan_mail.backend.router.test_smtp_login") as provider,
self.assertRaises(HTTPException) as captured,
):
router.test_profile_smtp(
"denied",
principal=principal, # type: ignore[arg-type]
session=session, # type: ignore[arg-type]
)
self.assertEqual(captured.exception.status_code, 404)
materialize.assert_not_called()
provider.assert_not_called()
def test_campaign_scoped_profile_requires_campaign_acl_before_policy_resolution(self) -> None:
profile = _profile("campaign-profile", scope_type="campaign", scope_id="campaign-1")
access = SimpleNamespace(can_read_campaign=lambda *_args, **_kwargs: False)
with (
patch("govoplan_mail.backend.mail_profiles._campaign_access_provider", return_value=access),
patch("govoplan_mail.backend.mail_profiles._campaign_policy_context") as policy_context,
):
visible = mail_profile_visible_to_actor(
_Session(), # type: ignore[arg-type]
profile=profile,
tenant_id="tenant-1",
user_id="user-1",
)
self.assertFalse(visible)
policy_context.assert_not_called()
def test_mail_profile_admin_does_not_bypass_campaign_acl_but_tenant_admin_does(self) -> None:
profile = _profile("campaign-profile", scope_type="campaign", scope_id="campaign-1")
tenant_admin_values: list[bool] = []
def can_read(*_args, **kwargs):
tenant_admin_values.append(bool(kwargs["tenant_admin"]))
return bool(kwargs["tenant_admin"])
access = SimpleNamespace(can_read_campaign=can_read)
with (
patch("govoplan_mail.backend.mail_profiles._campaign_access_provider", return_value=access),
patch("govoplan_mail.backend.mail_profiles._campaign_policy_context") as policy_context,
):
mail_admin_visible = mail_profile_visible_to_actor(
_Session(), # type: ignore[arg-type]
profile=profile,
tenant_id="tenant-1",
user_id="user-1",
can_manage_tenant_profiles=True,
administrative_visibility=True,
)
tenant_admin_visible = mail_profile_visible_to_actor(
_Session(), # type: ignore[arg-type]
profile=profile,
tenant_id="tenant-1",
user_id="user-1",
can_manage_tenant_profiles=True,
tenant_admin=True,
administrative_visibility=True,
)
self.assertFalse(mail_admin_visible)
self.assertTrue(tenant_admin_visible)
self.assertEqual(tenant_admin_values, [False, True])
policy_context.assert_not_called()
def test_policy_read_rejects_unshared_and_mismatched_campaign_contexts(self) -> None:
principal = _Principal({"mail:profile:read"})
with (
patch(
"govoplan_mail.backend.router.campaign_mail_context_visible_to_actor",
return_value=False,
),
patch("govoplan_mail.backend.router._policy_response") as policy_response,
self.assertRaises(HTTPException) as unshared,
):
router.read_mail_profile_policy(
"tenant",
scope_id="tenant-1",
campaign_id="campaign-1",
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(unshared.exception.status_code, 404)
policy_response.assert_not_called()
with self.assertRaises(HTTPException) as mismatched:
router._require_policy_campaign_context( # noqa: SLF001 - authorization seam regression
_Session(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
scope_type="campaign",
scope_id="campaign-1",
campaign_id="campaign-2",
)
self.assertEqual(mismatched.exception.status_code, 422)
def test_campaign_profile_crud_requires_campaign_acl(self) -> None:
principal = _Principal({"mail:profile:write", "mail:secret:manage"})
profile = _profile("campaign-profile", scope_type="campaign", scope_id="campaign-1")
create_payload = MailServerProfileCreateRequest.model_validate(
{
"name": "Campaign profile",
"scope_type": "campaign",
"scope_id": "campaign-1",
"smtp": {"host": "smtp.example.test"},
}
)
with (
patch(
"govoplan_mail.backend.router.campaign_mail_context_visible_to_actor",
return_value=False,
),
patch("govoplan_mail.backend.router.create_mail_server_profile") as create,
self.assertRaises(HTTPException) as create_denied,
):
router.create_profile(
create_payload,
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(create_denied.exception.status_code, 404)
create.assert_not_called()
for operation in ("update", "delete"):
with (
self.subTest(operation=operation),
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=profile),
patch(
"govoplan_mail.backend.router.campaign_mail_context_visible_to_actor",
return_value=False,
),
patch("govoplan_mail.backend.router.update_mail_server_profile") as update,
patch("govoplan_mail.backend.router.delete_mail_profile_credentials") as delete,
self.assertRaises(HTTPException) as denied,
):
if operation == "update":
router.update_profile(
profile.id,
MailServerProfileUpdateRequest(name="Renamed"),
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
else:
router.deactivate_profile(
profile.id,
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(denied.exception.status_code, 404)
update.assert_not_called()
delete.assert_not_called()
def test_inactive_visible_profile_change_remains_a_tombstone_candidate(self) -> None:
profile = _profile("deactivated", scope_type="tenant", scope_id="tenant-1")
profile.is_active = False
entry = SimpleNamespace(
resource_id=profile.id,
tenant_id="tenant-1",
payload={"scope_type": "tenant", "scope_id": "tenant-1"},
)
principal = _Principal({"mail:profile:read"})
with patch(
"govoplan_mail.backend.mail_profiles.effective_mail_profile_policy",
return_value=EffectiveMailProfilePolicy(),
):
visible = router._profile_change_visible_to_principal( # noqa: SLF001
_Session([profile]), # type: ignore[arg-type]
entry=entry, # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
)
self.assertTrue(visible)
def test_stale_mailbox_cache_refreshes_synchronously_under_request_authorization(self) -> None:
principal = _Principal({"mail:mailbox:read", "mail:profile:use"})
session = _Session()
imap = SimpleNamespace(
host="imap.example.test",
port=993,
security=SimpleNamespace(value="tls"),
)
stale = SimpleNamespace(stale=True)
provider_result = SimpleNamespace(
host="imap.example.test",
port=993,
security="tls",
folders=[
SimpleNamespace(
name="INBOX",
flags=[],
message_count=0,
unseen_count=0,
)
],
detected_sent_folder=None,
)
recovery = SimpleNamespace(
operation=SimpleNamespace(closed=False),
reject=Mock(),
complete_folders=Mock(),
)
with (
patch("govoplan_mail.backend.router._imap_config_for_principal", return_value=imap),
patch("govoplan_mail.backend.router.cached_mailbox_folders", return_value=stale),
patch("govoplan_mail.backend.router.list_imap_folders", return_value=provider_result) as provider,
patch("govoplan_mail.backend.router.cache_mailbox_folders") as cache,
patch(
"govoplan_mail.backend.router.begin_mailbox_refresh_recovery",
return_value=recovery,
),
):
response = router.list_profile_mailbox_folders(
"profile-1",
include_status=False,
refresh=False,
principal=principal, # type: ignore[arg-type]
session=session, # type: ignore[arg-type]
)
provider.assert_called_once_with(imap_config=imap, include_status=False)
cache.assert_called_once()
recovery.complete_folders.assert_called_once_with(provider_result)
self.assertFalse(response.from_cache)
self.assertFalse(response.refreshing)
self.assertEqual(session.commits, 1)
def test_jmap_mailbox_list_uses_protocol_neutral_response_and_provider_search(self) -> None:
principal = _Principal({"mail:mailbox:read", "mail:profile:use"})
jmap = SimpleNamespace(session_url="https://jmap.example.test/.well-known/jmap")
provider_result = SimpleNamespace(
host="jmap.example.test",
port=443,
security="https",
folder="INBOX",
messages=[],
total_count=0,
offset=0,
limit=25,
uidvalidity="query-state-1",
cursor_reset=False,
)
with (
patch("govoplan_mail.backend.router._jmap_config_for_principal", return_value=jmap),
patch("govoplan_mail.backend.router.list_jmap_messages", return_value=provider_result) as provider,
patch("govoplan_mail.backend.router._next_jmap_mailbox_cursor", return_value=(None, True)),
):
response = router.list_profile_mailbox_messages(
"profile-1",
folder="INBOX",
limit=25,
offset=0,
cursor=None,
refresh=False,
protocol="jmap",
q="budget",
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
provider.assert_called_once_with(
jmap_config=jmap,
folder="INBOX",
limit=25,
offset=0,
expected_query_state=None,
query="budget",
)
self.assertEqual("profile-1", response.profile_id)
self.assertEqual("INBOX", response.folder)
self.assertEqual([], response.messages)
self.assertTrue(response.cursor_stable)
def test_jmap_changes_maps_bounded_incremental_state(self) -> None:
principal = _Principal({"mail:mailbox:read", "mail:profile:use"})
jmap = SimpleNamespace(session_url="https://jmap.example.test/.well-known/jmap")
changes = SimpleNamespace(
account_id="account-1",
old_state="state-1",
new_state="state-2",
has_more_changes=False,
created=("email-1",),
updated=("email-2",),
destroyed=("email-3",),
)
with (
patch("govoplan_mail.backend.router._jmap_config_for_principal", return_value=jmap),
patch("govoplan_mail.backend.router.get_jmap_email_changes", return_value=changes) as provider,
):
response = router.get_profile_mailbox_changes(
"profile-1",
since_state="state-1",
max_changes=50,
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
provider.assert_called_once_with(
jmap_config=jmap,
since_state="state-1",
max_changes=50,
)
self.assertEqual("jmap", response.protocol)
self.assertEqual(["email-1"], response.created)
self.assertEqual("state-2", response.new_state)
def test_unauthorized_profile_test_fails_before_credentials_or_provider_effect(self) -> None:
principal = _Principal({"mail:profile:test", "mail:profile:use"})
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile_for_actor",
side_effect=MailProfileError("Mail-server profile not found"),
),
patch("govoplan_mail.backend.router.smtp_config_from_profile") as materialize,
patch("govoplan_mail.backend.router.test_smtp_login") as provider,
):
with self.assertRaises(HTTPException) as captured:
router.test_profile_smtp(
"other-user-profile",
principal=principal, # type: ignore[arg-type]
session=_Session(), # type: ignore[arg-type]
)
self.assertEqual(captured.exception.status_code, 404)
materialize.assert_not_called()
provider.assert_not_called()
def test_unauthorized_mailbox_read_fails_before_credential_decryption(self) -> None:
principal = _Principal({"mail:mailbox:read", "mail:profile:use"})
with (
patch(
"govoplan_mail.backend.router.get_mail_server_profile_for_actor",
side_effect=MailProfileError("Mail-server profile not found"),
),
patch("govoplan_mail.backend.router.imap_config_from_profile") as materialize,
):
with self.assertRaisesRegex(MailProfileError, "not found"):
router._imap_config_for_principal( # noqa: SLF001 - authorization seam regression
_Session(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
profile_id="other-group-profile",
)
materialize.assert_not_called()
if __name__ == "__main__":
unittest.main()
+240
View File
@@ -0,0 +1,240 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db import models as access_models # noqa: F401
from govoplan_core.core.provider_governance import ExternalProviderStateContext
from govoplan_core.db.base import Base
from govoplan_mail.backend.db.models import (
MailBounceSource,
MailDeliveryCommand,
MailMailboxFolderIndex,
MailMailboxMessageIndex,
MailPop3Import,
MailServerEndpoint,
MailServerProfile,
)
from govoplan_mail.backend.manifest import manifest
from govoplan_mail.backend.provider_state import (
IMAP_PROVIDER_ID,
JMAP_PROVIDER_ID,
POP3_PROVIDER_ID,
SMTP_PROVIDER_ID,
imap_provider_states,
jmap_provider_states,
pop3_provider_states,
smtp_provider_states,
)
class MailProviderStateTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
Base.metadata.create_all(
self.engine,
tables=(
MailServerProfile.__table__,
MailServerEndpoint.__table__,
MailDeliveryCommand.__table__,
MailMailboxFolderIndex.__table__,
MailMailboxMessageIndex.__table__,
MailBounceSource.__table__,
MailPop3Import.__table__,
),
)
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
self.profile = MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Mail",
slug="mail",
is_active=True,
smtp_config={"host": "smtp.example.test", "port": 587},
imap_config={"host": "imap.example.test", "port": 993},
)
self.session.add(
self.profile
)
self.session.add(
MailMailboxFolderIndex(
id="folder-1",
tenant_id="tenant-1",
profile_id=self.profile.id,
folder="INBOX",
indexed_at=datetime.now(UTC),
)
)
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_smtp_state_exposes_reconciliation_without_server_or_secrets(self) -> None:
self.session.add(
MailDeliveryCommand(
id="command-1",
tenant_id="tenant-1",
command_type="message",
source_module="notifications",
source_resource_type="notification",
idempotency_key="notification-1",
canonical_request_hash="a" * 64,
profile_id=self.profile.id,
expected_smtp_transport_revision="revision-1",
message_sha256="b" * 64,
message_size_bytes=10,
recipient_count=1,
status="outcome_unknown",
expires_at=datetime.now(UTC) + timedelta(days=1),
)
)
self.session.flush()
state = smtp_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual("warning", state.health)
self.assertEqual("pending", state.conflict)
self.assertEqual("attention", state.recovery)
self.assertEqual(1, state.metrics["outcome_unknown_commands"])
self.assertNotIn("smtp.example.test", str(state.to_dict()))
def test_imap_state_is_fresh_tenant_bounded_and_registered(self) -> None:
state = imap_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual("healthy", state.health)
self.assertEqual("current", state.freshness)
self.assertEqual("external_mirror", state.authority_mode)
self.assertNotIn("imap.example.test", str(state.to_dict()))
self.assertEqual(
(),
imap_provider_states(
ExternalProviderStateContext(
session=self.session,
tenant_id="tenant-2",
)
),
)
self.assertEqual(
{SMTP_PROVIDER_ID, IMAP_PROVIDER_ID, JMAP_PROVIDER_ID, POP3_PROVIDER_ID},
{item.id for item in manifest.external_providers},
)
self.assertEqual(
{SMTP_PROVIDER_ID, IMAP_PROVIDER_ID, JMAP_PROVIDER_ID, POP3_PROVIDER_ID},
{
item.provider_id
for item in manifest.external_provider_state_providers
},
)
def test_jmap_state_reuses_protocol_neutral_index_without_endpoint_details(self) -> None:
self.session.add(
MailServerEndpoint(
id="jmap-server-1",
profile_id=self.profile.id,
tenant_id="tenant-1",
protocol="jmap",
name="JMAP",
config={"session_url": "https://jmap.example.test/.well-known/jmap"},
scope_type="tenant",
scope_id="tenant-1",
is_active=True,
)
)
self.session.flush()
state = jmap_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual(JMAP_PROVIDER_ID, state.provider_id)
self.assertEqual("healthy", state.health)
self.assertEqual("current", state.freshness)
self.assertEqual("mail:profile:profile-1:jmap", state.binding_ref)
self.assertNotIn("jmap.example.test", str(state.to_dict()))
def test_pop3_state_is_disabled_by_default_and_projects_deletion_evidence(self) -> None:
endpoint = MailServerEndpoint(
id="pop3-server-1",
profile_id=self.profile.id,
tenant_id="tenant-1",
protocol="pop3",
name="Legacy POP3",
config={"host": "pop3.example.test", "legacy_import_enabled": False},
scope_type="tenant",
scope_id="tenant-1",
is_active=True,
)
self.session.add(endpoint)
self.session.flush()
disabled = pop3_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertFalse(disabled.active)
self.assertEqual("inactive", disabled.health)
endpoint.config = {
"host": "pop3.example.test",
"legacy_import_enabled": True,
}
self.session.add(
MailPop3Import(
id="pop3-import-1",
tenant_id="tenant-1",
profile_id=self.profile.id,
pop3_server_id=endpoint.id,
transport_revision=endpoint.transport_revision,
provider_uidl="uid-1",
fingerprint="a" * 64,
raw_sha256="b" * 64,
raw_message_encrypted="ciphertext-do-not-project",
size_bytes=42,
imported_at=datetime.now(UTC),
deletion_requested=True,
deletion_status="outcome_unknown",
)
)
self.session.add(
MailPop3Import(
id="pop3-import-tenant-2",
tenant_id="tenant-2",
profile_id=self.profile.id,
pop3_server_id=endpoint.id,
transport_revision=endpoint.transport_revision,
provider_uidl="uid-tenant-2",
fingerprint="c" * 64,
raw_sha256="d" * 64,
raw_message_encrypted="other-tenant-ciphertext",
size_bytes=42,
imported_at=datetime.now(UTC),
deletion_requested=True,
deletion_status="failed",
)
)
self.session.flush()
state = pop3_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertTrue(state.active)
self.assertEqual("warning", state.health)
self.assertEqual("pending", state.conflict)
self.assertEqual(1, state.metrics["outcome_unknown_deletions"])
self.assertEqual(0, state.metrics["failed_deletions"])
self.assertNotIn("pop3.example.test", str(state.to_dict()))
self.assertNotIn("ciphertext-do-not-project", str(state.to_dict()))
if __name__ == "__main__":
unittest.main()
+80
View File
@@ -0,0 +1,80 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from redis.exceptions import RedisError
from govoplan_mail.backend.sending import rate_limit
class RateLimitTests(unittest.TestCase):
def setUp(self) -> None:
rate_limit._local_next_allowed.clear()
def test_local_fallback_waits_between_sends(self) -> None:
now = [100.0]
sleeps: list[float] = []
def fake_time() -> float:
return now[0]
def fake_sleep(seconds: float) -> None:
sleeps.append(seconds)
now[0] += seconds
with (
patch.object(rate_limit, "_distributed_rate_limit_enabled", return_value=False),
patch.object(rate_limit.time, "time", fake_time),
patch.object(rate_limit.time, "sleep", fake_sleep),
):
first = rate_limit.wait_for_rate_limit(key="tenant:t:campaign:c", messages_per_minute=4)
second = rate_limit.wait_for_rate_limit(key="tenant:t:campaign:c", messages_per_minute=4)
self.assertEqual(first.waited_seconds, 0.0)
self.assertEqual(second.waited_seconds, 15.0)
self.assertEqual(sleeps, [15.0])
def test_disabled_rate_limit_does_not_reserve_local_slot(self) -> None:
now = [100.0]
sleeps: list[float] = []
with (
patch.object(rate_limit, "_distributed_rate_limit_enabled", return_value=False),
patch.object(rate_limit.time, "time", lambda: now[0]),
patch.object(rate_limit.time, "sleep", lambda seconds: sleeps.append(seconds)),
):
disabled = rate_limit.wait_for_rate_limit(key="k", messages_per_minute=4, enabled=False)
enabled = rate_limit.wait_for_rate_limit(key="k", messages_per_minute=4)
self.assertEqual(disabled.waited_seconds, 0.0)
self.assertEqual(enabled.waited_seconds, 0.0)
self.assertEqual(sleeps, [])
def test_redis_failure_falls_back_to_local_wait(self) -> None:
now = [100.0]
sleeps: list[float] = []
def fail_redis_client():
raise RedisError("redis unavailable")
def fake_sleep(seconds: float) -> None:
sleeps.append(seconds)
now[0] += seconds
with (
patch.object(rate_limit, "_distributed_rate_limit_enabled", return_value=True),
patch.object(rate_limit, "_redis_client", fail_redis_client),
patch.object(rate_limit.time, "time", lambda: now[0]),
patch.object(rate_limit.time, "sleep", fake_sleep),
):
first = rate_limit.wait_for_rate_limit(key="k", messages_per_minute=60)
second = rate_limit.wait_for_rate_limit(key="k", messages_per_minute=60)
self.assertEqual(first.waited_seconds, 0.0)
self.assertEqual(second.waited_seconds, 1.0)
self.assertEqual(sleeps, [1.0])
if __name__ == "__main__":
unittest.main()
+312
View File
@@ -0,0 +1,312 @@
from __future__ import annotations
import json
from pathlib import Path
import tempfile
import unittest
from sqlalchemy import Column, String, Table, create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from govoplan_core.core.recovery import (
RecoveryCheckpoint,
RecoveryGuaranteeError,
RecoveryOperation,
RecoveryStatus,
)
from govoplan_core.core.runtime_coordination import (
DistributedLease,
RuntimeIdentity,
bind_process_runtime_identity,
)
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, reset_database
from govoplan_mail.backend.db.models import (
MailMailboxFolderIndex,
MailMailboxMessageIndex,
MailServerProfile,
)
from govoplan_mail.backend.mailbox_index import cache_mailbox_folders
from govoplan_mail.backend.recovery import (
MailRecoveryError,
MailboxRefreshBusy,
begin_mailbox_refresh_recovery,
begin_provider_effect_recovery,
reconcile_outbox_provider_effect,
)
from govoplan_mail.backend.sending.imap import (
ImapFolderListResult,
ImapMailboxInfo,
)
class MailRecoveryTests(unittest.TestCase):
def setUp(self) -> None:
self.tempdir = tempfile.TemporaryDirectory()
self.addCleanup(self.tempdir.cleanup)
database_path = Path(self.tempdir.name) / "mail-recovery.sqlite3"
self.engine = create_engine(f"sqlite:///{database_path}")
access_users = Base.metadata.tables.get("access_users")
if access_users is None:
access_users = Table(
"access_users",
Base.metadata,
Column("id", String(36), primary_key=True),
)
Base.metadata.create_all(
self.engine,
tables=[
access_users,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
MailServerProfile.__table__,
MailMailboxFolderIndex.__table__,
MailMailboxMessageIndex.__table__,
],
)
configure_database(
f"sqlite:///{database_path}",
engine=self.engine,
dispose_previous=True,
)
bind_process_runtime_identity(
RuntimeIdentity(
installation_id="mail-recovery-tests",
node_id="mail-test-node",
incarnation="mail-test-incarnation",
role="worker",
software_version="test",
composition_hash="a" * 64,
)
)
self.SessionLocal = sessionmaker(
bind=self.engine,
class_=Session,
expire_on_commit=False,
)
with self.SessionLocal() as session:
session.add(
MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Recovery profile",
slug="recovery-profile",
smtp_config={},
)
)
session.commit()
self.addCleanup(self._cleanup_runtime)
def _cleanup_runtime(self) -> None:
bind_process_runtime_identity(None)
reset_database()
self.engine.dispose()
def test_smtp_effect_is_durable_before_provider_and_redacted_on_success(self) -> None:
recovery = begin_provider_effect_recovery(
kind="smtp-delivery",
effect_id="outbox:command-1:smtp-attempt:1",
tenant_id="tenant-1",
profile_id="profile-1",
message_bytes=b"Subject: Recovery\r\n\r\nBody",
expected_transport_revision="revision-1",
recipient_count=1,
resource_type="mail_delivery_command",
resource_id="command-1",
)
assert recovery is not None and recovery.operation is not None
with self.SessionLocal() as session:
operation = session.get(RecoveryOperation, recovery.operation_id)
assert operation is not None
self.assertEqual(RecoveryStatus.RUNNING.value, operation.status)
recovery.succeed_smtp(
accepted_count=1,
refused_recipients={},
)
with self.SessionLocal() as session:
operation = session.get(RecoveryOperation, recovery.operation_id)
assert operation is not None
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
evidence = json.dumps(
[
item.evidence
for item in session.scalars(
select(RecoveryCheckpoint).where(
RecoveryCheckpoint.operation_id == operation.id
)
)
]
)
self.assertNotIn("recipient@example.test", evidence)
self.assertNotIn("Subject: Recovery", evidence)
def test_unknown_smtp_outcome_blocks_replay_until_reconciled(self) -> None:
kwargs = {
"kind": "smtp-delivery",
"effect_id": "outbox:command-2:smtp-attempt:1",
"tenant_id": "tenant-1",
"profile_id": "profile-1",
"message_bytes": b"Subject: Unknown\r\n\r\nBody",
"expected_transport_revision": "revision-1",
"recipient_count": 1,
"resource_type": "mail_delivery_command",
"resource_id": "command-2",
}
recovery = begin_provider_effect_recovery(**kwargs)
assert recovery is not None
recovery.unknown(
code="socket_closed_after_data",
summary="SMTP outcome is unknown",
)
with self.assertRaises(MailRecoveryError):
begin_provider_effect_recovery(**kwargs)
self.assertTrue(
reconcile_outbox_provider_effect(
command_id="command-2",
effect_occurred=False,
evidence_reference="provider-case-42",
user_id="operator-1",
)
)
with self.SessionLocal() as session:
operation = session.get(RecoveryOperation, recovery.operation_id)
assert operation is not None
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
def test_mailbox_refresh_verifies_the_committed_index(self) -> None:
result = ImapFolderListResult(
host="imap.example.test",
port=993,
security="tls",
folders=[
ImapMailboxInfo(
name="INBOX",
flags=["\\HasNoChildren"],
message_count=4,
unseen_count=1,
)
],
)
recovery = begin_mailbox_refresh_recovery(
tenant_id="tenant-1",
profile_id="profile-1",
folder="*",
purpose="folders",
)
with self.SessionLocal() as session:
session.add(
MailMailboxFolderIndex(
tenant_id="tenant-1",
profile_id="profile-1",
folder="Removed",
flags=[],
)
)
session.commit()
cache_mailbox_folders(
session,
tenant_id="tenant-1",
profile_id="profile-1",
result=result,
)
session.commit()
recovery.complete_folders(result)
with self.SessionLocal() as session:
self.assertEqual(
["INBOX"],
list(
session.scalars(
select(MailMailboxFolderIndex.folder).order_by(
MailMailboxFolderIndex.folder
)
)
),
)
operation = session.get(
RecoveryOperation,
recovery.operation.operation_id,
)
assert operation is not None
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
def test_missing_runtime_identity_fails_before_a_provider_effect_can_start(self) -> None:
bind_process_runtime_identity(None)
with self.assertRaises(MailRecoveryError):
begin_provider_effect_recovery(
kind="imap-append",
effect_id="campaign-job:job-1:imap-attempt:1",
tenant_id="tenant-1",
profile_id="profile-1",
message_bytes=b"message",
expected_transport_revision="revision-1",
folder="Sent",
)
def test_tampered_provider_evidence_cannot_be_marked_successful(self) -> None:
recovery = begin_provider_effect_recovery(
kind="smtp-delivery",
effect_id="outbox:command-3:smtp-attempt:1",
tenant_id="tenant-1",
profile_id="profile-1",
message_bytes=b"message",
expected_transport_revision="revision-1",
recipient_count=1,
resource_type="mail_delivery_command",
resource_id="command-3",
)
assert recovery is not None
with self.SessionLocal() as session:
checkpoint = session.scalar(
select(RecoveryCheckpoint)
.where(RecoveryCheckpoint.operation_id == recovery.operation_id)
.order_by(RecoveryCheckpoint.sequence)
.limit(1)
)
assert checkpoint is not None
checkpoint.summary = "tampered"
session.commit()
with self.assertRaises(RecoveryGuaranteeError):
recovery.succeed_smtp(accepted_count=1, refused_recipients={})
with self.SessionLocal() as session:
operation = session.get(RecoveryOperation, recovery.operation_id)
assert operation is not None
self.assertNotEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
def test_mailbox_refresh_has_a_cross_runtime_fence(self) -> None:
recovery = begin_mailbox_refresh_recovery(
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
purpose="messages",
)
bind_process_runtime_identity(
RuntimeIdentity(
installation_id="mail-recovery-tests",
node_id="mail-test-node-2",
incarnation="mail-test-incarnation-2",
role="worker",
software_version="test",
composition_hash="a" * 64,
)
)
with self.assertRaises(MailboxRefreshBusy):
begin_mailbox_refresh_recovery(
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
purpose="messages",
)
recovery.reject(summary="Test refresh stopped", code="test")
if __name__ == "__main__":
unittest.main()
+201
View File
@@ -0,0 +1,201 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from fastapi import HTTPException
from govoplan_mail.backend.router import create_profile, deactivate_profile, update_profile
from govoplan_mail.backend.schemas import MailServerProfileCreateRequest, MailServerProfileUpdateRequest
class _Session:
def __init__(self) -> None:
self.commits = 0
self.rollbacks = 0
def add(self, _value) -> None:
return None
def commit(self) -> None:
self.commits += 1
def rollback(self) -> None:
self.rollbacks += 1
def refresh(self, _value) -> None:
return None
class MailProfileDeletionRouteTests(unittest.TestCase):
def setUp(self) -> None:
self.principal = SimpleNamespace(
tenant_id="tenant-1",
user=SimpleNamespace(id="user-1"),
api_key_id=None,
has=lambda _scope: False,
)
self.profile = SimpleNamespace(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
is_active=False,
)
def test_repeated_delete_does_not_emit_a_false_change(self) -> None:
session = _Session()
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=self.profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router._require_profile_credentials_scope") as require_credentials,
patch("govoplan_mail.backend.router.delete_mail_profile_credentials", return_value=()),
patch("govoplan_mail.backend.router.clear_mailbox_index") as clear_index,
patch("govoplan_mail.backend.router._record_mail_change") as record_change,
patch("govoplan_mail.backend.router._profile_response", return_value=self.profile),
):
result = deactivate_profile("profile-1", principal=self.principal, session=session)
self.assertIs(result, self.profile)
self.assertEqual(session.commits, 1)
self.assertEqual(session.rollbacks, 0)
clear_index.assert_called_once_with(session, profile_id="profile-1")
record_change.assert_not_called()
require_credentials.assert_not_called()
def test_delete_requires_secret_authority_when_credentials_will_be_deleted(self) -> None:
self.profile.is_active = True
self.profile.smtp_password_encrypted = "existing-ciphertext"
self.profile.imap_password_encrypted = None
session = _Session()
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=self.profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router._require_profile_credentials_scope") as require_credentials,
patch(
"govoplan_mail.backend.router.delete_mail_profile_credentials",
return_value=("smtp",),
),
patch("govoplan_mail.backend.router.clear_mailbox_index"),
patch("govoplan_mail.backend.router._record_mail_change"),
patch("govoplan_mail.backend.router._profile_response", return_value=self.profile),
):
deactivate_profile("profile-1", principal=self.principal, session=session)
require_credentials.assert_called_once_with(
self.principal,
"tenant",
"tenant-1",
)
def test_change_feed_reports_whether_credentials_were_deleted(self) -> None:
self.profile.is_active = True
session = _Session()
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=self.profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router._require_profile_credentials_scope"),
patch("govoplan_mail.backend.router.delete_mail_profile_credentials", return_value=()),
patch("govoplan_mail.backend.router.clear_mailbox_index") as clear_index,
patch("govoplan_mail.backend.router._record_mail_change") as record_change,
patch("govoplan_mail.backend.router._profile_response", return_value=self.profile),
):
deactivate_profile("profile-1", principal=self.principal, session=session)
self.assertFalse(record_change.call_args.kwargs["payload"]["credentials_deleted"])
self.assertEqual(record_change.call_args.kwargs["payload"]["deleted_credential_protocols"], [])
clear_index.assert_called_once_with(session, profile_id="profile-1")
def test_generic_secret_deletion_failure_rolls_back(self) -> None:
self.profile.is_active = True
session = _Session()
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=self.profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router._require_profile_credentials_scope"),
patch("govoplan_mail.backend.router.delete_mail_profile_credentials", side_effect=RuntimeError("audit unavailable")),
):
with self.assertRaisesRegex(RuntimeError, "audit unavailable"):
deactivate_profile("profile-1", principal=self.principal, session=session)
self.assertEqual(session.commits, 0)
self.assertEqual(session.rollbacks, 1)
def test_patch_deactivation_is_rejected_in_favor_of_audited_delete(self) -> None:
self.profile.is_active = True
self.profile.smtp_password_encrypted = "existing-ciphertext"
session = _Session()
payload = MailServerProfileUpdateRequest(is_active=False)
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=self.profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
self.assertRaises(HTTPException) as captured,
):
update_profile(
self.profile.id,
payload,
principal=self.principal,
session=session,
)
self.assertEqual(captured.exception.status_code, 422)
self.assertTrue(self.profile.is_active)
self.assertEqual(self.profile.smtp_password_encrypted, "existing-ciphertext")
self.assertEqual(session.commits, 0)
self.assertEqual(session.rollbacks, 1)
def test_system_profile_changes_are_instance_wide_in_the_change_feed(self) -> None:
system_profile = SimpleNamespace(
id="profile-system",
tenant_id=None,
scope_type="system",
scope_id=None,
is_active=True,
smtp_config={"host": "smtp.example.test"},
imap_config=None,
)
session = _Session()
create_payload = MailServerProfileCreateRequest.model_validate(
{
"name": "System profile",
"scope_type": "system",
"smtp": {"host": "smtp.example.test"},
}
)
with (
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router.create_mail_server_profile", return_value=system_profile),
patch("govoplan_mail.backend.router.initialize_profile_hierarchy"),
patch("govoplan_mail.backend.router._record_mail_change") as create_change,
patch("govoplan_mail.backend.router._profile_response", return_value=system_profile),
):
create_profile(create_payload, principal=self.principal, session=session)
self.assertIsNone(create_change.call_args.kwargs["tenant_id"])
update_payload = MailServerProfileUpdateRequest(name="Renamed system profile")
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=system_profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router.update_mail_server_profile", return_value=system_profile),
patch("govoplan_mail.backend.router._record_mail_change") as update_change,
patch("govoplan_mail.backend.router._profile_response", return_value=system_profile),
):
update_profile("profile-system", update_payload, principal=self.principal, session=session)
self.assertIsNone(update_change.call_args.kwargs["tenant_id"])
with (
patch("govoplan_mail.backend.router.get_mail_server_profile", return_value=system_profile),
patch("govoplan_mail.backend.router._require_profile_write_scope"),
patch("govoplan_mail.backend.router._require_profile_credentials_scope"),
patch("govoplan_mail.backend.router.delete_mail_profile_credentials", return_value=()),
patch("govoplan_mail.backend.router.clear_mailbox_index"),
patch("govoplan_mail.backend.router._record_mail_change") as delete_change,
patch("govoplan_mail.backend.router._profile_response", return_value=system_profile),
):
deactivate_profile("profile-system", principal=self.principal, session=session)
self.assertIsNone(delete_change.call_args.kwargs["tenant_id"])
if __name__ == "__main__":
unittest.main()
+144
View File
@@ -0,0 +1,144 @@
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_access.backend.db.models import Account, User
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.search import (
SearchAuthorizationRequest,
SearchBackfillRequest,
SearchResourceReference,
)
from govoplan_core.db.base import Base
from govoplan_mail.backend.db.models import (
MailMailboxMessageIndex,
MailServerProfile,
)
from govoplan_mail.backend.search_source import (
MailSearchSource,
PROVIDER_ID,
RESOURCE_TYPE,
)
class MailSearchSourceTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite://")
Base.metadata.create_all(
self.engine,
tables=(
Account.__table__,
User.__table__,
MailServerProfile.__table__,
MailMailboxMessageIndex.__table__,
),
)
self.session = Session(self.engine)
self.session.add_all(
(
Account(
id="account-1",
email="one@example.test",
normalized_email="one@example.test",
),
User(
id="user-1",
tenant_id="tenant-1",
account_id="account-1",
email="one@example.test",
),
MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="user",
scope_id="user-1",
name="Personal mailbox",
slug="personal",
smtp_config={},
imap_config={"host": "imap.example.test"},
),
MailMailboxMessageIndex(
id="message-1",
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
uid="42",
subject="Permit status",
from_header="service@example.test",
body_preview="Your permit is ready",
indexed_at=datetime(2026, 8, 5, tzinfo=timezone.utc),
),
)
)
self.session.commit()
self.source = MailSearchSource()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_mailbox_backfill_and_profile_recheck_are_bounded(self) -> None:
page = self.source.backfill(
self.session,
request=SearchBackfillRequest(
tenant_id="tenant-1",
provider_id=PROVIDER_ID,
resource_type=RESOURCE_TYPE,
rebuild_id="rebuild-1",
),
)
self.assertEqual(("message-1",), tuple(doc.resource_id for doc in page.documents))
self.assertNotIn("password", str(page.documents[0].metadata).casefold())
reference = SearchResourceReference(
tenant_id="tenant-1",
module_id="mail",
resource_type=RESOURCE_TYPE,
resource_id="message-1",
)
request = SearchAuthorizationRequest(reference=reference, source_revision="1")
with patch(
"govoplan_mail.backend.search_source.mail_profile_visible_to_actor",
return_value=True,
):
allowed = self.source.authorize(
self.session,
_principal(),
requests=(request,),
)
self.assertTrue(allowed[reference.key])
self.assertFalse(
self.source.authorize(
self.session,
_principal(scopes={"mail:mailbox:read"}),
requests=(request,),
)[reference.key]
)
def _principal(
*,
scopes: set[str] | None = None,
) -> ApiPrincipal:
return ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="user-1",
tenant_id="tenant-1",
scopes=frozenset(
scopes or {"mail:mailbox:read", "mail:profile:use"}
),
),
account=SimpleNamespace(id="account-1"),
user=SimpleNamespace(id="user-1"),
)
if __name__ == "__main__":
unittest.main()
+182
View File
@@ -0,0 +1,182 @@
from __future__ import annotations
import smtplib
import unittest
from unittest.mock import patch
from govoplan_core.security.outbound_http import OutboundHttpBlocked
from govoplan_mail.backend.config import SmtpConfig
from govoplan_mail.backend.sending.smtp import (
SmtpConfigurationError,
SmtpBatchPolicy,
SmtpBatchSession,
SmtpSendError,
_open_smtp,
_prepare_smtp_send,
_smtp_send_result,
)
class SmtpSendHelperTests(unittest.TestCase):
def test_real_smtp_connections_honor_deployment_egress_policy(self):
config = SmtpConfig(host="smtp.internal", port=587, security="starttls")
with patch(
"govoplan_mail.backend.sending.smtp.validate_outbound_host",
side_effect=OutboundHttpBlocked("private network blocked"),
), self.assertRaisesRegex(SmtpConfigurationError, "private network blocked"):
_open_smtp(config)
def test_smtp_revalidates_and_pins_at_connection_time(self):
config = SmtpConfig(host="smtp.example.test", port=587, security="starttls")
public = [(2, 1, 6, "", ("93.184.216.34", 587))]
private = [(2, 1, 6, "", ("127.0.0.1", 587))]
with patch.dict(
"os.environ",
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
side_effect=(public, private),
), patch("govoplan_core.security.outbound_http.socket.socket") as socket_factory, self.assertRaisesRegex(
OutboundHttpBlocked,
"non-public network",
):
_open_smtp(config)
socket_factory.assert_not_called()
def test_prepare_smtp_send_validates_envelope(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
with self.assertRaisesRegex(SmtpConfigurationError, "envelope sender"):
_prepare_smtp_send(smtp_config=config, envelope_from="", envelope_recipients=["user@example.org"])
with self.assertRaisesRegex(SmtpConfigurationError, "recipient"):
_prepare_smtp_send(smtp_config=config, envelope_from="sender@example.org", envelope_recipients=[""])
def test_prepare_smtp_send_filters_blank_recipients(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
self.assertEqual(
_prepare_smtp_send(
smtp_config=config,
envelope_from="sender@example.org",
envelope_recipients=["", "user@example.org"],
),
("smtp.example.org", 587, ["user@example.org"]),
)
def test_smtp_send_result_decodes_refused_recipients(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
result = _smtp_send_result(
smtp_config=config,
host="smtp.example.org",
port=587,
envelope_from="sender@example.org",
envelope_recipients=["ok@example.org", "blocked@example.org"],
refused={"blocked@example.org": (550, b"blocked")},
)
self.assertEqual(result.accepted_count, 1)
self.assertEqual(result.refused_recipients["blocked@example.org"], (550, "blocked"))
def test_batch_preflight_reuses_one_authenticated_connection(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
smtp = _FakeSmtp()
with patch("govoplan_mail.backend.sending.smtp._open_smtp", return_value=smtp) as opener:
with SmtpBatchSession(config) as batch:
first = batch.send(b"first", envelope_from="sender@example.org", envelope_recipients=["one@example.org"])
second = batch.send(b"second", envelope_from="sender@example.org", envelope_recipients=["two@example.org"])
opener.assert_called_once_with(config)
self.assertFalse(first.session_reused)
self.assertTrue(second.session_reused)
self.assertEqual(1, second.connection_sequence)
self.assertEqual([b"first", b"second"], smtp.messages)
self.assertTrue(smtp.quit_called)
def test_batch_reconnects_before_next_message_when_health_check_fails(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
first_smtp = _FakeSmtp(noop_error_on_call=1)
second_smtp = _FakeSmtp()
policy = SmtpBatchPolicy(reconnect_attempts=1)
with patch(
"govoplan_mail.backend.sending.smtp._open_smtp",
side_effect=[first_smtp, second_smtp],
) as opener:
with SmtpBatchSession(config, policy=policy) as batch:
batch.send(b"first", envelope_from="sender@example.org", envelope_recipients=["one@example.org"])
result = batch.send(b"second", envelope_from="sender@example.org", envelope_recipients=["two@example.org"])
self.assertEqual(2, opener.call_count)
self.assertEqual(2, result.connection_sequence)
self.assertEqual(1, result.reconnect_count)
self.assertEqual([b"first"], first_smtp.messages)
self.assertEqual([b"second"], second_smtp.messages)
def test_preflight_retries_a_transient_connection_failure(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
smtp = _FakeSmtp()
with patch(
"govoplan_mail.backend.sending.smtp._open_smtp",
side_effect=[OSError("temporary DNS failure"), smtp],
) as opener:
with SmtpBatchSession(config, policy=SmtpBatchPolicy(reconnect_attempts=1)) as batch:
result = batch.send(b"message", envelope_from="sender@example.org", envelope_recipients=["one@example.org"])
self.assertEqual(2, opener.call_count)
self.assertEqual(1, result.reconnect_count)
def test_connection_loss_after_send_starts_is_unknown_and_never_replayed(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
smtp = _FakeSmtp(send_error=smtplib.SMTPServerDisconnected("lost"))
with patch("govoplan_mail.backend.sending.smtp._open_smtp", return_value=smtp), self.assertRaises(SmtpSendError) as raised:
with SmtpBatchSession(config) as batch:
batch.send(b"one", envelope_from="sender@example.org", envelope_recipients=["one@example.org"])
self.assertTrue(raised.exception.outcome_unknown)
self.assertTrue(raised.exception.systemic)
self.assertEqual("smtp_connection_lost_after_transmission", raised.exception.reason_code)
self.assertEqual(1, smtp.send_calls)
def test_authentication_preflight_is_systemic_and_blocks_batch(self):
config = SmtpConfig(host="smtp.example.org", port=587, security="starttls")
error = smtplib.SMTPAuthenticationError(535, b"bad credentials")
with patch("govoplan_mail.backend.sending.smtp._open_smtp", side_effect=error), self.assertRaises(SmtpSendError) as raised:
SmtpBatchSession(config).preflight()
self.assertTrue(raised.exception.systemic)
self.assertFalse(raised.exception.temporary)
self.assertEqual("preflight", raised.exception.phase)
self.assertEqual("smtp_authentication_failed", raised.exception.reason_code)
class _FakeSmtp:
def __init__(self, *, noop_error_on_call: int | None = None, send_error: BaseException | None = None):
self.noop_error_on_call = noop_error_on_call
self.send_error = send_error
self.noop_calls = 0
self.send_calls = 0
self.messages: list[bytes] = []
self.quit_called = False
def noop(self):
self.noop_calls += 1
if self.noop_error_on_call == self.noop_calls:
raise smtplib.SMTPServerDisconnected("stale")
return 250, b"ok"
def sendmail(self, _sender, _recipients, message):
self.send_calls += 1
if self.send_error is not None:
raise self.send_error
self.messages.append(message)
return {}
def send_message(self, message, **_kwargs):
return self.sendmail(None, None, message.as_bytes())
def quit(self):
self.quit_called = True
return 221, b"bye"
def close(self):
return None
if __name__ == "__main__":
unittest.main()
+110
View File
@@ -0,0 +1,110 @@
{
"name": "@govoplan/mail-webui",
"version": "0.1.22",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@govoplan/mail-webui",
"version": "0.1.22",
"devDependencies": {
"typescript": "^5.7.2"
},
"peerDependencies": {
"@govoplan/core-webui": "^0.1.18",
"lucide-react": "^1.23.0",
"react": ">=19.2.7 <20",
"react-dom": ">=19.2.7 <20",
"react-router": ">=8.3.0 <9"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
"optional": true
}
}
},
"node_modules/cookie-es": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-3.1.1.tgz",
"integrity": "sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg==",
"license": "MIT",
"peer": true
},
"node_modules/lucide-react": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.28.0.tgz",
"integrity": "sha512-fARAFJULsGuDDydjp6+6blekG/sBIM29TerzLjc9bQUKAcEfrSc4ZQKb25KRz4OMKd87cZTb5dgq0w/T6KufVg==",
"license": "ISC",
"peer": true,
"peerDependencies": {
"react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/react": {
"version": "19.2.8",
"resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz",
"integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/react-dom": {
"version": "19.2.8",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz",
"integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"scheduler": "^0.27.0"
},
"peerDependencies": {
"react": "^19.2.8"
}
},
"node_modules/react-router": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/react-router/-/react-router-8.3.0.tgz",
"integrity": "sha512-qyPMvW83jGIct3yiieisxdk9M745anqhpIMKN5m1t6yBMfgVPpt77aHOqs5fUlEJRMCGffg9BaQLH9oPVOL7xQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"cookie-es": "^3.1.1"
},
"engines": {
"node": ">=22.22.0"
},
"peerDependencies": {
"react": ">=19.2.7",
"react-dom": ">=19.2.7"
},
"peerDependenciesMeta": {
"react-dom": {
"optional": true
}
}
},
"node_modules/scheduler": {
"version": "0.27.0",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz",
"integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==",
"license": "MIT",
"peer": true
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
}
}
}
+17 -6
View File
@@ -1,6 +1,6 @@
{
"name": "@govoplan/mail-webui",
"version": "0.1.0",
"version": "0.1.22",
"private": true,
"type": "module",
"main": "src/index.ts",
@@ -14,10 +14,21 @@
"./styles/mail-profiles.css": "./src/styles/mail-profiles.css"
},
"peerDependencies": {
"@govoplan/core-webui": "^0.1.0",
"lucide-react": "^0.555.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.1"
"@govoplan/core-webui": "^0.1.18",
"lucide-react": "^1.23.0",
"react": ">=19.2.7 <20",
"react-dom": ">=19.2.7 <20",
"react-router": ">=8.3.0 <9"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
"optional": true
}
},
"scripts": {
"test:mail-ui": "rm -rf .mail-test-build && mkdir -p .mail-test-build && printf '{\"type\":\"commonjs\"}\\n' > .mail-test-build/package.json && tsc -p tsconfig.mail-tests.json && node .mail-test-build/tests/mailbox-display.test.js && node .mail-test-build/tests/mailbox-folders.test.js && node .mail-test-build/tests/mailbox-launch.test.js && node .mail-test-build/tests/mail-profile-editor-model.test.js && node .mail-test-build/tests/mail-policy-validation.test.js && node .mail-test-build/tests/mail-address-integration.test.js && node scripts/test-mailbox-icon-button-structure.mjs && node scripts/test-interface-pattern-language.mjs"
},
"devDependencies": {
"typescript": "^5.7.2"
}
}
@@ -0,0 +1,86 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
function read(relativePath) {
return readFileSync(fileURLToPath(new URL(relativePath, import.meta.url)), "utf8");
}
const profiles = read("../src/features/mail/MailProfileManagement.tsx");
const mailbox = read("../src/features/mail/MailboxPage.tsx");
const bounces = read("../src/features/mail/MailBouncePage.tsx");
const legacyImport = read("../src/features/mail/MailLegacyImportPage.tsx");
const moduleSource = read("../src/module.ts");
const styles = read("../src/styles/mail-profiles.css");
const migration = read("../../docs/INTERFACE_PATTERN_MIGRATION.md");
assert.match(profiles, /ActionBlockerHint,[\s\S]*DocumentationHelpLink/);
for (const sharedComponent of ["ConnectionTree", "ConfirmDialog", "Dialog", "LoadingFrame"]) {
assert.match(profiles, new RegExp(`\\b${sharedComponent}\\b`));
}
assert.match(profiles, /topicId: "mail\.profiles-and-policy"/);
assert.match(profiles, /disabledReason: credentialMutationBlocker/);
assert.match(profiles, /disabledReason=\{editorSaveBlocker\}/);
assert.match(profiles, /disabledReason=\{policySaveBlocker\}/);
assert.match(profiles, /smtpActionDisabledReason=\{smtpTestBlocker\}/);
assert.match(profiles, /folder_mappings: draft\.imapFolderMappings/);
assert.match(profiles, /listMailProfileImapFolders[\s\S]*listImapFolders/);
assert.match(profiles, /onLookupImapFolders=\{\(\) => void runImapFolderLookup\(\)\}/);
assert.match(profiles, /imapFolderLookupResult=\{imapFolderResult\}/);
assert.match(mailbox, /ActionBlockerHint/);
assert.match(mailbox, /DocumentationHelpLink/);
assert.match(mailbox, /topicId: "mail\.workflow\.read-mailbox"/);
assert.match(mailbox, /disabledReason=\{folderReloadBlocker\}/);
assert.match(mailbox, /folder_mappings\?\.inbox/);
assert.match(mailbox, /detected_folder_mappings\?\.inbox/);
assert.match(mailbox, /onKeyDown=\{\(event\) => \{[\s\S]*event\.key === "Enter" \|\| event\.key === " "/);
assert.match(bounces, /DocumentationHelpLink/);
assert.match(bounces, /topicId: "mail\.bounce-processing"/);
assert.match(bounces, /<ConfirmDialog[\s\S]*confirmLabel="Remove watcher"[\s\S]*tone="danger"/);
assert.match(bounces, /disabledReason=\{saveWatcherBlocker\}/);
for (const sharedComponent of ["PageLayout", "PageActionBar", "SelectionList", "DataGrid", "Dialog", "ConfirmDialog", "ToggleSwitch", "StatusBadge"]) {
assert.match(legacyImport, new RegExp(`\\b${sharedComponent}\\b`));
}
assert.match(legacyImport, /archetype="collection"/);
assert.match(legacyImport, /variant="collection"[\s\S]*refreshable[\s\S]*reloadAction=/);
assert.match(legacyImport, /topicId: "mail\.workflow\.legacy-pop3-import"/);
assert.match(legacyImport, /helpContextId="mail\.pop3"[\s\S]*helpTopicId="mail\.workflow\.legacy-pop3-import"/);
for (const helpContextId of [
"mail.pop3.source-editor",
"mail.pop3.action.reload",
"mail.pop3.action.save-source",
"mail.pop3.action.import",
"mail.pop3.field.transport-security",
"mail.pop3.field.max-message-size",
"mail.pop3.field.max-batch-size",
"mail.pop3.field.password",
"mail.pop3.field.delete-after-import",
"mail.pop3.confirm-delete-source",
]) {
assert.match(
legacyImport,
new RegExp(`helpContextId(?:=|:\\s*)"${helpContextId.replaceAll(".", "\\.")}"`)
);
}
assert.match(legacyImport, /data-help-context-id="mail\.pop3\.field\.message-selection"/);
assert.match(legacyImport, /<ConfirmDialog[\s\S]*tone="danger"[\s\S]*onConfirm=\{\(\) => void runImport\(\)\}/);
assert.match(legacyImport, /legacy_import_enabled: false[\s\S]*is_active: false[\s\S]*createMailServerCredential[\s\S]*updateMailServerEndpoint/);
assert.match(legacyImport, /expected_transport_revision: preview\.transport_revision/);
assert.doesNotMatch(`${profiles}\n${mailbox}\n${bounces}\n${legacyImport}`, /window\.(?:alert|confirm)\(/);
assert.doesNotMatch(`${profiles}\n${mailbox}\n${bounces}\n${legacyImport}\n${moduleSource}`, /@govoplan\/(?:campaign|files|docs|calendar)-webui|govoplan_(?:campaign|files|docs|calendar)/);
assert.match(moduleSource, /"mail\.profiles"/);
assert.match(styles, /@media \(max-width: 900px\)[\s\S]*\.mail-profile-transport-summary[\s\S]*grid-template-columns: 1fr/);
assert.match(styles, /@media \(max-width: 1280px\)[\s\S]*\.mailbox-shell\.file-manager-shell[\s\S]*grid-template-columns:/);
assert.match(styles, /@media \(max-width: 760px\)[\s\S]*\.mailbox-toolbar\.file-manager-toolbar[\s\S]*grid-template-columns: 1fr/);
for (const archetype of ["Directory/explorer", "Administration/configuration", "Effective-policy editor", "Evidence/reporting"]) {
assert.match(migration, new RegExp(archetype.replace("/", "\\/")));
}
assert.match(migration, /Shared `Dialog` owns focus entry, Escape handling, focus containment, and focus\s+return/);
assert.match(migration, /Passwords are write-only[\s\S]*saved\s+state marker/);
console.log("Mail surfaces satisfy the recorded interface pattern-language contract.");
@@ -0,0 +1,37 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
const webuiDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const source = fs.readFileSync(path.join(webuiDir, "src/features/mail/MailboxPage.tsx"), "utf8");
const styles = fs.readFileSync(path.join(webuiDir, "src/styles/mail-profiles.css"), "utf8");
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(source.includes("IconButton,"), "MailboxPage must import the central IconButton");
assert(
source.includes('<IconButton label="i18n:govoplan-mail.clear_message_search.cc9f2800"'),
"the clear-search action must use the central IconButton"
);
assert(
!source.includes('<button type="button" onClick={() => setMessageQuery("")}'),
"the raw clear-search button must not return"
);
assert(
!styles.includes(".mailbox-search-field button"),
"Mail must not redefine the central icon-button appearance"
);
assert(
source.includes("isMailboxMessageRead(message.flags)"),
"mailbox rows must present provider-derived read/unread state"
);
assert(
source.includes("mailboxSyncState(messageProvenance)"),
"mailbox lists must present synchronization provenance"
);
assert(
styles.includes(".mailbox-message-row.is-unread") && styles.includes(".mailbox-sync-provenance"),
"read state and synchronization provenance must retain focused responsive styling"
);
+1 -1
View File
@@ -1 +1 @@
export { apiFetch, apiUrl, authHeaders, csrfToken, apiDownload } from "@govoplan/core-webui";
export { apiDownload, apiFetch, apiGetList, apiPath, apiPost, apiPostJson, apiQuery, apiUrl, authHeaders, csrfToken } from "@govoplan/core-webui";
+620 -166
View File
@@ -1,50 +1,104 @@
import type { ApiSettings } from "../types";
import { apiFetch } from "./client";
import type {
ApiSettings,
DeltaDeletedItem,
MailConnectionTestResponse,
MailImapFolderListResponse,
MailImapTestPayload,
MailCredentialEnvelope,
MailProfilePolicy,
MailProfilePolicyResponse,
MailProfileScope,
MailSecurity,
MailServerEndpoint,
MailServerProfile,
MailServerProfilePayload,
MailSmtpTestPayload,
MockMailboxMessage,
MockMailboxMessageResponse
} from "@govoplan/core-webui";
import { apiFetch, apiGetList, apiPath, apiPost, apiPostJson } from "./client";
export { mailProfilePatternKeys, mailProfilePolicyLimitKeys } from "@govoplan/core-webui";
export type {
MailConnectionTestResponse,
MailCredentialEnvelope,
MailCredentialPolicy,
MailImapFolderListResponse,
MailImapFolderResponse,
MailImapTestPayload,
MailProfilePatternKey,
MailProfilePatternRules,
MailProfilePolicy,
MailProfilePolicyLimitKey,
MailProfilePolicyLimitPermissions,
MailProfilePolicyResponse,
MailProfileScope,
MailSecurity,
MailServerEndpoint,
MailServerProfile,
MailServerProfileCredentialsPayload,
MailServerProfileListResponse,
MailServerProfilePayload,
MailSmtpTestPayload,
MailTransportCredentialsPayload,
MockMailboxMessage,
MockMailboxMessageResponse
} from "@govoplan/core-webui";
export type { MailPolicySourceStep as PolicySourceStep } from "@govoplan/core-webui";
export type MailSecurity = "plain" | "tls" | "starttls";
export type MailProfileScope = "system" | "tenant" | "user" | "group" | "campaign";
export type MailSmtpTestPayload = {
host?: string | null;
port?: number | null;
username?: string | null;
password?: string | null;
security?: MailSecurity;
timeout_seconds?: number;
export type MailAddressLookupCandidate = {
contact_id: string;
address_book_id: string;
display_name: string;
email?: string | null;
email_label?: string | null;
organization?: string | null;
role_title?: string | null;
tags: string[];
source_kind: string;
source_ref?: string | null;
source_revision?: string | null;
provenance: Record<string, unknown>;
};
export type MailImapTestPayload = MailSmtpTestPayload & {
enabled?: boolean;
sent_folder?: string | null;
export type MailAddressLookupResponse = {
available: boolean;
candidates: MailAddressLookupCandidate[];
};
export type MailConnectionTestResponse = {
ok: boolean;
protocol: "smtp" | "imap";
host?: string | null;
port?: number | null;
security?: MailSecurity | string | null;
export type MailAddressWriteTarget = {
address_book_id: string;
address_book_label?: string | null;
operation: string;
allowed: boolean;
reason: string;
message: string;
details?: Record<string, unknown>;
scope_type?: string | null;
scope_id?: string | null;
source_kind?: string | null;
read_only: boolean;
required_scopes: string[];
provenance: Record<string, unknown>;
};
export type MailImapFolderResponse = {
name: string;
flags?: string[];
export type MailAddressWriteTargetResponse = {
available: boolean;
targets: MailAddressWriteTarget[];
};
export type MailImapFolderListResponse = {
ok: boolean;
protocol: "imap";
host?: string | null;
port?: number | null;
security?: MailSecurity | string | null;
message: string;
folders: MailImapFolderResponse[];
detected_sent_folder?: string | null;
details?: Record<string, unknown>;
export type MailContactCreatePayload = {
address_book_id: string;
display_name?: string | null;
email: string;
};
export type MailContactCreateResponse = {
contact_id: string;
address_book_id: string;
display_name: string;
email?: string | null;
source_kind: string;
provenance: Record<string, unknown>;
};
export type MailMailboxAttachment = {
filename?: string | null;
@@ -59,6 +113,7 @@ export type MailMailboxMessageSummary = {
from_header?: string | null;
to_header?: string | null;
cc_header?: string | null;
bcc_header?: string | null;
date?: string | null;
message_id?: string | null;
flags?: string[];
@@ -81,9 +136,25 @@ export type MailMailboxMessageListResponse = {
port?: number | null;
security?: MailSecurity | string | null;
total_count?: number;
offset?: number;
limit?: number;
cursor?: string | null;
next_cursor?: string | null;
cursor_stable?: boolean;
full?: boolean;
from_cache?: boolean;
refreshing?: boolean;
indexed_at?: string | null;
messages: MailMailboxMessageSummary[];
};
export type MailMailboxBootstrapResponse = {
profile_id: string;
folder: string;
folders: MailImapFolderListResponse;
messages: MailMailboxMessageListResponse;
};
export type MailMailboxMessageResponse = {
profile_id: string;
folder: string;
@@ -93,87 +164,140 @@ export type MailMailboxMessageResponse = {
message: MailMailboxMessageDetail;
};
export type MailServerProfile = {
export type MailMailboxProtocol = "imap" | "jmap";
export type MailMailboxChangesResponse = {
profile_id: string;
protocol: "jmap";
account_id: string;
old_state: string;
new_state: string;
has_more_changes: boolean;
created: string[];
updated: string[];
destroyed: string[];
};
export type MailSettingsDeltaResponse = {
profiles: MailServerProfile[];
policy?: MailProfilePolicyResponse | null;
changed_sections?: string[];
deleted: DeltaDeletedItem[];
watermark?: string | null;
has_more: boolean;
full: boolean;
};
export type MailBounceSource = {
id: string;
tenant_id?: string | null;
scope_type: MailProfileScope;
scope_id?: string | null;
name: string;
slug: string;
description?: string | null;
tenant_id: string;
profile_id: string;
folder: string;
imap_server_id?: string | null;
imap_credential_id?: string | null;
expected_imap_transport_revision: string;
is_active: boolean;
smtp: MailSmtpTestPayload;
imap?: MailImapTestPayload | null;
smtp_password_configured: boolean;
imap_password_configured: boolean;
created_at: string;
updated_at: string;
uidvalidity?: string | null;
highest_processed_uid: number;
last_scanned_at?: string | null;
last_success_at?: string | null;
last_error?: string | null;
};
export type MailServerProfileListResponse = { profiles: MailServerProfile[] };
export const mailProfilePatternKeys = [
"smtp_hosts",
"imap_hosts",
"envelope_senders",
"from_headers",
"recipient_domains"
] as const;
export type MailProfilePatternKey = typeof mailProfilePatternKeys[number];
export type MailProfilePatternRules = Partial<Record<MailProfilePatternKey, string[]>>;
export type MailCredentialPolicy = {
inherit?: boolean | null;
allow_override?: boolean | null;
export type MailBounceObservation = {
id: string;
profile_id: string;
folder: string;
uid: string;
original_message_id?: string | null;
command_id?: string | null;
recipient?: string | null;
action: string;
status_code?: string | null;
diagnostic?: string | null;
permanent: boolean;
observed_at: string;
matched: boolean;
evidence: Record<string, unknown>;
};
export type MailProfilePolicy = {
allowed_profile_ids?: string[] | null;
allow_user_profiles?: boolean | null;
allow_group_profiles?: boolean | null;
allow_campaign_profiles?: boolean | null;
smtp_credentials?: MailCredentialPolicy | null;
imap_credentials?: MailCredentialPolicy | null;
whitelist?: MailProfilePatternRules | null;
blacklist?: MailProfilePatternRules | null;
export type MailBounceSourcePayload = {
profile_id: string;
folder: string;
imap_server_id?: string | null;
imap_credential_id?: string | null;
is_active: boolean;
};
export type PolicySourceStep = {
scope_type: string;
scope_id?: string | null;
label: string;
applied_fields?: string[];
};
export async function listMailBounceSources(settings: ApiSettings): Promise<MailBounceSource[]> {
const response = await apiFetch<{ sources: MailBounceSource[] }>(settings, "/api/v1/mail/bounce-sources");
return response.sources;
}
export type MailProfilePolicyResponse = {
scope_type: MailProfileScope;
scope_id?: string | null;
policy: MailProfilePolicy;
effective_policy?: MailProfilePolicy | null;
parent_policy?: MailProfilePolicy | null;
effective_policy_sources?: PolicySourceStep[];
parent_policy_sources?: PolicySourceStep[];
};
export async function saveMailBounceSource(settings: ApiSettings, payload: MailBounceSourcePayload): Promise<MailBounceSource> {
return apiFetch<MailBounceSource>(settings, "/api/v1/mail/bounce-sources", {
method: "POST",
body: JSON.stringify(payload)
});
}
export type MailServerProfilePayload = {
name: string;
slug?: string | null;
description?: string | null;
is_active?: boolean;
scope_type?: MailProfileScope;
scope_id?: string | null;
smtp: MailSmtpTestPayload;
imap?: MailImapTestPayload | null;
};
export async function removeMailBounceSource(settings: ApiSettings, sourceId: string): Promise<void> {
await apiFetch<void>(settings, `/api/v1/mail/bounce-sources/${encodeURIComponent(sourceId)}`, { method: "DELETE" });
}
export async function scanMailBounceSource(settings: ApiSettings, sourceId: string): Promise<{ sources: number; processed_messages: number; observations: number; failures: Array<Record<string, string>> }> {
return apiFetch(settings, `/api/v1/mail/bounce-sources/${encodeURIComponent(sourceId)}/scan`, { method: "POST" });
}
export async function listMailBounceObservations(settings: ApiSettings, limit = 100): Promise<MailBounceObservation[]> {
const response = await apiFetch<{ observations: MailBounceObservation[] }>(settings, apiPath("/api/v1/mail/bounce-observations", { limit }));
return response.observations;
}
export async function lookupMailAddresses(settings: ApiSettings, query: string, limit = 25): Promise<MailAddressLookupResponse> {
return apiFetch<MailAddressLookupResponse>(settings, apiPath("/api/v1/mail/address-lookup", { query, limit }));
}
export async function listMailAddressWriteTargets(settings: ApiSettings): Promise<MailAddressWriteTargetResponse> {
return apiFetch<MailAddressWriteTargetResponse>(settings, "/api/v1/mail/address-write-targets");
}
export async function createMailAddressContact(
settings: ApiSettings,
payload: MailContactCreatePayload
): Promise<MailContactCreateResponse> {
return apiFetch<MailContactCreateResponse>(settings, "/api/v1/mail/address-contacts", {
method: "POST",
body: JSON.stringify(payload)
});
}
export async function listMailServerProfiles(settings: ApiSettings, includeInactive = false, campaignId?: string): Promise<MailServerProfile[]> {
const params = new URLSearchParams();
if (includeInactive) params.set("include_inactive", "true");
if (campaignId) params.set("campaign_id", campaignId);
const suffix = params.toString() ? `?${params.toString()}` : "";
const response = await apiFetch<MailServerProfileListResponse>(settings, `/api/v1/mail/profiles${suffix}`);
return response.profiles ?? [];
return apiGetList<MailServerProfile, "profiles">(settings, "/api/v1/mail/profiles", "profiles", {
include_inactive: includeInactive ? true : undefined,
campaign_id: campaignId
});
}
export async function fetchMailSettingsDelta(
settings: ApiSettings,
params: {
scope_type?: MailProfileScope;
scope_id?: string | null;
include_inactive?: boolean;
campaign_id?: string | null;
since?: string | null;
limit?: number;
} = {}
): Promise<MailSettingsDeltaResponse> {
return apiFetch<MailSettingsDeltaResponse>(settings, apiPath("/api/v1/mail/settings/delta", {
scope_type: params.scope_type,
scope_id: params.scope_id,
include_inactive: params.include_inactive ? true : undefined,
campaign_id: params.campaign_id,
since: params.since,
limit: params.limit
}));
}
export async function createMailServerProfile(settings: ApiSettings, payload: MailServerProfilePayload): Promise<MailServerProfile> {
@@ -185,6 +309,104 @@ export async function createMailServerProfile(settings: ApiSettings, payload: Ma
export type MailServerProfileUpdatePayload = Partial<MailServerProfilePayload> & { clear_imap?: boolean };
export type MailServerEndpointPayload = {
protocol: "smtp" | "imap" | "jmap" | "pop3";
name: string;
config: Record<string, unknown>;
inherit_to_lower_scopes?: boolean | null;
is_default?: boolean;
is_active?: boolean;
};
export type MailPop3ServerConfig = {
host?: string | null;
port?: number | null;
security?: MailSecurity | string;
timeout_seconds?: number;
max_message_bytes?: number;
max_batch_bytes?: number;
preview_body_lines?: number;
legacy_import_enabled?: boolean;
allow_delete_after_import?: boolean;
};
export type MailPop3ServerEndpoint = Omit<MailServerEndpoint, "protocol" | "config"> & {
protocol: "pop3";
config: MailPop3ServerConfig;
};
export type MailPop3MessagePreview = {
message_number: number;
uidl: string;
subject?: string | null;
from_header?: string | null;
to_header?: string | null;
date?: string | null;
message_id?: string | null;
size_bytes: number;
body_preview?: string | null;
already_imported: boolean;
};
export type MailPop3PreviewResponse = {
profile_id: string;
server_id: string;
transport_revision: string;
host: string;
port: number;
security: string;
message_count: number;
mailbox_size_bytes: number;
delete_after_import_allowed: boolean;
messages: MailPop3MessagePreview[];
};
export type MailPop3ImportRecord = {
id: string;
profile_id: string;
pop3_server_id: string;
transport_revision: string;
provider_uidl: string;
message_id?: string | null;
subject?: string | null;
from_header?: string | null;
to_header?: string | null;
date?: string | null;
body_preview?: string | null;
size_bytes: number;
raw_sha256: string;
status: string;
imported_at: string;
deletion_requested: boolean;
deletion_status: string;
deletion_attempted_at?: string | null;
deletion_error?: string | null;
};
export type MailPop3ImportResponse = {
imports: MailPop3ImportRecord[];
duplicate_uidls: string[];
deletion_status: string;
};
export type MailCredentialCreatePayload = {
name: string;
description?: string | null;
credential_kind?: string;
username?: string | null;
password?: string | null;
public_data?: Record<string, unknown>;
secret_data?: Record<string, unknown>;
inherit_to_lower_scopes?: boolean | null;
allowed_modules?: string[];
allowed_server_refs?: string[];
is_default?: boolean;
};
export type MailCredentialUpdatePayload = Partial<MailCredentialCreatePayload> & {
is_active?: boolean;
};
export async function updateMailServerProfile(settings: ApiSettings, profileId: string, payload: MailServerProfileUpdatePayload): Promise<MailServerProfile> {
return apiFetch<MailServerProfile>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}`, {
method: "PATCH",
@@ -196,17 +418,128 @@ export async function deactivateMailServerProfile(settings: ApiSettings, profile
return apiFetch<MailServerProfile>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}`, { method: "DELETE" });
}
export async function createMailServerEndpoint(
settings: ApiSettings,
profileId: string,
payload: MailServerEndpointPayload
): Promise<MailServerEndpoint> {
return apiFetch<MailServerEndpoint>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers`,
{ method: "POST", body: JSON.stringify(payload) }
);
}
export async function updateMailServerEndpoint(
settings: ApiSettings,
profileId: string,
serverId: string,
payload: Partial<Omit<MailServerEndpointPayload, "protocol">>
): Promise<MailServerEndpoint> {
return apiFetch<MailServerEndpoint>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}`,
{ method: "PATCH", body: JSON.stringify(payload) }
);
}
export async function deactivateMailServerEndpoint(
settings: ApiSettings,
profileId: string,
serverId: string
): Promise<MailServerEndpoint> {
return apiFetch<MailServerEndpoint>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}`,
{ method: "DELETE" }
);
}
export async function listAvailableMailCredentials(
settings: ApiSettings,
profileId: string,
serverId: string,
includeInactive = false
): Promise<MailCredentialEnvelope[]> {
return apiGetList<MailCredentialEnvelope, "credentials">(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}/available-credentials`,
"credentials",
{ include_inactive: includeInactive ? true : undefined }
);
}
export async function createMailServerCredential(
settings: ApiSettings,
profileId: string,
serverId: string,
payload: MailCredentialCreatePayload
): Promise<MailCredentialEnvelope> {
return apiFetch<MailCredentialEnvelope>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}/credentials`,
{ method: "POST", body: JSON.stringify(payload) }
);
}
export async function bindMailServerCredential(
settings: ApiSettings,
profileId: string,
serverId: string,
credentialId: string,
isDefault = false
): Promise<MailCredentialEnvelope> {
return apiFetch<MailCredentialEnvelope>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}/credential-bindings`,
{
method: "POST",
body: JSON.stringify({ credential_id: credentialId, is_default: isDefault })
}
);
}
export async function updateMailServerCredential(
settings: ApiSettings,
profileId: string,
serverId: string,
credentialId: string,
payload: MailCredentialUpdatePayload
): Promise<MailCredentialEnvelope> {
return apiFetch<MailCredentialEnvelope>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}/credentials/${encodeURIComponent(credentialId)}`,
{ method: "PATCH", body: JSON.stringify(payload) }
);
}
export async function unlinkMailServerCredential(
settings: ApiSettings,
profileId: string,
serverId: string,
credentialId: string,
retireIfUnused = false
): Promise<void> {
await apiFetch<void>(
settings,
apiPath(
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/servers/${encodeURIComponent(serverId)}/credentials/${encodeURIComponent(credentialId)}`,
{ retire_if_unused: retireIfUnused ? true : undefined }
),
{ method: "DELETE" }
);
}
export async function getMailProfilePolicy(
settings: ApiSettings,
scopeType: MailProfileScope,
scopeId?: string | null,
campaignId?: string | null
): Promise<MailProfilePolicyResponse> {
const params = new URLSearchParams();
if (scopeId) params.set("scope_id", scopeId);
if (campaignId) params.set("campaign_id", campaignId);
const suffix = params.toString() ? `?${params.toString()}` : "";
return apiFetch<MailProfilePolicyResponse>(settings, `/api/v1/mail/policies/${encodeURIComponent(scopeType)}${suffix}`);
return apiFetch<MailProfilePolicyResponse>(settings, apiPath(`/api/v1/mail/policies/${encodeURIComponent(scopeType)}`, {
scope_id: scopeId,
campaign_id: campaignId
}));
}
export async function updateMailProfilePolicy(
@@ -215,110 +548,232 @@ export async function updateMailProfilePolicy(
policy: MailProfilePolicy,
scopeId?: string | null
): Promise<MailProfilePolicyResponse> {
const params = new URLSearchParams();
if (scopeId) params.set("scope_id", scopeId);
const suffix = params.toString() ? `?${params.toString()}` : "";
return apiFetch<MailProfilePolicyResponse>(settings, `/api/v1/mail/policies/${encodeURIComponent(scopeType)}${suffix}`, {
return apiFetch<MailProfilePolicyResponse>(settings, apiPath(`/api/v1/mail/policies/${encodeURIComponent(scopeType)}`, { scope_id: scopeId }), {
method: "PUT",
body: JSON.stringify({ policy })
});
}
export async function testMailProfileSmtp(settings: ApiSettings, profileId: string): Promise<MailConnectionTestResponse> {
return apiFetch<MailConnectionTestResponse>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-smtp`, { method: "POST" });
export async function testMailProfileSmtp(
settings: ApiSettings,
profileId: string,
serverId?: string | null,
credentialId?: string | null,
campaignId?: string | null
): Promise<MailConnectionTestResponse> {
return apiPost<MailConnectionTestResponse>(
settings,
apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-smtp`, {
server_id: serverId,
credential_id: credentialId,
campaign_id: campaignId
})
);
}
export async function testMailProfileImap(settings: ApiSettings, profileId: string): Promise<MailConnectionTestResponse> {
return apiFetch<MailConnectionTestResponse>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-imap`, { method: "POST" });
export async function testMailProfileImap(
settings: ApiSettings,
profileId: string,
serverId?: string | null,
credentialId?: string | null,
campaignId?: string | null
): Promise<MailConnectionTestResponse> {
return apiPost<MailConnectionTestResponse>(
settings,
apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-imap`, {
server_id: serverId,
credential_id: credentialId,
campaign_id: campaignId
})
);
}
export async function listMailProfileImapFolders(settings: ApiSettings, profileId: string): Promise<MailImapFolderListResponse> {
return apiFetch<MailImapFolderListResponse>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/list-imap-folders`, { method: "POST" });
export async function testMailProfileJmap(
settings: ApiSettings,
profileId: string,
serverId?: string | null,
credentialId?: string | null,
campaignId?: string | null
): Promise<MailConnectionTestResponse> {
return apiPost<MailConnectionTestResponse>(
settings,
apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-jmap`, {
server_id: serverId,
credential_id: credentialId,
campaign_id: campaignId
})
);
}
export async function listMailboxFolders(settings: ApiSettings, profileId: string): Promise<MailImapFolderListResponse> {
return apiFetch<MailImapFolderListResponse>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/folders`);
export async function testMailProfilePop3(
settings: ApiSettings,
profileId: string,
serverId: string,
credentialId?: string | null
): Promise<MailConnectionTestResponse> {
return apiPost<MailConnectionTestResponse>(
settings,
apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-pop3`, {
server_id: serverId,
credential_id: credentialId
})
);
}
export async function previewMailProfilePop3(
settings: ApiSettings,
profileId: string,
payload: { server_id: string; credential_id?: string | null; limit?: number }
): Promise<MailPop3PreviewResponse> {
return apiPostJson<MailPop3PreviewResponse>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/pop3/preview`,
payload
);
}
export async function importMailProfilePop3(
settings: ApiSettings,
profileId: string,
payload: {
server_id: string;
credential_id?: string | null;
expected_transport_revision: string;
uidls: string[];
delete_after_import?: boolean;
}
): Promise<MailPop3ImportResponse> {
return apiPostJson<MailPop3ImportResponse>(
settings,
`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/pop3/import`,
payload
);
}
export async function listMailPop3Imports(
settings: ApiSettings,
profileId?: string | null,
limit = 100
): Promise<MailPop3ImportRecord[]> {
const response = await apiFetch<{ imports: MailPop3ImportRecord[] }>(
settings,
apiPath("/api/v1/mail/pop3/imports", { profile_id: profileId, limit })
);
return response.imports;
}
export async function listMailProfileImapFolders(
settings: ApiSettings,
profileId: string,
serverId?: string | null,
credentialId?: string | null,
campaignId?: string | null
): Promise<MailImapFolderListResponse> {
return apiPost<MailImapFolderListResponse>(
settings,
apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/list-imap-folders`, {
server_id: serverId,
credential_id: credentialId,
campaign_id: campaignId
})
);
}
export async function listMailboxFolders(settings: ApiSettings, profileId: string, includeStatus = false, refresh = false, protocol: MailMailboxProtocol = "imap"): Promise<MailImapFolderListResponse> {
return apiFetch<MailImapFolderListResponse>(settings, apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/folders`, {
include_status: includeStatus ? true : undefined,
refresh: refresh ? true : undefined,
protocol
}));
}
export async function bootstrapMailbox(
settings: ApiSettings,
profileId: string,
folder = "INBOX",
limit = 50,
offset = 0,
refresh = false,
protocol: MailMailboxProtocol = "imap"
): Promise<MailMailboxBootstrapResponse> {
return apiFetch<MailMailboxBootstrapResponse>(settings, apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/bootstrap`, {
folder,
limit,
offset,
refresh: refresh ? true : undefined,
protocol
}));
}
export async function listMailboxMessages(
settings: ApiSettings,
profileId: string,
folder = "INBOX",
limit = 50
limit = 50,
offset = 0,
cursor?: string | null,
refresh = false,
protocol: MailMailboxProtocol = "imap",
query?: string | null
): Promise<MailMailboxMessageListResponse> {
const params = new URLSearchParams({ folder, limit: String(limit) });
return apiFetch<MailMailboxMessageListResponse>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/messages?${params.toString()}`);
return apiFetch<MailMailboxMessageListResponse>(settings, apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/messages`, {
folder,
limit,
offset,
cursor,
refresh: refresh ? true : undefined,
protocol,
q: query || undefined
}));
}
export async function getMailboxMessage(
settings: ApiSettings,
profileId: string,
folder: string,
uid: string
uid: string,
protocol: MailMailboxProtocol = "imap"
): Promise<MailMailboxMessageResponse> {
const params = new URLSearchParams({ folder });
return apiFetch<MailMailboxMessageResponse>(settings, `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/messages/${encodeURIComponent(uid)}?${params.toString()}`);
return apiFetch<MailMailboxMessageResponse>(settings, apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/messages/${encodeURIComponent(uid)}`, { folder, protocol }));
}
export async function getMailboxChanges(
settings: ApiSettings,
profileId: string,
sinceState: string,
maxChanges = 500
): Promise<MailMailboxChangesResponse> {
return apiFetch<MailMailboxChangesResponse>(settings, apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/mailbox/changes`, {
since_state: sinceState,
max_changes: maxChanges
}));
}
export async function testSmtpSettings(
settings: ApiSettings,
payload: MailSmtpTestPayload
): Promise<MailConnectionTestResponse> {
return apiFetch<MailConnectionTestResponse>(settings, "/api/v1/mail/test-smtp", {
method: "POST",
body: JSON.stringify(payload)
});
return apiPostJson<MailConnectionTestResponse>(settings, "/api/v1/mail/test-smtp", payload);
}
export async function testImapSettings(
settings: ApiSettings,
payload: MailImapTestPayload
): Promise<MailConnectionTestResponse> {
return apiFetch<MailConnectionTestResponse>(settings, "/api/v1/mail/test-imap", {
method: "POST",
body: JSON.stringify(payload)
});
return apiPostJson<MailConnectionTestResponse>(settings, "/api/v1/mail/test-imap", payload);
}
export async function listImapFolders(
settings: ApiSettings,
payload: MailImapTestPayload
): Promise<MailImapFolderListResponse> {
return apiFetch<MailImapFolderListResponse>(settings, "/api/v1/mail/list-imap-folders", {
method: "POST",
body: JSON.stringify(payload)
});
return apiPostJson<MailImapFolderListResponse>(settings, "/api/v1/mail/list-imap-folders", payload);
}
export type MockMailboxMessage = {
id: string;
kind: "smtp" | "imap_append" | string;
created_at: string;
envelope_from?: string | null;
envelope_recipients?: string[];
subject?: string | null;
from_header?: string | null;
to_header?: string | null;
cc_header?: string | null;
bcc_header?: string | null;
message_id?: string | null;
size_bytes?: number;
body_preview?: string | null;
attachment_count?: number;
folder?: string | null;
raw_eml?: string | null;
headers?: Record<string, string>;
attachments?: Array<{ filename?: string | null; content_type?: string | null; size_bytes?: number }>;
};
export type MockMailboxListResponse = {
messages: MockMailboxMessage[];
};
export type MockMailboxMessageResponse = {
message: MockMailboxMessage;
};
export type MockMailboxFailureConfig = {
fail_next_smtp?: boolean | null;
fail_next_imap?: boolean | null;
@@ -326,8 +781,7 @@ export type MockMailboxFailureConfig = {
};
export async function listMockMailboxMessages(settings: ApiSettings, kind?: string): Promise<MockMailboxListResponse> {
const suffix = kind ? `?kind=${encodeURIComponent(kind)}` : "";
return apiFetch<MockMailboxListResponse>(settings, `/api/v1/dev/mailbox/messages${suffix}`);
return apiFetch<MockMailboxListResponse>(settings, apiPath("/api/v1/dev/mailbox/messages", { kind }));
}
export async function getMockMailboxMessage(settings: ApiSettings, id: string): Promise<MockMailboxMessageResponse> {
-2
View File
@@ -1,2 +0,0 @@
import { Button } from "@govoplan/core-webui";
export default Button;
-2
View File
@@ -1,2 +0,0 @@
import { Card } from "@govoplan/core-webui";
export default Card;
-2
View File
@@ -1,2 +0,0 @@
import { ConfirmDialog } from "@govoplan/core-webui";
export default ConfirmDialog;
-2
View File
@@ -1,2 +0,0 @@
import { Dialog } from "@govoplan/core-webui";
export default Dialog;
@@ -1,2 +0,0 @@
import { DismissibleAlert } from "@govoplan/core-webui";
export default DismissibleAlert;
-2
View File
@@ -1,2 +0,0 @@
import { FormField } from "@govoplan/core-webui";
export default FormField;
-2
View File
@@ -1,2 +0,0 @@
import { LoadingFrame } from "@govoplan/core-webui";
export default LoadingFrame;
@@ -1,2 +0,0 @@
import { LoadingIndicator } from "@govoplan/core-webui";
export default LoadingIndicator;
-2
View File
@@ -1,2 +0,0 @@
import { MetricCard } from "@govoplan/core-webui";
export default MetricCard;
-2
View File
@@ -1,2 +0,0 @@
import { PageTitle } from "@govoplan/core-webui";
export default PageTitle;
-2
View File
@@ -1,2 +0,0 @@
import { StatusBadge } from "@govoplan/core-webui";
export default StatusBadge;
-2
View File
@@ -1,2 +0,0 @@
import { Stepper } from "@govoplan/core-webui";
export default Stepper;
-2
View File
@@ -1,2 +0,0 @@
import { ToggleSwitch } from "@govoplan/core-webui";
export default ToggleSwitch;
@@ -1,2 +0,0 @@
import { EmailAddressInput } from "@govoplan/core-webui";
export default EmailAddressInput;

Some files were not shown because too many files have changed in this diff Show More