Files
av-tools/scripts/package-release.mjs

784 lines
24 KiB
JavaScript

import {
access,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
realpath,
rename,
rm,
writeFile,
} from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { zipSync } from 'fflate';
import { checksumLine, sha256 } from './checksum-release.mjs';
const repositoryRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'..'
);
const zipEpoch = new Date(1980, 0, 1, 0, 0, 0);
const maximumEntryBytes = 256 * 1024 * 1024;
const maximumArchiveBytes = 2 * 1024 * 1024 * 1024;
const applicationLicenseSpdx = 'GPL-3.0-or-later';
const canonicalRepository = 'https://git.add-ideas.de/lotobo/av-tools';
const canonicalPackageRepository =
'git+https://git.add-ideas.de/lotobo/av-tools.git';
const requiredNativeLegalFiles = [
'LICENSES/native/FFmpeg-LICENSE.md',
'LICENSES/native/FreeType-LICENSE.txt',
'LICENSES/native/FriBidi-COPYING.txt',
'LICENSES/native/HarfBuzz-COPYING.txt',
'LICENSES/native/LAME-COPYING.txt',
'LICENSES/native/LAME-LICENSE.txt',
'LICENSES/native/Ogg-COPYING.txt',
'LICENSES/native/Opus-COPYING.txt',
'LICENSES/native/Theora-COPYING.txt',
'LICENSES/native/Vorbis-COPYING.txt',
'LICENSES/native/libass-COPYING.txt',
'LICENSES/native/libvpx-LICENSE.txt',
'LICENSES/native/libvpx-PATENTS.txt',
'LICENSES/native/libwebp-COPYING.txt',
'LICENSES/native/libwebp-PATENTS.txt',
'LICENSES/native/x264-COPYING.txt',
'LICENSES/native/x265-COPYING.txt',
'LICENSES/native/zimg-COPYING.txt',
'LICENSES/native/zlib-README.txt',
];
const ffmpegLicenseCoverage = new Map([
['@ffmpeg/ffmpeg', 'LICENSES/ffmpeg.wasm-MIT.txt'],
['@ffmpeg/util', 'LICENSES/ffmpeg.wasm-MIT.txt'],
['@ffmpeg/types', 'LICENSES/ffmpeg.wasm-MIT.txt'],
]);
function compareText(left, right) {
return left < right ? -1 : left > right ? 1 : 0;
}
export function safeArchivePath(fileName) {
if (
typeof fileName !== 'string' ||
fileName.length === 0 ||
fileName.includes('\0') ||
fileName.includes('\\')
) {
throw new Error(`Invalid release path: ${String(fileName)}`);
}
if (fileName.startsWith('/') || /^[A-Za-z]:\//.test(fileName)) {
throw new Error(`Absolute release path is forbidden: ${fileName}`);
}
const components = fileName.split('/');
if (
components.some(
(component) => component === '' || component === '.' || component === '..'
)
) {
throw new Error(`Ambiguous or traversing release path: ${fileName}`);
}
const normalized = path.posix.normalize(fileName);
if (normalized !== fileName) {
throw new Error(`Non-canonical release path: ${fileName}`);
}
return normalized;
}
function relativeArchivePath(root, file) {
return safeArchivePath(
path.relative(root, file).split(path.sep).join(path.posix.sep)
);
}
function isInside(root, candidate) {
return candidate === root || candidate.startsWith(`${root}${path.sep}`);
}
async function collectDirectoryRecursive(
canonicalRoot,
sourceRoot,
directory,
prefix,
entries
) {
const canonicalDirectory = await realpath(directory);
if (!isInside(canonicalRoot, canonicalDirectory)) {
throw new Error(`Directory resolves outside packaging root: ${directory}`);
}
const children = await readdir(directory, { withFileTypes: true });
children.sort((left, right) => compareText(left.name, right.name));
for (const child of children) {
const childPath = path.join(directory, child.name);
const details = await lstat(childPath);
if (details.isSymbolicLink()) {
throw new Error(`Refusing release symlink: ${childPath}`);
}
if (details.isDirectory()) {
await collectDirectoryRecursive(
canonicalRoot,
sourceRoot,
childPath,
prefix,
entries
);
continue;
}
if (!details.isFile()) {
throw new Error(`Refusing special release entry: ${childPath}`);
}
const canonicalFile = await realpath(childPath);
if (!isInside(canonicalRoot, canonicalFile)) {
throw new Error(`File resolves outside packaging root: ${childPath}`);
}
const relativeName = relativeArchivePath(sourceRoot, childPath);
const name = safeArchivePath(
prefix ? path.posix.join(prefix, relativeName) : relativeName
);
entries.push({ name, data: await readFile(childPath) });
}
}
export async function collectDirectoryEntries(directory, prefix = '') {
const details = await lstat(directory);
if (details.isSymbolicLink() || !details.isDirectory()) {
throw new Error(`Release input is not a real directory: ${directory}`);
}
const sourceRoot = path.resolve(directory);
const canonicalRoot = await realpath(directory);
const entries = [];
await collectDirectoryRecursive(
canonicalRoot,
sourceRoot,
sourceRoot,
prefix,
entries
);
return entries.sort((left, right) => compareText(left.name, right.name));
}
async function readRequired(file) {
try {
return await readFile(file);
} catch (error) {
throw new Error(`Required release file is missing: ${file}`, {
cause: error,
});
}
}
async function existingApplicationLicense(root) {
for (const fileName of ['LICENSE', 'LICENSE.txt', 'COPYING']) {
const candidate = path.join(root, fileName);
try {
const details = await lstat(candidate);
if (details.isSymbolicLink() || !details.isFile()) {
throw new Error(
`Application licence path must be a regular file: ${candidate}`
);
}
return candidate;
} catch (error) {
if (error && typeof error === 'object' && error.code === 'ENOENT') {
continue;
}
throw error;
}
}
return null;
}
export async function assertLicenseReady(root, allowDevelopmentSmoke = false) {
const applicationLicense = await existingApplicationLicense(root);
if (applicationLicense) {
const legalCode = await readFile(applicationLicense, 'utf8');
if (
!legalCode.includes('GNU GENERAL PUBLIC LICENSE') ||
!legalCode.includes('Version 3, 29 June 2007') ||
!legalCode.includes('END OF TERMS AND CONDITIONS')
) {
throw new Error(
`Application licence does not contain the complete GNU GPL version 3 legal code: ${applicationLicense}`
);
}
return applicationLicense;
}
if (allowDevelopmentSmoke) return null;
throw new Error(
[
'RELEASE BLOCKED: av-tools has no application licence.',
'The FFmpeg and dependency notices do not license the application itself.',
'For a local, non-publishable portal test only, pass',
'`--allow-unlicensed-development-smoke`.',
].join(' ')
);
}
function safePackageName(name) {
return name.replace(/^@/, '').replaceAll('/', '--');
}
async function findInstalledPackageDirectory(name, fromDirectory) {
let current = path.resolve(fromDirectory);
const packageSegments = name.split('/');
while (true) {
const candidate = path.join(current, 'node_modules', ...packageSegments);
try {
const details = await lstat(path.join(candidate, 'package.json'));
if (!details.isFile()) {
throw new Error(`Package manifest is not a file: ${candidate}`);
}
return await realpath(candidate);
} catch (error) {
if (!(error && typeof error === 'object' && error.code === 'ENOENT')) {
throw error;
}
}
const parent = path.dirname(current);
if (parent === current) return null;
current = parent;
}
}
async function collectRuntimePackages(root, packageJson) {
const queue = Object.keys(packageJson.dependencies ?? {})
.sort(compareText)
.map((name) => ({ name, fromDirectory: root }));
const visited = new Set();
const packages = [];
while (queue.length > 0) {
const dependency = queue.shift();
const directory = await findInstalledPackageDirectory(
dependency.name,
dependency.fromDirectory
);
if (!directory) {
throw new Error(
`Runtime dependency is not installed: ${dependency.name}`
);
}
const manifest = JSON.parse(
await readFile(path.join(directory, 'package.json'), 'utf8')
);
const identity = `${manifest.name}@${manifest.version}`;
if (visited.has(identity)) continue;
visited.add(identity);
packages.push({
directory,
name: manifest.name,
version: manifest.version,
});
const childNames = [
...new Set([
...Object.keys(manifest.dependencies ?? {}),
...Object.keys(manifest.optionalDependencies ?? {}),
]),
].sort(compareText);
for (const name of childNames) {
const childDirectory = await findInstalledPackageDirectory(
name,
directory
);
if (childDirectory) {
queue.push({ name, fromDirectory: directory });
}
}
}
return packages.sort((left, right) =>
compareText(
`${left.name}@${left.version}`,
`${right.name}@${right.version}`
)
);
}
export async function collectRuntimeLicenseEntries(root, packageJson) {
const packages = await collectRuntimePackages(root, packageJson);
const entries = [];
for (const runtimePackage of packages) {
const coveredBy = ffmpegLicenseCoverage.get(runtimePackage.name);
if (coveredBy) {
await access(path.join(root, coveredBy));
continue;
}
const legalFiles = (
await readdir(runtimePackage.directory, {
withFileTypes: true,
})
)
.filter(
(entry) =>
entry.isFile() &&
/^(?:licen[cs]e|copying|notice)(?:\.|$)/i.test(entry.name)
)
.sort((left, right) => compareText(left.name, right.name));
if (legalFiles.length === 0) {
throw new Error(
`Runtime dependency ${runtimePackage.name}@${runtimePackage.version} has no packaged licence file.`
);
}
const packageDirectory = `${safePackageName(runtimePackage.name)}-${runtimePackage.version}`;
for (const legalFile of legalFiles) {
entries.push({
name: safeArchivePath(
`LICENSES/npm/${packageDirectory}/${legalFile.name}`
),
data: await readFile(
path.join(runtimePackage.directory, legalFile.name)
),
});
}
}
return entries;
}
function scanReleaseEntry(entry) {
if (entry.name.endsWith('.map')) {
throw new Error(`Source map is not approved for release: ${entry.name}`);
}
const lowerParts = entry.name.toLowerCase().split('/');
if (
lowerParts.some((part) =>
[
'.env',
'.npmrc',
'.git',
'id_rsa',
'id_ed25519',
'credentials',
].includes(part)
) ||
/\.(?:pem|key|p12|pfx)$/i.test(entry.name)
) {
throw new Error(`Credential-like release path is forbidden: ${entry.name}`);
}
// Exact FFmpeg vendor bytes are checked against pinned SHA-256 values. The
// generated Emscripten glue contains upstream build paths such as
// /home/web_user that are not paths from this repository or workstation.
if (entry.name.startsWith('vendor/ffmpeg/')) return;
if (!/\.(?:css|html|js|json|md|mjs|svg|txt)$/i.test(entry.name)) return;
const text = entry.data.toString('utf8');
if (/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/.test(text)) {
throw new Error(`Private-key material detected in ${entry.name}`);
}
if (/https?:\/\/[^/@\s]+:[^/@\s]+@/i.test(text)) {
throw new Error(`Credential-bearing URL detected in ${entry.name}`);
}
if (
/(?:^|[\s"'`])(?:\/home\/|\/mnt\/|\/Users\/|[A-Za-z]:\\Users\\)/m.test(text)
) {
throw new Error(`Local absolute path detected in ${entry.name}`);
}
}
function assertUniqueSortedEntries(entries) {
entries.sort((left, right) => compareText(left.name, right.name));
let totalBytes = 0;
let previous;
for (const entry of entries) {
safeArchivePath(entry.name);
if (entry.name === previous) {
throw new Error(`Duplicate release path: ${entry.name}`);
}
previous = entry.name;
if (entry.data.byteLength > maximumEntryBytes) {
throw new Error(`Release entry exceeds 256 MiB: ${entry.name}`);
}
totalBytes += entry.data.byteLength;
if (totalBytes > maximumArchiveBytes) {
throw new Error('Release exceeds the 2 GiB uncompressed-size limit.');
}
scanReleaseEntry(entry);
}
}
function manifestPath(reference, field, directoryIndex = false) {
if (
typeof reference !== 'string' ||
/^[A-Za-z][A-Za-z0-9+.-]*:/.test(reference) ||
reference.startsWith('/') ||
reference.startsWith('//') ||
reference.includes('?') ||
reference.includes('#')
) {
throw new Error(`${field} must be a plain relative packaged path.`);
}
let normalized = reference.replace(/^\.\//, '');
if (normalized === '' && directoryIndex) normalized = 'index.html';
else if (normalized.endsWith('/') && directoryIndex) {
normalized = `${normalized}index.html`;
}
return safeArchivePath(normalized);
}
function validateReleaseLayout(entries, packageJson, applicationLicense) {
const entriesByName = new Map(entries.map((entry) => [entry.name, entry]));
const names = new Set(entries.map((entry) => entry.name));
for (const required of [
'index.html',
'toolbox-app.json',
'CHANGELOG.md',
'SOURCE.md',
'THIRD_PARTY_NOTICES.md',
...requiredNativeLegalFiles,
]) {
if (!names.has(required)) {
throw new Error(`Required release root file is missing: ${required}`);
}
}
for (const prefix of ['LICENSES/', 'assets/', 'vendor/']) {
if (![...names].some((name) => name.startsWith(prefix))) {
throw new Error(`Required release directory is empty: ${prefix}`);
}
}
const manifestEntry = entries.find(
(entry) => entry.name === 'toolbox-app.json'
);
const manifest = JSON.parse(manifestEntry.data.toString('utf8'));
if (
manifest.id !== 'de.add-ideas.av-tools' ||
manifest.version !== packageJson.version
) {
throw new Error(
'Packaged manifest id/version does not match av-tools package.json.'
);
}
if (
applicationLicense &&
(manifest.source?.repository !== canonicalRepository ||
manifest.source?.license !== applicationLicenseSpdx)
) {
throw new Error(
'A licensed release requires an accurate, non-placeholder manifest.source declaration.'
);
}
if (!applicationLicense && manifest.source !== undefined) {
throw new Error(
'An unlicensed development artifact must omit manifest.source.'
);
}
const references = [
[manifest.entry, 'manifest.entry', true],
[manifest.icon, 'manifest.icon', false],
...(manifest.assets ?? []).map((asset, index) => [
asset,
`manifest.assets[${index}]`,
false,
]),
];
for (const [reference, field, directoryIndex] of references) {
const requiredPath = manifestPath(reference, field, directoryIndex);
if (!names.has(requiredPath)) {
throw new Error(`${field} is missing from release: ${reference}`);
}
}
const vendorBuildId = '0.12.10-reviewed-stack5m.1';
const vendorManifestPath = `vendor/ffmpeg/${vendorBuildId}/version.json`;
const vendorManifestEntry = entriesByName.get(vendorManifestPath);
if (!vendorManifestEntry) {
throw new Error(
`Required FFmpeg verification file is missing: ${vendorManifestPath}`
);
}
const vendorManifest = JSON.parse(vendorManifestEntry.data.toString('utf8'));
if (
vendorManifest.schemaVersion !== 2 ||
vendorManifest.buildId !== vendorBuildId ||
vendorManifest.profile !== 'reviewed-stack5m' ||
vendorManifest.coreVersion !== '0.12.10' ||
vendorManifest.opusRegression?.singleThread !== true ||
vendorManifest.opusRegression?.multiThread !== true ||
!Array.isArray(vendorManifest.assets) ||
vendorManifest.assets.length !== 5
) {
throw new Error(
'Packaged FFmpeg version.json has an unexpected shape or version.'
);
}
const verifiedVendorPaths = new Set();
for (const asset of vendorManifest.assets) {
if (
typeof asset.path !== 'string' ||
!Number.isSafeInteger(asset.bytes) ||
!/^[a-f0-9]{64}$/.test(asset.sha256)
) {
throw new Error(
'Packaged FFmpeg version.json contains invalid asset metadata.'
);
}
const assetPath = safeArchivePath(
path.posix.join(`vendor/ffmpeg/${vendorBuildId}`, asset.path)
);
if (verifiedVendorPaths.has(assetPath)) {
throw new Error(`Duplicate FFmpeg verification path: ${assetPath}`);
}
verifiedVendorPaths.add(assetPath);
const packagedAsset = entriesByName.get(assetPath);
if (
!packagedAsset ||
packagedAsset.data.byteLength !== asset.bytes ||
sha256(packagedAsset.data) !== asset.sha256
) {
throw new Error(
`Packaged FFmpeg asset failed byte verification: ${assetPath}`
);
}
}
const declaredVendorPaths = new Set(
(manifest.assets ?? []).map((reference) =>
manifestPath(reference, 'manifest.assets', false)
)
);
if (
verifiedVendorPaths.size !== declaredVendorPaths.size ||
[...verifiedVendorPaths].some(
(assetPath) => !declaredVendorPaths.has(assetPath)
)
) {
throw new Error(
'Manifest FFmpeg assets and the verified vendor asset set do not match.'
);
}
const indexHtml = entries
.find((entry) => entry.name === 'index.html')
.data.toString('utf8');
if (/(?:src|href)\s*=\s*["']\/(?!\/)/i.test(indexHtml)) {
throw new Error('index.html contains a root-absolute asset reference.');
}
}
export function createDeterministicZip(inputEntries) {
const entries = inputEntries.map((entry) => ({
name: safeArchivePath(entry.name),
data: Buffer.from(entry.data),
}));
assertUniqueSortedEntries(entries);
const zippable = Object.create(null);
for (const entry of entries) {
zippable[entry.name] = [
new Uint8Array(entry.data),
{
attrs: (0o100644 << 16) >>> 0,
level: 6,
mtime: zipEpoch,
os: 3,
},
];
}
return Buffer.from(
zipSync(zippable, {
attrs: (0o100644 << 16) >>> 0,
level: 6,
mtime: zipEpoch,
os: 3,
})
);
}
export async function collectReleaseEntries(options = {}) {
const root = options.root ?? repositoryRoot;
const packageJson = JSON.parse(
await readFile(path.join(root, 'package.json'), 'utf8')
);
const applicationLicense = await assertLicenseReady(
root,
options.allowUnlicensedDevelopmentSmoke
);
if (
applicationLicense &&
(packageJson.license !== applicationLicenseSpdx ||
packageJson.repository?.url !== canonicalPackageRepository ||
packageJson.homepage !== canonicalRepository)
) {
throw new Error(
'Licensed release package metadata must identify GPL-3.0-or-later and the canonical av-tools repository.'
);
}
const entries = await collectDirectoryEntries(path.join(root, 'dist'));
for (const fileName of [
'README.md',
'CHANGELOG.md',
'SOURCE.md',
'THIRD_PARTY_NOTICES.md',
]) {
entries.push({
name: fileName,
data: await readRequired(path.join(root, fileName)),
});
}
entries.push(
...(await collectDirectoryEntries(path.join(root, 'LICENSES'), 'LICENSES'))
);
entries.push(...(await collectRuntimeLicenseEntries(root, packageJson)));
if (applicationLicense) {
const applicationLicenseBytes = await readFile(applicationLicense);
entries.push({
name: path.basename(applicationLicense),
data: applicationLicenseBytes,
});
entries.push({
name: 'LICENSES/av-tools-LICENSE.txt',
data: applicationLicenseBytes,
});
} else {
entries.push({
name: 'DEVELOPMENT-ONLY-NOT-FOR-DISTRIBUTION.txt',
data: Buffer.from(
[
'UNLICENSED DEVELOPMENT SMOKE ARTIFACT',
'',
'av-tools has no application licence. This ZIP exists only to test',
'local Toolbox Portal assembly. Do not publish or distribute it.',
'',
].join('\n')
),
});
}
assertUniqueSortedEntries(entries);
validateReleaseLayout(entries, packageJson, applicationLicense);
return { entries, packageJson, unlicensed: !applicationLicense };
}
export function parseReleaseArguments(argumentsList) {
const allowed = new Set(['--force', '--allow-unlicensed-development-smoke']);
const unknown = argumentsList.filter((argument) => !allowed.has(argument));
if (unknown.length > 0) {
throw new Error(`Unknown release argument: ${unknown.join(', ')}`);
}
return {
force: argumentsList.includes('--force'),
allowUnlicensedDevelopmentSmoke: argumentsList.includes(
'--allow-unlicensed-development-smoke'
),
};
}
async function assertWritableTarget(target, force) {
try {
const details = await lstat(target);
if (details.isSymbolicLink() || !details.isFile()) {
throw new Error(`Release target is not a regular file: ${target}`);
}
if (!force) {
throw new Error(
`${path.relative(repositoryRoot, target)} exists; pass --force to replace this exact version.`
);
}
return true;
} catch (error) {
if (error && typeof error === 'object' && error.code === 'ENOENT') {
return false;
}
throw error;
}
}
async function publishReleasePair({
archive,
archivePath,
checksum,
checksumPath,
archiveExists,
checksumExists,
}) {
const releaseDirectory = path.dirname(archivePath);
const staging = await mkdtemp(
path.join(releaseDirectory, '.av-tools-release-')
);
const stagedArchive = path.join(staging, path.basename(archivePath));
const stagedChecksum = path.join(staging, path.basename(checksumPath));
const previousArchive = path.join(staging, 'previous-archive');
const previousChecksum = path.join(staging, 'previous-checksum');
let archiveBackedUp = false;
let checksumBackedUp = false;
let archivePublished = false;
let checksumPublished = false;
try {
await writeFile(stagedArchive, archive, { flag: 'wx' });
await writeFile(stagedChecksum, checksum, {
encoding: 'utf8',
flag: 'wx',
});
if (archiveExists) {
await rename(archivePath, previousArchive);
archiveBackedUp = true;
}
if (checksumExists) {
await rename(checksumPath, previousChecksum);
checksumBackedUp = true;
}
await rename(stagedArchive, archivePath);
archivePublished = true;
await rename(stagedChecksum, checksumPath);
checksumPublished = true;
} catch (error) {
if (checksumPublished) await rm(checksumPath, { force: true });
if (archivePublished) await rm(archivePath, { force: true });
if (checksumBackedUp) await rename(previousChecksum, checksumPath);
if (archiveBackedUp) await rename(previousArchive, archivePath);
throw error;
} finally {
await rm(staging, { force: true, recursive: true });
}
}
async function main() {
const options = parseReleaseArguments(process.argv.slice(2));
const { entries, packageJson, unlicensed } = await collectReleaseEntries({
allowUnlicensedDevelopmentSmoke: options.allowUnlicensedDevelopmentSmoke,
});
const releaseDirectory = path.join(repositoryRoot, 'release');
await mkdir(releaseDirectory, { recursive: true });
const releaseDirectoryDetails = await lstat(releaseDirectory);
if (
releaseDirectoryDetails.isSymbolicLink() ||
!releaseDirectoryDetails.isDirectory()
) {
throw new Error('release/ must be a real directory.');
}
const archiveName = `av-tools-${packageJson.version}.zip`;
const archivePath = path.join(releaseDirectory, archiveName);
const checksumPath = `${archivePath}.sha256`;
const archiveExists = await assertWritableTarget(archivePath, options.force);
const checksumExists = await assertWritableTarget(
checksumPath,
options.force
);
const archive = createDeterministicZip(entries);
const digest = sha256(archive);
await publishReleasePair({
archive,
archiveExists,
archivePath,
checksum: checksumLine(digest, archiveName),
checksumExists,
checksumPath,
});
if (unlicensed) {
console.warn(
'WARNING: created an unlicensed development smoke artifact. Do not publish or distribute it.'
);
}
console.log(`Created release/${archiveName}`);
console.log(`Created release/${archiveName}.sha256`);
console.log(`SHA-256 ${digest}`);
}
if (
process.argv[1] &&
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
) {
await main();
}