import { access, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile, } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { zipSync } from 'fflate'; import { checksumLine, sha256 } from './checksum-release.mjs'; const repositoryRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), '..' ); const zipEpoch = new Date(1980, 0, 1, 0, 0, 0); const maximumEntryBytes = 256 * 1024 * 1024; const maximumArchiveBytes = 2 * 1024 * 1024 * 1024; const applicationLicenseSpdx = 'GPL-3.0-or-later'; const canonicalRepository = 'https://git.add-ideas.de/zemion/av-tools'; const canonicalPackageRepository = 'git+https://git.add-ideas.de/zemion/av-tools.git'; const requiredNativeLegalFiles = [ 'LICENSES/native/FFmpeg-LICENSE.md', 'LICENSES/native/FreeType-LICENSE.txt', 'LICENSES/native/FriBidi-COPYING.txt', 'LICENSES/native/HarfBuzz-COPYING.txt', 'LICENSES/native/LAME-COPYING.txt', 'LICENSES/native/LAME-LICENSE.txt', 'LICENSES/native/Ogg-COPYING.txt', 'LICENSES/native/Opus-COPYING.txt', 'LICENSES/native/Theora-COPYING.txt', 'LICENSES/native/Vorbis-COPYING.txt', 'LICENSES/native/libass-COPYING.txt', 'LICENSES/native/libvpx-LICENSE.txt', 'LICENSES/native/libvpx-PATENTS.txt', 'LICENSES/native/libwebp-COPYING.txt', 'LICENSES/native/libwebp-PATENTS.txt', 'LICENSES/native/x264-COPYING.txt', 'LICENSES/native/x265-COPYING.txt', 'LICENSES/native/zimg-COPYING.txt', 'LICENSES/native/zlib-README.txt', ]; const ffmpegLicenseCoverage = new Map([ ['@ffmpeg/ffmpeg', 'LICENSES/ffmpeg.wasm-MIT.txt'], ['@ffmpeg/util', 'LICENSES/ffmpeg.wasm-MIT.txt'], ['@ffmpeg/types', 'LICENSES/ffmpeg.wasm-MIT.txt'], ]); function compareText(left, right) { return left < right ? -1 : left > right ? 1 : 0; } export function safeArchivePath(fileName) { if ( typeof fileName !== 'string' || fileName.length === 0 || fileName.includes('\0') || fileName.includes('\\') ) { throw new Error(`Invalid release path: ${String(fileName)}`); } if (fileName.startsWith('/') || /^[A-Za-z]:\//.test(fileName)) { throw new Error(`Absolute release path is forbidden: ${fileName}`); } const components = fileName.split('/'); if ( components.some( (component) => component === '' || component === '.' || component === '..' ) ) { throw new Error(`Ambiguous or traversing release path: ${fileName}`); } const normalized = path.posix.normalize(fileName); if (normalized !== fileName) { throw new Error(`Non-canonical release path: ${fileName}`); } return normalized; } function relativeArchivePath(root, file) { return safeArchivePath( path.relative(root, file).split(path.sep).join(path.posix.sep) ); } function isInside(root, candidate) { return candidate === root || candidate.startsWith(`${root}${path.sep}`); } async function collectDirectoryRecursive( canonicalRoot, sourceRoot, directory, prefix, entries ) { const canonicalDirectory = await realpath(directory); if (!isInside(canonicalRoot, canonicalDirectory)) { throw new Error(`Directory resolves outside packaging root: ${directory}`); } const children = await readdir(directory, { withFileTypes: true }); children.sort((left, right) => compareText(left.name, right.name)); for (const child of children) { const childPath = path.join(directory, child.name); const details = await lstat(childPath); if (details.isSymbolicLink()) { throw new Error(`Refusing release symlink: ${childPath}`); } if (details.isDirectory()) { await collectDirectoryRecursive( canonicalRoot, sourceRoot, childPath, prefix, entries ); continue; } if (!details.isFile()) { throw new Error(`Refusing special release entry: ${childPath}`); } const canonicalFile = await realpath(childPath); if (!isInside(canonicalRoot, canonicalFile)) { throw new Error(`File resolves outside packaging root: ${childPath}`); } const relativeName = relativeArchivePath(sourceRoot, childPath); const name = safeArchivePath( prefix ? path.posix.join(prefix, relativeName) : relativeName ); entries.push({ name, data: await readFile(childPath) }); } } export async function collectDirectoryEntries(directory, prefix = '') { const details = await lstat(directory); if (details.isSymbolicLink() || !details.isDirectory()) { throw new Error(`Release input is not a real directory: ${directory}`); } const sourceRoot = path.resolve(directory); const canonicalRoot = await realpath(directory); const entries = []; await collectDirectoryRecursive( canonicalRoot, sourceRoot, sourceRoot, prefix, entries ); return entries.sort((left, right) => compareText(left.name, right.name)); } async function readRequired(file) { try { return await readFile(file); } catch (error) { throw new Error(`Required release file is missing: ${file}`, { cause: error, }); } } async function existingApplicationLicense(root) { for (const fileName of ['LICENSE', 'LICENSE.txt', 'COPYING']) { const candidate = path.join(root, fileName); try { const details = await lstat(candidate); if (details.isSymbolicLink() || !details.isFile()) { throw new Error( `Application licence path must be a regular file: ${candidate}` ); } return candidate; } catch (error) { if (error && typeof error === 'object' && error.code === 'ENOENT') { continue; } throw error; } } return null; } export async function assertLicenseReady(root, allowDevelopmentSmoke = false) { const applicationLicense = await existingApplicationLicense(root); if (applicationLicense) { const legalCode = await readFile(applicationLicense, 'utf8'); if ( !legalCode.includes('GNU GENERAL PUBLIC LICENSE') || !legalCode.includes('Version 3, 29 June 2007') || !legalCode.includes('END OF TERMS AND CONDITIONS') ) { throw new Error( `Application licence does not contain the complete GNU GPL version 3 legal code: ${applicationLicense}` ); } return applicationLicense; } if (allowDevelopmentSmoke) return null; throw new Error( [ 'RELEASE BLOCKED: av-tools has no application licence.', 'The FFmpeg and dependency notices do not license the application itself.', 'For a local, non-publishable portal test only, pass', '`--allow-unlicensed-development-smoke`.', ].join(' ') ); } function safePackageName(name) { return name.replace(/^@/, '').replaceAll('/', '--'); } async function findInstalledPackageDirectory(name, fromDirectory) { let current = path.resolve(fromDirectory); const packageSegments = name.split('/'); while (true) { const candidate = path.join(current, 'node_modules', ...packageSegments); try { const details = await lstat(path.join(candidate, 'package.json')); if (!details.isFile()) { throw new Error(`Package manifest is not a file: ${candidate}`); } return await realpath(candidate); } catch (error) { if (!(error && typeof error === 'object' && error.code === 'ENOENT')) { throw error; } } const parent = path.dirname(current); if (parent === current) return null; current = parent; } } async function collectRuntimePackages(root, packageJson) { const queue = Object.keys(packageJson.dependencies ?? {}) .sort(compareText) .map((name) => ({ name, fromDirectory: root })); const visited = new Set(); const packages = []; while (queue.length > 0) { const dependency = queue.shift(); const directory = await findInstalledPackageDirectory( dependency.name, dependency.fromDirectory ); if (!directory) { throw new Error( `Runtime dependency is not installed: ${dependency.name}` ); } const manifest = JSON.parse( await readFile(path.join(directory, 'package.json'), 'utf8') ); const identity = `${manifest.name}@${manifest.version}`; if (visited.has(identity)) continue; visited.add(identity); packages.push({ directory, name: manifest.name, version: manifest.version, }); const childNames = [ ...new Set([ ...Object.keys(manifest.dependencies ?? {}), ...Object.keys(manifest.optionalDependencies ?? {}), ]), ].sort(compareText); for (const name of childNames) { const childDirectory = await findInstalledPackageDirectory( name, directory ); if (childDirectory) { queue.push({ name, fromDirectory: directory }); } } } return packages.sort((left, right) => compareText( `${left.name}@${left.version}`, `${right.name}@${right.version}` ) ); } export async function collectRuntimeLicenseEntries(root, packageJson) { const packages = await collectRuntimePackages(root, packageJson); const entries = []; for (const runtimePackage of packages) { const coveredBy = ffmpegLicenseCoverage.get(runtimePackage.name); if (coveredBy) { await access(path.join(root, coveredBy)); continue; } const legalFiles = ( await readdir(runtimePackage.directory, { withFileTypes: true, }) ) .filter( (entry) => entry.isFile() && /^(?:licen[cs]e|copying|notice)(?:\.|$)/i.test(entry.name) ) .sort((left, right) => compareText(left.name, right.name)); if (legalFiles.length === 0) { throw new Error( `Runtime dependency ${runtimePackage.name}@${runtimePackage.version} has no packaged licence file.` ); } const packageDirectory = `${safePackageName(runtimePackage.name)}-${runtimePackage.version}`; for (const legalFile of legalFiles) { entries.push({ name: safeArchivePath( `LICENSES/npm/${packageDirectory}/${legalFile.name}` ), data: await readFile( path.join(runtimePackage.directory, legalFile.name) ), }); } } return entries; } function scanReleaseEntry(entry) { if (entry.name.endsWith('.map')) { throw new Error(`Source map is not approved for release: ${entry.name}`); } const lowerParts = entry.name.toLowerCase().split('/'); if ( lowerParts.some((part) => [ '.env', '.npmrc', '.git', 'id_rsa', 'id_ed25519', 'credentials', ].includes(part) ) || /\.(?:pem|key|p12|pfx)$/i.test(entry.name) ) { throw new Error(`Credential-like release path is forbidden: ${entry.name}`); } // Exact FFmpeg vendor bytes are checked against pinned SHA-256 values. The // generated Emscripten glue contains upstream build paths such as // /home/web_user that are not paths from this repository or workstation. if (entry.name.startsWith('vendor/ffmpeg/')) return; if (!/\.(?:css|html|js|json|md|mjs|svg|txt)$/i.test(entry.name)) return; const text = entry.data.toString('utf8'); if (/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/.test(text)) { throw new Error(`Private-key material detected in ${entry.name}`); } if (/https?:\/\/[^/@\s]+:[^/@\s]+@/i.test(text)) { throw new Error(`Credential-bearing URL detected in ${entry.name}`); } if ( /(?:^|[\s"'`])(?:\/home\/|\/mnt\/|\/Users\/|[A-Za-z]:\\Users\\)/m.test(text) ) { throw new Error(`Local absolute path detected in ${entry.name}`); } } function assertUniqueSortedEntries(entries) { entries.sort((left, right) => compareText(left.name, right.name)); let totalBytes = 0; let previous; for (const entry of entries) { safeArchivePath(entry.name); if (entry.name === previous) { throw new Error(`Duplicate release path: ${entry.name}`); } previous = entry.name; if (entry.data.byteLength > maximumEntryBytes) { throw new Error(`Release entry exceeds 256 MiB: ${entry.name}`); } totalBytes += entry.data.byteLength; if (totalBytes > maximumArchiveBytes) { throw new Error('Release exceeds the 2 GiB uncompressed-size limit.'); } scanReleaseEntry(entry); } } function manifestPath(reference, field, directoryIndex = false) { if ( typeof reference !== 'string' || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(reference) || reference.startsWith('/') || reference.startsWith('//') || reference.includes('?') || reference.includes('#') ) { throw new Error(`${field} must be a plain relative packaged path.`); } let normalized = reference.replace(/^\.\//, ''); if (normalized === '' && directoryIndex) normalized = 'index.html'; else if (normalized.endsWith('/') && directoryIndex) { normalized = `${normalized}index.html`; } return safeArchivePath(normalized); } function validateReleaseLayout(entries, packageJson, applicationLicense) { const entriesByName = new Map(entries.map((entry) => [entry.name, entry])); const names = new Set(entries.map((entry) => entry.name)); for (const required of [ 'index.html', 'toolbox-app.json', 'CHANGELOG.md', 'SOURCE.md', 'THIRD_PARTY_NOTICES.md', ...requiredNativeLegalFiles, ]) { if (!names.has(required)) { throw new Error(`Required release root file is missing: ${required}`); } } for (const prefix of ['LICENSES/', 'assets/', 'vendor/']) { if (![...names].some((name) => name.startsWith(prefix))) { throw new Error(`Required release directory is empty: ${prefix}`); } } const manifestEntry = entries.find( (entry) => entry.name === 'toolbox-app.json' ); const manifest = JSON.parse(manifestEntry.data.toString('utf8')); if ( manifest.id !== 'de.add-ideas.av-tools' || manifest.version !== packageJson.version ) { throw new Error( 'Packaged manifest id/version does not match av-tools package.json.' ); } if ( applicationLicense && (manifest.source?.repository !== canonicalRepository || manifest.source?.license !== applicationLicenseSpdx) ) { throw new Error( 'A licensed release requires an accurate, non-placeholder manifest.source declaration.' ); } if (!applicationLicense && manifest.source !== undefined) { throw new Error( 'An unlicensed development artifact must omit manifest.source.' ); } const references = [ [manifest.entry, 'manifest.entry', true], [manifest.icon, 'manifest.icon', false], ...(manifest.assets ?? []).map((asset, index) => [ asset, `manifest.assets[${index}]`, false, ]), ]; for (const [reference, field, directoryIndex] of references) { const requiredPath = manifestPath(reference, field, directoryIndex); if (!names.has(requiredPath)) { throw new Error(`${field} is missing from release: ${reference}`); } } const vendorBuildId = '0.12.10-reviewed-stack5m.1'; const vendorManifestPath = `vendor/ffmpeg/${vendorBuildId}/version.json`; const vendorManifestEntry = entriesByName.get(vendorManifestPath); if (!vendorManifestEntry) { throw new Error( `Required FFmpeg verification file is missing: ${vendorManifestPath}` ); } const vendorManifest = JSON.parse(vendorManifestEntry.data.toString('utf8')); if ( vendorManifest.schemaVersion !== 2 || vendorManifest.buildId !== vendorBuildId || vendorManifest.profile !== 'reviewed-stack5m' || vendorManifest.coreVersion !== '0.12.10' || vendorManifest.opusRegression?.singleThread !== true || vendorManifest.opusRegression?.multiThread !== true || !Array.isArray(vendorManifest.assets) || vendorManifest.assets.length !== 5 ) { throw new Error( 'Packaged FFmpeg version.json has an unexpected shape or version.' ); } const verifiedVendorPaths = new Set(); for (const asset of vendorManifest.assets) { if ( typeof asset.path !== 'string' || !Number.isSafeInteger(asset.bytes) || !/^[a-f0-9]{64}$/.test(asset.sha256) ) { throw new Error( 'Packaged FFmpeg version.json contains invalid asset metadata.' ); } const assetPath = safeArchivePath( path.posix.join(`vendor/ffmpeg/${vendorBuildId}`, asset.path) ); if (verifiedVendorPaths.has(assetPath)) { throw new Error(`Duplicate FFmpeg verification path: ${assetPath}`); } verifiedVendorPaths.add(assetPath); const packagedAsset = entriesByName.get(assetPath); if ( !packagedAsset || packagedAsset.data.byteLength !== asset.bytes || sha256(packagedAsset.data) !== asset.sha256 ) { throw new Error( `Packaged FFmpeg asset failed byte verification: ${assetPath}` ); } } const declaredVendorPaths = new Set( (manifest.assets ?? []).map((reference) => manifestPath(reference, 'manifest.assets', false) ) ); if ( verifiedVendorPaths.size !== declaredVendorPaths.size || [...verifiedVendorPaths].some( (assetPath) => !declaredVendorPaths.has(assetPath) ) ) { throw new Error( 'Manifest FFmpeg assets and the verified vendor asset set do not match.' ); } const indexHtml = entries .find((entry) => entry.name === 'index.html') .data.toString('utf8'); if (/(?:src|href)\s*=\s*["']\/(?!\/)/i.test(indexHtml)) { throw new Error('index.html contains a root-absolute asset reference.'); } } export function createDeterministicZip(inputEntries) { const entries = inputEntries.map((entry) => ({ name: safeArchivePath(entry.name), data: Buffer.from(entry.data), })); assertUniqueSortedEntries(entries); const zippable = Object.create(null); for (const entry of entries) { zippable[entry.name] = [ new Uint8Array(entry.data), { attrs: (0o100644 << 16) >>> 0, level: 6, mtime: zipEpoch, os: 3, }, ]; } return Buffer.from( zipSync(zippable, { attrs: (0o100644 << 16) >>> 0, level: 6, mtime: zipEpoch, os: 3, }) ); } export async function collectReleaseEntries(options = {}) { const root = options.root ?? repositoryRoot; const packageJson = JSON.parse( await readFile(path.join(root, 'package.json'), 'utf8') ); const applicationLicense = await assertLicenseReady( root, options.allowUnlicensedDevelopmentSmoke ); if ( applicationLicense && (packageJson.license !== applicationLicenseSpdx || packageJson.repository?.url !== canonicalPackageRepository || packageJson.homepage !== canonicalRepository) ) { throw new Error( 'Licensed release package metadata must identify GPL-3.0-or-later and the canonical av-tools repository.' ); } const entries = await collectDirectoryEntries(path.join(root, 'dist')); for (const fileName of [ 'README.md', 'CHANGELOG.md', 'SOURCE.md', 'THIRD_PARTY_NOTICES.md', ]) { entries.push({ name: fileName, data: await readRequired(path.join(root, fileName)), }); } entries.push( ...(await collectDirectoryEntries(path.join(root, 'LICENSES'), 'LICENSES')) ); entries.push(...(await collectRuntimeLicenseEntries(root, packageJson))); if (applicationLicense) { const applicationLicenseBytes = await readFile(applicationLicense); entries.push({ name: path.basename(applicationLicense), data: applicationLicenseBytes, }); entries.push({ name: 'LICENSES/av-tools-LICENSE.txt', data: applicationLicenseBytes, }); } else { entries.push({ name: 'DEVELOPMENT-ONLY-NOT-FOR-DISTRIBUTION.txt', data: Buffer.from( [ 'UNLICENSED DEVELOPMENT SMOKE ARTIFACT', '', 'av-tools has no application licence. This ZIP exists only to test', 'local Toolbox Portal assembly. Do not publish or distribute it.', '', ].join('\n') ), }); } assertUniqueSortedEntries(entries); validateReleaseLayout(entries, packageJson, applicationLicense); return { entries, packageJson, unlicensed: !applicationLicense }; } export function parseReleaseArguments(argumentsList) { const allowed = new Set(['--force', '--allow-unlicensed-development-smoke']); const unknown = argumentsList.filter((argument) => !allowed.has(argument)); if (unknown.length > 0) { throw new Error(`Unknown release argument: ${unknown.join(', ')}`); } return { force: argumentsList.includes('--force'), allowUnlicensedDevelopmentSmoke: argumentsList.includes( '--allow-unlicensed-development-smoke' ), }; } async function assertWritableTarget(target, force) { try { const details = await lstat(target); if (details.isSymbolicLink() || !details.isFile()) { throw new Error(`Release target is not a regular file: ${target}`); } if (!force) { throw new Error( `${path.relative(repositoryRoot, target)} exists; pass --force to replace this exact version.` ); } return true; } catch (error) { if (error && typeof error === 'object' && error.code === 'ENOENT') { return false; } throw error; } } async function publishReleasePair({ archive, archivePath, checksum, checksumPath, archiveExists, checksumExists, }) { const releaseDirectory = path.dirname(archivePath); const staging = await mkdtemp( path.join(releaseDirectory, '.av-tools-release-') ); const stagedArchive = path.join(staging, path.basename(archivePath)); const stagedChecksum = path.join(staging, path.basename(checksumPath)); const previousArchive = path.join(staging, 'previous-archive'); const previousChecksum = path.join(staging, 'previous-checksum'); let archiveBackedUp = false; let checksumBackedUp = false; let archivePublished = false; let checksumPublished = false; try { await writeFile(stagedArchive, archive, { flag: 'wx' }); await writeFile(stagedChecksum, checksum, { encoding: 'utf8', flag: 'wx', }); if (archiveExists) { await rename(archivePath, previousArchive); archiveBackedUp = true; } if (checksumExists) { await rename(checksumPath, previousChecksum); checksumBackedUp = true; } await rename(stagedArchive, archivePath); archivePublished = true; await rename(stagedChecksum, checksumPath); checksumPublished = true; } catch (error) { if (checksumPublished) await rm(checksumPath, { force: true }); if (archivePublished) await rm(archivePath, { force: true }); if (checksumBackedUp) await rename(previousChecksum, checksumPath); if (archiveBackedUp) await rename(previousArchive, archivePath); throw error; } finally { await rm(staging, { force: true, recursive: true }); } } async function main() { const options = parseReleaseArguments(process.argv.slice(2)); const { entries, packageJson, unlicensed } = await collectReleaseEntries({ allowUnlicensedDevelopmentSmoke: options.allowUnlicensedDevelopmentSmoke, }); const releaseDirectory = path.join(repositoryRoot, 'release'); await mkdir(releaseDirectory, { recursive: true }); const releaseDirectoryDetails = await lstat(releaseDirectory); if ( releaseDirectoryDetails.isSymbolicLink() || !releaseDirectoryDetails.isDirectory() ) { throw new Error('release/ must be a real directory.'); } const archiveName = `av-tools-${packageJson.version}.zip`; const archivePath = path.join(releaseDirectory, archiveName); const checksumPath = `${archivePath}.sha256`; const archiveExists = await assertWritableTarget(archivePath, options.force); const checksumExists = await assertWritableTarget( checksumPath, options.force ); const archive = createDeterministicZip(entries); const digest = sha256(archive); await publishReleasePair({ archive, archiveExists, archivePath, checksum: checksumLine(digest, archiveName), checksumExists, checksumPath, }); if (unlicensed) { console.warn( 'WARNING: created an unlicensed development smoke artifact. Do not publish or distribute it.' ); } console.log(`Created release/${archiveName}`); console.log(`Created release/${archiveName}.sha256`); console.log(`SHA-256 ${digest}`); } if ( process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href ) { await main(); }