FoXiMao/a-share-analysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(GOV-012): 升级 Schema 基线 format v3

Browse Source
This commit is contained in:
FoXiMao
2026-08-18 02:24:54 +08:00
parent dc8ad7d4b6
commit c11de09fa7
4 changed files with 396 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/database/schema-baseline.mjs
+256 -110
View File
@@ -12,12 +12,20 @@ import { createHash } from 'node:crypto';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
MYSQL2_EXPORTER_METADATA,
assertMysql57ServerMetadata as assertMysql57ServerMetadataV3,
exportMysql57Schema,
} from './mysql2-schema-exporter.mjs';
const DEVELOPMENT_DATABASE = 'a_share_analysis_dev';
const REQUIRED_CHARSET = 'utf8mb4';
const REQUIRED_COLLATION = 'utf8mb4_general_ci';
const BASELINE_FORMAT_VERSION = 2;
const BASELINE_FORMAT_VERSION = 3;
const PREVIOUS_BASELINE_FORMAT_VERSION = 2;
const LEGACY_BASELINE_FORMAT_VERSION = 1;
const TOOL_VERSION = '2.0.0';
const TOOL_VERSION = '3.0.0';
const PREVIOUS_TOOL_VERSION = '2.0.0';
const LEGACY_TOOL_VERSION = '1.0.0';
const VERSION_PATTERN = /^v\d+\.\d+\.\d+$/;
const DEVELOPMENT_HOSTS = new Set(['127.0.0.1', 'localhost', '::1', '112.192.20.11']);
@@ -283,54 +291,108 @@ function countSchemaObjects(sql) {
function buildManifest(fields) {
// JSON 是 YAML 1.2 的严格子集；使用标准解析器生成和读取，避免手写 YAML
// 对缩进、引号和同名嵌套字段产生歧义，同时仍保留约定的 manifest.yaml 文件名。
return `${JSON.stringify(
{
format_version: BASELINE_FORMAT_VERSION,
status: fields.status,
application_version: fields.applicationVersion,
git_commit: fields.gitCommit,
source_environment: 'development',
database_name: DEVELOPMENT_DATABASE,
mysql_client_version: fields.mysqlClientVersion,
mysqldump_client_version: fields.mysqldumpClientVersion,
mysql_server_version: fields.serverVersion,
charset: REQUIRED_CHARSET,
collation: REQUIRED_COLLATION,
generated_at: fields.generatedAt,
generator: `schema-baseline/${TOOL_VERSION}`,
reason: fields.reason ?? 'none',
schema_sha256: fields.schemaSha256 ?? 'not_applicable',
objects: {
tables: fields.captured,
views: fields.captured,
triggers: fields.captured,
routines: fields.captured,
events: fields.captured,
},
object_counts: fields.objectCounts,
const common = {
format_version: fields.formatVersion,
status: fields.status,
application_version: fields.applicationVersion,
git_commit: fields.gitCommit,
source_environment: 'development',
database_name: DEVELOPMENT_DATABASE,
charset: REQUIRED_CHARSET,
collation: REQUIRED_COLLATION,
generated_at: fields.generatedAt,
generator: `schema-baseline/${fields.formatVersion === 3 ? TOOL_VERSION : fields.formatVersion === 2 ? PREVIOUS_TOOL_VERSION : LEGACY_TOOL_VERSION}`,
reason: fields.reason ?? 'none',
schema_sha256: fields.schemaSha256 ?? 'not_applicable',
objects: {
tables: fields.captured,
views: fields.captured,
triggers: fields.captured,
routines: fields.captured,
events: fields.captured,
},
null,
2,
)}\n`;
object_counts: fields.objectCounts,
};
if (fields.formatVersion === 3) {
common.exporter_name = fields.exporterName;
common.exporter_version = fields.exporterVersion;
common.database_driver = fields.driverName;
common.database_driver_version = fields.driverVersion;
common.mysql_server_version = fields.serverVersion;
} else {
common.mysql_client_version = fields.mysqlClientVersion;
common.mysqldump_client_version = fields.mysqldumpClientVersion;
common.mysql_server_version = fields.serverVersion;
}
return `${JSON.stringify(common, null, 2)}\n`;
}
export function buildCapturedArtifacts(metadata, sql) {
function assertNodeMetadata(metadata) {
if (metadata.formatVersion !== 3) {
throw new Error('Node Schema 基线必须使用 format v3');
}
for (const key of ['exporterName', 'exporterVersion', 'driverName', 'driverVersion']) {
if (!String(metadata[key] ?? '').trim()) {
throw new Error(`Node Schema 元数据缺少 ${key}`);
}
}
assertMysql57ServerMetadataV3(metadata);
}
export function buildCapturedArtifacts(metadata, sql, objectCounts) {
if (metadata.formatVersion === 3) {
assertNodeMetadata(metadata);
if (!objectCounts || Object.keys(objectCounts).sort().join(',') !== [...OBJECT_KEYS].sort().join(',')) {
throw new Error('Node Schema 对象计数字段无效');
}
const normalizedSql = normalizeSchemaSql(sql);
const derivedCounts = countSchemaObjects(normalizedSql);
for (const key of OBJECT_KEYS) {
if (!Number.isInteger(objectCounts[key]) || objectCounts[key] < 0 || objectCounts[key] !== derivedCounts[key]) {
throw new Error(`Node Schema 对象计数与 SQL 不一致：${key}`);
}
}
if (String(sql).trim() && Object.values(objectCounts).every((count) => count === 0)) {
throw new Error('Node Schema SQL 非空但对象计数全为零');
}
if (Object.values(objectCounts).every((count) => count === 0)) {
return buildCapturedEmptyArtifacts(metadata);
}
const schemaSha256 = sha256(normalizedSql);
const manifest = buildManifest({
...metadata,
status: 'captured',
captured: true,
objectCounts,
schemaSha256,
});
return Object.freeze({
status: 'captured',
schemaSha256,
files: Object.freeze({
'manifest.yaml': manifest,
'schema.sql': normalizedSql,
'checksum.sha256': `${schemaSha256} schema.sql\n`,
}),
});
}
assertMysql57Metadata(metadata);
if (!VERSION_PATTERN.test(metadata.applicationVersion ?? '')) {
throw new Error(`应用版本格式无效：${metadata.applicationVersion || '(空)'}`);
}
const normalizedSql = normalizeSchemaSql(sql);
const objectCounts = countSchemaObjects(normalizedSql);
const derivedCounts = countSchemaObjects(normalizedSql);
// mysqldump 对空库仍会输出会话 SET 与头尾注释；结构对象计数为零才是空库判据。
if (Object.values(objectCounts).every((count) => count === 0)) {
if (Object.values(derivedCounts).every((count) => count === 0)) {
return buildCapturedEmptyArtifacts(metadata);
}
const schemaSha256 = sha256(normalizedSql);
const manifest = buildManifest({
...metadata,
formatVersion: metadata.formatVersion ?? PREVIOUS_BASELINE_FORMAT_VERSION,
status: 'captured',
captured: true,
objectCounts,
objectCounts: derivedCounts,
schemaSha256,
});
return Object.freeze({
@@ -345,6 +407,23 @@ export function buildCapturedArtifacts(metadata, sql) {
}
export function buildCapturedEmptyArtifacts(metadata) {
if (metadata.formatVersion === 3) {
assertNodeMetadata(metadata);
const schemaSha256 = sha256('');
const objectCounts = Object.freeze(Object.fromEntries(OBJECT_KEYS.map((key) => [key, 0])));
const manifest = buildManifest({
...metadata,
status: 'captured_empty',
captured: false,
objectCounts,
schemaSha256,
});
return Object.freeze({
status: 'captured_empty',
schemaSha256,
files: Object.freeze({ 'manifest.yaml': manifest }),
});
}
assertMysql57Metadata(metadata);
if (!VERSION_PATTERN.test(metadata.applicationVersion ?? '')) {
throw new Error(`应用版本格式无效：${metadata.applicationVersion || '(空)'}`);
@@ -353,6 +432,7 @@ export function buildCapturedEmptyArtifacts(metadata) {
const objectCounts = Object.freeze(Object.fromEntries(OBJECT_KEYS.map((key) => [key, 0])));
const manifest = buildManifest({
...metadata,
formatVersion: metadata.formatVersion ?? PREVIOUS_BASELINE_FORMAT_VERSION,
status: 'captured_empty',
captured: false,
objectCounts,
@@ -369,8 +449,14 @@ export function buildNotApplicableArtifacts(metadata) {
if (!VERSION_PATTERN.test(metadata.applicationVersion ?? '')) {
throw new Error(`应用版本格式无效：${metadata.applicationVersion || '(空)'}`);
}
const manifest = buildManifest({
const nodeMetadata = metadata.formatVersion === 3 ? metadata : {
...metadata,
formatVersion: PREVIOUS_BASELINE_FORMAT_VERSION,
};
const manifest = buildManifest({
...nodeMetadata,
...metadata,
formatVersion: nodeMetadata.formatVersion,
mysqlClientVersion: 'not_enabled',
mysqldumpClientVersion: 'not_enabled',
serverVersion: 'not_enabled',
@@ -403,28 +489,65 @@ function compareExistingArtifacts(targetDirectory, artifacts) {
* 先在目标父目录写入临时目录，再原子改名。这样异常不会留下半份基线；
* 已发布版本目录只允许逐字节相同的幂等重跑，任何差异都必须人工留痕修正。
*/
export function persistBaselineArtifacts(targetDirectory, artifacts) {
if (existsSync(targetDirectory)) {
if (compareExistingArtifacts(targetDirectory, artifacts)) {
const DEFAULT_FILE_OPERATIONS = Object.freeze({
existsSync,
mkdirSync,
readFileSync,
readdirSync,
renameSync,
rmSync,
writeFileSync,
});
export function persistBaselineArtifacts(
targetDirectory,
artifacts,
fileOperations = DEFAULT_FILE_OPERATIONS,
) {
const operations = { ...DEFAULT_FILE_OPERATIONS, ...fileOperations };
const {
existsSync: fileExists,
mkdirSync: makeDirectory,
readFileSync: readFile,
readdirSync: readDirectory,
renameSync: renameFile,
rmSync: removePath,
writeFileSync: writeFile,
} = operations;
const compare = () => {
const actualFiles = readDirectory(targetDirectory).sort();
const expectedFiles = Object.keys(artifacts.files).sort();
if (
actualFiles.length !== expectedFiles.length ||
actualFiles.some((file, index) => file !== expectedFiles[index])
) {
return false;
}
return expectedFiles.every(
(file) => readFile(path.join(targetDirectory, file), 'utf8') === artifacts.files[file],
);
};
if (fileExists(targetDirectory)) {
if (compare()) {
return { reused: true };
}
throw new Error(`目标版本基线已存在且内容不同，禁止覆盖：${targetDirectory}`);
}
const parentDirectory = path.dirname(targetDirectory);
mkdirSync(parentDirectory, { recursive: true });
makeDirectory(parentDirectory, { recursive: true });
const temporaryDirectory = path.join(
parentDirectory,
`.${path.basename(targetDirectory)}.tmp-${process.pid}-${Date.now()}`,
);
mkdirSync(temporaryDirectory);
makeDirectory(temporaryDirectory);
try {
for (const [file, content] of Object.entries(artifacts.files)) {
writeFileSync(path.join(temporaryDirectory, file), content, 'utf8');
writeFile(path.join(temporaryDirectory, file), content, 'utf8');
}
renameSync(temporaryDirectory, targetDirectory);
renameFile(temporaryDirectory, targetDirectory);
} catch (error) {
rmSync(temporaryDirectory, { recursive: true, force: true });
removePath(temporaryDirectory, { recursive: true, force: true });
throw error;
}
return { reused: false };
@@ -468,6 +591,8 @@ export function verifyBaselineDirectory(targetDirectory, expectedRelease, expect
const supportedFormat =
(manifest.format_version === LEGACY_BASELINE_FORMAT_VERSION &&
manifest.generator === `schema-baseline/${LEGACY_TOOL_VERSION}`) ||
(manifest.format_version === PREVIOUS_BASELINE_FORMAT_VERSION &&
manifest.generator === `schema-baseline/${PREVIOUS_TOOL_VERSION}`) ||
(manifest.format_version === BASELINE_FORMAT_VERSION &&
manifest.generator === `schema-baseline/${TOOL_VERSION}`);
if (
@@ -506,30 +631,49 @@ export function verifyBaselineDirectory(targetDirectory, expectedRelease, expect
assertExactFileSet(targetDirectory, NOT_APPLICABLE_FILES);
const reason = manifest.reason;
requireExactValue(reason, 'mysql_disabled', 'Schema 不适用原因无效');
if (
manifest.mysql_client_version !== 'not_enabled' ||
manifest.mysqldump_client_version !== 'not_enabled' ||
const commonInvalid =
manifest.mysql_server_version !== 'not_enabled' ||
manifest.schema_sha256 !== 'not_applicable' ||
Object.keys(manifest.objects ?? {}).sort().join(',') !== [...OBJECT_KEYS].sort().join(',') ||
OBJECT_KEYS.some((key) => manifest.objects[key] !== false)
) {
Object.keys(manifest.objects ?? {}).sort().join(',') !==
[...OBJECT_KEYS].sort().join(',') ||
OBJECT_KEYS.some((key) => manifest.objects[key] !== false);
const formatInvalid =
manifest.format_version === BASELINE_FORMAT_VERSION
? !String(manifest.exporter_name ?? '').trim() ||
!String(manifest.exporter_version ?? '').trim() ||
!String(manifest.database_driver ?? '').trim() ||
!String(manifest.database_driver_version ?? '').trim() ||
'mysql_client_version' in manifest ||
'mysqldump_client_version' in manifest
: manifest.mysql_client_version !== 'not_enabled' ||
manifest.mysqldump_client_version !== 'not_enabled';
if (commonInvalid || formatInvalid) {
throw new Error('Schema 不适用清单必须证明 MySQL 未启用且所有对象均未捕获');
}
return { status, applicationVersion, reason };
}
if (status === 'captured_empty') {
if (manifest.format_version !== BASELINE_FORMAT_VERSION) {
throw new Error('空库 Schema 基线只允许 format v2');
}
assertExactFileSet(targetDirectory, CAPTURED_EMPTY_FILES);
assertMysql57Metadata({
mysqlClientVersion: manifest.mysql_client_version,
mysqldumpClientVersion: manifest.mysqldump_client_version,
serverVersion: manifest.mysql_server_version,
databaseCharset: manifest.charset,
databaseCollation: manifest.collation,
});
if (manifest.format_version === BASELINE_FORMAT_VERSION) {
assertNodeMetadata({
formatVersion: 3,
exporterName: manifest.exporter_name,
exporterVersion: manifest.exporter_version,
driverName: manifest.database_driver,
driverVersion: manifest.database_driver_version,
serverVersion: manifest.mysql_server_version,
databaseCharset: manifest.charset,
databaseCollation: manifest.collation,
});
} else {
assertMysql57Metadata({
mysqlClientVersion: manifest.mysql_client_version,
mysqldumpClientVersion: manifest.mysqldump_client_version,
serverVersion: manifest.mysql_server_version,
databaseCharset: manifest.charset,
databaseCollation: manifest.collation,
});
}
const expectedEmptyChecksum = sha256('');
if (
manifest.schema_sha256 !== expectedEmptyChecksum ||
@@ -546,13 +690,26 @@ export function verifyBaselineDirectory(targetDirectory, expectedRelease, expect
}
assertExactFileSet(targetDirectory, CAPTURED_FILES);
assertMysql57Metadata({
mysqlClientVersion: manifest.mysql_client_version,
mysqldumpClientVersion: manifest.mysqldump_client_version,
serverVersion: manifest.mysql_server_version,
databaseCharset: manifest.charset,
databaseCollation: manifest.collation,
});
if (manifest.format_version === BASELINE_FORMAT_VERSION) {
assertNodeMetadata({
formatVersion: 3,
exporterName: manifest.exporter_name,
exporterVersion: manifest.exporter_version,
driverName: manifest.database_driver,
driverVersion: manifest.database_driver_version,
serverVersion: manifest.mysql_server_version,
databaseCharset: manifest.charset,
databaseCollation: manifest.collation,
});
} else {
assertMysql57Metadata({
mysqlClientVersion: manifest.mysql_client_version,
mysqldumpClientVersion: manifest.mysqldump_client_version,
serverVersion: manifest.mysql_server_version,
databaseCharset: manifest.charset,
databaseCollation: manifest.collation,
});
}
if (Object.keys(manifest.objects ?? {}).sort().join(',') !== [...OBJECT_KEYS].sort().join(',')) {
throw new Error('Schema 对象范围字段无效');
}
@@ -583,6 +740,14 @@ export function verifyBaselineDirectory(targetDirectory, expectedRelease, expect
if (manifestChecksum !== schemaSha256 || checksumFile !== `${schemaSha256} schema.sql\n`) {
throw new Error('Schema SQL、manifest 和 checksum 校验值不一致');
}
if (manifest.format_version === BASELINE_FORMAT_VERSION) {
const derivedCounts = countSchemaObjects(sql);
for (const key of OBJECT_KEYS) {
if (manifest.object_counts[key] !== derivedCounts[key]) {
throw new Error(`Schema 对象计数与 SQL 不一致：${key}`);
}
}
}
return { status, applicationVersion, schemaSha256 };
}
@@ -614,7 +779,7 @@ function extractClientVersion(output) {
* 真实执行器可注入是为了让测试证明“关闭时零数据库调用”和命令边界；默认执行器
* 仍使用 execFileSync 参数数组。密码仅放入子进程环境 MYSQL_PWD，不会进入 argv。
*/
export function runBaselineGeneration(options) {
export async function runBaselineGeneration(options) {
if ((options.commandArguments ?? []).length > 0) {
throw new Error('Schema 基线命令不接受命令参数，不能覆盖环境、数据库或 env 路径');
}
@@ -641,52 +806,33 @@ export function runBaselineGeneration(options) {
generatedAt: options.generatedAt ?? new Date().toISOString(),
};
if (!config.MYSQL_ENABLED) {
const artifacts = buildNotApplicableArtifacts(commonMetadata);
persistBaselineArtifacts(targetDirectory, artifacts);
const artifacts = buildNotApplicableArtifacts({
...commonMetadata,
formatVersion: BASELINE_FORMAT_VERSION,
exporterName: MYSQL2_EXPORTER_METADATA.name,
exporterVersion: MYSQL2_EXPORTER_METADATA.version,
driverName: MYSQL2_EXPORTER_METADATA.driverName,
driverVersion: MYSQL2_EXPORTER_METADATA.driverVersion,
serverVersion: 'not_enabled',
databaseCharset: REQUIRED_CHARSET,
databaseCollation: REQUIRED_COLLATION,
});
persistBaselineArtifacts(targetDirectory, artifacts, options.fileOperations);
return verifyBaselineDirectory(targetDirectory, options.applicationVersion);
}
const executeFile = options.executeFile ?? execFileSync;
const childEnvironment = { ...process.env, MYSQL_PWD: config.MYSQL_PASSWORD ?? '' };
const executionOptions = {
cwd: options.repositoryRoot,
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
env: childEnvironment,
};
const mysqlClientVersion = extractClientVersion(
executeFile('mysql', ['--version'], executionOptions),
);
const mysqldumpClientVersion = extractClientVersion(
executeFile('mysqldump', ['--version'], executionOptions),
);
assertMysql57Metadata({
mysqlClientVersion,
mysqldumpClientVersion,
serverVersion: '5.7.0-pending-server-check',
databaseCharset: REQUIRED_CHARSET,
databaseCollation: REQUIRED_COLLATION,
});
const serverMetadata = String(
executeFile('mysql', buildMysqlMetadataArguments(config), executionOptions),
)
.trim()
.split('\t');
if (serverMetadata.length !== 3) {
throw new Error('无法从固定开发库读取 MySQL 版本、字符集和排序规则');
}
const exported = await (options.exportSchema ?? exportMysql57Schema)({ config });
const exportedMetadata = exported.metadata ?? {};
const metadata = {
...commonMetadata,
mysqlClientVersion,
mysqldumpClientVersion,
serverVersion: serverMetadata[0],
databaseCharset: serverMetadata[1],
databaseCollation: serverMetadata[2],
...exportedMetadata,
applicationVersion: exportedMetadata.applicationVersion ?? commonMetadata.applicationVersion,
gitCommit: exportedMetadata.gitCommit ?? commonMetadata.gitCommit,
generatedAt: exportedMetadata.generatedAt ?? commonMetadata.generatedAt,
formatVersion: BASELINE_FORMAT_VERSION,
};
assertMysql57Metadata(metadata);
const rawSql = executeFile('mysqldump', buildMysqldumpArguments(config), executionOptions);
const artifacts = buildCapturedArtifacts(metadata, rawSql);
persistBaselineArtifacts(targetDirectory, artifacts);
const artifacts = buildCapturedArtifacts(metadata, exported.rawSql, exported.objectCounts);
persistBaselineArtifacts(targetDirectory, artifacts, options.fileOperations);
return verifyBaselineDirectory(targetDirectory, options.applicationVersion);
}
@@ -815,7 +961,7 @@ export function resolveFixedDevelopmentEnv(repositoryRoot) {
return path.join(repositoryRoot, 'deploy', 'env', 'development', 'api.env.example');
}
function runCommandLine() {
export async function runCommandLine() {
const [command, ...commandArguments] = process.argv.slice(2);
if (!['generate', 'verify'].includes(command) || commandArguments.length > 0) {
throw new Error('用法：node tools/database/schema-baseline.mjs <generate|verify>（不接受额外参数）');
@@ -824,7 +970,7 @@ function runCommandLine() {
const release = getCurrentReleasedVersion(repositoryRoot);
const result =
command === 'generate'
? runBaselineGeneration({
? await runBaselineGeneration({
repositoryRoot,
envFile: resolveFixedDevelopmentEnv(repositoryRoot),
...release,
@@ -837,5 +983,5 @@ function runCommandLine() {
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
runCommandLine();
await runCommandLine();
}