Files
a-share-analysis/tools/database/mysql2-schema-exporter.test.mjs
T

569 lines
18 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import assert from 'node:assert/strict';
import test from 'node:test';
import {
DEFAULT_EXPORT_LIMITS,
MYSQL2_EXPORTER_METADATA,
SchemaExportError,
assertMysql57ServerMetadata,
buildConnectionOptions,
exportMysql57Schema,
normalizeCreateDefinition,
orderViewNames,
quoteIdentifier,
readDefinitions,
readMysql57ServerMetadata,
readSchemaCatalog,
renderSchemaSql,
} from './mysql2-schema-exporter.mjs';
const config = Object.freeze({
MYSQL_HOST: '127.0.0.1',
MYSQL_PORT: '3306',
MYSQL_DATABASE: 'a_share_analysis_dev',
MYSQL_USERNAME: 'schema_reader',
MYSQL_PASSWORD: 'not-for-logs',
});
test('连接配置固定单连接、utf8mb4、东八区且禁用多语句', () => {
const options = buildConnectionOptions(config);
assert.deepEqual(options, {
host: '127.0.0.1',
port: 3306,
user: 'schema_reader',
password: 'not-for-logs',
database: 'a_share_analysis_dev',
charset: 'utf8mb4',
timezone: '+08:00',
connectTimeout: 5_000,
multipleStatements: false,
enableKeepAlive: true,
});
});
test('标识符转义反引号并拒绝空值和控制字符', () => {
assert.equal(quoteIdentifier('market`view'), '`market``view`');
for (const value of ['', 'line\nbreak', 'zero\0byte']) {
assert.throws(() => quoteIdentifier(value), SchemaExportError);
}
});
test('只接受 MySQL 5.7.x 与固定字符规则', () => {
const valid = {
serverVersion: '5.7.42-log',
databaseCharset: 'utf8mb4',
databaseCollation: 'utf8mb4_general_ci',
};
assert.doesNotThrow(() => assertMysql57ServerMetadata(valid));
for (const serverVersion of [
'5.6.51',
'8.0.40',
'5.7.42-MariaDB',
'10.6.18-MariaDB',
]) {
assert.throws(() => assertMysql57ServerMetadata({ ...valid, serverVersion }));
}
});
function createCatalogConnection(overrides = {}) {
const rowsBySource = {
'information_schema.TABLES': [
{ TABLE_NAME: 'market_snapshot', TABLE_TYPE: 'BASE TABLE' },
{ TABLE_NAME: 'market_dashboard', TABLE_TYPE: 'VIEW' },
{ TABLE_NAME: 'market_summary', TABLE_TYPE: 'VIEW' },
],
'information_schema.VIEW_TABLE_USAGE': [
{
VIEW_NAME: 'market_dashboard',
REFERENCED_SCHEMA_NAME: 'a_share_analysis_dev',
REFERENCED_TABLE_NAME: 'market_summary',
},
{
VIEW_NAME: 'market_summary',
REFERENCED_SCHEMA_NAME: 'a_share_analysis_dev',
REFERENCED_TABLE_NAME: 'market_snapshot',
},
],
'information_schema.ROUTINES': [
{ ROUTINE_NAME: 'calculate_breadth', ROUTINE_TYPE: 'FUNCTION' },
{ ROUTINE_NAME: 'refresh_market_snapshot', ROUTINE_TYPE: 'PROCEDURE' },
],
'information_schema.TRIGGERS': [
{
TRIGGER_NAME: 'snapshot_before_insert',
EVENT_OBJECT_TABLE: 'market_snapshot',
ACTION_TIMING: 'BEFORE',
EVENT_MANIPULATION: 'INSERT',
ACTION_ORDER: 1,
},
],
'information_schema.EVENTS': [{ EVENT_NAME: 'purge_market_snapshot' }],
...overrides,
};
const calls = [];
return {
calls,
async query(options) {
calls.push(options);
const source = Object.keys(rowsBySource).find((candidate) => options.sql.includes(candidate));
assert.ok(source, `未识别的目录查询:${options.sql}`);
return [rowsBySource[source], []];
},
};
}
test('目录查询使用固定参数和超时并返回稳定对象', async () => {
const connection = createCatalogConnection();
const catalog = await readSchemaCatalog(connection, 'a_share_analysis_dev', DEFAULT_EXPORT_LIMITS);
assert.deepEqual(catalog, {
tables: ['market_snapshot'],
views: ['market_dashboard', 'market_summary'],
viewDependencies: [
{
view: 'market_dashboard',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_summary',
},
{
view: 'market_summary',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_snapshot',
},
],
routines: [
{ name: 'calculate_breadth', type: 'FUNCTION' },
{ name: 'refresh_market_snapshot', type: 'PROCEDURE' },
],
triggers: [
{
name: 'snapshot_before_insert',
table: 'market_snapshot',
timing: 'BEFORE',
event: 'INSERT',
order: 1,
},
],
events: ['purge_market_snapshot'],
});
assert.equal(connection.calls.length, 5);
for (const call of connection.calls) {
assert.deepEqual(call.values, ['a_share_analysis_dev']);
assert.equal(call.timeout, DEFAULT_EXPORT_LIMITS.queryTimeoutMs);
}
});
test('视图按依赖拓扑排序且同层按名称稳定排序', () => {
const catalog = {
tables: ['market_snapshot'],
views: ['z_dashboard', 'market_summary', 'a_dashboard'],
viewDependencies: [
{
view: 'z_dashboard',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_summary',
},
{
view: 'a_dashboard',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_summary',
},
{
view: 'market_summary',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_snapshot',
},
],
};
assert.deepEqual(orderViewNames(catalog), ['market_summary', 'a_dashboard', 'z_dashboard']);
});
test('视图排序拒绝循环、跨库和未知依赖', () => {
const base = { tables: ['market_snapshot'], views: ['first_view', 'second_view'] };
const invalidCases = [
{
code: 'VIEW_DEPENDENCY_CYCLE',
dependencies: [
{ view: 'first_view', referencedSchema: 'a_share_analysis_dev', referencedObject: 'second_view' },
{ view: 'second_view', referencedSchema: 'a_share_analysis_dev', referencedObject: 'first_view' },
],
},
{
code: 'CROSS_SCHEMA_VIEW_DEPENDENCY',
dependencies: [
{ view: 'first_view', referencedSchema: 'production', referencedObject: 'market_snapshot' },
],
},
{
code: 'UNKNOWN_VIEW_DEPENDENCY',
dependencies: [
{ view: 'first_view', referencedSchema: 'a_share_analysis_dev', referencedObject: 'missing_object' },
],
},
];
for (const invalid of invalidCases) {
assert.throws(
() => orderViewNames({ ...base, viewDependencies: invalid.dependencies }),
(error) => error instanceof SchemaExportError && error.code === invalid.code,
);
}
});
test('目录拒绝未知表类型和超过对象上限', async () => {
const unknownType = createCatalogConnection({
'information_schema.TABLES': [{ TABLE_NAME: 'temporary_data', TABLE_TYPE: 'SYSTEM VIEW' }],
});
await assert.rejects(
() => readSchemaCatalog(unknownType, 'a_share_analysis_dev', DEFAULT_EXPORT_LIMITS),
(error) => error instanceof SchemaExportError && error.code === 'UNKNOWN_TABLE_TYPE',
);
const overLimit = createCatalogConnection({
'information_schema.TABLES': [
{ TABLE_NAME: 'first_table', TABLE_TYPE: 'BASE TABLE' },
{ TABLE_NAME: 'second_table', TABLE_TYPE: 'BASE TABLE' },
],
});
await assert.rejects(
() => readSchemaCatalog(overLimit, 'a_share_analysis_dev', { ...DEFAULT_EXPORT_LIMITS, maxObjects: 1 }),
(error) => error instanceof SchemaExportError && error.code === 'OBJECT_LIMIT_EXCEEDED',
);
});
const completeCatalog = Object.freeze({
tables: ['market_snapshot'],
views: ['market_dashboard', 'market_summary'],
viewDependencies: [
{
view: 'market_dashboard',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_summary',
},
{
view: 'market_summary',
referencedSchema: 'a_share_analysis_dev',
referencedObject: 'market_snapshot',
},
],
routines: [
{ name: 'calculate_breadth', type: 'FUNCTION' },
{ name: 'refresh_market_snapshot', type: 'PROCEDURE' },
],
triggers: [
{
name: 'snapshot_before_insert',
table: 'market_snapshot',
timing: 'BEFORE',
event: 'INSERT',
order: 1,
},
],
events: ['purge_market_snapshot'],
});
const createRows = Object.freeze({
'SHOW CREATE TABLE `a_share_analysis_dev`.`market_snapshot`': {
'Create Table':
'CREATE TABLE `market_snapshot` (`id` bigint NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8mb4',
},
'SHOW CREATE VIEW `a_share_analysis_dev`.`market_summary`': {
'Create View':
'CREATE ALGORITHM=UNDEFINED DEFINER=`schema_user`@`192.168.1.10` SQL SECURITY DEFINER VIEW `market_summary` AS select `market_snapshot`.`id` AS `id` from `market_snapshot`',
},
'SHOW CREATE VIEW `a_share_analysis_dev`.`market_dashboard`': {
'Create View':
'CREATE ALGORITHM=UNDEFINED DEFINER=`schema_user`@`192.168.1.10` SQL SECURITY DEFINER VIEW `market_dashboard` AS select `market_summary`.`id` AS `id` from `market_summary`',
},
'SHOW CREATE FUNCTION `a_share_analysis_dev`.`calculate_breadth`': {
'Create Function':
"CREATE DEFINER=`schema_user`@`192.168.1.10` FUNCTION `calculate_breadth`() RETURNS int RETURN 1",
},
'SHOW CREATE PROCEDURE `a_share_analysis_dev`.`refresh_market_snapshot`': {
'Create Procedure':
'CREATE DEFINER=`schema_user`@`192.168.1.10` PROCEDURE `refresh_market_snapshot`()\nBEGIN\n UPDATE `market_snapshot` SET `id` = `id`;\nEND',
},
'SHOW CREATE TRIGGER `a_share_analysis_dev`.`snapshot_before_insert`': {
'SQL Original Statement':
'CREATE DEFINER=`schema_user`@`192.168.1.10` TRIGGER `snapshot_before_insert` BEFORE INSERT ON `market_snapshot` FOR EACH ROW SET NEW.id = NEW.id',
},
'SHOW CREATE EVENT `a_share_analysis_dev`.`purge_market_snapshot`': {
'Create Event':
'CREATE DEFINER=`schema_user`@`192.168.1.10` EVENT `purge_market_snapshot` ON SCHEDULE EVERY 1 DAY DO DELETE FROM `market_snapshot` WHERE `id` < 0',
},
});
function catalogRowsFor(sql) {
if (sql.includes('information_schema.TABLES')) {
return [
{ TABLE_NAME: 'market_snapshot', TABLE_TYPE: 'BASE TABLE' },
{ TABLE_NAME: 'market_dashboard', TABLE_TYPE: 'VIEW' },
{ TABLE_NAME: 'market_summary', TABLE_TYPE: 'VIEW' },
];
}
if (sql.includes('information_schema.VIEW_TABLE_USAGE')) {
return [
{
VIEW_NAME: 'market_dashboard',
REFERENCED_SCHEMA_NAME: 'a_share_analysis_dev',
REFERENCED_TABLE_NAME: 'market_summary',
},
{
VIEW_NAME: 'market_summary',
REFERENCED_SCHEMA_NAME: 'a_share_analysis_dev',
REFERENCED_TABLE_NAME: 'market_snapshot',
},
];
}
if (sql.includes('information_schema.ROUTINES')) {
return [
{ ROUTINE_NAME: 'calculate_breadth', ROUTINE_TYPE: 'FUNCTION' },
{ ROUTINE_NAME: 'refresh_market_snapshot', ROUTINE_TYPE: 'PROCEDURE' },
];
}
if (sql.includes('information_schema.TRIGGERS')) {
return [
{
TRIGGER_NAME: 'snapshot_before_insert',
EVENT_OBJECT_TABLE: 'market_snapshot',
ACTION_TIMING: 'BEFORE',
EVENT_MANIPULATION: 'INSERT',
ACTION_ORDER: 1,
},
];
}
if (sql.includes('information_schema.EVENTS')) {
return [{ EVENT_NAME: 'purge_market_snapshot' }];
}
return undefined;
}
function createExportConnection({ mutateCatalog, mutateDefinition, queryError } = {}) {
let catalogRound = 0;
let definitionRound = 0;
let tableCatalogCalls = 0;
let tableDefinitionCalls = 0;
const connection = {
endCalls: 0,
async query(options) {
if (queryError) throw queryError;
if (options.sql.includes('information_schema.SCHEMATA')) {
return [
[
{
SERVER_VERSION: '5.7.42-log',
DATABASE_CHARSET: 'utf8mb4',
DATABASE_COLLATION: 'utf8mb4_general_ci',
},
],
[],
];
}
const catalogRows = catalogRowsFor(options.sql);
if (catalogRows) {
if (options.sql.includes('information_schema.TABLES')) {
tableCatalogCalls += 1;
catalogRound = tableCatalogCalls > 1 ? 1 : 0;
}
const rows = mutateCatalog?.(options.sql, catalogRows, catalogRound) ?? catalogRows;
return [rows, []];
}
const row = createRows[options.sql];
assert.ok(row, `未识别的 SHOW CREATE:${options.sql}`);
if (options.sql.includes('SHOW CREATE TABLE')) {
tableDefinitionCalls += 1;
definitionRound = tableDefinitionCalls > 1 ? 1 : 0;
}
const changed = mutateDefinition?.(options.sql, row, definitionRound);
return [changed === undefined ? [row] : changed, []];
},
async end() {
connection.endCalls += 1;
},
};
return connection;
}
test('读取并校验固定开发库服务端元数据', async () => {
const connection = createExportConnection();
assert.deepEqual(
await readMysql57ServerMetadata(connection, 'a_share_analysis_dev', DEFAULT_EXPORT_LIMITS),
{
serverVersion: '5.7.42-log',
databaseCharset: 'utf8mb4',
databaseCollation: 'utf8mb4_general_ci',
},
);
});
test('SHOW CREATE 覆盖全部对象并按可恢复顺序渲染', async () => {
const connection = createExportConnection();
const definitions = await readDefinitions(connection, completeCatalog, DEFAULT_EXPORT_LIMITS);
const sql = renderSchemaSql(completeCatalog, definitions);
const order = [
'CREATE TABLE `market_snapshot`',
'FUNCTION `calculate_breadth`',
'PROCEDURE `refresh_market_snapshot`',
'VIEW `market_summary`',
'VIEW `market_dashboard`',
'TRIGGER `snapshot_before_insert`',
'EVENT `purge_market_snapshot`',
].map((fragment) => sql.indexOf(fragment));
assert.ok(order.every((index) => index >= 0));
assert.deepEqual(order, [...order].sort((left, right) => left - right));
assert.match(sql, /DELIMITER ;;\nCREATE PROCEDURE[\s\S]+END ;;\nDELIMITER ;/);
assert.doesNotMatch(sql, /DEFINER\s*=/i);
assert.doesNotMatch(sql, /AUTO_INCREMENT=12/);
assert.match(sql, /`id` bigint NOT NULL AUTO_INCREMENT/);
assert.match(sql, /UPDATE `market_snapshot`/);
});
test('定义规范化只移除账号绑定和当前自增值', () => {
const routine = normalizeCreateDefinition(
"CREATE DEFINER=`user`@`host` PROCEDURE `sample`() SELECT 'DEFINER=`kept`@`text` AUTO_INCREMENT=99'",
);
const table = normalizeCreateDefinition(
'CREATE TABLE `sample` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=99 DEFAULT CHARSET=utf8mb4;',
);
assert.equal(
routine,
"CREATE PROCEDURE `sample`() SELECT 'DEFINER=`kept`@`text` AUTO_INCREMENT=99'",
);
assert.equal(
table,
'CREATE TABLE `sample` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4',
);
});
test('完整导出双重读取一致并始终关闭连接', async () => {
const connection = createExportConnection();
let receivedOptions;
const result = await exportMysql57Schema({
config,
createConnection: async (options) => {
receivedOptions = options;
return connection;
},
now: () => 100,
});
assert.equal(receivedOptions.multipleStatements, false);
assert.equal(connection.endCalls, 1);
assert.deepEqual(result.objectCounts, {
tables: 1,
views: 2,
triggers: 1,
routines: 2,
events: 1,
});
assert.equal(result.durationMs, 0);
assert.equal(result.metadata.driverName, 'mysql2');
});
test('导出拒绝目录变化和定义变化', async () => {
const catalogConnection = createExportConnection({
mutateCatalog(sql, rows, round) {
if (round === 1 && sql.includes('information_schema.TABLES')) {
return [...rows, { TABLE_NAME: 'new_table', TABLE_TYPE: 'BASE TABLE' }];
}
return rows;
},
});
await assert.rejects(
() =>
exportMysql57Schema({
config,
createConnection: async () => catalogConnection,
now: () => 100,
}),
(error) => error.code === 'CATALOG_CHANGED_DURING_EXPORT',
);
assert.equal(catalogConnection.endCalls, 1);
const definitionConnection = createExportConnection({
mutateDefinition(sql, row, round) {
if (round === 1 && sql.includes('SHOW CREATE TABLE')) {
return [{ ...row, 'Create Table': `${row['Create Table']} COMMENT='changed'` }];
}
return undefined;
},
});
await assert.rejects(
() =>
exportMysql57Schema({
config,
createConnection: async () => definitionConnection,
now: () => 100,
}),
(error) => error.code === 'DEFINITION_CHANGED_DURING_EXPORT',
);
assert.equal(definitionConnection.endCalls, 1);
});
test('SHOW CREATE 拒绝空结果、错误形状和过大定义', async () => {
for (const [changed, expectedCode] of [
[[], 'SHOW_CREATE_EMPTY'],
[[{ unexpected: 'value' }], 'SHOW_CREATE_SHAPE_INVALID'],
]) {
const connection = createExportConnection({
mutateDefinition(sql) {
return sql.includes('SHOW CREATE TABLE') ? changed : undefined;
},
});
await assert.rejects(
() => readDefinitions(connection, completeCatalog, DEFAULT_EXPORT_LIMITS),
(error) => error.code === expectedCode,
);
}
const oversized = createExportConnection({
mutateDefinition(sql, row) {
return sql.includes('SHOW CREATE TABLE')
? [{ ...row, 'Create Table': 'x'.repeat(100) }]
: undefined;
},
});
await assert.rejects(
() =>
readDefinitions(oversized, completeCatalog, {
...DEFAULT_EXPORT_LIMITS,
maxDefinitionBytes: 32,
}),
(error) => error.code === 'DEFINITION_LIMIT_EXCEEDED',
);
});
test('查询失败和总超时只返回脱敏错误并关闭连接', async () => {
const failed = createExportConnection({
queryError: new Error('connect 127.0.0.1 password=not-for-logs'),
});
await assert.rejects(
() =>
exportMysql57Schema({
config,
createConnection: async () => failed,
now: () => 100,
}),
(error) =>
error.code === 'DATABASE_QUERY_FAILED' &&
!error.message.includes('127.0.0.1') &&
!error.message.includes('not-for-logs'),
);
assert.equal(failed.endCalls, 1);
const timedOut = createExportConnection();
let tick = 0;
await assert.rejects(
() =>
exportMysql57Schema({
config,
createConnection: async () => timedOut,
now: () => (tick++ === 0 ? 0 : 30_001),
}),
(error) => error.code === 'EXPORT_TIMEOUT',
);
assert.equal(timedOut.endCalls, 1);
});