Normalize database encoding during SQL import

This commit is contained in:
王鹏
2026-10-09 11:33:06 +08:00
parent 822c518534
commit 2f3d275021
3 changed files with 206 additions and 8 deletions

View File

@@ -68,7 +68,7 @@ python main.py
1. 使用 `mvn clean package -DskipTests` 或 Gradle `bootJar`/`war` 构建后端。 1. 使用 `mvn clean package -DskipTests` 或 Gradle `bootJar`/`war` 构建后端。
2. 为尚未安装依赖的前端执行 `npm install`。 2. 为尚未安装依赖的前端执行 `npm install`。
3. 使用 MySQL 命令行依次导入 SQL 文件。 3. 自动创建填写的 MySQL 数据库并设置为 `utf8mb4`,识别 UTF-8、UTF-16、GBK/GB18030 SQL 文件后依次导入。
4. Spring Boot 使用 `java -jar` 启动;SSM 将 WAR 发布到 Tomcat 9;随后等待 8080 端口就绪。 4. Spring Boot 使用 `java -jar` 启动;SSM 将 WAR 发布到 Tomcat 9;随后等待 8080 端口就绪。
5. 使用 npm 启动各前端开发服务器,并等待对应端口就绪。 5. 使用 npm 启动各前端开发服务器,并等待对应端口就绪。
6. 在运行日志中输出所有服务的访问地址。 6. 在运行日志中输出所有服务的访问地址。

View File

@@ -1,12 +1,14 @@
from __future__ import annotations from __future__ import annotations
import json import json
import codecs
import os import os
import re import re
import socket import socket
import shutil import shutil
import subprocess import subprocess
import sys import sys
import tempfile
import time import time
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime from datetime import datetime
@@ -103,8 +105,38 @@ def build_deployment_plan(scan: ScanResult, config: DeploymentConfig) -> list[De
) )
if scan.sql_files and config.mysql_database: if scan.sql_files and config.mysql_database:
mysql_command = _tool_command(config, "MySQL", "mysql")
database_identifier = _quote_mysql_identifier(config.mysql_database)
prepare_command = [
mysql_command,
"--default-character-set=utf8mb4",
"-u",
config.mysql_user,
]
if config.mysql_password:
prepare_command.append(f"-p{config.mysql_password}")
prepare_command.extend(
[
"-e",
(
f"CREATE DATABASE IF NOT EXISTS {database_identifier} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; "
f"ALTER DATABASE {database_identifier} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
),
]
)
steps.append(
DeploymentStep(
"准备数据库",
prepare_command,
scan.root,
f"创建数据库 {config.mysql_database} 并设置 utf8mb4 字符集",
kind="sql_prepare",
)
)
for sql_file in scan.sql_files: for sql_file in scan.sql_files:
command = [_tool_command(config, "MySQL", "mysql"), "-u", config.mysql_user] command = [mysql_command, "--default-character-set=utf8mb4", "-u", config.mysql_user]
if config.mysql_password: if config.mysql_password:
command.append(f"-p{config.mysql_password}") command.append(f"-p{config.mysql_password}")
if not _sql_file_selects_database(sql_file): if not _sql_file_selects_database(sql_file):
@@ -286,6 +318,79 @@ def _sql_file_selects_database(sql_file: Path) -> bool:
return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized
def _quote_mysql_identifier(value: str) -> str:
if not value or "\x00" in value:
raise ValueError("数据库名不能为空或包含空字符")
if len(value) > 64:
raise ValueError("数据库名不能超过 64 个字符")
return "`" + value.replace("`", "``") + "`"
def _detect_sql_encoding(path: Path) -> str:
with path.open("rb") as source:
prefix = source.read(4)
if prefix.startswith(codecs.BOM_UTF8):
return "utf-8-sig"
if prefix.startswith(codecs.BOM_UTF16_LE) or prefix.startswith(codecs.BOM_UTF16_BE):
return "utf-16"
for encoding in ("utf-8", "gb18030"):
decoder = codecs.getincrementaldecoder(encoding)(errors="strict")
try:
with path.open("rb") as source:
while chunk := source.read(1024 * 1024):
decoder.decode(chunk)
decoder.decode(b"", final=True)
return encoding
except UnicodeDecodeError:
continue
raise UnicodeError(f"无法识别 SQL 文件编码,仅支持 UTF-8、UTF-16 和 GBK/GB18030: {path}")
SQL_CHARACTER_SET_PATTERN = re.compile(
r"(?i)(\b(?:DEFAULT\s+)?(?:CHARSET|CHARACTER\s+SET)\s*=?\s*)"
r"(?:utf8mb4|utf8|latin1|gbk|gb18030)\b"
)
SQL_COLLATION_PATTERN = re.compile(
r"(?i)(\bCOLLATE\s*=?\s*)(?:utf8mb4|utf8|latin1|gbk|gb18030)_[A-Za-z0-9_]+\b"
)
SQL_SET_NAMES_PATTERN = re.compile(
r"(?i)(\bSET\s+NAMES\s+)[`'\"]?(?:utf8mb4|utf8|latin1|gbk|gb18030)[`'\"]?"
)
def _normalize_sql_character_sets(line: str) -> tuple[str, int]:
stripped = line.lstrip()
if stripped.startswith("--") or stripped.startswith("#"):
return line, 0
normalized, count_names = SQL_SET_NAMES_PATTERN.subn(r"\1utf8mb4", line)
normalized, count_charset = SQL_CHARACTER_SET_PATTERN.subn(r"\1utf8mb4", normalized)
normalized, count_collation = SQL_COLLATION_PATTERN.subn(r"\1utf8mb4_unicode_ci", normalized)
return normalized, count_names + count_charset + count_collation
def _open_utf8_sql_input(path: Path, on_output: OutputHandler):
encoding = _detect_sql_encoding(path)
temporary = tempfile.TemporaryFile(mode="w+b")
replacement_count = 0
try:
temporary.write(b"SET NAMES utf8mb4;\n")
with path.open("r", encoding=encoding, errors="strict", newline="") as source:
for line in source:
normalized, count = _normalize_sql_character_sets(line)
replacement_count += count
temporary.write(normalized.encode("utf-8"))
temporary.seek(0)
except Exception:
temporary.close()
raise
display_encoding = "GBK/GB18030" if encoding == "gb18030" else encoding.upper()
on_output(f"SQL 文件编码: {display_encoding},已按 UTF-8 交给 MySQL。")
if replacement_count:
on_output(f"已将 SQL 中 {replacement_count} 处字符集声明统一为 utf8mb4。")
return temporary
def resolve_process_command(command: list[str], path: str | None = None) -> list[str]: def resolve_process_command(command: list[str], path: str | None = None) -> list[str]:
if not command: if not command:
return command return command
@@ -724,7 +829,7 @@ def _run_process(
if background: if background:
return _restart_backend(command, cwd, on_output) return _restart_backend(command, cwd, on_output)
input_file = input_path.open("r", encoding="utf-8", errors="ignore") if input_path else None input_file = _open_utf8_sql_input(input_path, on_output) if input_path else None
try: try:
process = subprocess.Popen( process = subprocess.Popen(
command, command,

View File

@@ -346,6 +346,7 @@ class DeployerTests(unittest.TestCase):
[ [
"backend_build", "backend_build",
"frontend_install", "frontend_install",
"sql_prepare",
"sql_import", "sql_import",
"backend_start", "backend_start",
"frontend_start", "frontend_start",
@@ -353,7 +354,7 @@ class DeployerTests(unittest.TestCase):
], ],
) )
self.assertIn("mvn", plan[0].command[0]) self.assertIn("mvn", plan[0].command[0])
self.assertEqual(plan[2].cwd, root) self.assertEqual(plan[3].cwd, root)
def test_builds_and_starts_user_and_admin_frontends_then_logs_all_addresses(self): def test_builds_and_starts_user_and_admin_frontends_then_logs_all_addresses(self):
with TemporaryDirectory() as temp: with TemporaryDirectory() as temp:
@@ -456,7 +457,7 @@ class DeployerTests(unittest.TestCase):
kinds = [step.kind for step in plan] kinds = [step.kind for step in plan]
self.assertEqual( self.assertEqual(
kinds, kinds,
["backend_build_skip", "sql_import", "backend_start", "deployment_summary"], ["backend_build_skip", "sql_prepare", "sql_import", "backend_start", "deployment_summary"],
) )
self.assertFalse(any(kind.startswith("frontend_") for kind in kinds)) self.assertFalse(any(kind.startswith("frontend_") for kind in kinds))
self.assertEqual(plan[-1].description.count("http://"), 1) self.assertEqual(plan[-1].description.count("http://"), 1)
@@ -543,7 +544,15 @@ class DeployerTests(unittest.TestCase):
exit_code = run_steps(plan, lambda value: None, process_runner=fake_runner) exit_code = run_steps(plan, lambda value: None, process_runner=fake_runner)
self.assertEqual(exit_code, 0) self.assertEqual(exit_code, 0)
self.assertIn((["mysql", "-u", "root", "-psecret", "demo"], root, sql_file, False), calls) self.assertIn(
(
["mysql", "--default-character-set=utf8mb4", "-u", "root", "-psecret", "demo"],
root,
sql_file,
False,
),
calls,
)
self.assertIn((["java", "-Dserver.port=9090", "-jar", str(app_jar)], root / "deploy", None, True), calls) self.assertIn((["java", "-Dserver.port=9090", "-jar", str(app_jar)], root / "deploy", None, True), calls)
def test_plan_uses_configured_maven_path_for_backend_build(self): def test_plan_uses_configured_maven_path_for_backend_build(self):
@@ -645,7 +654,12 @@ class DeployerTests(unittest.TestCase):
plan = build_deployment_plan(scan, config) plan = build_deployment_plan(scan, config)
self.assertEqual(plan[0].command, [str(mysql), "-u", "root", "demo"]) self.assertEqual(plan[0].kind, "sql_prepare")
self.assertIn("CREATE DATABASE IF NOT EXISTS `demo`", plan[0].command[-1])
self.assertEqual(
plan[1].command,
[str(mysql), "--default-character-set=utf8mb4", "-u", "root", "demo"],
)
def test_sql_dump_with_database_selection_does_not_require_existing_database(self): def test_sql_dump_with_database_selection_does_not_require_existing_database(self):
with TemporaryDirectory() as temp: with TemporaryDirectory() as temp:
@@ -667,7 +681,86 @@ class DeployerTests(unittest.TestCase):
plan = build_deployment_plan(scan, config) plan = build_deployment_plan(scan, config)
self.assertEqual(plan[0].command, ["mysql", "-u", "root"]) self.assertEqual(plan[0].kind, "sql_prepare")
self.assertEqual(
plan[1].command,
["mysql", "--default-character-set=utf8mb4", "-u", "root"],
)
def test_database_preparation_quotes_name_and_sets_utf8mb4(self):
with TemporaryDirectory() as temp:
root = Path(temp)
sql_file = root / "schema.sql"
sql_file.write_text("CREATE TABLE demo(id INT);", encoding="utf-8")
scan = ScanResult(root=root, backend=None, frontend=None, sql_files=[sql_file])
plan = build_deployment_plan(
scan,
DeploymentConfig(mysql_database="医院`资源", tool_search_roots=[root / "missing"]),
)
prepare = plan[0]
self.assertEqual(prepare.kind, "sql_prepare")
self.assertIn("`医院``资源`", prepare.command[-1])
self.assertIn("CHARACTER SET utf8mb4", prepare.command[-1])
self.assertIn("ALTER DATABASE", prepare.command[-1])
def test_sql_input_converts_gbk_to_utf8_without_losing_chinese(self):
with TemporaryDirectory() as temp:
sql_file = Path(temp) / "legacy.sql"
sql = "INSERT INTO bingli(bingshi_text) VALUES ('病史单1');\r\n"
sql_file.write_bytes(sql.encode("gb18030"))
messages = []
stream = deployer_module._open_utf8_sql_input(sql_file, messages.append)
try:
converted = stream.read()
finally:
stream.close()
self.assertEqual(converted.decode("utf-8"), "SET NAMES utf8mb4;\n" + sql)
self.assertTrue(any("GBK/GB18030" in message for message in messages))
def test_sql_input_removes_utf8_bom_and_preserves_text(self):
with TemporaryDirectory() as temp:
sql_file = Path(temp) / "utf8.sql"
sql_file.write_bytes("SET NAMES utf8mb4;\n插入内容".encode("utf-8-sig"))
stream = deployer_module._open_utf8_sql_input(sql_file, lambda _: None)
try:
converted = stream.read()
finally:
stream.close()
self.assertFalse(converted.startswith(b"\xef\xbb\xbf"))
self.assertEqual(
converted.decode("utf-8"),
"SET NAMES utf8mb4;\nSET NAMES utf8mb4;\n插入内容",
)
def test_sql_input_normalizes_table_character_set_and_collation(self):
with TemporaryDirectory() as temp:
sql_file = Path(temp) / "latin1.sql"
sql_file.write_text(
"/*!40101 SET NAMES gbk */;\n"
"CREATE TABLE bingli (bingshi_text TEXT CHARACTER SET latin1) "
"DEFAULT CHARSET=utf8 COLLATE=latin1_swedish_ci;\n",
encoding="utf-8",
)
messages = []
stream = deployer_module._open_utf8_sql_input(sql_file, messages.append)
try:
converted = stream.read().decode("utf-8")
finally:
stream.close()
self.assertNotIn("latin1", converted.lower())
self.assertNotIn("SET NAMES gbk", converted)
self.assertIn("CHARACTER SET utf8mb4", converted)
self.assertIn("DEFAULT CHARSET=utf8mb4", converted)
self.assertIn("COLLATE=utf8mb4_unicode_ci", converted)
self.assertTrue(any("4 处" in message for message in messages))
def test_plan_skips_backend_build_when_runnable_jar_already_exists(self): def test_plan_skips_backend_build_when_runnable_jar_already_exists(self):
with TemporaryDirectory() as temp: with TemporaryDirectory() as temp: