Normalize database encoding during SQL import
This commit is contained in:
@@ -68,7 +68,7 @@ python main.py
|
|||||||
|
|
||||||
1. 使用 `mvn clean package -DskipTests` 或 Gradle `bootJar`/`war` 构建后端。
|
1. 使用 `mvn clean package -DskipTests` 或 Gradle `bootJar`/`war` 构建后端。
|
||||||
2. 为尚未安装依赖的前端执行 `npm install`。
|
2. 为尚未安装依赖的前端执行 `npm install`。
|
||||||
3. 使用 MySQL 命令行依次导入 SQL 文件。
|
3. 自动创建填写的 MySQL 数据库并设置为 `utf8mb4`,识别 UTF-8、UTF-16、GBK/GB18030 SQL 文件后依次导入。
|
||||||
4. Spring Boot 使用 `java -jar` 启动;SSM 将 WAR 发布到 Tomcat 9;随后等待 8080 端口就绪。
|
4. Spring Boot 使用 `java -jar` 启动;SSM 将 WAR 发布到 Tomcat 9;随后等待 8080 端口就绪。
|
||||||
5. 使用 npm 启动各前端开发服务器,并等待对应端口就绪。
|
5. 使用 npm 启动各前端开发服务器,并等待对应端口就绪。
|
||||||
6. 在运行日志中输出所有服务的访问地址。
|
6. 在运行日志中输出所有服务的访问地址。
|
||||||
|
|||||||
@@ -1,12 +1,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import codecs
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import socket
|
import socket
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
@@ -103,8 +105,38 @@ def build_deployment_plan(scan: ScanResult, config: DeploymentConfig) -> list[De
|
|||||||
)
|
)
|
||||||
|
|
||||||
if scan.sql_files and config.mysql_database:
|
if scan.sql_files and config.mysql_database:
|
||||||
|
mysql_command = _tool_command(config, "MySQL", "mysql")
|
||||||
|
database_identifier = _quote_mysql_identifier(config.mysql_database)
|
||||||
|
prepare_command = [
|
||||||
|
mysql_command,
|
||||||
|
"--default-character-set=utf8mb4",
|
||||||
|
"-u",
|
||||||
|
config.mysql_user,
|
||||||
|
]
|
||||||
|
if config.mysql_password:
|
||||||
|
prepare_command.append(f"-p{config.mysql_password}")
|
||||||
|
prepare_command.extend(
|
||||||
|
[
|
||||||
|
"-e",
|
||||||
|
(
|
||||||
|
f"CREATE DATABASE IF NOT EXISTS {database_identifier} "
|
||||||
|
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; "
|
||||||
|
f"ALTER DATABASE {database_identifier} "
|
||||||
|
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
steps.append(
|
||||||
|
DeploymentStep(
|
||||||
|
"准备数据库",
|
||||||
|
prepare_command,
|
||||||
|
scan.root,
|
||||||
|
f"创建数据库 {config.mysql_database} 并设置 utf8mb4 字符集",
|
||||||
|
kind="sql_prepare",
|
||||||
|
)
|
||||||
|
)
|
||||||
for sql_file in scan.sql_files:
|
for sql_file in scan.sql_files:
|
||||||
command = [_tool_command(config, "MySQL", "mysql"), "-u", config.mysql_user]
|
command = [mysql_command, "--default-character-set=utf8mb4", "-u", config.mysql_user]
|
||||||
if config.mysql_password:
|
if config.mysql_password:
|
||||||
command.append(f"-p{config.mysql_password}")
|
command.append(f"-p{config.mysql_password}")
|
||||||
if not _sql_file_selects_database(sql_file):
|
if not _sql_file_selects_database(sql_file):
|
||||||
@@ -286,6 +318,79 @@ def _sql_file_selects_database(sql_file: Path) -> bool:
|
|||||||
return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized
|
return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _quote_mysql_identifier(value: str) -> str:
|
||||||
|
if not value or "\x00" in value:
|
||||||
|
raise ValueError("数据库名不能为空或包含空字符")
|
||||||
|
if len(value) > 64:
|
||||||
|
raise ValueError("数据库名不能超过 64 个字符")
|
||||||
|
return "`" + value.replace("`", "``") + "`"
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_sql_encoding(path: Path) -> str:
|
||||||
|
with path.open("rb") as source:
|
||||||
|
prefix = source.read(4)
|
||||||
|
if prefix.startswith(codecs.BOM_UTF8):
|
||||||
|
return "utf-8-sig"
|
||||||
|
if prefix.startswith(codecs.BOM_UTF16_LE) or prefix.startswith(codecs.BOM_UTF16_BE):
|
||||||
|
return "utf-16"
|
||||||
|
|
||||||
|
for encoding in ("utf-8", "gb18030"):
|
||||||
|
decoder = codecs.getincrementaldecoder(encoding)(errors="strict")
|
||||||
|
try:
|
||||||
|
with path.open("rb") as source:
|
||||||
|
while chunk := source.read(1024 * 1024):
|
||||||
|
decoder.decode(chunk)
|
||||||
|
decoder.decode(b"", final=True)
|
||||||
|
return encoding
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
continue
|
||||||
|
raise UnicodeError(f"无法识别 SQL 文件编码,仅支持 UTF-8、UTF-16 和 GBK/GB18030: {path}")
|
||||||
|
|
||||||
|
|
||||||
|
SQL_CHARACTER_SET_PATTERN = re.compile(
|
||||||
|
r"(?i)(\b(?:DEFAULT\s+)?(?:CHARSET|CHARACTER\s+SET)\s*=?\s*)"
|
||||||
|
r"(?:utf8mb4|utf8|latin1|gbk|gb18030)\b"
|
||||||
|
)
|
||||||
|
SQL_COLLATION_PATTERN = re.compile(
|
||||||
|
r"(?i)(\bCOLLATE\s*=?\s*)(?:utf8mb4|utf8|latin1|gbk|gb18030)_[A-Za-z0-9_]+\b"
|
||||||
|
)
|
||||||
|
SQL_SET_NAMES_PATTERN = re.compile(
|
||||||
|
r"(?i)(\bSET\s+NAMES\s+)[`'\"]?(?:utf8mb4|utf8|latin1|gbk|gb18030)[`'\"]?"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_sql_character_sets(line: str) -> tuple[str, int]:
|
||||||
|
stripped = line.lstrip()
|
||||||
|
if stripped.startswith("--") or stripped.startswith("#"):
|
||||||
|
return line, 0
|
||||||
|
normalized, count_names = SQL_SET_NAMES_PATTERN.subn(r"\1utf8mb4", line)
|
||||||
|
normalized, count_charset = SQL_CHARACTER_SET_PATTERN.subn(r"\1utf8mb4", normalized)
|
||||||
|
normalized, count_collation = SQL_COLLATION_PATTERN.subn(r"\1utf8mb4_unicode_ci", normalized)
|
||||||
|
return normalized, count_names + count_charset + count_collation
|
||||||
|
|
||||||
|
|
||||||
|
def _open_utf8_sql_input(path: Path, on_output: OutputHandler):
|
||||||
|
encoding = _detect_sql_encoding(path)
|
||||||
|
temporary = tempfile.TemporaryFile(mode="w+b")
|
||||||
|
replacement_count = 0
|
||||||
|
try:
|
||||||
|
temporary.write(b"SET NAMES utf8mb4;\n")
|
||||||
|
with path.open("r", encoding=encoding, errors="strict", newline="") as source:
|
||||||
|
for line in source:
|
||||||
|
normalized, count = _normalize_sql_character_sets(line)
|
||||||
|
replacement_count += count
|
||||||
|
temporary.write(normalized.encode("utf-8"))
|
||||||
|
temporary.seek(0)
|
||||||
|
except Exception:
|
||||||
|
temporary.close()
|
||||||
|
raise
|
||||||
|
display_encoding = "GBK/GB18030" if encoding == "gb18030" else encoding.upper()
|
||||||
|
on_output(f"SQL 文件编码: {display_encoding},已按 UTF-8 交给 MySQL。")
|
||||||
|
if replacement_count:
|
||||||
|
on_output(f"已将 SQL 中 {replacement_count} 处字符集声明统一为 utf8mb4。")
|
||||||
|
return temporary
|
||||||
|
|
||||||
|
|
||||||
def resolve_process_command(command: list[str], path: str | None = None) -> list[str]:
|
def resolve_process_command(command: list[str], path: str | None = None) -> list[str]:
|
||||||
if not command:
|
if not command:
|
||||||
return command
|
return command
|
||||||
@@ -724,7 +829,7 @@ def _run_process(
|
|||||||
if background:
|
if background:
|
||||||
return _restart_backend(command, cwd, on_output)
|
return _restart_backend(command, cwd, on_output)
|
||||||
|
|
||||||
input_file = input_path.open("r", encoding="utf-8", errors="ignore") if input_path else None
|
input_file = _open_utf8_sql_input(input_path, on_output) if input_path else None
|
||||||
try:
|
try:
|
||||||
process = subprocess.Popen(
|
process = subprocess.Popen(
|
||||||
command,
|
command,
|
||||||
|
|||||||
@@ -346,6 +346,7 @@ class DeployerTests(unittest.TestCase):
|
|||||||
[
|
[
|
||||||
"backend_build",
|
"backend_build",
|
||||||
"frontend_install",
|
"frontend_install",
|
||||||
|
"sql_prepare",
|
||||||
"sql_import",
|
"sql_import",
|
||||||
"backend_start",
|
"backend_start",
|
||||||
"frontend_start",
|
"frontend_start",
|
||||||
@@ -353,7 +354,7 @@ class DeployerTests(unittest.TestCase):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
self.assertIn("mvn", plan[0].command[0])
|
self.assertIn("mvn", plan[0].command[0])
|
||||||
self.assertEqual(plan[2].cwd, root)
|
self.assertEqual(plan[3].cwd, root)
|
||||||
|
|
||||||
def test_builds_and_starts_user_and_admin_frontends_then_logs_all_addresses(self):
|
def test_builds_and_starts_user_and_admin_frontends_then_logs_all_addresses(self):
|
||||||
with TemporaryDirectory() as temp:
|
with TemporaryDirectory() as temp:
|
||||||
@@ -456,7 +457,7 @@ class DeployerTests(unittest.TestCase):
|
|||||||
kinds = [step.kind for step in plan]
|
kinds = [step.kind for step in plan]
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
kinds,
|
kinds,
|
||||||
["backend_build_skip", "sql_import", "backend_start", "deployment_summary"],
|
["backend_build_skip", "sql_prepare", "sql_import", "backend_start", "deployment_summary"],
|
||||||
)
|
)
|
||||||
self.assertFalse(any(kind.startswith("frontend_") for kind in kinds))
|
self.assertFalse(any(kind.startswith("frontend_") for kind in kinds))
|
||||||
self.assertEqual(plan[-1].description.count("http://"), 1)
|
self.assertEqual(plan[-1].description.count("http://"), 1)
|
||||||
@@ -543,7 +544,15 @@ class DeployerTests(unittest.TestCase):
|
|||||||
exit_code = run_steps(plan, lambda value: None, process_runner=fake_runner)
|
exit_code = run_steps(plan, lambda value: None, process_runner=fake_runner)
|
||||||
|
|
||||||
self.assertEqual(exit_code, 0)
|
self.assertEqual(exit_code, 0)
|
||||||
self.assertIn((["mysql", "-u", "root", "-psecret", "demo"], root, sql_file, False), calls)
|
self.assertIn(
|
||||||
|
(
|
||||||
|
["mysql", "--default-character-set=utf8mb4", "-u", "root", "-psecret", "demo"],
|
||||||
|
root,
|
||||||
|
sql_file,
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
calls,
|
||||||
|
)
|
||||||
self.assertIn((["java", "-Dserver.port=9090", "-jar", str(app_jar)], root / "deploy", None, True), calls)
|
self.assertIn((["java", "-Dserver.port=9090", "-jar", str(app_jar)], root / "deploy", None, True), calls)
|
||||||
|
|
||||||
def test_plan_uses_configured_maven_path_for_backend_build(self):
|
def test_plan_uses_configured_maven_path_for_backend_build(self):
|
||||||
@@ -645,7 +654,12 @@ class DeployerTests(unittest.TestCase):
|
|||||||
|
|
||||||
plan = build_deployment_plan(scan, config)
|
plan = build_deployment_plan(scan, config)
|
||||||
|
|
||||||
self.assertEqual(plan[0].command, [str(mysql), "-u", "root", "demo"])
|
self.assertEqual(plan[0].kind, "sql_prepare")
|
||||||
|
self.assertIn("CREATE DATABASE IF NOT EXISTS `demo`", plan[0].command[-1])
|
||||||
|
self.assertEqual(
|
||||||
|
plan[1].command,
|
||||||
|
[str(mysql), "--default-character-set=utf8mb4", "-u", "root", "demo"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_sql_dump_with_database_selection_does_not_require_existing_database(self):
|
def test_sql_dump_with_database_selection_does_not_require_existing_database(self):
|
||||||
with TemporaryDirectory() as temp:
|
with TemporaryDirectory() as temp:
|
||||||
@@ -667,7 +681,86 @@ class DeployerTests(unittest.TestCase):
|
|||||||
|
|
||||||
plan = build_deployment_plan(scan, config)
|
plan = build_deployment_plan(scan, config)
|
||||||
|
|
||||||
self.assertEqual(plan[0].command, ["mysql", "-u", "root"])
|
self.assertEqual(plan[0].kind, "sql_prepare")
|
||||||
|
self.assertEqual(
|
||||||
|
plan[1].command,
|
||||||
|
["mysql", "--default-character-set=utf8mb4", "-u", "root"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_database_preparation_quotes_name_and_sets_utf8mb4(self):
|
||||||
|
with TemporaryDirectory() as temp:
|
||||||
|
root = Path(temp)
|
||||||
|
sql_file = root / "schema.sql"
|
||||||
|
sql_file.write_text("CREATE TABLE demo(id INT);", encoding="utf-8")
|
||||||
|
scan = ScanResult(root=root, backend=None, frontend=None, sql_files=[sql_file])
|
||||||
|
|
||||||
|
plan = build_deployment_plan(
|
||||||
|
scan,
|
||||||
|
DeploymentConfig(mysql_database="医院`资源", tool_search_roots=[root / "missing"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
prepare = plan[0]
|
||||||
|
self.assertEqual(prepare.kind, "sql_prepare")
|
||||||
|
self.assertIn("`医院``资源`", prepare.command[-1])
|
||||||
|
self.assertIn("CHARACTER SET utf8mb4", prepare.command[-1])
|
||||||
|
self.assertIn("ALTER DATABASE", prepare.command[-1])
|
||||||
|
|
||||||
|
def test_sql_input_converts_gbk_to_utf8_without_losing_chinese(self):
|
||||||
|
with TemporaryDirectory() as temp:
|
||||||
|
sql_file = Path(temp) / "legacy.sql"
|
||||||
|
sql = "INSERT INTO bingli(bingshi_text) VALUES ('病史单1');\r\n"
|
||||||
|
sql_file.write_bytes(sql.encode("gb18030"))
|
||||||
|
messages = []
|
||||||
|
|
||||||
|
stream = deployer_module._open_utf8_sql_input(sql_file, messages.append)
|
||||||
|
try:
|
||||||
|
converted = stream.read()
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
self.assertEqual(converted.decode("utf-8"), "SET NAMES utf8mb4;\n" + sql)
|
||||||
|
self.assertTrue(any("GBK/GB18030" in message for message in messages))
|
||||||
|
|
||||||
|
def test_sql_input_removes_utf8_bom_and_preserves_text(self):
|
||||||
|
with TemporaryDirectory() as temp:
|
||||||
|
sql_file = Path(temp) / "utf8.sql"
|
||||||
|
sql_file.write_bytes("SET NAMES utf8mb4;\n插入内容".encode("utf-8-sig"))
|
||||||
|
|
||||||
|
stream = deployer_module._open_utf8_sql_input(sql_file, lambda _: None)
|
||||||
|
try:
|
||||||
|
converted = stream.read()
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
self.assertFalse(converted.startswith(b"\xef\xbb\xbf"))
|
||||||
|
self.assertEqual(
|
||||||
|
converted.decode("utf-8"),
|
||||||
|
"SET NAMES utf8mb4;\nSET NAMES utf8mb4;\n插入内容",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_sql_input_normalizes_table_character_set_and_collation(self):
|
||||||
|
with TemporaryDirectory() as temp:
|
||||||
|
sql_file = Path(temp) / "latin1.sql"
|
||||||
|
sql_file.write_text(
|
||||||
|
"/*!40101 SET NAMES gbk */;\n"
|
||||||
|
"CREATE TABLE bingli (bingshi_text TEXT CHARACTER SET latin1) "
|
||||||
|
"DEFAULT CHARSET=utf8 COLLATE=latin1_swedish_ci;\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
messages = []
|
||||||
|
|
||||||
|
stream = deployer_module._open_utf8_sql_input(sql_file, messages.append)
|
||||||
|
try:
|
||||||
|
converted = stream.read().decode("utf-8")
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
self.assertNotIn("latin1", converted.lower())
|
||||||
|
self.assertNotIn("SET NAMES gbk", converted)
|
||||||
|
self.assertIn("CHARACTER SET utf8mb4", converted)
|
||||||
|
self.assertIn("DEFAULT CHARSET=utf8mb4", converted)
|
||||||
|
self.assertIn("COLLATE=utf8mb4_unicode_ci", converted)
|
||||||
|
self.assertTrue(any("4 处" in message for message in messages))
|
||||||
|
|
||||||
def test_plan_skips_backend_build_when_runnable_jar_already_exists(self):
|
def test_plan_skips_backend_build_when_runnable_jar_already_exists(self):
|
||||||
with TemporaryDirectory() as temp:
|
with TemporaryDirectory() as temp:
|
||||||
|
|||||||
Reference in New Issue
Block a user