diff --git a/README.md b/README.md index 076feb5..8b270d8 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ python main.py 1. 使用 `mvn clean package -DskipTests` 或 Gradle `bootJar`/`war` 构建后端。 2. 为尚未安装依赖的前端执行 `npm install`。 -3. 使用 MySQL 命令行依次导入 SQL 文件。 +3. 自动创建填写的 MySQL 数据库并设置为 `utf8mb4`,识别 UTF-8、UTF-16、GBK/GB18030 SQL 文件后依次导入。 4. Spring Boot 使用 `java -jar` 启动;SSM 将 WAR 发布到 Tomcat 9;随后等待 8080 端口就绪。 5. 使用 npm 启动各前端开发服务器,并等待对应端口就绪。 6. 在运行日志中输出所有服务的访问地址。 diff --git a/deploy_tool/deployer.py b/deploy_tool/deployer.py index 2ecede5..d86d1f8 100644 --- a/deploy_tool/deployer.py +++ b/deploy_tool/deployer.py @@ -1,12 +1,14 @@ from __future__ import annotations import json +import codecs import os import re import socket import shutil import subprocess import sys +import tempfile import time from dataclasses import dataclass from datetime import datetime @@ -103,8 +105,38 @@ def build_deployment_plan(scan: ScanResult, config: DeploymentConfig) -> list[De ) if scan.sql_files and config.mysql_database: + mysql_command = _tool_command(config, "MySQL", "mysql") + database_identifier = _quote_mysql_identifier(config.mysql_database) + prepare_command = [ + mysql_command, + "--default-character-set=utf8mb4", + "-u", + config.mysql_user, + ] + if config.mysql_password: + prepare_command.append(f"-p{config.mysql_password}") + prepare_command.extend( + [ + "-e", + ( + f"CREATE DATABASE IF NOT EXISTS {database_identifier} " + "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; " + f"ALTER DATABASE {database_identifier} " + "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" + ), + ] + ) + steps.append( + DeploymentStep( + "准备数据库", + prepare_command, + scan.root, + f"创建数据库 {config.mysql_database} 并设置 utf8mb4 字符集", + kind="sql_prepare", + ) + ) for sql_file in scan.sql_files: - command = [_tool_command(config, "MySQL", "mysql"), "-u", config.mysql_user] + command = [mysql_command, "--default-character-set=utf8mb4", "-u", config.mysql_user] if config.mysql_password: command.append(f"-p{config.mysql_password}") if not _sql_file_selects_database(sql_file): @@ -286,6 +318,79 @@ def _sql_file_selects_database(sql_file: Path) -> bool: return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized +def _quote_mysql_identifier(value: str) -> str: + if not value or "\x00" in value: + raise ValueError("数据库名不能为空或包含空字符") + if len(value) > 64: + raise ValueError("数据库名不能超过 64 个字符") + return "`" + value.replace("`", "``") + "`" + + +def _detect_sql_encoding(path: Path) -> str: + with path.open("rb") as source: + prefix = source.read(4) + if prefix.startswith(codecs.BOM_UTF8): + return "utf-8-sig" + if prefix.startswith(codecs.BOM_UTF16_LE) or prefix.startswith(codecs.BOM_UTF16_BE): + return "utf-16" + + for encoding in ("utf-8", "gb18030"): + decoder = codecs.getincrementaldecoder(encoding)(errors="strict") + try: + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + decoder.decode(chunk) + decoder.decode(b"", final=True) + return encoding + except UnicodeDecodeError: + continue + raise UnicodeError(f"无法识别 SQL 文件编码,仅支持 UTF-8、UTF-16 和 GBK/GB18030: {path}") + + +SQL_CHARACTER_SET_PATTERN = re.compile( + r"(?i)(\b(?:DEFAULT\s+)?(?:CHARSET|CHARACTER\s+SET)\s*=?\s*)" + r"(?:utf8mb4|utf8|latin1|gbk|gb18030)\b" +) +SQL_COLLATION_PATTERN = re.compile( + r"(?i)(\bCOLLATE\s*=?\s*)(?:utf8mb4|utf8|latin1|gbk|gb18030)_[A-Za-z0-9_]+\b" +) +SQL_SET_NAMES_PATTERN = re.compile( + r"(?i)(\bSET\s+NAMES\s+)[`'\"]?(?:utf8mb4|utf8|latin1|gbk|gb18030)[`'\"]?" +) + + +def _normalize_sql_character_sets(line: str) -> tuple[str, int]: + stripped = line.lstrip() + if stripped.startswith("--") or stripped.startswith("#"): + return line, 0 + normalized, count_names = SQL_SET_NAMES_PATTERN.subn(r"\1utf8mb4", line) + normalized, count_charset = SQL_CHARACTER_SET_PATTERN.subn(r"\1utf8mb4", normalized) + normalized, count_collation = SQL_COLLATION_PATTERN.subn(r"\1utf8mb4_unicode_ci", normalized) + return normalized, count_names + count_charset + count_collation + + +def _open_utf8_sql_input(path: Path, on_output: OutputHandler): + encoding = _detect_sql_encoding(path) + temporary = tempfile.TemporaryFile(mode="w+b") + replacement_count = 0 + try: + temporary.write(b"SET NAMES utf8mb4;\n") + with path.open("r", encoding=encoding, errors="strict", newline="") as source: + for line in source: + normalized, count = _normalize_sql_character_sets(line) + replacement_count += count + temporary.write(normalized.encode("utf-8")) + temporary.seek(0) + except Exception: + temporary.close() + raise + display_encoding = "GBK/GB18030" if encoding == "gb18030" else encoding.upper() + on_output(f"SQL 文件编码: {display_encoding},已按 UTF-8 交给 MySQL。") + if replacement_count: + on_output(f"已将 SQL 中 {replacement_count} 处字符集声明统一为 utf8mb4。") + return temporary + + def resolve_process_command(command: list[str], path: str | None = None) -> list[str]: if not command: return command @@ -724,7 +829,7 @@ def _run_process( if background: return _restart_backend(command, cwd, on_output) - input_file = input_path.open("r", encoding="utf-8", errors="ignore") if input_path else None + input_file = _open_utf8_sql_input(input_path, on_output) if input_path else None try: process = subprocess.Popen( command, diff --git a/tests/test_deployer.py b/tests/test_deployer.py index 42ce5e2..b6b230f 100644 --- a/tests/test_deployer.py +++ b/tests/test_deployer.py @@ -346,6 +346,7 @@ class DeployerTests(unittest.TestCase): [ "backend_build", "frontend_install", + "sql_prepare", "sql_import", "backend_start", "frontend_start", @@ -353,7 +354,7 @@ class DeployerTests(unittest.TestCase): ], ) self.assertIn("mvn", plan[0].command[0]) - self.assertEqual(plan[2].cwd, root) + self.assertEqual(plan[3].cwd, root) def test_builds_and_starts_user_and_admin_frontends_then_logs_all_addresses(self): with TemporaryDirectory() as temp: @@ -456,7 +457,7 @@ class DeployerTests(unittest.TestCase): kinds = [step.kind for step in plan] self.assertEqual( kinds, - ["backend_build_skip", "sql_import", "backend_start", "deployment_summary"], + ["backend_build_skip", "sql_prepare", "sql_import", "backend_start", "deployment_summary"], ) self.assertFalse(any(kind.startswith("frontend_") for kind in kinds)) self.assertEqual(plan[-1].description.count("http://"), 1) @@ -543,7 +544,15 @@ class DeployerTests(unittest.TestCase): exit_code = run_steps(plan, lambda value: None, process_runner=fake_runner) self.assertEqual(exit_code, 0) - self.assertIn((["mysql", "-u", "root", "-psecret", "demo"], root, sql_file, False), calls) + self.assertIn( + ( + ["mysql", "--default-character-set=utf8mb4", "-u", "root", "-psecret", "demo"], + root, + sql_file, + False, + ), + calls, + ) self.assertIn((["java", "-Dserver.port=9090", "-jar", str(app_jar)], root / "deploy", None, True), calls) def test_plan_uses_configured_maven_path_for_backend_build(self): @@ -645,7 +654,12 @@ class DeployerTests(unittest.TestCase): plan = build_deployment_plan(scan, config) - self.assertEqual(plan[0].command, [str(mysql), "-u", "root", "demo"]) + self.assertEqual(plan[0].kind, "sql_prepare") + self.assertIn("CREATE DATABASE IF NOT EXISTS `demo`", plan[0].command[-1]) + self.assertEqual( + plan[1].command, + [str(mysql), "--default-character-set=utf8mb4", "-u", "root", "demo"], + ) def test_sql_dump_with_database_selection_does_not_require_existing_database(self): with TemporaryDirectory() as temp: @@ -667,7 +681,86 @@ class DeployerTests(unittest.TestCase): plan = build_deployment_plan(scan, config) - self.assertEqual(plan[0].command, ["mysql", "-u", "root"]) + self.assertEqual(plan[0].kind, "sql_prepare") + self.assertEqual( + plan[1].command, + ["mysql", "--default-character-set=utf8mb4", "-u", "root"], + ) + + def test_database_preparation_quotes_name_and_sets_utf8mb4(self): + with TemporaryDirectory() as temp: + root = Path(temp) + sql_file = root / "schema.sql" + sql_file.write_text("CREATE TABLE demo(id INT);", encoding="utf-8") + scan = ScanResult(root=root, backend=None, frontend=None, sql_files=[sql_file]) + + plan = build_deployment_plan( + scan, + DeploymentConfig(mysql_database="医院`资源", tool_search_roots=[root / "missing"]), + ) + + prepare = plan[0] + self.assertEqual(prepare.kind, "sql_prepare") + self.assertIn("`医院``资源`", prepare.command[-1]) + self.assertIn("CHARACTER SET utf8mb4", prepare.command[-1]) + self.assertIn("ALTER DATABASE", prepare.command[-1]) + + def test_sql_input_converts_gbk_to_utf8_without_losing_chinese(self): + with TemporaryDirectory() as temp: + sql_file = Path(temp) / "legacy.sql" + sql = "INSERT INTO bingli(bingshi_text) VALUES ('病史单1');\r\n" + sql_file.write_bytes(sql.encode("gb18030")) + messages = [] + + stream = deployer_module._open_utf8_sql_input(sql_file, messages.append) + try: + converted = stream.read() + finally: + stream.close() + + self.assertEqual(converted.decode("utf-8"), "SET NAMES utf8mb4;\n" + sql) + self.assertTrue(any("GBK/GB18030" in message for message in messages)) + + def test_sql_input_removes_utf8_bom_and_preserves_text(self): + with TemporaryDirectory() as temp: + sql_file = Path(temp) / "utf8.sql" + sql_file.write_bytes("SET NAMES utf8mb4;\n插入内容".encode("utf-8-sig")) + + stream = deployer_module._open_utf8_sql_input(sql_file, lambda _: None) + try: + converted = stream.read() + finally: + stream.close() + + self.assertFalse(converted.startswith(b"\xef\xbb\xbf")) + self.assertEqual( + converted.decode("utf-8"), + "SET NAMES utf8mb4;\nSET NAMES utf8mb4;\n插入内容", + ) + + def test_sql_input_normalizes_table_character_set_and_collation(self): + with TemporaryDirectory() as temp: + sql_file = Path(temp) / "latin1.sql" + sql_file.write_text( + "/*!40101 SET NAMES gbk */;\n" + "CREATE TABLE bingli (bingshi_text TEXT CHARACTER SET latin1) " + "DEFAULT CHARSET=utf8 COLLATE=latin1_swedish_ci;\n", + encoding="utf-8", + ) + messages = [] + + stream = deployer_module._open_utf8_sql_input(sql_file, messages.append) + try: + converted = stream.read().decode("utf-8") + finally: + stream.close() + + self.assertNotIn("latin1", converted.lower()) + self.assertNotIn("SET NAMES gbk", converted) + self.assertIn("CHARACTER SET utf8mb4", converted) + self.assertIn("DEFAULT CHARSET=utf8mb4", converted) + self.assertIn("COLLATE=utf8mb4_unicode_ci", converted) + self.assertTrue(any("4 处" in message for message in messages)) def test_plan_skips_backend_build_when_runnable_jar_already_exists(self): with TemporaryDirectory() as temp: