Normalize database encoding during SQL import

This commit is contained in:
王鹏
2026-10-09 11:33:06 +08:00
parent 822c518534
commit 2f3d275021
3 changed files with 206 additions and 8 deletions

View File

@@ -346,6 +346,7 @@ class DeployerTests(unittest.TestCase):
[
"backend_build",
"frontend_install",
"sql_prepare",
"sql_import",
"backend_start",
"frontend_start",
@@ -353,7 +354,7 @@ class DeployerTests(unittest.TestCase):
],
)
self.assertIn("mvn", plan[0].command[0])
self.assertEqual(plan[2].cwd, root)
self.assertEqual(plan[3].cwd, root)
def test_builds_and_starts_user_and_admin_frontends_then_logs_all_addresses(self):
with TemporaryDirectory() as temp:
@@ -456,7 +457,7 @@ class DeployerTests(unittest.TestCase):
kinds = [step.kind for step in plan]
self.assertEqual(
kinds,
["backend_build_skip", "sql_import", "backend_start", "deployment_summary"],
["backend_build_skip", "sql_prepare", "sql_import", "backend_start", "deployment_summary"],
)
self.assertFalse(any(kind.startswith("frontend_") for kind in kinds))
self.assertEqual(plan[-1].description.count("http://"), 1)
@@ -543,7 +544,15 @@ class DeployerTests(unittest.TestCase):
exit_code = run_steps(plan, lambda value: None, process_runner=fake_runner)
self.assertEqual(exit_code, 0)
self.assertIn((["mysql", "-u", "root", "-psecret", "demo"], root, sql_file, False), calls)
self.assertIn(
(
["mysql", "--default-character-set=utf8mb4", "-u", "root", "-psecret", "demo"],
root,
sql_file,
False,
),
calls,
)
self.assertIn((["java", "-Dserver.port=9090", "-jar", str(app_jar)], root / "deploy", None, True), calls)
def test_plan_uses_configured_maven_path_for_backend_build(self):
@@ -645,7 +654,12 @@ class DeployerTests(unittest.TestCase):
plan = build_deployment_plan(scan, config)
self.assertEqual(plan[0].command, [str(mysql), "-u", "root", "demo"])
self.assertEqual(plan[0].kind, "sql_prepare")
self.assertIn("CREATE DATABASE IF NOT EXISTS `demo`", plan[0].command[-1])
self.assertEqual(
plan[1].command,
[str(mysql), "--default-character-set=utf8mb4", "-u", "root", "demo"],
)
def test_sql_dump_with_database_selection_does_not_require_existing_database(self):
with TemporaryDirectory() as temp:
@@ -667,7 +681,86 @@ class DeployerTests(unittest.TestCase):
plan = build_deployment_plan(scan, config)
self.assertEqual(plan[0].command, ["mysql", "-u", "root"])
self.assertEqual(plan[0].kind, "sql_prepare")
self.assertEqual(
plan[1].command,
["mysql", "--default-character-set=utf8mb4", "-u", "root"],
)
def test_database_preparation_quotes_name_and_sets_utf8mb4(self):
with TemporaryDirectory() as temp:
root = Path(temp)
sql_file = root / "schema.sql"
sql_file.write_text("CREATE TABLE demo(id INT);", encoding="utf-8")
scan = ScanResult(root=root, backend=None, frontend=None, sql_files=[sql_file])
plan = build_deployment_plan(
scan,
DeploymentConfig(mysql_database="医院`资源", tool_search_roots=[root / "missing"]),
)
prepare = plan[0]
self.assertEqual(prepare.kind, "sql_prepare")
self.assertIn("`医院``资源`", prepare.command[-1])
self.assertIn("CHARACTER SET utf8mb4", prepare.command[-1])
self.assertIn("ALTER DATABASE", prepare.command[-1])
def test_sql_input_converts_gbk_to_utf8_without_losing_chinese(self):
with TemporaryDirectory() as temp:
sql_file = Path(temp) / "legacy.sql"
sql = "INSERT INTO bingli(bingshi_text) VALUES ('病史单1');\r\n"
sql_file.write_bytes(sql.encode("gb18030"))
messages = []
stream = deployer_module._open_utf8_sql_input(sql_file, messages.append)
try:
converted = stream.read()
finally:
stream.close()
self.assertEqual(converted.decode("utf-8"), "SET NAMES utf8mb4;\n" + sql)
self.assertTrue(any("GBK/GB18030" in message for message in messages))
def test_sql_input_removes_utf8_bom_and_preserves_text(self):
with TemporaryDirectory() as temp:
sql_file = Path(temp) / "utf8.sql"
sql_file.write_bytes("SET NAMES utf8mb4;\n插入内容".encode("utf-8-sig"))
stream = deployer_module._open_utf8_sql_input(sql_file, lambda _: None)
try:
converted = stream.read()
finally:
stream.close()
self.assertFalse(converted.startswith(b"\xef\xbb\xbf"))
self.assertEqual(
converted.decode("utf-8"),
"SET NAMES utf8mb4;\nSET NAMES utf8mb4;\n插入内容",
)
def test_sql_input_normalizes_table_character_set_and_collation(self):
with TemporaryDirectory() as temp:
sql_file = Path(temp) / "latin1.sql"
sql_file.write_text(
"/*!40101 SET NAMES gbk */;\n"
"CREATE TABLE bingli (bingshi_text TEXT CHARACTER SET latin1) "
"DEFAULT CHARSET=utf8 COLLATE=latin1_swedish_ci;\n",
encoding="utf-8",
)
messages = []
stream = deployer_module._open_utf8_sql_input(sql_file, messages.append)
try:
converted = stream.read().decode("utf-8")
finally:
stream.close()
self.assertNotIn("latin1", converted.lower())
self.assertNotIn("SET NAMES gbk", converted)
self.assertIn("CHARACTER SET utf8mb4", converted)
self.assertIn("DEFAULT CHARSET=utf8mb4", converted)
self.assertIn("COLLATE=utf8mb4_unicode_ci", converted)
self.assertTrue(any("4 处" in message for message in messages))
def test_plan_skips_backend_build_when_runnable_jar_already_exists(self):
with TemporaryDirectory() as temp: