Normalize database encoding during SQL import

This commit is contained in:
王鹏
2026-10-09 11:33:06 +08:00
parent 822c518534
commit 2f3d275021
3 changed files with 206 additions and 8 deletions

View File

@@ -1,12 +1,14 @@
from __future__ import annotations
import json
import codecs
import os
import re
import socket
import shutil
import subprocess
import sys
import tempfile
import time
from dataclasses import dataclass
from datetime import datetime
@@ -103,8 +105,38 @@ def build_deployment_plan(scan: ScanResult, config: DeploymentConfig) -> list[De
)
if scan.sql_files and config.mysql_database:
mysql_command = _tool_command(config, "MySQL", "mysql")
database_identifier = _quote_mysql_identifier(config.mysql_database)
prepare_command = [
mysql_command,
"--default-character-set=utf8mb4",
"-u",
config.mysql_user,
]
if config.mysql_password:
prepare_command.append(f"-p{config.mysql_password}")
prepare_command.extend(
[
"-e",
(
f"CREATE DATABASE IF NOT EXISTS {database_identifier} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; "
f"ALTER DATABASE {database_identifier} "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
),
]
)
steps.append(
DeploymentStep(
"准备数据库",
prepare_command,
scan.root,
f"创建数据库 {config.mysql_database} 并设置 utf8mb4 字符集",
kind="sql_prepare",
)
)
for sql_file in scan.sql_files:
command = [_tool_command(config, "MySQL", "mysql"), "-u", config.mysql_user]
command = [mysql_command, "--default-character-set=utf8mb4", "-u", config.mysql_user]
if config.mysql_password:
command.append(f"-p{config.mysql_password}")
if not _sql_file_selects_database(sql_file):
@@ -286,6 +318,79 @@ def _sql_file_selects_database(sql_file: Path) -> bool:
return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized
def _quote_mysql_identifier(value: str) -> str:
if not value or "\x00" in value:
raise ValueError("数据库名不能为空或包含空字符")
if len(value) > 64:
raise ValueError("数据库名不能超过 64 个字符")
return "`" + value.replace("`", "``") + "`"
def _detect_sql_encoding(path: Path) -> str:
with path.open("rb") as source:
prefix = source.read(4)
if prefix.startswith(codecs.BOM_UTF8):
return "utf-8-sig"
if prefix.startswith(codecs.BOM_UTF16_LE) or prefix.startswith(codecs.BOM_UTF16_BE):
return "utf-16"
for encoding in ("utf-8", "gb18030"):
decoder = codecs.getincrementaldecoder(encoding)(errors="strict")
try:
with path.open("rb") as source:
while chunk := source.read(1024 * 1024):
decoder.decode(chunk)
decoder.decode(b"", final=True)
return encoding
except UnicodeDecodeError:
continue
raise UnicodeError(f"无法识别 SQL 文件编码,仅支持 UTF-8、UTF-16 和 GBK/GB18030: {path}")
SQL_CHARACTER_SET_PATTERN = re.compile(
r"(?i)(\b(?:DEFAULT\s+)?(?:CHARSET|CHARACTER\s+SET)\s*=?\s*)"
r"(?:utf8mb4|utf8|latin1|gbk|gb18030)\b"
)
SQL_COLLATION_PATTERN = re.compile(
r"(?i)(\bCOLLATE\s*=?\s*)(?:utf8mb4|utf8|latin1|gbk|gb18030)_[A-Za-z0-9_]+\b"
)
SQL_SET_NAMES_PATTERN = re.compile(
r"(?i)(\bSET\s+NAMES\s+)[`'\"]?(?:utf8mb4|utf8|latin1|gbk|gb18030)[`'\"]?"
)
def _normalize_sql_character_sets(line: str) -> tuple[str, int]:
stripped = line.lstrip()
if stripped.startswith("--") or stripped.startswith("#"):
return line, 0
normalized, count_names = SQL_SET_NAMES_PATTERN.subn(r"\1utf8mb4", line)
normalized, count_charset = SQL_CHARACTER_SET_PATTERN.subn(r"\1utf8mb4", normalized)
normalized, count_collation = SQL_COLLATION_PATTERN.subn(r"\1utf8mb4_unicode_ci", normalized)
return normalized, count_names + count_charset + count_collation
def _open_utf8_sql_input(path: Path, on_output: OutputHandler):
encoding = _detect_sql_encoding(path)
temporary = tempfile.TemporaryFile(mode="w+b")
replacement_count = 0
try:
temporary.write(b"SET NAMES utf8mb4;\n")
with path.open("r", encoding=encoding, errors="strict", newline="") as source:
for line in source:
normalized, count = _normalize_sql_character_sets(line)
replacement_count += count
temporary.write(normalized.encode("utf-8"))
temporary.seek(0)
except Exception:
temporary.close()
raise
display_encoding = "GBK/GB18030" if encoding == "gb18030" else encoding.upper()
on_output(f"SQL 文件编码: {display_encoding},已按 UTF-8 交给 MySQL。")
if replacement_count:
on_output(f"已将 SQL 中 {replacement_count} 处字符集声明统一为 utf8mb4。")
return temporary
def resolve_process_command(command: list[str], path: str | None = None) -> list[str]:
if not command:
return command
@@ -724,7 +829,7 @@ def _run_process(
if background:
return _restart_backend(command, cwd, on_output)
input_file = input_path.open("r", encoding="utf-8", errors="ignore") if input_path else None
input_file = _open_utf8_sql_input(input_path, on_output) if input_path else None
try:
process = subprocess.Popen(
command,