Normalize database encoding during SQL import
This commit is contained in:
@@ -1,12 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import codecs
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
@@ -103,8 +105,38 @@ def build_deployment_plan(scan: ScanResult, config: DeploymentConfig) -> list[De
|
||||
)
|
||||
|
||||
if scan.sql_files and config.mysql_database:
|
||||
mysql_command = _tool_command(config, "MySQL", "mysql")
|
||||
database_identifier = _quote_mysql_identifier(config.mysql_database)
|
||||
prepare_command = [
|
||||
mysql_command,
|
||||
"--default-character-set=utf8mb4",
|
||||
"-u",
|
||||
config.mysql_user,
|
||||
]
|
||||
if config.mysql_password:
|
||||
prepare_command.append(f"-p{config.mysql_password}")
|
||||
prepare_command.extend(
|
||||
[
|
||||
"-e",
|
||||
(
|
||||
f"CREATE DATABASE IF NOT EXISTS {database_identifier} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; "
|
||||
f"ALTER DATABASE {database_identifier} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
|
||||
),
|
||||
]
|
||||
)
|
||||
steps.append(
|
||||
DeploymentStep(
|
||||
"准备数据库",
|
||||
prepare_command,
|
||||
scan.root,
|
||||
f"创建数据库 {config.mysql_database} 并设置 utf8mb4 字符集",
|
||||
kind="sql_prepare",
|
||||
)
|
||||
)
|
||||
for sql_file in scan.sql_files:
|
||||
command = [_tool_command(config, "MySQL", "mysql"), "-u", config.mysql_user]
|
||||
command = [mysql_command, "--default-character-set=utf8mb4", "-u", config.mysql_user]
|
||||
if config.mysql_password:
|
||||
command.append(f"-p{config.mysql_password}")
|
||||
if not _sql_file_selects_database(sql_file):
|
||||
@@ -286,6 +318,79 @@ def _sql_file_selects_database(sql_file: Path) -> bool:
|
||||
return "create database" in normalized and "use `" in normalized or "create database" in normalized and "use " in normalized
|
||||
|
||||
|
||||
def _quote_mysql_identifier(value: str) -> str:
|
||||
if not value or "\x00" in value:
|
||||
raise ValueError("数据库名不能为空或包含空字符")
|
||||
if len(value) > 64:
|
||||
raise ValueError("数据库名不能超过 64 个字符")
|
||||
return "`" + value.replace("`", "``") + "`"
|
||||
|
||||
|
||||
def _detect_sql_encoding(path: Path) -> str:
|
||||
with path.open("rb") as source:
|
||||
prefix = source.read(4)
|
||||
if prefix.startswith(codecs.BOM_UTF8):
|
||||
return "utf-8-sig"
|
||||
if prefix.startswith(codecs.BOM_UTF16_LE) or prefix.startswith(codecs.BOM_UTF16_BE):
|
||||
return "utf-16"
|
||||
|
||||
for encoding in ("utf-8", "gb18030"):
|
||||
decoder = codecs.getincrementaldecoder(encoding)(errors="strict")
|
||||
try:
|
||||
with path.open("rb") as source:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
decoder.decode(chunk)
|
||||
decoder.decode(b"", final=True)
|
||||
return encoding
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
raise UnicodeError(f"无法识别 SQL 文件编码,仅支持 UTF-8、UTF-16 和 GBK/GB18030: {path}")
|
||||
|
||||
|
||||
SQL_CHARACTER_SET_PATTERN = re.compile(
|
||||
r"(?i)(\b(?:DEFAULT\s+)?(?:CHARSET|CHARACTER\s+SET)\s*=?\s*)"
|
||||
r"(?:utf8mb4|utf8|latin1|gbk|gb18030)\b"
|
||||
)
|
||||
SQL_COLLATION_PATTERN = re.compile(
|
||||
r"(?i)(\bCOLLATE\s*=?\s*)(?:utf8mb4|utf8|latin1|gbk|gb18030)_[A-Za-z0-9_]+\b"
|
||||
)
|
||||
SQL_SET_NAMES_PATTERN = re.compile(
|
||||
r"(?i)(\bSET\s+NAMES\s+)[`'\"]?(?:utf8mb4|utf8|latin1|gbk|gb18030)[`'\"]?"
|
||||
)
|
||||
|
||||
|
||||
def _normalize_sql_character_sets(line: str) -> tuple[str, int]:
|
||||
stripped = line.lstrip()
|
||||
if stripped.startswith("--") or stripped.startswith("#"):
|
||||
return line, 0
|
||||
normalized, count_names = SQL_SET_NAMES_PATTERN.subn(r"\1utf8mb4", line)
|
||||
normalized, count_charset = SQL_CHARACTER_SET_PATTERN.subn(r"\1utf8mb4", normalized)
|
||||
normalized, count_collation = SQL_COLLATION_PATTERN.subn(r"\1utf8mb4_unicode_ci", normalized)
|
||||
return normalized, count_names + count_charset + count_collation
|
||||
|
||||
|
||||
def _open_utf8_sql_input(path: Path, on_output: OutputHandler):
|
||||
encoding = _detect_sql_encoding(path)
|
||||
temporary = tempfile.TemporaryFile(mode="w+b")
|
||||
replacement_count = 0
|
||||
try:
|
||||
temporary.write(b"SET NAMES utf8mb4;\n")
|
||||
with path.open("r", encoding=encoding, errors="strict", newline="") as source:
|
||||
for line in source:
|
||||
normalized, count = _normalize_sql_character_sets(line)
|
||||
replacement_count += count
|
||||
temporary.write(normalized.encode("utf-8"))
|
||||
temporary.seek(0)
|
||||
except Exception:
|
||||
temporary.close()
|
||||
raise
|
||||
display_encoding = "GBK/GB18030" if encoding == "gb18030" else encoding.upper()
|
||||
on_output(f"SQL 文件编码: {display_encoding},已按 UTF-8 交给 MySQL。")
|
||||
if replacement_count:
|
||||
on_output(f"已将 SQL 中 {replacement_count} 处字符集声明统一为 utf8mb4。")
|
||||
return temporary
|
||||
|
||||
|
||||
def resolve_process_command(command: list[str], path: str | None = None) -> list[str]:
|
||||
if not command:
|
||||
return command
|
||||
@@ -724,7 +829,7 @@ def _run_process(
|
||||
if background:
|
||||
return _restart_backend(command, cwd, on_output)
|
||||
|
||||
input_file = input_path.open("r", encoding="utf-8", errors="ignore") if input_path else None
|
||||
input_file = _open_utf8_sql_input(input_path, on_output) if input_path else None
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
|
||||
Reference in New Issue
Block a user