702 lines
28 KiB
Python
702 lines
28 KiB
Python
|
|
"""Self-contained Python 3 deployment runner, executed on the Linux server.
|
|||
|
|
|
|||
|
|
Only paths in the supplied deployment configuration are mutated. Credentials never
|
|||
|
|
leave the SSH client. JSON line events form the desktop/runner protocol.
|
|||
|
|
"""
|
|||
|
|
import base64
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
from pathlib import Path, PurePosixPath
|
|||
|
|
import re
|
|||
|
|
import shutil
|
|||
|
|
import signal
|
|||
|
|
import subprocess
|
|||
|
|
import sys
|
|||
|
|
import tarfile
|
|||
|
|
import time
|
|||
|
|
import urllib.request
|
|||
|
|
import uuid
|
|||
|
|
|
|||
|
|
|
|||
|
|
def event(message, progress=None, **extra):
|
|||
|
|
try:
|
|||
|
|
print(json.dumps(dict(message=message, progress=progress, **extra), ensure_ascii=False), flush=True)
|
|||
|
|
except BrokenPipeError:
|
|||
|
|
# SSH may disappear mid-transaction. Do not let a logging failure stop recovery.
|
|||
|
|
sys.stdout = open(os.devnull, 'w')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate(c):
|
|||
|
|
keys = ('front', 'war', 'tomcat', 'startup', 'shutdown', 'temp', 'backups')
|
|||
|
|
for key in keys:
|
|||
|
|
value = c[key]
|
|||
|
|
p = PurePosixPath(value)
|
|||
|
|
if not p.is_absolute() or '..' in p.parts or '\x00' in value or '\n' in value or str(p) in ('/', '/home', '/usr', '/usr/local', '/tmp', '/var', '/etc', '/root'):
|
|||
|
|
raise ValueError('路径必须为明确的绝对路径,不能是系统根目录:' + key)
|
|||
|
|
# Reject symlink parents too: backups and renames must stay in the configured tree.
|
|||
|
|
if os.name == 'posix' and any(Path(x).is_symlink() for x in [p, *p.parents]):
|
|||
|
|
raise ValueError('部署路径不能包含软链接:' + value)
|
|||
|
|
if PurePosixPath(c['war']).name != 'ROOT.war':
|
|||
|
|
raise ValueError('后端目标文件名必须为 ROOT.war')
|
|||
|
|
roots = [PurePosixPath(c[k]) for k in ('front', 'temp', 'backups')] + [PurePosixPath(c['war']).parent]
|
|||
|
|
for i, a in enumerate(roots):
|
|||
|
|
for b in roots[i + 1:]:
|
|||
|
|
if a == b or a in b.parents or b in a.parents:
|
|||
|
|
raise ValueError('前端、webapps、临时目录和备份目录不能相互包含')
|
|||
|
|
front = PurePosixPath(c['front'])
|
|||
|
|
tc = PurePosixPath(c['tomcat'])
|
|||
|
|
if front == tc or front in tc.parents or tc in front.parents:
|
|||
|
|
raise ValueError('前端目录不能与 Tomcat 实例目录重叠')
|
|||
|
|
for key in ('stop_timeout', 'start_timeout'):
|
|||
|
|
if not 1 <= int(c[key]) <= 3600:
|
|||
|
|
raise ValueError('超时必须介于 1 到 3600 秒')
|
|||
|
|
if c.get('service') and not re.fullmatch(r'[A-Za-z0-9_@.:-]+', c['service']):
|
|||
|
|
raise ValueError('systemd 服务名格式无效')
|
|||
|
|
if c.get('health') and not c['health'].startswith(('http://', 'https://')):
|
|||
|
|
raise ValueError('健康检查地址必须为 HTTP 或 HTTPS')
|
|||
|
|
validate_nginx(c)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate_nginx(c, require_files=False):
|
|||
|
|
service = c.get('nginx_service', 'nginx.service')
|
|||
|
|
if not service or not re.fullmatch(r'[A-Za-z0-9_@.:-]+', service):
|
|||
|
|
raise ValueError('Nginx systemd 服务名格式无效')
|
|||
|
|
for key, default in (('nginx_binary', '/usr/sbin/nginx'), ('nginx_cert_dir', '/home/ssl')):
|
|||
|
|
value = c.get(key, default)
|
|||
|
|
p = PurePosixPath(value)
|
|||
|
|
if not p.is_absolute() or '..' in p.parts or '\x00' in value or '\n' in value or str(p) in ('/', '/home', '/usr', '/etc', '/root'):
|
|||
|
|
raise ValueError('Nginx 路径必须为明确的绝对路径:' + key)
|
|||
|
|
names = (c.get('nginx_cert_name', ''), c.get('nginx_key_name', ''))
|
|||
|
|
if require_files and not all(names):
|
|||
|
|
raise ValueError('请填写服务器证书和私钥文件名')
|
|||
|
|
for name in names:
|
|||
|
|
if name and (PurePosixPath(name).name != name or not re.fullmatch(r'[A-Za-z0-9._-]+', name) or name in ('.', '..')):
|
|||
|
|
raise ValueError('证书和私钥只能填写安全的文件名,不能包含路径')
|
|||
|
|
if all(names) and names[0] == names[1]:
|
|||
|
|
raise ValueError('证书和私钥不能使用同一个目标文件名')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def project_key(c):
|
|||
|
|
return hashlib.sha256((c['front'] + '\n' + c['war']).encode()).hexdigest()[:16]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def time_nonce():
|
|||
|
|
"""Return a nanosecond-style suffix on Python 3.6 and newer."""
|
|||
|
|
clock = getattr(time, 'time_ns', None)
|
|||
|
|
now = clock() if clock else int(time.time() * 1000000000)
|
|||
|
|
return now % 1000000000
|
|||
|
|
|
|||
|
|
|
|||
|
|
def pids(c):
|
|||
|
|
result = []
|
|||
|
|
expected = '-Dcatalina.base=' + c['tomcat'].rstrip('/')
|
|||
|
|
for p in Path('/proc').glob('[0-9]*/cmdline'):
|
|||
|
|
try:
|
|||
|
|
args = p.read_bytes().decode(errors='replace').split('\0')
|
|||
|
|
if expected in args and 'org.apache.catalina.startup.Bootstrap' in args and 'start' in args:
|
|||
|
|
result.append(int(p.parent.name))
|
|||
|
|
except (OSError, ValueError):
|
|||
|
|
pass
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def script(c, start):
|
|||
|
|
env = os.environ.copy()
|
|||
|
|
env['CATALINA_BASE'] = c['tomcat']
|
|||
|
|
env['CATALINA_HOME'] = c['tomcat']
|
|||
|
|
if c.get('java_home'):
|
|||
|
|
env['JAVA_HOME'] = c['java_home']
|
|||
|
|
cmd = ['systemctl', 'start' if start else 'stop', c['service']] if c.get('service') else ['sh', c['startup' if start else 'shutdown']]
|
|||
|
|
# Files instead of pipes: a daemon inheriting stdout must not keep communicate() open.
|
|||
|
|
import tempfile
|
|||
|
|
with tempfile.TemporaryFile() as output:
|
|||
|
|
proc = subprocess.Popen(cmd, cwd=c['tomcat'], env=env, stdout=output, stderr=output,
|
|||
|
|
stdin=subprocess.DEVNULL, start_new_session=True)
|
|||
|
|
try:
|
|||
|
|
proc.wait(timeout=int(c['start_timeout' if start else 'stop_timeout']))
|
|||
|
|
except subprocess.TimeoutExpired:
|
|||
|
|
proc.kill()
|
|||
|
|
proc.wait()
|
|||
|
|
raise RuntimeError('启动/停止命令执行超时')
|
|||
|
|
output.seek(0)
|
|||
|
|
msg = output.read(16000).decode(errors='replace').strip()
|
|||
|
|
if msg:
|
|||
|
|
event(msg)
|
|||
|
|
if proc.returncode:
|
|||
|
|
raise RuntimeError('命令退出码:' + str(proc.returncode))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def command(args, timeout=60, error='命令执行失败'):
|
|||
|
|
proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|||
|
|
stdin=subprocess.DEVNULL, timeout=timeout)
|
|||
|
|
output = proc.stdout.decode(errors='replace').strip()
|
|||
|
|
if output:
|
|||
|
|
event(output)
|
|||
|
|
if proc.returncode:
|
|||
|
|
raise RuntimeError('{}(退出码 {}){}'.format(error, proc.returncode,
|
|||
|
|
':' + output if output else ''))
|
|||
|
|
return proc.stdout
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_test(c, progress=70):
|
|||
|
|
event('正在检查 Nginx 配置…', progress)
|
|||
|
|
command([c.get('nginx_binary', '/usr/sbin/nginx'), '-t'], error='Nginx 配置检查失败')
|
|||
|
|
event('Nginx 配置检查通过')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_active(c):
|
|||
|
|
proc = subprocess.run(['systemctl', 'is-active', '--quiet', c.get('nginx_service', 'nginx.service')],
|
|||
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|||
|
|
stdin=subprocess.DEVNULL, timeout=30)
|
|||
|
|
return proc.returncode == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def restart_nginx(c, progress=85):
|
|||
|
|
event('正在重启 Nginx…', progress)
|
|||
|
|
command(['systemctl', 'restart', c.get('nginx_service', 'nginx.service')],
|
|||
|
|
timeout=int(c.get('start_timeout', 120)), error='Nginx 重启失败')
|
|||
|
|
if not nginx_active(c):
|
|||
|
|
raise RuntimeError('Nginx 重启后未处于 active 状态')
|
|||
|
|
event('Nginx 已正常运行')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def wait_stopped(c, timeout):
|
|||
|
|
end = time.monotonic() + timeout
|
|||
|
|
while time.monotonic() < end:
|
|||
|
|
if not pids(c):
|
|||
|
|
return True
|
|||
|
|
time.sleep(.5)
|
|||
|
|
return not pids(c)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def stop(c):
|
|||
|
|
event('正在停止目标 Tomcat…', 55)
|
|||
|
|
try:
|
|||
|
|
script(c, False)
|
|||
|
|
except Exception as exc:
|
|||
|
|
if c.get('service'):
|
|||
|
|
raise # Never fight a service manager's automatic restart policy.
|
|||
|
|
event('停止脚本未成功,继续核对进程:' + str(exc))
|
|||
|
|
if not wait_stopped(c, int(c['stop_timeout'])):
|
|||
|
|
for sig, seconds in ((signal.SIGTERM, 5), (signal.SIGKILL, 5)):
|
|||
|
|
for pid in pids(c):
|
|||
|
|
try:
|
|||
|
|
os.kill(pid, sig)
|
|||
|
|
event('终止 Tomcat PID {}({})'.format(pid, sig.name))
|
|||
|
|
except ProcessLookupError:
|
|||
|
|
pass
|
|||
|
|
if wait_stopped(c, seconds):
|
|||
|
|
break
|
|||
|
|
if pids(c):
|
|||
|
|
raise RuntimeError('Tomcat 未完全停止,已中止文件替换')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def start(c):
|
|||
|
|
event('正在启动 Tomcat…', 85)
|
|||
|
|
script(c, True)
|
|||
|
|
end = time.monotonic() + int(c['start_timeout'])
|
|||
|
|
stable = 0
|
|||
|
|
while time.monotonic() < end:
|
|||
|
|
if pids(c):
|
|||
|
|
if c.get('health'):
|
|||
|
|
try:
|
|||
|
|
with urllib.request.urlopen(c['health'], timeout=3) as response:
|
|||
|
|
if 200 <= response.status < 300:
|
|||
|
|
event('Tomcat 进程与 HTTP 健康检查通过', 96)
|
|||
|
|
return
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
else:
|
|||
|
|
stable += 1
|
|||
|
|
if stable >= 5:
|
|||
|
|
event('Tomcat 进程已连续运行 5 秒;未配置 HTTP 检查', 96)
|
|||
|
|
return
|
|||
|
|
else:
|
|||
|
|
stable = 0
|
|||
|
|
time.sleep(1)
|
|||
|
|
log = Path(c['tomcat']) / 'logs/catalina.out'
|
|||
|
|
if log.is_file():
|
|||
|
|
with log.open('rb') as f:
|
|||
|
|
f.seek(max(0, log.stat().st_size - 12000))
|
|||
|
|
event(f.read().decode(errors='replace'))
|
|||
|
|
raise RuntimeError('Tomcat 启动验证超时')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def remove(p):
|
|||
|
|
if p.is_symlink() or p.is_file():
|
|||
|
|
p.unlink()
|
|||
|
|
elif p.exists():
|
|||
|
|
shutil.rmtree(p)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def copy(src, dst):
|
|||
|
|
if src.is_symlink():
|
|||
|
|
raise RuntimeError('发布文件不能是软链接:' + str(src))
|
|||
|
|
if src.is_dir():
|
|||
|
|
shutil.copytree(src, dst, symlinks=True)
|
|||
|
|
else:
|
|||
|
|
shutil.copy2(src, dst)
|
|||
|
|
if os.name == 'posix' and os.geteuid() == 0:
|
|||
|
|
for p in [src] + (list(src.rglob('*')) if src.is_dir() else []):
|
|||
|
|
dest = dst if p == src else dst / p.relative_to(src)
|
|||
|
|
stat = p.lstat()
|
|||
|
|
os.chown(str(dest), stat.st_uid, stat.st_gid, follow_symlinks=False)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def tree_size(p):
|
|||
|
|
if not p.exists():
|
|||
|
|
return 0
|
|||
|
|
if p.is_file():
|
|||
|
|
return p.stat().st_size
|
|||
|
|
return sum(x.stat().st_size for x in p.rglob('*') if x.is_file() and not x.is_symlink())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def targets(c, mode):
|
|||
|
|
entries = {}
|
|||
|
|
if mode in ('all', 'backend'):
|
|||
|
|
entries.update(war=c['war'], root=str(Path(c['war']).with_suffix('')))
|
|||
|
|
if mode in ('all', 'frontend'):
|
|||
|
|
entries['front'] = c['front']
|
|||
|
|
return entries
|
|||
|
|
|
|||
|
|
|
|||
|
|
def preflight(c, mode, size=0):
|
|||
|
|
validate(c)
|
|||
|
|
if mode not in ('all', 'backend', 'frontend'):
|
|||
|
|
raise ValueError('未知部署模式')
|
|||
|
|
for target in targets(c, mode).values():
|
|||
|
|
path = Path(target)
|
|||
|
|
if any(p.is_symlink() for p in [path, *path.parents]):
|
|||
|
|
raise ValueError('目标路径不能包含软链接:' + target)
|
|||
|
|
if mode != 'frontend':
|
|||
|
|
if not Path(c['tomcat']).is_dir():
|
|||
|
|
raise RuntimeError('Tomcat 目录不存在')
|
|||
|
|
if not c.get('service'):
|
|||
|
|
for k in ('startup', 'shutdown'):
|
|||
|
|
if not Path(c[k]).is_file():
|
|||
|
|
raise RuntimeError('脚本不存在:' + c[k])
|
|||
|
|
paths = [Path(c['temp']), Path(c['backups'])] + [Path(p).parent for p in targets(c, mode).values()]
|
|||
|
|
old_size = sum(tree_size(Path(p)) for p in targets(c, mode).values())
|
|||
|
|
for path in paths:
|
|||
|
|
path.mkdir(parents=True, exist_ok=True)
|
|||
|
|
probe = path / ('.autodeploy-probe-' + uuid.uuid4().hex)
|
|||
|
|
probe.write_text('ok')
|
|||
|
|
probe.unlink()
|
|||
|
|
if shutil.disk_usage(path).free < old_size * 2 + int(size) * 3 + 32 * 1024 * 1024:
|
|||
|
|
raise RuntimeError('磁盘可用空间不足:' + str(path))
|
|||
|
|
if c.get('owner'):
|
|||
|
|
import pwd
|
|||
|
|
import grp
|
|||
|
|
user, _, group = c['owner'].partition(':')
|
|||
|
|
pwd.getpwnam(user)
|
|||
|
|
if group:
|
|||
|
|
grp.getgrnam(group)
|
|||
|
|
if os.geteuid() != 0:
|
|||
|
|
raise RuntimeError('配置文件属主需要 root 权限')
|
|||
|
|
event('路径、权限与磁盘检查通过', 12)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_targets(c):
|
|||
|
|
root = Path(c.get('nginx_cert_dir', '/home/ssl'))
|
|||
|
|
return {'certificate': str(root / c['nginx_cert_name']),
|
|||
|
|
'private_key': str(root / c['nginx_key_name'])}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_preflight(c, certificate=False, size=0):
|
|||
|
|
validate_nginx(c, require_files=certificate)
|
|||
|
|
binary = Path(c.get('nginx_binary', '/usr/sbin/nginx'))
|
|||
|
|
if not binary.is_file() or not os.access(str(binary), os.X_OK):
|
|||
|
|
raise RuntimeError('Nginx 可执行文件不存在或不可执行:' + str(binary))
|
|||
|
|
if not shutil.which('systemctl'):
|
|||
|
|
raise RuntimeError('服务器缺少 systemctl,当前版本只支持 systemd 管理的 Nginx')
|
|||
|
|
if certificate:
|
|||
|
|
if not shutil.which('openssl'):
|
|||
|
|
raise RuntimeError('服务器缺少 openssl,无法校验证书和私钥')
|
|||
|
|
root = Path(c.get('nginx_cert_dir', '/home/ssl'))
|
|||
|
|
backup_root = nginx_backup_root(c)
|
|||
|
|
for directory, caption in ((root, '证书目录'), (backup_root, '证书备份目录')):
|
|||
|
|
if any(p.is_symlink() for p in [directory, *directory.parents]):
|
|||
|
|
raise ValueError(caption + '不能包含软链接:' + str(directory))
|
|||
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|||
|
|
probe = directory / ('.autodeploy-probe-' + uuid.uuid4().hex)
|
|||
|
|
probe.write_text('ok')
|
|||
|
|
probe.unlink()
|
|||
|
|
if shutil.disk_usage(directory).free < int(size) * 3 + 1024 * 1024:
|
|||
|
|
raise RuntimeError(caption + '可用空间不足:' + str(directory))
|
|||
|
|
for target in nginx_targets(c).values():
|
|||
|
|
path = Path(target)
|
|||
|
|
if path.is_symlink():
|
|||
|
|
raise ValueError('证书目标不能是软链接:' + target)
|
|||
|
|
if path.exists() and not path.is_file():
|
|||
|
|
raise ValueError('证书目标不是普通文件:' + target)
|
|||
|
|
event('Nginx、路径与权限检查通过', 12)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def snapshot(c, mode):
|
|||
|
|
root = Path(c['backups']) / project_key(c)
|
|||
|
|
root.mkdir(parents=True, exist_ok=True)
|
|||
|
|
backup = root / (time.strftime('%Y%m%d-%H%M%S') + '-{:09d}-'.format(time_nonce()) + uuid.uuid4().hex[:8])
|
|||
|
|
backup.mkdir()
|
|||
|
|
manifest = dict(mode=mode, targets=targets(c, mode), present={}, complete=False)
|
|||
|
|
for name, path in manifest['targets'].items():
|
|||
|
|
src = Path(path)
|
|||
|
|
manifest['present'][name] = src.exists()
|
|||
|
|
if src.exists():
|
|||
|
|
copy(src, backup / name)
|
|||
|
|
manifest['complete'] = True
|
|||
|
|
(backup / 'manifest.json').write_text(json.dumps(manifest), encoding='utf-8')
|
|||
|
|
event('已备份当前版本', 48, backup=str(backup))
|
|||
|
|
return backup
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_backup(c, path):
|
|||
|
|
p = Path(path)
|
|||
|
|
expected = Path(c['backups']) / project_key(c)
|
|||
|
|
if p.parent != expected or p.is_symlink() or p.resolve().parent != expected.resolve():
|
|||
|
|
raise ValueError('备份不属于当前项目')
|
|||
|
|
m = json.loads((p / 'manifest.json').read_text(encoding='utf-8'))
|
|||
|
|
if not m.get('complete') or m['targets'] != targets(c, m['mode']):
|
|||
|
|
raise ValueError('备份不完整或目标路径不匹配')
|
|||
|
|
for name, present in m['present'].items():
|
|||
|
|
if present and not (p / name).exists():
|
|||
|
|
raise ValueError('备份文件缺失:' + name)
|
|||
|
|
return m
|
|||
|
|
|
|||
|
|
|
|||
|
|
def restore(c, backup):
|
|||
|
|
m = load_backup(c, str(backup))
|
|||
|
|
for name, path in m['targets'].items():
|
|||
|
|
dst = Path(path)
|
|||
|
|
remove(dst)
|
|||
|
|
if m['present'][name]:
|
|||
|
|
copy(backup / name, dst)
|
|||
|
|
ownership(c, dst)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def ownership(c, path):
|
|||
|
|
if not c.get('owner'):
|
|||
|
|
return
|
|||
|
|
import pwd
|
|||
|
|
import grp
|
|||
|
|
user, _, group = c['owner'].partition(':')
|
|||
|
|
account = pwd.getpwnam(user)
|
|||
|
|
uid, gid = account.pw_uid, grp.getgrnam(group).gr_gid if group else account.pw_gid
|
|||
|
|
for p in [path] + (list(path.rglob('*')) if path.is_dir() else []):
|
|||
|
|
os.chown(str(p), uid, gid, follow_symlinks=False)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def unpack(archive, destination):
|
|||
|
|
destination.mkdir()
|
|||
|
|
with tarfile.open(archive, 'r:gz') as tf:
|
|||
|
|
for member in tf.getmembers():
|
|||
|
|
p = Path(member.name)
|
|||
|
|
if p.is_absolute() or '..' in p.parts or not (member.isfile() or member.isdir()):
|
|||
|
|
raise ValueError('前端压缩包包含不安全条目')
|
|||
|
|
tf.extractall(destination)
|
|||
|
|
if not (destination / 'index.html').is_file():
|
|||
|
|
raise ValueError('前端目录中找不到 index.html')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def install(src, dst, c):
|
|||
|
|
staged = dst.parent / ('.autodeploy-new-' + uuid.uuid4().hex)
|
|||
|
|
try:
|
|||
|
|
copy(src, staged)
|
|||
|
|
ownership(c, staged)
|
|||
|
|
remove(dst)
|
|||
|
|
os.replace(staged, dst)
|
|||
|
|
finally:
|
|||
|
|
remove(staged)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def prune(c):
|
|||
|
|
root = Path(c['backups']) / project_key(c)
|
|||
|
|
completed = sorted(p for p in root.iterdir() if p.is_dir() and not p.is_symlink() and (p / 'manifest.json').is_file())
|
|||
|
|
for p in completed[:-2]:
|
|||
|
|
remove(p)
|
|||
|
|
event('备份已保留最近 2 次')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_backup_root(c):
|
|||
|
|
targets = nginx_targets(c)
|
|||
|
|
key = hashlib.sha256((targets['certificate'] + '\n' + targets['private_key']).encode()).hexdigest()[:16]
|
|||
|
|
return Path(c['backups']) / ('nginx-' + key)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_snapshot(c):
|
|||
|
|
root = nginx_backup_root(c)
|
|||
|
|
root.mkdir(parents=True, exist_ok=True)
|
|||
|
|
backup = root / (time.strftime('%Y%m%d-%H%M%S') + '-{:09d}-'.format(time_nonce()) + uuid.uuid4().hex[:8])
|
|||
|
|
backup.mkdir()
|
|||
|
|
manifest = dict(kind='nginx-certificate', targets=nginx_targets(c), present={}, complete=False)
|
|||
|
|
for name, path in manifest['targets'].items():
|
|||
|
|
src = Path(path)
|
|||
|
|
manifest['present'][name] = src.exists()
|
|||
|
|
if src.exists():
|
|||
|
|
copy(src, backup / name)
|
|||
|
|
manifest['complete'] = True
|
|||
|
|
(backup / 'manifest.json').write_text(json.dumps(manifest), encoding='utf-8')
|
|||
|
|
event('已备份当前证书', 48, backup=str(backup))
|
|||
|
|
return backup
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_nginx_backup(c, backup):
|
|||
|
|
path = Path(backup)
|
|||
|
|
expected = nginx_backup_root(c)
|
|||
|
|
if path.parent != expected or path.is_symlink() or path.resolve().parent != expected.resolve():
|
|||
|
|
raise ValueError('证书备份不属于当前配置')
|
|||
|
|
manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8'))
|
|||
|
|
if (not manifest.get('complete') or manifest.get('kind') != 'nginx-certificate' or
|
|||
|
|
manifest.get('targets') != nginx_targets(c)):
|
|||
|
|
raise ValueError('证书备份不完整或目标路径不匹配')
|
|||
|
|
for name, present in manifest['present'].items():
|
|||
|
|
if present and not (path / name).is_file():
|
|||
|
|
raise ValueError('证书备份文件缺失:' + name)
|
|||
|
|
return manifest
|
|||
|
|
|
|||
|
|
|
|||
|
|
def replace_secure(src, dst, mode, preserve_from=None):
|
|||
|
|
if dst.is_symlink() or any(p.is_symlink() for p in dst.parents):
|
|||
|
|
raise ValueError('证书目标不能包含软链接:' + str(dst))
|
|||
|
|
staged = dst.parent / ('.autodeploy-new-' + uuid.uuid4().hex)
|
|||
|
|
try:
|
|||
|
|
shutil.copy2(str(src), str(staged))
|
|||
|
|
os.chmod(str(staged), mode)
|
|||
|
|
reference = preserve_from if preserve_from and preserve_from.exists() else (dst if dst.exists() else None)
|
|||
|
|
if reference is not None and os.name == 'posix' and os.geteuid() == 0:
|
|||
|
|
stat = reference.stat()
|
|||
|
|
os.chown(str(staged), stat.st_uid, stat.st_gid)
|
|||
|
|
os.replace(str(staged), str(dst))
|
|||
|
|
finally:
|
|||
|
|
if staged.exists():
|
|||
|
|
staged.unlink()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def restore_nginx_certificate(c, backup):
|
|||
|
|
manifest = load_nginx_backup(c, backup)
|
|||
|
|
for name, target in manifest['targets'].items():
|
|||
|
|
dst = Path(target)
|
|||
|
|
if manifest['present'][name]:
|
|||
|
|
source = Path(backup) / name
|
|||
|
|
replace_secure(source, dst, source.stat().st_mode & 0o777, source)
|
|||
|
|
elif dst.exists():
|
|||
|
|
dst.unlink()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def prune_nginx(c):
|
|||
|
|
root = nginx_backup_root(c)
|
|||
|
|
completed = sorted(p for p in root.iterdir() if p.is_dir() and not p.is_symlink() and (p / 'manifest.json').is_file())
|
|||
|
|
for path in completed[:-2]:
|
|||
|
|
remove(path)
|
|||
|
|
event('证书备份已保留最近 2 次')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def openssl_output(args, error):
|
|||
|
|
proc = subprocess.run(['openssl'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|||
|
|
stdin=subprocess.DEVNULL, timeout=30)
|
|||
|
|
if proc.returncode:
|
|||
|
|
detail = proc.stderr.decode(errors='replace').strip()
|
|||
|
|
raise ValueError(error + (':' + detail if detail else ''))
|
|||
|
|
return proc.stdout
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate_certificate_pair(cert, key):
|
|||
|
|
openssl_output(['x509', '-in', str(cert), '-noout'], '证书格式无效')
|
|||
|
|
openssl_output(['pkey', '-in', str(key), '-noout'], '私钥格式无效或私钥已加密')
|
|||
|
|
if openssl_output(['x509', '-in', str(cert), '-pubkey', '-noout'], '无法读取证书公钥').strip() != \
|
|||
|
|
openssl_output(['pkey', '-in', str(key), '-pubout'], '无法读取私钥公钥').strip():
|
|||
|
|
raise ValueError('证书与私钥不匹配')
|
|||
|
|
openssl_output(['x509', '-in', str(cert), '-checkend', '0', '-noout'], '证书已经过期')
|
|||
|
|
details = openssl_output(['x509', '-in', str(cert), '-subject', '-issuer', '-dates', '-noout'],
|
|||
|
|
'无法读取证书信息').decode(errors='replace').strip()
|
|||
|
|
event('证书与私钥校验通过' + (':\n' + details if details else ''), 40)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_lock_path(c):
|
|||
|
|
name = c.get('nginx_service', 'nginx.service') + '\n' + c.get('nginx_cert_dir', '/home/ssl')
|
|||
|
|
return Path('/tmp') / ('.autodeploy-nginx-' + hashlib.sha256(name.encode()).hexdigest()[:16] + '.lock')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def nginx_restart_transaction(c):
|
|||
|
|
import fcntl
|
|||
|
|
with nginx_lock_path(c).open('a') as lock:
|
|||
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|||
|
|
nginx_test(c, 45)
|
|||
|
|
restart_nginx(c, 75)
|
|||
|
|
event('Nginx 重启成功', 100, result='success')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def certificate_transaction(c, stage):
|
|||
|
|
import fcntl
|
|||
|
|
cert = stage / 'certificate.pem'
|
|||
|
|
key = stage / 'private.key'
|
|||
|
|
validate_certificate_pair(cert, key)
|
|||
|
|
backup = None
|
|||
|
|
touched = False
|
|||
|
|
was_active = nginx_active(c)
|
|||
|
|
with nginx_lock_path(c).open('a') as lock:
|
|||
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|||
|
|
try:
|
|||
|
|
backup = nginx_snapshot(c)
|
|||
|
|
touched = True
|
|||
|
|
targets = nginx_targets(c)
|
|||
|
|
event('正在原子替换证书和私钥…', 62)
|
|||
|
|
replace_secure(cert, Path(targets['certificate']), 0o644)
|
|||
|
|
replace_secure(key, Path(targets['private_key']), 0o600)
|
|||
|
|
nginx_test(c, 72)
|
|||
|
|
restart_nginx(c, 86)
|
|||
|
|
event('证书更新并重启 Nginx 成功', 100, result='success')
|
|||
|
|
try:
|
|||
|
|
prune_nginx(c)
|
|||
|
|
except Exception as exc:
|
|||
|
|
event('证书更新已完成,旧备份清理失败:' + str(exc))
|
|||
|
|
except Exception as exc:
|
|||
|
|
event('证书更新失败:' + str(exc))
|
|||
|
|
if touched and backup:
|
|||
|
|
try:
|
|||
|
|
event('正在恢复更新前的证书…')
|
|||
|
|
restore_nginx_certificate(c, backup)
|
|||
|
|
if was_active:
|
|||
|
|
nginx_test(c)
|
|||
|
|
restart_nginx(c)
|
|||
|
|
event('已恢复更新前的证书' + (',Nginx 已恢复' if was_active else ''), result='rolled_back')
|
|||
|
|
except Exception as restore_error:
|
|||
|
|
event('证书自动恢复失败,请查看备份:{};{}'.format(backup, restore_error), result='recovery_failed')
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
|
|||
|
|
def transaction(c, mode, stage, rollback=None):
|
|||
|
|
import fcntl
|
|||
|
|
# Lock by each target AND instance, so overlapping profiles cannot deploy together.
|
|||
|
|
lock_paths = [Path(p).parent / ('.autodeploy-' + Path(p).name + '.lock') for p in targets(c, mode).values()]
|
|||
|
|
if mode != 'frontend':
|
|||
|
|
lock_paths.append(Path(c['tomcat']) / '.autodeploy.lock')
|
|||
|
|
locks = []
|
|||
|
|
backup = None
|
|||
|
|
touched = False
|
|||
|
|
stopped = False
|
|||
|
|
was_running = bool(pids(c)) if mode != 'frontend' else False
|
|||
|
|
try:
|
|||
|
|
for path in sorted(set(lock_paths)):
|
|||
|
|
handle = path.open('a')
|
|||
|
|
locks.append(handle)
|
|||
|
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|||
|
|
if mode != 'frontend':
|
|||
|
|
stopped = True
|
|||
|
|
stop(c)
|
|||
|
|
# Snapshot after shutdown so the exploded application is consistent.
|
|||
|
|
backup = snapshot(c, mode)
|
|||
|
|
touched = True
|
|||
|
|
event('正在替换部署文件…', 68)
|
|||
|
|
if rollback:
|
|||
|
|
restore(c, Path(rollback))
|
|||
|
|
else:
|
|||
|
|
if mode != 'frontend':
|
|||
|
|
remove(Path(c['war']).with_suffix(''))
|
|||
|
|
install(stage / 'ROOT.war', Path(c['war']), c)
|
|||
|
|
if mode != 'backend':
|
|||
|
|
install(stage / 'dist', Path(c['front']), c)
|
|||
|
|
if mode != 'frontend':
|
|||
|
|
start(c)
|
|||
|
|
event('回滚成功' if rollback else '部署成功', 100, result='success')
|
|||
|
|
try:
|
|||
|
|
prune(c)
|
|||
|
|
except Exception as exc:
|
|||
|
|
event('发布已完成,旧备份清理失败:' + str(exc))
|
|||
|
|
except Exception as exc:
|
|||
|
|
event('操作失败:' + str(exc))
|
|||
|
|
if touched and backup and c.get('auto_rollback', True):
|
|||
|
|
try:
|
|||
|
|
event('正在自动恢复部署前版本…')
|
|||
|
|
if mode != 'frontend':
|
|||
|
|
stop(c)
|
|||
|
|
restore(c, backup)
|
|||
|
|
if mode != 'frontend' and was_running:
|
|||
|
|
start(c)
|
|||
|
|
event('已恢复部署前文件' + (',服务已恢复' if was_running else ''), result='rolled_back')
|
|||
|
|
except Exception as restore_error:
|
|||
|
|
event('自动恢复失败,请查看备份:{};{}'.format(backup, restore_error), result='recovery_failed')
|
|||
|
|
elif stopped and not touched and was_running:
|
|||
|
|
try:
|
|||
|
|
if not pids(c):
|
|||
|
|
start(c)
|
|||
|
|
except Exception as restart_error:
|
|||
|
|
event('恢复原服务失败:' + str(restart_error))
|
|||
|
|
raise
|
|||
|
|
finally:
|
|||
|
|
for handle in locks:
|
|||
|
|
handle.close()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
if hasattr(signal, 'SIGHUP'):
|
|||
|
|
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
|||
|
|
args = json.loads(base64.b64decode(sys.argv[1]))
|
|||
|
|
c, action = args['config'], args['action']
|
|||
|
|
mode = args.get('mode', 'all')
|
|||
|
|
if action == 'preflight':
|
|||
|
|
if args.get('operation') in ('cert_deploy', 'nginx_restart'):
|
|||
|
|
nginx_preflight(c, args.get('operation') == 'cert_deploy', args.get('size', 0))
|
|||
|
|
else:
|
|||
|
|
preflight(c, mode, args.get('size', 0))
|
|||
|
|
return
|
|||
|
|
if action == 'nginx_restart':
|
|||
|
|
nginx_preflight(c)
|
|||
|
|
nginx_restart_transaction(c)
|
|||
|
|
return
|
|||
|
|
if action == 'cert_deploy':
|
|||
|
|
validate_nginx(c, require_files=True)
|
|||
|
|
else:
|
|||
|
|
validate(c)
|
|||
|
|
stage = Path(args['stage'])
|
|||
|
|
if stage.parent != Path(c['temp']) or not re.fullmatch(r'autodeploy-[a-f0-9]{32}', stage.name) or stage.is_symlink():
|
|||
|
|
raise ValueError('临时目录无效')
|
|||
|
|
try:
|
|||
|
|
if action == 'cert_deploy':
|
|||
|
|
expected = {'certificate.pem', 'private.key'}
|
|||
|
|
if set(args.get('hashes', {})) != expected:
|
|||
|
|
raise ValueError('证书上传文件不完整')
|
|||
|
|
for filename, digest in args['hashes'].items():
|
|||
|
|
h = hashlib.sha256()
|
|||
|
|
with (stage / filename).open('rb') as f:
|
|||
|
|
for chunk in iter(lambda: f.read(1024 * 1024), b''):
|
|||
|
|
h.update(chunk)
|
|||
|
|
if h.hexdigest() != digest:
|
|||
|
|
raise ValueError('上传文件校验失败:' + filename)
|
|||
|
|
nginx_preflight(c, True, args.get('size', 0))
|
|||
|
|
event('上传内容校验通过', 36)
|
|||
|
|
certificate_transaction(c, stage)
|
|||
|
|
elif action == 'rollback':
|
|||
|
|
mode = load_backup(c, args['backup'])['mode']
|
|||
|
|
elif action == 'deploy':
|
|||
|
|
for filename, digest in args['hashes'].items():
|
|||
|
|
if filename not in ('ROOT.war', 'dist.tar.gz'):
|
|||
|
|
raise ValueError('未知部署文件')
|
|||
|
|
h = hashlib.sha256()
|
|||
|
|
with (stage / filename).open('rb') as f:
|
|||
|
|
for chunk in iter(lambda: f.read(1024 * 1024), b''):
|
|||
|
|
h.update(chunk)
|
|||
|
|
if h.hexdigest() != digest:
|
|||
|
|
raise ValueError('上传文件校验失败:' + filename)
|
|||
|
|
if mode != 'backend':
|
|||
|
|
unpack(stage / 'dist.tar.gz', stage / 'dist')
|
|||
|
|
else:
|
|||
|
|
raise ValueError('未知操作')
|
|||
|
|
if action in ('deploy', 'rollback'):
|
|||
|
|
preflight(c, mode, args.get('size', 0))
|
|||
|
|
event('上传内容校验通过', 40)
|
|||
|
|
transaction(c, mode, stage, args.get('backup') if action == 'rollback' else None)
|
|||
|
|
finally:
|
|||
|
|
try:
|
|||
|
|
remove(stage)
|
|||
|
|
except OSError as exc:
|
|||
|
|
event('临时文件清理失败:' + str(exc))
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
try:
|
|||
|
|
main()
|
|||
|
|
except Exception as error:
|
|||
|
|
event(str(error), result='failed')
|
|||
|
|
sys.exit(1)
|