#!/bin/bash
# kvcs-ctl — KVCS 服务原子操作控制器（systemd 托管版）
#
# 安装方式:
#   RPM/DEB: 安装后位于 /usr/bin/kvcs-ctl，配置文件在 /etc/kvcs/
#   tarball: 位于解包目录，配置文件在同目录
#
# 交付件:
#   1. 二进制包 (kvcs_efc_server / kvcs-master / kvcs-node-manager)
#   2. 配置文件 (efc.yaml / master.yaml / nodemanager.yaml)
#   3. kvcs-ctl（本脚本）
#
# 目录布局 (tarball 部署, $KVCS_ROOT = 本脚本所在目录):
#   $KVCS_ROOT/
#   ├── kvcs-ctl             # 本脚本
#   ├── efc.yaml             # EFC 配置
#   ├── master.yaml          # Master 配置
#   ├── nodemanager.yaml     # Node-Manager 配置
#   ├── kvcs_efc_server      # EFC 二进制
#   ├── spdk_fs_tool         # NVMe 格式化工具（init-efc/reset-efc 需要）
#   ├── kvcs-master          # Master 二进制
#   ├── kvcs-node-manager    # Node-Manager 二进制
#   └── log/                 # 日志
#
# 目录布局 (RPM/DEB 安装):
#   /usr/bin/kvcs-ctl          # 本脚本
#   /usr/bin/kvcs_efc_server   # EFC 二进制
#   /usr/bin/spdk_fs_tool      # NVMe 格式化工具
#   /etc/kvcs/efc.yaml         # EFC 配置
#   /etc/kvcs/master.yaml      # Master 配置
#   /etc/kvcs/nodemanager.yaml # Node-Manager 配置
#   /var/lib/kvcs/log/         # 日志
#
# 用法:
#   kvcs-ctl <command> [command2 ...]
#
# 原子命令:
#   install-systemd    安装 systemd unit 文件到 /etc/systemd/system/ 并 enable
#   start-efc          systemctl start kvcs-efc
#   stop-efc           systemctl stop kvcs-efc
#   init-disks         初始化后端存储（tmpfs 挂载 / disk blkdiscard + 格式化）
#   reset-efc          重置 disk EFC（要求 EFC 已停止；清 SHM + 全盘trim + 强制 format）
#                      执行前打印计划并需输入 yes 确认，-y/--yes 可跳过
#   start-master       systemctl start kvcs-master
#   stop-master        systemctl stop kvcs-master
#   start-nm           systemctl start kvcs-node-manager
#   stop-nm            systemctl stop kvcs-node-manager
#   status             显示全部服务状态
#   help               显示本帮助信息
#
# systemd 钩子命令（由 unit 文件自动调用，一般不手动执行）:
#   efc-prestart        ExecStartPre: 恢复 tmpfs mount + 清信号量 + 建 socket 目录
#   efc-poststart       ExecStartPost: 等 socket 就绪 + chmod 777
#
# 组合示例:
#   kvcs-ctl install-systemd                # 首次安装 unit 文件
#   kvcs-ctl start-efc start-master start-nm  # 启动全部
#   kvcs-ctl stop-nm stop-master stop-efc     # 停止全部
#

# ================================================================
#  路径常量
#  系统安装（RPM/DEB）：/etc/kvcs/ 存在 → 用标准路径
#  tarball 部署：基于脚本自身位置
# ================================================================
if [ -d /etc/kvcs ]; then
    KVCS_ROOT="/var/lib/kvcs"
    KVCS_CTL_BIN="/usr/bin/kvcs-ctl"
    EFC_CONFIG="/etc/kvcs/efc.yaml"
    EFC_BIN="/usr/bin/kvcs_efc_server"
    MASTER_BIN="/usr/bin/kvcs-master"
    MASTER_CONFIG="/etc/kvcs/master.yaml"
    NM_BIN="/usr/bin/kvcs-node-manager"
    NM_CONFIG="/etc/kvcs/nodemanager.yaml"
else
    KVCS_ROOT="$(cd "$(dirname "$0")" && pwd)"
    KVCS_CTL_BIN="$KVCS_ROOT/kvcs-ctl"
    EFC_CONFIG="$KVCS_ROOT/efc.yaml"
    EFC_BIN="$KVCS_ROOT/kvcs_efc_server"
    MASTER_BIN="$KVCS_ROOT/kvcs-master"
    MASTER_CONFIG="$KVCS_ROOT/master.yaml"
    NM_BIN="$KVCS_ROOT/kvcs-node-manager"
    NM_CONFIG="$KVCS_ROOT/nodemanager.yaml"
fi
LOG_DIR="$KVCS_ROOT/log"

# systemd 常量
SYSTEMD_DIR="/etc/systemd/system"
SVC_EFC="kvcs-efc"
SVC_MASTER="kvcs-master"
SVC_NM="kvcs-node-manager"

# 破坏性命令的确认开关：-y/--yes 或 KVCS_ASSUME_YES=true
ASSUME_YES="${KVCS_ASSUME_YES:-false}"

# ================================================================
#  辅助函数
# ================================================================
log() { echo "[$(date +%H:%M:%S)] $*"; }

die() { log "ERROR: $*"; exit 1; }

# 抽象块设备判断，便于单元测试 mock
is_block_device() { [ -b "$1" ]; }

# yaml_scalar <file> <key> <default> — 读取顶层 YAML 标量并解析 ${VAR:-default}。
yaml_scalar() {
    local file="$1" key="$2" default_value="$3" val
    val=$(grep "^${key}:" "$file" 2>/dev/null | head -1 | sed "s/^${key}:[[:space:]]*//" | sed 's/[[:space:]]*#.*//' || true)
    val=$(echo "$val" | sed 's/^"//;s/"$//;s/^'"'"'//;s/'"'"'$//')
    if [[ "$val" =~ ^\$\{([A-Za-z_][A-Za-z0-9_]*)\:-([^\}]*)\}$ ]]; then
        local env_name="${BASH_REMATCH[1]}" env_default="${BASH_REMATCH[2]}"
        val="${!env_name:-$env_default}"
    elif [[ "$val" =~ ^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$ ]]; then
        local env_name="${BASH_REMATCH[1]}"
        val="${!env_name:-}"
    fi
    echo "${val:-$default_value}"
}

# health_url <config> <default_addr> — 将 health 监听地址转换为本机 readiness URL。
health_url() {
    local file="$1" default_addr="$2" addr
    addr=$(yaml_scalar "$file" health_addr "$default_addr")
    case "$addr" in
        :*) addr="127.0.0.1${addr}" ;;
        0.0.0.0:*) addr="127.0.0.1:${addr#0.0.0.0:}" ;;
    esac
    echo "http://${addr}/readyz"
}

http_probe() {
    local url="$1"
    if command -v curl >/dev/null 2>&1; then
        curl -fsS --max-time 2 "$url" >/dev/null 2>&1
    elif command -v wget >/dev/null 2>&1; then
        wget -q -T 2 -O /dev/null "$url" >/dev/null 2>&1
    else
        log "  ✗ health check requires curl or wget"
        return 1
    fi
}

# service_health_check <name> <service> <config> <default_addr>
service_health_check() {
    local name="$1" service="$2" config="$3" default_addr="$4"
    local url waited=0 max_wait=30
    url=$(health_url "$config" "$default_addr")
    while [ "$waited" -lt "$max_wait" ]; do
        if ! systemctl is-active --quiet "$service"; then
            log "  ✗ $name service exited while waiting for readiness"
            journalctl -u "$service" --no-pager -n 10 2>/dev/null | sed 's/^/    /'
            return 1
        fi
        if http_probe "$url"; then
            log "  ✓ $name ready: $url"
            return 0
        fi
        sleep 2
        waited=$((waited + 2))
        log "  waiting for $name readiness ... (${waited}s)"
    done
    log "  ✗ $name NOT ready after ${max_wait}s: $url"
    journalctl -u "$service" --no-pager -n 10 2>/dev/null | sed 's/^/    /'
    return 1
}

# stop_process <pattern> — 通过 ps -ef | grep 定位 PID，sudo kill -9（reset-efc 用）
stop_process() {
    local pattern="$1"
    local pids
    pids=$(ps -ef | grep "$pattern" | grep -v grep | grep -v $$ | awk '{print $2}' || true)
    if [ -z "$pids" ]; then
        log "  $pattern: not running"
        return 0
    fi
    log "  Stopping $pattern (PIDs: $pids) ..."
    sudo kill -9 $pids 2>/dev/null || true
    sleep 1
}

# ensure_unit <svc> — 检查 unit 是否已安装
ensure_unit() {
    local svc="$1"
    if ! systemctl list-unit-files "$svc.service" 2>/dev/null | grep -q "$svc"; then
        die "unit $svc 未安装，请先执行: kvcs-ctl install-systemd"
    fi
}

# ================================================================
#  efc.yaml 配置解析
# ================================================================

# yaml_get <key> — 读取 YAML 平层 key: value（支持 ${VAR:-default} 解析，并去除 YAML 字符串引号）
yaml_get() {
    local val
    val=$(grep "^${1}:" "$EFC_CONFIG" | sed "s/^${1}:[[:space:]]*//" | sed 's/[[:space:]]*#.*//')
    # 解析 ${VAR:-default} → 取 default 值
    val=$(echo "$val" | sed 's/\${[^:]*:-\([^}]*\)}/\1/g')
    # 去除 YAML 字符串首尾引号（"..." 或 '...'）
    val=$(echo "$val" | sed 's/^["'\''"]//; s/["'\''"]$//')
    echo "$val"
}

# yaml_list <key> — 读取 YAML 列表项
yaml_list() {
    local items
    items=$(awk "/^${1}:/{found=1;next} found && /^[[:space:]]*-/{line=\$0; gsub(/^[[:space:]]*-[[:space:]]*/,\"\",line); gsub(/[[:space:]]*#.*/,\"\",line); print line} found && /^[^[:space:]]/{exit}" "$EFC_CONFIG")
    # 解析 ${VAR:-default} → 取 default 值，并去除首尾引号
    echo "$items" | sed 's/\${[^:]*:-\([^}]*\)}/\1/g' | sed 's/^["'\''"]//; s/["'\''"]$//'
}

# 从 efc.yaml 加载核心配置到全局变量
load_config() {
    [ -f "$EFC_CONFIG" ] || die "配置文件不存在: $EFC_CONFIG"

    CFG_SOCKET=$(yaml_get grpc_socket)
    CFG_DATA_DIR=$(yaml_get data_dir)
    CFG_BACKEND=$(yaml_get backend)
    CFG_CAPACITY_BYTES=$(yaml_get capacity_bytes)
    CFG_MOUNT_UUID=$(yaml_get mount_uuid)
    CFG_DISKS=$(yaml_list disks)

    # 填充默认值
    CFG_SOCKET=${CFG_SOCKET:-/var/run/kvcs/efc-grpc.sock}
    CFG_DATA_DIR=${CFG_DATA_DIR:-/apsarapangu/}
    CFG_BACKEND=${CFG_BACKEND:-mem}
    CFG_CAPACITY_BYTES=${CFG_CAPACITY_BYTES:-0}
    CFG_MOUNT_UUID=${CFG_MOUNT_UUID:-1}
}

# ================================================================
#  tmpfs backend: 部署层挂载与校验
# ================================================================
validate_tmpfs_config() {
    case "$CFG_DATA_DIR" in
        /*) ;;
        *) die "backend=tmpfs 要求 data_dir 是绝对路径: $CFG_DATA_DIR" ;;
    esac
    case "$CFG_DATA_DIR" in
        /|/dev/shm|/dev/shm/)
            die "backend=tmpfs 要求独立挂载目录，不能直接使用 $CFG_DATA_DIR"
            ;;
        *"/../"*|*/..|*"/./"*|*/.)
            die "backend=tmpfs 的 data_dir 不能包含 . 或 .. 路径分量: $CFG_DATA_DIR"
            ;;
    esac
    [[ "$CFG_CAPACITY_BYTES" =~ ^[1-9][0-9]*$ ]] &&
        [ "${#CFG_CAPACITY_BYTES}" -le 18 ] ||
        die "backend=tmpfs 要求 capacity_bytes 是不超过 18 位的正整数字节数"
}

tmpfs_is_mountpoint() {
    mountpoint -q "$1"
}

tmpfs_mount_fstype() {
    stat -f -c '%T' "$1" 2>/dev/null
}

tmpfs_mount_size() {
    local block_size block_count max_signed
    block_size=$(stat -f -c '%S' "$1" 2>/dev/null) || return 1
    block_count=$(stat -f -c '%b' "$1" 2>/dev/null) || return 1
    [[ "$block_size" =~ ^[1-9][0-9]*$ ]] || return 1
    [[ "$block_count" =~ ^[0-9]+$ ]] || return 1

    # Bash arithmetic is signed 64-bit on supported Linux platforms.
    max_signed=9223372036854775807
    [ "$block_count" -le $((max_signed / block_size)) ] || return 1
    printf '%s\n' "$((block_size * block_count))"
}

tmpfs_capacity_matches() {
    local actual="$1" expected="$2" page_size difference
    [[ "$actual" =~ ^[0-9]+$ ]] || return 1
    page_size=$(getconf PAGESIZE 2>/dev/null || echo 4096)
    [[ "$page_size" =~ ^[1-9][0-9]*$ ]] || page_size=4096
    if [ "$actual" -ge "$expected" ]; then
        difference=$((actual - expected))
    else
        difference=$((expected - actual))
    fi
    [ "$difference" -le "$page_size" ]
}

# ensure_tmpfs_mount — 幂等准备独立 tmpfs mount。
# 已有 mount 的类型或容量不匹配时直接失败，不自动 remount，避免隐藏已有数据。
ensure_tmpfs_mount() {
    validate_tmpfs_config
    command -v mountpoint >/dev/null 2>&1 ||
        die "backend=tmpfs 需要 util-linux mountpoint"
    command -v stat >/dev/null 2>&1 ||
        die "backend=tmpfs 需要 coreutils stat"

    if [ -L "$CFG_DATA_DIR" ]; then
        die "backend=tmpfs 的 data_dir 不能是符号链接: $CFG_DATA_DIR"
    fi
    sudo mkdir -p "$CFG_DATA_DIR"

    # mkdir 后再物理规范化，覆盖重复斜杠和祖先符号链接；所有后续操作都只使用
    # 解析后的精确目标，避免诸如 /dev//shm 绕过共享 IPC 目录保护。
    local canonical_dir
    if ! canonical_dir=$(sudo sh -c 'cd -P "$1" && pwd -P' sh "$CFG_DATA_DIR"); then
        die "无法解析 tmpfs data_dir: $CFG_DATA_DIR"
    fi
    case "$canonical_dir" in
        /|/dev/shm)
            die "backend=tmpfs 要求独立挂载目录，不能直接使用 $canonical_dir"
            ;;
    esac
    CFG_DATA_DIR="$canonical_dir"

    local fstype mounted_size first_entry
    if tmpfs_is_mountpoint "$CFG_DATA_DIR"; then
        if ! fstype=$(tmpfs_mount_fstype "$CFG_DATA_DIR"); then
            die "无法检查 tmpfs 文件系统类型: $CFG_DATA_DIR"
        fi
        [ "$fstype" = "tmpfs" ] ||
            die "$CFG_DATA_DIR 已是 $fstype 挂载点，期望 tmpfs"
        if ! mounted_size=$(tmpfs_mount_size "$CFG_DATA_DIR"); then
            die "无法检查 tmpfs 容量: $CFG_DATA_DIR"
        fi
        tmpfs_capacity_matches "$mounted_size" "$CFG_CAPACITY_BYTES" ||
            die "$CFG_DATA_DIR tmpfs 容量不匹配: mounted=$mounted_size configured=$CFG_CAPACITY_BYTES"
        sudo chmod 0750 "$CFG_DATA_DIR"
        log "  ✓ tmpfs ready: $CFG_DATA_DIR (${mounted_size} bytes)"
        return 0
    fi

    if ! first_entry=$(sudo find "$CFG_DATA_DIR" -mindepth 1 -maxdepth 1 -print -quit); then
        die "无法检查 tmpfs 挂载目录内容: $CFG_DATA_DIR"
    fi
    [ -z "$first_entry" ] ||
        die "拒绝在非空目录上挂载 tmpfs（会隐藏数据）: $CFG_DATA_DIR"

    log "  mounting dedicated tmpfs: $CFG_DATA_DIR (${CFG_CAPACITY_BYTES} bytes) ..."
    if ! sudo mount -t tmpfs \
        -o "size=${CFG_CAPACITY_BYTES},mode=0750,nodev,nosuid,noexec" \
        kvcs-tmpfs "$CFG_DATA_DIR"; then
        die "tmpfs mount 失败: $CFG_DATA_DIR"
    fi

    if ! tmpfs_is_mountpoint "$CFG_DATA_DIR"; then
        sudo umount "$CFG_DATA_DIR" >/dev/null 2>&1 || true
        die "tmpfs mount 后校验失败: $CFG_DATA_DIR 不是独立挂载点"
    fi
    if ! fstype=$(tmpfs_mount_fstype "$CFG_DATA_DIR"); then
        sudo umount "$CFG_DATA_DIR" >/dev/null 2>&1 || true
        die "tmpfs mount 后无法检查文件系统类型: $CFG_DATA_DIR"
    fi
    if [ "$fstype" != "tmpfs" ]; then
        sudo umount "$CFG_DATA_DIR" >/dev/null 2>&1 || true
        die "tmpfs mount 后校验失败: $CFG_DATA_DIR fstype=${fstype:-unknown}"
    fi
    if ! mounted_size=$(tmpfs_mount_size "$CFG_DATA_DIR"); then
        sudo umount "$CFG_DATA_DIR" >/dev/null 2>&1 || true
        die "tmpfs mount 后无法检查容量: $CFG_DATA_DIR"
    fi
    if ! tmpfs_capacity_matches "$mounted_size" "$CFG_CAPACITY_BYTES"; then
        sudo umount "$CFG_DATA_DIR" >/dev/null 2>&1 || true
        die "tmpfs mount 后容量校验失败: mounted=$mounted_size configured=$CFG_CAPACITY_BYTES"
    fi
    log "  ✓ tmpfs mounted: $CFG_DATA_DIR (${mounted_size} bytes)"
}

# ================================================================
#  format-disk: 用 blkdiscard + spdk_fs_tool 幂等格式化 NVMe 盘
#  已格式化则跳过；单盘失败记录后继续处理其余磁盘，全部结束后汇总。
# ================================================================
format_disk_if_needed() {
    local spdk_tool
    if [ -x /usr/bin/spdk_fs_tool ]; then
        spdk_tool="/usr/bin/spdk_fs_tool"
    elif [ -x "$KVCS_ROOT/spdk_fs_tool" ]; then
        spdk_tool="$KVCS_ROOT/spdk_fs_tool"
    else
        log "  ⏭ spdk_fs_tool not found, skip format"
        return
    fi
    if [ -z "$CFG_DISKS" ]; then
        log "  ⏭ no disks configured, skip format"
        return
    fi

    local failed_disks=""
    for dev in $CFG_DISKS; do
        if ! is_block_device "$dev"; then
            log "  ✗ $dev is not a block device, skipped"
            failed_disks="$failed_disks $dev"
            continue
        fi
        if sudo "$spdk_tool" --check --addr="$dev" --access_mode=2 --partition=0 --enable_lvm_start >/dev/null 2>&1; then
            log "  ⏭ $dev already formatted, skip"
            continue
        fi
        log "  blkdiscard (full-disk trim) $dev ..."
        if ! sudo blkdiscard "$dev"; then
            log "  ✗ blkdiscard $dev failed"
            failed_disks="$failed_disks $dev"
            continue
        fi
        log "  ✓ $dev trimmed"

        log "  Formatting $dev ..."
        if ! sudo "$spdk_tool" \
            --format --addr="$dev" --access_mode=2 --alloc_block_size=256 --partition=0 --force --enable_lvm_start 2>&1; then
            log "  ✗ format $dev failed (spdk_fs_tool returned non-zero)"
            failed_disks="$failed_disks $dev"
            continue
        fi
        log "  ✓ $dev formatted"

        # Verify the disk is recognized as formatted after formatting.
        if ! sudo "$spdk_tool" --check --addr="$dev" --access_mode=2 --partition=0 >/dev/null 2>&1; then
            log "  ✗ $dev format verification failed (spdk_fs_tool --check did not pass)"
            failed_disks="$failed_disks $dev"
            continue
        fi
        log "  ✓ $dev format verified"
    done

    if [ -n "$failed_disks" ]; then
        die "init-disks failed for the following disks:$failed_disks"
    fi
}

# ================================================================
#  install-systemd: 安装 unit 文件到 /etc/systemd/system/
# ================================================================
cmd_install_systemd() {
    log ">>> install-systemd"
    load_config

    # --- EFC unit ---
    local efc_unit="$SYSTEMD_DIR/${SVC_EFC}.service"
    log "  installing $SVC_EFC.service ..."
    sudo tee "$efc_unit" >/dev/null <<EOF
[Unit]
Description=KVCacheStore EFC Server
After=network.target

[Service]
Type=simple
User=root
Group=root
ExecStartPre=$KVCS_CTL_BIN efc-prestart
ExecStart=$EFC_BIN --config=$EFC_CONFIG
ExecStartPost=$KVCS_CTL_BIN efc-poststart
WorkingDirectory=$KVCS_ROOT
Restart=on-failure
RestartSec=5s
LimitNOFILE=65536
LimitMEMLOCK=infinity
LimitCORE=infinity
StandardOutput=journal
StandardError=journal
SyslogIdentifier=kvcs-efc

[Install]
WantedBy=multi-user.target
EOF
    log "  ✓ $SVC_EFC.service installed"

    # --- Master unit (二进制存在才装) ---
    if [ -x "$MASTER_BIN" ]; then
        local master_unit="$SYSTEMD_DIR/${SVC_MASTER}.service"
        log "  installing $SVC_MASTER.service ..."
        sudo tee "$master_unit" >/dev/null <<EOF
[Unit]
Description=KVCacheStore Master
After=network.target

[Service]
Type=simple
User=root
Group=root
ExecStart=$MASTER_BIN -config $MASTER_CONFIG
WorkingDirectory=$KVCS_ROOT
Restart=on-failure
RestartSec=5s
LimitNOFILE=65536
StandardOutput=journal
StandardError=journal
SyslogIdentifier=kvcs-master

[Install]
WantedBy=multi-user.target
EOF
        log "  ✓ $SVC_MASTER.service installed"
    else
        log "  ⏭ $MASTER_BIN not found, skip $SVC_MASTER.service"
    fi

    # --- Node-Manager unit (二进制存在才装) ---
    if [ -x "$NM_BIN" ]; then
        local nm_unit="$SYSTEMD_DIR/${SVC_NM}.service"
        log "  installing $SVC_NM.service ..."
        sudo tee "$nm_unit" >/dev/null <<EOF
[Unit]
Description=KVCacheStore Node-Manager
After=network.target

[Service]
Type=simple
User=root
Group=root
ExecStart=$NM_BIN -config $NM_CONFIG
WorkingDirectory=$KVCS_ROOT
Restart=on-failure
RestartSec=5s
LimitNOFILE=65536
StandardOutput=journal
StandardError=journal
SyslogIdentifier=kvcs-node-manager

[Install]
WantedBy=multi-user.target
EOF
        log "  ✓ $SVC_NM.service installed"
    else
        log "  ⏭ $NM_BIN not found, skip $SVC_NM.service"
    fi

    # --- daemon-reload + enable ---
    sudo systemctl daemon-reload
    log "  ✓ daemon-reload"
    sudo systemctl enable "$SVC_EFC" 2>/dev/null || true
    [ -x "$MASTER_BIN" ] && sudo systemctl enable "$SVC_MASTER" 2>/dev/null || true
    [ -x "$NM_BIN" ] && sudo systemctl enable "$SVC_NM" 2>/dev/null || true
    log "  ✓ services enabled"

    log "<<< install-systemd done"
}

# ================================================================
#  efc-prestart: ExecStartPre 钩子
#  恢复 tmpfs mount + 清信号量 + 建 socket 目录
#  注意: 不含磁盘格式化（需手动执行 kvcs-ctl init-disks 或 reset-efc）
# ================================================================
cmd_efc_prestart() {
    log ">>> efc-prestart"
    load_config

    if [ "$CFG_BACKEND" = "tmpfs" ]; then
        ensure_tmpfs_mount
    fi

    # 只清理本节点残留信号量（防止 register_iov EEXIST）
    # 严禁删除 IOV/IOR 共享内存段：EFC epoch 重连机制依赖同名 SHM 恢复链路
    log "  cleaning stale semaphores (keep IOV/IOR SHM for epoch reconnect) ..."
    sudo rm -f /dev/shm/sem.kvc-s${CFG_MOUNT_UUID}* 2>/dev/null || true
    log "  ✓ semaphores cleaned"

    # 确保 socket 目录存在
    local socket_dir
    socket_dir=$(dirname "$CFG_SOCKET")
    if [ ! -d "$socket_dir" ]; then
        sudo mkdir -p "$socket_dir"
        sudo chmod 777 "$socket_dir"
        log "  ✓ created $socket_dir"
    fi

    log "<<< efc-prestart done"
}

# ================================================================
#  init-disks: 按 backend 初始化存储。
#  ⚠️ disk backend 会先 blkdiscard 再格式化配置的磁盘，确认磁盘路径无误后再执行。
#  单盘失败记录后继续处理其余磁盘，全部结束后汇总失败盘。
# ================================================================
cmd_init_disks() {
    log ">>> init-disks"
    load_config

    if [ "$CFG_BACKEND" = "tmpfs" ]; then
        ensure_tmpfs_mount
        log "  ✓ tmpfs mounted"
        log "<<< init-disks done"
        return 0
    fi
    [ "$CFG_BACKEND" = "disk" ] ||
        die "init-disks 仅支持 disk/tmpfs backend；当前 backend=$CFG_BACKEND"

    if [ -z "$CFG_DISKS" ]; then
        die "未从 $EFC_CONFIG 读到 disks 配置"
    fi

    # 若 EFC 仍在运行，提示先停止；init-disks 会写盘，在线执行可能引发数据面异常
    if systemctl is-active --quiet "$SVC_EFC" 2>/dev/null || pgrep -x "$(basename "$EFC_BIN")" >/dev/null 2>&1; then
        log "  ⚠️  WARNING: $SVC_EFC 仍在运行，建议先执行 stop-efc 后再 init-disks"
    fi

    log "  ⚠️  WARNING: init-disks 将对以下尚未格式化的磁盘执行 blkdiscard + format："
    log "    config: $EFC_CONFIG (backend=$CFG_BACKEND)"
    for dev in $CFG_DISKS; do
        if is_block_device "$dev"; then
            log "         - $dev"
        else
            log "         - $dev  (非块设备)"
        fi
    done
    echo ""

    if [ "$ASSUME_YES" = true ]; then
        log "  --yes 已指定，跳过确认"
    else
        local confirm=""
        read -rp "  Continue? [y/N] " confirm || true
        if [[ "$confirm" != "y" && "$confirm" != "Y" ]]; then
            log "  Aborted."
            return 1
        fi
    fi

    format_disk_if_needed
    log "  ✓ init-disks 完成"
    log "<<< init-disks done"
}

# ================================================================
#  efc-poststart: ExecStartPost 钩子
#  等 socket 就绪 + chmod 777
# ================================================================
cmd_efc_poststart() {
    log ">>> efc-poststart"
    load_config

    local max_wait=30
    local waited=0
    while [ "$waited" -lt "$max_wait" ]; do
        if [ -S "$CFG_SOCKET" ]; then
            sudo chmod 777 "$CFG_SOCKET"
            log "  ✓ socket ready: $CFG_SOCKET (chmod 777)"
            log "<<< efc-poststart done"
            return 0
        fi
        sleep 2
        waited=$((waited + 2))
        log "  waiting for socket ... (${waited}s)"
    done
    log "  ✗ socket NOT ready after ${max_wait}s"
    # ExecStartPost 返回非 0 会让 systemd 标记服务为 failed
    # 但 socket 未就绪不一定是致命错误，返回 0 避免误杀
    return 0
}

# ================================================================
#  start-efc: systemctl start kvcs-efc
# ================================================================
cmd_start_efc() {
    log ">>> start-efc"
    ensure_unit "$SVC_EFC"
    sudo systemctl start "$SVC_EFC"
    if systemctl is-active --quiet "$SVC_EFC"; then
        log "  ✓ $SVC_EFC active"
    else
        log "  ✗ $SVC_EFC failed"
        journalctl -u "$SVC_EFC" --no-pager -n 10 | sed 's/^/    /'
        return 1
    fi
    log "<<< start-efc done"
}

# ================================================================
#  stop-efc: systemctl stop kvcs-efc
# ================================================================
cmd_stop_efc() {
    log ">>> stop-efc"
    ensure_unit "$SVC_EFC"
    sudo systemctl stop "$SVC_EFC"
    log "<<< stop-efc done"
}

# ================================================================
#  reset-efc: 要求 EFC 已停止；清 SHM + 全盘trim + format
#  仅适用于 disk backend，避免把其他 backend 误带入破坏性磁盘流程。
#  要求 EFC 必须先停止；如果仍在运行则拒绝执行，防止在线破坏数据。
# ================================================================
cmd_reset_efc() {
    log ">>> reset-efc"
    load_config

    [ "$CFG_BACKEND" = "disk" ] ||
        die "reset-efc 仅支持 disk backend；当前 backend=$CFG_BACKEND"

    if [ -z "$CFG_DISKS" ]; then
        die "未从 $EFC_CONFIG 读到 disks 配置"
    fi

    if systemctl is-active --quiet "$SVC_EFC" 2>/dev/null; then
        die "$SVC_EFC 仍在运行，请先执行 stop-efc 后再 reset"
    fi
    if pgrep -x "$(basename "$EFC_BIN")" >/dev/null 2>&1; then
        die "$EFC_BIN 进程仍在运行，请先停止 EFC 后再 reset"
    fi

    local spdk_tool=""
    if [ -x /usr/bin/spdk_fs_tool ]; then
        spdk_tool="/usr/bin/spdk_fs_tool"
    elif [ -x "$KVCS_ROOT/spdk_fs_tool" ]; then
        spdk_tool="$KVCS_ROOT/spdk_fs_tool"
    fi

    # 全盘 trim + 强制格式化不可恢复，先把计划和盘符打出来让操作者核对
    log "  ⚠️  DANGER: reset-efc 将执行以下不可恢复的操作"
    log "    config: $EFC_CONFIG (backend=$CFG_BACKEND)"
    log "    1) 删除 /dev/shm 下 EFC 共享内存与信号量 (kvcs_*, sem.kvc-s${CFG_MOUNT_UUID}*)"
    log "    2) 清空数据目录 $CFG_DATA_DIR/ 下全部内容"
    log "    3) blkdiscard 全盘 TRIM:"
    for dev in $CFG_DISKS; do
        if is_block_device "$dev"; then
            log "         - $dev"
        else
            log "         - $dev  (非块设备)"
        fi
    done
    if [ -n "$spdk_tool" ]; then
        log "    4) $spdk_tool --format --force 强制格式化:"
        for dev in $CFG_DISKS; do
            if is_block_device "$dev"; then
                log "         - $dev"
            else
                log "         - $dev  (非块设备)"
            fi
        done
    else
        log "    4) 强制格式化: 跳过（未找到 spdk_fs_tool）"
    fi
    echo ""

    if [ "$ASSUME_YES" = true ]; then
        log "  --yes 已指定，跳过确认"
    else
        local confirm=""
        read -rp "  盘符确认无误？输入 yes 继续（其他任意输入取消）: " confirm || true
        if [ "$confirm" != "yes" ]; then
            log "  Aborted."
            return 1
        fi
    fi

    # 1. 清理 /dev/shm 下残留的 EFC 共享内存段和信号量
    log "  cleaning stale SHM segments and semaphores ..."
    sudo rm -f /dev/shm/kvcs_iov_* /dev/shm/kvcs_ior_* 2>/dev/null || true
    sudo rm -f /dev/shm/kvcs_* 2>/dev/null || true
    sudo rm -f /dev/shm/sem.kvc-s${CFG_MOUNT_UUID}* 2>/dev/null || true
    log "  ✓ SHM cleaned"

    # 1.5 清理数据目录下所有残留数据
    log "  cleaning all data in $CFG_DATA_DIR ..."
    sudo rm -rf "${CFG_DATA_DIR:?}"/* 2>/dev/null || true
    sudo rm -rf "${CFG_DATA_DIR:?}"/.[!.]* 2>/dev/null || true
    log "  ✓ data dir cleaned"

    # 2. blkdiscard 全盘 TRIM（NVMe DSM Deallocate），让 SSD 回收所有块
    failed_disks=""
    for dev in $CFG_DISKS; do
        if is_block_device "$dev"; then
            log "  blkdiscard (full-disk trim) $dev ..."
            if ! sudo blkdiscard "$dev"; then
                log "  ✗ blkdiscard $dev 失败"
                failed_disks="$failed_disks $dev"
                continue
            fi
            log "  ✓ $dev trimmed"
        else
            log "  ✗ $dev 不是块设备，跳过"
            failed_disks="$failed_disks $dev"
        fi
    done

    # 3. 强制格式化磁盘（单盘失败记录后继续处理其余磁盘，全部结束后汇总失败盘）
    #    跳过 blkdiscard 阶段已失败的盘。
    if [ -n "$spdk_tool" ]; then
        for dev in $CFG_DISKS; do
            if ! is_block_device "$dev"; then
                log "  ⏭ $dev 非块设备，跳过格式化"
                continue
            fi
            case " $failed_disks " in
                *" $dev "*)
                    log "  ⏭ $dev blkdiscard 失败，跳过格式化"
                    continue
                    ;;
            esac
            log "  Formatting $dev ..."
            if ! sudo "$spdk_tool" \
                --format --addr="$dev" --access_mode=2 --alloc_block_size=256 --partition=0 --force --enable_lvm_start 2>&1; then
                log "  ✗ 格式化 $dev 失败（spdk_fs_tool 返回非零）"
                failed_disks="$failed_disks $dev"
                continue
            fi
            log "  ✓ $dev formatted"

            # 格式化成功后校验，确保盘可被识别为已格式化
            if ! sudo "$spdk_tool" --check --addr="$dev" --access_mode=2 --partition=0 >/dev/null 2>&1; then
                log "  ✗ $dev 格式化校验失败（spdk_fs_tool --check 未通过）"
                failed_disks="$failed_disks $dev"
                continue
            fi
            log "  ✓ $dev format verified"
        done
        if [ -n "$failed_disks" ]; then
            die "以下磁盘 reset 失败:$failed_disks"
        fi
    else
        log "  ⚠ spdk_fs_tool 不存在，跳过格式化"
    fi

    log "  ✓ reset 完成"
    log "<<< reset-efc done"
}

# ================================================================
#  start-master: systemctl start kvcs-master
# ================================================================
cmd_start_master() {
    log ">>> start-master"
    ensure_unit "$SVC_MASTER"
    sudo systemctl start "$SVC_MASTER"
    service_health_check "master" "$SVC_MASTER" "$MASTER_CONFIG" "127.0.0.1:9191" || return 1
    log "<<< start-master done"
}

# ================================================================
#  stop-master: systemctl stop kvcs-master
# ================================================================
cmd_stop_master() {
    log ">>> stop-master"
    ensure_unit "$SVC_MASTER"
    sudo systemctl stop "$SVC_MASTER"
    log "<<< stop-master done"
}

# ================================================================
#  start-nm: systemctl start kvcs-node-manager
# ================================================================
cmd_start_nm() {
    log ">>> start-nm"
    ensure_unit "$SVC_NM"
    sudo systemctl start "$SVC_NM"
    service_health_check "node-manager" "$SVC_NM" "$NM_CONFIG" "127.0.0.1:9192" || return 1
    log "<<< start-nm done"
}

# ================================================================
#  stop-nm: systemctl stop kvcs-node-manager
# ================================================================
cmd_stop_nm() {
    log ">>> stop-nm"
    ensure_unit "$SVC_NM"
    sudo systemctl stop "$SVC_NM"
    log "<<< stop-nm done"
}

# ================================================================
#  status: 显示全部服务状态
# ================================================================
cmd_status() {
    log ">>> status"

    # EFC
    local efc_state
    efc_state=$(systemctl is-active "$SVC_EFC" 2>/dev/null || true)
    log "  EFC: ${efc_state:-unknown}"

    # Master
    local master_state
    master_state=$(systemctl is-active "$SVC_MASTER" 2>/dev/null || true)
    if [ "$master_state" = "active" ]; then
        local master_readyz
        master_readyz=$(health_url "$MASTER_CONFIG" "127.0.0.1:9191")
        if http_probe "$master_readyz"; then
            log "  Master: READY"
        else
            log "  Master: active, NOT READY"
        fi
    else
        log "  Master: ${master_state:-unknown}"
    fi

    # Node-Manager
    local nm_state
    nm_state=$(systemctl is-active "$SVC_NM" 2>/dev/null || true)
    if [ "$nm_state" = "active" ]; then
        local nm_readyz
        nm_readyz=$(health_url "$NM_CONFIG" "127.0.0.1:9192")
        if http_probe "$nm_readyz"; then
            log "  Node-Manager: READY"
        else
            log "  Node-Manager: active, NOT READY"
        fi
    else
        log "  Node-Manager: ${nm_state:-unknown}"
    fi

    log "<<< status done"
}

# ================================================================
#  使用说明
# ================================================================
usage() {
    cat <<'EOF'
kvcs-ctl — KVCS 服务原子操作控制器（systemd 托管版）

用法: kvcs-ctl [选项] <command> [command2 ...]

选项:
  -y, --yes          跳过破坏性命令（reset-efc）的交互确认
                     等价于环境变量 KVCS_ASSUME_YES=true

Systemd 安装:
  install-systemd    安装 unit 文件到 /etc/systemd/system/ 并 enable

EFC 命令:
  start-efc          systemctl start kvcs-efc
  stop-efc           systemctl stop kvcs-efc
  init-disks         初始化后端存储（tmpfs mount / disk blkdiscard + 格式化）
  reset-efc          重置 disk EFC（要求 EFC 已停止；清 SHM + 全盘trim + 强制 format）
                     执行前会打印计划与待格式化盘符，需输入 yes 确认

Master / Node-Manager 命令:
  start-master       systemctl start kvcs-master
  stop-master        systemctl stop kvcs-master
  start-nm           systemctl start kvcs-node-manager
  stop-nm            systemctl stop kvcs-node-manager

通用:
  status             显示全部服务状态
  help               显示本帮助信息

systemd 钩子（自动调用，一般不手动执行）:
  efc-prestart       ExecStartPre: 恢复 tmpfs mount + 清信号量 + 建 socket 目录
  efc-poststart      ExecStartPost: 等 socket 就绪 + chmod 777

组合示例:
  kvcs-ctl install-systemd                    # 首次安装（tarball 模式）
  kvcs-ctl start-efc start-master start-nm    # 启动全部
  kvcs-ctl stop-nm stop-master stop-efc       # 停止全部
  kvcs-ctl -y reset-efc                       # 非交互重置（跳过确认）
EOF
}

# ================================================================
#  Main — 按顺序执行传入的命令
# ================================================================
main() {
    if [ $# -eq 0 ]; then
        usage
        return 0
    fi

    local cmd status
    for cmd in "$@"; do
        case "$cmd" in
            install-systemd) cmd_install_systemd ;;
            start-efc)       cmd_start_efc ;;
            stop-efc)        cmd_stop_efc ;;
            reset-efc)       cmd_reset_efc ;;
            start-master)    cmd_start_master ;;
            stop-master)     cmd_stop_master ;;
            start-nm)        cmd_start_nm ;;
            stop-nm)         cmd_stop_nm ;;
            status)          cmd_status ;;
            init-disks)        cmd_init_disks ;;
            efc-prestart)     cmd_efc_prestart ;;
            efc-poststart)    cmd_efc_poststart ;;
            -y|--yes) ASSUME_YES=true ;;
            -h|--help|help) usage; return 0 ;;
            *)
                log "ERROR: unknown command '$cmd'"
                usage
                return 1
                ;;
        esac
        status=$?
        if [ "$status" -ne 0 ]; then
            log "=== FAILED: $cmd (exit $status) ==="
            return "$status"
        fi
    done

    log "=== SUCCESS ==="
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
    main "$@"
fi
