feat: v1.4.0 — 同步失败自动重试（每 60s 一次，次数可调默认 5）

- 独立重试线程：整轮同步有仓库失败时，把失败仓库排入重试队列，每隔 60 秒并行重试一次（复用并行度设置），与同步间隔无关—— 即使同步间隔是 1 天，网络抖动也能在几分钟内自愈 - 重试计数：每个仓库连续失败计数，达到上限（默认 5，可调 0~20， 0=不重试）后放弃并记日志，等下一轮同步或手动同步再次尝试；手动同步成功立即退出重试队列；被其他同步占用（busy）不计次数 - 设置页新增「失败重试次数」；状态页新增「失败重试」统计卡片，重试中的仓库显示「⏳ 重试中 x/N」；日志记录排程/重试/放弃全过程 - 修复 retry_next_at 缺少 global 声明导致重试每 2 秒空转的 bug （线上实测发现：next_in 显示 0 而非 60） - manifest 1.3.0 → 1.4.0
2026-10-13 20:32:46 +08:00
parent 1acd60894f
commit 62f059921d
3 changed files with 187 additions and 9 deletions
@@ -54,6 +54,10 @@ MAX_INTERVAL = 86400               # 最大同步间隔（秒，1 天）
 DEFAULT_WORKERS = 3                # 同时同步的项目数默认值
 MIN_WORKERS = 1
 MAX_WORKERS = 8
+RETRY_INTERVAL = 60                # 同步失败后的自动重试间隔（秒）
+DEFAULT_MAX_RETRIES = 5            # 失败重试次数默认值
+MIN_MAX_RETRIES = 0                # 0 = 不重试
+MAX_MAX_RETRIES = 20
 PANEL_PORT = int(os.environ.get("PANEL_PORT", "8899"))
 START_TS = time.time()
 NAME_RE = re.compile(r"^[A-Za-z0-9._-]{1,100}$")
@@ -67,7 +71,7 @@ ENV["GIT_TERMINAL_PROMPT"] = "0"
 # ---------------- 应用设置（settings.json） ----------------
 cfg = {"gitea_user": LEGACY_GITEA_USER, "gitea_pass": "", "auto_create": True,
       "interval": DEFAULT_INTERVAL, "gitea_host": DEFAULT_GITEA_HOST,
-       "max_workers": DEFAULT_WORKERS}
+       "max_workers": DEFAULT_WORKERS, "max_retries": DEFAULT_MAX_RETRIES}


 def norm_gitea_host(h):
@@ -141,6 +145,22 @@ def valid_workers(v):
        return False


+def max_retries():
+    """失败自动重试次数，始终返回合法值"""
+    try:
+        n = int(cfg.get("max_retries", DEFAULT_MAX_RETRIES))
+    except Exception:
+        n = DEFAULT_MAX_RETRIES
+    return min(MAX_MAX_RETRIES, max(MIN_MAX_RETRIES, n))
+
+
+def valid_retries(v):
+    try:
+        return MIN_MAX_RETRIES <= int(v) <= MAX_MAX_RETRIES
+    except Exception:
+        return False
+
+
 def load_settings():
    try:
        d = json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
@@ -152,6 +172,8 @@ def load_settings():
            cfg["interval"] = int(d["interval"])
        if valid_workers(d.get("max_workers")):
            cfg["max_workers"] = int(d["max_workers"])
+        if valid_retries(d.get("max_retries")):
+            cfg["max_retries"] = int(d["max_retries"])
        h = norm_gitea_host(d.get("gitea_host"))
        if h:
            cfg["gitea_host"] = h
@@ -263,7 +285,7 @@ def scrub(s, acc):

 state = {"enabled": True, "syncing": False, "last_check": None, "checks": 0,
         "last_result": "尚未运行", "loop_last": None}
-stats = {"total_checks": 0, "total_pushes": 0, "total_failures": 0}
+stats = {"total_checks": 0, "total_pushes": 0, "total_failures": 0, "total_retries": 0}
 op_lock = threading.Lock()          # 整轮同步互斥（同一轮只跑一次）
 stats_lock = threading.Lock()       # 保护 stats 计数
 log_lock = threading.Lock()         # 保护日志文件写入
@@ -511,7 +533,7 @@ def fingerprint(repo):


 def sync_repo(name):
-    """返回 'changed' / 'same' / 'error:...'；同一仓库并发调用时后到者直接返回"""
+    """返回 'changed' / 'same' / 'busy' / 'error:...'；同一仓库并发调用时后到者返回 busy"""
    if not valid_name(name):
        return "error: 非法仓库名"
    acc = active_account()
@@ -519,7 +541,7 @@ def sync_repo(name):
        return "error: 没有可用的 GitHub 账号，请到设置页添加"
    with repo_busy_lock:
        if name in repo_in_progress:
-            return "error: 该仓库正在同步中，请稍候"
+            return "busy"
        repo_in_progress.add(name)
    try:
        return _sync_repo_locked(name, acc)
@@ -603,6 +625,13 @@ def run_sync(repos=None):
                errs.append(f"{r} {res}")
            elif res == "changed":
                changed += 1
+            # 'same' / 'busy' 不计失败
+        failed_names = [r for r in repos if results.get(r, "").startswith("error")]
+        with retry_lock:
+            for r in repos:
+                if r not in failed_names:
+                    retry_queue.pop(r, None)   # 成功的仓库退出重试队列
+        schedule_retries(failed_names)
        if errs:
            bump_stats(total_failures=len(errs))
        if changed:
@@ -622,6 +651,102 @@ def run_sync(repos=None):
        state["syncing"] = False


+# ---------------- 失败自动重试（独立于同步间隔，默认每 60s 一次） ----------------
+retry_lock = threading.Lock()
+retry_queue = {}        # {仓库名: 已重试次数}
+retry_next_at = 0.0     # 下次重试时间戳
+
+
+def clear_retry(name):
+    with retry_lock:
+        retry_queue.pop(name, None)
+
+
+def schedule_retries(failed):
+    """整轮同步有失败时，把失败仓库排入自动重试（每轮重新计数）"""
+    global retry_next_at
+    mr = max_retries()
+    if mr <= 0 or not failed:
+        return
+    with retry_lock:
+        for name in failed:
+            retry_queue.setdefault(name, 0)
+        retry_next_at = time.time() + RETRY_INTERVAL
+    log(f"同步失败，已排程自动重试: {', '.join(failed)}"
+        f"（每 {RETRY_INTERVAL} 秒一次，最多 {mr} 次）")
+
+
+def retry_snapshot():
+    """重试状态（供前端展示）"""
+    with retry_lock:
+        if not retry_queue:
+            return {"pending": [], "attempts": {}, "max": max_retries(), "next_in": None}
+        return {"pending": sorted(retry_queue.keys()),
+                "attempts": dict(retry_queue),
+                "max": max_retries(),
+                "next_in": max(0, int(retry_next_at - time.time()))}
+
+
+def retry_round():
+    """跑一轮重试：并行重试队列中的仓库；成功移出，busy 不计数，失败计数+1，超上限放弃"""
+    global retry_next_at
+    with retry_lock:
+        batch = dict(retry_queue)
+    if not batch:
+        return
+    results = {}
+    workers = max(1, min(sync_workers(), len(batch)))
+
+    def worker(name):
+        try:
+            results[name] = sync_repo(name)
+        except Exception as e:
+            results[name] = f"error: {e}"
+
+    with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="ghs-retry") as ex:
+        list(ex.map(worker, batch))
+    mr = max_retries()
+    ok_names, failed, busy, gave_up = [], [], [], []
+    with retry_lock:
+        for name, res in results.items():
+            if res in ("changed", "same"):
+                retry_queue.pop(name, None)
+                ok_names.append(name)
+            elif res == "busy":
+                busy.append(name)            # 被其他同步占用，不计数
+            else:
+                n = batch.get(name, 0) + 1
+                if n >= mr:
+                    retry_queue.pop(name, None)
+                    gave_up.append((name, n))
+                else:
+                    retry_queue[name] = n
+                    failed.append((name, n))
+        attempts = len(results) - len(busy)
+        if retry_queue:
+            retry_next_at = time.time() + RETRY_INTERVAL
+    if attempts:
+        bump_stats(total_retries=attempts)
+    if ok_names:
+        log(f"重试成功: {', '.join(ok_names)}")
+    for name, n in failed:
+        log(f"重试失败: {name}（第 {n}/{mr} 次，{RETRY_INTERVAL} 秒后再试）")
+    for name, n in gave_up:
+        log(f"重试 {n} 次仍失败，放弃自动重试: {name}（下一轮同步或手动同步会再次尝试）")
+
+
+def retry_loop():
+    while True:
+        time.sleep(2)
+        try:
+            with retry_lock:
+                due = bool(retry_queue) and time.time() >= retry_next_at
+            if due:
+                retry_round()
+        except Exception as e:
+            log(f"重试线程异常: {e}")
+
+
 def loop():
    while True:
        state["loop_last"] = time.time()
@@ -716,6 +841,7 @@ def global_status(ui_url=None):
        "uptime": fmt_uptime(now - START_TS),
        "interval": sync_interval(),
        "max_workers": sync_workers(),
+        "retry": retry_snapshot(),
        "last_check": state["last_check"],
        "next_in": next_in,
        "checks": state["checks"],
@@ -962,6 +1088,7 @@ PAGE_STATUS = """
      <div class="stat"><div class="k">累计检查轮数</div><div class="v" id="sChecks">—</div></div>
      <div class="stat"><div class="k">累计实际同步</div><div class="v" id="sPushes">—</div></div>
      <div class="stat"><div class="k">累计失败次数</div><div class="v" id="sFails">—</div></div>
+      <div class="stat"><div class="k">失败重试</div><div class="v" id="sRetry">—</div></div>
      <div class="stat"><div class="k">镜像总占用</div><div class="v" id="sSize">—</div></div>
    </div>
  </div>
@@ -992,12 +1119,19 @@ async function refresh(){
    document.getElementById('sChecks').textContent=s.stats.total_checks;
    document.getElementById('sPushes').textContent=s.stats.total_pushes;
    document.getElementById('sFails').textContent=s.stats.total_failures;
+    const rt=s.retry||{};
+    document.getElementById('sRetry').textContent=(rt.pending&&rt.pending.length)
+      ?(rt.pending.length+' 个待重试 · '+rt.next_in+'s后（最多'+rt.max+'次）')
+      :'无待重试';
    document.getElementById('sSize').textContent=s.total_size_mb+' MB';
    document.getElementById('sTarget').textContent='同步目标: '+(s.account?s.account.gh_user+'（'+s.account.method+'）':'未配置')+' · '+(s.max_workers||1)+' 个并行 · Gitea '+(s.gitea_online?'在线 ✓':'离线 ✗');
    document.getElementById('tb').innerHTML=s.repos.map(r=>{
-      const st=r.status==='已同步'?'<td class="ok">✓ 已同步</td>':'<td class="warn">⏳ 未同步</td>';
+      const att=(rt.attempts||{})[r.name];
+      const stTxt=r.status==='已同步'?'✓ 已同步':(att!=null?('⏳ 重试中 '+att+'/'+(rt.max||5)):'⏳ 未同步');
+      const stCls=r.status==='已同步'?'ok':'warn';
+      const st=`<td class="${stCls}">${stTxt}</td>`;
      const c=r.last_commit?`<div class="commit">${esc(r.last_commit.hash)} ${esc(r.last_commit.msg)} · ${esc(r.last_commit.author)} · ${esc(r.last_commit.time)}</div>`:'<div class="commit">—</div>';
-      return `<tr><td data-label="项目"><b>${esc(r.name)}</b></td><td data-label="状态" class="${r.status==='已同步'?'ok':'warn'}">${r.status==='已同步'?'✓ 已同步':'⏳ 未同步'}</td><td data-label="默认分支">${esc(r.head||'—')}</td><td data-label="分支/标签">${r.branches} / ${r.tags}</td><td data-label="最近推送同步">${r.last_push||'—'}</td><td data-label="最新提交">${c}</td><td data-label="大小">${r.size_mb} MB</td><td data-label="操作"><button class="b-mini" onclick="syncOne('${esc(r.name)}')">同步</button></td></tr>`;
+      return `<tr><td data-label="项目"><b>${esc(r.name)}</b></td>${st}<td data-label="默认分支">${esc(r.head||'—')}</td><td data-label="分支/标签">${r.branches} / ${r.tags}</td><td data-label="最近推送同步">${r.last_push||'—'}</td><td data-label="最新提交">${c}</td><td data-label="大小">${r.size_mb} MB</td><td data-label="操作"><button class="b-mini" onclick="syncOne('${esc(r.name)}')">同步</button></td></tr>`;
    }).join('')||'<tr><td colspan="8" style="color:#646a73">还没有同步项目，去<a href="manage">项目管理</a>添加</td></tr>';
  }catch(e){document.getElementById('tb').innerHTML='<tr><td colspan="8">刷新失败: '+e+'</td></tr>'}
 }
@@ -1144,6 +1278,14 @@ PAGE_SETTINGS = """
          <div class="swsub" id="workersTip"></div>
        </div>
      </div>
+      <div class="optrow">
+        <input id="syncRetries" type="number" min="0" max="20" step="1" class="numin">
+        <button class="b-mini" id="btnRetries" onclick="saveRetries()">保存次数</button>
+        <div class="opttext">
+          <div class="swlabel">失败重试次数</div>
+          <div class="swsub" id="retriesTip"></div>
+        </div>
+      </div>
    </div>
    <div class="msg" id="msgCfg"></div>
  </div>
@@ -1166,6 +1308,9 @@ async function refresh(){
    const wk=document.getElementById('syncWorkers');
    wk.value=s.max_workers||3;wk.min=s.min_workers||1;wk.max=s.max_workers_limit||8;
    document.getElementById('workersTip').textContent='每轮最多 '+(s.max_workers||3)+' 个项目同时克隆/推送（1~'+(s.max_workers_limit||8)+'），项目多时能显著加快；网络或 NAS 负载高就调小';
+    const rt=document.getElementById('syncRetries');
+    rt.value=(s.max_retries==null?5:s.max_retries);rt.min=0;rt.max=20;
+    document.getElementById('retriesTip').textContent='同步失败后每隔 '+(s.retry_interval||60)+' 秒自动重试失败的仓库，最多 '+(s.max_retries==null?5:s.max_retries)+' 次（0 = 不重试）；与同步间隔无关，专治网络抖动';
    const cur=(s.accounts||[]).find(a=>a.id===s.active);
    // 当前账号无 Token 时，顶部提示为什么不能自动建库、怎么解锁
    document.getElementById('accBanner').innerHTML=(cur&&!cur.has_token)
@@ -1273,6 +1418,15 @@ async function saveWorkers(){
  }catch(e){showMsg('msgCfg','请求失败: '+e,false)}
  finally{b.disabled=false;b.textContent='保存并发';refresh();}
 }
+async function saveRetries(){
+  const v=parseInt(document.getElementById('syncRetries').value,10);
+  const b=document.getElementById('btnRetries');b.disabled=true;b.textContent='⏳ 保存中…';
+  try{
+    const r=await (await fetch('api/settings/save',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({max_retries:v})})).json();
+    showMsg('msgCfg',r.msg||JSON.stringify(r),!!r.ok);
+  }catch(e){showMsg('msgCfg','请求失败: '+e,false)}
+  finally{b.disabled=false;b.textContent='保存次数';refresh();}
+}
 refresh();
 </script>
 """
@@ -1346,6 +1500,8 @@ class Handler(BaseHTTPRequestHandler):
                        "max_workers": sync_workers(),
                        "min_workers": MIN_WORKERS,
                        "max_workers_limit": MAX_WORKERS,
+                        "max_retries": max_retries(),
+                        "retry_interval": RETRY_INTERVAL,
                        "gitea_host": gitea_host(),
                        "gitea_url": gitea_ui(self.headers.get("Host")),
                        "active": active_id,
@@ -1390,6 +1546,10 @@ class Handler(BaseHTTPRequestHandler):
                if not valid_workers(data.get("max_workers")):
                    self._json({"ok": False, "msg": f"同时同步项目数须为 {MIN_WORKERS} ~ {MAX_WORKERS} 之间的整数"})
                    return
+            if "max_retries" in data:
+                if not valid_retries(data.get("max_retries")):
+                    self._json({"ok": False, "msg": f"失败重试次数须为 {MIN_MAX_RETRIES} ~ {MAX_MAX_RETRIES} 之间的整数"})
+                    return
            # 预校验：新地址先探连通性；涉及账号的改动再用新凭据调一次 /user 验证
            trial = dict(cfg)
            if new_user is not None:
@@ -1441,6 +1601,11 @@ class Handler(BaseHTTPRequestHandler):
                cfg["max_workers"] = int(data.get("max_workers"))
                if cfg["max_workers"] != old_w:
                    log(f"同时同步项目数已调整: {old_w} -> {cfg['max_workers']}")
+            if "max_retries" in data:
+                old_r = max_retries()
+                cfg["max_retries"] = int(data.get("max_retries"))
+                if cfg["max_retries"] != old_r:
+                    log(f"失败重试次数已调整: {old_r} -> {cfg['max_retries']}")
            save_settings()
            notes = []
            if new_host is not None:
@@ -1545,13 +1710,17 @@ class Handler(BaseHTTPRequestHandler):
            # 与正在进行的整轮同步互不阻塞，实现多项目同时同步
            res = sync_repo(name)
            state["last_check"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            if res.startswith("error"):
+            if res == "busy":
+                self._json({"ok": False, "msg": f"{name} 正在同步中，请稍候"})
+            elif res.startswith("error"):
                bump_stats(total_failures=1)
                self._json({"ok": False, "msg": f"{name} {res}"})
            elif res == "changed":
                bump_stats(total_pushes=1)
+                clear_retry(name)   # 手动同步成功，退出自动重试队列
                self._json({"ok": True, "msg": f"{name} 已同步到 GitHub"})
            else:
+                clear_retry(name)
                self._json({"ok": True, "msg": f"{name} 已是最新，无需同步"})
        elif path == "/api/repos/add":
            name = (data.get("name") or "").strip()
@@ -1580,12 +1749,16 @@ class Handler(BaseHTTPRequestHandler):
            save_repos(repos)
            log(f"{name}: 已加入同步列表")
            res = sync_repo(name)
-            if res.startswith("error"):
+            if res == "busy":
+                self._json({"ok": True, "msg": f"{name} 添加成功（该仓库正在同步中，稍后刷新看结果）"})
+            elif res.startswith("error"):
                self._json({"ok": False, "msg": f"已加入列表，但首次同步失败：{res}"})
            elif res == "changed":
                bump_stats(total_pushes=1)
+                clear_retry(name)
                self._json({"ok": True, "msg": f"{name} 添加成功，已完成首次同步"})
            else:
+                clear_retry(name)
                self._json({"ok": True, "msg": f"{name} 添加成功，内容已是最新"})
        elif path == "/api/repos/remove":
            name = (data.get("name") or "").strip()
@@ -1595,6 +1768,7 @@ class Handler(BaseHTTPRequestHandler):
                return
            repos = [r for r in repos if r != name]
            save_repos(repos)
+            clear_retry(name)
            log(f"{name}: 已从同步列表移除")
            self._json({"ok": True, "msg": f"{name} 已移除（GitHub 仓库与本地数据保留）"})
        else:
@@ -1642,6 +1816,7 @@ def main():
        f"({acc['gh_user'] if acc else '-'}/{'Token' if acc and acc.get('token') else 'SSH'}), "
        f"Gitea 用户={cfg['gitea_user']}, 地址={gitea_host()}, 自动创建={'开' if cfg.get('auto_create') else '关'})")
    threading.Thread(target=loop, daemon=True).start()
+    threading.Thread(target=retry_loop, daemon=True).start()
    server = ThreadingHTTPServer(("0.0.0.0", PANEL_PORT), Handler)
    log(f"控制面板监听 :{PANEL_PORT}")
    server.serve_forever()
@@ -1,5 +1,5 @@
 appname               = GiteaHubSync
-version               = 1.3.0
+version               = 1.4.0
 display_name          = Gitea→GitHub 同步助手
 desc                  = 每次推送到飞牛 Gitea 的代码，10 秒内自动镜像到 GitHub 的 7 个私有仓库。内置控制面板：自动同步开关、立即同步、仓库状态、同步日志，一键打开 Gitea。数据全部保存在 NAS 本地。
 platform              = all
@@ -16,6 +16,9 @@ NAS 重启自动恢复。控制面板带自动同步开关、立即同步、仓
 - **同步间隔可调**：设置页可改检查间隔（5 秒 ~ 1 天，默认 10 秒），改完立即生效
 - **多项目并行同步**：每轮最多 N 个项目同时克隆/推送（默认 3，可调 1~8）；
  在状态页点单个仓库的「同步」也不再被整轮同步阻塞（同一仓库有防重入）
+- **失败自动重试**：同步失败后每隔 1 分钟自动重试失败的仓库，最多 N 次
+  （默认 5，可调 0~20，0 = 不重试）；重试与同步间隔无关，专治网络抖动，
+  重试中的仓库在状态页显示「重试中 x/N」
 - **Gitea 地址可改**：设置页可改同步引擎访问的 Gitea 地址（默认 `http://127.0.0.1:3000`），
  保存前自动探测连通性；改地址后已有镜像的拉取地址自动更新，「打开 Gitea」按钮随访问地址自适应
 - **自动创建仓库**：Gitea 上的新仓库加入同步时，若 GitHub 上不存在同名仓库，