From 1acd60894fa51e13028adcf105bf16ccb478f4af Mon Sep 17 00:00:00 2001 From: cbb Date: Tue, 13 Oct 2026 20:14:57 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20v1.3.0=20=E2=80=94=20=E5=A4=9A=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E5=90=8C=E6=97=B6=E5=90=8C=E6=AD=A5=EF=BC=88=E5=B9=B6?= =?UTF-8?q?=E8=A1=8C=E5=BA=A6=E5=8F=AF=E8=B0=83=201~8=EF=BC=8C=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=203=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_sync 改线程池并行：每轮最多 N 个项目同时克隆/推送，项目多时 显著缩短单轮耗时（实测 6 个仓库 3 并发 3 倍加速） - 按仓库防重入：同一仓库同时只允许一个同步，后到者提示 「该仓库正在同步中，请稍候」，避免并发 git 操作损坏镜像 - 状态页单仓库「同步」按钮不再被整轮同步全局拦截（旧版报 「已有同步任务正在进行」），与进行中的整轮真正并行 - 新增「同时同步项目数」设置（1~8），设置页可调，立即生效 - 统计计数与日志写入加锁，修正并行下的计数漂移与日志交错风险 - 首页/状态页显示当前并行数；manifest 1.2.0 → 1.3.0 --- GiteaHubSync/app/backend/panel.py | 165 +++++++++++++++++++++++------- GiteaHubSync/manifest | 2 +- README.md | 2 + 3 files changed, 130 insertions(+), 39 deletions(-) diff --git a/GiteaHubSync/app/backend/panel.py b/GiteaHubSync/app/backend/panel.py index 888b0d8..3629fbe 100644 --- a/GiteaHubSync/app/backend/panel.py +++ b/GiteaHubSync/app/backend/panel.py @@ -20,6 +20,7 @@ import subprocess import threading import time import uuid +from concurrent.futures import ThreadPoolExecutor from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -50,6 +51,9 @@ LEGACY_GITEA_USER = "cbb" # 旧版硬编码的 Gitea 用户名，仅作 DEFAULT_INTERVAL = 10 # 同步间隔默认值（秒） MIN_INTERVAL = 5 # 最小同步间隔（秒） MAX_INTERVAL = 86400 # 最大同步间隔（秒，1 天） +DEFAULT_WORKERS = 3 # 同时同步的项目数默认值 +MIN_WORKERS = 1 +MAX_WORKERS = 8 PANEL_PORT = int(os.environ.get("PANEL_PORT", "8899")) START_TS = time.time() NAME_RE = re.compile(r"^[A-Za-z0-9._-]{1,100}$") @@ -62,7 +66,8 @@ ENV["GIT_TERMINAL_PROMPT"] = "0" # ---------------- 应用设置（settings.json） ---------------- cfg = {"gitea_user": LEGACY_GITEA_USER, "gitea_pass": "", "auto_create": True, - "interval": DEFAULT_INTERVAL, "gitea_host": DEFAULT_GITEA_HOST} + "interval": DEFAULT_INTERVAL, "gitea_host": DEFAULT_GITEA_HOST, + "max_workers": DEFAULT_WORKERS} def norm_gitea_host(h): @@ -120,6 +125,22 @@ def valid_interval(v): return False +def sync_workers(): + """同时同步的项目数，始终返回合法值""" + try: + n = int(cfg.get("max_workers", DEFAULT_WORKERS)) + except Exception: + n = DEFAULT_WORKERS + return min(MAX_WORKERS, max(MIN_WORKERS, n)) + + +def valid_workers(v): + try: + return MIN_WORKERS <= int(v) <= MAX_WORKERS + except Exception: + return False + + def load_settings(): try: d = json.loads(SETTINGS_FILE.read_text(encoding="utf-8")) @@ -129,6 +150,8 @@ def load_settings(): cfg["auto_create"] = bool(d.get("auto_create", True)) if valid_interval(d.get("interval")): cfg["interval"] = int(d["interval"]) + if valid_workers(d.get("max_workers")): + cfg["max_workers"] = int(d["max_workers"]) h = norm_gitea_host(d.get("gitea_host")) if h: cfg["gitea_host"] = h @@ -241,17 +264,30 @@ def scrub(s, acc): state = {"enabled": True, "syncing": False, "last_check": None, "checks": 0, "last_result": "尚未运行", "loop_last": None} stats = {"total_checks": 0, "total_pushes": 0, "total_failures": 0} -op_lock = threading.Lock() +op_lock = threading.Lock() # 整轮同步互斥（同一轮只跑一次） +stats_lock = threading.Lock() # 保护 stats 计数 +log_lock = threading.Lock() # 保护日志文件写入 +repo_busy_lock = threading.Lock() # 保护 repo_in_progress +repo_in_progress = set() # 正在同步的仓库名（按仓库防重入） # ---------------- 基础工具 ---------------- def log(msg): - try: - DATA_DIR.mkdir(parents=True, exist_ok=True) - with open(LOG_FILE, "a", encoding="utf-8") as f: - f.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n") - except Exception: - pass + with log_lock: + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + with open(LOG_FILE, "a", encoding="utf-8") as f: + f.write(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n") + except Exception: + pass + + +def bump_stats(**kw): + """线程安全地累加统计计数""" + with stats_lock: + for k, v in kw.items(): + stats[k] = stats.get(k, 0) + v + save_stats() def write_gitcreds(): @@ -475,12 +511,24 @@ def fingerprint(repo): def sync_repo(name): - """返回 'changed' / 'same' / 'error:...' """ + """返回 'changed' / 'same' / 'error:...'；同一仓库并发调用时后到者直接返回""" if not valid_name(name): return "error: 非法仓库名" acc = active_account() if not acc: return "error: 没有可用的 GitHub 账号，请到设置页添加" + with repo_busy_lock: + if name in repo_in_progress: + return "error: 该仓库正在同步中，请稍候" + repo_in_progress.add(name) + try: + return _sync_repo_locked(name, acc) + finally: + with repo_busy_lock: + repo_in_progress.discard(name) + + +def _sync_repo_locked(name, acc): repo = REPO_DIR / f"{name}.git" try: if not repo.exists(): @@ -527,25 +575,41 @@ def sync_repo(name): def run_sync(repos=None): + """跑一轮同步：多个项目按设置并行（默认 3 个同时）""" with op_lock: if state["syncing"]: return False, "已有同步任务正在进行" state["syncing"] = True try: repos = repos if repos is not None else load_repos() + results = {} + workers = sync_workers() + + def worker(name): + try: + results[name] = sync_repo(name) + except Exception as e: + results[name] = f"error: {e}" + + if repos: + with ThreadPoolExecutor(max_workers=workers, + thread_name_prefix="ghs-sync") as ex: + # map 会阻塞到全部完成；异常已在 worker 内捕获 + list(ex.map(worker, repos)) errs, changed = [], 0 for r in repos: - res = sync_repo(r) + res = results.get(r, "error: 未执行") if res.startswith("error"): errs.append(f"{r} {res}") - stats["total_failures"] += 1 elif res == "changed": changed += 1 - stats["total_pushes"] += 1 + if errs: + bump_stats(total_failures=len(errs)) + if changed: + bump_stats(total_pushes=changed) state["last_check"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") state["checks"] += 1 - stats["total_checks"] += 1 - save_stats() + bump_stats(total_checks=1) if errs: state["last_result"] = "失败: " + "; ".join(errs)[:200] return False, state["last_result"] @@ -651,6 +715,7 @@ def global_status(ui_url=None): "syncing": state["syncing"], "uptime": fmt_uptime(now - START_TS), "interval": sync_interval(), + "max_workers": sync_workers(), "last_check": state["last_check"], "next_in": next_in, "checks": state["checks"], @@ -863,7 +928,7 @@ async function refresh(){ const s=await jget('api/state'); document.getElementById('sw').checked=s.enabled; document.getElementById('swlabel').textContent='自动同步：'+(s.enabled?'开':'关'); - document.getElementById('swsub').textContent='开启后每 '+(s.interval||10)+' 秒检查一次 Gitea，有新提交自动推送到 GitHub（间隔可在设置页修改）'; + document.getElementById('swsub').textContent='开启后每 '+(s.interval||10)+' 秒检查一次 Gitea，'+(s.max_workers||1)+' 个项目并行同步，有新提交自动推送到 GitHub（间隔/并发可在设置页调整）'; const p=document.getElementById('pill'); p.textContent=s.enabled?'运行中':'已停止';p.className='pill '+(s.enabled?'on':'off'); document.getElementById('sEnabled').textContent=s.enabled?('开启（'+(s.interval||10)+'秒/次）'):'已停止'; @@ -928,7 +993,7 @@ async function refresh(){ document.getElementById('sPushes').textContent=s.stats.total_pushes; document.getElementById('sFails').textContent=s.stats.total_failures; document.getElementById('sSize').textContent=s.total_size_mb+' MB'; - document.getElementById('sTarget').textContent='同步目标: '+(s.account?s.account.gh_user+'（'+s.account.method+'）':'未配置')+' · Gitea '+(s.gitea_online?'在线 ✓':'离线 ✗'); + document.getElementById('sTarget').textContent='同步目标: '+(s.account?s.account.gh_user+'（'+s.account.method+'）':'未配置')+' · '+(s.max_workers||1)+' 个并行 · Gitea '+(s.gitea_online?'在线 ✓':'离线 ✗'); document.getElementById('tb').innerHTML=s.repos.map(r=>{ const st=r.status==='已同步'?'✓ 已同步':'⏳ 未同步'; const c=r.last_commit?`
${esc(r.last_commit.hash)} ${esc(r.last_commit.msg)} · ${esc(r.last_commit.author)} · ${esc(r.last_commit.time)}
`:'
—
'; @@ -1071,6 +1136,14 @@ PAGE_SETTINGS = """
+
+ + +
+
同时同步项目数
+
+
+
@@ -1090,6 +1163,9 @@ async function refresh(){ const iv=document.getElementById('syncInterval'); iv.value=s.interval||10;iv.min=s.min_interval||5;iv.max=s.max_interval||86400; document.getElementById('intervalTip').textContent='每隔 '+fmtInterval(s.interval||10)+' 检查一次 Gitea 是否有新提交（'+(s.min_interval||5)+'~'+(s.max_interval||86400)+' 秒），改完立即生效'; + const wk=document.getElementById('syncWorkers'); + wk.value=s.max_workers||3;wk.min=s.min_workers||1;wk.max=s.max_workers_limit||8; + document.getElementById('workersTip').textContent='每轮最多 '+(s.max_workers||3)+' 个项目同时克隆/推送（1~'+(s.max_workers_limit||8)+'），项目多时能显著加快；网络或 NAS 负载高就调小'; const cur=(s.accounts||[]).find(a=>a.id===s.active); // 当前账号无 Token 时，顶部提示为什么不能自动建库、怎么解锁 document.getElementById('accBanner').innerHTML=(cur&&!cur.has_token) @@ -1188,6 +1264,15 @@ async function saveInterval(){ }catch(e){showMsg('msgCfg','请求失败: '+e,false)} finally{b.disabled=false;b.textContent='保存间隔';refresh();} } +async function saveWorkers(){ + const v=parseInt(document.getElementById('syncWorkers').value,10); + const b=document.getElementById('btnWorkers');b.disabled=true;b.textContent='⏳ 保存中…'; + try{ + const r=await (await fetch('api/settings/save',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({max_workers:v})})).json(); + showMsg('msgCfg',r.msg||JSON.stringify(r),!!r.ok); + }catch(e){showMsg('msgCfg','请求失败: '+e,false)} + finally{b.disabled=false;b.textContent='保存并发';refresh();} +} refresh(); """ @@ -1242,6 +1327,7 @@ class Handler(BaseHTTPRequestHandler): "last_check": state["last_check"], "checks": state["checks"], "last_result": state["last_result"], "interval": sync_interval(), + "max_workers": sync_workers(), "repo_count": len(repos), "synced_count": sum(1 for r in repos if state_path(acc, r).exists()), "uptime": fmt_uptime(time.time() - START_TS), @@ -1257,6 +1343,9 @@ class Handler(BaseHTTPRequestHandler): "interval": sync_interval(), "min_interval": MIN_INTERVAL, "max_interval": MAX_INTERVAL, + "max_workers": sync_workers(), + "min_workers": MIN_WORKERS, + "max_workers_limit": MAX_WORKERS, "gitea_host": gitea_host(), "gitea_url": gitea_ui(self.headers.get("Host")), "active": active_id, @@ -1297,6 +1386,10 @@ class Handler(BaseHTTPRequestHandler): if not valid_interval(data.get("interval")): self._json({"ok": False, "msg": f"同步间隔须为 {MIN_INTERVAL} ~ {MAX_INTERVAL} 之间的整数（秒）"}) return + if "max_workers" in data: + if not valid_workers(data.get("max_workers")): + self._json({"ok": False, "msg": f"同时同步项目数须为 {MIN_WORKERS} ~ {MAX_WORKERS} 之间的整数"}) + return # 预校验：新地址先探连通性；涉及账号的改动再用新凭据调一次 /user 验证 trial = dict(cfg) if new_user is not None: @@ -1343,6 +1436,11 @@ class Handler(BaseHTTPRequestHandler): cfg["interval"] = int(data.get("interval")) if cfg["interval"] != old_iv: log(f"同步间隔已调整: {old_iv}s -> {cfg['interval']}s") + if "max_workers" in data: + old_w = sync_workers() + cfg["max_workers"] = int(data.get("max_workers")) + if cfg["max_workers"] != old_w: + log(f"同时同步项目数已调整: {old_w} -> {cfg['max_workers']}") save_settings() notes = [] if new_host is not None: @@ -1443,26 +1541,18 @@ class Handler(BaseHTTPRequestHandler): if not valid_name(name): self._json({"ok": False, "msg": "非法仓库名"}) return - with op_lock: - if state["syncing"]: - self._json({"ok": False, "msg": "已有同步任务正在进行"}) - return - state["syncing"] = True - try: - res = sync_repo(name) - state["last_check"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if res.startswith("error"): - stats["total_failures"] += 1 - save_stats() - self._json({"ok": False, "msg": f"{name} {res}"}) - elif res == "changed": - stats["total_pushes"] += 1 - save_stats() - self._json({"ok": True, "msg": f"{name} 已同步到 GitHub"}) - else: - self._json({"ok": True, "msg": f"{name} 已是最新，无需同步"}) - finally: - state["syncing"] = False + # 不再被整轮同步全局拦截：只禁止同一仓库重复同步（sync_repo 内部按仓库防重入）， + # 与正在进行的整轮同步互不阻塞，实现多项目同时同步 + res = sync_repo(name) + state["last_check"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + if res.startswith("error"): + bump_stats(total_failures=1) + self._json({"ok": False, "msg": f"{name} {res}"}) + elif res == "changed": + bump_stats(total_pushes=1) + self._json({"ok": True, "msg": f"{name} 已同步到 GitHub"}) + else: + self._json({"ok": True, "msg": f"{name} 已是最新，无需同步"}) elif path == "/api/repos/add": name = (data.get("name") or "").strip() if not valid_name(name): @@ -1493,8 +1583,7 @@ class Handler(BaseHTTPRequestHandler): if res.startswith("error"): self._json({"ok": False, "msg": f"已加入列表，但首次同步失败：{res}"}) elif res == "changed": - stats["total_pushes"] += 1 - save_stats() + bump_stats(total_pushes=1) self._json({"ok": True, "msg": f"{name} 添加成功，已完成首次同步"}) else: self._json({"ok": True, "msg": f"{name} 添加成功，内容已是最新"}) diff --git a/GiteaHubSync/manifest b/GiteaHubSync/manifest index 5ebc27f..5b9f50b 100644 --- a/GiteaHubSync/manifest +++ b/GiteaHubSync/manifest @@ -1,5 +1,5 @@ appname = GiteaHubSync -version = 1.2.0 +version = 1.3.0 display_name = Gitea→GitHub 同步助手 desc = 每次推送到飞牛 Gitea 的代码，10 秒内自动镜像到 GitHub 的 7 个私有仓库。内置控制面板：自动同步开关、立即同步、仓库状态、同步日志，一键打开 Gitea。数据全部保存在 NAS 本地。 platform = all diff --git a/README.md b/README.md index 9c80f9d..28c8d26 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ NAS 重启自动恢复。控制面板带自动同步开关、立即同步、仓 - **多 GitHub 账号**：设置页可添加多个 GitHub 账号（用户名 + Personal Access Token）， 一键切换推送目标；每个账号独立记录同步进度，切回原账号不会重复推送 - **同步间隔可调**：设置页可改检查间隔（5 秒 ~ 1 天，默认 10 秒），改完立即生效 +- **多项目并行同步**：每轮最多 N 个项目同时克隆/推送（默认 3，可调 1~8）； + 在状态页点单个仓库的「同步」也不再被整轮同步阻塞（同一仓库有防重入） - **Gitea 地址可改**：设置页可改同步引擎访问的 Gitea 地址（默认 `http://127.0.0.1:3000`）， 保存前自动探测连通性；改地址后已有镜像的拉取地址自动更新，「打开 Gitea」按钮随访问地址自适应 - **自动创建仓库**：Gitea 上的新仓库加入同步时，若 GitHub 上不存在同名仓库，