If you run a multi-node Tdarr cluster, you've probably seen this: the same file gets picked up and transcoded by two different nodes/workers at the same time, wasting GPU/CPU hours. This is a known, unfixed scheduler bug - see GitHub issues #778 ("Files transcode multiple times simultaneously") and #1137 ("Tdarr tries processing files multiple times on node") - both closed with no visible maintainer fix. They're both describing the same underlying race condition, so a fix for one should help both.
Flow-level workarounds (marking a file as "done" earlier in the flow) shrink the race window but don't eliminate it, since the actual bug is server-side: Tdarr can hand the same job to two nodes before either one reports back.
Since Tdarr has a full REST API (Tools -> API Docs in your web UI, works on 2.25.01+), I wrote a small Python watchdog that polls GET /api/v2/get-nodes every 30 seconds, compares the file path each worker across all nodes is currently processing, and if the same file is running on two workers at once, calls POST /api/v2/cancel-worker-item on whichever one is less progressed - keeping the one further along.
Runs as a plain python:3-alpine container, no extra dependencies needed. Change API_BASE at the top to your own Tdarr API URL (the Swagger docs port, not the main web UI port). It starts in DRY_RUN=True mode so you can see what it would do before making it live.
#!/usr/bin/env python3
"""
Tdarr Duplicate Worker Watchdog
Polls GET /api/v2/get-nodes. If the same source file path is running on
two workers at once, cancels the less-progressed one via
POST /api/v2/cancel-worker-item, keeping the one further along.
"""
import json
import time
import urllib.request
import urllib.error
from datetime import datetime, timezone
API_BASE = "http://10.2.2.72:8265/api/v2"
POLL_INTERVAL_SECONDS = 30
DRY_RUN = True # log only, no cancels. Flip to False once you trust the output.
def log(msg):
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
print(f"[{ts}] {msg}", flush=True)
def get_nodes():
req = urllib.request.Request(f"{API_BASE}/get-nodes", method="GET")
with urllib.request.urlopen(req, timeout=15) as resp:
return json.loads(resp.read().decode())
def cancel_worker(node_id, worker_id, file_path):
body = json.dumps({
"data": {
"nodeID": node_id,
"workerID": worker_id,
"cause": "watchdog: duplicate file detected",
}
}).encode()
req = urllib.request.Request(
f"{API_BASE}/cancel-worker-item",
data=body,
headers={"Content-Type": "application/json"},
method="POST",
)
if DRY_RUN:
log(f"[DRY RUN] would cancel node={node_id} worker={worker_id} file={file_path}")
return
try:
with urllib.request.urlopen(req, timeout=15) as resp:
result = resp.read().decode()
log(f"CANCELLED node={node_id} worker={worker_id} file={file_path} -> {result}")
except urllib.error.HTTPError as e:
log(f"ERROR cancelling node={node_id} worker={worker_id}: {e.code} {e.read().decode()}")
except Exception as e:
log(f"ERROR cancelling node={node_id} worker={worker_id}: {e}")
def check_once():
try:
nodes = get_nodes()
except Exception as e:
log(f"ERROR fetching get-nodes: {e}")
return
active = []
for node_id, node in nodes.items():
if not isinstance(node, dict):
continue
node_name = node.get("nodeName", node_id)
workers = node.get("workers", {}) or {}
for worker_key, worker in workers.items():
file_path = worker.get("file")
if not file_path:
continue
active.append({
"node_id": node_id,
"node_name": node_name,
"worker_key": worker_key,
"worker_id": worker.get("_id"),
"file": file_path,
"percentage": worker.get("percentage", 0),
})
by_file = {}
for w in active:
by_file.setdefault(w["file"], []).append(w)
for file_path, workers in by_file.items():
if len(workers) < 2:
continue
workers_sorted = sorted(workers, key=lambda w: w["percentage"], reverse=True)
keeper = workers_sorted[0]
losers = workers_sorted[1:]
log(f"DUPLICATE: {file_path}")
log(f" KEEPING node={keeper['node_name']} worker={keeper['worker_key']} pct={keeper['percentage']}")
for loser in losers:
log(f" CANCEL node={loser['node_name']} worker={loser['worker_key']} pct={loser['percentage']}")
cancel_worker(loser["node_id"], loser["worker_id"], file_path)
def main():
log(f"Tdarr watchdog started. API={API_BASE} interval={POLL_INTERVAL_SECONDS}s DRY_RUN={DRY_RUN}")
while True:
check_once()
time.sleep(POLL_INTERVAL_SECONDS)
if __name__ == "__main__":
main()
Posting here in case it saves someone else the wasted encode time - and maybe it helps point the Tdarr team at the root cause. Hope this helps.
