From 71b603eb1e82012a6f78d59010a3b0d209b1acd0 Mon Sep 17 00:00:00 2001 From: George Stykalin Date: Thu, 16 Apr 2026 16:34:11 +0300 Subject: [PATCH] Set NCCL_DEBUG=TRACE for transport visibility NET/IB transport lines only appear at TRACE level. Training output remains clean with per-epoch summaries only. Co-Authored-By: Claude --- dags/test-train-pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index e388d1c..7429a20 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -44,7 +44,7 @@ def run_training_node_func(rank, world_size): os.dup2(sys.stdout.fileno(), sys.stderr.fileno()) # CRITICAL: Set NCCL environment variables BEFORE importing torch - os.environ['NCCL_DEBUG'] = 'INFO' + os.environ['NCCL_DEBUG'] = 'TRACE' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand