From dc2ffc270c8367205eaad1a916de2484ef6069f7 Mon Sep 17 00:00:00 2001 From: George Stykalin Date: Thu, 16 Apr 2026 16:27:45 +0300 Subject: [PATCH] Restore NCCL INFO debug level, remove per-batch noise TRACE was too verbose. INFO shows clean transport selection lines (NET/IB, GPU Direct RDMA status) for both ranks via stderr redirect. Per-batch logging removed; per-epoch summary kept with timing. Co-Authored-By: Claude --- dags/test-train-pytorch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index d9e4a5c..e388d1c 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -44,7 +44,7 @@ def run_training_node_func(rank, world_size): os.dup2(sys.stdout.fileno(), sys.stderr.fileno()) # CRITICAL: Set NCCL environment variables BEFORE importing torch - os.environ['NCCL_DEBUG'] = 'TRACE' + os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand @@ -285,7 +285,6 @@ def run_training_node_func(rank, world_size): optimizer.step() epoch_loss += loss.item() - print(f"[{rank}] Epoch {epoch+1}/{num_epochs} Batch {batch_idx+1}/{num_batches} | Loss: {loss.item():.6f}") avg_loss = epoch_loss / num_batches epoch_time = time.time() - epoch_start