From dad7b2cd1a89afcca2340cc0f8abb996dc099dfb Mon Sep 17 00:00:00 2001 From: George Stykalin Date: Thu, 16 Apr 2026 16:18:18 +0300 Subject: [PATCH] Enable GPU Direct RDMA over IB, restore INFO debug level Set NCCL_IB_GPU_DIRECT_RDMA_MAX_DISTANCE=6 to allow GPU Direct RDMA despite the 6-hop PCI distance between GPU (bus 0) and IB HCA (bus 6). Restore NCCL_DEBUG=INFO since IB transport is confirmed working. Co-Authored-By: Claude --- dags/test-train-pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index 61b7043..788af8a 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -40,11 +40,11 @@ def run_training_node_func(rank, world_size): import time # CRITICAL: Set NCCL environment variables BEFORE importing torch - os.environ['NCCL_DEBUG'] = 'TRACE' - os.environ['NCCL_DEBUG_SUBSYS'] = 'NET,COLL' + os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand + os.environ['NCCL_IB_GPU_DIRECT_RDMA_MAX_DISTANCE'] = '6' # GPU at bus 0, IB at bus 6 os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility