From 7620477bb68dc344289074a826ee9af3bb1ae7fb Mon Sep 17 00:00:00 2001 From: George Stykalin Date: Wed, 15 Apr 2026 12:39:29 +0300 Subject: [PATCH] Enable InfiniBand with RDMA device support The RDMA device plugin is running and devices are exposed (/dev/infiniband/*). Try enabling IB with settings that work with containerized IB where /proc/drivers/infiniband may not be accessible. Co-Authored-By: Claude --- dags/test-train-pytorch.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index 67d99b5..501508b 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -115,19 +115,20 @@ def run_training_node_func(rank, world_size): os.environ['WORLD_SIZE'] = str(world_size) os.environ['RANK'] = str(rank) - # NCCL Configuration - Handle duplicate GPU IDs across nodes + # NCCL Configuration - Try IB with RDMA devices os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['NCCL_BLOCKING_WAIT'] = '1' - os.environ['NCCL_SOCKET_IFNAME'] = 'eth0' # Use ethernet (reliable) - os.environ['NCCL_IB_DISABLE'] = '1' # Disable IB for now + os.environ['NCCL_IB_DISABLE'] = '0' # Enable IB + # Don't set SOCKET_IFNAME - let NCCL auto-detect os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility - # IMPORTANT: This fixes the "Duplicate GPU detected" error - # Force each node to use its local rank instead of global device ID - os.environ['NCCL_LL_THRESHOLD'] = '0' # Disable LL for better compatibility - os.environ['NCCL_ALGO'] = 'Ring' # Use ring algorithm to avoid device ID conflicts + os.environ['NCCL_LL_THRESHOLD'] = '0' # Disable LL for compatibility + os.environ['NCCL_ALGO'] = 'Ring' # Use ring algorithm os.environ['NCCL_PROTO'] = 'Simple' # Use simple protocol + # These help with IB in containers where /proc/drivers/infiniband isn't available + os.environ['NCCL_IB_HCA'] = '^irdma[0-9]+$|^mlx5_[0-9]+$|^hfi1_[0-9]+$|^qib0$' # Match any IB HCA + os.environ['NCCL_SOCKET_NTHREADS'] = '4' # More threads for better performance print(f"[{rank}] Environment configured:") print(f" MASTER_ADDR: {MASTER_ADDR}")