gstykalin/test-dags
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable InfiniBand with RDMA device support

Browse Source 
The RDMA device plugin is running and devices are exposed
(/dev/infiniband/*). Try enabling IB with settings that
work with containerized IB where /proc/drivers/infiniband
may not be accessible.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
George Stykalin
2026-04-15 12:39:29 +03:00
parent 8b91012567
commit 7620477bb6
1 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
dags/test-train-pytorch.py
View File

@@ -115,19 +115,20 @@ def run_training_node_func(rank, world_size):
os.environ['WORLD_SIZE'] = str(world_size) os.environ['WORLD_SIZE'] = str(world_size)
os.environ['RANK'] = str(rank) os.environ['RANK'] = str(rank)
# NCCL Configuration - Handle duplicate GPU IDs across nodes # NCCL Configuration - Try IB with RDMA devices
os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_DEBUG'] = 'INFO'
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
os.environ['NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_BLOCKING_WAIT'] = '1'
os.environ['NCCL_SOCKET_IFNAME'] = 'eth0' # Use ethernet (reliable) os.environ['NCCL_IB_DISABLE'] = '0' # Enable IB
os.environ['NCCL_IB_DISABLE'] = '1' # Disable IB for now # Don't set SOCKET_IFNAME - let NCCL auto-detect
os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P
os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility
# IMPORTANT: This fixes the "Duplicate GPU detected" error os.environ['NCCL_LL_THRESHOLD'] = '0' # Disable LL for compatibility
# Force each node to use its local rank instead of global device ID os.environ['NCCL_ALGO'] = 'Ring' # Use ring algorithm
os.environ['NCCL_LL_THRESHOLD'] = '0' # Disable LL for better compatibility
os.environ['NCCL_ALGO'] = 'Ring' # Use ring algorithm to avoid device ID conflicts
os.environ['NCCL_PROTO'] = 'Simple' # Use simple protocol os.environ['NCCL_PROTO'] = 'Simple' # Use simple protocol
# These help with IB in containers where /proc/drivers/infiniband isn't available
os.environ['NCCL_IB_HCA'] = '^irdma[0-9]+$|^mlx5_[0-9]+$|^hfi1_[0-9]+$|^qib0$' # Match any IB HCA
os.environ['NCCL_SOCKET_NTHREADS'] = '4' # More threads for better performance
print(f"[{rank}] Environment configured:") print(f"[{rank}] Environment configured:")
print(f" MASTER_ADDR: {MASTER_ADDR}") print(f" MASTER_ADDR: {MASTER_ADDR}")