gstykalin/test-dags
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Let NCCL auto-detect IB instead of hardcoding interfaces

Browse Source 
The regex patterns for IB interfaces weren't working. Now letting
NCCL auto-detect the best available transport (IB or ethernet).

- Removed NCCL_IB_DISABLE (let NCCL decide)
- Removed NCCL_SOCKET_IFNAME (let NCCL auto-detect)
- Kept RoCE v2 configuration

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
George Stykalin
2026-04-15 12:26:58 +03:00
parent 1c4c663431
commit ebb3284853
1 changed files with 6 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
dags/test-train-pytorch.py
View File

@@ -113,18 +113,17 @@ def run_training_node_func(rank, world_size):
os.environ['WORLD_SIZE'] = str(world_size) os.environ['WORLD_SIZE'] = str(world_size)
os.environ['RANK'] = str(rank) os.environ['RANK'] = str(rank)
# NCCL Configuration for InfiniBand # NCCL Configuration - Let NCCL auto-detect best transport
os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_DEBUG'] = 'INFO'
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
os.environ['NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_BLOCKING_WAIT'] = '1'
os.environ['NCCL_IB_DISABLE'] = '0' # Enable IB # Don't disable IB - let NCCL auto-detect
os.environ['NCCL_IB_HCA'] = '^mlx5_[0-9]+$' # Regex for Mellanox HCAs # Don't set SOCKET_IFNAME - let NCCL find the best interface
os.environ['NCCL_SOCKET_IFNAME'] = '^ibp[0-9]+s[0-9]+$' # Regex for IB interfaces
os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P
os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility
os.environ['NCCL_NET_GDR_LEVEL'] = '5' # Enable GPUDirect RDMA os.environ['NCCL_IB_GID_INDEX'] = '3' # Use RoCE v2
os.environ['NCCL_IB_GID_INDEX'] = '3' # Use RoCE v2 (common for modern IB) os.environ['NCCL_IB_TC'] = '106' # Traffic class (lower value for compatibility)
os.environ['NCCL_IB_TC'] = '160' # Traffic class for low latency os.environ['NCCL_IB_TIMEOUT'] = '22' # IB timeout in microseconds
print(f"[{rank}] Environment configured:") print(f"[{rank}] Environment configured:")
print(f" MASTER_ADDR: {MASTER_ADDR}") print(f" MASTER_ADDR: {MASTER_ADDR}")