gstykalin/test-dags
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert to ethernet for reliable DDP

Browse Source 
IB configuration is causing failures. Reverting to ethernet
which we confirmed works. IB can be enabled later once the
network setup is verified.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
George Stykalin
2026-04-15 12:27:36 +03:00
parent ebb3284853
commit d7aecf3cd8
1 changed files with 3 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
dags/test-train-pytorch.py
View File

@@ -113,17 +113,14 @@ def run_training_node_func(rank, world_size):
os.environ['WORLD_SIZE'] = str(world_size) os.environ['WORLD_SIZE'] = str(world_size)
os.environ['RANK'] = str(rank) os.environ['RANK'] = str(rank)
# NCCL Configuration - Let NCCL auto-detect best transport # NCCL Configuration - Use ethernet for reliability
os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_DEBUG'] = 'INFO'
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
os.environ['NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_BLOCKING_WAIT'] = '1'
# Don't disable IB - let NCCL auto-detect os.environ['NCCL_SOCKET_IFNAME'] = 'eth0' # Use ethernet (reliable)
# Don't set SOCKET_IFNAME - let NCCL find the best interface os.environ['NCCL_IB_DISABLE'] = '1' # Disable IB for now
os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P
os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility
os.environ['NCCL_IB_GID_INDEX'] = '3' # Use RoCE v2
os.environ['NCCL_IB_TC'] = '106' # Traffic class (lower value for compatibility)
os.environ['NCCL_IB_TIMEOUT'] = '22' # IB timeout in microseconds
print(f"[{rank}] Environment configured:") print(f"[{rank}] Environment configured:")
print(f" MASTER_ADDR: {MASTER_ADDR}") print(f" MASTER_ADDR: {MASTER_ADDR}")