gstykalin/test-dags
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix duplicate GPU detected error

Browse Source 
Two critical fixes:
1. Set device BEFORE initializing process group (PyTorch best practice)
2. Add NCCL environment variables to handle same GPU IDs across nodes:
   - NCCL_LL_THRESHOLD=0
   - NCCL_ALGO=Ring
   - NCCL_PROTO=Simple

This fixes the "Duplicate GPU detected" error when both nodes
have GPUs with the same PCI bus ID.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
George Stykalin
2026-04-15 12:37:02 +03:00
parent 4fff9658d7
commit 8b91012567
1 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
dags/test-train-pytorch.py
View File

@@ -115,7 +115,7 @@ def run_training_node_func(rank, world_size):
os.environ['WORLD_SIZE'] = str(world_size) os.environ['WORLD_SIZE'] = str(world_size)
os.environ['RANK'] = str(rank) os.environ['RANK'] = str(rank)
# NCCL Configuration - Use ethernet for reliability # NCCL Configuration - Handle duplicate GPU IDs across nodes
os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_DEBUG'] = 'INFO'
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
os.environ['NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_BLOCKING_WAIT'] = '1'
@@ -123,6 +123,11 @@ def run_training_node_func(rank, world_size):
os.environ['NCCL_IB_DISABLE'] = '1' # Disable IB for now os.environ['NCCL_IB_DISABLE'] = '1' # Disable IB for now
os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P
os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility
# IMPORTANT: This fixes the "Duplicate GPU detected" error
# Force each node to use its local rank instead of global device ID
os.environ['NCCL_LL_THRESHOLD'] = '0' # Disable LL for better compatibility
os.environ['NCCL_ALGO'] = 'Ring' # Use ring algorithm to avoid device ID conflicts
os.environ['NCCL_PROTO'] = 'Simple' # Use simple protocol
print(f"[{rank}] Environment configured:") print(f"[{rank}] Environment configured:")
print(f" MASTER_ADDR: {MASTER_ADDR}") print(f" MASTER_ADDR: {MASTER_ADDR}")
@@ -155,7 +160,16 @@ def run_training_node_func(rank, world_size):
except Exception as e: except Exception as e:
print(f"[{rank}] Warning: Could not list interfaces: {e}") print(f"[{rank}] Warning: Could not list interfaces: {e}")
# STEP 4: Initialize process group # STEP 4: Setup GPU FIRST (before process group init)
device = torch.device("cuda:0")
torch.cuda.set_device(device)
print(f"[{rank}] GPU Configuration:")
print(f" Device: {torch.cuda.get_device_name(0)}")
print(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB")
print(f" Current device: {torch.cuda.current_device()}")
# STEP 5: Initialize process group
print(f"[{rank}] Initializing process group (backend=nccl)...") print(f"[{rank}] Initializing process group (backend=nccl)...")
print(f"[{rank}] This may take 30-60 seconds...") print(f"[{rank}] This may take 30-60 seconds...")
print(f"[{rank}] NCCL_DEBUG={os.environ.get('NCCL_DEBUG', 'not set')}") print(f"[{rank}] NCCL_DEBUG={os.environ.get('NCCL_DEBUG', 'not set')}")
@@ -185,14 +199,6 @@ def run_training_node_func(rank, world_size):
traceback.print_exc() traceback.print_exc()
raise raise
# STEP 5: Setup GPU
device = torch.device("cuda:0")
torch.cuda.set_device(device)
print(f"[{rank}] GPU Configuration:")
print(f" Device: {torch.cuda.get_device_name(0)}")
print(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB")
# STEP 6: Define model # STEP 6: Define model
model = nn.Sequential( model = nn.Sequential(
nn.Linear(10, 128), nn.Linear(10, 128),