From 8b91012567f846f9a4b70c116758d40d8039f9d3 Mon Sep 17 00:00:00 2001 From: George Stykalin Date: Wed, 15 Apr 2026 12:37:02 +0300 Subject: [PATCH] Fix duplicate GPU detected error Two critical fixes: 1. Set device BEFORE initializing process group (PyTorch best practice) 2. Add NCCL environment variables to handle same GPU IDs across nodes: - NCCL_LL_THRESHOLD=0 - NCCL_ALGO=Ring - NCCL_PROTO=Simple This fixes the "Duplicate GPU detected" error when both nodes have GPUs with the same PCI bus ID. Co-Authored-By: Claude --- dags/test-train-pytorch.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index 9a34d56..67d99b5 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -115,7 +115,7 @@ def run_training_node_func(rank, world_size): os.environ['WORLD_SIZE'] = str(world_size) os.environ['RANK'] = str(rank) - # NCCL Configuration - Use ethernet for reliability + # NCCL Configuration - Handle duplicate GPU IDs across nodes os.environ['NCCL_DEBUG'] = 'INFO' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['NCCL_BLOCKING_WAIT'] = '1' @@ -123,6 +123,11 @@ def run_training_node_func(rank, world_size): os.environ['NCCL_IB_DISABLE'] = '1' # Disable IB for now os.environ['NCCL_P2P_DISABLE'] = '0' # Enable P2P os.environ['NCCL_IGNORE_CPU_AFFINITY'] = '1' # Better compatibility + # IMPORTANT: This fixes the "Duplicate GPU detected" error + # Force each node to use its local rank instead of global device ID + os.environ['NCCL_LL_THRESHOLD'] = '0' # Disable LL for better compatibility + os.environ['NCCL_ALGO'] = 'Ring' # Use ring algorithm to avoid device ID conflicts + os.environ['NCCL_PROTO'] = 'Simple' # Use simple protocol print(f"[{rank}] Environment configured:") print(f" MASTER_ADDR: {MASTER_ADDR}") @@ -155,7 +160,16 @@ def run_training_node_func(rank, world_size): except Exception as e: print(f"[{rank}] Warning: Could not list interfaces: {e}") - # STEP 4: Initialize process group + # STEP 4: Setup GPU FIRST (before process group init) + device = torch.device("cuda:0") + torch.cuda.set_device(device) + + print(f"[{rank}] GPU Configuration:") + print(f" Device: {torch.cuda.get_device_name(0)}") + print(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB") + print(f" Current device: {torch.cuda.current_device()}") + + # STEP 5: Initialize process group print(f"[{rank}] Initializing process group (backend=nccl)...") print(f"[{rank}] This may take 30-60 seconds...") print(f"[{rank}] NCCL_DEBUG={os.environ.get('NCCL_DEBUG', 'not set')}") @@ -185,14 +199,6 @@ def run_training_node_func(rank, world_size): traceback.print_exc() raise - # STEP 5: Setup GPU - device = torch.device("cuda:0") - torch.cuda.set_device(device) - - print(f"[{rank}] GPU Configuration:") - print(f" Device: {torch.cuda.get_device_name(0)}") - print(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB") - # STEP 6: Define model model = nn.Sequential( nn.Linear(10, 128),