diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index 5b88623..61b7043 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -40,7 +40,8 @@ def run_training_node_func(rank, world_size): import time # CRITICAL: Set NCCL environment variables BEFORE importing torch - os.environ['NCCL_DEBUG'] = 'INFO' + os.environ['NCCL_DEBUG'] = 'TRACE' + os.environ['NCCL_DEBUG_SUBSYS'] = 'NET,COLL' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand @@ -154,9 +155,48 @@ def run_training_node_func(rank, world_size): print(f"[{rank}] Found IB interfaces:") for line in ib_lines[:5]: print(f" {line.strip()}") + else: + print(f"[{rank}] No IB interfaces found via 'ip addr'") except Exception as e: print(f"[{rank}] Warning: Could not list interfaces: {e}") + # Check /dev/infiniband devices + print(f"[{rank}] Checking IB device access...") + ib_dev_path = '/dev/infiniband' + if os.path.exists(ib_dev_path): + ib_devices = os.listdir(ib_dev_path) + print(f"[{rank}] /dev/infiniband contents: {ib_devices if ib_devices else 'EMPTY'}") + else: + print(f"[{rank}] /dev/infiniband does NOT exist - IB devices not mounted in container") + + # Check /sys/class/infiniband + ib_sys_path = '/sys/class/infiniband' + if os.path.exists(ib_sys_path): + ib_sys_devices = os.listdir(ib_sys_path) + print(f"[{rank}] /sys/class/infiniband contents: {ib_sys_devices if ib_sys_devices else 'EMPTY'}") + else: + print(f"[{rank}] /sys/class/infiniband does NOT exist") + + # Try ibv_devinfo + try: + result = subprocess.run(['ibv_devinfo'], capture_output=True, text=True, timeout=10) + print(f"[{rank}] ibv_devinfo output:") + for line in result.stdout.split('\n')[:20]: + print(f" {line}") + if result.returncode != 0: + print(f"[{rank}] ibv_devinfo stderr: {result.stderr.strip()}") + except FileNotFoundError: + print(f"[{rank}] ibv_devinfo not found - rdma-core tools not installed") + except Exception as e: + print(f"[{rank}] ibv_devinfo failed: {e}") + + # Check RDMA devices via sysfs + rdma_path = '/sys/class/infiniband_verbs' + if os.path.exists(rdma_path): + print(f"[{rank}] RDMA verbs devices: {os.listdir(rdma_path)}") + else: + print(f"[{rank}] /sys/class/infiniband_verbs does NOT exist") + # STEP 4: Setup GPU FIRST (before process group init) device = torch.device("cuda:0") torch.cuda.set_device(device)