Add IB diagnostics and NCCL TRACE debug for IB transport detection

- Set NCCL_DEBUG=TRACE with NET,COLL subsystem filter
- Add pre-flight checks: /dev/infiniband, /sys/class/infiniband, ibv_devinfo, RDMA verbs
- Report whether IB devices are visible inside the container

Co-Authored-By: Claude <noreply@anthropic.com>

dags/test-train-pytorch.py

@@ -40,7 +40,8 @@ def run_training_node_func(rank, world_size):
   import time
 
   # CRITICAL: Set NCCL environment variables BEFORE importing torch
-  os.environ['NCCL_DEBUG'] = 'INFO'
+  os.environ['NCCL_DEBUG'] = 'TRACE'
+  os.environ['NCCL_DEBUG_SUBSYS'] = 'NET,COLL'
   os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
   os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1'
   os.environ['NCCL_IB_DISABLE'] = '0'  # Enable InfiniBand
@@ -154,9 +155,48 @@ def run_training_node_func(rank, world_size):
           print(f"[{rank}] Found IB interfaces:")
           for line in ib_lines[:5]:
               print(f"  {line.strip()}")
+      else:
+          print(f"[{rank}] No IB interfaces found via 'ip addr'")
   except Exception as e:
       print(f"[{rank}] Warning: Could not list interfaces: {e}")
 
+  # Check /dev/infiniband devices
+  print(f"[{rank}] Checking IB device access...")
+  ib_dev_path = '/dev/infiniband'
+  if os.path.exists(ib_dev_path):
+      ib_devices = os.listdir(ib_dev_path)
+      print(f"[{rank}] /dev/infiniband contents: {ib_devices if ib_devices else 'EMPTY'}")
+  else:
+      print(f"[{rank}] /dev/infiniband does NOT exist - IB devices not mounted in container")
+
+  # Check /sys/class/infiniband
+  ib_sys_path = '/sys/class/infiniband'
+  if os.path.exists(ib_sys_path):
+      ib_sys_devices = os.listdir(ib_sys_path)
+      print(f"[{rank}] /sys/class/infiniband contents: {ib_sys_devices if ib_sys_devices else 'EMPTY'}")
+  else:
+      print(f"[{rank}] /sys/class/infiniband does NOT exist")
+
+  # Try ibv_devinfo
+  try:
+      result = subprocess.run(['ibv_devinfo'], capture_output=True, text=True, timeout=10)
+      print(f"[{rank}] ibv_devinfo output:")
+      for line in result.stdout.split('\n')[:20]:
+          print(f"  {line}")
+      if result.returncode != 0:
+          print(f"[{rank}] ibv_devinfo stderr: {result.stderr.strip()}")
+  except FileNotFoundError:
+      print(f"[{rank}] ibv_devinfo not found - rdma-core tools not installed")
+  except Exception as e:
+      print(f"[{rank}] ibv_devinfo failed: {e}")
+
+  # Check RDMA devices via sysfs
+  rdma_path = '/sys/class/infiniband_verbs'
+  if os.path.exists(rdma_path):
+      print(f"[{rank}] RDMA verbs devices: {os.listdir(rdma_path)}")
+  else:
+      print(f"[{rank}] /sys/class/infiniband_verbs does NOT exist")
+
   # STEP 4: Setup GPU FIRST (before process group init)
   device = torch.device("cuda:0")
   torch.cuda.set_device(device)