diff --git a/dags/test-train-pytorch.py b/dags/test-train-pytorch.py index 788af8a..95fa60f 100644 --- a/dags/test-train-pytorch.py +++ b/dags/test-train-pytorch.py @@ -40,7 +40,7 @@ def run_training_node_func(rank, world_size): import time # CRITICAL: Set NCCL environment variables BEFORE importing torch - os.environ['NCCL_DEBUG'] = 'INFO' + os.environ['NCCL_DEBUG'] = 'TRACE' os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT) os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1' os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand