Set NCCL_DEBUG=TRACE for transport visibility
NET/IB transport lines only appear at TRACE level. Training output remains clean with per-epoch summaries only. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -44,7 +44,7 @@ def run_training_node_func(rank, world_size):
|
||||
os.dup2(sys.stdout.fileno(), sys.stderr.fileno())
|
||||
|
||||
# CRITICAL: Set NCCL environment variables BEFORE importing torch
|
||||
os.environ['NCCL_DEBUG'] = 'INFO'
|
||||
os.environ['NCCL_DEBUG'] = 'TRACE'
|
||||
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
|
||||
os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1'
|
||||
os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand
|
||||
|
||||
Reference in New Issue
Block a user