Set NCCL_DEBUG=TRACE for transport visibility

NET/IB transport lines only appear at TRACE level. Training output
remains clean with per-epoch summaries only.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-04-16 16:34:11 +03:00
parent dc2ffc270c
commit 71b603eb1e

View File

@@ -44,7 +44,7 @@ def run_training_node_func(rank, world_size):
os.dup2(sys.stdout.fileno(), sys.stderr.fileno())
# CRITICAL: Set NCCL environment variables BEFORE importing torch
os.environ['NCCL_DEBUG'] = 'INFO'
os.environ['NCCL_DEBUG'] = 'TRACE'
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1'
os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand