update dags
This commit is contained in:
@@ -40,7 +40,7 @@ def run_training_node_func(rank, world_size):
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
# CRITICAL: Set NCCL environment variables BEFORE importing torch
|
# CRITICAL: Set NCCL environment variables BEFORE importing torch
|
||||||
os.environ['NCCL_DEBUG'] = 'INFO'
|
os.environ['NCCL_DEBUG'] = 'TRACE'
|
||||||
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
|
os.environ['NCCL_TIMEOUT'] = str(NCCL_TIMEOUT)
|
||||||
os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1'
|
os.environ['TORCH_NCCL_BLOCKING_WAIT'] = '1'
|
||||||
os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand
|
os.environ['NCCL_IB_DISABLE'] = '0' # Enable InfiniBand
|
||||||
|
|||||||
Reference in New Issue
Block a user