|
|
|
@ -198,8 +198,8 @@ def run(rank, n_gpus, hps):
|
|
|
|
|
scaler = GradScaler(enabled=hps.train.fp16_run)
|
|
|
|
|
|
|
|
|
|
net_d=optim_d=scheduler_d=None
|
|
|
|
|
print("start training from epoch %s"%epoch_str)
|
|
|
|
|
for epoch in range(epoch_str, hps.train.epochs + 1):
|
|
|
|
|
print("start training from epoch %s"%epoch)
|
|
|
|
|
if rank == 0:
|
|
|
|
|
train_and_evaluate(
|
|
|
|
|
rank,
|
|
|
|
@ -228,7 +228,7 @@ def run(rank, n_gpus, hps):
|
|
|
|
|
None,
|
|
|
|
|
)
|
|
|
|
|
scheduler_g.step()
|
|
|
|
|
print("training done")
|
|
|
|
|
print("training done")
|
|
|
|
|
|
|
|
|
|
def train_and_evaluate(
|
|
|
|
|
rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers
|
|
|
|
|