import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
X = a * X + B @ X
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
backend='newtonschulz5', backend_steps=5):
self.num_process = int(os.environ['WORLD_SIZE'])
self.rank = int(os.environ["RANK"])
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
params: "list[torch.Tensor]" = list(params)
assert all(isinstance(p, torch.Tensor) for p in params)
sizes = {p.numel() for p in params}
param_groups = [
{
"params": [p for p in params if p.numel() == size],
"update_buffer": [
torch.empty(size, device="cuda", dtype=torch.bfloat16)
for _ in range(self.num_process)
],
}
for size in sizes
]
super().__init__(param_groups, defaults)
def step(self):
for group in self.param_groups:
lr: float = group["lr"]
momentum: float = group["momentum"]
nesterov: bool = group["nesterov"]
zeropower_backend = zeropower_backends[group["backend"]]
backend_steps: int = group["backend_steps"]
update_buffers: "list[torch.Tensor]" = group["update_buffer"]
# generate weight updates in distributed fashion
params: "list[torch.Tensor]" = group["params"]
assert len(params) % self.num_process == 0
handle = None
params_world = None
def update_prev():
if params_world is None:
return
assert handle is not None
handle.wait()
for p_world, g_world in zip(params_world, update_buffers):
p_world.data.add_(
g_world.view_as(p_world),
alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
)
for base_i in range(len(params))[::self.num_process]:
p = params[base_i + self.rank]
g = p.grad
assert g is not None
state = self.state[p]
if "momentum_buffer" not in state:
state["momentum_buffer"] = torch.zeros_like(g)
buf: torch.Tensor = state["momentum_buffer"]
buf.lerp_(g, 1 - momentum)
g = g.lerp_(buf, momentum) if nesterov else buf
g = zeropower_backend(g, steps=backend_steps).flatten()
update_prev()
handle = dist.all_gather(update_buffers, g, async_op=True)
params_world = params[base_i : base_i + self.num_process]
update_prev()
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
def norm(x):
return F.rms_norm(x, (x.size(-1),))
class CastedLinear(nn.Linear):
def __init__(self, in_features, out_features):
super().__init__(in_features, out_features, bias=False)
def forward(self, x):
return F.linear(x, self.weight.to(x.dtype))
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
t = torch.arange(seq_len, device=x.device)
freqs = torch.outer(t, self.inv_freq)
self.seq_len_cached = seq_len
self.cos_cached = freqs.cos()
self.sin_cached = freqs.sin()
cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
# apply_rotary_emb(x, cos, sin)
x1, x2 = x.chunk(2, dim=3)
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat((y1, y2), 3).type_as(x)
class CausalSelfAttention(nn.Module):
def __init__(self, dim, n_head):
super().__init__()
assert dim % n_head == 0
self.n_head = n_head
self.c_q = CastedLinear(dim, dim)
self.c_k = CastedLinear(dim, dim)
self.c_v = CastedLinear(dim, dim)
# value residual lambda
self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
# rotary embeddings
self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
# output projection
self.c_proj = CastedLinear(dim, dim)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
B, T = x.size(0), x.size(1) # batch size, sequence length
assert B == 1, "Must use batch size = 1 for FlexAttention"
q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
q, k = self.rotary(q), self.rotary(k)
y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y
class MLP(nn.Module):
def __init__(self, dim: int):
super().__init__()
self.c_fc = CastedLinear(dim, 4 * dim)
self.c_proj = CastedLinear(4 * dim, dim)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config.n_embd, config.n_head)
self.mlp = MLP(config.n_embd)
self.lambdas = nn.Parameter(torch.tensor([1., 0.]))
def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
x = self.lambdas[0] * x + self.lambdas[1] * x0
x = x + self.attn(norm(x), vi, block_mask)
x = x + self.mlp(norm(x))
return x
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
lm_head_softcap : int = 30
class GPT(nn.Module):
def __init__(self, config: GPTConfig):
super().__init__()
self.n_layer = config.n_layer
self.lm_head_softcap = config.lm_head_softcap
# U-net design by @brendanh0gan
self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
# Add learnable skip connection weights for decoder layers
self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
# token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
# U-net structure on token value embeddings by @leloykun
vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
self.lm_head.weight.data.zero_() # @Grad62304977
def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
BLOCK_SIZE = 128
assert idx.ndim == 1
docs = (idx == 50256).cumsum(0)
docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
def document_sliding_window_causal(b, h, q_idx, kv_idx):
causal_mask = q_idx >= kv_idx
document_mask = docs[q_idx] == docs[kv_idx]
window_mask = q_idx - kv_idx < sliding_window
return causal_mask & document_mask & window_mask
S = len(idx)
def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
q_idx = block_idx[:, None]
causal_mask = q_idx >= kv_idx
document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
dense_mask = causal_mask & document_mask & window_mask
dense_mask = dense_mask.to(torch.int32)
num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
num_blocks = num_blocks[None, None, :].contiguous()
indices = indices[None, None, :].contiguous()
return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
block_mask = create_sliding_window_causal_mask(S, sliding_window)
# forward the GPT model itself
x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
x = norm(x) # @Grad62304977
x0 = x
vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)
# Store outputs for U-Net skip connections
skip_connections = []
# Encoder pass - process only the first half of the blocks
for i in range(self.num_encoder_layers):
x = self.transformer.h[i](x, vi[i], x0, block_mask)
skip_connections.append(x)
# Decoder pass - process the remaining blocks with weighted skip connections
for i in range(self.num_decoder_layers):
x = x + self.skip_weights[i] * skip_connections.pop()
# U-net structure on token value embeddings by @leloykun
x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)
x = norm(x)
logits = self.lm_head(x)
logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
logits = logits.float()
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
return loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(file: Path):
# only reads the header, returns header data
# header is 256 int32
header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
return int(header[2]) # number of tokens (claimed)
def _load_data_shard(file: Path, ntok: int):
with file.open("rb") as f:
tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
f.seek(256 * 4)
nbytes = f.readinto(tokens.numpy())
assert nbytes == 2 * ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.T = T
# glob files that match the pattern
self.files = sorted(Path.cwd().glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
self.ntoks = [_peek_data_shard(file) for file in self.files]
assert min(self.ntoks) >= num_processes * T + 1
self.ntok_total = sum(self.ntoks)
self.reset()
def reset(self):
self.current_shard = -1
self.advance()
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.T
self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])
def next_batch(self):
batch_size = self.T * self.num_processes
buf = self.tokens[self.current_position:self.current_position+self.T+1]
# host side async is sufficient;
# no performance improvement was observed when introducing a separate stream.
x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
# advance current position and load next shard if necessary
self.current_position += batch_size
if self.current_position + batch_size + 1 >= len(self.tokens):
self.advance()
return x, y
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8 # batch size, in sequences, across all devices
sequence_length : int = 64*1024 # sequence length, in tokens
num_iterations : int = 1480 # number of iterations to run
warmup_iters : int = 0
cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# begin logging
logfile = None
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
# os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write(code)
f.write('='*100 + '\n')
def print0(s, logonly=False):
if master_process:
with open(logfile, "a") as f:
if not logonly:
print(s)
f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)
# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1
# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
if isinstance(m, CastedLinear):
m.float()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.cooldown_iters:
return 1.0
# 3) linear cooldown
else:
decay_ratio = (args.num_iterations - it) / args.cooldown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.perf_counter()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
sw_size = 64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
if sw_size != sw_size_prev:
sliding_window_size.copy_(sw_size, non_blocking=True)
sw_size_prev = sw_size
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.perf_counter() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
with torch.no_grad():
x_val, y_val = val_loader.next_batch()
val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
# start the clock again
torch.cuda.synchronize()
t0 = time.perf_counter()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.perf_counter() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
# torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.perf_counter()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
loss = model(x, y, sliding_window=sliding_window_size)
loss.backward()
del loss
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# momentum warmup for Muon
frac = min(step/300, 1)
for group in optimizer3.param_groups:
group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec 8 10:26:09 2024
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03 Driver Version: 535.129.03 CUDA Version: 12.6 |
|-----------------------------------------+----------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+======================+======================|
| 0 NVIDIA H100 80GB HBM3 On | 00000000:65:02.0 Off | 0 |
| N/A 36C P0 73W / 700W | 7MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 On | 00000000:67:02.0 Off | 0 |
| N/A 46C P0 129W / 700W | 533MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 On | 00000000:69:02.0 Off | 0 |
| N/A 45C P0 84W / 700W | 26MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 On | 00000000:6B:02.0 Off | 0 |
| N/A 39C P0 114W / 700W | 533MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 On | 00000000:6F:02.0 Off | 0 |
| N/A 39C P0 117W / 700W | 533MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 On | 00000000:71:02.0 Off | 0 |
| N/A 45C P0 122W / 700W | 533MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 On | 00000000:73:02.0 Off | 0 |
| N/A 46C P0 127W / 700W | 533MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 On | 00000000:75:02.0 Off | 0 |
| N/A 38C P0 124W / 700W | 533MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+----------------------+----------------------+
+---------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=======================================================================================|
+---------------------------------------------------------------------------------------+
====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:23612ms step_avg:nanms
step:2/1480 train_time:23841ms step_avg:nanms
step:3/1480 train_time:23980ms step_avg:nanms
step:4/1480 train_time:24121ms step_avg:nanms
step:5/1480 train_time:24262ms step_avg:nanms
step:6/1480 train_time:24403ms step_avg:nanms
step:7/1480 train_time:24543ms step_avg:nanms
step:8/1480 train_time:24686ms step_avg:nanms
step:9/1480 train_time:24831ms step_avg:nanms
step:10/1480 train_time:24975ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:284ms step_avg:nanms
step:13/1480 train_time:425ms step_avg:141.81ms
step:14/1480 train_time:567ms step_avg:141.64ms
step:15/1480 train_time:709ms step_avg:141.71ms
step:16/1480 train_time:851ms step_avg:141.85ms
step:17/1480 train_time:995ms step_avg:142.21ms
step:18/1480 train_time:1138ms step_avg:142.27ms
step:19/1480 train_time:1281ms step_avg:142.32ms
step:20/1480 train_time:1424ms step_avg:142.38ms
step:21/1480 train_time:1565ms step_avg:142.28ms
step:22/1480 train_time:1708ms step_avg:142.35ms
step:23/1480 train_time:1851ms step_avg:142.39ms
step:24/1480 train_time:1996ms step_avg:142.54ms
step:25/1480 train_time:2140ms step_avg:142.63ms
step:26/1480 train_time:2284ms step_avg:142.76ms
step:27/1480 train_time:2426ms step_avg:142.70ms
step:28/1480 train_time:2567ms step_avg:142.62ms
step:29/1480 train_time:2709ms step_avg:142.58ms
step:30/1480 train_time:2850ms step_avg:142.52ms
step:31/1480 train_time:2995ms step_avg:142.64ms
step:32/1480 train_time:3140ms step_avg:142.71ms
step:33/1480 train_time:3285ms step_avg:142.84ms
step:34/1480 train_time:3428ms step_avg:142.83ms
step:35/1480 train_time:3569ms step_avg:142.77ms
step:36/1480 train_time:3711ms step_avg:142.72ms
step:37/1480 train_time:3852ms step_avg:142.66ms
step:38/1480 train_time:3995ms step_avg:142.66ms
step:39/1480 train_time:4137ms step_avg:142.65ms
step:40/1480 train_time:4282ms step_avg:142.72ms
step:41/1480 train_time:4425ms step_avg:142.74ms
step:42/1480 train_time:4567ms step_avg:142.71ms
step:43/1480 train_time:4709ms step_avg:142.70ms
step:44/1480 train_time:4852ms step_avg:142.71ms
step:45/1480 train_time:4997ms step_avg:142.76ms
step:46/1480 train_time:5140ms step_avg:142.78ms
step:47/1480 train_time:5285ms step_avg:142.83ms
step:48/1480 train_time:5427ms step_avg:142.83ms
step:49/1480 train_time:5569ms step_avg:142.78ms
step:50/1480 train_time:5712ms step_avg:142.79ms
step:51/1480 train_time:5853ms step_avg:142.76ms
step:52/1480 train_time:5995ms step_avg:142.73ms
step:53/1480 train_time:6138ms step_avg:142.74ms
step:54/1480 train_time:6283ms step_avg:142.79ms
step:55/1480 train_time:6425ms step_avg:142.78ms
step:56/1480 train_time:6567ms step_avg:142.75ms
step:57/1480 train_time:6710ms step_avg:142.77ms
step:58/1480 train_time:6853ms step_avg:142.77ms
step:59/1480 train_time:6998ms step_avg:142.81ms
step:60/1480 train_time:7141ms step_avg:142.82ms
step:61/1480 train_time:7285ms step_avg:142.84ms
step:62/1480 train_time:7428ms step_avg:142.85ms
step:63/1480 train_time:7570ms step_avg:142.82ms
step:64/1480 train_time:7713ms step_avg:142.83ms
step:65/1480 train_time:7856ms step_avg:142.84ms
step:66/1480 train_time:8001ms step_avg:142.87ms
step:67/1480 train_time:8145ms step_avg:142.90ms
step:68/1480 train_time:8288ms step_avg:142.90ms
step:69/1480 train_time:8430ms step_avg:142.87ms
step:70/1480 train_time:8571ms step_avg:142.86ms
step:71/1480 train_time:8716ms step_avg:142.88ms
step:72/1480 train_time:8859ms step_avg:142.88ms
step:73/1480 train_time:9003ms step_avg:142.90ms
step:74/1480 train_time:9146ms step_avg:142.90ms
step:75/1480 train_time:9289ms step_avg:142.90ms
step:76/1480 train_time:9430ms step_avg:142.88ms
step:77/1480 train_time:9572ms step_avg:142.87ms
step:78/1480 train_time:9715ms step_avg:142.86ms
step:79/1480 train_time:9858ms step_avg:142.87ms
step:80/1480 train_time:10002ms step_avg:142.88ms
step:81/1480 train_time:10145ms step_avg:142.89ms
step:82/1480 train_time:10287ms step_avg:142.87ms
step:83/1480 train_time:10429ms step_avg:142.86ms
step:84/1480 train_time:10570ms step_avg:142.83ms
step:85/1480 train_time:10711ms step_avg:142.82ms
step:86/1480 train_time:10853ms step_avg:142.80ms
step:87/1480 train_time:10994ms step_avg:142.78ms
step:88/1480 train_time:11135ms step_avg:142.76ms
step:89/1480 train_time:11277ms step_avg:142.75ms
step:90/1480 train_time:11420ms step_avg:142.75ms
step:91/1480 train_time:11562ms step_avg:142.75ms
step:92/1480 train_time:11705ms step_avg:142.75ms
step:93/1480 train_time:11847ms step_avg:142.73ms
step:94/1480 train_time:11989ms step_avg:142.73ms
step:95/1480 train_time:12130ms step_avg:142.70ms
step:96/1480 train_time:12271ms step_avg:142.68ms
step:97/1480 train_time:12414ms step_avg:142.69ms
step:98/1480 train_time:12556ms step_avg:142.68ms
step:99/1480 train_time:12699ms step_avg:142.69ms
step:100/1480 train_time:12843ms step_avg:142.70ms
step:101/1480 train_time:12986ms step_avg:142.70ms
step:102/1480 train_time:13127ms step_avg:142.68ms
step:103/1480 train_time:13268ms step_avg:142.66ms
step:104/1480 train_time:13409ms step_avg:142.65ms
step:105/1480 train_time:13552ms step_avg:142.65ms
step:106/1480 train_time:13697ms step_avg:142.67ms
step:107/1480 train_time:13840ms step_avg:142.68ms
step:108/1480 train_time:13984ms step_avg:142.70ms
step:109/1480 train_time:14126ms step_avg:142.69ms
step:110/1480 train_time:14268ms step_avg:142.68ms
step:111/1480 train_time:14412ms step_avg:142.69ms
step:112/1480 train_time:14559ms step_avg:142.74ms
step:113/1480 train_time:14708ms step_avg:142.79ms
step:114/1480 train_time:14853ms step_avg:142.81ms
step:115/1480 train_time:15002ms step_avg:142.87ms
step:116/1480 train_time:15149ms step_avg:142.91ms
step:117/1480 train_time:15295ms step_avg:142.95ms
step:118/1480 train_time:15442ms step_avg:142.98ms
step:119/1480 train_time:15589ms step_avg:143.02ms
step:120/1480 train_time:15735ms step_avg:143.04ms
step:121/1480 train_time:15882ms step_avg:143.08ms
step:122/1480 train_time:16030ms step_avg:143.12ms
step:123/1480 train_time:16176ms step_avg:143.15ms
step:124/1480 train_time:16324ms step_avg:143.19ms
step:125/1480 train_time:16470ms step_avg:143.22ms
step:125/1480 val_loss:4.4109 train_time:16527ms step_avg:143.72ms
step:126/1480 train_time:16625ms step_avg:143.32ms
step:127/1480 train_time:16772ms step_avg:143.35ms
step:128/1480 train_time:16920ms step_avg:143.39ms
step:129/1480 train_time:17065ms step_avg:143.40ms
step:130/1480 train_time:17211ms step_avg:143.42ms
step:131/1480 train_time:17358ms step_avg:143.45ms
step:132/1480 train_time:17504ms step_avg:143.48ms
step:133/1480 train_time:17654ms step_avg:143.53ms
step:134/1480 train_time:17802ms step_avg:143.56ms
step:135/1480 train_time:17948ms step_avg:143.58ms
step:136/1480 train_time:18094ms step_avg:143.60ms
step:137/1480 train_time:18242ms step_avg:143.64ms
step:138/1480 train_time:18387ms step_avg:143.65ms
step:139/1480 train_time:18534ms step_avg:143.67ms
step:140/1480 train_time:18682ms step_avg:143.71ms
step:141/1480 train_time:18829ms step_avg:143.73ms
step:142/1480 train_time:18976ms step_avg:143.76ms
step:143/1480 train_time:19123ms step_avg:143.78ms
step:144/1480 train_time:19269ms step_avg:143.80ms
step:145/1480 train_time:19416ms step_avg:143.82ms
step:146/1480 train_time:19563ms step_avg:143.85ms
step:147/1480 train_time:19709ms step_avg:143.86ms
step:148/1480 train_time:19857ms step_avg:143.89ms
step:149/1480 train_time:20003ms step_avg:143.91ms
step:150/1480 train_time:20149ms step_avg:143.92ms
step:151/1480 train_time:20296ms step_avg:143.94ms
step:152/1480 train_time:20442ms step_avg:143.96ms
step:153/1480 train_time:20588ms step_avg:143.97ms
step:154/1480 train_time:20736ms step_avg:144.00ms
step:155/1480 train_time:20883ms step_avg:144.02ms
step:156/1480 train_time:21029ms step_avg:144.03ms
step:157/1480 train_time:21176ms step_avg:144.05ms
step:158/1480 train_time:21323ms step_avg:144.07ms
step:159/1480 train_time:21468ms step_avg:144.08ms
step:160/1480 train_time:21615ms step_avg:144.10ms
step:161/1480 train_time:21762ms step_avg:144.12ms
step:162/1480 train_time:21908ms step_avg:144.13ms
step:163/1480 train_time:22056ms step_avg:144.15ms
step:164/1480 train_time:22202ms step_avg:144.17ms
step:165/1480 train_time:22348ms step_avg:144.18ms
step:166/1480 train_time:22495ms step_avg:144.20ms
step:167/1480 train_time:22642ms step_avg:144.22ms
step:168/1480 train_time:22789ms step_avg:144.23ms
step:169/1480 train_time:22936ms step_avg:144.25ms
step:170/1480 train_time:23083ms step_avg:144.27ms
step:171/1480 train_time:23229ms step_avg:144.28ms
step:172/1480 train_time:23376ms step_avg:144.30ms
step:173/1480 train_time:23522ms step_avg:144.31ms
step:174/1480 train_time:23669ms step_avg:144.32ms
step:175/1480 train_time:23816ms step_avg:144.34ms
step:176/1480 train_time:23963ms step_avg:144.36ms
step:177/1480 train_time:24109ms step_avg:144.37ms
step:178/1480 train_time:24257ms step_avg:144.39ms
step:179/1480 train_time:24404ms step_avg:144.40ms
step:180/1480 train_time:24551ms step_avg:144.42ms
step:181/1480 train_time:24698ms step_avg:144.43ms
step:182/1480 train_time:24844ms step_avg:144.44ms
step:183/1480 train_time:24993ms step_avg:144.47ms
step:184/1480 train_time:25141ms step_avg:144.49ms
step:185/1480 train_time:25287ms step_avg:144.50ms
step:186/1480 train_time:25435ms step_avg:144.52ms
step:187/1480 train_time:25582ms step_avg:144.53ms
step:188/1480 train_time:25727ms step_avg:144.53ms
step:189/1480 train_time:25875ms step_avg:144.55ms
step:190/1480 train_time:26023ms step_avg:144.57ms
step:191/1480 train_time:26170ms step_avg:144.58ms
step:192/1480 train_time:26317ms step_avg:144.60ms
step:193/1480 train_time:26464ms step_avg:144.61ms
step:194/1480 train_time:26613ms step_avg:144.63ms
step:195/1480 train_time:26760ms step_avg:144.65ms
step:196/1480 train_time:26905ms step_avg:144.65ms
step:197/1480 train_time:27052ms step_avg:144.66ms
step:198/1480 train_time:27199ms step_avg:144.68ms
step:199/1480 train_time:27345ms step_avg:144.68ms
step:200/1480 train_time:27492ms step_avg:144.70ms
step:201/1480 train_time:27640ms step_avg:144.71ms
step:202/1480 train_time:27785ms step_avg:144.72ms
step:203/1480 train_time:27932ms step_avg:144.72ms
step:204/1480 train_time:28079ms step_avg:144.74ms
step:205/1480 train_time:28225ms step_avg:144.74ms
step:206/1480 train_time:28371ms step_avg:144.75ms
step:207/1480 train_time:28520ms step_avg:144.77ms
step:208/1480 train_time:28666ms step_avg:144.78ms
step:209/1480 train_time:28813ms step_avg:144.79ms
step:210/1480 train_time:28960ms step_avg:144.80ms
step:211/1480 train_time:29105ms step_avg:144.80ms
step:212/1480 train_time:29251ms step_avg:144.81ms
step:213/1480 train_time:29398ms step_avg:144.82ms
step:214/1480 train_time:29545ms step_avg:144.83ms
step:215/1480 train_time:29692ms step_avg:144.84ms
step:216/1480 train_time:29839ms step_avg:144.85ms
step:217/1480 train_time:29985ms step_avg:144.86ms
step:218/1480 train_time:30133ms step_avg:144.87ms
step:219/1480 train_time:30279ms step_avg:144.88ms
step:220/1480 train_time:30425ms step_avg:144.88ms
step:221/1480 train_time:30574ms step_avg:144.90ms
step:222/1480 train_time:30725ms step_avg:144.93ms
step:223/1480 train_time:30874ms step_avg:144.95ms
step:224/1480 train_time:31024ms step_avg:144.97ms
step:225/1480 train_time:31174ms step_avg:144.99ms
step:226/1480 train_time:31324ms step_avg:145.02ms
step:227/1480 train_time:31474ms step_avg:145.04ms
step:228/1480 train_time:31625ms step_avg:145.07ms
step:229/1480 train_time:31776ms step_avg:145.10ms
step:230/1480 train_time:31927ms step_avg:145.12ms
step:231/1480 train_time:32078ms step_avg:145.15ms
step:232/1480 train_time:32230ms step_avg:145.18ms
step:233/1480 train_time:32381ms step_avg:145.20ms
step:234/1480 train_time:32531ms step_avg:145.23ms
step:235/1480 train_time:32682ms step_avg:145.25ms
step:236/1480 train_time:32832ms step_avg:145.27ms
step:237/1480 train_time:32983ms step_avg:145.30ms
step:238/1480 train_time:33134ms step_avg:145.32ms
step:239/1480 train_time:33285ms step_avg:145.35ms
step:240/1480 train_time:33434ms step_avg:145.37ms
step:241/1480 train_time:33584ms step_avg:145.39ms
step:242/1480 train_time:33736ms step_avg:145.42ms
step:243/1480 train_time:33886ms step_avg:145.43ms
step:244/1480 train_time:34037ms step_avg:145.46ms
step:245/1480 train_time:34188ms step_avg:145.48ms
step:246/1480 train_time:34338ms step_avg:145.50ms
step:247/1480 train_time:34487ms step_avg:145.51ms
step:248/1480 train_time:34637ms step_avg:145.53ms
step:249/1480 train_time:34786ms step_avg:145.55ms
step:250/1480 train_time:34936ms step_avg:145.57ms
step:250/1480 val_loss:3.9909 train_time:34995ms step_avg:145.81ms
step:251/1480 train_time:35093ms step_avg:145.61ms
step:252/1480 train_time:35244ms step_avg:145.63ms
step:253/1480 train_time:35394ms step_avg:145.65ms
step:254/1480 train_time:35542ms step_avg:145.67ms
step:255/1480 train_time:35692ms step_avg:145.68ms
step:256/1480 train_time:35841ms step_avg:145.69ms
step:257/1480 train_time:35992ms step_avg:145.72ms
step:258/1480 train_time:36142ms step_avg:145.73ms
step:259/1480 train_time:36294ms step_avg:145.76ms
step:260/1480 train_time:36445ms step_avg:145.78ms
step:261/1480 train_time:36596ms step_avg:145.80ms
step:262/1480 train_time:36745ms step_avg:145.81ms
step:263/1480 train_time:36895ms step_avg:145.83ms
step:264/1480 train_time:37045ms step_avg:145.85ms
step:265/1480 train_time:37197ms step_avg:145.87ms
step:266/1480 train_time:37348ms step_avg:145.89ms
step:267/1480 train_time:37498ms step_avg:145.91ms
step:268/1480 train_time:37648ms step_avg:145.92ms
step:269/1480 train_time:37798ms step_avg:145.94ms
step:270/1480 train_time:37949ms step_avg:145.96ms
step:271/1480 train_time:38100ms step_avg:145.98ms
step:272/1480 train_time:38251ms step_avg:146.00ms
step:273/1480 train_time:38401ms step_avg:146.01ms
step:274/1480 train_time:38551ms step_avg:146.03ms
step:275/1480 train_time:38701ms step_avg:146.04ms
step:276/1480 train_time:38852ms step_avg:146.06ms
step:277/1480 train_time:39002ms step_avg:146.07ms
step:278/1480 train_time:39154ms step_avg:146.10ms
step:279/1480 train_time:39304ms step_avg:146.11ms
step:280/1480 train_time:39455ms step_avg:146.13ms
step:281/1480 train_time:39605ms step_avg:146.14ms
step:282/1480 train_time:39756ms step_avg:146.16ms
step:283/1480 train_time:39906ms step_avg:146.18ms
step:284/1480 train_time:40057ms step_avg:146.19ms
step:285/1480 train_time:40207ms step_avg:146.21ms
step:286/1480 train_time:40359ms step_avg:146.23ms
step:287/1480 train_time:40510ms step_avg:146.24ms
step:288/1480 train_time:40661ms step_avg:146.26ms
step:289/1480 train_time:40812ms step_avg:146.28ms
step:290/1480 train_time:40963ms step_avg:146.30ms
step:291/1480 train_time:41113ms step_avg:146.31ms
step:292/1480 train_time:41262ms step_avg:146.32ms
step:293/1480 train_time:41413ms step_avg:146.34ms
step:294/1480 train_time:41563ms step_avg:146.35ms
step:295/1480 train_time:41713ms step_avg:146.36ms
step:296/1480 train_time:41865ms step_avg:146.38ms
step:297/1480 train_time:42014ms step_avg:146.39ms
step:298/1480 train_time:42164ms step_avg:146.40ms
step:299/1480 train_time:42314ms step_avg:146.42ms
step:300/1480 train_time:42464ms step_avg:146.43ms
step:301/1480 train_time:42615ms step_avg:146.44ms
step:302/1480 train_time:42766ms step_avg:146.46ms
step:303/1480 train_time:42915ms step_avg:146.47ms
step:304/1480 train_time:43066ms step_avg:146.48ms
step:305/1480 train_time:43216ms step_avg:146.50ms
step:306/1480 train_time:43365ms step_avg:146.50ms
step:307/1480 train_time:43516ms step_avg:146.52ms
step:308/1480 train_time:43667ms step_avg:146.53ms
step:309/1480 train_time:43820ms step_avg:146.55ms
step:310/1480 train_time:43971ms step_avg:146.57ms
step:311/1480 train_time:44122ms step_avg:146.58ms
step:312/1480 train_time:44273ms step_avg:146.60ms
step:313/1480 train_time:44424ms step_avg:146.61ms
step:314/1480 train_time:44574ms step_avg:146.62ms
step:315/1480 train_time:44722ms step_avg:146.63ms
step:316/1480 train_time:44873ms step_avg:146.64ms
step:317/1480 train_time:45022ms step_avg:146.65ms
step:318/1480 train_time:45174ms step_avg:146.67ms
step:319/1480 train_time:45323ms step_avg:146.68ms
step:320/1480 train_time:45475ms step_avg:146.69ms
step:321/1480 train_time:45626ms step_avg:146.71ms
step:322/1480 train_time:45777ms step_avg:146.72ms
step:323/1480 train_time:45927ms step_avg:146.73ms
step:324/1480 train_time:46078ms step_avg:146.74ms
step:325/1480 train_time:46228ms step_avg:146.75ms
step:326/1480 train_time:46379ms step_avg:146.77ms
step:327/1480 train_time:46529ms step_avg:146.78ms
step:328/1480 train_time:46679ms step_avg:146.79ms
step:329/1480 train_time:46831ms step_avg:146.81ms
step:330/1480 train_time:46982ms step_avg:146.82ms
step:331/1480 train_time:47136ms step_avg:146.84ms
step:332/1480 train_time:47290ms step_avg:146.86ms
step:333/1480 train_time:47444ms step_avg:146.88ms
step:334/1480 train_time:47598ms step_avg:146.91ms
step:335/1480 train_time:47753ms step_avg:146.93ms
step:336/1480 train_time:47907ms step_avg:146.95ms
step:337/1480 train_time:48062ms step_avg:146.98ms
step:338/1480 train_time:48215ms step_avg:147.00ms
step:339/1480 train_time:48371ms step_avg:147.02ms
step:340/1480 train_time:48525ms step_avg:147.05ms
step:341/1480 train_time:48678ms step_avg:147.06ms
step:342/1480 train_time:48831ms step_avg:147.08ms
step:343/1480 train_time:48985ms step_avg:147.10ms
step:344/1480 train_time:49139ms step_avg:147.12ms
step:345/1480 train_time:49293ms step_avg:147.14ms
step:346/1480 train_time:49446ms step_avg:147.16ms
step:347/1480 train_time:49599ms step_avg:147.18ms
step:348/1480 train_time:49753ms step_avg:147.20ms
step:349/1480 train_time:49908ms step_avg:147.22ms
step:350/1480 train_time:50064ms step_avg:147.25ms
step:351/1480 train_time:50216ms step_avg:147.26ms
step:352/1480 train_time:50371ms step_avg:147.28ms
step:353/1480 train_time:50525ms step_avg:147.30ms
step:354/1480 train_time:50679ms step_avg:147.32ms
step:355/1480 train_time:50833ms step_avg:147.34ms
step:356/1480 train_time:50988ms step_avg:147.36ms
step:357/1480 train_time:51141ms step_avg:147.38ms
step:358/1480 train_time:51295ms step_avg:147.40ms
step:359/1480 train_time:51449ms step_avg:147.42ms
step:360/1480 train_time:51602ms step_avg:147.44ms
step:361/1480 train_time:51757ms step_avg:147.45ms
step:362/1480 train_time:51908ms step_avg:147.47ms
step:363/1480 train_time:52063ms step_avg:147.49ms
step:364/1480 train_time:52216ms step_avg:147.50ms
step:365/1480 train_time:52370ms step_avg:147.52ms
step:366/1480 train_time:52525ms step_avg:147.54ms
step:367/1480 train_time:52678ms step_avg:147.56ms
step:368/1480 train_time:52832ms step_avg:147.58ms
step:369/1480 train_time:52985ms step_avg:147.59ms
step:370/1480 train_time:53137ms step_avg:147.60ms
step:371/1480 train_time:53291ms step_avg:147.62ms
step:372/1480 train_time:53444ms step_avg:147.64ms
step:373/1480 train_time:53598ms step_avg:147.65ms
step:374/1480 train_time:53753ms step_avg:147.67ms
step:375/1480 train_time:53906ms step_avg:147.69ms
step:375/1480 val_loss:3.7970 train_time:53966ms step_avg:147.85ms
step:376/1480 train_time:54064ms step_avg:147.72ms
step:377/1480 train_time:54219ms step_avg:147.74ms
step:378/1480 train_time:54373ms step_avg:147.75ms
step:379/1480 train_time:54525ms step_avg:147.76ms
step:380/1480 train_time:54677ms step_avg:147.78ms
step:381/1480 train_time:54830ms step_avg:147.79ms
step:382/1480 train_time:54983ms step_avg:147.80ms
step:383/1480 train_time:55139ms step_avg:147.83ms
step:384/1480 train_time:55293ms step_avg:147.84ms
step:385/1480 train_time:55448ms step_avg:147.86ms
step:386/1480 train_time:55601ms step_avg:147.87ms
step:387/1480 train_time:55754ms step_avg:147.89ms
step:388/1480 train_time:55907ms step_avg:147.90ms
step:389/1480 train_time:56060ms step_avg:147.92ms
step:390/1480 train_time:56213ms step_avg:147.93ms
step:391/1480 train_time:56369ms step_avg:147.95ms
step:392/1480 train_time:56522ms step_avg:147.96ms
step:393/1480 train_time:56676ms step_avg:147.98ms
step:394/1480 train_time:56830ms step_avg:147.99ms
step:395/1480 train_time:56984ms step_avg:148.01ms
step:396/1480 train_time:57138ms step_avg:148.03ms
step:397/1480 train_time:57292ms step_avg:148.04ms
step:398/1480 train_time:57448ms step_avg:148.06ms
step:399/1480 train_time:57602ms step_avg:148.08ms
step:400/1480 train_time:57755ms step_avg:148.09ms
step:401/1480 train_time:57909ms step_avg:148.10ms
step:402/1480 train_time:58063ms step_avg:148.12ms
step:403/1480 train_time:58217ms step_avg:148.13ms
step:404/1480 train_time:58372ms step_avg:148.15ms
step:405/1480 train_time:58527ms step_avg:148.17ms
step:406/1480 train_time:58680ms step_avg:148.18ms
step:407/1480 train_time:58834ms step_avg:148.20ms
step:408/1480 train_time:58987ms step_avg:148.21ms
step:409/1480 train_time:59141ms step_avg:148.22ms
step:410/1480 train_time:59294ms step_avg:148.24ms
step:411/1480 train_time:59449ms step_avg:148.25ms
step:412/1480 train_time:59603ms step_avg:148.27ms
step:413/1480 train_time:59755ms step_avg:148.28ms
step:414/1480 train_time:59909ms step_avg:148.29ms
step:415/1480 train_time:60064ms step_avg:148.31ms
step:416/1480 train_time:60217ms step_avg:148.32ms
step:417/1480 train_time:60373ms step_avg:148.34ms
step:418/1480 train_time:60527ms step_avg:148.35ms
step:419/1480 train_time:60681ms step_avg:148.36ms
step:420/1480 train_time:60834ms step_avg:148.38ms
step:421/1480 train_time:60988ms step_avg:148.39ms
step:422/1480 train_time:61142ms step_avg:148.40ms
step:423/1480 train_time:61295ms step_avg:148.41ms
step:424/1480 train_time:61450ms step_avg:148.43ms
step:425/1480 train_time:61605ms step_avg:148.44ms
step:426/1480 train_time:61758ms step_avg:148.46ms
step:427/1480 train_time:61911ms step_avg:148.47ms
step:428/1480 train_time:62065ms step_avg:148.48ms
step:429/1480 train_time:62217ms step_avg:148.49ms
step:430/1480 train_time:62371ms step_avg:148.50ms
step:431/1480 train_time:62526ms step_avg:148.52ms
step:432/1480 train_time:62681ms step_avg:148.53ms
step:433/1480 train_time:62835ms step_avg:148.55ms
step:434/1480 train_time:62989ms step_avg:148.56ms
step:435/1480 train_time:63142ms step_avg:148.57ms
step:436/1480 train_time:63296ms step_avg:148.58ms
step:437/1480 train_time:63450ms step_avg:148.59ms
step:438/1480 train_time:63603ms step_avg:148.61ms
step:439/1480 train_time:63759ms step_avg:148.62ms
step:440/1480 train_time:63912ms step_avg:148.63ms
step:441/1480 train_time:64071ms step_avg:148.66ms
step:442/1480 train_time:64230ms step_avg:148.68ms
step:443/1480 train_time:64387ms step_avg:148.70ms
step:444/1480 train_time:64544ms step_avg:148.72ms
step:445/1480 train_time:64699ms step_avg:148.73ms
step:446/1480 train_time:64855ms step_avg:148.75ms
step:447/1480 train_time:65010ms step_avg:148.76ms
step:448/1480 train_time:65166ms step_avg:148.78ms
step:449/1480 train_time:65322ms step_avg:148.80ms
step:450/1480 train_time:65479ms step_avg:148.82ms
step:451/1480 train_time:65637ms step_avg:148.84ms
step:452/1480 train_time:65794ms step_avg:148.86ms
step:453/1480 train_time:65952ms step_avg:148.88ms
step:454/1480 train_time:66110ms step_avg:148.90ms
step:455/1480 train_time:66267ms step_avg:148.92ms
step:456/1480 train_time:66423ms step_avg:148.93ms
step:457/1480 train_time:66580ms step_avg:148.95ms
step:458/1480 train_time:66735ms step_avg:148.96ms
step:459/1480 train_time:66893ms step_avg:148.98ms
step:460/1480 train_time:67051ms step_avg:149.00ms
step:461/1480 train_time:67210ms step_avg:149.02ms
step:462/1480 train_time:67368ms step_avg:149.04ms
step:463/1480 train_time:67526ms step_avg:149.06ms
step:464/1480 train_time:67683ms step_avg:149.08ms
step:465/1480 train_time:67838ms step_avg:149.10ms
step:466/1480 train_time:67994ms step_avg:149.11ms
step:467/1480 train_time:68152ms step_avg:149.13ms
step:468/1480 train_time:68309ms step_avg:149.15ms
step:469/1480 train_time:68467ms step_avg:149.17ms
step:470/1480 train_time:68625ms step_avg:149.19ms
step:471/1480 train_time:68783ms step_avg:149.20ms
step:472/1480 train_time:68940ms step_avg:149.22ms
step:473/1480 train_time:69095ms step_avg:149.23ms
step:474/1480 train_time:69252ms step_avg:149.25ms
step:475/1480 train_time:69409ms step_avg:149.27ms
step:476/1480 train_time:69566ms step_avg:149.28ms
step:477/1480 train_time:69724ms step_avg:149.30ms
step:478/1480 train_time:69880ms step_avg:149.32ms
step:479/1480 train_time:70037ms step_avg:149.33ms
step:480/1480 train_time:70194ms step_avg:149.35ms
step:481/1480 train_time:70350ms step_avg:149.36ms
step:482/1480 train_time:70507ms step_avg:149.38ms
step:483/1480 train_time:70663ms step_avg:149.39ms
step:484/1480 train_time:70819ms step_avg:149.41ms
step:485/1480 train_time:70976ms step_avg:149.42ms
step:486/1480 train_time:71133ms step_avg:149.44ms
step:487/1480 train_time:71292ms step_avg:149.46ms
step:488/1480 train_time:71450ms step_avg:149.48ms
step:489/1480 train_time:71606ms step_avg:149.49ms
step:490/1480 train_time:71762ms step_avg:149.50ms
step:491/1480 train_time:71917ms step_avg:149.52ms
step:492/1480 train_time:72075ms step_avg:149.53ms
step:493/1480 train_time:72232ms step_avg:149.55ms
step:494/1480 train_time:72390ms step_avg:149.57ms
step:495/1480 train_time:72549ms step_avg:149.59ms
step:496/1480 train_time:72707ms step_avg:149.60ms
step:497/1480 train_time:72864ms step_avg:149.62ms
step:498/1480 train_time:73021ms step_avg:149.63ms
step:499/1480 train_time:73179ms step_avg:149.65ms
step:500/1480 train_time:73336ms step_avg:149.66ms
step:500/1480 val_loss:3.6814 train_time:73398ms step_avg:149.79ms
step:501/1480 train_time:73499ms step_avg:149.69ms
step:502/1480 train_time:73657ms step_avg:149.71ms
step:503/1480 train_time:73812ms step_avg:149.72ms
step:504/1480 train_time:73968ms step_avg:149.73ms
step:505/1480 train_time:74124ms step_avg:149.75ms
step:506/1480 train_time:74280ms step_avg:149.76ms
step:507/1480 train_time:74436ms step_avg:149.77ms
step:508/1480 train_time:74592ms step_avg:149.78ms
step:509/1480 train_time:74748ms step_avg:149.80ms
step:510/1480 train_time:74905ms step_avg:149.81ms
step:511/1480 train_time:75062ms step_avg:149.82ms
step:512/1480 train_time:75219ms step_avg:149.84ms
step:513/1480 train_time:75374ms step_avg:149.85ms
step:514/1480 train_time:75531ms step_avg:149.86ms
step:515/1480 train_time:75688ms step_avg:149.88ms
step:516/1480 train_time:75846ms step_avg:149.89ms
step:517/1480 train_time:76003ms step_avg:149.91ms
step:518/1480 train_time:76162ms step_avg:149.92ms
step:519/1480 train_time:76318ms step_avg:149.94ms
step:520/1480 train_time:76474ms step_avg:149.95ms
step:521/1480 train_time:76630ms step_avg:149.96ms
step:522/1480 train_time:76788ms step_avg:149.98ms
step:523/1480 train_time:76946ms step_avg:149.99ms
step:524/1480 train_time:77103ms step_avg:150.01ms
step:525/1480 train_time:77260ms step_avg:150.02ms
step:526/1480 train_time:77418ms step_avg:150.03ms
step:527/1480 train_time:77574ms step_avg:150.05ms
step:528/1480 train_time:77730ms step_avg:150.06ms
step:529/1480 train_time:77885ms step_avg:150.07ms
step:530/1480 train_time:78042ms step_avg:150.08ms
step:531/1480 train_time:78199ms step_avg:150.09ms
step:532/1480 train_time:78358ms step_avg:150.11ms
step:533/1480 train_time:78514ms step_avg:150.12ms
step:534/1480 train_time:78669ms step_avg:150.13ms
step:535/1480 train_time:78826ms step_avg:150.14ms
step:536/1480 train_time:78984ms step_avg:150.16ms
step:537/1480 train_time:79142ms step_avg:150.18ms
step:538/1480 train_time:79301ms step_avg:150.19ms
step:539/1480 train_time:79459ms step_avg:150.21ms
step:540/1480 train_time:79617ms step_avg:150.22ms
step:541/1480 train_time:79771ms step_avg:150.23ms
step:542/1480 train_time:79929ms step_avg:150.24ms
step:543/1480 train_time:80085ms step_avg:150.25ms
step:544/1480 train_time:80242ms step_avg:150.27ms
step:545/1480 train_time:80398ms step_avg:150.28ms
step:546/1480 train_time:80554ms step_avg:150.29ms
step:547/1480 train_time:80710ms step_avg:150.30ms
step:548/1480 train_time:80869ms step_avg:150.31ms
step:549/1480 train_time:81026ms step_avg:150.33ms
step:550/1480 train_time:81183ms step_avg:150.34ms
step:551/1480 train_time:81343ms step_avg:150.36ms
step:552/1480 train_time:81503ms step_avg:150.37ms
step:553/1480 train_time:81664ms step_avg:150.39ms
step:554/1480 train_time:81825ms step_avg:150.41ms
step:555/1480 train_time:81985ms step_avg:150.43ms
step:556/1480 train_time:82144ms step_avg:150.45ms
step:557/1480 train_time:82304ms step_avg:150.46ms
step:558/1480 train_time:82465ms step_avg:150.48ms
step:559/1480 train_time:82625ms step_avg:150.50ms
step:560/1480 train_time:82785ms step_avg:150.52ms
step:561/1480 train_time:82944ms step_avg:150.53ms
step:562/1480 train_time:83103ms step_avg:150.55ms
step:563/1480 train_time:83262ms step_avg:150.56ms
step:564/1480 train_time:83421ms step_avg:150.58ms
step:565/1480 train_time:83580ms step_avg:150.59ms
step:566/1480 train_time:83740ms step_avg:150.61ms
step:567/1480 train_time:83899ms step_avg:150.63ms
step:568/1480 train_time:84058ms step_avg:150.64ms
step:569/1480 train_time:84216ms step_avg:150.66ms
step:570/1480 train_time:84374ms step_avg:150.67ms
step:571/1480 train_time:84532ms step_avg:150.68ms
step:572/1480 train_time:84691ms step_avg:150.70ms
step:573/1480 train_time:84851ms step_avg:150.71ms
step:574/1480 train_time:85010ms step_avg:150.73ms
step:575/1480 train_time:85169ms step_avg:150.74ms
step:576/1480 train_time:85328ms step_avg:150.76ms
step:577/1480 train_time:85488ms step_avg:150.77ms
step:578/1480 train_time:85647ms step_avg:150.79ms
step:579/1480 train_time:85806ms step_avg:150.80ms
step:580/1480 train_time:85966ms step_avg:150.82ms
step:581/1480 train_time:86126ms step_avg:150.83ms
step:582/1480 train_time:86287ms step_avg:150.85ms
step:583/1480 train_time:86447ms step_avg:150.87ms
step:584/1480 train_time:86607ms step_avg:150.88ms
step:585/1480 train_time:86766ms step_avg:150.90ms
step:586/1480 train_time:86926ms step_avg:150.91ms
step:587/1480 train_time:87085ms step_avg:150.93ms
step:588/1480 train_time:87245ms step_avg:150.94ms
step:589/1480 train_time:87405ms step_avg:150.96ms
step:590/1480 train_time:87566ms step_avg:150.98ms
step:591/1480 train_time:87726ms step_avg:150.99ms
step:592/1480 train_time:87885ms step_avg:151.01ms
step:593/1480 train_time:88045ms step_avg:151.02ms
step:594/1480 train_time:88205ms step_avg:151.04ms
step:595/1480 train_time:88367ms step_avg:151.05ms
step:596/1480 train_time:88528ms step_avg:151.07ms
step:597/1480 train_time:88687ms step_avg:151.08ms
step:598/1480 train_time:88845ms step_avg:151.10ms
step:599/1480 train_time:89004ms step_avg:151.11ms
step:600/1480 train_time:89164ms step_avg:151.13ms
step:601/1480 train_time:89324ms step_avg:151.14ms
step:602/1480 train_time:89484ms step_avg:151.16ms
step:603/1480 train_time:89646ms step_avg:151.17ms
step:604/1480 train_time:89804ms step_avg:151.19ms
step:605/1480 train_time:89965ms step_avg:151.20ms
step:606/1480 train_time:90125ms step_avg:151.22ms
step:607/1480 train_time:90287ms step_avg:151.23ms
step:608/1480 train_time:90447ms step_avg:151.25ms
step:609/1480 train_time:90606ms step_avg:151.26ms
step:610/1480 train_time:90765ms step_avg:151.27ms
step:611/1480 train_time:90925ms step_avg:151.29ms
step:612/1480 train_time:91086ms step_avg:151.31ms
step:613/1480 train_time:91246ms step_avg:151.32ms
step:614/1480 train_time:91405ms step_avg:151.33ms
step:615/1480 train_time:91566ms step_avg:151.35ms
step:616/1480 train_time:91724ms step_avg:151.36ms
step:617/1480 train_time:91884ms step_avg:151.37ms
step:618/1480 train_time:92043ms step_avg:151.39ms
step:619/1480 train_time:92203ms step_avg:151.40ms
step:620/1480 train_time:92364ms step_avg:151.42ms
step:621/1480 train_time:92524ms step_avg:151.43ms
step:622/1480 train_time:92684ms step_avg:151.44ms
step:623/1480 train_time:92846ms step_avg:151.46ms
step:624/1480 train_time:93005ms step_avg:151.47ms
step:625/1480 train_time:93165ms step_avg:151.49ms
step:625/1480 val_loss:3.6017 train_time:93229ms step_avg:151.59ms
step:626/1480 train_time:93328ms step_avg:151.51ms
step:627/1480 train_time:93488ms step_avg:151.52ms
step:628/1480 train_time:93646ms step_avg:151.53ms
step:629/1480 train_time:93803ms step_avg:151.54ms
step:630/1480 train_time:93960ms step_avg:151.55ms
step:631/1480 train_time:94117ms step_avg:151.56ms
step:632/1480 train_time:94276ms step_avg:151.57ms
step:633/1480 train_time:94437ms step_avg:151.58ms
step:634/1480 train_time:94597ms step_avg:151.60ms
step:635/1480 train_time:94756ms step_avg:151.61ms
step:636/1480 train_time:94915ms step_avg:151.62ms
step:637/1480 train_time:95074ms step_avg:151.63ms
step:638/1480 train_time:95234ms step_avg:151.65ms
step:639/1480 train_time:95393ms step_avg:151.66ms
step:640/1480 train_time:95554ms step_avg:151.67ms
step:641/1480 train_time:95714ms step_avg:151.69ms
step:642/1480 train_time:95874ms step_avg:151.70ms
step:643/1480 train_time:96035ms step_avg:151.71ms
step:644/1480 train_time:96193ms step_avg:151.72ms
step:645/1480 train_time:96352ms step_avg:151.74ms
step:646/1480 train_time:96511ms step_avg:151.75ms
step:647/1480 train_time:96671ms step_avg:151.76ms
step:648/1480 train_time:96832ms step_avg:151.77ms
step:649/1480 train_time:96992ms step_avg:151.79ms
step:650/1480 train_time:97152ms step_avg:151.80ms
step:651/1480 train_time:97313ms step_avg:151.81ms
step:652/1480 train_time:97473ms step_avg:151.83ms
step:653/1480 train_time:97633ms step_avg:151.84ms
step:654/1480 train_time:97793ms step_avg:151.85ms
step:655/1480 train_time:97953ms step_avg:151.86ms
step:656/1480 train_time:98113ms step_avg:151.88ms
step:657/1480 train_time:98273ms step_avg:151.89ms
step:658/1480 train_time:98434ms step_avg:151.90ms
step:659/1480 train_time:98595ms step_avg:151.92ms
step:660/1480 train_time:98757ms step_avg:151.93ms
step:661/1480 train_time:98919ms step_avg:151.95ms
step:662/1480 train_time:99079ms step_avg:151.96ms
step:663/1480 train_time:99238ms step_avg:151.97ms
step:664/1480 train_time:99400ms step_avg:151.99ms
step:665/1480 train_time:99562ms step_avg:152.00ms
step:666/1480 train_time:99721ms step_avg:152.01ms
step:667/1480 train_time:99882ms step_avg:152.03ms
step:668/1480 train_time:100045ms step_avg:152.04ms
step:669/1480 train_time:100208ms step_avg:152.06ms
step:670/1480 train_time:100369ms step_avg:152.07ms
step:671/1480 train_time:100531ms step_avg:152.09ms
step:672/1480 train_time:100695ms step_avg:152.11ms
step:673/1480 train_time:100858ms step_avg:152.12ms
step:674/1480 train_time:101020ms step_avg:152.14ms
step:675/1480 train_time:101181ms step_avg:152.15ms
step:676/1480 train_time:101343ms step_avg:152.17ms
step:677/1480 train_time:101502ms step_avg:152.18ms
step:678/1480 train_time:101662ms step_avg:152.19ms
step:679/1480 train_time:101824ms step_avg:152.20ms
step:680/1480 train_time:101986ms step_avg:152.22ms
step:681/1480 train_time:102148ms step_avg:152.23ms
step:682/1480 train_time:102313ms step_avg:152.25ms
step:683/1480 train_time:102475ms step_avg:152.27ms
step:684/1480 train_time:102635ms step_avg:152.28ms
step:685/1480 train_time:102798ms step_avg:152.29ms
step:686/1480 train_time:102959ms step_avg:152.31ms
step:687/1480 train_time:103120ms step_avg:152.32ms
step:688/1480 train_time:103282ms step_avg:152.33ms
step:689/1480 train_time:103444ms step_avg:152.35ms
step:690/1480 train_time:103608ms step_avg:152.36ms
step:691/1480 train_time:103769ms step_avg:152.38ms
step:692/1480 train_time:103932ms step_avg:152.39ms
step:693/1480 train_time:104095ms step_avg:152.41ms
step:694/1480 train_time:104256ms step_avg:152.42ms
step:695/1480 train_time:104417ms step_avg:152.43ms
step:696/1480 train_time:104576ms step_avg:152.44ms
step:697/1480 train_time:104739ms step_avg:152.46ms
step:698/1480 train_time:104900ms step_avg:152.47ms
step:699/1480 train_time:105063ms step_avg:152.49ms
step:700/1480 train_time:105226ms step_avg:152.50ms
step:701/1480 train_time:105387ms step_avg:152.51ms
step:702/1480 train_time:105546ms step_avg:152.52ms
step:703/1480 train_time:105707ms step_avg:152.54ms
step:704/1480 train_time:105868ms step_avg:152.55ms
step:705/1480 train_time:106033ms step_avg:152.57ms
step:706/1480 train_time:106197ms step_avg:152.58ms
step:707/1480 train_time:106358ms step_avg:152.59ms
step:708/1480 train_time:106520ms step_avg:152.61ms
step:709/1480 train_time:106681ms step_avg:152.62ms
step:710/1480 train_time:106840ms step_avg:152.63ms
step:711/1480 train_time:107002ms step_avg:152.64ms
step:712/1480 train_time:107165ms step_avg:152.66ms
step:713/1480 train_time:107329ms step_avg:152.67ms
step:714/1480 train_time:107492ms step_avg:152.69ms
step:715/1480 train_time:107653ms step_avg:152.70ms
step:716/1480 train_time:107814ms step_avg:152.71ms
step:717/1480 train_time:107976ms step_avg:152.72ms
step:718/1480 train_time:108135ms step_avg:152.73ms
step:719/1480 train_time:108295ms step_avg:152.74ms
step:720/1480 train_time:108457ms step_avg:152.76ms
step:721/1480 train_time:108619ms step_avg:152.77ms
step:722/1480 train_time:108780ms step_avg:152.78ms
step:723/1480 train_time:108940ms step_avg:152.79ms
step:724/1480 train_time:109102ms step_avg:152.80ms
step:725/1480 train_time:109264ms step_avg:152.82ms
step:726/1480 train_time:109429ms step_avg:152.83ms
step:727/1480 train_time:109592ms step_avg:152.85ms
step:728/1480 train_time:109754ms step_avg:152.86ms
step:729/1480 train_time:109914ms step_avg:152.87ms
step:730/1480 train_time:110076ms step_avg:152.88ms
step:731/1480 train_time:110238ms step_avg:152.90ms
step:732/1480 train_time:110398ms step_avg:152.91ms
step:733/1480 train_time:110560ms step_avg:152.92ms
step:734/1480 train_time:110721ms step_avg:152.93ms
step:735/1480 train_time:110882ms step_avg:152.94ms
step:736/1480 train_time:111044ms step_avg:152.95ms
step:737/1480 train_time:111205ms step_avg:152.96ms
step:738/1480 train_time:111367ms step_avg:152.98ms
step:739/1480 train_time:111527ms step_avg:152.99ms
step:740/1480 train_time:111693ms step_avg:153.00ms
step:741/1480 train_time:111857ms step_avg:153.02ms
step:742/1480 train_time:112021ms step_avg:153.03ms
step:743/1480 train_time:112181ms step_avg:153.04ms
step:744/1480 train_time:112343ms step_avg:153.06ms
step:745/1480 train_time:112507ms step_avg:153.07ms
step:746/1480 train_time:112668ms step_avg:153.08ms
step:747/1480 train_time:112829ms step_avg:153.09ms
step:748/1480 train_time:112996ms step_avg:153.11ms
step:749/1480 train_time:113161ms step_avg:153.13ms
step:750/1480 train_time:113320ms step_avg:153.14ms
step:750/1480 val_loss:3.5459 train_time:113384ms step_avg:153.22ms
step:751/1480 train_time:113484ms step_avg:153.15ms
step:752/1480 train_time:113643ms step_avg:153.16ms
step:753/1480 train_time:113803ms step_avg:153.17ms
step:754/1480 train_time:113964ms step_avg:153.18ms
step:755/1480 train_time:114124ms step_avg:153.19ms
step:756/1480 train_time:114285ms step_avg:153.20ms
step:757/1480 train_time:114451ms step_avg:153.21ms
step:758/1480 train_time:114612ms step_avg:153.22ms
step:759/1480 train_time:114776ms step_avg:153.24ms
step:760/1480 train_time:114938ms step_avg:153.25ms
step:761/1480 train_time:115100ms step_avg:153.26ms
step:762/1480 train_time:115261ms step_avg:153.27ms
step:763/1480 train_time:115423ms step_avg:153.28ms
step:764/1480 train_time:115585ms step_avg:153.30ms
step:765/1480 train_time:115745ms step_avg:153.30ms
step:766/1480 train_time:115907ms step_avg:153.32ms
step:767/1480 train_time:116068ms step_avg:153.33ms
step:768/1480 train_time:116230ms step_avg:153.34ms
step:769/1480 train_time:116395ms step_avg:153.35ms
step:770/1480 train_time:116559ms step_avg:153.37ms
step:771/1480 train_time:116721ms step_avg:153.38ms
step:772/1480 train_time:116883ms step_avg:153.39ms
step:773/1480 train_time:117045ms step_avg:153.40ms
step:774/1480 train_time:117206ms step_avg:153.41ms
step:775/1480 train_time:117367ms step_avg:153.42ms
step:776/1480 train_time:117532ms step_avg:153.44ms
step:777/1480 train_time:117700ms step_avg:153.45ms
step:778/1480 train_time:117862ms step_avg:153.47ms
step:779/1480 train_time:118024ms step_avg:153.48ms
step:780/1480 train_time:118187ms step_avg:153.49ms
step:781/1480 train_time:118351ms step_avg:153.50ms
step:782/1480 train_time:118514ms step_avg:153.52ms
step:783/1480 train_time:118677ms step_avg:153.53ms
step:784/1480 train_time:118841ms step_avg:153.54ms
step:785/1480 train_time:119003ms step_avg:153.55ms
step:786/1480 train_time:119168ms step_avg:153.57ms
step:787/1480 train_time:119332ms step_avg:153.58ms
step:788/1480 train_time:119497ms step_avg:153.60ms
step:789/1480 train_time:119659ms step_avg:153.61ms
step:790/1480 train_time:119823ms step_avg:153.62ms
step:791/1480 train_time:119990ms step_avg:153.64ms
step:792/1480 train_time:120157ms step_avg:153.65ms
step:793/1480 train_time:120319ms step_avg:153.66ms
step:794/1480 train_time:120483ms step_avg:153.68ms
step:795/1480 train_time:120650ms step_avg:153.69ms
step:796/1480 train_time:120817ms step_avg:153.71ms
step:797/1480 train_time:120981ms step_avg:153.72ms
step:798/1480 train_time:121144ms step_avg:153.74ms
step:799/1480 train_time:121309ms step_avg:153.75ms
step:800/1480 train_time:121472ms step_avg:153.76ms
step:801/1480 train_time:121636ms step_avg:153.78ms
step:802/1480 train_time:121804ms step_avg:153.79ms
step:803/1480 train_time:121966ms step_avg:153.80ms
step:804/1480 train_time:122126ms step_avg:153.81ms
step:805/1480 train_time:122293ms step_avg:153.83ms
step:806/1480 train_time:122455ms step_avg:153.84ms
step:807/1480 train_time:122617ms step_avg:153.85ms
step:808/1480 train_time:122780ms step_avg:153.86ms
step:809/1480 train_time:122942ms step_avg:153.87ms
step:810/1480 train_time:123104ms step_avg:153.88ms
step:811/1480 train_time:123267ms step_avg:153.89ms
step:812/1480 train_time:123431ms step_avg:153.90ms
step:813/1480 train_time:123593ms step_avg:153.91ms
step:814/1480 train_time:123756ms step_avg:153.93ms
step:815/1480 train_time:123920ms step_avg:153.94ms
step:816/1480 train_time:124085ms step_avg:153.95ms
step:817/1480 train_time:124248ms step_avg:153.96ms
step:818/1480 train_time:124409ms step_avg:153.97ms
step:819/1480 train_time:124573ms step_avg:153.98ms
step:820/1480 train_time:124736ms step_avg:154.00ms
step:821/1480 train_time:124900ms step_avg:154.01ms
step:822/1480 train_time:125063ms step_avg:154.02ms
step:823/1480 train_time:125225ms step_avg:154.03ms
step:824/1480 train_time:125386ms step_avg:154.04ms
step:825/1480 train_time:125551ms step_avg:154.05ms
step:826/1480 train_time:125719ms step_avg:154.07ms
step:827/1480 train_time:125882ms step_avg:154.08ms
step:828/1480 train_time:126045ms step_avg:154.09ms
step:829/1480 train_time:126207ms step_avg:154.10ms
step:830/1480 train_time:126371ms step_avg:154.11ms
step:831/1480 train_time:126536ms step_avg:154.12ms
step:832/1480 train_time:126701ms step_avg:154.14ms
step:833/1480 train_time:126864ms step_avg:154.15ms
step:834/1480 train_time:127029ms step_avg:154.16ms
step:835/1480 train_time:127193ms step_avg:154.17ms
step:836/1480 train_time:127358ms step_avg:154.19ms
step:837/1480 train_time:127522ms step_avg:154.20ms
step:838/1480 train_time:127686ms step_avg:154.21ms
step:839/1480 train_time:127847ms step_avg:154.22ms
step:840/1480 train_time:128007ms step_avg:154.23ms
step:841/1480 train_time:128167ms step_avg:154.23ms
step:842/1480 train_time:128331ms step_avg:154.24ms
step:843/1480 train_time:128495ms step_avg:154.26ms
step:844/1480 train_time:128658ms step_avg:154.27ms
step:845/1480 train_time:128822ms step_avg:154.28ms
step:846/1480 train_time:128987ms step_avg:154.29ms
step:847/1480 train_time:129150ms step_avg:154.30ms
step:848/1480 train_time:129312ms step_avg:154.31ms
step:849/1480 train_time:129476ms step_avg:154.32ms
step:850/1480 train_time:129638ms step_avg:154.33ms
step:851/1480 train_time:129803ms step_avg:154.34ms
step:852/1480 train_time:129965ms step_avg:154.35ms
step:853/1480 train_time:130125ms step_avg:154.36ms
step:854/1480 train_time:130290ms step_avg:154.37ms
step:855/1480 train_time:130456ms step_avg:154.39ms
step:856/1480 train_time:130618ms step_avg:154.39ms
step:857/1480 train_time:130783ms step_avg:154.41ms
step:858/1480 train_time:130948ms step_avg:154.42ms
step:859/1480 train_time:131113ms step_avg:154.43ms
step:860/1480 train_time:131276ms step_avg:154.44ms
step:861/1480 train_time:131441ms step_avg:154.46ms
step:862/1480 train_time:131610ms step_avg:154.47ms
step:863/1480 train_time:131781ms step_avg:154.49ms
step:864/1480 train_time:131944ms step_avg:154.50ms
step:865/1480 train_time:132104ms step_avg:154.51ms
step:866/1480 train_time:132271ms step_avg:154.52ms
step:867/1480 train_time:132435ms step_avg:154.53ms
step:868/1480 train_time:132598ms step_avg:154.54ms
step:869/1480 train_time:132760ms step_avg:154.55ms
step:870/1480 train_time:132925ms step_avg:154.56ms
step:871/1480 train_time:133088ms step_avg:154.57ms
step:872/1480 train_time:133250ms step_avg:154.58ms
step:873/1480 train_time:133412ms step_avg:154.59ms
step:874/1480 train_time:133580ms step_avg:154.61ms
step:875/1480 train_time:133745ms step_avg:154.62ms
step:875/1480 val_loss:3.5012 train_time:133809ms step_avg:154.69ms
step:876/1480 train_time:133910ms step_avg:154.63ms
step:877/1480 train_time:134074ms step_avg:154.64ms
step:878/1480 train_time:134236ms step_avg:154.65ms
step:879/1480 train_time:134400ms step_avg:154.66ms
step:880/1480 train_time:134563ms step_avg:154.67ms
step:881/1480 train_time:134727ms step_avg:154.68ms
step:882/1480 train_time:134891ms step_avg:154.69ms
step:883/1480 train_time:135057ms step_avg:154.70ms
step:884/1480 train_time:135224ms step_avg:154.72ms
step:885/1480 train_time:135390ms step_avg:154.73ms
step:886/1480 train_time:135555ms step_avg:154.74ms
step:887/1480 train_time:135723ms step_avg:154.76ms
step:888/1480 train_time:135896ms step_avg:154.78ms
step:889/1480 train_time:136065ms step_avg:154.79ms
step:890/1480 train_time:136228ms step_avg:154.80ms
step:891/1480 train_time:136392ms step_avg:154.82ms
step:892/1480 train_time:136557ms step_avg:154.83ms
step:893/1480 train_time:136719ms step_avg:154.83ms
step:894/1480 train_time:136886ms step_avg:154.85ms
step:895/1480 train_time:137052ms step_avg:154.86ms
step:896/1480 train_time:137216ms step_avg:154.87ms
step:897/1480 train_time:137383ms step_avg:154.88ms
step:898/1480 train_time:137549ms step_avg:154.90ms
step:899/1480 train_time:137713ms step_avg:154.91ms
step:900/1480 train_time:137876ms step_avg:154.92ms
step:901/1480 train_time:138040ms step_avg:154.93ms
step:902/1480 train_time:138205ms step_avg:154.94ms
step:903/1480 train_time:138376ms step_avg:154.96ms
step:904/1480 train_time:138542ms step_avg:154.97ms
step:905/1480 train_time:138705ms step_avg:154.98ms
step:906/1480 train_time:138873ms step_avg:154.99ms
step:907/1480 train_time:139041ms step_avg:155.01ms
step:908/1480 train_time:139206ms step_avg:155.02ms
step:909/1480 train_time:139371ms step_avg:155.03ms
step:910/1480 train_time:139543ms step_avg:155.05ms
step:911/1480 train_time:139708ms step_avg:155.06ms
step:912/1480 train_time:139874ms step_avg:155.07ms
step:913/1480 train_time:140040ms step_avg:155.08ms
step:914/1480 train_time:140209ms step_avg:155.10ms
step:915/1480 train_time:140379ms step_avg:155.11ms
step:916/1480 train_time:140544ms step_avg:155.13ms
step:917/1480 train_time:140708ms step_avg:155.14ms
step:918/1480 train_time:140876ms step_avg:155.15ms
step:919/1480 train_time:141048ms step_avg:155.17ms
step:920/1480 train_time:141213ms step_avg:155.18ms
step:921/1480 train_time:141379ms step_avg:155.19ms
step:922/1480 train_time:141548ms step_avg:155.21ms
step:923/1480 train_time:141710ms step_avg:155.21ms
step:924/1480 train_time:141873ms step_avg:155.22ms
step:925/1480 train_time:142039ms step_avg:155.23ms
step:926/1480 train_time:142203ms step_avg:155.24ms
step:927/1480 train_time:142367ms step_avg:155.25ms
step:928/1480 train_time:142534ms step_avg:155.27ms
step:929/1480 train_time:142699ms step_avg:155.28ms
step:930/1480 train_time:142866ms step_avg:155.29ms
step:931/1480 train_time:143030ms step_avg:155.30ms
step:932/1480 train_time:143195ms step_avg:155.31ms
step:933/1480 train_time:143363ms step_avg:155.32ms
step:934/1480 train_time:143531ms step_avg:155.34ms
step:935/1480 train_time:143700ms step_avg:155.35ms
step:936/1480 train_time:143869ms step_avg:155.37ms
step:937/1480 train_time:144040ms step_avg:155.38ms
step:938/1480 train_time:144202ms step_avg:155.39ms
step:939/1480 train_time:144371ms step_avg:155.41ms
step:940/1480 train_time:144537ms step_avg:155.42ms
step:941/1480 train_time:144701ms step_avg:155.42ms
step:942/1480 train_time:144867ms step_avg:155.44ms
step:943/1480 train_time:145038ms step_avg:155.45ms
step:944/1480 train_time:145210ms step_avg:155.47ms
step:945/1480 train_time:145374ms step_avg:155.48ms
step:946/1480 train_time:145542ms step_avg:155.49ms
step:947/1480 train_time:145710ms step_avg:155.51ms
step:948/1480 train_time:145874ms step_avg:155.52ms
step:949/1480 train_time:146039ms step_avg:155.53ms
step:950/1480 train_time:146204ms step_avg:155.54ms
step:951/1480 train_time:146374ms step_avg:155.55ms
step:952/1480 train_time:146539ms step_avg:155.56ms
step:953/1480 train_time:146708ms step_avg:155.58ms
step:954/1480 train_time:146875ms step_avg:155.59ms
step:955/1480 train_time:147040ms step_avg:155.60ms
step:956/1480 train_time:147205ms step_avg:155.61ms
step:957/1480 train_time:147372ms step_avg:155.62ms
step:958/1480 train_time:147542ms step_avg:155.64ms
step:959/1480 train_time:147707ms step_avg:155.65ms
step:960/1480 train_time:147874ms step_avg:155.66ms
step:961/1480 train_time:148039ms step_avg:155.67ms
step:962/1480 train_time:148205ms step_avg:155.68ms
step:963/1480 train_time:148370ms step_avg:155.69ms
step:964/1480 train_time:148537ms step_avg:155.70ms
step:965/1480 train_time:148702ms step_avg:155.71ms
step:966/1480 train_time:148867ms step_avg:155.72ms
step:967/1480 train_time:149031ms step_avg:155.73ms
step:968/1480 train_time:149196ms step_avg:155.74ms
step:969/1480 train_time:149365ms step_avg:155.75ms
step:970/1480 train_time:149529ms step_avg:155.76ms
step:971/1480 train_time:149694ms step_avg:155.77ms
step:972/1480 train_time:149856ms step_avg:155.78ms
step:973/1480 train_time:150020ms step_avg:155.78ms
step:974/1480 train_time:150189ms step_avg:155.80ms
step:975/1480 train_time:150354ms step_avg:155.81ms
step:976/1480 train_time:150519ms step_avg:155.82ms
step:977/1480 train_time:150683ms step_avg:155.83ms
step:978/1480 train_time:150850ms step_avg:155.84ms
step:979/1480 train_time:151014ms step_avg:155.85ms
step:980/1480 train_time:151180ms step_avg:155.86ms
step:981/1480 train_time:151350ms step_avg:155.87ms
step:982/1480 train_time:151513ms step_avg:155.88ms
step:983/1480 train_time:151677ms step_avg:155.89ms
step:984/1480 train_time:151842ms step_avg:155.89ms
step:985/1480 train_time:152009ms step_avg:155.91ms
step:986/1480 train_time:152174ms step_avg:155.92ms
step:987/1480 train_time:152339ms step_avg:155.92ms
step:988/1480 train_time:152507ms step_avg:155.94ms
step:989/1480 train_time:152673ms step_avg:155.95ms
step:990/1480 train_time:152844ms step_avg:155.96ms
step:991/1480 train_time:153011ms step_avg:155.97ms
step:992/1480 train_time:153185ms step_avg:155.99ms
step:993/1480 train_time:153361ms step_avg:156.01ms
step:994/1480 train_time:153527ms step_avg:156.02ms
step:995/1480 train_time:153691ms step_avg:156.03ms
step:996/1480 train_time:153853ms step_avg:156.04ms
step:997/1480 train_time:154017ms step_avg:156.05ms
step:998/1480 train_time:154181ms step_avg:156.05ms
step:999/1480 train_time:154347ms step_avg:156.06ms
step:1000/1480 train_time:154515ms step_avg:156.08ms
step:1000/1480 val_loss:3.4385 train_time:154583ms step_avg:156.14ms
step:1001/1480 train_time:154684ms step_avg:156.09ms
step:1002/1480 train_time:154851ms step_avg:156.10ms
step:1003/1480 train_time:155022ms step_avg:156.11ms
step:1004/1480 train_time:155190ms step_avg:156.13ms
step:1005/1480 train_time:155360ms step_avg:156.14ms
step:1006/1480 train_time:155527ms step_avg:156.15ms
step:1007/1480 train_time:155692ms step_avg:156.16ms
step:1008/1480 train_time:155860ms step_avg:156.17ms
step:1009/1480 train_time:156035ms step_avg:156.19ms
step:1010/1480 train_time:156201ms step_avg:156.20ms
step:1011/1480 train_time:156367ms step_avg:156.21ms
step:1012/1480 train_time:156532ms step_avg:156.22ms
step:1013/1480 train_time:156702ms step_avg:156.23ms
step:1014/1480 train_time:156868ms step_avg:156.24ms
step:1015/1480 train_time:157038ms step_avg:156.26ms
step:1016/1480 train_time:157207ms step_avg:156.27ms
step:1017/1480 train_time:157378ms step_avg:156.28ms
step:1018/1480 train_time:157546ms step_avg:156.30ms
step:1019/1480 train_time:157714ms step_avg:156.31ms
step:1020/1480 train_time:157884ms step_avg:156.32ms
step:1021/1480 train_time:158047ms step_avg:156.33ms
step:1022/1480 train_time:158214ms step_avg:156.34ms
step:1023/1480 train_time:158381ms step_avg:156.35ms
step:1024/1480 train_time:158547ms step_avg:156.36ms
step:1025/1480 train_time:158718ms step_avg:156.37ms
step:1026/1480 train_time:158885ms step_avg:156.38ms
step:1027/1480 train_time:159052ms step_avg:156.39ms
step:1028/1480 train_time:159225ms step_avg:156.41ms
step:1029/1480 train_time:159400ms step_avg:156.43ms
step:1030/1480 train_time:159567ms step_avg:156.44ms
step:1031/1480 train_time:159730ms step_avg:156.45ms
step:1032/1480 train_time:159904ms step_avg:156.46ms
step:1033/1480 train_time:160069ms step_avg:156.47ms
step:1034/1480 train_time:160238ms step_avg:156.48ms
step:1035/1480 train_time:160405ms step_avg:156.49ms
step:1036/1480 train_time:160570ms step_avg:156.50ms
step:1037/1480 train_time:160738ms step_avg:156.51ms
step:1038/1480 train_time:160905ms step_avg:156.52ms
step:1039/1480 train_time:161076ms step_avg:156.54ms
step:1040/1480 train_time:161242ms step_avg:156.55ms
step:1041/1480 train_time:161409ms step_avg:156.56ms
step:1042/1480 train_time:161573ms step_avg:156.56ms
step:1043/1480 train_time:161740ms step_avg:156.57ms
step:1044/1480 train_time:161905ms step_avg:156.58ms
step:1045/1480 train_time:162076ms step_avg:156.60ms
step:1046/1480 train_time:162244ms step_avg:156.61ms
step:1047/1480 train_time:162410ms step_avg:156.61ms
step:1048/1480 train_time:162576ms step_avg:156.62ms
step:1049/1480 train_time:162743ms step_avg:156.63ms
step:1050/1480 train_time:162911ms step_avg:156.65ms
step:1051/1480 train_time:163082ms step_avg:156.66ms
step:1052/1480 train_time:163248ms step_avg:156.67ms
step:1053/1480 train_time:163414ms step_avg:156.68ms
step:1054/1480 train_time:163583ms step_avg:156.69ms
step:1055/1480 train_time:163748ms step_avg:156.70ms
step:1056/1480 train_time:163912ms step_avg:156.70ms
step:1057/1480 train_time:164080ms step_avg:156.71ms
step:1058/1480 train_time:164248ms step_avg:156.73ms
step:1059/1480 train_time:164421ms step_avg:156.74ms
step:1060/1480 train_time:164589ms step_avg:156.75ms
step:1061/1480 train_time:164752ms step_avg:156.76ms
step:1062/1480 train_time:164919ms step_avg:156.77ms
step:1063/1480 train_time:165085ms step_avg:156.78ms
step:1064/1480 train_time:165249ms step_avg:156.78ms
step:1065/1480 train_time:165416ms step_avg:156.79ms
step:1066/1480 train_time:165583ms step_avg:156.80ms
step:1067/1480 train_time:165752ms step_avg:156.81ms
step:1068/1480 train_time:165919ms step_avg:156.82ms
step:1069/1480 train_time:166089ms step_avg:156.84ms
step:1070/1480 train_time:166255ms step_avg:156.84ms
step:1071/1480 train_time:166428ms step_avg:156.86ms
step:1072/1480 train_time:166594ms step_avg:156.87ms
step:1073/1480 train_time:166758ms step_avg:156.87ms
step:1074/1480 train_time:166925ms step_avg:156.88ms
step:1075/1480 train_time:167097ms step_avg:156.90ms
step:1076/1480 train_time:167265ms step_avg:156.91ms
step:1077/1480 train_time:167430ms step_avg:156.92ms
step:1078/1480 train_time:167605ms step_avg:156.93ms
step:1079/1480 train_time:167779ms step_avg:156.95ms
step:1080/1480 train_time:167948ms step_avg:156.96ms
step:1081/1480 train_time:168114ms step_avg:156.97ms
step:1082/1480 train_time:168281ms step_avg:156.98ms
step:1083/1480 train_time:168447ms step_avg:156.99ms
step:1084/1480 train_time:168614ms step_avg:157.00ms
step:1085/1480 train_time:168783ms step_avg:157.01ms
step:1086/1480 train_time:168949ms step_avg:157.02ms
step:1087/1480 train_time:169115ms step_avg:157.02ms
step:1088/1480 train_time:169286ms step_avg:157.04ms
step:1089/1480 train_time:169460ms step_avg:157.05ms
step:1090/1480 train_time:169630ms step_avg:157.07ms
step:1091/1480 train_time:169799ms step_avg:157.08ms
step:1092/1480 train_time:169967ms step_avg:157.09ms
step:1093/1480 train_time:170136ms step_avg:157.10ms
step:1094/1480 train_time:170302ms step_avg:157.11ms
step:1095/1480 train_time:170468ms step_avg:157.11ms
step:1096/1480 train_time:170639ms step_avg:157.13ms
step:1097/1480 train_time:170808ms step_avg:157.14ms
step:1098/1480 train_time:170981ms step_avg:157.15ms
step:1099/1480 train_time:171151ms step_avg:157.16ms
step:1100/1480 train_time:171323ms step_avg:157.18ms
step:1101/1480 train_time:171493ms step_avg:157.19ms
step:1102/1480 train_time:171666ms step_avg:157.20ms
step:1103/1480 train_time:171842ms step_avg:157.22ms
step:1104/1480 train_time:172010ms step_avg:157.23ms
step:1105/1480 train_time:172181ms step_avg:157.24ms
step:1106/1480 train_time:172349ms step_avg:157.25ms
step:1107/1480 train_time:172517ms step_avg:157.26ms
step:1108/1480 train_time:172683ms step_avg:157.27ms
step:1109/1480 train_time:172848ms step_avg:157.28ms
step:1110/1480 train_time:173014ms step_avg:157.29ms
step:1111/1480 train_time:173181ms step_avg:157.29ms
step:1112/1480 train_time:173350ms step_avg:157.31ms
step:1113/1480 train_time:173531ms step_avg:157.33ms
step:1114/1480 train_time:173704ms step_avg:157.34ms
step:1115/1480 train_time:173877ms step_avg:157.35ms
step:1116/1480 train_time:174044ms step_avg:157.36ms
step:1117/1480 train_time:174217ms step_avg:157.38ms
step:1118/1480 train_time:174393ms step_avg:157.39ms
step:1119/1480 train_time:174560ms step_avg:157.40ms
step:1120/1480 train_time:174727ms step_avg:157.41ms
step:1121/1480 train_time:174897ms step_avg:157.42ms
step:1122/1480 train_time:175063ms step_avg:157.43ms
step:1123/1480 train_time:175230ms step_avg:157.44ms
step:1124/1480 train_time:175398ms step_avg:157.45ms
step:1125/1480 train_time:175566ms step_avg:157.46ms
step:1125/1480 val_loss:3.3832 train_time:175633ms step_avg:157.52ms
step:1126/1480 train_time:175735ms step_avg:157.47ms
step:1127/1480 train_time:175905ms step_avg:157.48ms
step:1128/1480 train_time:176076ms step_avg:157.49ms
step:1129/1480 train_time:176249ms step_avg:157.51ms
step:1130/1480 train_time:176418ms step_avg:157.52ms
step:1131/1480 train_time:176595ms step_avg:157.53ms
step:1132/1480 train_time:176760ms step_avg:157.54ms
step:1133/1480 train_time:176933ms step_avg:157.55ms
step:1134/1480 train_time:177103ms step_avg:157.57ms
step:1135/1480 train_time:177271ms step_avg:157.57ms
step:1136/1480 train_time:177440ms step_avg:157.58ms
step:1137/1480 train_time:177610ms step_avg:157.60ms
step:1138/1480 train_time:177778ms step_avg:157.60ms
step:1139/1480 train_time:177947ms step_avg:157.61ms
step:1140/1480 train_time:178115ms step_avg:157.62ms
step:1141/1480 train_time:178289ms step_avg:157.64ms
step:1142/1480 train_time:178455ms step_avg:157.65ms
step:1143/1480 train_time:178627ms step_avg:157.66ms
step:1144/1480 train_time:178796ms step_avg:157.67ms
step:1145/1480 train_time:178960ms step_avg:157.67ms
step:1146/1480 train_time:179130ms step_avg:157.69ms
step:1147/1480 train_time:179298ms step_avg:157.69ms
step:1148/1480 train_time:179468ms step_avg:157.70ms
step:1149/1480 train_time:179637ms step_avg:157.71ms
step:1150/1480 train_time:179805ms step_avg:157.72ms
step:1151/1480 train_time:179977ms step_avg:157.74ms
step:1152/1480 train_time:180149ms step_avg:157.75ms
step:1153/1480 train_time:180321ms step_avg:157.76ms
step:1154/1480 train_time:180489ms step_avg:157.77ms
step:1155/1480 train_time:180660ms step_avg:157.78ms
step:1156/1480 train_time:180839ms step_avg:157.80ms
step:1157/1480 train_time:181009ms step_avg:157.81ms
step:1158/1480 train_time:181176ms step_avg:157.82ms
step:1159/1480 train_time:181343ms step_avg:157.83ms
step:1160/1480 train_time:181509ms step_avg:157.83ms
step:1161/1480 train_time:181678ms step_avg:157.84ms
step:1162/1480 train_time:181848ms step_avg:157.85ms
step:1163/1480 train_time:182018ms step_avg:157.86ms
step:1164/1480 train_time:182186ms step_avg:157.87ms
step:1165/1480 train_time:182351ms step_avg:157.88ms
step:1166/1480 train_time:182520ms step_avg:157.89ms
step:1167/1480 train_time:182690ms step_avg:157.90ms
step:1168/1480 train_time:182858ms step_avg:157.91ms
step:1169/1480 train_time:183028ms step_avg:157.92ms
step:1170/1480 train_time:183196ms step_avg:157.93ms
step:1171/1480 train_time:183363ms step_avg:157.94ms
step:1172/1480 train_time:183531ms step_avg:157.94ms
step:1173/1480 train_time:183702ms step_avg:157.96ms
step:1174/1480 train_time:183885ms step_avg:157.98ms
step:1175/1480 train_time:184056ms step_avg:157.99ms
step:1176/1480 train_time:184227ms step_avg:158.00ms
step:1177/1480 train_time:184405ms step_avg:158.02ms
step:1178/1480 train_time:184573ms step_avg:158.02ms
step:1179/1480 train_time:184738ms step_avg:158.03ms
step:1180/1480 train_time:184918ms step_avg:158.05ms
step:1181/1480 train_time:185088ms step_avg:158.06ms
step:1182/1480 train_time:185256ms step_avg:158.07ms
step:1183/1480 train_time:185426ms step_avg:158.08ms
step:1184/1480 train_time:185594ms step_avg:158.09ms
step:1185/1480 train_time:185768ms step_avg:158.10ms
step:1186/1480 train_time:185939ms step_avg:158.11ms
step:1187/1480 train_time:186122ms step_avg:158.13ms
step:1188/1480 train_time:186290ms step_avg:158.14ms
step:1189/1480 train_time:186463ms step_avg:158.15ms
step:1190/1480 train_time:186630ms step_avg:158.16ms
step:1191/1480 train_time:186801ms step_avg:158.17ms
step:1192/1480 train_time:186968ms step_avg:158.18ms
step:1193/1480 train_time:187133ms step_avg:158.19ms
step:1194/1480 train_time:187304ms step_avg:158.20ms
step:1195/1480 train_time:187477ms step_avg:158.21ms
step:1196/1480 train_time:187660ms step_avg:158.23ms
step:1197/1480 train_time:187832ms step_avg:158.24ms
step:1198/1480 train_time:188014ms step_avg:158.26ms
step:1199/1480 train_time:188185ms step_avg:158.27ms
step:1200/1480 train_time:188353ms step_avg:158.28ms
step:1201/1480 train_time:188519ms step_avg:158.29ms
step:1202/1480 train_time:188701ms step_avg:158.31ms
step:1203/1480 train_time:188876ms step_avg:158.32ms
step:1204/1480 train_time:189049ms step_avg:158.33ms
step:1205/1480 train_time:189217ms step_avg:158.34ms
step:1206/1480 train_time:189385ms step_avg:158.35ms
step:1207/1480 train_time:189553ms step_avg:158.36ms
step:1208/1480 train_time:189720ms step_avg:158.36ms
step:1209/1480 train_time:189894ms step_avg:158.38ms
step:1210/1480 train_time:190071ms step_avg:158.39ms
step:1211/1480 train_time:190244ms step_avg:158.40ms
step:1212/1480 train_time:190415ms step_avg:158.42ms
step:1213/1480 train_time:190588ms step_avg:158.43ms
step:1214/1480 train_time:190765ms step_avg:158.44ms
step:1215/1480 train_time:190936ms step_avg:158.45ms
step:1216/1480 train_time:191106ms step_avg:158.46ms
step:1217/1480 train_time:191281ms step_avg:158.48ms
step:1218/1480 train_time:191451ms step_avg:158.49ms
step:1219/1480 train_time:191630ms step_avg:158.50ms
step:1220/1480 train_time:191799ms step_avg:158.51ms
step:1221/1480 train_time:191968ms step_avg:158.52ms
step:1222/1480 train_time:192134ms step_avg:158.53ms
step:1223/1480 train_time:192305ms step_avg:158.54ms
step:1224/1480 train_time:192480ms step_avg:158.55ms
step:1225/1480 train_time:192652ms step_avg:158.56ms
step:1226/1480 train_time:192825ms step_avg:158.57ms
step:1227/1480 train_time:192998ms step_avg:158.58ms
step:1228/1480 train_time:193168ms step_avg:158.59ms
step:1229/1480 train_time:193341ms step_avg:158.61ms
step:1230/1480 train_time:193520ms step_avg:158.62ms
step:1231/1480 train_time:193695ms step_avg:158.64ms
step:1232/1480 train_time:193870ms step_avg:158.65ms
step:1233/1480 train_time:194040ms step_avg:158.66ms
step:1234/1480 train_time:194211ms step_avg:158.67ms
step:1235/1480 train_time:194385ms step_avg:158.68ms
step:1236/1480 train_time:194552ms step_avg:158.69ms
step:1237/1480 train_time:194723ms step_avg:158.70ms
step:1238/1480 train_time:194909ms step_avg:158.72ms
step:1239/1480 train_time:195079ms step_avg:158.73ms
step:1240/1480 train_time:195249ms step_avg:158.74ms
step:1241/1480 train_time:195422ms step_avg:158.75ms
step:1242/1480 train_time:195592ms step_avg:158.76ms
step:1243/1480 train_time:195767ms step_avg:158.77ms
step:1244/1480 train_time:195933ms step_avg:158.78ms
step:1245/1480 train_time:196102ms step_avg:158.79ms
step:1246/1480 train_time:196271ms step_avg:158.80ms
step:1247/1480 train_time:196442ms step_avg:158.81ms
step:1248/1480 train_time:196612ms step_avg:158.81ms
step:1249/1480 train_time:196779ms step_avg:158.82ms
step:1250/1480 train_time:196948ms step_avg:158.83ms
step:1250/1480 val_loss:3.3343 train_time:197018ms step_avg:158.89ms
step:1251/1480 train_time:197129ms step_avg:158.85ms
step:1252/1480 train_time:197299ms step_avg:158.86ms
step:1253/1480 train_time:197466ms step_avg:158.86ms
step:1254/1480 train_time:197640ms step_avg:158.87ms
step:1255/1480 train_time:197826ms step_avg:158.90ms
step:1256/1480 train_time:197999ms step_avg:158.91ms
step:1257/1480 train_time:198169ms step_avg:158.92ms
step:1258/1480 train_time:198342ms step_avg:158.93ms
step:1259/1480 train_time:198513ms step_avg:158.94ms
step:1260/1480 train_time:198680ms step_avg:158.94ms
step:1261/1480 train_time:198853ms step_avg:158.96ms
step:1262/1480 train_time:199029ms step_avg:158.97ms
step:1263/1480 train_time:199203ms step_avg:158.98ms
step:1264/1480 train_time:199369ms step_avg:158.99ms
step:1265/1480 train_time:199537ms step_avg:158.99ms
step:1266/1480 train_time:199708ms step_avg:159.00ms
step:1267/1480 train_time:199879ms step_avg:159.01ms
step:1268/1480 train_time:200052ms step_avg:159.02ms
step:1269/1480 train_time:200228ms step_avg:159.04ms
step:1270/1480 train_time:200399ms step_avg:159.05ms
step:1271/1480 train_time:200569ms step_avg:159.06ms
step:1272/1480 train_time:200735ms step_avg:159.06ms
step:1273/1480 train_time:200905ms step_avg:159.07ms
step:1274/1480 train_time:201077ms step_avg:159.08ms
step:1275/1480 train_time:201245ms step_avg:159.09ms
step:1276/1480 train_time:201412ms step_avg:159.09ms
step:1277/1480 train_time:201582ms step_avg:159.10ms
step:1278/1480 train_time:201750ms step_avg:159.11ms
step:1279/1480 train_time:201922ms step_avg:159.12ms
step:1280/1480 train_time:202102ms step_avg:159.14ms
step:1281/1480 train_time:202271ms step_avg:159.14ms
step:1282/1480 train_time:202437ms step_avg:159.15ms
step:1283/1480 train_time:202608ms step_avg:159.16ms
step:1284/1480 train_time:202778ms step_avg:159.17ms
step:1285/1480 train_time:202946ms step_avg:159.17ms
step:1286/1480 train_time:203115ms step_avg:159.18ms
step:1287/1480 train_time:203286ms step_avg:159.19ms
step:1288/1480 train_time:203458ms step_avg:159.20ms
step:1289/1480 train_time:203641ms step_avg:159.22ms
step:1290/1480 train_time:203822ms step_avg:159.24ms
step:1291/1480 train_time:203995ms step_avg:159.25ms
step:1292/1480 train_time:204169ms step_avg:159.26ms
step:1293/1480 train_time:204343ms step_avg:159.27ms
step:1294/1480 train_time:204515ms step_avg:159.28ms
step:1295/1480 train_time:204687ms step_avg:159.29ms
step:1296/1480 train_time:204860ms step_avg:159.30ms
step:1297/1480 train_time:205032ms step_avg:159.31ms
step:1298/1480 train_time:205201ms step_avg:159.32ms
step:1299/1480 train_time:205372ms step_avg:159.33ms
step:1300/1480 train_time:205540ms step_avg:159.33ms
step:1301/1480 train_time:205708ms step_avg:159.34ms
step:1302/1480 train_time:205884ms step_avg:159.35ms
step:1303/1480 train_time:206061ms step_avg:159.37ms
step:1304/1480 train_time:206236ms step_avg:159.38ms
step:1305/1480 train_time:206404ms step_avg:159.39ms
step:1306/1480 train_time:206578ms step_avg:159.40ms
step:1307/1480 train_time:206745ms step_avg:159.40ms
step:1308/1480 train_time:206915ms step_avg:159.41ms
step:1309/1480 train_time:207088ms step_avg:159.42ms
step:1310/1480 train_time:207256ms step_avg:159.43ms
step:1311/1480 train_time:207423ms step_avg:159.43ms
step:1312/1480 train_time:207596ms step_avg:159.44ms
step:1313/1480 train_time:207764ms step_avg:159.45ms
step:1314/1480 train_time:207937ms step_avg:159.46ms
step:1315/1480 train_time:208109ms step_avg:159.47ms
step:1316/1480 train_time:208276ms step_avg:159.48ms
step:1317/1480 train_time:208446ms step_avg:159.48ms
step:1318/1480 train_time:208625ms step_avg:159.50ms
step:1319/1480 train_time:208799ms step_avg:159.51ms
step:1320/1480 train_time:208976ms step_avg:159.52ms
step:1321/1480 train_time:209148ms step_avg:159.53ms
step:1322/1480 train_time:209330ms step_avg:159.55ms
step:1323/1480 train_time:209501ms step_avg:159.56ms
step:1324/1480 train_time:209676ms step_avg:159.57ms
step:1325/1480 train_time:209858ms step_avg:159.59ms
step:1326/1480 train_time:210035ms step_avg:159.60ms
step:1327/1480 train_time:210204ms step_avg:159.61ms
step:1328/1480 train_time:210376ms step_avg:159.62ms
step:1329/1480 train_time:210571ms step_avg:159.64ms
step:1330/1480 train_time:210750ms step_avg:159.66ms
step:1331/1480 train_time:210920ms step_avg:159.67ms
step:1332/1480 train_time:211095ms step_avg:159.68ms
step:1333/1480 train_time:211271ms step_avg:159.69ms
step:1334/1480 train_time:211442ms step_avg:159.70ms
step:1335/1480 train_time:211611ms step_avg:159.71ms
step:1336/1480 train_time:211795ms step_avg:159.72ms
step:1337/1480 train_time:211970ms step_avg:159.74ms
step:1338/1480 train_time:212141ms step_avg:159.74ms
step:1339/1480 train_time:212316ms step_avg:159.76ms
step:1340/1480 train_time:212487ms step_avg:159.76ms
step:1341/1480 train_time:212656ms step_avg:159.77ms
step:1342/1480 train_time:212830ms step_avg:159.78ms
step:1343/1480 train_time:213000ms step_avg:159.79ms
step:1344/1480 train_time:213171ms step_avg:159.80ms
step:1345/1480 train_time:213351ms step_avg:159.81ms
step:1346/1480 train_time:213520ms step_avg:159.82ms
step:1347/1480 train_time:213690ms step_avg:159.83ms
step:1348/1480 train_time:213861ms step_avg:159.84ms
step:1349/1480 train_time:214030ms step_avg:159.84ms
step:1350/1480 train_time:214204ms step_avg:159.85ms
step:1351/1480 train_time:214374ms step_avg:159.86ms
step:1352/1480 train_time:214543ms step_avg:159.87ms
step:1353/1480 train_time:214720ms step_avg:159.88ms
step:1354/1480 train_time:214891ms step_avg:159.89ms
step:1355/1480 train_time:215058ms step_avg:159.89ms
step:1356/1480 train_time:215232ms step_avg:159.90ms
step:1357/1480 train_time:215404ms step_avg:159.91ms
step:1358/1480 train_time:215576ms step_avg:159.92ms
step:1359/1480 train_time:215748ms step_avg:159.93ms
step:1360/1480 train_time:215923ms step_avg:159.94ms
step:1361/1480 train_time:216101ms step_avg:159.96ms
step:1362/1480 train_time:216276ms step_avg:159.97ms
step:1363/1480 train_time:216457ms step_avg:159.98ms
step:1364/1480 train_time:216626ms step_avg:159.99ms
step:1365/1480 train_time:216794ms step_avg:160.00ms
step:1366/1480 train_time:216966ms step_avg:160.00ms
step:1367/1480 train_time:217136ms step_avg:160.01ms
step:1368/1480 train_time:217311ms step_avg:160.02ms
step:1369/1480 train_time:217493ms step_avg:160.04ms
step:1370/1480 train_time:217671ms step_avg:160.05ms
step:1371/1480 train_time:217843ms step_avg:160.06ms
step:1372/1480 train_time:218020ms step_avg:160.07ms
step:1373/1480 train_time:218188ms step_avg:160.08ms
step:1374/1480 train_time:218364ms step_avg:160.09ms
step:1375/1480 train_time:218535ms step_avg:160.10ms
step:1375/1480 val_loss:3.2957 train_time:218603ms step_avg:160.15ms
step:1376/1480 train_time:218709ms step_avg:160.11ms
step:1377/1480 train_time:218883ms step_avg:160.12ms
step:1378/1480 train_time:219051ms step_avg:160.13ms
step:1379/1480 train_time:219227ms step_avg:160.14ms
step:1380/1480 train_time:219400ms step_avg:160.15ms
step:1381/1480 train_time:219583ms step_avg:160.16ms
step:1382/1480 train_time:219754ms step_avg:160.17ms
step:1383/1480 train_time:219926ms step_avg:160.18ms
step:1384/1480 train_time:220104ms step_avg:160.19ms
step:1385/1480 train_time:220270ms step_avg:160.20ms
step:1386/1480 train_time:220442ms step_avg:160.20ms
step:1387/1480 train_time:220613ms step_avg:160.21ms
step:1388/1480 train_time:220781ms step_avg:160.22ms
step:1389/1480 train_time:220954ms step_avg:160.23ms
step:1390/1480 train_time:221123ms step_avg:160.23ms
step:1391/1480 train_time:221292ms step_avg:160.24ms
step:1392/1480 train_time:221465ms step_avg:160.25ms
step:1393/1480 train_time:221635ms step_avg:160.26ms
step:1394/1480 train_time:221806ms step_avg:160.26ms
step:1395/1480 train_time:221974ms step_avg:160.27ms
step:1396/1480 train_time:222144ms step_avg:160.28ms
step:1397/1480 train_time:222311ms step_avg:160.28ms
step:1398/1480 train_time:222478ms step_avg:160.29ms
step:1399/1480 train_time:222648ms step_avg:160.29ms
step:1400/1480 train_time:222825ms step_avg:160.31ms
step:1401/1480 train_time:222990ms step_avg:160.31ms
step:1402/1480 train_time:223163ms step_avg:160.32ms
step:1403/1480 train_time:223340ms step_avg:160.33ms
step:1404/1480 train_time:223511ms step_avg:160.34ms
step:1405/1480 train_time:223687ms step_avg:160.35ms
step:1406/1480 train_time:223860ms step_avg:160.36ms
step:1407/1480 train_time:224028ms step_avg:160.36ms
step:1408/1480 train_time:224195ms step_avg:160.37ms
step:1409/1480 train_time:224379ms step_avg:160.39ms
step:1410/1480 train_time:224548ms step_avg:160.39ms
step:1411/1480 train_time:224717ms step_avg:160.40ms
step:1412/1480 train_time:224888ms step_avg:160.40ms
step:1413/1480 train_time:225058ms step_avg:160.41ms
step:1414/1480 train_time:225230ms step_avg:160.42ms
step:1415/1480 train_time:225407ms step_avg:160.43ms
step:1416/1480 train_time:225593ms step_avg:160.45ms
step:1417/1480 train_time:225767ms step_avg:160.46ms
step:1418/1480 train_time:225939ms step_avg:160.47ms
step:1419/1480 train_time:226113ms step_avg:160.48ms
step:1420/1480 train_time:226287ms step_avg:160.49ms
step:1421/1480 train_time:226462ms step_avg:160.50ms
step:1422/1480 train_time:226633ms step_avg:160.51ms
step:1423/1480 train_time:226804ms step_avg:160.51ms
step:1424/1480 train_time:226982ms step_avg:160.52ms
step:1425/1480 train_time:227162ms step_avg:160.54ms
step:1426/1480 train_time:227333ms step_avg:160.55ms
step:1427/1480 train_time:227509ms step_avg:160.56ms
step:1428/1480 train_time:227682ms step_avg:160.57ms
step:1429/1480 train_time:227850ms step_avg:160.57ms
step:1430/1480 train_time:228025ms step_avg:160.58ms
step:1431/1480 train_time:228201ms step_avg:160.59ms
step:1432/1480 train_time:228378ms step_avg:160.60ms
step:1433/1480 train_time:228555ms step_avg:160.61ms
step:1434/1480 train_time:228736ms step_avg:160.63ms
step:1435/1480 train_time:228910ms step_avg:160.64ms
step:1436/1480 train_time:229085ms step_avg:160.65ms
step:1437/1480 train_time:229255ms step_avg:160.66ms
step:1438/1480 train_time:229426ms step_avg:160.66ms
step:1439/1480 train_time:229600ms step_avg:160.67ms
step:1440/1480 train_time:229769ms step_avg:160.68ms
step:1441/1480 train_time:229942ms step_avg:160.69ms
step:1442/1480 train_time:230117ms step_avg:160.70ms
step:1443/1480 train_time:230307ms step_avg:160.72ms
step:1444/1480 train_time:230478ms step_avg:160.72ms
step:1445/1480 train_time:230650ms step_avg:160.73ms
step:1446/1480 train_time:230826ms step_avg:160.74ms
step:1447/1480 train_time:231005ms step_avg:160.75ms
step:1448/1480 train_time:231175ms step_avg:160.76ms
step:1449/1480 train_time:231349ms step_avg:160.77ms
step:1450/1480 train_time:231521ms step_avg:160.78ms
step:1451/1480 train_time:231692ms step_avg:160.79ms
step:1452/1480 train_time:231865ms step_avg:160.79ms
step:1453/1480 train_time:232035ms step_avg:160.80ms
step:1454/1480 train_time:232208ms step_avg:160.81ms
step:1455/1480 train_time:232386ms step_avg:160.82ms
step:1456/1480 train_time:232558ms step_avg:160.83ms
step:1457/1480 train_time:232730ms step_avg:160.84ms
step:1458/1480 train_time:232905ms step_avg:160.85ms
step:1459/1480 train_time:233080ms step_avg:160.86ms
step:1460/1480 train_time:233254ms step_avg:160.86ms
step:1461/1480 train_time:233430ms step_avg:160.88ms
step:1462/1480 train_time:233602ms step_avg:160.88ms
step:1463/1480 train_time:233779ms step_avg:160.89ms
step:1464/1480 train_time:233952ms step_avg:160.90ms
step:1465/1480 train_time:234126ms step_avg:160.91ms
step:1466/1480 train_time:234296ms step_avg:160.92ms
step:1467/1480 train_time:234471ms step_avg:160.93ms
step:1468/1480 train_time:234642ms step_avg:160.93ms
step:1469/1480 train_time:234815ms step_avg:160.94ms
step:1470/1480 train_time:234995ms step_avg:160.96ms
step:1471/1480 train_time:235182ms step_avg:160.97ms
step:1472/1480 train_time:235362ms step_avg:160.99ms
step:1473/1480 train_time:235533ms step_avg:160.99ms
step:1474/1480 train_time:235712ms step_avg:161.01ms
step:1475/1480 train_time:235891ms step_avg:161.02ms
step:1476/1480 train_time:236063ms step_avg:161.03ms
step:1477/1480 train_time:236247ms step_avg:161.04ms
step:1478/1480 train_time:236429ms step_avg:161.06ms
step:1479/1480 train_time:236603ms step_avg:161.06ms
step:1480/1480 train_time:236778ms step_avg:161.07ms
step:1480/1480 val_loss:3.2767 train_time:236849ms step_avg:161.12ms