lab_events.jsonl

{"seq": 1, "type": "trial_started", "trial_id": 0, "timestamp": "2026-04-04T23:42:26", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 4000, "val_games": 500, "min_ply": 10, "total_steps": 100, "batch_size": 64, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.05, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": null, "log_interval": 20, "pause_after_steps": null, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "float16", "no_compile": false, "sdpa_math": true, "num_workers": 0, "device": "cuda", "log_dir": "/workspace/logs/trial_0000", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": null, "bottleneck_dim": 16, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 2, "type": "trial_completed", "trial_id": 0, "timestamp": "2026-04-04T23:43:12", "data": {"best_val_loss": 2.6815053782271594, "best_accuracy": 0.25373134694665184, "param_count": 262144, "steps": 100}}
{"seq": 3, "type": "gpu_idle", "trial_id": null, "timestamp": "2026-04-04T23:43:12", "data": {"message": "All GPUs are idle"}}
{"seq": 4, "type": "trial_started", "trial_id": 1, "timestamp": "2026-04-04T23:43:26", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 4000, "val_games": 500, "min_ply": 10, "total_steps": 30, "batch_size": 64, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.05, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": null, "log_interval": 20, "pause_after_steps": null, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "float16", "no_compile": false, "sdpa_math": true, "num_workers": 0, "device": "cuda", "log_dir": "/workspace/logs/trial_0000", "hf_repo": null, "local_checkpoints": true, "resume": "/workspace/logs/trial_0000/bottleneck_20260404_234228_keen-bison/checkpoints/best", "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": null, "bottleneck_dim": 16, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 5, "type": "trial_killed", "trial_id": 1, "timestamp": "2026-04-04T23:43:29", "data": {}}
{"seq": 6, "type": "trial_started", "trial_id": 2, "timestamp": "2026-04-05T01:03:32", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 3500000, "val_games": 50000, "min_ply": 10, "total_steps": 200000, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 50, "pause_after_steps": 20000, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": false, "sdpa_math": true, "num_workers": 4, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 7, "type": "trial_killed", "trial_id": 2, "timestamp": "2026-04-05T01:04:40", "data": {}}
{"seq": 8, "type": "trial_started", "trial_id": 3, "timestamp": "2026-04-05T01:05:11", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 50000, "min_ply": 10, "total_steps": 200000, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 50, "pause_after_steps": 20000, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": false, "sdpa_math": true, "num_workers": 4, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 9, "type": "trial_killed", "trial_id": 3, "timestamp": "2026-04-05T01:17:15", "data": {}}
{"seq": 10, "type": "trial_started", "trial_id": 4, "timestamp": "2026-04-05T01:17:28", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 50000, "min_ply": 10, "total_steps": 200000, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 50, "pause_after_steps": 15000, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 4, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 11, "type": "trial_killed", "trial_id": 4, "timestamp": "2026-04-05T01:22:13", "data": {}}
{"seq": 12, "type": "trial_started", "trial_id": 5, "timestamp": "2026-04-05T01:22:34", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 50000, "min_ply": 10, "total_steps": 200000, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 50, "pause_after_steps": 15000, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 2, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 13, "type": "trial_killed", "trial_id": 5, "timestamp": "2026-04-05T01:24:48", "data": {}}
{"seq": 14, "type": "trial_started", "trial_id": 6, "timestamp": "2026-04-05T01:25:12", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 5000, "min_ply": 10, "total_steps": 100, "batch_size": 64, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 20, "pause_after_steps": null, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 2, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 15, "type": "trial_completed", "trial_id": 6, "timestamp": "2026-04-05T01:25:42", "data": {"best_val_loss": null, "best_accuracy": null, "param_count": null, "steps": 0}}
{"seq": 16, "type": "gpu_idle", "trial_id": null, "timestamp": "2026-04-05T01:25:42", "data": {"message": "All GPUs are idle"}}
{"seq": 17, "type": "trial_started", "trial_id": 7, "timestamp": "2026-04-05T01:26:24", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 5000, "min_ply": 10, "total_steps": 100, "batch_size": 128, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 20, "pause_after_steps": null, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 2, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 18, "type": "trial_completed", "trial_id": 7, "timestamp": "2026-04-05T01:27:00", "data": {"best_val_loss": null, "best_accuracy": null, "param_count": null, "steps": 0}}
{"seq": 19, "type": "gpu_idle", "trial_id": null, "timestamp": "2026-04-05T01:27:00", "data": {"message": "All GPUs are idle"}}
{"seq": 20, "type": "trial_started", "trial_id": 8, "timestamp": "2026-04-05T01:27:20", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 5000, "min_ply": 10, "total_steps": 50, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 10, "pause_after_steps": null, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 2, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 21, "type": "trial_completed", "trial_id": 8, "timestamp": "2026-04-05T01:27:55", "data": {"best_val_loss": null, "best_accuracy": null, "param_count": null, "steps": 0}}
{"seq": 22, "type": "gpu_idle", "trial_id": null, "timestamp": "2026-04-05T01:27:55", "data": {"message": "All GPUs are idle"}}
{"seq": 23, "type": "trial_started", "trial_id": 9, "timestamp": "2026-04-05T01:29:33", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 50000, "min_ply": 10, "total_steps": 200000, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 10, "pause_after_steps": 15000, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 2, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 24, "type": "trial_killed", "trial_id": 9, "timestamp": "2026-04-05T01:33:37", "data": {}}
{"seq": 25, "type": "trial_started", "trial_id": 10, "timestamp": "2026-04-05T01:34:21", "data": {"strategy": "bottleneck", "gpu": 0, "config": {"elo_min": 1800, "elo_max": 1900, "max_games": 30000000, "val_games": 50000, "min_ply": 10, "total_steps": 200000, "batch_size": 256, "lr": 0.0003, "weight_decay": 0.0, "warmup_frac": 0.02, "warmup_steps": null, "max_grad_norm": 1.0, "patience": 9999, "eval_interval": 2500, "log_interval": 100, "pause_after_steps": 15000, "mate_boost": 0.0, "no_outcome_token": false, "discard_ply_limit": false, "amp_dtype": "bfloat16", "no_compile": true, "sdpa_math": true, "num_workers": 2, "device": "cuda", "log_dir": "/workspace/logs", "hf_repo": null, "local_checkpoints": true, "resume": null, "wandb": false, "cache_dir": "/dev/shm/pawn_cache", "run_type": "adapter", "strategy": "bottleneck", "checkpoint": "thomas-schweich/pawn-base", "pgn": "thomas-schweich/pawn-lichess-full", "adapter_layers": "4,5,6,7", "bottleneck_dim": 2440, "no_adapt_attn": false, "no_adapt_ffn": false, "lora_rank": null, "lora_targets": null, "lora_ffn": false, "density": null, "sparse_targets": null, "sparse_ffn": false, "use_output_film": false, "rosa_mode": null, "rosa_warmup_steps": 128, "mask_samples": 32, "grad_alpha": 2, "unfreeze_layers": null, "d_model": null, "n_layers": null, "n_heads": null, "epochs": 9999, "val_every": 9999}}}
{"seq": 26, "type": "trial_completed", "trial_id": 10, "timestamp": "2026-04-05T02:30:43", "data": {"best_val_loss": null, "best_accuracy": null, "param_count": null, "steps": 0}}
{"seq": 27, "type": "gpu_idle", "trial_id": null, "timestamp": "2026-04-05T02:30:43", "data": {"message": "All GPUs are idle"}}