| run_name: AMD-OLMo-1B-SFT-2nd-phase |
| seed: 6198 |
| dry_run: false |
|
|
| wandb: |
| name: ${run_name} |
| project: AMD-OLMo |
| group: SFT |
|
|
| model: |
| d_model: 2048 |
| n_heads: 16 |
| n_layers: 16 |
| mlp_ratio: 8 |
| weight_tying: true |
| alibi: false |
| rope: true |
| flash_attention: false |
| attention_dropout: 0.0 |
| attention_layer_norm: false |
| multi_query_attention: false |
| include_bias: false |
| block_type: sequential |
| layer_norm_type: default |
| layer_norm_with_affine: false |
| bias_for_layer_norm: false |
| attention_layer_norm_with_affine: false |
| activation_type: swiglu |
| residual_dropout: 0.0 |
| embedding_dropout: 0.0 |
| max_sequence_length: 2048 |
| vocab_size: 50280 |
| embedding_size: 50304 |
| eos_token_id: 50279 |
| pad_token_id: 1 |
| init_device: meta |
| init_fn: mitchell |
|
|
| compile: |
| fullgraph: false |
|
|
| optimizer: |
| name: adamw |
| learning_rate: 2.0e-5 |
| weight_decay: 0 |
| betas: |
| - 0.9 |
| - 0.95 |
| metrics_log_interval: 10 |
|
|
| scheduler: |
| name: linear_with_warmup |
| t_warmup: 200 |
| alpha_f: 0.001 |
|
|
| tokenizer: |
| identifier: tokenizers/allenai_eleuther-ai-gpt-neox-20b-pii-special.json |
| truncate_direction: right |
|
|
| save_folder: ./outputs/${run_name}/ |
| save_overwrite: true |
| |
| save_interval: 1000 |
| save_num_checkpoints_to_keep: -1 |
| |
| save_interval_unsharded: 10000 |
| save_num_unsharded_checkpoints_to_keep: -1 |
|
|
| load_path: path_to_unsharded_1st_phase_SFT_checkpoint |
| reset_trainer_state: true |
|
|
| max_duration: 3ep |
| global_train_batch_size: 512 |
| device_train_microbatch_size: 8 |
|
|
| precision: amp_bf16 |
|
|
| fsdp: |
| wrapping_strategy: null |
| precision: mixed |
|
|
| max_grad_norm: 1.0 |
| max_grad_norm_ratio: null |
|
|
| speed_monitor: |
| window_size: 20 |
|
|
| eval_interval: ${save_interval} |
| eval_subset_num_batches: -1 |
| device_eval_batch_size: ${device_train_microbatch_size} |
| evaluators: |
| - label: piqa |
| type: downstream |
|
|
| - label: hellaswag |
| type: downstream |
|
|
| - label: winogrande |
| type: downstream |
|
|
| - label: openbook_qa |
| type: downstream |
|
|
| |
| |
|
|
| - label: sciq |
| type: downstream |
|
|
| - label: arc_easy |
| type: downstream |
|
|
| |
| |
|
|
| - label: copa |
| type: downstream |
|
|
| - label: rte |
| type: downstream |
|
|
| - label: commitment_bank |
| type: downstream |
|
|
| - label: mrpc |
| type: downstream |
|
|
| - label: sst2 |
| type: downstream |
|
|
| data: |
| pad_direction: right |
| num_workers: 0 |
| drop_last: true |
| pin_memory: true |
| prefetch_factor: 1 |
| persistent_workers: true |
| timeout: 0 |
| generate_attention_mask: true |
| paths: |
| - ./datasets/OpenHermes_WebInstructSub_CodeFeedBack/input_ids.npy |
| label_mask_paths: |
| - ./datasets/OpenHermes_WebInstructSub_CodeFeedBack/label_mask.npy |
