Upload folder using huggingface_hub

configs/cls_schedule/cls_vit_b16_s1.28B_bs16k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_b16_s1.28B_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-B-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_b16_s1.28B_bs16k_val.yaml

@@ -0,0 +1,34 @@
+save_frequency: 1
+name: "cls_vit_b16_s1.28B_bs16k"
+train_data: '' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-B-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_frequency: 1
+val_frequency: 1
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_b16_s512m_bs16k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_b16_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-B-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l14_224_s12.8B_bs90k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l14_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-14"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+resume: latest

configs/cls_schedule/cls_vit_l14_s1.28B_bs16k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l14_s1.28B_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 10
+model: "CLS-ViT-L-14"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_1_s512m_bs16k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_1_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16-1"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_224_s12.8B_bs90k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+resume: latest

configs/cls_schedule/cls_vit_l16_s1.28B_bs16k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s1.28B_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 10
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s1.28B_bs16k_noidf.yaml

@@ -0,0 +1,37 @@
+save_frequency: 1
+name: "cls_vit_l16_s1.28B_bs16k_noidf"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 10
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+use_idf: false
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s4B_bs32k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s4B_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s4B_bs32k_2b.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s4B_bs32k_2b"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s4B_bs32k_3b.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s4B_bs32k_3b"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s4B_bs32k_cls.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s4B_bs32k_cls"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16-cls"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s512m_bs16k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/cls_vit_l16_s512m_bs32k.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s512m_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 32768
+batch_size: 0
+epochs: 1
+lr: 1e-3
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_clip_vit_b16_s512m_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_clip_vit_b16_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-B-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: '/mnt/bn/seed-aws-va/zilonghuang/code/ViTamin/ViTamin/logs/vit_b16_s512m_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_b16_s1.28B_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_b16_s1.28B_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-B-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_b16_s1.28B_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_b16_s4B_bs32k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s4B_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_b16_s4B_bs32k/checkpoints/epoch_3.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_b16_s512m_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_b16_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-B-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_b16_s512m_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l14_224_s12.8B_bs90k.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "lit_cls_vit_l14_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-14-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l14_224_s12.8B_bs90k/checkpoints/epoch_10.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+zeroshot_frequency: 1
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+resume: latest

configs/cls_schedule/lit_vit_l14_s1.28B_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l14_s1.28B_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-14"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l14_s1.28B_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_1_s512m_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_1_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_1_s512m_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_224_s12.8B_bs90k.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_224_s12.8B_bs90k/checkpoints/epoch_10.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+zeroshot_frequency: 1
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+resume: latest

configs/cls_schedule/lit_vit_l16_s1.28B_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s1.28B_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s1.28B_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s1.28B_bs16k_noidf.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s1.28B_bs16k_noidf"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s1.28B_bs16k_noidf/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s4B_bs32k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s4B_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s4B_bs32k/checkpoints/epoch_3.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s4B_bs32k_2b.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s4B_bs32k_2b"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s4B_bs32k_2b/checkpoints/epoch_3.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s4B_bs32k_3b.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s4B_bs32k_3b"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s4B_bs32k_3b/checkpoints/epoch_3.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s4B_bs32k_cls.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s4B_bs32k_cls"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s4B_bs32k_cls/checkpoints/epoch_3.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s512m_bs16k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_clip_vit_l16_s512m_bs16k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 16384
+batch_size: 0
+epochs: 1
+lr: 5e-4
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s512m_bs16k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 128
+zeroshot_steps: 6104
+val_steps: 6104
+save_every_n_steps: 6104
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/lit_vit_l16_s512m_bs32k.yaml

@@ -0,0 +1,38 @@
+save_frequency: 1
+name: "lit_cls_vit_l16_s512m_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..51200}.tar'
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 32768
+batch_size: 0
+epochs: 1
+lr: 1e-3
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-avg"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+lock_image: true
+lock_image_unlocked_groups: 1
+pretrained_image: './logs/cls_vit_l16_s512m_bs32k/checkpoints/epoch_1.pt'
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/cls_schedule/test.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_vit_l16_s512m_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 500
+global_batch_size: 3200
+batch_size: 0
+epochs: 1
+lr: 1e-3
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 64
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+
+resume: latest

configs/exp_schedule/cls_vit_l16_s12.8B_bs90k.yaml

@@ -0,0 +1,37 @@
+save_frequency: 1
+name: "clsclip_l16_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+coca_caption_loss_weight: 0.5
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/exp_schedule/cls_vit_l16_s12.8B_bs90k_w1.0.yaml

@@ -0,0 +1,37 @@
+save_frequency: 1
+name: "clsclip_l16_224_s12.8B_bs90k_w1.0"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+coca_caption_loss_weight: 1.0
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/exp_schedule/test.yaml

@@ -0,0 +1,37 @@
+save_frequency: 1
+name: "clsclip_test"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 8000
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+coca_caption_loss_weight: 0.5
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/clip_test.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "clip_test"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/cls_test.yaml

@@ -0,0 +1,36 @@
+save_frequency: 1
+name: "cls_test"
+train_data: '/datasets/datacomp_1b/data/{00000..128000}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 90112
+batch_size: 0
+epochs: 10
+lr: 1e-3
+beta1: 0.9
+beta2: 0.98
+eps: 1.0e-6
+workers: 6
+model: "CLS-ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 0
+val_steps: 0
+zeroshot_frequency: 0
+val_frequency: 0
+save_every_n_steps: 1500
+delete_prev_step_ckpt: true
+
+resume: latest

configs/long_schedule/vit_l16_224_s12.8B_bs90k.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vit_l_224_s12.8B_bs90k_recap_cl128"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16-cl128"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vit_l16_224_s4B_bs32k.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vit_l16_224_s4B_bs32k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vit_l16_224_s4B_bs32k_3b.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vit_l16_224_s4B_bs32k_3B"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 32768
+batch_size: 0
+epochs: 3
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vit_l16_224_s4B_bs64k.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vit_l16_224_s4B_bs64k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 1000
+global_batch_size: 64000
+batch_size: 0
+epochs: 3
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-16"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+# aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vit_l_224_s12.8B_bs90k.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vit_l_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 10000
+global_batch_size: 90112 
+batch_size: 0
+epochs: 10
+lr: 1.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViT-L-14"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vitamin_l2_224_s12.8B_bs90k.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vitamin_l2_224_s12.8B_bs90k" 
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 4436 # 400M
+global_batch_size: 90160
+batch_size: 0
+epochs: 10
+lr: 2.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViTamin-L2"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 500
+val_steps: 500
+save_every_n_steps: 500
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vitamin_l2_224_s12.8B_bs90k_ft256.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "vitamin_l2_224_s12.8B_bs90k_ft256"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 0
+global_batch_size: 90160
+batch_size: 0
+epochs: 1
+lr: 1.0e-5
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViTamin-L2-256"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 256
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 200
+val_steps: 200
+save_every_n_steps: 200
+delete_prev_step_ckpt: false
+zeroshot_frequency: 1
+resume: latest
+
+pretrained: './logs/vitamin_l2_224_s12.8B_bs90k/checkpoints/epoch_10.pt' # please modify to your own path
+pretrained_optim_scaler: false 
+lr_scheduler: const
+wd: 0.0

configs/long_schedule/vitamin_l2_224_s12.8B_bs90k_ft336.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "vitamin_l2_224_s12.8B_bs90k_ft336"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 0
+global_batch_size: 90160 
+batch_size: 0
+epochs: 1
+lr: 1.0e-5
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViTamin-L2-336"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 336
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 200
+val_steps: 200
+save_every_n_steps: 200
+delete_prev_step_ckpt: false
+zeroshot_frequency: 1
+resume: latest
+
+pretrained: './logs/vitamin_l2_224_s12.8B_bs90k/checkpoints/epoch_10.pt' # please modify to your own path
+pretrained_optim_scaler: false
+lr_scheduler: const
+wd: 0.0

configs/long_schedule/vitamin_l2_224_s12.8B_bs90k_ft384.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "vitamin_l2_224_s12.8B_bs90k_ft384" 
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 0
+global_batch_size: 90160 # 322*280gpu = 90160
+batch_size: 0
+epochs: 1
+lr: 1.0e-5
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "ViTamin-L2-384"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 384
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 200
+val_steps: 200
+save_every_n_steps: 200
+delete_prev_step_ckpt: false
+zeroshot_frequency: 1
+resume: latest
+
+pretrained: './logs/vitamin_l2_224_s12.8B_bs90k/checkpoints/epoch_10.pt' # please modify to your own path
+pretrained_optim_scaler: false 
+lr_scheduler: const
+wd: 0.0

configs/long_schedule/vitamin_l_224_s12.8B_bs90k.yaml

@@ -0,0 +1,35 @@
+save_frequency: 1
+name: "vitamin_l_224_s12.8B_bs90k"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 782
+global_batch_size: 90160 
+batch_size: 0
+epochs: 10
+lr: 2.0e-3
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "vit_l16_mbconv_glu_d31_224"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 224
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 3052
+val_steps: 3052
+save_every_n_steps: 3052
+delete_prev_step_ckpt: true
+aug_cfg: {'scale': [0.4, 1.0], 'color_jitter': [0.32, 0.32, 0.32, 0.08], 'color_jitter_prob': 0.8, 'gray_scale_prob': 0.2}
+zeroshot_frequency: 1
+resume: latest

configs/long_schedule/vitamin_l_224_s12.8B_bs90k_ft256.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "vitamin_l_224_s12.8B_bs90k_ft256"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 0
+global_batch_size: 90240
+batch_size: 0
+epochs: 1
+lr: 1.0e-5
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "vit_l16_mbconv_glu_d31_ft256"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 256
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 200
+val_steps: 200
+save_every_n_steps: 200
+delete_prev_step_ckpt: false
+zeroshot_frequency: 1
+resume: latest
+
+pretrained: './logs/vitamin_l_224_s12.8B_bs90k/checkpoints/epoch_10.pt' # please modify to your own path
+pretrained_optim_scaler: false 
+lr_scheduler: const
+wd: 0.0

configs/long_schedule/vitamin_l_224_s12.8B_bs90k_ft336.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "vitamin_l_224_s12.8B_bs90k_ft336"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 512_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 0
+global_batch_size: 90200
+batch_size: 0
+epochs: 10
+lr: 1.0e-5
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "vit_l16_mbconv_glu_d31_ft336"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 336
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 500
+val_steps: 500
+save_every_n_steps: 500
+delete_prev_step_ckpt: false
+zeroshot_frequency: 1
+resume: latest
+
+pretrained: './logs/vitamin_l_224_s12.8B_bs90k/checkpoints/epoch_10.pt' # please modify to your own path
+pretrained_optim_scaler: false 
+lr_scheduler: const
+wd: 0.0

configs/long_schedule/vitamin_l_224_s12.8B_bs90k_ft384.yaml

@@ -0,0 +1,39 @@
+save_frequency: 1
+name: "vitamin_l_224_s12.8B_bs90k_ft384"
+train_data: '/datasets/datacomp_1b/data/{00000..140089}.tar' # please modify to your own path
+train_num_samples: 1_280_000_000
+dataset_type: webdataset
+precision: 'amp_bfloat16'
+warmup: 0
+global_batch_size: 90200
+batch_size: 0
+epochs: 10
+lr: 1.0e-5
+beta1: 0.9
+beta2: 0.95
+eps: 1.0e-6
+workers: 6
+model: "vit_l16_mbconv_glu_d31_ft384"
+seed: 0
+ddp_static_graph: true
+local_loss: true
+gather_with_grad: true
+force_image_size: 384
+grad_checkpointing: true
+
+logs: './logs'
+imagenet_val: './imagenet1k/val' # please modify to your own path
+
+report_to: "tensorboard"
+log_every_n_steps: 32
+zeroshot_steps: 500
+val_steps: 500
+save_every_n_steps: 500
+delete_prev_step_ckpt: false
+zeroshot_frequency: 1
+resume: latest
+
+pretrained: './logs/vitamin_l_224_s12.8B_bs90k/checkpoints/epoch_10.pt' # please modify to your own path
+pretrained_optim_scaler: false 
+lr_scheduler: const
+wd: 0.0