-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathtrain_script_mquake.sh
124 lines (124 loc) · 4.26 KB
/
train_script_mquake.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# !/bin/bash
## Uncomment the following to run experiments for: gemma_7b
# model_name="gemma_7b";
# task="mquake";
# seed=42;
# CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component A \
# --use_topk_heads 160 \
# --lr 5e-3 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 5e-4 \
# --eval_batch 32 \
# --seed $seed;
# CUDA_VISIBLE_DEVICES=0 python3 lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component v \
# --use_topk_heads 16 \
# --lofit_heads "./top_heads/${model_name}_${task}_Aonly_top160heads_${seed}.npy"\
# --lr 8e-3 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 0 \
# --eval_batch 32 \
# --seed $seed;
## Uncomment the following to run experiments for: llama2_7B
model_name="llama2_7B";
task="mquake";
seed=42;
CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
--task $task \
--base_model_name $model_name \
--apply_chat_template False \
--ft_method lofit \
--lofit_component A \
--use_topk_heads 160 \
--lr 5e-3 \
--train_batch 8 \
--num_epoch 5 \
--output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_Aonly_seed${seed}"\
--run_mode train \
--output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_Aonly_seed${seed}"\
--applied_module attention \
--save_strategy no \
--l1_lambda 1e-3 \
--eval_batch 32 \
--seed $seed;
CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
--task $task \
--base_model_name $model_name \
--apply_chat_template False \
--ft_method lofit \
--lofit_component v \
--use_topk_heads 32 \
--lofit_heads "./top_heads/${model_name}_${task}_Aonly_top160heads_${seed}.npy"\
--lr 1e-2 \
--train_batch 8 \
--num_epoch 5 \
--output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_lofit_seed${seed}"\
--run_mode train \
--output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_lofit_seed${seed}"\
--applied_module attention \
--save_strategy no \
--l1_lambda 0 \
--eval_batch 32 \
--seed $seed;
## Uncomment the following to run experiments for: llama2_13B
# model_name="llama2_13B";
# task="mquake";
# seed=42;
# CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component A \
# --use_topk_heads 160 \
# --lr 1e-3 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 1e-3 \
# --eval_batch 32 \
# --seed $seed;
# CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component v \
# --use_topk_heads 48 \
# --lofit_heads "./top_heads/${model_name}_${task}_Aonly_top160heads_${seed}.npy"\
# --lr 8e-3 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 0 \
# --eval_batch 32 \
# --seed $seed;