mirror of
https://github.com/CodedotAl/gpt-code-clippy.git
synced 2024-09-11 15:05:44 +03:00
upd apps fine-tuning script
This commit is contained in:
parent
6020b3cb86
commit
ec042fd0fb
58
EDA.ipynb
58
EDA.ipynb
@ -1,28 +1,4 @@
|
|||||||
{
|
{
|
||||||
"metadata": {
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.8.10"
|
|
||||||
},
|
|
||||||
"orig_nbformat": 4,
|
|
||||||
"kernelspec": {
|
|
||||||
"name": "python3",
|
|
||||||
"display_name": "Python 3.8.10 64-bit ('.venv': venv)"
|
|
||||||
},
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "25d32b45eddbc1e23c07b06a2c9ff49f418e028170a37fc806346e2c2002bf83"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2,
|
|
||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
@ -30,8 +6,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n"
|
"None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n"
|
||||||
]
|
]
|
||||||
@ -54,8 +30,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n",
|
"Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n",
|
||||||
"Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n",
|
"Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n",
|
||||||
@ -183,8 +159,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"0it [00:00, ?it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (2708 > 2048). Running this sequence through the model will result in indexing errors\n",
|
"0it [00:00, ?it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (2708 > 2048). Running this sequence through the model will result in indexing errors\n",
|
||||||
"61it [00:55, 1.36it/s]"
|
"61it [00:55, 1.36it/s]"
|
||||||
@ -204,5 +180,29 @@
|
|||||||
"collapse_metrics(eval_dataset, 'eval_metrics.pkl')"
|
"collapse_metrics(eval_dataset, 'eval_metrics.pkl')"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
}
|
"metadata": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "25d32b45eddbc1e23c07b06a2c9ff49f418e028170a37fc806346e2c2002bf83"
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.8.10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
|
@ -1,29 +1,29 @@
|
|||||||
#! /bin/bash
|
#! /bin/bash
|
||||||
./run_clm_apps.py \
|
./run_clm_apps.py \
|
||||||
--output_dir $HOME/gpt-code-clippy-apps-2 \
|
--output_dir $HOME/gpt-code-clippy-apps-125m-2048-raw \
|
||||||
--model_name_or_path EleutherAI/gpt-neo-1.3B \
|
--model_name_or_path $HOME/gpt-code-clippy-125M-bs2048-raw \
|
||||||
--dataset_name ./apps.py \
|
--dataset_name ./apps.py \
|
||||||
--do_train --do_eval \
|
--do_train --do_eval \
|
||||||
--block_size="1024" \
|
--block_size="1024" \
|
||||||
--per_device_train_batch_size="2" \
|
--per_device_train_batch_size="16" \
|
||||||
--per_device_eval_batch_size="2" \
|
--per_device_eval_batch_size="16" \
|
||||||
--preprocessing_num_workers="16" \
|
--preprocessing_num_workers="16" \
|
||||||
--learning_rate="2e-5" \
|
--learning_rate="5e-5" \
|
||||||
--warmup_steps="5000" \
|
--warmup_steps="800" \
|
||||||
--adam_beta1="0.9" \
|
--adam_beta1="0.9" \
|
||||||
--adam_beta2="0.98" \
|
--adam_beta2="0.98" \
|
||||||
--weight_decay="0.1" \
|
--weight_decay="0.1" \
|
||||||
--overwrite_output_dir \
|
--overwrite_output_dir \
|
||||||
--num_train_epochs="5" \
|
--num_train_epochs="5" \
|
||||||
--logging_steps="20" \
|
--logging_steps="20" \
|
||||||
--eval_steps="1000" \
|
--eval_steps="100" \
|
||||||
--push_to_hub="False" \
|
--push_to_hub="False" \
|
||||||
--report_to="wandb" \
|
--report_to="wandb" \
|
||||||
--dtype="bfloat16" \
|
--dtype="bfloat16" \
|
||||||
--skip_memory_metrics="False" \
|
--skip_memory_metrics="False" \
|
||||||
--save_steps="1000" \
|
--save_steps="100" \
|
||||||
--save_strategy epoch \
|
--save_strategy epoch \
|
||||||
--save_total_limit 2 \
|
--save_total_limit 5 \
|
||||||
--gradient_accumulation_steps 2 \
|
--gradient_accumulation_steps 2 \
|
||||||
--adafactor \
|
--adafactor \
|
||||||
# --resume_from_checkpoint $HOME/gpt-neo-125M-code-clippy/ckpt_201 \
|
# --resume_from_checkpoint $HOME/gpt-neo-125M-code-clippy/ckpt_201 \
|
||||||
|
35
run_clm_apps.py
Normal file → Executable file
35
run_clm_apps.py
Normal file → Executable file
@ -165,9 +165,13 @@ class DataTrainingArguments:
|
|||||||
metadata={"help": "The number of processes to use for the preprocessing."},
|
metadata={"help": "The number of processes to use for the preprocessing."},
|
||||||
)
|
)
|
||||||
text_column_name: Optional[str] = field(
|
text_column_name: Optional[str] = field(
|
||||||
default='text',
|
default='text',
|
||||||
metadata={"help": "Column containing main text data."},
|
metadata={"help": "Column containing main text data."},
|
||||||
)
|
)
|
||||||
|
all_data: Optional[bool] = field(
|
||||||
|
default=False,
|
||||||
|
metadata={"help":"If True will use all data ignoring original APPS splits."}
|
||||||
|
)
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
if self.dataset_name is None and self.train_file is None and self.validation_file is None:
|
if self.dataset_name is None and self.train_file is None and self.validation_file is None:
|
||||||
@ -381,18 +385,23 @@ def main():
|
|||||||
# download the dataset.
|
# download the dataset.
|
||||||
if data_args.dataset_name is not None:
|
if data_args.dataset_name is not None:
|
||||||
# Downloading and loading a dataset from the hub.
|
# Downloading and loading a dataset from the hub.
|
||||||
whole_dataset = load_dataset(
|
if data_args.all_data:
|
||||||
data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir, keep_in_memory=False
|
whole_dataset = load_dataset(
|
||||||
)
|
data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir, keep_in_memory=False
|
||||||
|
)
|
||||||
|
|
||||||
whole_dataset = concatenate_datasets([whole_dataset["train"], whole_dataset["test"]])
|
whole_dataset = concatenate_datasets([whole_dataset["train"], whole_dataset["test"]])
|
||||||
split_id = int(0.9*len(whole_dataset))
|
split_id = int(0.9*len(whole_dataset))
|
||||||
train_idx = list(range(split_id))
|
train_idx = list(range(split_id))
|
||||||
valid_idx = list(range(split_id, len(whole_dataset)))
|
valid_idx = list(range(split_id, len(whole_dataset)))
|
||||||
dataset = DatasetDict({
|
dataset = DatasetDict({
|
||||||
"train":whole_dataset.select(train_idx),
|
"train":whole_dataset.select(train_idx),
|
||||||
"validation":whole_dataset.select(valid_idx)
|
"validation":whole_dataset.select(valid_idx)
|
||||||
})
|
})
|
||||||
|
else:
|
||||||
|
dataset = load_dataset(
|
||||||
|
data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir, keep_in_memory=False
|
||||||
|
)
|
||||||
|
|
||||||
if "validation" not in dataset.keys():
|
if "validation" not in dataset.keys():
|
||||||
dataset["validation"] = load_dataset(
|
dataset["validation"] = load_dataset(
|
||||||
|
Loading…
Reference in New Issue
Block a user