最后是这么使用的：
import argparse from data_loader import load_and_cache_examples from trainer import Trainer from utils import init_logger, load_tokenizer, set_seed def main(args): init_logger() set_seed(args) tokenizer = load_tokenizer(args) train_dataset = load_and_cache_examples(args, tokenizer, mode="train") 其中用到了utils.py中的init_logger，load_tokenizer,set_seed： ```python import logging import os import random import numpy as np import torch from transformers import BertTokenizer ADDITIONAL_SPECIAL_TOKENS = ["<e1>", "</e1>", "<e2>", "</e2>"] def get_label(args): return [label.strip() for label in open(os.path.join(args.data_dir, args.label_file), "r", encoding="utf-8")] def load_tokenizer(args): tokenizer = BertTokenizer.from_pretrained(args.model_name_or_path) tokenizer.add_special_tokens({"additional_special_tokens": ADDITIONAL_SPECIAL_TOKENS}) return tokenizer其中使用的相关参数的定义如下：
parser = argparse.ArgumentParser() parser.add_argument("--task", default="semeval", type=str, help="The name of the task to train") parser.add_argument( "--data_dir", default="./data", type=str, help="The input data dir. Should contain the .tsv files (or other data files) for the task.", ) parser.add_argument("--model_dir", default="./model", type=str, help="Path to model") parser.add_argument( "--eval_dir", default="./eval", type=str, help="Evaluation script, result directory", ) parser.add_argument("--train_file", default="train.tsv", type=str, help="Train file") parser.add_argument("--test_file", default="test.tsv", type=str, help="Test file") parser.add_argument("--label_file", default="label.txt", type=str, help="Label file") parser.add_argument( "--model_name_or_path", type=str, default="bert-base-uncased", help="Model Name or Path", ) parser.add_argument("--seed", type=int, default=77, help="random seed for initialization") parser.add_argument("--train_batch_size", default=16, type=int, help="Batch size for training.") parser.add_argument("--eval_batch_size", default=32, type=int, help="Batch size for evaluation.") parser.add_argument( "--max_seq_len", default=384, type=int, help="The maximum total input sequence length after tokenization.", ) parser.add_argument( "--learning_rate", default=2e-5, type=float, help="The initial learning rate for Adam.", ) parser.add_argument( "--num_train_epochs", default=10.0, type=float, help="Total number of training epochs to perform.", ) parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.") parser.add_argument( "--gradient_accumulation_steps", type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.", ) parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument( "--max_steps", default=-1, type=int, help="If > 0: set total number of training steps to perform. Override num_train_epochs.", ) parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.") parser.add_argument( "--dropout_rate", default=0.1, type=float, help="Dropout for fully-connected layers", ) parser.add_argument("--logging_steps", type=int, default=250, help="Log every X updates steps.") parser.add_argument( "--save_steps", type=int, default=250, help="Save checkpoint every X updates steps.", ) parser.add_argument("--do_train", action="store_true", help="Whether to run training.") parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the test set.") parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available") parser.add_argument( "--add_sep_token", action="store_true", help="Add [SEP] token at the end of the sentence", ) args = parser.parse_args() main(args) 分步解析数据处理代码