我想在 disaster-tweets 数据集上微调模型“FacebookAI/roberta-large”。我在 transformers.Trainer 中遇到问题，并收到此错误：

TypeError                                 Traceback (most recent call last)
Cell In[53], line 25
     17 trainer = transformers.Trainer(
     18     model=model,
     19     args=training_args,
   (...)
     22     data_collator=transformers.DataCollatorWithPadding(tokenizer),
     23 )
     24 model.train()
---> 25 trainer.train()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1991, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1989         hf_hub_utils.enable_progress_bars()
   1990 else:
-> 1991     return inner_training_loop(
   1992         args=args,
   1993         resume_from_checkpoint=resume_from_checkpoint,
   1994         trial=trial,
   1995         ignore_keys_for_eval=ignore_keys_for_eval,
   1996     )

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2255, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   2253 model.zero_grad()
   2254 grad_norm: Optional[float] = None
-> 2255 self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
   2257 if args.eval_on_start:
   2258     self._evaluate(trial, ignore_keys_for_eval, skip_scheduler=True)

File /opt/conda/lib/python3.10/site-packages/transformers/trainer_callback.py:460, in CallbackHandler.on_train_begin(self, args, state, control)
    458 def on_train_begin(self, args: TrainingArguments, state: TrainerState, control: TrainerControl):
    459     control.should_training_stop = False
--> 460     return self.call_event("on_train_begin", args, state, control)

File /opt/conda/lib/python3.10/site-packages/transformers/trainer_callback.py:507, in CallbackHandler.call_event(self, event, args, state, control, **kwargs)
    505 def call_event(self, event, args, state, control, **kwargs):
    506     for callback in self.callbacks:
--> 507         result = getattr(callback, event)(
    508             args,
    509             state,
    510             control,
    511             model=self.model,
    512             tokenizer=self.tokenizer,
    513             optimizer=self.optimizer,
    514             lr_scheduler=self.lr_scheduler,
    515             train_dataloader=self.train_dataloader,
    516             eval_dataloader=self.eval_dataloader,
    517             **kwargs,
    518         )
    519         # A Callback can skip the return of `control` if it doesn't change it.
    520         if result is not None:

File /opt/conda/lib/python3.10/site-packages/transformers/integrations/integration_utils.py:900, in WandbCallback.on_train_begin(self, args, state, control, model, **kwargs)
    898     args.run_name = None
    899 if not self._initialized:
--> 900     self.setup(args, state, model, **kwargs)

File /opt/conda/lib/python3.10/site-packages/transformers/integrations/integration_utils.py:838, in WandbCallback.setup(self, args, state, model, **kwargs)
    833     self._wandb.init(
    834         project=os.getenv("WANDB_PROJECT", "huggingface"),
    835         **init_args,
    836     )
    837 # add config parameters (run may have been created manually)
--> 838 self._wandb.config.update(combined_dict, allow_val_change=True)
    840 # define default x-axis (for latest wandb versions)
    841 if getattr(self._wandb, "define_metric", None):

File /opt/conda/lib/python3.10/site-packages/wandb/sdk/wandb_config.py:186, in Config.update(self, d, allow_val_change)
    184 sanitized = self._update(d, allow_val_change)
    185 if self._callback:
--> 186     self._callback(data=sanitized)

File /opt/conda/lib/python3.10/site-packages/wandb/sdk/wandb_run.py:340, in _run_decorator._noop_on_finish.<locals>.decorator_fn.<locals>.wrapper_fn(self, *args, **kwargs)
    337 @functools.wraps(func)
    338 def wrapper_fn(self: Type["Run"], *args: Any, **kwargs: Any) -> Any:
    339     if not getattr(self, "_is_finished", False):
--> 340         return func(self, *args, **kwargs)
    342     default_message = (
    343         f"Run ({self.id}) is finished. The call to `{func.__name__}` will be ignored. "
    344         f"Please make sure that you are using an active run."
    345     )
    346     resolved_message = message or default_message

File /opt/conda/lib/python3.10/site-packages/wandb/sdk/wandb_run.py:1283, in Run._config_callback(self, key, val, data)
   1281 logger.info(f"config_cb {key} {val} {data}")
   1282 if self._backend and self._backend.interface:
-> 1283     self._backend.interface.publish_config(key=key, val=val, data=data)

File /opt/conda/lib/python3.10/site-packages/wandb/sdk/interface/interface.py:205, in InterfaceBase.publish_config(self, data, key, val)
    199 def publish_config(
    200     self,
    201     data: Optional[dict] = None,
    202     key: Optional[Union[Tuple[str, ...], str]] = None,
    203     val: Optional[Any] = None,
    204 ) -> None:
--> 205     cfg = self._make_config(data=data, key=key, val=val)
    207     self._publish_config(cfg)

File /opt/conda/lib/python3.10/site-packages/wandb/sdk/interface/interface.py:160, in InterfaceBase._make_config(self, data, key, val, obj)
    158         update = config.update.add()
    159         update.key = k
--> 160         update.value_json = json_dumps_safer(json_friendly(v)[0])
    161 if key:
    162     update = config.update.add()

File /opt/conda/lib/python3.10/site-packages/wandb/util.py:810, in json_dumps_safer(obj, **kwargs)
    808 def json_dumps_safer(obj: Any, **kwargs: Any) -> str:
    809     """Convert obj to json, with some extra encodable types."""
--> 810     return json.dumps(obj, cls=WandBJSONEncoder, **kwargs)

File /opt/conda/lib/python3.10/json/__init__.py:238, in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
    232 if cls is None:
    233     cls = JSONEncoder
    234 return cls(
    235     skipkeys=skipkeys, ensure_ascii=ensure_ascii,
    236     check_circular=check_circular, allow_nan=allow_nan, indent=indent,
    237     separators=separators, default=default, sort_keys=sort_keys,
--> 238     **kw).encode(obj)

File /opt/conda/lib/python3.10/json/encoder.py:199, in JSONEncoder.encode(self, o)
    195         return encode_basestring(o)
    196 # This doesn't pass the iterator directly to ''.join() because the
    197 # exceptions aren't as detailed.  The list call should be roughly
    198 # equivalent to the PySequence_Fast that ''.join() would do.
--> 199 chunks = self.iterencode(o, _one_shot=True)
    200 if not isinstance(chunks, (list, tuple)):
    201     chunks = list(chunks)

File /opt/conda/lib/python3.10/json/encoder.py:257, in JSONEncoder.iterencode(self, o, _one_shot)
    252 else:
    253     _iterencode = _make_iterencode(
    254         markers, self.default, _encoder, self.indent, floatstr,
    255         self.key_separator, self.item_separator, self.sort_keys,
    256         self.skipkeys, _one_shot)
--> 257 return _iterencode(o, 0)

File /opt/conda/lib/python3.10/site-packages/wandb/util.py:761, in WandBJSONEncoder.default(self, obj)
    759 if converted:
    760     return tmp_obj
--> 761 return json.JSONEncoder.default(self, obj)

File /opt/conda/lib/python3.10/json/encoder.py:179, in JSONEncoder.default(self, o)
    160 def default(self, o):
    161     """Implement this method in a subclass such that it returns
    162     a serializable object for ``o``, or calls the base implementation
    163     (to raise a ``TypeError``).
   (...)
    177 
    178     """
--> 179     raise TypeError(f'Object of type {o.__class__.__name__} '
    180                     f'is not JSON serializable')

TypeError: Object of type set is not JSON serializable

我不知道如何解决。请帮帮我！

数据集在这里： https： //www.kaggle.com/competitions/nlp-getting-started/data

我使用模型的标记器对其进行了预处理，如下所示：

if any(k in model_name for k in ("gpt", "opt", "bloom")):
    padding_side = "left"
else:
    padding_side = "right"

tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side=padding_side, trust_remote_code=True)
if getattr(tokenizer, "pad_token_id") is None:
    tokenizer.pad_token_id = tokenizer.eos_token_id


def tokenize_function(example):
    # max_length=None => use the model max length (it's actually the default)
    return tokenizer(example['text'], truncation=True, max_length=None)  

以下是模型配置和训练器：

peft_config = LoraConfig(
    lora_alpha= 8,
    lora_dropout= 0.1,
    r= 16,
    bias="none",
    task_type="SEQ_CLS"
)

model = get_peft_model(model, peft_config)

output_dir1 = "results1"
os.mkdir(output_dir1)

accuracy = evaluate.load("accuracy")
def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    return accuracy.compute(predictions=predictions, references=labels)

EPOCHS=1
training_args = TrainingArguments(
    output_dir=output_dir1,
    save_strategy="epoch",
    num_train_epochs=EPOCHS,
    logging_steps= 1000,
    learning_rate= 2e-4,
    weight_decay= 0.01,
    fp16= False,
    bf16= False,
    max_grad_norm= 0.3,
    warmup_ratio= 0.3,
    group_by_length= True,
    lr_scheduler_type= "linear",
)

trainer = transformers.Trainer(
    model=model,
    args=training_args,
    compute_metrics=compute_metrics, 
    train_dataset=train_tokenized,
    data_collator=transformers.DataCollatorWithPadding(tokenizer),
)
model.train()
trainer.train()

解决方案

暂无回答