CorDA on ms-swift #4239
Description
I'm trying CORDA on MS-SWIFT.
trying to link a multimodal model(Ovis2) in corda to match the format of the run_model() function.
The problem I am facing is that I need to apply the data_collator function to tensorize the input_ids, pixel_values data, but I am getting an error.
preprocess_corda(model, lora_config, run_model=run_model)
def run_model():
for batch in sampled_dataset:
input_ids = batch["input_ids"]
pixel_values = batch["pixel_values"]
input_ids = input_ids.to(model.device)
attention_mask = torch.ne(input_ids, text_tokenizer.pad_token_id)
input_ids = input_ids.unsqueeze(0).to(device=model.device)
attention_mask = attention_mask.unsqueeze(0).to(device=model.device)
if pixel_values is not None:
pixel_values = pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device)
pixel_values = [pixel_values]
# generate output
with torch.inference_mode():
output_ids = model.generate(input_ids, pixel_values=pixel_values, attention_mask=attention_mask, **gen_kwargs)[0]