are they compatible and what precautions should I take if I use it to fine-tune a bf16 model.
No, bfloat16
is supported starting from Ampere GPUs.
sorry to bother but can it be reason why i am getting error
None type object is non iterable. because rest of code looks fine to me. I am using qwen2.5-3B-Instruct.
import os
import pandas as pd
from torch.utils.data import Dataset, DataLoader
from PIL import Image
from transformers import AutoProcessor, AutoTokenizer, AdamW , AutoModelForCausalLM
import torch
from tqdm import tqdm
class ImageTextDataset(Dataset):
def init(self, csv_file, images_folder, processor, max_length=128):
self.data = pd.read_csv(csv_file)
self.images_folder = images_folder
self.processor = processor
self.max_length = max_length
self.image_size = (224, 224)
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
image_id = self.data.iloc[idx]['id'].strip()
question = self.data.iloc[idx]['user']
answer = self.data.iloc[idx]['assistant']
image_path = os.path.join(self.images_folder, f"{image_id}.png")
image = Image.open(image_path).convert("RGB").resize(self.image_size)
# Process inputs
inputs = self.processor(
text=question,
images=image,
return_tensors="pt",
padding='max_length',
max_length=self.max_length,
truncation=True
)
# Process labels
label_output = self.processor(
text=answer,
return_tensors="pt",
padding='max_length',
max_length=self.max_length,
truncation=True
)
labels = label_output.input_ids if label_output is not None else torch.zeros((1, self.max_length), dtype=torch.long)
return {
"input_ids": inputs.input_ids.squeeze(),
"pixel_values": inputs.pixel_values.squeeze(),
"labels": labels.squeeze()
}
Check if the CSV file and images folder exist
if not os.path.exists(“mapping.csv”):
raise FileNotFoundError(“mapping.csv not found”)
if not os.path.exists(“images_folder”):
raise FileNotFoundError(“images_folder not found”)
dataset = ImageTextDataset(csv_file=“mapping.csv”, images_folder=“images_folder”, processor=processor, max_length=128)
Verify dataset is not empty
if len(dataset) == 0:
raise ValueError(“Dataset is empty”)
def collate_fn(batch):
if not batch:
raise ValueError(“Batch is empty”)
input_ids = torch.stack([item[‘input_ids’] for item in batch if ‘input_ids’ in item])
pixel_values = torch.stack([item[‘pixel_values’] for item in batch if ‘pixel_values’ in item])
labels = torch.stack([item[‘labels’] for item in batch if ‘labels’ in item])
# Ensure all tensors have the same batch dimension
if len(input_ids) != len(pixel_values) or len(input_ids) != len(labels):
raise ValueError("Inconsistent batch sizes")
return {
"input_ids": input_ids,
"pixel_values": pixel_values,
"labels": labels
}
dataloader = DataLoader(
dataset,
batch_size=2,
shuffle=True,
collate_fn=collate_fn,
num_workers=0 # Set to 0 to make debugging easier
)
Verify dataloader is not empty
try:
first_batch = next(iter(dataloader))
except StopIteration:
raise ValueError(“DataLoader is empty”)
Set model to training mode
model.train()
optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5)
epochs = 1
Training loop
total_steps = len(dataloader)
for epoch in range(epochs):
running_loss = 0.0
progress_bar = tqdm(enumerate(dataloader), total=total_steps, desc=f"Epoch {epoch + 1}/{epochs}")
for step, batch in progress_bar:
# Zero the gradients
optimizer.zero_grad()
# Move data to device
input_ids = batch["input_ids"].to(device)
pixel_values = batch["pixel_values"].to(device)
labels = batch["labels"].to(device)
# Forward pass
outputs = model(input_ids=input_ids, pixel_values=pixel_values, labels=labels)
loss = outputs.loss
# Backward pass
loss.backward()
# Update weights
optimizer.step()
# Update progress bar
running_loss += loss.item()
avg_loss = running_loss / (step + 1)
progress_bar.set_postfix({"avg_loss": f"{avg_loss:.4f}", "loss": f"{loss.item():.4f}"})