Implementing Xlnet classification class

1

Could you please let me know where i am going wrong.

 def __init__(self, n_classes):
   super(SentimentClassifier, self).__init__()
   self.xlnet = XLNetModel.from_pretrained(PRE_TRAINED_MODEL_NAME)
   self.drop = nn.Dropout(p=0.3)
   self.out = nn.Linear(self.xlnet.config.hidden_size, n_classes)
 def forward(self, input_ids, attention_mask):
   _, pooled_output = self.xlnet(self,
       input_ids=input_ids,
       attention_mask=attention_mask,
   )
   output = self.drop(pooled_output)
   return self.out(output)
class Classification(Dataset):
    def __init__(self, texts, labels, tokenizer, max_len):
       self.texts = texts
       self.labels = labels
       self.tokenizer = tokenizer
       self.max_len = max_len
   
   def __len__(self):
       return len(self.texts)
   
   def __getitem__(self, item):
       text = str(self.texts[item])
       label = self.labels[item]

       encoding = self.tokenizer.encode_plus(
       text,
       add_special_tokens=True,
       max_length=self.max_len,
       return_token_type_ids=False,
       pad_to_max_length=False,
       return_attention_mask=True,
       return_tensors='pt',
       )

       input_ids = pad_sequences(encoding['input_ids'], maxlen=MAX_LEN, dtype=torch.Tensor ,truncating="post",padding="post")
       input_ids = input_ids.astype(dtype = 'int64')
       input_ids = torch.tensor(input_ids) 

       attention_mask = pad_sequences(encoding['attention_mask'], maxlen=MAX_LEN, dtype=torch.Tensor ,truncating="post",padding="post")
       attention_mask = attention_mask.astype(dtype = 'int64')
       attention_mask = torch.tensor(attention_mask)       

       return {
       'review_text': text,
       'input_ids': input_ids,
       'attention_mask': attention_mask.flatten(),
       'labels': torch.tensor(label, dtype=torch.long)
       }
def train_epoch(
 model,
 data_loader,
 loss_fn,
 optimizer,
 device,
 scheduler,
 n_examples
):
 model = model.train()
 losses = []
 correct_predictions = 0
 for d in data_loader:
   input_ids = d["input_ids"].to(device)
   attention_mask = d["attention_mask"].to(device)
   labels = d["labels"].to(device)
   outputs = model(
     input_ids=input_ids,
     attention_mask=attention_mask)
   _, preds = torch.max(outputs[1], dim=1)
   loss = loss_fn(outputs, lables)
   correct_predictions += torch.sum(preds == labels)
   losses.append(loss.item())
   loss.backward()
   nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
   optimizer.step()
   scheduler.step()
   optimizer.zero_grad()
   
 return correct_predictions.double() / n_examples, np.mean(losses)
train_acc, train_loss = train_epoch(
   model,
   train_data_loader,
   loss_fn,
   optimizer,
   device,
   scheduler,
   len(df_train)
 )
print(f'Train loss {train_loss} accuracy {train_acc}') ```


error:TypeError: forward() got multiple values for argument 'input_ids'
2

Could you print out input_ids and its shape in the train loop?

3

thanks, i am getting below error trying to print shape
def train_epoch(
model,
data_loader,
loss_fn,
optimizer,
device,
scheduler,
n_examples
):
model = xlnet_model.train()
losses = []
correct_predictions = 0
for d in data_loader:
input_ids = d[“input_ids”].reshape(4,512).to(device)
print(d[‘input_ids’].shape)
attention_mask = d[“attention_mask”].to(device)
labels = d[“labels”].to(device)
outputs = xlnet_model(input_ids=input_ids, attention_mask=attention_mask)

_, preds = torch.max(outputs, dim=1)
loss = loss_fn(outputs, labels)
correct_predictions += torch.sum(preds == labels)
losses.append(loss.item())
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
optimizer.step()
scheduler.step()
optimizer.zero_grad()

return correct_predictions.double() / n_examples, np.mean(losses)

error:ValueError: not enough values to unpack (expected 2, got 1)

4

which line is this error on

5

at this line outputs = xlnet_model(input_ids=input_ids, attention_mask=attention_mask)

I am using xlnet pertained model.

6

ValueError Traceback (most recent call last)

in ()
6 device,
7 scheduler,
----> 8 len(df_train)
9 )
10 print(f’Train loss {train_loss} accuracy {train_acc}’)

2 frames

in forward(self, input_ids, attention_mask)
9 _, pooled_output = self.xlnet(
10 input_ids=input_ids,
—> 11 attention_mask=attention_mask)
12 output = self.drop(pooled_output)
13 return self.out(output)

ValueError: not enough values to unpack (expected 2, got 1)

7

Ok well in you original code in this line there is an extra comma

     attention_mask=attention_mask, <---- here
   )

but I don’t think that is the problem. What is the shape of your attention mask and what github are you using for the xlnet?

8

@Dwight_Foster
input_id and Attention mask shape size:
torch.Size([4, 160])
torch.Size([4, 160])

9

Ok what repository are you using for the xlnet model? The error means that for some reason either input ids or attention masks should have two different things in it and you are only passing one tensor for each.

10

I didn’t get , what you are saying, Could you please explain to me
link:https://curiousily.com/posts/sentiment-analysis-with-bert-and-hugging-face-using-pytorch-and-python/