Hello,
i using tansformer network.
This code bring in “GitHub - LiamMaclean216/Pytorch-Transfomer: My implementation of the transformer architecture from the Attention is All you need paper applied to time series.”
I verify CPU version, so using GPU version.
Add to cuda .to(device)
BUt this ERROR
train_predict_outputs = model(train_data)
Traceback (most recent call last):
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/IPython/core/interactiveshell.py”, line 3417, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File “”, line 1, in
train_predict_outputs = model(train_data)
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 550, in call
result = self.forward(*input, **kwargs)
File “/home/woon/06_pytorch/Reference_code_/Pytorch-Transfomer-master/Network.py”, line 74, in forward
e = self.encs0
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 550, in call
result = self.forward(*input, **kwargs)
File “/home/woon/06_pytorch/Reference_code_/Pytorch-Transfomer-master/Network.py”, line 19, in forward
a = self.attn(x)
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 550, in call
result = self.forward(*input, **kwargs)
File “/home/woon/06_pytorch/Reference_code_/Pytorch-Transfomer-master/utils.py”, line 50, in forward
a.append(h(x, kv = kv))
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 550, in call
result = self.forward(*input, **kwargs)
File “/home/woon/06_pytorch/Reference_code_/Pytorch-Transfomer-master/utils.py”, line 30, in forward
return attention(self.query(x), self.key(x), self.value(x))
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 550, in call
result = self.forward(*input, **kwargs)
File “/home/woon/06_pytorch/Reference_code_/Pytorch-Transfomer-master/utils.py”, line 97, in forward
x = self.fc1(x)
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 550, in call
result = self.forward(*input, **kwargs)
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/linear.py”, line 87, in forward
return F.linear(input, self.weight, self.bias)
File “/home/woon/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/functional.py”, line 1612, in linear
output = input.matmul(weight.t())
RuntimeError: Expected object of device type cuda but got device type cpu for argument #2 ‘mat2’ in call to _th_mm
i think this class error
class AttentionBlock(torch.nn.Module):
def init(self, dim_val, dim_attn):
super(AttentionBlock, self).init()
self.value = Value(dim_val, dim_val)
self.key = Key(dim_val, dim_attn)
self.query = Query(dim_val, dim_attn)
def forward(self, x, kv = None):
if(kv is None):
#Attention with x connected to Q,K and V (For encoder)
return attention(self.query(x), self.key(x), self.value(x))
#Attention with x as Q, external vector kv as K an V (For decoder)
return attention(self.query(x), self.key(kv), self.value(kv))
class MultiHeadAttentionBlock(torch.nn.Module):
def init(self, dim_val, dim_attn, n_heads):
super(MultiHeadAttentionBlock, self).init()
self.heads = []
for i in range(n_heads):
self.heads.append(AttentionBlock(dim_val, dim_attn))
self.heads = nn.ModuleList(self.heads)
self.fc = nn.Linear(n_heads * dim_val, dim_val, bias = False)
def forward(self, x, kv = None):
a = []
for h in self.heads:
a.append(h(x, kv = kv))
a = torch.stack(a, dim = -1) #combine heads
a = a.flatten(start_dim = 2) #flatten all head outputs
x = self.fc(a)
return x
but I dont know what to change.