pytorch中的多元输入LSTM

Question

我想在Pytorch中实现多变量输入的LSTM。

在这篇使用keras的文章https://machinelearningmastery.com/how-to-develop-lstm-models-for-time-series-forecasting/之后，输入数据的形状是（样本数、时间步数、并行特征数）

in_seq1 = array([10, 20, 30, 40, 50, 60, 70, 80, 90])
in_seq2 = array([15, 25, 35, 45, 55, 65, 75, 85, 95])
out_seq = array([in_seq1[i]+in_seq2[i] for i in range(len(in_seq1))])
. . . 
Input     Output
[[10 15]
 [20 25]
 [30 35]] 65
[[20 25]
 [30 35]
 [40 45]] 85
[[30 35]
 [40 45]
 [50 55]] 105
[[40 45]
 [50 55]
 [60 65]] 125
[[50 55]
 [60 65]
 [70 75]] 145
[[60 65]
 [70 75]
 [80 85]] 165
[[70 75]
 [80 85]
 [90 95]] 185

n_timesteps = 3
n_features = 2

在keras，这似乎很容易：

model.add（LSTM（50，激活='relu'，input_shape=（n_timesteps，n_features））

除了创建LSTM的n_features作为第一层并分别馈送每个（想象为多个序列流），然后将它们的输出平坦到线性层之外，还能用其他方式实现吗？

我不是100%确定，但由于LSTM的性质，输入不能被展平并作为1D数组传递，因为每个序列“按照不同的规则运行”，LSTM应该学习这些规则。

那么，这样的实现与keras等于PyTorch输入的形状（seq_len，批处理，input_size）（源https://pytorch.org/docs/stable/nn.html#lstm）

编辑：

除了将LSTM的n_特征创建为第一层，分别为每个层馈电（想象为多个序列流），然后将其输出平坦化为线性层之外，是否可以用其他方式实现？

根据PyTorch文档，输入的_size参数实际上意味着特征的数量（如果它意味着并行序列的数量）




Answer 1

pytorch中任何rnn单元格中的输入都是3d输入，格式为（seq_len，batch，input_size）或（batch，seq_len，input_size），如果您喜欢第二个（也像me lol）init lstm层）或其他rnn层）和arg

bach_first = True

https://discuss.pytorch.org/t/could-someone-explain-batch-first-true-in-lstm/15402

此外，您在设置中没有任何reccurent关系。如果要创建多对一计数器，请在大小（-1，n，1）时创建输入，其中-1是所需的大小，n是位数，每个刻度一位数，如输入[[10][20][30]]，输出-60，输入[[30，][70]]输出100等，输入必须具有从1到某个最大值的不同长度，以便了解rnn关系

import random

import numpy as np

import torch


def rnd_io():    
    return  np.random.randint(100, size=(random.randint(1,10), 1))


class CountRNN(torch.nn.Module):

def __init__(self):
    super(CountRNN, self).__init__()

    self.rnn = torch.nn.RNN(1, 20,num_layers=1, batch_first=True)
    self.fc = torch.nn.Linear(20, 1)


def forward(self, x):        
    full_out, last_out = self.rnn(x)
    return self.fc(last_out)


nnet = CountRNN()

criterion = torch.nn.MSELoss(reduction='sum')

optimizer = torch.optim.Adam(nnet.parameters(), lr=0.0005)

batch_size = 100

batches = 10000 * 1000

printout = max(batches //(20* 1000),1)

for t in range(batches):

optimizer.zero_grad()

x_batch = torch.unsqueeze(torch.from_numpy(rnd_io()).float(),0)

y_batch = torch.unsqueeze(torch.sum(x_batch),0)

output = nnet.forward(x_batch) 

loss = criterion(output, y_batch)

if t % printout == 0:
    print('step : ' , t , 'loss : ' , loss.item())  
    torch.save(nnet.state_dict(), './rnn_summ.pth')  

loss.backward()

optimizer.step()

Answer 2

我希望对有问题的部分进行评论，使其有意义：

import random
import numpy as np
import torch

# multivariate data preparation
from numpy import array
from numpy import hstack
 
# split a multivariate sequence into samples
def split_sequences(sequences, n_steps):
    X, y = list(), list()
    for i in range(len(sequences)):
        # find the end of this pattern
        end_ix = i + n_steps
        # check if we are beyond the dataset
        if end_ix > len(sequences):
            break
        # gather input and output parts of the pattern
        seq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1, -1]
        X.append(seq_x)
        y.append(seq_y)
    return array(X), array(y)
 
# define input sequence
in_seq1 = array([x for x in range(0,100,10)])
in_seq2 = array([x for x in range(5,105,10)])
out_seq = array([in_seq1[i]+in_seq2[i] for i in range(len(in_seq1))])
# convert to [rows, columns] structure
in_seq1 = in_seq1.reshape((len(in_seq1), 1))
in_seq2 = in_seq2.reshape((len(in_seq2), 1))
out_seq = out_seq.reshape((len(out_seq), 1))
# horizontally stack columns
dataset = hstack((in_seq1, in_seq2, out_seq))

class MV_LSTM(torch.nn.Module):
    def __init__(self,n_features,seq_length):
        super(MV_LSTM, self).__init__()
        self.n_features = n_features
        self.seq_len = seq_length
        self.n_hidden = 20 # number of hidden states
        self.n_layers = 1 # number of LSTM layers (stacked)
    
        self.l_lstm = torch.nn.LSTM(input_size = n_features, 
                                 hidden_size = self.n_hidden,
                                 num_layers = self.n_layers, 
                                 batch_first = True)
        # according to pytorch docs LSTM output is 
        # (batch_size,seq_len, num_directions * hidden_size)
        # when considering batch_first = True
        self.l_linear = torch.nn.Linear(self.n_hidden*self.seq_len, 1)
        
    
    def init_hidden(self, batch_size):
        # even with batch_first = True this remains same as docs
        hidden_state = torch.zeros(self.n_layers,batch_size,self.n_hidden)
        cell_state = torch.zeros(self.n_layers,batch_size,self.n_hidden)
        self.hidden = (hidden_state, cell_state)
    
    
    def forward(self, x):        
        batch_size, seq_len, _ = x.size()
        
        lstm_out, self.hidden = self.l_lstm(x,self.hidden)
        # lstm_out(with batch_first = True) is 
        # (batch_size,seq_len,num_directions * hidden_size)
        # for following linear layer we want to keep batch_size dimension and merge rest       
        # .contiguous() -> solves tensor compatibility error
        x = lstm_out.contiguous().view(batch_size,-1)
        return self.l_linear(x)

n_features = 2 # this is number of parallel inputs
n_timesteps = 3 # this is number of timesteps

# convert dataset into input/output
X, y = split_sequences(dataset, n_timesteps)
print(X.shape, y.shape)

# create NN
mv_net = MV_LSTM(n_features,n_timesteps)
criterion = torch.nn.MSELoss() # reduction='sum' created huge loss value
optimizer = torch.optim.Adam(mv_net.parameters(), lr=1e-1)

train_episodes = 500
batch_size = 16

mv_net.train()
for t in range(train_episodes):
    for b in range(0,len(X),batch_size):
        inpt = X[b:b+batch_size,:,:]
        target = y[b:b+batch_size]    
        
        x_batch = torch.tensor(inpt,dtype=torch.float32)    
        y_batch = torch.tensor(target,dtype=torch.float32)
    
        mv_net.init_hidden(x_batch.size(0))
    #    lstm_out, _ = mv_net.l_lstm(x_batch,nnet.hidden)    
    #    lstm_out.contiguous().view(x_batch.size(0),-1)
        output = mv_net(x_batch) 
        loss = criterion(output.view(-1), y_batch)  
        
        loss.backward()
        optimizer.step()        
        optimizer.zero_grad() 
    print('step : ' , t , 'loss : ' , loss.item())

step :  499 loss :  0.0010267728939652443 # probably overfitted due to 500 training episodes