Python与自然语言处理（NLP）：文本生成和机器翻译

from keras.models import Sequential
from keras.layers import LSTM, Dense
import numpy as np

# 假设data为训练数据
# 将文本数据转换为数值表示
data = "Hello, how are you?"
chars = list(set(data))
char_to_int = {char: i for i, char in enumerate(chars)}
int_to_char = {i: char for i, char in enumerate(chars)}
data_int = [char_to_int[char] for char in data]

# 构建输入和输出序列
seq_length = 10
X = []
y = []
for i in range(len(data_int) - seq_length):
    X.append(data_int[i:i+seq_length])
    y.append(data_int[i+seq_length])

X = np.array(X)
y = np.array(y)

# 构建LSTM模型
model = Sequential()
model.add(LSTM(128, input_shape=(seq_length, 1)))
model.add(Dense(len(chars), activation='softmax'))

# 编译模型并进行训练
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')
model.fit(X, y, epochs=100, batch_size=64)

# 使用模型生成文本
seed = "Hello, how"
for _ in range(50):
    x = [char_to_int[char] for char in seed]
    x = np.array(x)
    x = x.reshape(1, seq_length, 1)
    prediction = model.predict(x)
    index = np.argmax(prediction)
    result_char = int_to_char[index]
    seed += result_char
print(seed)

import torch
import torch.nn as nn
from torchtext.legacy.data import Field, TabularDataset, BucketIterator

# 假设data为翻译数据
# 使用torchtext库处理数据
src_field = Field(tokenize='spacy', tokenizer_language='en', init_token='<sos>', eos_token='<eos>')
trg_field = Field(tokenize='spacy', tokenizer_language='de', init_token='<sos>', eos_token='<eos>')

fields = {'src': ('src', src_field), 'trg': ('trg', trg_field)}

train_data, valid_data, test_data = TabularDataset.splits(
    path='path_to_data_folder',
    train='train.csv',
    validation='valid.csv',
    test='test.csv',
    format='csv',
    fields=fields
)

src_field.build_vocab(train_data, min_freq=2)
trg_field.build_vocab(train_data, min_freq=2)

# 构建Transformer模型
class TransformerModel(nn.Module):
    def __init__(self, src_vocab_size, trg_vocab_size, src_pad_idx, trg_pad_idx):
        # 定义模型结构
        pass

    def forward(self, src, trg):
        # 定义前向传播
        pass

# 定义损失函数和优化器
criterion = nn.CrossEntropyLoss(ignore_index=trg_pad_idx)
optimizer = torch.optim.Adam(model.parameters())

# 训练模型
model = TransformerModel(len(src_field.vocab), len(trg_field.vocab), src_field.vocab.stoi['<pad>'], trg_field.vocab.stoi['<pad>'])
for epoch in range(num_epochs):
    for batch in train_iterator:
        src = batch.src
        trg = batch.trg
        output = model(src, trg[:, :-1])
        output_dim = output.shape[-1]
        output = output.contiguous().view(-1, output_dim)
        trg = trg[:, 1:].contiguous().view(-1)
        loss = criterion(output, trg)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

# 使用模型进行翻译
def translate_sentence(model, sentence, src_field, trg_field):
    # 定义翻译函数
    pass