参考文章：序列模型——搭建循环神经网络及其应用
这周的编程作业好难啊，明明原理都懂的一实践就完蛋，模棱两可的码了好久。
问题：在执行LSTM网络即兴演奏爵士乐代码时出现了AssertError，我将preprocess.py文件第110行的assert len(chords) == len(measures)注释掉了，加了一行del measures[len(measures) - 1]，并将preprocess.py、data_utils.py、music_utils.py文件中所有的78换成了80，成功运行。
1.搭建循环神经网络
main.py
import numpy as npimport Deep_Learning.test5_1.rnn_utils# 1.1RNN单元def rnn_cell_forward(xt, a_prev, parameters):"""实现RNN单元的单步前向传播:param xt:-时间步t输入的数据，维度为(n_x,m):param a_prev: -时间步t-1的隐藏状态，维度为(n_a,m):param parameters: -字典，包含以下内容：Wax-矩阵，输入乘以权重，维度为(n_a,n_x)Waa-矩阵，隐藏状态乘以权重，维度为(n_a,n_a)Wya-矩阵，隐藏状态与输出相关的权重矩阵，维度为(n_y,n_a)ba-偏置，维度为(n_a,1)by-偏置，维度为(n_y,1):return: a_next-下一个隐藏状态，维度为(n_a,m)yt_pred -在时间步t的预测，维度为(n_y,m)cache-反向传播需要的元组，包含了(a_next,a_prev,xt,parameters)"""# 从parameters获取参数Wax = parameters["Wax"]Waa = parameters["Waa"]Wya = parameters["Wya"]ba = parameters["ba"]by = parameters["by"]# 计算下一个激活值a_next = np.tanh(np.dot(Waa, a_prev) + np.dot(Wax, xt) + ba)# 计算当前单元的输出yt_pred = Deep_Learning.test5_1.rnn_utils.softmax(np.dot(Wya, a_next) + by)# 保存反向传播需要的值cache = (a_next, a_prev, xt, parameters)return a_next, yt_pred, cacheprint("==================测试rnn_cell_forward================")np.random.seed(1)xt = np.random.rand(3, 10)a_prev = np.random.randn(5, 10)Waa = np.random.randn(5, 5)Wax = np.random.randn(5, 3)Wya = np.random.randn(2, 5)ba = np.random.randn(5, 1)by = np.random.randn(2, 1)parameters = {"Waa": Waa, "Wax": Wax, "Wya": Wya, "ba": ba, "by": by}a_next, yt_pred, cache = rnn_cell_forward(xt, a_prev, parameters)print("a_next[4] = ", a_next[4])print("a_next.shape = ", a_next.shape)print("yt_pred[1] = ", yt_pred[1])print("yt_pred.shape = ", yt_pred.shape)# 1.2RNN的前向传播def rnn_forward(x, a0, parameters):"""实现循环神经网络的前向传播:param x: -输入的全部数据，维度为(n_x,m,T_x):param a0: -初始化隐藏状态，维度为(n_a,m):param parameters: 字典，包含以下内容：Wax-矩阵，输入乘以权重，维度为(n_a,n_x)Waa-矩阵，隐藏状态乘以权重，维度为(n_a,n_a)Wya-矩阵，隐藏状态与输出相关的权重矩阵，维度为(n_y,n_a)ba-偏置，维度为(n_a,1)by-偏置，维度为(n_y,1):return: a -所有时间步的隐藏状态，维度为(n_a,m,T_x)y_pred-所有时间步的预测，维度为(n_y,m,T_x)caches-为反向传播的保存的元组，维度为([列表类型]cache,x)"""# 初始化caches，它将以列表类型包含所有的cachecaches = []# 获取x与Wya的维度信息n_x, m, T_x = x.shapen_y, n_a = parameters["Wya"].shape# 使用0初始化"a"与"y"a = np.zeros([n_a, m, T_x])y_pred = np.zeros([n_y, m, T_x])# 初始化nexta_next = a0# 遍历所有时间步for t in range(T_x):# 1.使用rnn_cell_forward函数更新next隐藏状态与cachea_next, yt_pred, cache = rnn_cell_forward(x[:, :, t], a_next, parameters)# 2.使用a保存next隐藏状态(第t)个位置a[:, :, t] = a_next# 3.使用y保存预测值y_pred[:, :, t] = yt_pred# 4.把cache保存到caches列表中caches.append(cache)# 保存反向传播所需要的参数caches = (caches, x)return a, y_pred, cachesprint("=======================测试rnn_forward====================")np.random.seed(1)x = np.random.rand(3, 10, 4)a0 = np.random.randn(5, 10)Waa = np.random.randn(5, 5)Wax = np.random.randn(5, 3)Wya = np.random.randn(2, 5)ba = np.random.randn(5, 1)by = np.random.randn(2, 1)parameters = {"Waa": Waa, "Wax": Wax, "Wya": Wya, "ba": ba, "by": by}a, y_pred, caches = rnn_forward(x, a0, parameters)print("a[4][1] = ", a[4][1])print("a.shape = ", a.shape)print("y_pred[1][3] = ", y_pred[1][3])print("y_pred.shape = ", y_pred.shape)print("caches[1][1][3] = ", caches[1][1][3])print("len(caches) = ", len(caches))# 长短时记忆网络LSTM# 2.1LSTM单元def lstm_cell_forward(xt, a_prev, c_prev, parameters):"""实现一个LSTM单元的前向传播:param xt:-在时间步t输入的数据，维度为(n_x,m):param a_prev: -上一个时间步t-1的隐藏状态，维度为(n_a,m):param c_prev: -上一个时间步t-1的记忆状态，维度为(n_a,m):param parameters: -字典类型的变量，包含了:Wf -遗忘门的权值，维度为(n_a,n_a+n_x)bf -遗忘门的偏置，维度为(n_a,1)Wu -更新门的权值，维度为(n_a,n_a+n_x)bu -更新门的偏置，维度为(n_a,1)Wc -第一个tanh的权值，维度为(n_a,n_a+n_x)bc -第一个tanh的偏置，维度为(n_a,n_a+n_x)Wo -输出门的权值，维度为(n_a,n_a+n_x)bo -输出门的偏置，维度为(n_a,1)Wy -隐藏状态与输出相关的权值，维度为(n_y,n_a)by -隐藏状态与输出相关的权值，维度为(n_y,1):return: a_next-下一个隐藏状态，维度为(n_a,m)c_next-下一个记忆状态，维度为(n_a,m)yt_pred -在时间步t的预测，维度为(n_y,m)cache-包含了反向传播所需要的参数，包含了(a_next,c_next,a_prev,c_prev,xt,parameters)"""# 从parameters中获取相关值Wf = parameters["Wf"]bf = parameters["bf"]Wu = parameters["Wu"]bu = parameters["bu"]Wc = parameters["Wc"]bc = parameters["bc"]Wo = parameters["Wo"]bo = parameters["bo"]Wy = parameters["Wy"]by = parameters["by"]# 获取xt与Wy的维度信息n_x, m = xt.shapen_y, n_a = Wy.shape# 1.连接a_prev与xtcontact = np.zeros([n_a + n_x, m])contact[: n_a, :] = a_prevcontact[n_a:, :] = xt# 2.计算遗忘门ft，更新门ut，候选值(c tilda)，下一个记忆状态c_next，输出门ot，下一个激活状态a_next# 遗忘门ft = Deep_Learning.test5_1.rnn_utils.sigmoid(np.dot(Wf, contact) + bf)# 更新门ut = Deep_Learning.test5_1.rnn_utils.sigmoid(np.dot(Wu, contact) + bu)# 更新单元cct = np.tanh(np.dot(Wc, contact) + bc)# c_next = np.multiply(ft, c_prev) + np.multiply(ut, cct)c_next = ft * c_prev + ut * cct# 输出门ot = Deep_Learning.test5_1.rnn_utils.sigmoid(np.dot(Wo, contact) + bo)# a_next = np.multiply(ot, np.tanh(c_next))a_next = ot * np.tanh(c_next)# 3.计算LSTM单元的预测值yt_pred = Deep_Learning.test5_1.rnn_utils.softmax(np.dot(Wy, a_next) + by)# 保存包含了反向传播需要的参数cache = (a_next, c_next, a_prev, c_prev, ft, ut, cct, ot, xt, parameters)return a_next, c_next, yt_pred, cacheprint("====================测试lstm_cell_forward=====================")np.random.seed(1)xt = np.random.rand(3, 10)a_prev = np.random.randn(5, 10)c_prev = np.random.randn(5, 10)Wf = np.random.randn(5, 5+3)bf = np.random.randn(5, 1)Wu = np.random.randn(5, 5+3)bu = np.random.randn(5, 1)Wo = np.random.randn(5, 5+3)bo = np.random.randn(5, 1)Wc = np.random.randn(5, 5+3)bc = np.random.randn(5, 1)Wy = np.random.randn(2, 5)by = np.random.randn(2, 1)parameters = {"Wf": Wf, "Wu": Wu, "Wo": Wo, "Wc": Wc, "Wy": Wy, "bf": bf, "bu": bu, "bo": bo, "bc": bc, "by": by}a_next, c_next, yt, cache = lstm_cell_forward(xt, a_prev, c_prev, parameters)print("a_next[4] = ", a_next[4])print("a_next.shape = ", a_next.shape)print("c_next[2] = ", c_next[2])print("c_next.shape = ", c_next.shape)print("yt[1] = ", yt[1])print("yt.shape = ", yt.shape)print("cache[1][3] = ", cache[1][3])print("len(cache) = ", len(cache))# 2.2Lstm前向传播，c^<0>使用0初始化def lstm_forward(x, a0, parameters):"""实现LSTM单元组成的循环神经网络:param x: -所有时间步的输入数据，维度为(n_x,m,T_x):param a0: -初始化隐藏状态，维度为(n_a,m):param parameters: -字典，包含以下参数：Wf -遗忘门的权值，维度为(n_a,n_a+n_x)bf -遗忘门的偏置，维度为(n_a,1)Wu -更新门的权值，维度为(n_a,n_a+n_x)bu -更新门的偏置，维度为(n_a,1)Wc -第一个tanh的权值，维度为(n_a,n_a+n_x)bc -第一个tanh的偏置，维度为(n_a,n_a+n_x)Wo -输出门的权值，维度为(n_a,n_a+n_x)bo -输出门的偏置，维度为(n_a,1)Wy -隐藏状态与输出相关的权值，维度为(n_y,n_a)by -隐藏状态与输出相关的权值，维度为(n_y,1):return: a -所有时间步的隐藏状态，维度为(n_a,m,T_x)y -所有时间步的预测值，维度为(n_y,m,T_x)caches -为反向传播保存的元组，维度为([列表类型]cache,x)"""# 初始化“caches”caches = []# 获取xt和Wy的维度信息n_x, m, T_x = x.shapen_y, n_a = parameters["Wy"].shape# 使用0初始化"a","c","y"a = np.zeros([n_a, m, T_x])c = np.zeros([n_a, m, T_x])y = np.zeros([n_y, m, T_x])# 初始化"a_next"、"c_next"a_next = a0c_next = np.zeros([n_a, m])# 遍历所有时间步for t in range(T_x):# 更新下一个隐藏状态，下一个记忆状态，计算预测值，获取cachea_next, c_next, yt_pred, cache = lstm_cell_forward(x[:, :, t], a_next, c_next, parameters)# 保存新的下一个隐藏状态到变量a中a[:, :, t] = a_next# 保存预测值到变量y中y[:, :, t] = yt_pred# 保存下一个单元状态到变量c中c[:, :, t] = c_next# 把cache添加到caches中caches.append(cache)# 保存反向传播需要的参数caches = (caches, x)return a, y, c, cachesprint("===========================测试lstm_forward====================")np.random.seed(1)x = np.random.rand(3, 10, 7)a0 = np.random.randn(5, 10)Wf = np.random.randn(5, 5+3)bf = np.random.randn(5, 1)Wu = np.random.randn(5, 5+3)bu = np.random.randn(5, 1)Wo = np.random.randn(5, 5+3)bo = np.random.randn(5, 1)Wc = np.random.randn(5, 5+3)bc = np.random.randn(5, 1)Wy = np.random.randn(2, 5)by = np.random.randn(2, 1)parameters = {"Wf": Wf, "Wu": Wu, "Wo": Wo, "Wc": Wc, "Wy": Wy, "bf": bf, "bu": bu, "bo": bo, "bc": bc, "by": by}a, y, c, caches = lstm_forward(x, a0, parameters)print("a[4][3][6] = ", a[4][3][6])print("a.shape = ", a.shape)print("y[1][4][3] = ", y[1][4][3])print("y.shape = ", y.shape)print("caches[1][1][1] = ", caches[1][1][1])print("c[1][2][1] = ", c[1][2][1])print("len(caches) = ", len(caches))## 2.3RNN反向传播（选学）
运行结果
==================测试rnn_cell_forward================a_next[4] = [ 0.78343681 0.99841326 0.3412785 0.74815333 0.06904651 -0.4412712-0.96311151 -0.9255633 -0.39331674 0.8259313 ]a_next.shape = (5, 10)yt_pred[1] = [0.61103326 0.62075852 0.54066066 0.49386023 0.89878639 0.719153860.73411439 0.88215484 0.60354066 0.62300476]yt_pred.shape = (2, 10)=======================测试rnn_forward====================a[4][1] = [ 0.74340266 -0.15802485 0.53481814 0.39883487]a.shape = (5, 10, 4)y_pred[1][3] = [0.59596259 0.44898213 0.44507391 0.50777107]y_pred.shape = (2, 10, 4)caches[1][1][3] = [0.21162812 0.26554666 0.49157316 0.05336255]len(caches) = 2====================测试lstm_cell_forward=====================a_next[4] = [ 0.00801245 -0.04933657 -0.01185145 0.01026473 -0.10713704 0.062705730.06914978 -0.01141511 0.01495696 -0.38133372]a_next.shape = (5, 10)c_next[2] = [ 0.60879308 0.71075478 -0.63003263 1.55707745 -0.12389442 0.95036850.07566992 -1.22707045 0.25010194 -0.15193039]c_next.shape = (5, 10)yt[1] = [0.24208087 0.01697087 0.35355734 0.25499077 0.3899106 0.431510560.320404 0.28227159 0.28307044 0.48093818]yt.shape = (2, 10)cache[1][3] = [-0.36843733 0.92140172 -0.03508767 0.54797075 0.14707127 0.304599330.02522788 0.54096181 -0.22560049 -0.53332628]len(cache) = 10===========================测试lstm_forward====================a[4][3][6] = -0.01086643920336303a.shape = (5, 10, 7)y[1][4][3] = 0.7850075501598014y.shape = (2, 10, 7)caches[1][1][1] = [0.34776586 0.7508121 0.72599799 0.88330609 0.62367221 0.750942430.34889834]c[1][2][1] = 0.6424527847446263len(caches) = 2
2.字符级语言模型
main.py
"""代码实现：1.存储文本数据以便使用RNN进行处理2.合成数据，通过每次采样预测，并将其传递给下一个RNN单元3.构建字符级文本生成循环神经网络4.梯度修剪：避免梯度爆炸"""import numpy as npimport timeimport Deep_Learning.test5_1.cllm_utils# 1.读取恐龙名称的数据集，创建一个唯一字符列表（如AZ），并计算数据集和词汇量大小# 获取名称data = open("dinos.txt", "r").read()# 转化为小写字符data = data.lower()# 转化为无序且不重复的元素列表chars = list(set(data))# 获取大小信息data_size, vocab_size = len(data), len(chars)print(chars)print("共计有%d个字符，唯一字符有%d个"%(data_size, vocab_size))# 创建一个字典，每个字符映射到0-26的索引char_to_ix = {ch: i for i, ch in enumerate(sorted(chars))}# 将字典每个索引映射回相应的字符ix_to_char = {i: ch for i, ch in enumerate(sorted(chars))}print(char_to_ix)print(ix_to_char)"""运行结果：['c', 'a', 'l', 'm', 'n', 'h', 'b', 'k', 'd', 'u', 'f', 's', '\n', 'v', 'g', 'e', 'o', 'i', 'q', 'y', 'z', 'j', 'r', 'w', 'p', 't', 'x']共计有19909个字符，唯一字符有27个{'\n': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26}{0: '\n', 1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z'}"""# 2.构建模型中的模块# 2.1梯度修剪，函数接受最大阈值def clip(gradients, maxValue):"""使用maxValue来修剪梯度:param gradients: -字典类型，包含了以下参数："dWaa","dWax","dWya","db","dby":param maxValue: -阈值，把梯度值限制在[-maxValue,maxValue]内:return:gradients -修剪后的梯度"""# 获取参数dWaa, dWax, dWya, db, dby = gradients['dWaa'], gradients['dWax'], gradients['dWya'], gradients['db'], gradients['dby']# 修剪梯度for gradient in [dWaa, dWax, dWya, db, dby]:np.clip(gradient, -maxValue, maxValue, out=gradient)gradients = {"dWaa": dWaa, "dWax": dWax, "dWya": dWya, "db": db, "dby": dby}return gradientsprint("======================测试clip=========================")np.random.seed(3)dWax = np.random.randn(5, 3) * 10dWaa = np.random.randn(5, 5) * 10dWya = np.random.randn(2, 5) * 10db = np.random.randn(5, 1) * 10dby = np.random.randn(2, 1) * 10gradients = {"dWaa": dWaa,"dWax": dWax,"dWya": dWya,"db": db,"dby": dby}gradients = clip(gradients, 10)print("gradients[\"dWaa\"][1][2] = ", gradients["dWaa"][1][2])print("gradients[\"dWax\"][3][1] = ", gradients["dWax"][3][1])print("gradients[\"dWya\"][1][2] = ", gradients["dWya"][1][2])print("gradients[\"db\"][4] = ", gradients["db"][4])print("gradients[\"dby\"][1] = ", gradients["dby"][1])"""运行结果：gradients["dWaa"][1][2] = 10.0gradients["dWax"][3][1] = -10.0gradients["dWya"][1][2] = 0.2971381536101662gradients["db"][4] = [10.]gradients["dby"][1] = [8.45833407] """# 2.2采样def sample(parameters, char_to_ix, seed):"""根据RNN输出的概率分布序列对字符序列进行采样:param parameters: -包含了Waa,Wax,Wya,by,b的字典:param char_to_ix: -字符映射到索引的字典:param seed: -随机种子:return: indices -包含采样字符索引的长度为n的列表"""# 从parameters中获取参数Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']vocab_size = by.shape[0]n_a = Waa.shape[1]# 1.1创建独热向量xx = np.zeros((vocab_size, 1))# 1.2使用0初始化a_preva_prev = np.zeros((n_a, 1))# 创建索引的空列表，这是包含要生成的字符的索引的列表indices = []# IDX是检测换行符的标志，将其初始化为-1idx = -1# 循环遍历时间步t，在每个时间步中，从概率分布中抽取一个字符，并将其索引附加到indices上# 如果我们达到50个字符（我们应该不太可能有一个训练好的模型），将停止循环，这有助于调试并防止进入无限循环counter = 0newline_character = char_to_ix["\n"]while(idx != newline_character and counter < 50):# 2.进行前向传播a = np.tanh(np.dot(Wax, x) + np.dot(Waa, a_prev) + b)z = np.dot(Wya, a) + byy = Deep_Learning.test5_1.cllm_utils.softmax(z)# 设定随机种子np.random.seed(counter + seed)# 3.从概率分布y中抽取词汇表中字符的索引？？？？？？？？？？？？idx = np.random.choice(list(range(vocab_size)), p=y.ravel())# 添加到索引中indices.append(idx)# 4.将输入字符重写为与采样索引相对应的字符x = np.zeros((vocab_size, 1))x[idx] = 1# 更新a_prev为aa_prev = a# 累加器seed += 1counter += 1if(counter == 50):indices.append(char_to_ix["\n"])return indicesprint("==================测试sample===============")np.random.seed(2)_, n_a = 20, 100Wax = np.random.randn(n_a, vocab_size)Waa = np.random.randn(n_a, n_a)Wya = np.random.randn(vocab_size, n_a)b = np.random.randn(n_a, 1)by = np.random.randn(vocab_size, 1)parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b, "by": by}indices = sample(parameters, char_to_ix, 0)print("Sampling:")print("list of sampled indices:", indices)print("list of sampled characters:", [ix_to_char[i] for i in indices])"""运行结果：list of sampled indices: [12, 17, 24, 14, 13, 9, 10, 22, 24, 6, 13, 11, 12, 6, 21, 15, 21, 14, 3, 2, 1, 21, 18, 24, 7, 25, 6, 25, 18, 10, 16, 2, 3, 8, 15, 12, 11, 7, 1, 12, 10, 2, 7, 7, 11, 17, 24, 1, 13, 0, 0]list of sampled characters: ['l', 'q', 'x', 'n', 'm', 'i', 'j', 'v', 'x', 'f', 'm', 'k', 'l', 'f', 'u', 'o', 'u', 'n', 'c', 'b', 'a', 'u', 'r', 'x', 'g', 'y', 'f', 'y', 'r', 'j', 'p', 'b', 'c', 'h', 'o', 'l', 'k', 'g', 'a', 'l', 'j', 'b', 'g', 'g', 'k', 'q', 'x', 'a', 'm', '\n', '\n']"""# 3.构建语言模型# 3.1梯度下降def optimize(X, Y, a_prev, parameters, learning_rate = 0.01):"""执行训练模型的单步优化:param X: -整数列表，其中每个整数映射到词汇表中的字符:param Y: -整数列表，与X完全相同，但向左移动了一个索引:param a_prev: -上一个隐藏状态:param parameters: -字典，包含以下参数：Wax-权重矩阵乘以输入，维度为(n_a,n_x)Waa-权重矩阵乘以隐藏状态，维度为(n_a,n_a)Wya-隐藏状态与输出相关的权重矩阵，维度为(n_y,n_a)b -偏置，维度为(n_a,1)by-隐藏状态与输出相关的权重偏置，维度为(n_y,1):param learning_rate: -学习率:return:loss -损失函数的值(交叉熵损失)gradients -字典，包含以下参数dWax-输入到隐藏的权值的梯度，维度为(n_a,n_x)dWaa-隐藏到隐藏的权值的梯度，维度为(n_a,n_a)dWya-隐藏到输出的权值的梯度，维度为(n_y,n_a)db -偏置的梯度，维度为(n_a,1)dby-输出偏置向量的梯度，维度为(n_y,1)a[len(X)-1]-最后的隐藏状态，维度为(n_a,1)"""# 前向传播loss, cache = Deep_Learning.test5_1.cllm_utils.rnn_forward(X, Y, a_prev, parameters)# 反向传播gradients, a = Deep_Learning.test5_1.cllm_utils.rnn_backward(X, Y, parameters, cache)# 梯度修剪，[-5,5]gradients = clip(gradients, 5)# 更新参数parameters = Deep_Learning.test5_1.cllm_utils.update_parameters(parameters, gradients, learning_rate)return loss, gradients, a[len(X) - 1]print("=====================测试optimize===============")np.random.seed(1)vocab_size, n_a = 27, 100a_prev = np.random.randn(n_a, 1)Wax = np.random.randn(n_a, vocab_size)Waa = np.random.randn(n_a, n_a)Wya = np.random.randn(vocab_size, n_a)b = np.random.randn(n_a, 1)by = np.random.randn(vocab_size, 1)parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b, "by": by}X = [12, 3, 5, 11, 22, 3]Y = [4, 14, 11, 22, 25, 26]loss, gradients, a_last = optimize(X, Y, a_prev, parameters, learning_rate=0.01)print("loss = ", loss)print("gradients[\"dWaa\"][1][2] = ", gradients["dWaa"][1][2])print("np.argmax(gradients[\"dWax\"]) = ", np.argmax(gradients["dWax"]))print("gradients[\"dWya\"][1][2] = ", gradients["dWya"][1][2])print("gradients[\"db\"][4] = ", gradients["db"][4])print("gradients[\"dby\"][1] = ", gradients["dby"][1])print("a_last[4] = ", a_last[4])"""运行结果：loss = 126.50397572165382gradients["dWaa"][1][2] = 0.1947093153471637np.argmax(gradients["dWax"]) = 93gradients["dWya"][1][2] = -0.007773876032002977gradients["db"][4] = [-0.06809825]gradients["dby"][1] = [0.01538192]a_last[4] = [-1.]"""# 3.2训练模型def model(data, ix_to_char, char_to_ix, num_iterations=3500,n_a=50, dino_names=7, vocab_size=27):"""训练模型并生成恐龙名字:param data: -语料库:param ix_to_char:-索引映射字符字典:param char_to_ix:-字符映射索引字典:param num_iterations: -迭代次数:param n_a: -RNN单元数量:param dino_names:-每次迭代中采样的数量:param vocab_size:-在文本中的唯一字符的数量:return: parameters-学习后了的参数"""# 从vocab_size中获取n_x, n_yn_x, n_y = vocab_size, vocab_size# 初始化参数parameters = Deep_Learning.test5_1.cllm_utils.initialize_parameters(n_a, n_x, n_y)# 初始化损失loss = Deep_Learning.test5_1.cllm_utils.get_initial_loss(vocab_size, dino_names)# 构建恐龙名称列表with open("dinos.txt") as f:examples = f.readlines()examples = [x.lower().strip() for x in examples]# 打乱全部的恐龙名称np.random.seed(0)np.random.shuffle(examples)# 初始化LSTM隐藏状态a_prev = np.zeros((n_a, 1))# 循环for j in range(num_iterations):# 定义一个训练样本index = j % len(examples)X = [None] + [char_to_ix[ch] for ch in examples[index]]Y = X[1:] + [char_to_ix["\n"]]# 执行单步优化：前向传播 -> 反向传播 -> 梯度修剪 -> 更新参数# 选择学习率为0.01curr_loss, gradients, a_prev = optimize(X, Y, a_prev, parameters)# 使用延迟来保持损失平衡，这是为了加速训练loss = Deep_Learning.test5_1.cllm_utils.smooth(loss, curr_loss)# 每2000次迭代，通过sample()生成"\n"字符，检查模型是否学习正确if j % 2000 == 0:print("第" + str(j + 1) + "次迭代，损失值为：" + str(loss))seed = 0for name in range(dino_names):# 采样sampled_indices = sample(parameters, char_to_ix, seed)Deep_Learning.test5_1.cllm_utils.print_sample(sampled_indices, ix_to_char)# 为了得到相同的效果，随机种子＋1seed += 1print("\n")return parameters# 训练# 开始时间start_time = time.clock()# 开始训练parameters = model(data, ix_to_char, char_to_ix, num_iterations=3500)# 结束时间end_time = time.clock()# 计算时差minium = end_time - start_timeprint("执行了：" + str(int(minium / 60)) + "分" + str(int(minium % 60)) + "秒")"""运行结果：第1次迭代，损失值为：23.087336085484605NkzxwtdmfqoeyhsqwasjkjvuKnebKzxwtdmfqoeyhsqwasjkjvuNebZxwtdmfqoeyhsqwasjkjvuEbXwtdmfqoeyhsqwasjkjvu第2001次迭代，损失值为：27.884160491415777LiusskeomnolxerosHmdaairusHytroligoraurusLecalosapausXusicikoraurusAbalpsamantisaurusTpraneronxeros执行了：0分4秒"""
3.写出莎士比亚风格的文字
main.py
import time# 开始时间start_time = time.clock()import numpy as npfrom keras.callbacks import LambdaCallbackfrom keras.models import Model, load_model, Sequentialfrom keras.layers import Dense, Activation, Dropout, Input, Maskingfrom keras.layers import LSTMfrom keras.utils.data_utils import get_filefrom keras.preprocessing.sequence import pad_sequencesfrom Deep_Learning.test5_1.shakespeare_utils import *import sysimport iofrom IPython.display import SVGfrom keras.utils.vis_utils import model_to_dotfrom keras.utils import plot_model# 结束时间end_time = time.clock()# 计算时差minium = end_time - start_timeprint("执行了：" + str(int(minium / 60)) + "分" + str(int(minium % 60)) + "秒")print_callback = LambdaCallback(on_epoch_end=on_epoch_end)model.fit(x, y, batch_size=128, epochs=1, callbacks=[print_callback])# 运行此代码尝试不同的输入，而不必重新训练模型generate_output()# 绘制模型细节plot_model(model, to_file='shakespeare.png')SVG(model_to_dot(model).create(prog='dot', format='svg'))
运行结果
Loading text data...Creating training set...number of training examples: 31412Vectorizing training set...Loading model...执行了：0分42秒246/246 [==============================] - 47s 190ms/step - loss: 2.5501Write the beginning of your poem, the Shakespeare machine will complete it. Your input is: Why don't we start overHere is your poem: Why don't we start overfreprite,on cithons he to e'su, on my my luvings do dield,her bilthou of make pu isfla sompter me autelone dyet man rehained.hif tray to preary is meling that cry gunntfich beend.to evevy aguhes to cossance for nowher hin try, mand my all my desting fide,that the pary suy i well my comprave chess,mistur kelps the to fate,, and my,do be of farmeded as to: the creballe you entery.so much
4.用LSTM网络即兴演奏爵士乐
main.py
"""代码实现：使用LSTM生成音乐使用深度学习生成自己的爵士乐"""import tensorflow as pat.v1.disable_eager_execution()import timefrom keras.models import load_model, Modelfrom keras.layers import Dense, Activation, Dropout, Input, LSTM, Reshape, Lambda, RepeatVectorfrom keras.initializers import glorot_uniformfrom keras.utils import to_categoricalfrom keras.optimizers import Adamfrom keras import backend as Kimport numpy as npimport IPythonimport sysfrom music21 import *from Deep_Learning.test5_1.grammar import *from Deep_Learning.test5_1.qa import *from Deep_Learning.test5_1.preprocess import *from Deep_Learning.test5_1.music_utils import *from Deep_Learning.test5_1.data_utils import *# 1.1查看训练集的音乐片段 pycharm不支持播放音乐，jupyter支持IPython.display.Audio('./data/30s_seq.mp3')# 加载原始音乐数据并将其预处理为值X, Y, n_values, indices_values = load_music_utils()print('shape of X:', X.shape)print('number of training examples:', X.shape[0])print('Tx (length of sequence):', X.shape[1])print('total # of unique values:', n_values)print('Shape of Y:', Y.shape)# 2.构建模型# 使用64维隐藏状态的LSTM模块n_a = 64# 将需要的层对象定义为全局变量reshapor = Reshape((1, 80))LSTM_cell = LSTM(n_a, return_state=True)densor = Dense(n_values, activation='softmax')def djmodel(Tx, n_a, n_values):"""实现模型:param Tx:-语料库的长度:param n_a:-激活值的数量:param n_values: -音乐数据中唯一数据的数量:return: model-Keras模型实体"""# 定义输入数据的维度X = Input((Tx, n_values))# 定义a0，初始化隐藏状态a0 = Input(shape=(n_a, ), name="a0")c0 = Input(shape=(n_a, ), name="c0")a = a0c = c0# 1.创建一个空的outputs列表来保持LSTM的所有时间步的输出outputs = []# 2.循环for t in range(Tx):# 2.1从X中选择第t个时间步向量x = Lambda(lambda x: X[:, t, :])(X)# 2.2使用reshapor来对x进行重构为(1,n_values)x = reshapor(x)# 2.3单步传播a, _, c = LSTM_cell(x, initial_state=[a, c])# 2.4使用densor()应用于LSTM_Cell的隐藏状态输出out = densor(a)# 2.5把预测值添加到outputs列表中outputs.append(out)# 3.创建模型实体model = Model(inputs=[X, a0, c0], outputs=outputs)return model# 获取模型，这里Tx=30,n_a=64,n_values=80model = djmodel(Tx=30, n_a=64, n_values=80)# 编译模型，使用Adam优化器与分类熵损失opt = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, decay=0.01)pile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])# 初始化a0和c0，使LSTM的初始化状态为零m = 60a0 = np.zeros((m, n_a))c0 = np.zeros((m, n_a))# 开始时间start_time = time.clock()# 开始拟合model.fit([X, a0, c0], list(Y), epochs=100)# 结束时间end_time = time.clock()# 计算时差minium = end_time - start_timeprint("执行了：" + str(int(minium / 60)) + "分" + str(int(minium % 60)) + "秒")# 3.生成音乐def music_inference_model(LSTM_cell, densor, n_values=80, n_a=64, Ty=100):""":param LSTM_cell: -来自model()的训练过后的LSTM单元，是keras层对象:param densor:-来自model()的训练过后的densor，是keras层对象:param n_values: -整数，唯一值的数量:param n_a: -LSTM单元的数量:param Ty:-整数，生成的是时间步的数量:return:inference_model-keras模型实体"""# 定义模型输入的维度x0 = Input(shape=(1, n_values))# 初始化隐藏状态a0 = Input(shape=(n_a,), name="a0")c0 = Input(shape=(n_a,), name="c0")a = a0c = c0x = x0# 1.创建一个空的outputs列表保存预测值outputs = []# 2.遍历Ty,生成所有时间步的输出for t in range(Ty):# 2.1在LSTM中单步传播a, _, c = LSTM_cell(x, initial_state=[a, c])# 2.2使用densor()应用于LSTM_Cell的隐藏状态输出out = densor(a)# 2.3预测值添加到outputs列表中outputs.append(out)# 根据out选择下一个值，并将x设置为所选值的一个独热编码# 该值将在下一步作为输入传递给LSTM_cell,我们已经提供了执行此操作所需的代码x = Lambda(one_hot)(out)# 创建模型实体inference_model = Model(inputs=[x0, a0, c0], outputs=outputs)return inference_model# 获取模型实体，模型被硬编码产生50个值inference_model = music_inference_model(LSTM_cell, densor, n_values=80, n_a=64, Ty=50)# 创建用于初始化x和LSTM状态变量a和c的零向量x_initializer = np.zeros((1, 1, 80))a_initializer = np.zeros((1, n_a))c_initializer = np.zeros((1, n_a))# 预测该输入对应的输出def predict_and_sample(inference_model, x_initializer=x_initializer, a_initializer=a_initializer,c_initializer=c_initializer):"""使用模型预测当前值的下一个值:param inference_model:-keras的实体模型:param x_initializer: -初始化的独热编码，维度为(1,1,80):param a_initializer: -LSTM单元的隐藏状态初始化，维度为(1,n_a):param c_initializer: -LSTM单元的状态初始化，维度为(1,n_a):return: results -生成值的独热编码向量，维度为(Ty,80)indices -所生成值的索引矩阵，维度为(Ty,1)"""# 1.模型来预测给定x_initializer,a_initializer,c_initializer的输出序列pred = inference_model.predict([x_initializer, a_initializer, c_initializer])# 2.将pred转换为具有最大概率的索引数组np.array()indices = np.argmax(pred, axis=-1)# 3.将索引转换为它们的一个独热编码results = to_categorical(indices, num_classes=80)return results, indicesresults, indices = predict_and_sample(inference_model, x_initializer, a_initializer, c_initializer)print("np.argmax(results[12]) = ", np.argmax(results[12]))print("np.argmax(results[17]) = ", np.argmax(results[17]))print("list(indices[12:18]) = ", list(indices[12:18]))# 生成音乐out_stream = generate_music(inference_model)IPython.display.Audio('./data/30s_trained_model.mp3')
运行结果
shape of X: (60, 30, 80)number of training examples: 60Tx (length of sequence): 30total # of unique values: 80Shape of Y: (30, 60, 80)Train on 60 samplesEpoch 1/10060/60 [==============================] - ...执行了：1分40秒np.argmax(results[12]) = 19np.argmax(results[17]) = 68list(indices[12:18]) = [array([19], dtype=int64), array([20], dtype=int64), array([20], dtype=int64), array([14], dtype=int64), array([19], dtype=int64), array([68], dtype=int64)]Predicting new values for different set of chords.Generated 50 sounds using the predicted values for the set of chords ("1") and after pruningGenerated 50 sounds using the predicted values for the set of chords ("2") and after pruningGenerated 51 sounds using the predicted values for the set of chords ("3") and after pruningGenerated 50 sounds using the predicted values for the set of chords ("4") and after pruningGenerated 50 sounds using the predicted values for the set of chords ("5") and after pruningYour generated music is saved in output/my_music.midi
