exten"t" instead "d" on p.23
This commit is contained in:
rasbt
+174
-203
@@ -120,6 +120,17 @@
|
||||
"from IPython.display import Image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -341,11 +352,9 @@
|
||||
"def load_mnist(path, kind='train'):\n",
|
||||
" \"\"\"Load MNIST data from `path`\"\"\"\n",
|
||||
" labels_path = os.path.join(path, \n",
|
||||
" '%s-labels-idx1-ubyte' \n",
|
||||
" % kind)\n",
|
||||
" '%s-labels-idx1-ubyte' % kind)\n",
|
||||
" images_path = os.path.join(path, \n",
|
||||
" '%s-images-idx3-ubyte' \n",
|
||||
" % kind)\n",
|
||||
" '%s-images-idx3-ubyte' % kind)\n",
|
||||
" \n",
|
||||
" with open(labels_path, 'rb') as lbpath:\n",
|
||||
" magic, n = struct.unpack('>II', \n",
|
||||
@@ -429,7 +438,6 @@
|
||||
],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"fig, ax = plt.subplots(nrows=2, ncols=5, sharex=True, sharey=True,)\n",
|
||||
"ax = ax.flatten()\n",
|
||||
@@ -504,15 +512,15 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#np.savetxt('train_img.csv', X_train, fmt='%i', delimiter=',')\n",
|
||||
"#np.savetxt('train_labels.csv', y_train, fmt='%i', delimiter=',')\n",
|
||||
"#X_train = np.genfromtxt('train_img.csv', dtype=int, delimiter=',')\n",
|
||||
"#y_train = np.genfromtxt('train_labels.csv', dtype=int, delimiter=',')\n",
|
||||
"# np.savetxt('train_img.csv', X_train, fmt='%i', delimiter=',')\n",
|
||||
"# np.savetxt('train_labels.csv', y_train, fmt='%i', delimiter=',')\n",
|
||||
"# X_train = np.genfromtxt('train_img.csv', dtype=int, delimiter=',')\n",
|
||||
"# y_train = np.genfromtxt('train_labels.csv', dtype=int, delimiter=',')\n",
|
||||
"\n",
|
||||
"#np.savetxt('test_img.csv', X_test, fmt='%i', delimiter=',')\n",
|
||||
"#np.savetxt('test_labels.csv', y_test, fmt='%i', delimiter=',')\n",
|
||||
"#X_test = np.genfromtxt('test_img.csv', dtype=int, delimiter=',')\n",
|
||||
"#y_test = np.genfromtxt('test_labels.csv', dtype=int, delimiter=',')\n"
|
||||
"# np.savetxt('test_img.csv', X_test, fmt='%i', delimiter=',')\n",
|
||||
"# np.savetxt('test_labels.csv', y_test, fmt='%i', delimiter=',')\n",
|
||||
"# X_test = np.genfromtxt('test_img.csv', dtype=int, delimiter=',')\n",
|
||||
"# y_test = np.genfromtxt('test_labels.csv', dtype=int, delimiter=',')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -553,49 +561,38 @@
|
||||
" Parameters\n",
|
||||
" ------------\n",
|
||||
" n_output : int\n",
|
||||
" Number of output units, should be equal to the\n",
|
||||
" number of unique class labels.\n",
|
||||
"\n",
|
||||
" Number of output units, should be equal to the\n",
|
||||
" number of unique class labels.\n",
|
||||
" n_features : int\n",
|
||||
" Number of features (dimensions) in the target dataset.\n",
|
||||
" Should be equal to the number of columns in the X array.\n",
|
||||
"\n",
|
||||
" Number of features (dimensions) in the target dataset.\n",
|
||||
" Should be equal to the number of columns in the X array.\n",
|
||||
" n_hidden : int (default: 30)\n",
|
||||
" Number of hidden units.\n",
|
||||
"\n",
|
||||
" Number of hidden units.\n",
|
||||
" l1 : float (default: 0.0)\n",
|
||||
" Lambda value for L1-regularization.\n",
|
||||
" No regularization if l1=0.0 (default)\n",
|
||||
"\n",
|
||||
" Lambda value for L1-regularization.\n",
|
||||
" No regularization if l1=0.0 (default)\n",
|
||||
" l2 : float (default: 0.0)\n",
|
||||
" Lambda value for L2-regularization.\n",
|
||||
" No regularization if l2=0.0 (default)\n",
|
||||
"\n",
|
||||
" Lambda value for L2-regularization.\n",
|
||||
" No regularization if l2=0.0 (default)\n",
|
||||
" epochs : int (default: 500)\n",
|
||||
" Number of passes over the training set.\n",
|
||||
"\n",
|
||||
" Number of passes over the training set.\n",
|
||||
" eta : float (default: 0.001)\n",
|
||||
" Learning rate.\n",
|
||||
"\n",
|
||||
" Learning rate.\n",
|
||||
" alpha : float (default: 0.0)\n",
|
||||
" Momentum constant. Factor multiplied with the\n",
|
||||
" gradient of the previous epoch t-1 to improve\n",
|
||||
" learning speed\n",
|
||||
" w(t) := w(t) - (grad(t) + alpha*grad(t-1))\n",
|
||||
" \n",
|
||||
" Momentum constant. Factor multiplied with the\n",
|
||||
" gradient of the previous epoch t-1 to improve\n",
|
||||
" learning speed\n",
|
||||
" w(t) := w(t) - (grad(t) + alpha*grad(t-1))\n",
|
||||
" decrease_const : float (default: 0.0)\n",
|
||||
" Decrease constant. Shrinks the learning rate\n",
|
||||
" after each epoch via eta / (1 + epoch*decrease_const)\n",
|
||||
"\n",
|
||||
" Decrease constant. Shrinks the learning rate\n",
|
||||
" after each epoch via eta / (1 + epoch*decrease_const)\n",
|
||||
" shuffle : bool (default: True)\n",
|
||||
" Shuffles training data every epoch if True to prevent circles.\n",
|
||||
"\n",
|
||||
" Shuffles training data every epoch if True to prevent circles.\n",
|
||||
" minibatches : int (default: 1)\n",
|
||||
" Divides training data into k minibatches for efficiency.\n",
|
||||
" Normal gradient descent learning if k=1 (default).\n",
|
||||
"\n",
|
||||
" Divides training data into k minibatches for efficiency.\n",
|
||||
" Normal gradient descent learning if k=1 (default).\n",
|
||||
" random_state : int (default: None)\n",
|
||||
" Set random state for shuffling and initializing the weights.\n",
|
||||
" Set random state for shuffling and initializing the weights.\n",
|
||||
"\n",
|
||||
" Attributes\n",
|
||||
" -----------\n",
|
||||
@@ -604,8 +601,8 @@
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" def __init__(self, n_output, n_features, n_hidden=30,\n",
|
||||
" l1=0.0, l2=0.0, epochs=500, eta=0.001, \n",
|
||||
" alpha=0.0, decrease_const=0.0, shuffle=True, \n",
|
||||
" l1=0.0, l2=0.0, epochs=500, eta=0.001,\n",
|
||||
" alpha=0.0, decrease_const=0.0, shuffle=True,\n",
|
||||
" minibatches=1, random_state=None):\n",
|
||||
"\n",
|
||||
" np.random.seed(random_state)\n",
|
||||
@@ -642,9 +639,11 @@
|
||||
"\n",
|
||||
" def _initialize_weights(self):\n",
|
||||
" \"\"\"Initialize weights with small random numbers.\"\"\"\n",
|
||||
" w1 = np.random.uniform(-1.0, 1.0, size=self.n_hidden*(self.n_features + 1))\n",
|
||||
" w1 = np.random.uniform(-1.0, 1.0,\n",
|
||||
" size=self.n_hidden*(self.n_features + 1))\n",
|
||||
" w1 = w1.reshape(self.n_hidden, self.n_features + 1)\n",
|
||||
" w2 = np.random.uniform(-1.0, 1.0, size=self.n_output*(self.n_hidden + 1))\n",
|
||||
" w2 = np.random.uniform(-1.0, 1.0,\n",
|
||||
" size=self.n_output*(self.n_hidden + 1))\n",
|
||||
" w2 = w2.reshape(self.n_output, self.n_hidden + 1)\n",
|
||||
" return w1, w2\n",
|
||||
"\n",
|
||||
@@ -681,30 +680,24 @@
|
||||
" Parameters\n",
|
||||
" -----------\n",
|
||||
" X : array, shape = [n_samples, n_features]\n",
|
||||
" Input layer with original features.\n",
|
||||
"\n",
|
||||
" Input layer with original features.\n",
|
||||
" w1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
"\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
" w2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" ----------\n",
|
||||
" a1 : array, shape = [n_samples, n_features+1]\n",
|
||||
" Input values with bias unit.\n",
|
||||
"\n",
|
||||
" Input values with bias unit.\n",
|
||||
" z2 : array, shape = [n_hidden, n_samples]\n",
|
||||
" Net input of hidden layer.\n",
|
||||
"\n",
|
||||
" Net input of hidden layer.\n",
|
||||
" a2 : array, shape = [n_hidden+1, n_samples]\n",
|
||||
" Activation of hidden layer.\n",
|
||||
"\n",
|
||||
" Activation of hidden layer.\n",
|
||||
" z3 : array, shape = [n_output_units, n_samples]\n",
|
||||
" Net input of output layer.\n",
|
||||
"\n",
|
||||
" Net input of output layer.\n",
|
||||
" a3 : array, shape = [n_output_units, n_samples]\n",
|
||||
" Activation of output layer.\n",
|
||||
" Activation of output layer.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" a1 = self._add_bias_unit(X, how='column')\n",
|
||||
@@ -717,31 +710,32 @@
|
||||
"\n",
|
||||
" def _L2_reg(self, lambda_, w1, w2):\n",
|
||||
" \"\"\"Compute L2-regularization cost\"\"\"\n",
|
||||
" return (lambda_/2.0) * (np.sum(w1[:, 1:] ** 2) + np.sum(w2[:, 1:] ** 2))\n",
|
||||
" return (lambda_/2.0) * (np.sum(w1[:, 1:] ** 2) +\n",
|
||||
" np.sum(w2[:, 1:] ** 2))\n",
|
||||
"\n",
|
||||
" def _L1_reg(self, lambda_, w1, w2):\n",
|
||||
" \"\"\"Compute L1-regularization cost\"\"\"\n",
|
||||
" return (lambda_/2.0) * (np.abs(w1[:, 1:]).sum() + np.abs(w2[:, 1:]).sum())\n",
|
||||
" return (lambda_/2.0) * (np.abs(w1[:, 1:]).sum() +\n",
|
||||
" np.abs(w2[:, 1:]).sum())\n",
|
||||
"\n",
|
||||
" def _get_cost(self, y_enc, output, w1, w2):\n",
|
||||
" \"\"\"Compute cost function.\n",
|
||||
"\n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" y_enc : array, shape = (n_labels, n_samples)\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
"\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
" output : array, shape = [n_output_units, n_samples]\n",
|
||||
" Activation of the output layer (feedforward)\n",
|
||||
"\n",
|
||||
" Activation of the output layer (feedforward)\n",
|
||||
" w1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
"\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
" w2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" ---------\n",
|
||||
" cost : float\n",
|
||||
" Regularized cost.\n",
|
||||
" Regularized cost.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" term1 = -y_enc * (np.log(output))\n",
|
||||
@@ -758,32 +752,24 @@
|
||||
" Parameters\n",
|
||||
" ------------\n",
|
||||
" a1 : array, shape = [n_samples, n_features+1]\n",
|
||||
" Input values with bias unit.\n",
|
||||
"\n",
|
||||
" Input values with bias unit.\n",
|
||||
" a2 : array, shape = [n_hidden+1, n_samples]\n",
|
||||
" Activation of hidden layer.\n",
|
||||
"\n",
|
||||
" Activation of hidden layer.\n",
|
||||
" a3 : array, shape = [n_output_units, n_samples]\n",
|
||||
" Activation of output layer.\n",
|
||||
"\n",
|
||||
" Activation of output layer.\n",
|
||||
" z2 : array, shape = [n_hidden, n_samples]\n",
|
||||
" Net input of hidden layer.\n",
|
||||
"\n",
|
||||
" Net input of hidden layer.\n",
|
||||
" y_enc : array, shape = (n_labels, n_samples)\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
"\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
" w1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
"\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
" w2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" ---------\n",
|
||||
"\n",
|
||||
" grad1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Gradient of the weight matrix w1.\n",
|
||||
"\n",
|
||||
" Gradient of the weight matrix w1.\n",
|
||||
" grad2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Gradient of the weight matrix w2.\n",
|
||||
"\n",
|
||||
@@ -808,12 +794,12 @@
|
||||
" Parameters\n",
|
||||
" -----------\n",
|
||||
" X : array, shape = [n_samples, n_features]\n",
|
||||
" Input layer with original features.\n",
|
||||
" Input layer with original features.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" ----------\n",
|
||||
" y_pred : array, shape = [n_samples]\n",
|
||||
" Predicted class labels.\n",
|
||||
" Predicted class labels.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" if len(X.shape) != 2:\n",
|
||||
@@ -831,14 +817,12 @@
|
||||
" Parameters\n",
|
||||
" -----------\n",
|
||||
" X : array, shape = [n_samples, n_features]\n",
|
||||
" Input layer with original features.\n",
|
||||
"\n",
|
||||
" Input layer with original features.\n",
|
||||
" y : array, shape = [n_samples]\n",
|
||||
" Target class labels.\n",
|
||||
"\n",
|
||||
" Target class labels.\n",
|
||||
" print_progress : bool (default: False)\n",
|
||||
" Prints progress as the number of epochs\n",
|
||||
" to stderr.\n",
|
||||
" Prints progress as the number of epochs\n",
|
||||
" to stderr.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" ----------\n",
|
||||
@@ -853,7 +837,7 @@
|
||||
" delta_w2_prev = np.zeros(self.w2.shape)\n",
|
||||
"\n",
|
||||
" for i in range(self.epochs):\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # adaptive learning rate\n",
|
||||
" self.eta /= (1 + self.decrease_const*i)\n",
|
||||
"\n",
|
||||
@@ -869,7 +853,9 @@
|
||||
" for idx in mini:\n",
|
||||
"\n",
|
||||
" # feedforward\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X_data[idx], self.w1, self.w2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X_data[idx],\n",
|
||||
" self.w1,\n",
|
||||
" self.w2)\n",
|
||||
" cost = self._get_cost(y_enc=y_enc[:, idx],\n",
|
||||
" output=a3,\n",
|
||||
" w1=self.w1,\n",
|
||||
@@ -961,8 +947,8 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"plt.plot(range(len(nn.cost_)), nn.cost_)\n",
|
||||
"plt.ylim([0, 2000])\n",
|
||||
"plt.ylabel('Cost')\n",
|
||||
@@ -1029,12 +1015,11 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"y_train_pred = nn.predict(X_train)\n",
|
||||
"\n",
|
||||
"if sys.version_info < (3, 0):\n",
|
||||
" acc = (np.sum(y_train == y_train_pred, axis=0)).astype('float') / X_train.shape[0]\n",
|
||||
" acc = ((np.sum(y_train == y_train_pred, axis=0)).astype('float') /\n",
|
||||
" X_train.shape[0])\n",
|
||||
"else:\n",
|
||||
" acc = np.sum(y_train == y_train_pred, axis=0) / X_train.shape[0]\n",
|
||||
"\n",
|
||||
@@ -1060,7 +1045,8 @@
|
||||
"y_test_pred = nn.predict(X_test)\n",
|
||||
"\n",
|
||||
"if sys.version_info < (3, 0):\n",
|
||||
" acc = (np.sum(y_test == y_test_pred, axis=0)).astype('float') / X_test.shape[0]\n",
|
||||
" acc = ((np.sum(y_test == y_test_pred, axis=0)).astype('float') /\n",
|
||||
" X_test.shape[0])\n",
|
||||
"else:\n",
|
||||
" acc = np.sum(y_test == y_test_pred, axis=0) / X_test.shape[0]\n",
|
||||
"\n",
|
||||
@@ -1088,7 +1074,7 @@
|
||||
"source": [
|
||||
"miscl_img = X_test[y_test != y_test_pred][:25]\n",
|
||||
"correct_lab = y_test[y_test != y_test_pred][:25]\n",
|
||||
"miscl_lab= y_test_pred[y_test != y_test_pred][:25]\n",
|
||||
"miscl_lab = y_test_pred[y_test != y_test_pred][:25]\n",
|
||||
"\n",
|
||||
"fig, ax = plt.subplots(nrows=5, ncols=5, sharex=True, sharey=True,)\n",
|
||||
"ax = ax.flatten()\n",
|
||||
@@ -1345,59 +1331,48 @@
|
||||
" Parameters\n",
|
||||
" ------------\n",
|
||||
" n_output : int\n",
|
||||
" Number of output units, should be equal to the\n",
|
||||
" number of unique class labels.\n",
|
||||
"\n",
|
||||
" Number of output units, should be equal to the\n",
|
||||
" number of unique class labels.\n",
|
||||
" n_features : int\n",
|
||||
" Number of features (dimensions) in the target dataset.\n",
|
||||
" Should be equal to the number of columns in the X array.\n",
|
||||
"\n",
|
||||
" Number of features (dimensions) in the target dataset.\n",
|
||||
" Should be equal to the number of columns in the X array.\n",
|
||||
" n_hidden : int (default: 30)\n",
|
||||
" Number of hidden units.\n",
|
||||
"\n",
|
||||
" Number of hidden units.\n",
|
||||
" l1 : float (default: 0.0)\n",
|
||||
" Lambda value for L1-regularization.\n",
|
||||
" No regularization if l1=0.0 (default)\n",
|
||||
"\n",
|
||||
" Lambda value for L1-regularization.\n",
|
||||
" No regularization if l1=0.0 (default)\n",
|
||||
" l2 : float (default: 0.0)\n",
|
||||
" Lambda value for L2-regularization.\n",
|
||||
" No regularization if l2=0.0 (default)\n",
|
||||
"\n",
|
||||
" Lambda value for L2-regularization.\n",
|
||||
" No regularization if l2=0.0 (default)\n",
|
||||
" epochs : int (default: 500)\n",
|
||||
" Number of passes over the training set.\n",
|
||||
"\n",
|
||||
" Number of passes over the training set.\n",
|
||||
" eta : float (default: 0.001)\n",
|
||||
" Learning rate.\n",
|
||||
"\n",
|
||||
" Learning rate.\n",
|
||||
" alpha : float (default: 0.0)\n",
|
||||
" Momentum constant. Factor multiplied with the\n",
|
||||
" gradient of the previous epoch t-1 to improve\n",
|
||||
" learning speed\n",
|
||||
" w(t) := w(t) - (grad(t) + alpha*grad(t-1))\n",
|
||||
" \n",
|
||||
" Momentum constant. Factor multiplied with the\n",
|
||||
" gradient of the previous epoch t-1 to improve\n",
|
||||
" learning speed\n",
|
||||
" w(t) := w(t) - (grad(t) + alpha*grad(t-1))\n",
|
||||
" decrease_const : float (default: 0.0)\n",
|
||||
" Decrease constant. Shrinks the learning rate\n",
|
||||
" after each epoch via eta / (1 + epoch*decrease_const)\n",
|
||||
"\n",
|
||||
" Decrease constant. Shrinks the learning rate\n",
|
||||
" after each epoch via eta / (1 + epoch*decrease_const)\n",
|
||||
" shuffle : bool (default: False)\n",
|
||||
" Shuffles training data every epoch if True to prevent circles.\n",
|
||||
"\n",
|
||||
" Shuffles training data every epoch if True to prevent circles.\n",
|
||||
" minibatches : int (default: 1)\n",
|
||||
" Divides training data into k minibatches for efficiency.\n",
|
||||
" Normal gradient descent learning if k=1 (default).\n",
|
||||
"\n",
|
||||
" Divides training data into k minibatches for efficiency.\n",
|
||||
" Normal gradient descent learning if k=1 (default).\n",
|
||||
" random_state : int (default: None)\n",
|
||||
" Set random state for shuffling and initializing the weights.\n",
|
||||
" Set random state for shuffling and initializing the weights.\n",
|
||||
"\n",
|
||||
" Attributes\n",
|
||||
" -----------\n",
|
||||
" cost_ : list\n",
|
||||
" Sum of squared errors after each epoch.\n",
|
||||
" Sum of squared errors after each epoch.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" def __init__(self, n_output, n_features, n_hidden=30,\n",
|
||||
" l1=0.0, l2=0.0, epochs=500, eta=0.001, \n",
|
||||
" alpha=0.0, decrease_const=0.0, shuffle=True, \n",
|
||||
" l1=0.0, l2=0.0, epochs=500, eta=0.001,\n",
|
||||
" alpha=0.0, decrease_const=0.0, shuffle=True,\n",
|
||||
" minibatches=1, random_state=None):\n",
|
||||
"\n",
|
||||
" np.random.seed(random_state)\n",
|
||||
@@ -1434,9 +1409,11 @@
|
||||
"\n",
|
||||
" def _initialize_weights(self):\n",
|
||||
" \"\"\"Initialize weights with small random numbers.\"\"\"\n",
|
||||
" w1 = np.random.uniform(-1.0, 1.0, size=self.n_hidden*(self.n_features + 1))\n",
|
||||
" w1 = np.random.uniform(-1.0, 1.0,\n",
|
||||
" size=self.n_hidden*(self.n_features + 1))\n",
|
||||
" w1 = w1.reshape(self.n_hidden, self.n_features + 1)\n",
|
||||
" w2 = np.random.uniform(-1.0, 1.0, size=self.n_output*(self.n_hidden + 1))\n",
|
||||
" w2 = np.random.uniform(-1.0, 1.0,\n",
|
||||
" size=self.n_output*(self.n_hidden + 1))\n",
|
||||
" w2 = w2.reshape(self.n_output, self.n_hidden + 1)\n",
|
||||
" return w1, w2\n",
|
||||
"\n",
|
||||
@@ -1473,30 +1450,24 @@
|
||||
" Parameters\n",
|
||||
" -----------\n",
|
||||
" X : array, shape = [n_samples, n_features]\n",
|
||||
" Input layer with original features.\n",
|
||||
"\n",
|
||||
" Input layer with original features.\n",
|
||||
" w1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
"\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
" w2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" ----------\n",
|
||||
" a1 : array, shape = [n_samples, n_features+1]\n",
|
||||
" Input values with bias unit.\n",
|
||||
"\n",
|
||||
" Input values with bias unit.\n",
|
||||
" z2 : array, shape = [n_hidden, n_samples]\n",
|
||||
" Net input of hidden layer.\n",
|
||||
"\n",
|
||||
" Net input of hidden layer.\n",
|
||||
" a2 : array, shape = [n_hidden+1, n_samples]\n",
|
||||
" Activation of hidden layer.\n",
|
||||
"\n",
|
||||
" Activation of hidden layer.\n",
|
||||
" z3 : array, shape = [n_output_units, n_samples]\n",
|
||||
" Net input of output layer.\n",
|
||||
"\n",
|
||||
" Net input of output layer.\n",
|
||||
" a3 : array, shape = [n_output_units, n_samples]\n",
|
||||
" Activation of output layer.\n",
|
||||
" Activation of output layer.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" a1 = self._add_bias_unit(X, how='column')\n",
|
||||
@@ -1509,31 +1480,32 @@
|
||||
"\n",
|
||||
" def _L2_reg(self, lambda_, w1, w2):\n",
|
||||
" \"\"\"Compute L2-regularization cost\"\"\"\n",
|
||||
" return (lambda_/2.0) * (np.sum(w1[:, 1:] ** 2) + np.sum(w2[:, 1:] ** 2))\n",
|
||||
" return (lambda_/2.0) * (np.sum(w1[:, 1:] ** 2) +\n",
|
||||
" np.sum(w2[:, 1:] ** 2))\n",
|
||||
"\n",
|
||||
" def _L1_reg(self, lambda_, w1, w2):\n",
|
||||
" \"\"\"Compute L1-regularization cost\"\"\"\n",
|
||||
" return (lambda_/2.0) * (np.abs(w1[:, 1:]).sum() + np.abs(w2[:, 1:]).sum())\n",
|
||||
" return (lambda_/2.0) * (np.abs(w1[:, 1:]).sum() +\n",
|
||||
" np.abs(w2[:, 1:]).sum())\n",
|
||||
"\n",
|
||||
" def _get_cost(self, y_enc, output, w1, w2):\n",
|
||||
" \"\"\"Compute cost function.\n",
|
||||
"\n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" y_enc : array, shape = (n_labels, n_samples)\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
"\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
" output : array, shape = [n_output_units, n_samples]\n",
|
||||
" Activation of the output layer (feedforward)\n",
|
||||
"\n",
|
||||
" Activation of the output layer (feedforward)\n",
|
||||
" w1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
"\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
" w2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" ---------\n",
|
||||
" cost : float\n",
|
||||
" Regularized cost.\n",
|
||||
" Regularized cost.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" term1 = -y_enc * (np.log(output))\n",
|
||||
@@ -1550,32 +1522,24 @@
|
||||
" Parameters\n",
|
||||
" ------------\n",
|
||||
" a1 : array, shape = [n_samples, n_features+1]\n",
|
||||
" Input values with bias unit.\n",
|
||||
"\n",
|
||||
" Input values with bias unit.\n",
|
||||
" a2 : array, shape = [n_hidden+1, n_samples]\n",
|
||||
" Activation of hidden layer.\n",
|
||||
"\n",
|
||||
" Activation of hidden layer.\n",
|
||||
" a3 : array, shape = [n_output_units, n_samples]\n",
|
||||
" Activation of output layer.\n",
|
||||
"\n",
|
||||
" Activation of output layer.\n",
|
||||
" z2 : array, shape = [n_hidden, n_samples]\n",
|
||||
" Net input of hidden layer.\n",
|
||||
"\n",
|
||||
" Net input of hidden layer.\n",
|
||||
" y_enc : array, shape = (n_labels, n_samples)\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
"\n",
|
||||
" one-hot encoded class labels.\n",
|
||||
" w1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
"\n",
|
||||
" Weight matrix for input layer -> hidden layer.\n",
|
||||
" w2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
" Weight matrix for hidden layer -> output layer.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" ---------\n",
|
||||
"\n",
|
||||
" grad1 : array, shape = [n_hidden_units, n_features]\n",
|
||||
" Gradient of the weight matrix w1.\n",
|
||||
"\n",
|
||||
" Gradient of the weight matrix w1.\n",
|
||||
" grad2 : array, shape = [n_output_units, n_hidden_units]\n",
|
||||
" Gradient of the weight matrix w2.\n",
|
||||
"\n",
|
||||
@@ -1609,9 +1573,11 @@
|
||||
" for i in range(w1.shape[0]):\n",
|
||||
" for j in range(w1.shape[1]):\n",
|
||||
" epsilon_ary1[i, j] = epsilon\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X, w1 - epsilon_ary1, w2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X,\n",
|
||||
" w1 - epsilon_ary1, w2)\n",
|
||||
" cost1 = self._get_cost(y_enc, a3, w1-epsilon_ary1, w2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X, w1 + epsilon_ary1, w2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X,\n",
|
||||
" w1 + epsilon_ary1, w2)\n",
|
||||
" cost2 = self._get_cost(y_enc, a3, w1 + epsilon_ary1, w2)\n",
|
||||
" num_grad1[i, j] = (cost2 - cost1) / (2 * epsilon)\n",
|
||||
" epsilon_ary1[i, j] = 0\n",
|
||||
@@ -1621,9 +1587,11 @@
|
||||
" for i in range(w2.shape[0]):\n",
|
||||
" for j in range(w2.shape[1]):\n",
|
||||
" epsilon_ary2[i, j] = epsilon\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X, w1, w2 - epsilon_ary2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X, w1,\n",
|
||||
" w2 - epsilon_ary2)\n",
|
||||
" cost1 = self._get_cost(y_enc, a3, w1, w2 - epsilon_ary2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X, w1, w2 + epsilon_ary2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X, w1,\n",
|
||||
" w2 + epsilon_ary2)\n",
|
||||
" cost2 = self._get_cost(y_enc, a3, w1, w2 + epsilon_ary2)\n",
|
||||
" num_grad2[i, j] = (cost2 - cost1) / (2 * epsilon)\n",
|
||||
" epsilon_ary2[i, j] = 0\n",
|
||||
@@ -1642,12 +1610,12 @@
|
||||
" Parameters\n",
|
||||
" -----------\n",
|
||||
" X : array, shape = [n_samples, n_features]\n",
|
||||
" Input layer with original features.\n",
|
||||
" Input layer with original features.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" ----------\n",
|
||||
" y_pred : array, shape = [n_samples]\n",
|
||||
" Predicted class labels.\n",
|
||||
" Predicted class labels.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
" if len(X.shape) != 2:\n",
|
||||
@@ -1665,14 +1633,12 @@
|
||||
" Parameters\n",
|
||||
" -----------\n",
|
||||
" X : array, shape = [n_samples, n_features]\n",
|
||||
" Input layer with original features.\n",
|
||||
"\n",
|
||||
" Input layer with original features.\n",
|
||||
" y : array, shape = [n_samples]\n",
|
||||
" Target class labels.\n",
|
||||
"\n",
|
||||
" Target class labels.\n",
|
||||
" print_progress : bool (default: False)\n",
|
||||
" Prints progress as the number of epochs\n",
|
||||
" to stderr.\n",
|
||||
" Prints progress as the number of epochs\n",
|
||||
" to stderr.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" ----------\n",
|
||||
@@ -1687,7 +1653,7 @@
|
||||
" delta_w2_prev = np.zeros(self.w2.shape)\n",
|
||||
"\n",
|
||||
" for i in range(self.epochs):\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # adaptive learning rate\n",
|
||||
" self.eta /= (1 + self.decrease_const*i)\n",
|
||||
"\n",
|
||||
@@ -1703,7 +1669,9 @@
|
||||
" for idx in mini:\n",
|
||||
"\n",
|
||||
" # feedforward\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X[idx], self.w1, self.w2)\n",
|
||||
" a1, z2, a2, z3, a3 = self._feedforward(X[idx],\n",
|
||||
" self.w1,\n",
|
||||
" self.w2)\n",
|
||||
" cost = self._get_cost(y_enc=y_enc[:, idx],\n",
|
||||
" output=a3,\n",
|
||||
" w1=self.w1,\n",
|
||||
@@ -1716,13 +1684,16 @@
|
||||
" y_enc=y_enc[:, idx],\n",
|
||||
" w1=self.w1,\n",
|
||||
" w2=self.w2)\n",
|
||||
" \n",
|
||||
" ## start gradient checking\n",
|
||||
" grad_diff = self._gradient_checking(X=X_data[idx], y_enc=y_enc[:, idx],\n",
|
||||
" w1=self.w1, w2=self.w2,\n",
|
||||
" epsilon=1e-5,\n",
|
||||
" grad1=grad1, grad2=grad2)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # start gradient checking\n",
|
||||
" grad_diff = self._gradient_checking(X=X_data[idx],\n",
|
||||
" y_enc=y_enc[:, idx],\n",
|
||||
" w1=self.w1,\n",
|
||||
" w2=self.w2,\n",
|
||||
" epsilon=1e-5,\n",
|
||||
" grad1=grad1,\n",
|
||||
" grad2=grad2)\n",
|
||||
"\n",
|
||||
" if grad_diff <= 1e-7:\n",
|
||||
" print('Ok: %s' % grad_diff)\n",
|
||||
" elif grad_diff <= 1e-4:\n",
|
||||
|
||||
Reference in New Issue
Block a user