From 8a584d9b1c85af7933eea0bfd69c309d37fefa6d Mon Sep 17 00:00:00 2001 From: Reza Mahmoudi <108347344+R-Mahmoudi@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:19:53 +0330 Subject: [PATCH 1/2] Add Course README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2ae9571..3f66b40 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,9 @@ In this repository, I will share some useful notes and references about deployin - [MLOPs-Zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp) [Great] - [A collection of resources to learn about MLOPs](https://github.com/dair-ai/MLOPs-Primer) [Great] - [MLEM: package and deploy machine learning models](https://github.com/iterative/mlem) +- [Full Stack Deep Learning](https://www.youtube.com/watch?v=-Iob-FW5jVM&list=PL1T8fO7ArWleMMI8KPJ_5D5XSlovTW_Ur) +- [Made With ML](https://madewithml.com/#mlops) +- [Machine Learning Engineering for Production (MLOps)](https://www.youtube.com/playlist?list=PLkDaE6sCZn6GMoA0wbpJLi3t34Gd8l0aK) ## Other: - [A Guide to Production Level Deep Learning](https://github.com/alirezadir/Production-Level-Deep-Learning) From fc308b84f5bb61b311752d66eede60f22c060e09 Mon Sep 17 00:00:00 2001 From: Reza Mahmoudi <108347344+R-Mahmoudi@users.noreply.github.com> Date: Thu, 2 Mar 2023 15:19:33 +0330 Subject: [PATCH 2/2] Created using Colaboratory --- ...Forward_Forward_Algorithm_by_Pytorch.ipynb | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 A_Implemention_Hintons_Forward_Forward_Algorithm_by_Pytorch.ipynb diff --git a/A_Implemention_Hintons_Forward_Forward_Algorithm_by_Pytorch.ipynb b/A_Implemention_Hintons_Forward_Forward_Algorithm_by_Pytorch.ipynb new file mode 100644 index 0000000..6e3da19 --- /dev/null +++ b/A_Implemention_Hintons_Forward_Forward_Algorithm_by_Pytorch.ipynb @@ -0,0 +1,246 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "authorship_tag": "ABX9TyNY9rI/YuyhTw18U6jZV0fc" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU", + "gpuClass": "standard" + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "##A Implemention Hintons Forward-Forward Algorithm\n", + "\n", + "\n", + "\n" + ], + "metadata": { + "id": "H1GWQHX4CPfq" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "_butDyfHBSHi" + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from tqdm import tqdm\n", + "from torch.optim import Adam\n", + "from torchvision.datasets import MNIST\n", + "from torch.utils.data import DataLoader\n", + "from torchvision.transforms import Compose,Normalize,ToTensor,Lambda" + ] + }, + { + "cell_type": "code", + "source": [ + "def loaders(train_batch_size=50000, test_batch_size=10000):\n", + " transform = Compose([ToTensor(),Normalize((0.1307,), (0.3081,)),Lambda(lambda x: torch.flatten(x))])\n", + "\n", + " train_loader = DataLoader(MNIST('./data/', train=True,\n", + " download=True,\n", + " transform=transform),\n", + " batch_size=train_batch_size, shuffle=True)\n", + " \n", + " test_loader = DataLoader(\n", + " MNIST('./data/', train=False,\n", + " download=True,\n", + " transform=transform),\n", + " batch_size=test_batch_size, shuffle=False)\n", + "\n", + " return train_loader, test_loader" + ], + "metadata": { + "id": "3ou_2dSDDfok" + }, + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "def overlay_y_on_x(x, y):\n", + " x_ = x.clone()\n", + " x_[:, :10] *= 0.0\n", + " x_[range(x.shape[0]), y] = x.max()\n", + " return x_" + ], + "metadata": { + "id": "s-bhEMmoFkJy" + }, + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "\n", + "class Net(torch.nn.Module):\n", + "\n", + " def __init__(self, dims):\n", + " super().__init__()\n", + " self.layers = []\n", + " for d in range(len(dims) - 1):\n", + " self.layers += [Layer(dims[d], dims[d + 1]).cuda()]\n", + "\n", + " def predict(self, x):\n", + " goodness_per_label = []\n", + " for label in range(10):\n", + " h = overlay_y_on_x(x, label)\n", + " goodness = []\n", + " for layer in self.layers:\n", + " h = layer(h)\n", + " goodness += [h.pow(2).mean(1)]\n", + " goodness_per_label += [sum(goodness).unsqueeze(1)]\n", + " goodness_per_label = torch.cat(goodness_per_label, 1)\n", + " return goodness_per_label.argmax(1)\n", + "\n", + " def train(self, x_pos, x_neg):\n", + " h_pos, h_neg = x_pos, x_neg\n", + " for i, layer in enumerate(self.layers):\n", + " print('Training layer..', i, '...')\n", + " h_pos, h_neg = layer.train(h_pos, h_neg)\n", + "\n" + ], + "metadata": { + "id": "BkyOAx26GACi" + }, + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "class Layer(nn.Linear):\n", + " def __init__(self, in_features, out_features,\n", + " bias=True, device=None, dtype=None):\n", + " super().__init__(in_features, out_features, bias, device, dtype)\n", + " self.relu = torch.nn.ReLU()\n", + " self.opt = Adam(self.parameters(), lr=0.03)\n", + " self.threshold = 2.0\n", + " self.num_epochs = 1000\n", + "\n", + " def forward(self, x):\n", + " x_direction = x / (x.norm(2, 1, keepdim=True) + 1e-4)\n", + " return self.relu(\n", + " torch.mm(x_direction, self.weight.T) +\n", + " self.bias.unsqueeze(0))\n", + "\n", + " def train(self, x_pos, x_neg):\n", + " for i in tqdm(range(self.num_epochs)):\n", + " g_pos = self.forward(x_pos).pow(2).mean(1)\n", + " g_neg = self.forward(x_neg).pow(2).mean(1)\n", + " \n", + " loss = torch.log(1 + torch.exp(torch.cat([\n", + " -g_pos + self.threshold,\n", + " g_neg - self.threshold]))).mean()\n", + " self.opt.zero_grad()\n", + " \n", + " loss.backward()\n", + " self.opt.step()\n", + " return self.forward(x_pos).detach(), self.forward(x_neg).detach()\n" + ], + "metadata": { + "id": "D2bE_NpXGq4Z" + }, + "execution_count": 5, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "if __name__ == \"__main__\":\n", + " torch.manual_seed(1234)\n", + " train_loader, test_loader = loaders()\n", + "\n", + " net = Net([784, 500, 500])\n", + " x, y = next(iter(train_loader))\n", + " x, y = x.cuda(), y.cuda()\n", + " x_pos = overlay_y_on_x(x, y)\n", + " rnd = torch.randperm(x.size(0))\n", + " x_neg = overlay_y_on_x(x, y[rnd])\n", + " \n", + " \n", + " \n", + " net.train(x_pos, x_neg)\n", + "\n", + " print('train error:', 1.0 - net.predict(x).eq(y).float().mean().item())\n", + "\n", + " x_te, y_te = next(iter(test_loader))\n", + " x_te, y_te = x_te.cuda(), y_te.cuda()\n", + "\n", + " print('test error:', 1.0 - net.predict(x_te).eq(y_te).float().mean().item())" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KDnN8rRMHPq4", + "outputId": "509d95f0-0cbe-499e-a001-eda39cf8a399" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Training layer.. 0 ...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "100%|██████████| 1000/1000 [00:57<00:00, 17.47it/s]\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Training layer.. 1 ...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "100%|██████████| 1000/1000 [00:38<00:00, 26.07it/s]\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "train error: 0.06754004955291748\n", + "test error: 0.06840002536773682\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "S_3Zya99IjM0" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file