{ "cells": [ { "attachments": {}, "cell_type": "markdown", "id": "b773bf8e-c420-44e1-80a6-99f75dd12268", "metadata": {}, "source": [ "# 3. Using Pytorch with CapyMOA\n", "* This notebook demonstrates how use Pytorch with CapyMOA.\n", "* It contains examples showing \n", " * How to define a Pytorch Network to be used with CapyMOA\n", " * How a simple Pytorch model can be used in a CapyMOA ```Instance``` loop\n", " * How to define a Pytorch CapyMOA Classifier based on CapyMOA ```Classifier``` framework and how to use it with ```prequential_evaluation()```\n", " * How to use a Pytorch dataset with a CapyMOA classifier\n", "* **Tutorial 6**: `Exploring Advanced Features` includes an example using TensorBoard and a `PyTorchClassifier`\n", " \n", "---\n", "\n", "*More information about CapyMOA can be found in* https://www.capymoa.org\n", "\n", "**last update on 25/07/2024**" ] }, { "attachments": {}, "cell_type": "markdown", "id": "cd9a931c-7b86-4bd6-8cda-179154e4b513", "metadata": {}, "source": [ "## 1. Setup\n", "* Sets random seed for reproducibility\n", "* Sets Pytorch network " ] }, { "attachments": {}, "cell_type": "markdown", "id": "9b6d4a63467b23b5", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "source": [ "### 1.1 Set random seeds" ] }, { "cell_type": "code", "execution_count": 1, "id": "2242896e95964ef4", "metadata": { "ExecuteTime": { "end_time": "2024-04-29T11:56:23.668712Z", "start_time": "2024-04-29T11:56:23.659844Z" }, "collapsed": false, "execution": { "iopub.execute_input": "2024-09-23T00:28:34.435464Z", "iopub.status.busy": "2024-09-23T00:28:34.434857Z", "iopub.status.idle": "2024-09-23T00:28:34.440944Z", "shell.execute_reply": "2024-09-23T00:28:34.440612Z" }, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "import random\n", "\n", "random_seed = 1\n", "random.seed(random_seed)" ] }, { "attachments": {}, "cell_type": "markdown", "id": "ccbccfce41f2f18", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "source": [ "### 1.2 Define network structure\n", "* Here, network uses the CPU device" ] }, { "cell_type": "code", "execution_count": 2, "id": "fbcd4d91-2b00-4d5a-9c54-f9dd6a291aa5", "metadata": { "ExecuteTime": { "end_time": "2024-04-29T11:24:39.060139Z", "start_time": "2024-04-29T11:24:39.035090Z" }, "execution": { "iopub.execute_input": "2024-09-23T00:28:34.442657Z", "iopub.status.busy": "2024-09-23T00:28:34.442418Z", "iopub.status.idle": "2024-09-23T00:28:35.180762Z", "shell.execute_reply": "2024-09-23T00:28:35.180261Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using cpu device\n" ] } ], "source": [ "import torch\n", "from torch import nn\n", "\n", "torch.manual_seed(random_seed)\n", "torch.use_deterministic_algorithms(True)\n", "\n", "# Get cpu device for training.\n", "device = \"cpu\"\n", "print(f\"Using {device} device\")\n", "\n", "\n", "# Define model\n", "class NeuralNetwork(nn.Module):\n", " def __init__(self, input_size=0, number_of_classes=0):\n", " super().__init__()\n", " self.flatten = nn.Flatten()\n", " self.linear_relu_stack = nn.Sequential(\n", " nn.Linear(input_size, 512),\n", " nn.ReLU(),\n", " nn.Linear(512, 512),\n", " nn.ReLU(),\n", " nn.Linear(512, number_of_classes),\n", " )\n", "\n", " def forward(self, x):\n", " x = self.flatten(x)\n", " logits = self.linear_relu_stack(x)\n", " return logits" ] }, { "attachments": {}, "cell_type": "markdown", "id": "882b6b292a2de100", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "source": [ "### 1.3 Using instance loop\n", "* Model is initialized after receiving the first instance" ] }, { "cell_type": "code", "execution_count": 3, "id": "81092940-5a6b-4377-b550-ed6f5fee711a", "metadata": { "ExecuteTime": { "end_time": "2024-04-29T11:24:41.344634Z", "start_time": "2024-04-29T11:24:39.063330Z" }, "execution": { "iopub.execute_input": "2024-09-23T00:28:35.182576Z", "iopub.status.busy": "2024-09-23T00:28:35.182380Z", "iopub.status.idle": "2024-09-23T00:28:38.088429Z", "shell.execute_reply": "2024-09-23T00:28:38.088007Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "NeuralNetwork(\n", " (flatten): Flatten(start_dim=1, end_dim=-1)\n", " (linear_relu_stack): Sequential(\n", " (0): Linear(in_features=6, out_features=512, bias=True)\n", " (1): ReLU()\n", " (2): Linear(in_features=512, out_features=512, bias=True)\n", " (3): ReLU()\n", " (4): Linear(in_features=512, out_features=2, bias=True)\n", " )\n", ")\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Accuracy at 500 : 50.4\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Accuracy at 1000 : 55.2\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Accuracy at 1500 : 61.199999999999996\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Accuracy at 2000 : 61.1\n", "Accuracy at 2000 : 61.1\n" ] } ], "source": [ "from capymoa.evaluation import ClassificationEvaluator\n", "from capymoa.datasets import ElectricityTiny\n", "\n", "elec_stream = ElectricityTiny()\n", "\n", "# Creating the evaluator\n", "evaluator = ClassificationEvaluator(schema=elec_stream.get_schema())\n", "\n", "model = None\n", "optimizer = None\n", "loss_fn = nn.CrossEntropyLoss()\n", "\n", "i = 0\n", "while elec_stream.has_more_instances():\n", " i += 1\n", " instance = elec_stream.next_instance()\n", " if model is None:\n", " moa_instance = instance.java_instance.getData()\n", " # initialize the model and send it to the device\n", " model = NeuralNetwork(\n", " input_size=elec_stream.get_schema().get_num_attributes(),\n", " number_of_classes=elec_stream.get_schema().get_num_classes(),\n", " ).to(device)\n", " # set the optimizer\n", " optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n", " print(model)\n", "\n", " X = torch.tensor(instance.x, dtype=torch.float32)\n", " y = torch.tensor(instance.y_index, dtype=torch.long)\n", " # set the device and add a dimension to the tensor\n", " X, y = torch.unsqueeze(X.to(device), 0), torch.unsqueeze(y.to(device), 0)\n", "\n", " # turn off gradient collection for test\n", " with torch.no_grad():\n", " pred = model(X)\n", " prediction = torch.argmax(pred)\n", "\n", " # update evaluator with predicted class\n", " evaluator.update(instance.y_index, prediction.item())\n", "\n", " # Compute prediction error\n", " pred = model(X)\n", " loss = loss_fn(pred, y)\n", "\n", " # Backpropagation\n", " loss.backward()\n", " optimizer.step()\n", " optimizer.zero_grad()\n", "\n", " if i % 500 == 0:\n", " print(f\"Accuracy at {i} : {evaluator.accuracy()}\")\n", "\n", "print(f\"Accuracy at {i} : {evaluator.accuracy()}\")" ] }, { "attachments": {}, "cell_type": "markdown", "id": "2803eaab-eb2b-4c6e-a246-67f9ac71f70f", "metadata": {}, "source": [ "## 2. PyTorchClassifier\n", "* Defining a `PyTorchClassifier` using CapyMOA API makes it **compatibility** with CapyMOA functions like `prequential_evaluation()` without losing the **flexibility** of specifying the `architecture` and the `training` method\n", "* Model is initialized after receiving the first instance\n", "* `PyTorchClassifier` is based on `capymoa.base` ```Classifier``` abstract class\n", "\n", "* **Important**: We can access information about the stream through any of its instances. See `set_model(self, instance)` for an example: \n", "\n", "```python\n", "...\n", "moa_instance = instance.java_instance.getData()\n", "self.model = NeuralNetwork(input_size=moa_instance.get_num_attributes(), \n", " number_of_classes=moa_instance.get_num_classes()).to(self.device)\n", "...\n", "```" ] }, { "cell_type": "code", "execution_count": 4, "id": "1c75513c-58e8-4499-bb19-2c58aea4567b", "metadata": { "ExecuteTime": { "end_time": "2024-04-29T11:24:41.367535Z", "start_time": "2024-04-29T11:24:41.347463Z" }, "execution": { "iopub.execute_input": "2024-09-23T00:28:38.090257Z", "iopub.status.busy": "2024-09-23T00:28:38.090039Z", "iopub.status.idle": "2024-09-23T00:28:38.099048Z", "shell.execute_reply": "2024-09-23T00:28:38.098639Z" } }, "outputs": [], "source": [ "from capymoa.base import Classifier\n", "import numpy as np\n", "\n", "\n", "class PyTorchClassifier(Classifier):\n", " def __init__(\n", " self,\n", " schema=None,\n", " random_seed=1,\n", " nn_model: nn.Module = None,\n", " optimizer=None,\n", " loss_fn=nn.CrossEntropyLoss(),\n", " device=(\"cpu\"),\n", " lr=1e-3,\n", " ):\n", " super().__init__(schema, random_seed)\n", " self.model = None\n", " self.optimizer = None\n", " self.loss_fn = loss_fn\n", " self.lr = lr\n", " self.device = device\n", "\n", " torch.manual_seed(random_seed)\n", "\n", " if nn_model is None:\n", " self.set_model(None)\n", " else:\n", " self.model = nn_model.to(device)\n", " if optimizer is None:\n", " if self.model is not None:\n", " self.optimizer = torch.optim.SGD(self.model.parameters(), lr=lr)\n", " else:\n", " self.optimizer = optimizer\n", "\n", " def __str__(self):\n", " return str(self.model)\n", "\n", " def CLI_help(self):\n", " return str(\n", " 'schema=None, random_seed=1, nn_model: nn.Module = None, optimizer=None, loss_fn=nn.CrossEntropyLoss(), device=(\"cpu\"), lr=1e-3'\n", " )\n", "\n", " def set_model(self, instance):\n", " if self.schema is None:\n", " moa_instance = instance.java_instance.getData()\n", " self.model = NeuralNetwork(\n", " input_size=moa_instance.get_num_attributes(),\n", " number_of_classes=moa_instance.get_num_classes(),\n", " ).to(self.device)\n", " elif instance is not None:\n", " self.model = NeuralNetwork(\n", " input_size=self.schema.get_num_attributes(),\n", " number_of_classes=self.schema.get_num_classes(),\n", " ).to(self.device)\n", "\n", " def train(self, instance):\n", " if self.model is None:\n", " self.set_model(instance)\n", "\n", " X = torch.tensor(instance.x, dtype=torch.float32)\n", " y = torch.tensor(instance.y_index, dtype=torch.long)\n", " # set the device and add a dimension to the tensor\n", " X, y = (\n", " torch.unsqueeze(X.to(self.device), 0),\n", " torch.unsqueeze(y.to(self.device), 0),\n", " )\n", "\n", " # Compute prediction error\n", " pred = self.model(X)\n", " loss = self.loss_fn(pred, y)\n", "\n", " # Backpropagation\n", " loss.backward()\n", " self.optimizer.step()\n", " self.optimizer.zero_grad()\n", "\n", " def predict(self, instance):\n", " return np.argmax(self.predict_proba(instance))\n", "\n", " def predict_proba(self, instance):\n", " if self.model is None:\n", " self.set_model(instance)\n", " X = torch.unsqueeze(\n", " torch.tensor(instance.x, dtype=torch.float32).to(self.device), 0\n", " )\n", " # turn off gradient collection\n", " with torch.no_grad():\n", " pred = np.asarray(self.model(X).numpy(), dtype=np.double)\n", " return pred" ] }, { "attachments": {}, "cell_type": "markdown", "id": "75ee1eb154c01996", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "source": [ "### 2.2 Using PyTorchClassifier + prequential_evaluation\n", "\n", "* We can access information about the stream through the `schema` directly, from the example below: \n", "```python\n", "...\n", "nn_model=NeuralNetwork(input_size=elec_stream.get_schema().get_num_attributes(),\n", " number_of_classes=elec_stream.get_schema().get_num_classes()).to(device)\n", "...\n", "```" ] }, { "cell_type": "code", "execution_count": 5, "id": "ece5b5ff-be24-439c-bf53-7bf5d4fbf858", "metadata": { "ExecuteTime": { "end_time": "2024-04-29T11:24:43.665700Z", "start_time": "2024-04-29T11:24:41.371991Z" }, "execution": { "iopub.execute_input": "2024-09-23T00:28:38.100830Z", "iopub.status.busy": "2024-09-23T00:28:38.100659Z", "iopub.status.idle": "2024-09-23T00:28:39.633560Z", "shell.execute_reply": "2024-09-23T00:28:39.633201Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy: 62.849999999999994\n" ] } ], "source": [ "from capymoa.evaluation import prequential_evaluation\n", "\n", "## Opening a file as a stream\n", "elec_stream = ElectricityTiny()\n", "\n", "# Creating a learner\n", "simple_pyTorch_classifier = PyTorchClassifier(\n", " schema=elec_stream.get_schema(),\n", " nn_model=NeuralNetwork(\n", " input_size=elec_stream.get_schema().get_num_attributes(),\n", " number_of_classes=elec_stream.get_schema().get_num_classes(),\n", " ).to(device),\n", ")\n", "\n", "evaluator = prequential_evaluation(\n", " stream=elec_stream,\n", " learner=simple_pyTorch_classifier,\n", " window_size=4500,\n", " optimise=False,\n", ")\n", "\n", "print(f\"Accuracy: {evaluator.cumulative.accuracy()}\")" ] }, { "cell_type": "markdown", "id": "a3a752afb7e7698d", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "source": [ "## 3. How to use a Pytorch dataset with a CapyMOA classifier\n", "* One may want to use various Pytorch datasets with different CapyMOA classifiers\n", "* In this example we use Pytorch Dataset + prequential evaluation + CapyMOA Classifier\n", "\n", "**Observation**: *Using a learner like Online Bagging without any feature extraction is not going to yield meaningful performance*" ] }, { "cell_type": "code", "execution_count": 6, "id": "8a6fdd873625b07b", "metadata": { "ExecuteTime": { "end_time": "2024-04-29T11:24:56.571969Z", "start_time": "2024-04-29T11:24:49.646899Z" }, "execution": { "iopub.execute_input": "2024-09-23T00:28:39.635436Z", "iopub.status.busy": "2024-09-23T00:28:39.635283Z", "iopub.status.idle": "2024-09-23T00:28:43.949481Z", "shell.execute_reply": "2024-09-23T00:28:43.948975Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy: 43.5\n" ] }, { "data": { "text/html": [ "
\n", " | instances | \n", "accuracy | \n", "kappa | \n", "kappa_t | \n", "kappa_m | \n", "f1_score | \n", "f1_score_0 | \n", "f1_score_1 | \n", "f1_score_2 | \n", "f1_score_3 | \n", "... | \n", "recall_0 | \n", "recall_1 | \n", "recall_2 | \n", "recall_3 | \n", "recall_4 | \n", "recall_5 | \n", "recall_6 | \n", "recall_7 | \n", "recall_8 | \n", "recall_9 | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "100.0 | \n", "17.0 | \n", "6.267645 | \n", "0.000000 | \n", "0.000000 | \n", "NaN | \n", "23.188406 | \n", "30.769231 | \n", "46.153846 | \n", "12.500000 | \n", "... | \n", "66.666667 | \n", "18.181818 | \n", "33.333333 | \n", "6.666667 | \n", "0.000000 | \n", "0.000000 | \n", "0.000000 | \n", "12.500000 | \n", "0.000000 | \n", "18.181818 | \n", "
1 | \n", "200.0 | \n", "41.0 | \n", "33.288105 | \n", "32.954545 | \n", "30.588235 | \n", "45.879666 | \n", "39.130435 | \n", "57.142857 | \n", "36.363636 | \n", "NaN | \n", "... | \n", "75.000000 | \n", "40.000000 | \n", "22.222222 | \n", "0.000000 | \n", "22.222222 | \n", "22.222222 | \n", "63.636364 | \n", "23.076923 | \n", "75.000000 | \n", "12.500000 | \n", "
2 | \n", "300.0 | \n", "30.0 | \n", "22.127044 | \n", "23.076923 | \n", "23.076923 | \n", "NaN | \n", "24.390244 | \n", "72.727273 | \n", "51.851852 | \n", "NaN | \n", "... | \n", "62.500000 | \n", "57.142857 | \n", "53.846154 | \n", "0.000000 | \n", "9.090909 | \n", "18.181818 | \n", "33.333333 | \n", "22.222222 | \n", "18.181818 | \n", "50.000000 | \n", "
3 | \n", "400.0 | \n", "48.0 | \n", "41.775837 | \n", "43.478261 | \n", "42.222222 | \n", "54.216958 | \n", "36.363636 | \n", "70.588235 | \n", "22.222222 | \n", "15.384615 | \n", "... | \n", "72.727273 | \n", "54.545455 | \n", "25.000000 | \n", "8.333333 | \n", "25.000000 | \n", "66.666667 | \n", "50.000000 | \n", "63.636364 | \n", "66.666667 | \n", "33.333333 | \n", "
4 | \n", "500.0 | \n", "51.0 | \n", "45.591828 | \n", "45.555556 | \n", "42.352941 | \n", "56.932958 | \n", "39.024390 | \n", "50.000000 | \n", "23.529412 | \n", "22.222222 | \n", "... | \n", "88.888889 | \n", "40.000000 | \n", "16.666667 | \n", "12.500000 | \n", "50.000000 | \n", "75.000000 | \n", "25.000000 | \n", "62.500000 | \n", "78.571429 | \n", "54.545455 | \n", "
5 | \n", "600.0 | \n", "48.0 | \n", "42.164387 | \n", "39.534884 | \n", "36.585366 | \n", "NaN | \n", "38.888889 | \n", "50.000000 | \n", "47.058824 | \n", "NaN | \n", "... | \n", "70.000000 | \n", "33.333333 | \n", "40.000000 | \n", "0.000000 | \n", "50.000000 | \n", "71.428571 | \n", "46.153846 | \n", "50.000000 | \n", "62.500000 | \n", "61.538462 | \n", "
6 | \n", "700.0 | \n", "46.0 | \n", "39.366719 | \n", "36.470588 | \n", "29.870130 | \n", "NaN | \n", "42.105263 | \n", "80.000000 | \n", "50.000000 | \n", "NaN | \n", "... | \n", "88.888889 | \n", "66.666667 | \n", "57.142857 | \n", "0.000000 | \n", "61.538462 | \n", "25.000000 | \n", "33.333333 | \n", "22.222222 | \n", "68.750000 | \n", "42.857143 | \n", "
7 | \n", "800.0 | \n", "55.0 | \n", "49.301487 | \n", "49.438202 | \n", "47.058824 | \n", "NaN | \n", "58.823529 | \n", "96.551724 | \n", "60.000000 | \n", "NaN | \n", "... | \n", "90.909091 | \n", "93.333333 | \n", "50.000000 | \n", "0.000000 | \n", "66.666667 | \n", "53.846154 | \n", "27.272727 | \n", "20.000000 | \n", "63.636364 | \n", "37.500000 | \n", "
8 | \n", "900.0 | \n", "50.0 | \n", "43.813912 | \n", "45.652174 | \n", "42.528736 | \n", "48.261276 | \n", "66.666667 | \n", "81.818182 | \n", "15.384615 | \n", "62.500000 | \n", "... | \n", "87.500000 | \n", "69.230769 | \n", "16.666667 | \n", "45.454545 | \n", "16.666667 | \n", "50.000000 | \n", "33.333333 | \n", "30.000000 | \n", "44.444444 | \n", "50.000000 | \n", "
9 | \n", "1000.0 | \n", "49.0 | \n", "43.118447 | \n", "46.875000 | \n", "42.045455 | \n", "54.455830 | \n", "59.259259 | \n", "85.714286 | \n", "55.555556 | \n", "22.222222 | \n", "... | \n", "88.888889 | \n", "85.714286 | \n", "50.000000 | \n", "12.500000 | \n", "75.000000 | \n", "66.666667 | \n", "27.272727 | \n", "12.500000 | \n", "87.500000 | \n", "12.500000 | \n", "
10 rows × 38 columns
\n", "