Create notebook for making embedding og sentence via BERT

bert_to_vec.ipynb

@@ -0,0 +1,118 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# BERT to Vector"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Load model and tokenizer from https://huggingface.co/NbAiLab/nb-bert-base/tree/main"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at NbAiLab/nb-bert-base were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias']\n",
+      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch \n",
+    "from transformers import AutoModel\n",
+    "from transformers import BertTokenizer\n",
+    "\n",
+    "model = AutoModel.from_pretrained(\"NbAiLab/nb-bert-base\")\n",
+    "tokenizer = BertTokenizer.from_pretrained(\"NbAiLab/nb-bert-base\")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Get embedding vector from arbitrary input"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sequence = f\"La oss prise Jesus for at han er den eneste veien til Gud.\" # <- copilot autocomplete from 'La oss prise'\n",
+    "tokenized = tokenizer(sequence, return_tensors=\"pt\")\n",
+    "vector = model(**tokenized, output_hidden_states=True)['hidden_states'][0][0, :, :]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(torch.Size([16, 768]),\n",
+       " tensor([[-0.2137,  0.2956,  0.0240,  ..., -0.0604, -0.3877, -0.2480],\n",
+       "         [ 0.4414,  0.0984,  0.9899,  ..., -0.4446, -0.1912,  0.4375],\n",
+       "         [ 0.9543, -0.1275,  0.6129,  ..., -0.8141,  0.8328,  0.1206],\n",
+       "         ...,\n",
+       "         [-0.1980, -0.0070, -0.1090,  ...,  0.4496,  0.7290,  0.0152],\n",
+       "         [-0.2790,  0.2548,  0.1130,  ..., -0.2711, -0.0275, -0.1722],\n",
+       "         [ 0.4214, -0.0558,  0.4787,  ..., -0.1377, -0.0362, -0.3224]],\n",
+       "        grad_fn=<SliceBackward0>))"
+      ]
+     },
+     "execution_count": 71,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vector.shape, vector"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "8a67875c7f50c1c171b962999a73a6eec154dfdf907f821fd501db0eb225404a"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -1,10 +1,13 @@
 appnope==0.1.3
 asttokens==2.2.1
 backcall==0.2.0
+certifi==2022.12.7
+charset-normalizer==3.0.1
 comm==0.1.2
 debugpy==1.6.6
 decorator==5.1.1
 executing==1.2.0
+idna==3.4
 ipykernel==6.21.2
 ipython==8.10.0
 jedi==0.18.2
@@ -18,6 +21,7 @@ pandas==1.5.3
 parso==0.8.3
 pexpect==4.8.0
 pickleshare==0.7.5
+Pillow==9.4.0
 platformdirs==3.0.0
 prompt-toolkit==3.0.36
 psutil==5.9.4
@@ -27,8 +31,13 @@ Pygments==2.14.0
 python-dateutil==2.8.2
 pytz==2022.7.1
 pyzmq==25.0.0
+requests==2.28.2
 six==1.16.0
 stack-data==0.6.2
+torch==1.13.1
+torchvision==0.14.1
 tornado==6.2
 traitlets==5.9.0
+typing_extensions==4.4.0
+urllib3==1.26.14
 wcwidth==0.2.6