add custom handler

README.md

@@ -0,0 +1,219 @@
+---
+license: mit
+tags:
+- sentence-embeddings
+- endpoints-template
+- optimum
+library_name: generic
+---
+
+# Optimized and Quantized [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) with a custom pipeline.py
+
+
+This repository implements a `custom` task for `sentence-embeddings` for 🤗 Inference Endpoints for accelerated inference using [🤗 Optiumum](https://huggingface.co/docs/optimum/index). The code for the customized pipeline is in the [pipeline.py](https://huggingface.co/philschmid/all-MiniLM-L6-v2-optimum-embeddings/blob/main/pipeline.py).
+
+Below is also describe how we converted & optimized the model, based on the [Accelerate Sentence Transformers with Hugging Face Optimum](https://www.philschmid.de/optimize-sentence-transformers) blog post. You can also check out the [notebook](https://huggingface.co/philschmid/all-MiniLM-L6-v2-optimum-embeddings/blob/main/convert.ipynb).
+
+To use deploy this model a an Inference Endpoint you have to select `Custom` as task to use the `pipeline.py` file. -> _double check if it is selected_
+
+### expected Request payload
+
+```json
+{
+  "inputs": "The sky is a blue today and not gray", 
+}
+```
+
+below is an example on how to run a request using Python and `requests`.
+
+## Run Request 
+
+```python
+import json
+from typing import List
+import requests as r
+import base64
+
+ENDPOINT_URL = ""
+HF_TOKEN = ""
+
+
+def predict(document_string:str=None):
+
+    payload = {"inputs": document_string}
+    response = r.post(
+        ENDPOINT_URL, headers={"Authorization": f"Bearer {HF_TOKEN}"}, json=payload
+    )
+    return response.json()
+
+
+prediction = predict(
+    path_to_image="The sky is a blue today and not gray"
+)
+```
+
+expected output
+
+```python
+{'embeddings': [[-0.021580450236797333,
+   0.021715054288506508,
+   0.00979710929095745,
+   -0.0005379787762649357,
+   0.04682469740509987,
+   -0.013600599952042103,
+   ...
+}
+```
+
+
+
+## How to create your own optimized and quantized model
+
+Steps:
+[1. Convert model to ONNX](#1-convert-model-to-onnx)  
+[2. Optimize & quantize model with Optimum](#2-optimize--quantize-model-with-optimum)  
+[3. Create Custom Handler for Inference Endpoints](#3-create-custom-handler-for-inference-endpoints)  
+
+Helpful links:
+* [Accelerate Sentence Transformers with Hugging Face Optimum](https://www.philschmid.de/optimize-sentence-transformers)
+* [Create Custom Handler Endpoints](https://link-to-docs)
+
+## Setup & Installation
+
+```python
+%%writefile requirements.txt
+optimum[onnxruntime]==1.3.0
+mkl-include
+mkl
+```
+
+install requirements
+
+```python
+!pip install -r requirements.txt
+```
+
+## 1. Convert model to ONNX
+
+
+```python
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+from transformers import AutoTokenizer
+from pathlib import Path
+
+
+model_id="sentence-transformers/all-MiniLM-L6-v2"
+onnx_path = Path(".")
+
+# load vanilla transformers and convert to onnx
+model = ORTModelForFeatureExtraction.from_pretrained(model_id, from_transformers=True)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+# save onnx checkpoint and tokenizer
+model.save_pretrained(onnx_path)
+tokenizer.save_pretrained(onnx_path)
+```
+
+
+## 2. Optimize & quantize model with Optimum
+
+
+```python
+from optimum.onnxruntime import ORTOptimizer, ORTQuantizer
+from optimum.onnxruntime.configuration import OptimizationConfig, AutoQuantizationConfig
+
+# create ORTOptimizer and define optimization configuration
+optimizer = ORTOptimizer.from_pretrained(model_id, feature=model.pipeline_task)
+optimization_config = OptimizationConfig(optimization_level=99) # enable all optimizations
+
+# apply the optimization configuration to the model
+optimizer.export(
+    onnx_model_path=onnx_path / "model.onnx",
+    onnx_optimized_model_output_path=onnx_path / "model-optimized.onnx",
+    optimization_config=optimization_config,
+)
+
+
+# create ORTQuantizer and define quantization configuration
+dynamic_quantizer = ORTQuantizer.from_pretrained(model_id, feature=model.pipeline_task)
+dqconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=False)
+
+# apply the quantization configuration to the model
+model_quantized_path = dynamic_quantizer.export(
+    onnx_model_path=onnx_path / "model-optimized.onnx",
+    onnx_quantized_model_output_path=onnx_path / "model-quantized.onnx",
+    quantization_config=dqconfig,
+)
+
+
+```
+
+## 3. Create Custom Handler for Inference Endpoints
+
+
+```python
+%%writefile pipeline.py
+from typing import  Dict, List, Any
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+from transformers import AutoTokenizer
+import torch.nn.functional as F
+import torch
+
+# copied from the model card
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+class PreTrainedPipeline():
+    def __init__(self, path=""):
+        # load the optimized model
+        self.model = ORTModelForFeatureExtraction.from_pretrained(path, file_name="model-quantized.onnx")
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+
+    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
+        """
+        Args:
+            data (:obj:):
+                includes the input data and the parameters for the inference.
+        Return:
+            A :obj:`list`:. The list contains the embeddings of the inference inputs
+        """
+        inputs = data.get("inputs", data)
+
+        # tokenize the input
+        encoded_inputs = self.tokenizer(inputs, padding=True, truncation=True, return_tensors='pt')
+        # run the model
+        outputs = self.model(**encoded_inputs)
+        # Perform pooling
+        sentence_embeddings = mean_pooling(outputs, encoded_inputs['attention_mask'])
+        # Normalize embeddings
+        sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+        # postprocess the prediction
+        return {"embeddings": sentence_embeddings.tolist()}
+```
+
+test custom pipeline
+
+
+```python
+from pipeline import PreTrainedPipeline
+
+# init handler
+my_handler = PreTrainedPipeline(path=".")
+
+# prepare sample payload
+request = {"inputs": "I am quite excited how this will turn out"}
+
+# test the handler
+%timeit my_handler(request)
+
+```
+
+results
+
+```
+1.55 ms ± 2.04 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)
+```
+

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "deepset/roberta-base-squad2",
+  "architectures": [
+    "RobertaForQuestionAnswering"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "language": "english",
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "name": "Roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.21.3",
+  "type_vocab_size": 1,
+  "use_cache": false,
+  "vocab_size": 50265
+}

handler.py

@@ -0,0 +1,26 @@
+from typing import  Dict, List, Any
+from optimum.onnxruntime import ORTModelForQuestionAnswering
+from transformers import AutoTokenizer, pipeline
+
+
+class EndpointHandler():
+    def __init__(self, path=""):
+        # load the optimized model
+        self.model = ORTModelForQuestionAnswering.from_pretrained(path, file_name="model_optimized_quantized.onnx")
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        # create pipeline
+        self.pipeline = pipeline("question-answering", model=self.model, tokenizer=self.tokenizer)
+
+    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
+        """
+        Args:
+            data (:obj:):
+                includes the input data and the parameters for the inference.
+        Return:
+            A :obj:`list`:. The list contains the answer and scores of the inference inputs
+        """
+        inputs = data.get("inputs", data)
+        # run the model
+        prediction = self.pipeline(**inputs)
+        # return prediction
+        return prediction

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:570afefbc8642150310e46c10a252dd091c8f44449e8a3a65a425f77991dc2ab
+size 496337664

model_optimized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c0577c4bb3afdb2a88e21807d5722511b6aa678d6d8275a7ba73c5cd8f88b1
+size 496254364

model_optimized_quantized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a27adda924cc0cd34fde41f606da51673ebefb7132a5518e41f1196ebc362f1
+size 305175132

optimize_model.ipynb

@@ -0,0 +1,438 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Convert & Optimize model with Optimum \n",
+    "\n",
+    "\n",
+    "Steps:\n",
+    "1. Convert model to ONNX\n",
+    "2. Optimize & quantize model with Optimum\n",
+    "3. Create Custom Handler for Inference Endpoints\n",
+    "4. Test Custom Handler Locally\n",
+    "5. Push to repository and create Inference Endpoint\n",
+    "\n",
+    "Helpful links:\n",
+    "* [Accelerate Transformers with Hugging Face Optimum](https://huggingface.co/blog/optimum-inference)\n",
+    "* [Optimizing Transformers for GPUs with Optimum](https://www.philschmid.de/optimizing-transformers-with-optimum-gpu)\n",
+    "* [Optimum Documentation](https://huggingface.co/docs/optimum/onnxruntime/modeling_ort)\n",
+    "* [Create Custom Handler Endpoints](https://link-to-docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setup & Installation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Writing requirements.txt\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile requirements.txt\n",
+    "optimum[onnxruntime]==1.4.0\n",
+    "mkl-include\n",
+    "mkl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install -r requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 0. Base line Performance\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "qa = pipeline(\"question-answering\",model=\"deepset/roberta-base-squad2\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Okay, let's test the performance (latency) with sequence length of 128."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context=\"Hello, my name is Philipp and I live in Nuremberg, Germany. Currently I am working as a Technical Lead at Hugging Face to democratize artificial intelligence through open source and open science. In the past I designed and implemented cloud-native machine learning architectures for fin-tech and insurance companies. I found my passion for cloud concepts and machine learning 5 years ago. Since then I never stopped learning. Currently, I am focusing myself in the area NLP and how to leverage models like BERT, Roberta, T5, ViT, and GPT2 to generate business value.\" \n",
+    "question=\"As what is Philipp working?\" \n",
+    "\n",
+    "payload = {\"inputs\": {\"question\": question, \"context\": context}}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Vanilla model Average latency (ms) - 64.15 +\\- 2.44\n"
+     ]
+    }
+   ],
+   "source": [
+    "from time import perf_counter\n",
+    "import numpy as np \n",
+    "\n",
+    "def measure_latency(pipe,payload):\n",
+    "    latencies = []\n",
+    "    # warm up\n",
+    "    for _ in range(10):\n",
+    "        _ = pipe(question=payload[\"inputs\"][\"question\"], context=payload[\"inputs\"][\"context\"])\n",
+    "    # Timed run\n",
+    "    for _ in range(50):\n",
+    "        start_time = perf_counter()\n",
+    "        _ =  pipe(question=payload[\"inputs\"][\"question\"], context=payload[\"inputs\"][\"context\"])\n",
+    "        latency = perf_counter() - start_time\n",
+    "        latencies.append(latency)\n",
+    "    # Compute run statistics\n",
+    "    time_avg_ms = 1000 * np.mean(latencies)\n",
+    "    time_std_ms = 1000 * np.std(latencies)\n",
+    "    return f\"Average latency (ms) - {time_avg_ms:.2f} +\\- {time_std_ms:.2f}\"\n",
+    "\n",
+    "print(f\"Vanilla model {measure_latency(qa,payload)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Convert model to ONNX"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "df00c03d67b546bf8a3d1a327b9380f5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/571 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "('./tokenizer_config.json',\n",
+       " './special_tokens_map.json',\n",
+       " './vocab.json',\n",
+       " './merges.txt',\n",
+       " './added_tokens.json',\n",
+       " './tokenizer.json')"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from optimum.onnxruntime import ORTModelForQuestionAnswering\n",
+    "from transformers import AutoTokenizer\n",
+    "from pathlib import Path\n",
+    "\n",
+    "\n",
+    "model_id=\"deepset/roberta-base-squad2\"\n",
+    "onnx_path = Path(\".\")\n",
+    "\n",
+    "# load vanilla transformers and convert to onnx\n",
+    "model = ORTModelForQuestionAnswering.from_pretrained(model_id, from_transformers=True)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
+    "\n",
+    "# save onnx checkpoint and tokenizer\n",
+    "model.save_pretrained(onnx_path)\n",
+    "tokenizer.save_pretrained(onnx_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Optimize & quantize model with Optimum"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2022-09-12 18:47:03.240390005 [W:onnxruntime:, inference_session.cc:1488 Initialize] Serializing optimized model with Graph Optimization level greater than ORT_ENABLE_EXTENDED and the NchwcTransformer enabled. The generated model may contain hardware specific optimizations, and should only be used in the same environment the model was optimized in.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "PosixPath('.')"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from optimum.onnxruntime import ORTOptimizer, ORTQuantizer\n",
+    "from optimum.onnxruntime.configuration import OptimizationConfig, AutoQuantizationConfig\n",
+    "\n",
+    "# Create the optimizer\n",
+    "optimizer = ORTOptimizer.from_pretrained(model)\n",
+    "\n",
+    "# Define the optimization strategy by creating the appropriate configuration\n",
+    "optimization_config = OptimizationConfig(optimization_level=99) # enable all optimizations\n",
+    "\n",
+    "# Optimize the model\n",
+    "optimizer.optimize(save_dir=onnx_path, optimization_config=optimization_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create ORTQuantizer and define quantization configuration\n",
+    "dynamic_quantizer = ORTQuantizer.from_pretrained(onnx_path, file_name=\"model_optimized.onnx\")\n",
+    "dqconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=False)\n",
+    "\n",
+    "# apply the quantization configuration to the model\n",
+    "model_quantized_path = dynamic_quantizer.quantize(\n",
+    "    save_dir=onnx_path,\n",
+    "    quantization_config=dqconfig,\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Create Custom Handler for Inference Endpoints\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting handler.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile handler.py\n",
+    "from typing import  Dict, List, Any\n",
+    "from optimum.onnxruntime import ORTModelForQuestionAnswering\n",
+    "from transformers import AutoTokenizer, pipeline\n",
+    "\n",
+    "\n",
+    "class EndpointHandler():\n",
+    "    def __init__(self, path=\"\"):\n",
+    "        # load the optimized model\n",
+    "        self.model = ORTModelForQuestionAnswering.from_pretrained(path, file_name=\"model_optimized_quantized.onnx\")\n",
+    "        self.tokenizer = AutoTokenizer.from_pretrained(path)\n",
+    "        # create pipeline\n",
+    "        self.pipeline = pipeline(\"question-answering\", model=self.model, tokenizer=self.tokenizer)\n",
+    "\n",
+    "    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:\n",
+    "        \"\"\"\n",
+    "        Args:\n",
+    "            data (:obj:):\n",
+    "                includes the input data and the parameters for the inference.\n",
+    "        Return:\n",
+    "            A :obj:`list`:. The list contains the answer and scores of the inference inputs\n",
+    "        \"\"\"\n",
+    "        inputs = data.get(\"inputs\", data)\n",
+    "        # run the model\n",
+    "        prediction = self.pipeline(**inputs)\n",
+    "        # return prediction\n",
+    "        return prediction"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Test Custom Handler Locally\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 0.4749588668346405,\n",
+       " 'start': 88,\n",
+       " 'end': 102,\n",
+       " 'answer': 'Technical Lead'}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from handler import EndpointHandler\n",
+    "\n",
+    "# init handler\n",
+    "my_handler = EndpointHandler(path=\".\")\n",
+    "\n",
+    "# prepare sample payload\n",
+    "context=\"Hello, my name is Philipp and I live in Nuremberg, Germany. Currently I am working as a Technical Lead at Hugging Face to democratize artificial intelligence through open source and open science. In the past I designed and implemented cloud-native machine learning architectures for fin-tech and insurance companies. I found my passion for cloud concepts and machine learning 5 years ago. Since then I never stopped learning. Currently, I am focusing myself in the area NLP and how to leverage models like BERT, Roberta, T5, ViT, and GPT2 to generate business value.\" \n",
+    "question=\"As what is Philipp working?\" \n",
+    "\n",
+    "payload = {\"inputs\": {\"question\": question, \"context\": context}}\n",
+    "\n",
+    "# test the handler\n",
+    "my_handler(payload)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Optimized & Quantized model Average latency (ms) - 29.90 +\\- 0.53\n"
+     ]
+    }
+   ],
+   "source": [
+    "from time import perf_counter\n",
+    "import numpy as np \n",
+    "\n",
+    "def measure_latency(handler,payload):\n",
+    "    latencies = []\n",
+    "    # warm up\n",
+    "    for _ in range(10):\n",
+    "        _ = handler(payload)\n",
+    "    # Timed run\n",
+    "    for _ in range(50):\n",
+    "        start_time = perf_counter()\n",
+    "        _ =  handler(payload)\n",
+    "        latency = perf_counter() - start_time\n",
+    "        latencies.append(latency)\n",
+    "    # Compute run statistics\n",
+    "    time_avg_ms = 1000 * np.mean(latencies)\n",
+    "    time_std_ms = 1000 * np.std(latencies)\n",
+    "    return f\"Average latency (ms) - {time_avg_ms:.2f} +\\- {time_std_ms:.2f}\"\n",
+    "\n",
+    "print(f\"Optimized & Quantized model {measure_latency(my_handler,payload)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`Vanilla model Average latency (ms) - 64.15 +\\- 2.44`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. Push to repository and create Inference Endpoint\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# add all our new files\n",
+    "!git add * \n",
+    "# commit our files\n",
+    "!git commit -m \"add custom handler\"\n",
+    "# push the files to the hub\n",
+    "!git push"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.12 ('az': conda)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "bddb99ecda5b40a820d97bf37f3ff3a89fb9dbcf726ae84d28624ac628a665b4"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

ort_config.json

@@ -0,0 +1,29 @@
+{
+  "opset": null,
+  "optimization": {},
+  "optimum_version": "1.4.0",
+  "quantization": {
+    "activations_dtype": "QUInt8",
+    "activations_symmetric": false,
+    "format": "QOperator",
+    "is_static": false,
+    "mode": "IntegerOps",
+    "nodes_to_exclude": [],
+    "nodes_to_quantize": [],
+    "operators_to_quantize": [
+      "MatMul",
+      "Add"
+    ],
+    "per_channel": false,
+    "qdq_add_pair_to_weight": false,
+    "qdq_dedicated_pair": false,
+    "qdq_op_type_per_channel_support_to_axis": {
+      "MatMul": 1
+    },
+    "reduce_range": false,
+    "weights_dtype": "QInt8",
+    "weights_symmetric": true
+  },
+  "transformers_version": "4.21.3",
+  "use_external_data_format": false
+}

requirements.txt

@@ -0,0 +1,3 @@
+optimum[onnxruntime]==1.4.0
+mkl-include
+mkl

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,67 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "do_lower_case": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "full_tokenizer_file": null,
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "deepset/roberta-base-squad2",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": "/home/ubuntu/.cache/huggingface/transformers/c9d2c178fac8d40234baa1833a3b1903d393729bf93ea34da247c07db24900d0.cb2244924ab24d706b02fd7fcedaea4531566537687a539ebb94db511fd122a0",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}