optimum-neuron/v0.4.4/en/_app/immutable/nodes/14.32d51f78.js

import{s as qe,o as Xe,n as Ve}from"../chunks/scheduler.56725da7.js";import{S as He,i as _e,e as p,s as l,c as d,h as Ne,a as o,d as s,b as a,f as Ie,g as f,j as c,k as ye,l as Le,m as n,n as h,t as M,o as g,p as y}from"../chunks/index.18a26576.js";import{T as Ce}from"../chunks/Tip.5b941656.js";import{C as Se}from"../chunks/CopyLLMTxtMenu.4513c8ed.js";import{C as Q}from"../chunks/CodeBlock.58e3e98b.js";import{H as ge}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.049405bf.js";function ze(E){let i,w=`You can also use the
<a href="https://huggingface.co/docs/transformers/main/en/main_classes/pipelines#pipelines" rel="nofollow">pipeline()</a> function from
Transformers and provide your NeuronModel model class.`;return{c(){i=p("p"),i.innerHTML=w},l(r){i=o(r,"P",{"data-svelte-h":!0}),c(i)!=="svelte-zitk2r"&&(i.innerHTML=w)},m(r,m){n(r,i,m)},p:Ve,d(r){r&&s(i)}}}function Ee(E){let i,w=`To be able to load the model with the Neuron Runtime, the export to neuron needs
to be supported for the considered architecture.`,r,m,j=`You can check the list of supported architectures
<a href="../package_reference/configuration#supported-architectures">here</a>.`;return{c(){i=p("p"),i.textContent=w,r=l(),m=p("p"),m.innerHTML=j},l(u){i=o(u,"P",{"data-svelte-h":!0}),c(i)!=="svelte-1fvrwrq"&&(i.textContent=w),r=a(u),m=o(u,"P",{"data-svelte-h":!0}),c(m)!=="svelte-1w5ukc2"&&(m.innerHTML=j)},m(u,T){n(u,i,T),n(u,r,T),n(u,m,T)},p:Ve,d(u){u&&(s(i),s(r),s(m))}}}function Re(E){let i,w,r,m,j,u,T,F,U,Te=`The <code>pipeline()</code> function makes it simple to use models from the <a href="https://huggingface.co/models" rel="nofollow">Model Hub</a>
for accelerated inference on a variety of tasks such as text classification, question answering and image classification.`,Y,J,P,v,we="Currently the supported tasks are:",A,W,je="<li><code>feature-extraction</code></li> <li><code>fill-mask</code></li> <li><code>text-classification</code></li> <li><code>token-classification</code></li> <li><code>question-answering</code></li> <li><code>text-generation</code></li> <li><code>image-classification</code></li> <li><code>image-segmentation</code></li> <li><code>object-detection</code></li> <li><code>automatic-speech-recognition</code></li> <li><code>audio-classification</code></li>",D,x,O,k,Je=`While each task has an associated pipeline class, it is simpler to use the general <code>pipeline()</code> function which wraps all the task-specific pipelines in one object.
The <code>pipeline()</code> function automatically loads a default model and tokenizer/feature-extractor capable of performing inference for your task.`,K,G,be="<li>Start by creating a pipeline by specifying an inference task:</li>",ee,B,te,b,$e="<li>Pass your input text/image to the <code>pipeline()</code> function:</li>",se,Z,ne,I,Ue="<em>Note: The default models used in the <code>pipeline()</code> function are not optimized for inference or quantized, so there won’t be a performance improvement compared to their PyTorch counterparts.</em>",le,C,ae,V,ve=`The <code>pipeline()</code> function accepts any supported model from the <a href="https://huggingface.co/models" rel="nofollow">Hugging Face Hub</a>.
There are tags on the Model Hub that allow you to filter for a model you’d like to use for your task.`,ie,$,pe,q,We="Once you have picked an appropriate model, you can create the <code>pipeline()</code> by specifying the model repo:",oe,X,re,H,xe=`It is also possible to load it with the <code>from_pretrained(model_name_or_path, export=True)</code>
method associated with the <code>NeuronModelForXXX</code> class.`,ce,_,ke="For example, here is how you can load the <code>~neuron.NeuronModelForQuestionAnswering</code> class for question answering:",me,N,ue,L,de,S,Ge=`NeuronModels currently require static <code>input_shapes</code> to run inference. The default input shapes will be used if you are not providing input shapes when providing the <code>export=True</code> parameter.
Below is an example of how to specify the input shapes for the sequence length and batch size.`,fe,z,he,R,Me;return j=new Se({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new ge({props:{title:"Inference pipelines with AWS Neuron (Inf2/Trn1)",local:"inference-pipelines-with-aws-neuron-inf2trn1",headingTag:"h1"}}),J=new Ce({props:{$$slots:{default:[ze]},$$scope:{ctx:E}}}),x=new ge({props:{title:"Optimum pipeline usage",local:"optimum-pipeline-usage",headingTag:"h2"}}),B=new Q({props:{code:"ZnJvbSUyMG9wdGltdW0ubmV1cm9uLnBpcGVsaW5lcyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKHRhc2slM0QlMjJ0ZXh0LWNsYXNzaWZpY2F0aW9uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.neuron.pipelines <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>classifier = pipeline(task=<span class="hljs-string">&quot;text-classification&quot;</span>)`,wrap:!1}}),Z=new Q({props:{code:"Y2xhc3NpZmllciglMjJJJTIwbGlrZSUyMHlvdS4lMjBJJTIwbG92ZSUyMHlvdS4lMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>classifier(<span class="hljs-string">&quot;I like you. I love you.&quot;</span>)
[{<span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-string">&#x27;POSITIVE&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9998838901519775</span>}]`,wrap:!1}}),C=new ge({props:{title:"Using vanilla Transformers model and converting to AWS Neuron",local:"using-vanilla-transformers-model-and-converting-to-aws-neuron",headingTag:"h3"}}),$=new Ce({props:{$$slots:{default:[Ee]},$$scope:{ctx:E}}}),X=new Q({props:{code:"ZnJvbSUyMG9wdGltdW0ubmV1cm9uLnBpcGVsaW5lcyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBbmV1cm9uX3FhJTIwJTNEJTIwcGlwZWxpbmUoJTIycXVlc3Rpb24tYW5zd2VyaW5nJTIyJTJDJTIwbW9kZWwlM0QlMjJkZWVwc2V0JTJGcm9iZXJ0YS1iYXNlLXNxdWFkMiUyMiUyQyUyMGV4cG9ydCUzRFRydWUpJTBBcXVlc3Rpb24lMjAlM0QlMjAlMjJXaGF0J3MlMjBteSUyMG5hbWUlM0YlMjIlMEFjb250ZXh0JTIwJTNEJTIwJTIyTXklMjBuYW1lJTIwaXMlMjBQaGlsaXBwJTIwYW5kJTIwSSUyMGxpdmUlMjBpbiUyME51cmVtYmVyZy4lMjIlMEElMEFwcmVkJTIwJTNEJTIwbmV1cm9uX3FhKHF1ZXN0aW9uJTNEcXVlc3Rpb24lMkMlMjBjb250ZXh0JTNEY29udGV4dCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.neuron.pipelines <span class="hljs-keyword">import</span> pipeline

<span class="hljs-comment"># The model will be loaded to an NeuronModelForQuestionAnswering.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>neuron_qa = pipeline(<span class="hljs-string">&quot;question-answering&quot;</span>, model=<span class="hljs-string">&quot;deepset/roberta-base-squad2&quot;</span>, export=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;What&#x27;s my name?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>context = <span class="hljs-string">&quot;My name is Philipp and I live in Nuremberg.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>pred = neuron_qa(question=question, context=context)`,wrap:!1}}),N=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEFmcm9tJTIwb3B0aW11bS5uZXVyb24lMjBpbXBvcnQlMjBOZXVyb25Nb2RlbEZvclF1ZXN0aW9uQW5zd2VyaW5nJTJDJTIwcGlwZWxpbmUlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkZWVwc2V0JTJGcm9iZXJ0YS1iYXNlLXNxdWFkMiUyMiklMEElMEElMjMlMjBMb2FkaW5nJTIwdGhlJTIwUHlUb3JjaCUyMGNoZWNrcG9pbnQlMjBhbmQlMjBjb252ZXJ0aW5nJTIwdG8lMjB0aGUlMjBuZXVyb24lMjBmb3JtYXQlMjBieSUyMHByb3ZpZGluZyUyMGV4cG9ydCUzRFRydWUlMEFtb2RlbCUyMCUzRCUyME5ldXJvbk1vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmcuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRlZXBzZXQlMkZyb2JlcnRhLWJhc2Utc3F1YWQyJTIyJTJDJTBBJTIwJTIwJTIwJTIwZXhwb3J0JTNEVHJ1ZSUwQSklMEElMEFuZXVyb25fcWElMjAlM0QlMjBwaXBlbGluZSglMjJxdWVzdGlvbi1hbnN3ZXJpbmclMjIlMkMlMjBtb2RlbCUzRG1vZGVsJTJDJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyKSUwQXF1ZXN0aW9uJTIwJTNEJTIwJTIyV2hhdCdzJTIwbXklMjBuYW1lJTNGJTIyJTBBY29udGV4dCUyMCUzRCUyMCUyMk15JTIwbmFtZSUyMGlzJTIwUGhpbGlwcCUyMGFuZCUyMEklMjBsaXZlJTIwaW4lMjBOdXJlbWJlcmcuJTIyJTBBJTBBcHJlZCUyMCUzRCUyMG5ldXJvbl9xYShxdWVzdGlvbiUzRHF1ZXN0aW9uJTJDJTIwY29udGV4dCUzRGNvbnRleHQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronModelForQuestionAnswering, pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;deepset/roberta-base-squad2&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Loading the PyTorch checkpoint and converting to the neuron format by providing export=True</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = NeuronModelForQuestionAnswering.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;deepset/roberta-base-squad2&quot;</span>,
<span class="hljs-meta">... </span>    export=<span class="hljs-literal">True</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>neuron_qa = pipeline(<span class="hljs-string">&quot;question-answering&quot;</span>, model=model, tokenizer=tokenizer)
<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;What&#x27;s my name?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>context = <span class="hljs-string">&quot;My name is Philipp and I live in Nuremberg.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>pred = neuron_qa(question=question, context=context)`,wrap:!1}}),L=new ge({props:{title:"Defining Input Shapes",local:"defining-input-shapes",headingTag:"h3"}}),z=new Q({props:{code:"ZnJvbSUyMG9wdGltdW0ubmV1cm9uLnBpcGVsaW5lcyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBaW5wdXRfc2hhcGVzJTIwJTNEJTIwJTdCJTIyYmF0Y2hfc2l6ZSUyMiUzQSUyMDElMkMlMjAlMjJzZXF1ZW5jZV9sZW5ndGglMjIlM0ElMjA2NCU3RCUwQWNsdCUyMCUzRCUyMHBpcGVsaW5lKCUyMnRva2VuLWNsYXNzaWZpY2F0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJkc2xpbSUyRmJlcnQtYmFzZS1ORVIlMjIlMkMlMjBleHBvcnQlM0RUcnVlJTJDaW5wdXRfc2hhcGVzJTNEaW5wdXRfc2hhcGVzKSUwQWNvbnRleHQlMjAlM0QlMjAlMjJNeSUyMG5hbWUlMjBpcyUyMFBoaWxpcHAlMjBhbmQlMjBJJTIwbGl2ZSUyMGluJTIwTnVyZW1iZXJnLiUyMiUwQSUwQXByZWQlMjAlM0QlMjBjbHQoY29udGV4dCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.neuron.pipelines <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>input_shapes = {<span class="hljs-string">&quot;batch_size&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;sequence_length&quot;</span>: <span class="hljs-number">64</span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>clt = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>, model=<span class="hljs-string">&quot;dslim/bert-base-NER&quot;</span>, export=<span class="hljs-literal">True</span>,input_shapes=input_shapes)
<span class="hljs-meta">&gt;&gt;&gt; </span>context = <span class="hljs-string">&quot;My name is Philipp and I live in Nuremberg.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>pred = clt(context)`,wrap:!1}}),{c(){i=p("meta"),w=l(),r=p("p"),m=l(),d(j.$$.fragment),u=l(),d(T.$$.fragment),F=l(),U=p("p"),U.innerHTML=Te,Y=l(),d(J.$$.fragment),P=l(),v=p("p"),v.textContent=we,A=l(),W=p("ul"),W.innerHTML=je,D=l(),d(x.$$.fragment),O=l(),k=p("p"),k.innerHTML=Je,K=l(),G=p("ol"),G.innerHTML=be,ee=l(),d(B.$$.fragment),te=l(),b=p("ol"),b.innerHTML=$e,se=l(),d(Z.$$.fragment),ne=l(),I=p("p"),I.innerHTML=Ue,le=l(),d(C.$$.fragment),ae=l(),V=p("p"),V.innerHTML=ve,ie=l(),d($.$$.fragment),pe=l(),q=p("p"),q.innerHTML=We,oe=l(),d(X.$$.fragment),re=l(),H=p("p"),H.innerHTML=xe,ce=l(),_=p("p"),_.innerHTML=ke,me=l(),d(N.$$.fragment),ue=l(),d(L.$$.fragment),de=l(),S=p("p"),S.innerHTML=Ge,fe=l(),d(z.$$.fragment),he=l(),R=p("p"),this.h()},l(e){const t=Ne("svelte-u9bgzb",document.head);i=o(t,"META",{name:!0,content:!0}),t.forEach(s),w=a(e),r=o(e,"P",{}),Ie(r).forEach(s),m=a(e),f(j.$$.fragment,e),u=a(e),f(T.$$.fragment,e),F=a(e),U=o(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-grle1x"&&(U.innerHTML=Te),Y=a(e),f(J.$$.fragment,e),P=a(e),v=o(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-1rl24tl"&&(v.textContent=we),A=a(e),W=o(e,"UL",{"data-svelte-h":!0}),c(W)!=="svelte-4fbr41"&&(W.innerHTML=je),D=a(e),f(x.$$.fragment,e),O=a(e),k=o(e,"P",{"data-svelte-h":!0}),c(k)!=="svelte-j14w9v"&&(k.innerHTML=Je),K=a(e),G=o(e,"OL",{"data-svelte-h":!0}),c(G)!=="svelte-khfcvb"&&(G.innerHTML=be),ee=a(e),f(B.$$.fragment,e),te=a(e),b=o(e,"OL",{start:!0,"data-svelte-h":!0}),c(b)!=="svelte-8j6vxl"&&(b.innerHTML=$e),se=a(e),f(Z.$$.fragment,e),ne=a(e),I=o(e,"P",{"data-svelte-h":!0}),c(I)!=="svelte-1o2tkpw"&&(I.innerHTML=Ue),le=a(e),f(C.$$.fragment,e),ae=a(e),V=o(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-ddng3e"&&(V.innerHTML=ve),ie=a(e),f($.$$.fragment,e),pe=a(e),q=o(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-1ig4n3p"&&(q.innerHTML=We),oe=a(e),f(X.$$.fragment,e),re=a(e),H=o(e,"P",{"data-svelte-h":!0}),c(H)!=="svelte-djskr"&&(H.innerHTML=xe),ce=a(e),_=o(e,"P",{"data-svelte-h":!0}),c(_)!=="svelte-1iyza81"&&(_.innerHTML=ke),me=a(e),f(N.$$.fragment,e),ue=a(e),f(L.$$.fragment,e),de=a(e),S=o(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-xm6kib"&&(S.innerHTML=Ge),fe=a(e),f(z.$$.fragment,e),he=a(e),R=o(e,"P",{}),Ie(R).forEach(s),this.h()},h(){ye(i,"name","hf:doc:metadata"),ye(i,"content",Qe),ye(b,"start","2")},m(e,t){Le(document.head,i),n(e,w,t),n(e,r,t),n(e,m,t),h(j,e,t),n(e,u,t),h(T,e,t),n(e,F,t),n(e,U,t),n(e,Y,t),h(J,e,t),n(e,P,t),n(e,v,t),n(e,A,t),n(e,W,t),n(e,D,t),h(x,e,t),n(e,O,t),n(e,k,t),n(e,K,t),n(e,G,t),n(e,ee,t),h(B,e,t),n(e,te,t),n(e,b,t),n(e,se,t),h(Z,e,t),n(e,ne,t),n(e,I,t),n(e,le,t),h(C,e,t),n(e,ae,t),n(e,V,t),n(e,ie,t),h($,e,t),n(e,pe,t),n(e,q,t),n(e,oe,t),h(X,e,t),n(e,re,t),n(e,H,t),n(e,ce,t),n(e,_,t),n(e,me,t),h(N,e,t),n(e,ue,t),h(L,e,t),n(e,de,t),n(e,S,t),n(e,fe,t),h(z,e,t),n(e,he,t),n(e,R,t),Me=!0},p(e,[t]){const Be={};t&2&&(Be.$$scope={dirty:t,ctx:e}),J.$set(Be);const Ze={};t&2&&(Ze.$$scope={dirty:t,ctx:e}),$.$set(Ze)},i(e){Me||(M(j.$$.fragment,e),M(T.$$.fragment,e),M(J.$$.fragment,e),M(x.$$.fragment,e),M(B.$$.fragment,e),M(Z.$$.fragment,e),M(C.$$.fragment,e),M($.$$.fragment,e),M(X.$$.fragment,e),M(N.$$.fragment,e),M(L.$$.fragment,e),M(z.$$.fragment,e),Me=!0)},o(e){g(j.$$.fragment,e),g(T.$$.fragment,e),g(J.$$.fragment,e),g(x.$$.fragment,e),g(B.$$.fragment,e),g(Z.$$.fragment,e),g(C.$$.fragment,e),g($.$$.fragment,e),g(X.$$.fragment,e),g(N.$$.fragment,e),g(L.$$.fragment,e),g(z.$$.fragment,e),Me=!1},d(e){e&&(s(w),s(r),s(m),s(u),s(F),s(U),s(Y),s(P),s(v),s(A),s(W),s(D),s(O),s(k),s(K),s(G),s(ee),s(te),s(b),s(se),s(ne),s(I),s(le),s(ae),s(V),s(ie),s(pe),s(q),s(oe),s(re),s(H),s(ce),s(_),s(me),s(ue),s(de),s(S),s(fe),s(he),s(R)),s(i),y(j,e),y(T,e),y(J,e),y(x,e),y(B,e),y(Z,e),y(C,e),y($,e),y(X,e),y(N,e),y(L,e),y(z,e)}}}const Qe='{"title":"Inference pipelines with AWS Neuron (Inf2/Trn1)","local":"inference-pipelines-with-aws-neuron-inf2trn1","sections":[{"title":"Optimum pipeline usage","local":"optimum-pipeline-usage","sections":[{"title":"Using vanilla Transformers model and converting to AWS Neuron","local":"using-vanilla-transformers-model-and-converting-to-aws-neuron","sections":[],"depth":3},{"title":"Defining Input Shapes","local":"defining-input-shapes","sections":[],"depth":3}],"depth":2}],"depth":1}';function Fe(E){return Xe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class et extends He{constructor(i){super(),_e(this,i,Fe,Re,qe,{})}}export{et as component};