Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"EC2 Setup","local":"ec2-setup","sections":[{"title":"Recommended: AWS EC2 with Deep Learning AMI","local":"recommended-aws-ec2-with-deep-learning-ami","sections":[{"title":"Create an AWS Trainium or Inferentia instance on Amazon EC2 with the HF DLAMI","local":"create-an-aws-trainium-or-inferentia-instance-on-amazon-ec2-with-the-hf-dlami","sections":[{"title":"Find a supported region","local":"find-a-supported-region","sections":[],"depth":4},{"title":"Increase service quota","local":"increase-service-quota","sections":[],"depth":4},{"title":"Launch the Amazon EC2 instance with the HF DLAMI","local":"launch-the-amazon-ec2-instance-with-the-hf-dlami","sections":[],"depth":4},{"title":"Connect through SSH","local":"connect-through-ssh","sections":[],"depth":4}],"depth":3},{"title":"Set up your remote development environment","local":"set-up-your-remote-development-environment","sections":[{"title":"Access through Jupyter Notebook","local":"access-through-jupyter-notebook","sections":[],"depth":4},{"title":"Access through VS Code remote server","local":"access-through-vs-code-remote-server","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Alternative: Manual Installation","local":"alternative-manual-installation","sections":[{"title":"Adding pip packages URL","local":"adding-pip-packages-url","sections":[],"depth":3},{"title":"Installing optimum-neuron for AWS Trainium ( trn1 ) or AWS inferentia2 ( inf2 )","local":"installing-optimum-neuron-for-aws-trainium--trn1--or-aws-inferentia2--inf2-","sections":[],"depth":3},{"title":"Installing optimum-neuron for AWS inferentia ( inf1 )","local":"installing-optimum-neuron-for-aws-inferentia--inf1-","sections":[],"depth":3}],"depth":2},{"title":"What’s Next?","local":"whats-next","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/entry/start.a52ea6f7.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/scheduler.56725da7.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/singletons.dd296e23.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/paths.ad75736c.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/entry/app.3fdd64a6.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/preload-helper.c3fe681f.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/index.18a26576.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/nodes/0.6ece8d77.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/nodes/8.4bb40b54.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/CopyLLMTxtMenu.3134fcef.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/globals.7f7f1b26.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.868449a1.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.4.3/en/_app/immutable/chunks/CodeBlock.b87ef962.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"EC2 Setup","local":"ec2-setup","sections":[{"title":"Recommended: AWS EC2 with Deep Learning AMI","local":"recommended-aws-ec2-with-deep-learning-ami","sections":[{"title":"Create an AWS Trainium or Inferentia instance on Amazon EC2 with the HF DLAMI","local":"create-an-aws-trainium-or-inferentia-instance-on-amazon-ec2-with-the-hf-dlami","sections":[{"title":"Find a supported region","local":"find-a-supported-region","sections":[],"depth":4},{"title":"Increase service quota","local":"increase-service-quota","sections":[],"depth":4},{"title":"Launch the Amazon EC2 instance with the HF DLAMI","local":"launch-the-amazon-ec2-instance-with-the-hf-dlami","sections":[],"depth":4},{"title":"Connect through SSH","local":"connect-through-ssh","sections":[],"depth":4}],"depth":3},{"title":"Set up your remote development environment","local":"set-up-your-remote-development-environment","sections":[{"title":"Access through Jupyter Notebook","local":"access-through-jupyter-notebook","sections":[],"depth":4},{"title":"Access through VS Code remote server","local":"access-through-vs-code-remote-server","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Alternative: Manual Installation","local":"alternative-manual-installation","sections":[{"title":"Adding pip packages URL","local":"adding-pip-packages-url","sections":[],"depth":3},{"title":"Installing optimum-neuron for AWS Trainium ( trn1 ) or AWS inferentia2 ( inf2 )","local":"installing-optimum-neuron-for-aws-trainium--trn1--or-aws-inferentia2--inf2-","sections":[],"depth":3},{"title":"Installing optimum-neuron for AWS inferentia ( inf1 )","local":"installing-optimum-neuron-for-aws-inferentia--inf1-","sections":[],"depth":3}],"depth":2},{"title":"What’s Next?","local":"whats-next","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 max-sm:gap-0.5 h-6 max-sm:h-5 px-2 max-sm:px-1.5 text-[11px] max-sm:text-[9px] font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0"><svg class="w-3 h-3 max-sm:w-2.5 max-sm:h-2.5" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-6 max-sm:h-5 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible w-3 h-3 max-sm:w-2.5 max-sm:h-2.5 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="ec2-setup" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ec2-setup"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>EC2 Setup</span></h1> <p data-svelte-h="svelte-ppiuvg">This guide will help you get Optimum Neuron up and running. There are two main approaches:</p> <ol data-svelte-h="svelte-bfl7te"><li><strong>🚀 Recommended: AWS EC2 with Deep Learning AMI</strong> - The simplest way to get started with pre-configured environment</li> <li><strong>⚙️ Manual Installation</strong> - Install Optimum Neuron on existing infrastructure</li></ol> <h2 class="relative group"><a id="recommended-aws-ec2-with-deep-learning-ami" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#recommended-aws-ec2-with-deep-learning-ami"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Recommended: AWS EC2 with Deep Learning AMI</span></h2> <p data-svelte-h="svelte-1gnzyei">The simplest way to work with AWS Trainium or Inferentia and Optimum Neuron on Amazon EC2 is the <a href="https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2" rel="nofollow">Hugging Face Neuron Deep Learning AMI</a> (DLAMI). The DLAMI comes with all required libraries pre-packaged for you, including the Optimum Neuron, Neuron Drivers, Transformers, Datasets, and Accelerate. The HF DLAMI is provided at no additional charge to Amazon EC2 users.</p> <p data-svelte-h="svelte-1g253ra">Optimum Neuron supports Inf1, Inf2, Trn1 and Trn2, all accessible on Amazon EC2. You can find all the specifications of the Trn and Inf instances <a href="https://aws.amazon.com/ec2/instance-types/" rel="nofollow">here</a>, in the “Accelerated Computing” section.</p> <p data-svelte-h="svelte-zcjb5i">In this section, we will show you:</p> <ol data-svelte-h="svelte-myip9t"><li><a href="#create-an-aws-trainium-or-inferentia-instance-on-amazon-ec2-with-the-hf-dlami">How to create an AWS Trainium or Inferentia instance on Amazon EC2 with the HF DLAMI</a> <ol><li><a href="#find-a-supported-region">Find a supported region</a></li> <li><a href="#increase-service-quota">Increase service quota</a></li> <li><a href="#launch-the-amazon-ec2-instance-with-the-hf-dlami">Launch the Amazon EC2 instance with the HF DLAMI</a></li> <li><a href="#connect-through-ssh">Connect through SSH</a></li></ol></li> <li><a href="#set-up-your-remote-development-environment">How to set up your remote development environment</a> <ol><li><a href="#access-through-jupyter-notebook">Access through Jupyter Notebook</a></li> <li><a href="#access-through-vs-code-remote-server">Access through VS Code remote server</a></li></ol></li></ol> <h3 class="relative group"><a id="create-an-aws-trainium-or-inferentia-instance-on-amazon-ec2-with-the-hf-dlami" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-an-aws-trainium-or-inferentia-instance-on-amazon-ec2-with-the-hf-dlami"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Create an AWS Trainium or Inferentia instance on Amazon EC2 with the HF DLAMI</span></h3> <p data-svelte-h="svelte-1y0w0e0">Before creating the EC2 instance, make sure you are in a supported region for the instance you selected and that you have quota in your AWS account.</p> <h4 class="relative group"><a id="find-a-supported-region" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#find-a-supported-region"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Find a supported region</span></h4> <p data-svelte-h="svelte-19mf0fu">Here is the list of regions that support at least one type of Trainium or Inferentia2 instance, as of February 2025:</p> <ul data-svelte-h="svelte-1ur9ox2"><li>us-east-1: US East (N. Virginia)</li> <li>us-east-2: US East (Ohio)</li> <li>us-west-2: US West (Oregon)</li> <li>ap-south-1: Asia Pacific (Mumbai)</li> <li>ap-northeast-1: Asia Pacific (Tokyo)</li> <li>ap-southeast-1: Asia Pacific (Singapore)</li> <li>ap-southeast-2: Asia Pacific (Sydney)</li> <li>ap-southeast-4: Asia Pacific (Melbourne)</li> <li>eu-north-1: Europe (Stockholm)</li> <li>eu-west-3: Europe (Paris)</li> <li>eu-west-2: Europe (London)</li> <li>eu-west-1: Europe (Ireland)</li> <li>eu-central-1: Europe (Frankfurt)</li> <li>sa-east-1: South America (Sao Paulo)</li></ul> <p data-svelte-h="svelte-cd8ooq">Here is a Python script that lets you pull the latest supported instance type in each region you have enabled:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> boto3 | |
| <span class="hljs-keyword">from</span> datetime <span class="hljs-keyword">import</span> datetime | |
| ec2 = boto3.client(<span class="hljs-string">'ec2'</span>) | |
| regions = [region[<span class="hljs-string">'RegionName'</span>] <span class="hljs-keyword">for</span> region <span class="hljs-keyword">in</span> ec2.describe_regions()[<span class="hljs-string">'Regions'</span>]] | |
| <span class="hljs-comment">#Edit this line to change the instance types displayed</span> | |
| instance_types = [<span class="hljs-string">'trn1.32xlarge'</span>, <span class="hljs-string">'trn1.2xlarge'</span>, <span class="hljs-string">'inf2.48xlarge'</span>, <span class="hljs-string">'inf2.24xlarge'</span>, <span class="hljs-string">'inf2.8xlarge'</span>, <span class="hljs-string">'inf2.xlarge'</span>, <span class="hljs-string">'trn2.48xlarge'</span>] | |
| supported_regions = {} | |
| <span class="hljs-keyword">for</span> region <span class="hljs-keyword">in</span> regions: | |
| ec2_region = boto3.client(<span class="hljs-string">'ec2'</span>, region_name=region) | |
| response = ec2_region.describe_instance_type_offerings( | |
| <span class="hljs-comment">#LocationType='availability-zone',</span> | |
| Filters=[ | |
| {<span class="hljs-string">'Name'</span>: <span class="hljs-string">'instance-type'</span>, <span class="hljs-string">'Values'</span>: instance_types}, | |
| ] | |
| ) | |
| <span class="hljs-keyword">if</span> response[<span class="hljs-string">'InstanceTypeOfferings'</span>]: | |
| supported_regions[region] = [offer[<span class="hljs-string">'InstanceType'</span>] <span class="hljs-keyword">for</span> offer <span class="hljs-keyword">in</span> response[<span class="hljs-string">'InstanceTypeOfferings'</span>]] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">'# Supported Regions as of'</span>,datetime.now().strftime(<span class="hljs-string">'%B %d, %Y'</span>)) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">'================'</span>) | |
| client = boto3.client(<span class="hljs-string">'ssm'</span>) | |
| <span class="hljs-keyword">for</span> region, instance_types <span class="hljs-keyword">in</span> supported_regions.items(): | |
| <span class="hljs-keyword">try</span>: | |
| response = client.get_parameter(Name=<span class="hljs-string">f'/aws/service/global-infrastructure/regions/<span class="hljs-subst">{region}</span>/longName'</span>) | |
| region_long_name = response[<span class="hljs-string">'Parameter'</span>][<span class="hljs-string">'Value'</span>] | |
| <span class="hljs-keyword">except</span> (client.exceptions.ParameterNotFound, KeyError): | |
| region_long_name = region | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f' * <span class="hljs-subst">{region}</span>: <span class="hljs-subst">{region_long_name}</span>'</span>) | |
| <span class="hljs-keyword">for</span> instance_type <span class="hljs-keyword">in</span> instance_types: | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f' - <span class="hljs-subst">{instance_type}</span>'</span>) | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">'\n'</span>)<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="increase-service-quota" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#increase-service-quota"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Increase service quota</span></h4> <p data-svelte-h="svelte-mu91xn">Now that you selected your region and that you switched to it, you can request a Service Quota increase through the AWS Console by navigating to Service Quota, AWS services in the left panel, search for Amazon EC2, then “trn” or “inf”. You are able to request quota increase for On-Demand and Spot instances separately.</p> <p data-svelte-h="svelte-1oqd42e">By default, all quotas are 0 for Inferentia and Trainium. There is no charge for increased quotas. There are separate quotas for Inferentia and Trainium, and separate quotas for spot and on-demand. Quotas refer to the maximum TOTAL number of vCPUs assigned to each instance type.</p> <p data-svelte-h="svelte-tmf6ga">For example, a quota of 192 will let you run a single inf2.48xlarge, two inf2.24xlarges, six inf2.8xlarges, or forty-eight of the inf2.xlarges. It will also let you run inf1 instance types. Similarly for Trainium, a quota of 128 will let you run a single trn1n.32xlarge or trn1.32xlarge, but it will also let you run sixteen trn1.2xlarge.</p> <h4 class="relative group"><a id="launch-the-amazon-ec2-instance-with-the-hf-dlami" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#launch-the-amazon-ec2-instance-with-the-hf-dlami"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Launch the Amazon EC2 instance with the HF DLAMI</span></h4> <p data-svelte-h="svelte-r3bcy6">Let’s deploy a trn1.2xlarge instance in the us-east-1 region (North Virginia) through the EC2 console.</p> <p data-svelte-h="svelte-50wzvt">First, click on <strong>Launch instance</strong> and define a name for the instance (<code>trainium-huggingface-demo</code>).</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/01-name-instance.png" alt="name instance"> <p data-svelte-h="svelte-7wwfcf">Next, you search the Amazon Marketplace for Hugging Face AMIs. Entering “Hugging Face” in the search bar for “Application and OS Images” and hitting “enter”.</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/02-search-ami.png" alt="search ami"> <p data-svelte-h="svelte-cgjl8t">This should now open the “Choose an Amazon Machine Image” view with the search. You can now navigate to “AWS Marketplace AMIs” and find the <a href="https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2" rel="nofollow">Hugging Face Neuron Deep Learning AMI</a> and click select.</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/03-select-ami.png" alt="select ami"> <p data-svelte-h="svelte-11azj2p"><em>You will be asked to subscribe if you aren’t. The AMI is completely free of charge, and you will only pay for the EC2 compute.</em></p> <p data-svelte-h="svelte-1j7ptbp">Then you need to define a key pair, which will be used to connect to the instance via <code>ssh</code>. You can create one in place if you don’t have a key pair.</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/04-select-key.png" alt="select ssh key"> <p data-svelte-h="svelte-jwe09q">After that, create or select a <a href="https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html" rel="nofollow">security group</a> which allows <code>ssh</code> traffic.</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/05-select-sg.png" alt="select security group"> <p data-svelte-h="svelte-1hnrw8h">You are ready to launch the instance. Therefore click on “Launch Instance” on the right side.</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/06-launch-instance.png" alt="select ssh key"> <p data-svelte-h="svelte-yyg7kn">AWS will now provision the instance using the <a href="https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2" rel="nofollow">Hugging Face Neuron Deep Learning AMI</a>.</p> <h4 class="relative group"><a id="connect-through-ssh" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#connect-through-ssh"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Connect through SSH</span></h4> <p data-svelte-h="svelte-dpbyx7">Once the instance is ready, you can view and copy the public IPv4 address to <code>ssh</code> into the machine.</p> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/setup_aws_instance/07-copy-dns.png" alt="select public dns"> <p data-svelte-h="svelte-pmoe5">Replace the empty strings <code>""</code> in the snippet below with the IP address of your instances and the path to the key pair you created/selected when launching the instance.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->PUBLIC_DNS=<span class="hljs-string">""</span> <span class="hljs-comment"># IP address</span> | |
| KEY_PATH=<span class="hljs-string">""</span> <span class="hljs-comment"># local path to key pair</span> | |
| ssh -i <span class="hljs-variable">$KEY_PATH</span> ubuntu@<span class="hljs-variable">$PUBLIC_DNS</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jd2fkg">Once you are connected, you can run <code>neuron-ls</code> to ensure you have access to the Trainium accelerators. You should see a similar output than below.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->ubuntu@ip<span class="hljs-number">-172</span><span class="hljs-number">-31</span><span class="hljs-number">-79</span><span class="hljs-number">-164</span><span class="hljs-punctuation">:</span>~$ neuron-ls | |
| instance-type<span class="hljs-punctuation">:</span> trn1<span class="hljs-number">.2</span>xlarge | |
| instance-id<span class="hljs-punctuation">:</span> i<span class="hljs-number">-0570615e41700</span>a481 | |
| +--------+--------+--------+---------+ | |
| | NEURON | NEURON | NEURON | PCI | | |
| | DEVICE | CORES | MEMORY | BDF | | |
| +--------+--------+--------+---------+ | |
| | <span class="hljs-number">0</span> | <span class="hljs-number">2</span> | <span class="hljs-number">32</span> GB | <span class="hljs-number">00</span><span class="hljs-punctuation">:</span><span class="hljs-number">1</span>e<span class="hljs-number">.0</span> | | |
| +--------+--------+--------+---------+<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="set-up-your-remote-development-environment" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#set-up-your-remote-development-environment"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Set up your remote development environment</span></h3> <p data-svelte-h="svelte-va0ciz">We will walk through setting up Jupyter Notebooks or VS Code remote server on the Amazon EC2 instance.</p> <p data-svelte-h="svelte-20tg3z">These two methods require an SSH connection of some kind. These instructions were written for a Mac, but should work on a Linux system as well. A PC may require using Putty.</p> <p data-svelte-h="svelte-14ylm5t">You should have a .pem file that you created when you deployed your instance or had from previous deployments. You can connect to your system using:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->ssh -i <span class="hljs-string">"/path/to/sshkey.pem"</span> ubuntu@instance_ip_address<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="access-through-jupyter-notebook" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#access-through-jupyter-notebook"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Access through Jupyter Notebook</span></h4> <p data-svelte-h="svelte-16s9nvc">This method involves running the Jupyter notebook server on the Neuron instance, mapping a port locally, then using the browser on your desktop to access the notebook server.</p> <p data-svelte-h="svelte-kl6qgs">Start by mapping a port on your local machine to the Neuron instance. From a terminal on your system, run</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->ssh -i <span class="hljs-string">"/path/to/sshkey.pem"</span> -N -f -L localhost:8888:localhost:8888 ubuntu@instance_ip_address<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-l0sx97">Then connect to your Amazon EC2 instance using SSH from your computer. Once connected, from the command prompt, run</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">nohup</span> jupyter notebook --no-browser --port=8888<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1fn6l5y">After a few seconds, check the nohup.out file to find your server’s token:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">cat</span> nohup.out | grep localhost<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ou1jyu">Copy the connection string and paste it into your browser. After a few seconds, you should eventually see the Jupyter Notebook browser. It should look like http://localhost:8888/tree?token=337fc8de2aenot_a_real_tokene952c43946e4fb57131</p> <p data-svelte-h="svelte-1965vx">This process works because you have mapped the 8888 port on your local machine to the 8888 port on the Neuron instance, so when you connect to localhost:8888, you end up accessing the Jupyter server on the Neuron instance.</p> <p data-svelte-h="svelte-165fa3p">If you have problems, make sure the initial port mapping was successful. If you already have something running on port 8888 on your machine, this may give you an error. You can always change the port (e.g. 8885) in all the instructions if you need to.</p> <h4 class="relative group"><a id="access-through-vs-code-remote-server" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#access-through-vs-code-remote-server"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Access through VS Code remote server</span></h4> <p data-svelte-h="svelte-bzwe4m">With Visual Studio Code installed on your local machine, you can use the Remote-SSH command to edit and run files that are stored on a Neuron instance. See the VS Code article for additional details.</p> <ol data-svelte-h="svelte-c41b6w"><li>Select Remote-SSH: Connect to Host… from the Command Palette (F1, ⇧⌘P)</li> <li>Enter in the full connection string from the ssh section above: ssh -i “/path/to/sshkey.pem” ubuntu@instance_ip_address</li> <li>VS Code should connect and automatically set up the VS Code server.</li> <li>Eventually, you should be prompted for a base directory. You can browse to a directory on the Neuron instance.</li> <li>In case you find that some commands seem greyed out in the menus, but the keyboard commands still work (⌘S to save or ^⇧` for terminal), you may need to restart VS Code.</li></ol> <h2 class="relative group"><a id="alternative-manual-installation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#alternative-manual-installation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Alternative: Manual Installation</span></h2> <p data-svelte-h="svelte-1qjg0mm">Manual installation is useful in several scenarios:</p> <ul data-svelte-h="svelte-204tif"><li><strong>Using a newer version</strong>: Install the latest Optimum Neuron version that may not yet be available in the DLAMI</li> <li><strong>Custom AMI requirements</strong>: Working with your organization’s standard AMI or security-hardened images</li> <li><strong>Existing infrastructure</strong>: Adding Neuron support to pre-configured environments or Docker containers</li> <li><strong>Development setup</strong>: Installing pre-release or development versions for testing</li> <li><strong>Minimal installations</strong>: Creating lightweight environments with only required dependencies</li></ul> <p data-svelte-h="svelte-134n78l">If you choose manual installation, you will need to ensure the Neuron drivers and tools are properly installed first.</p> <p data-svelte-h="svelte-1xl7fn1">Before installing <code>optimum-neuron</code> make sure that you have installed the Neuron driver and tools, check out <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/setup/torch-neuronx.html#setup-torch-neuronx" rel="nofollow">more detailed guide here</a>.</p> <h3 class="relative group"><a id="adding-pip-packages-url" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#adding-pip-packages-url"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Adding pip packages URL</span></h3> <p data-svelte-h="svelte-1fuys5u">Pointing to the AWS Neuron repository:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python -m pip config <span class="hljs-built_in">set</span> global.extra-index-url https://pip.repos.neuron.amazonaws.com<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="installing-optimum-neuron-for-aws-trainium--trn1--or-aws-inferentia2--inf2-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#installing-optimum-neuron-for-aws-trainium--trn1--or-aws-inferentia2--inf2-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Installing optimum-neuron for AWS Trainium ( trn1 ) or AWS inferentia2 ( inf2 )</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python -m pip install optimum-neuron[neuronx]<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="installing-optimum-neuron-for-aws-inferentia--inf1-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#installing-optimum-neuron-for-aws-inferentia--inf1-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Installing optimum-neuron for AWS inferentia ( inf1 )</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python -m pip install optimum-neuron[neuron]<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="whats-next" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#whats-next"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What’s Next?</span></h2> <p data-svelte-h="svelte-1yywgqb">Now that you have Optimum Neuron set up, check out the <strong><a href="./quickstart">Quickstart Guide</a></strong> to learn the basics of training and inference with Optimum Neuron.</p> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1v4goo = { | |
| assets: "/docs/optimum.neuron/v0.4.3/en", | |
| base: "/docs/optimum.neuron/v0.4.3/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/optimum.neuron/v0.4.3/en/_app/immutable/entry/start.a52ea6f7.js"), | |
| import("/docs/optimum.neuron/v0.4.3/en/_app/immutable/entry/app.3fdd64a6.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 8], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 56.4 kB
- Xet hash:
- 0c6b04ace9661ef7eded66356ec167318007c6041bcc991059fdbd5a212de555
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.