optimum-neuron/v0.4.4/en/_app/immutable/nodes/49.6204c76b.js

import{s as At,n as Gt,o as Bt}from"../chunks/scheduler.56725da7.js";import{S as Ft,i as Lt,e as s,s as a,c as p,q as Vt,h as Qt,a as r,d as l,b as i,f as nt,g as c,j as o,r as xt,k as Et,l as g,m as n,n as u,t as m,o as d,p as y}from"../chunks/index.18a26576.js";import{C as Zt}from"../chunks/CopyLLMTxtMenu.4513c8ed.js";import{C as T}from"../chunks/CodeBlock.58e3e98b.js";import{H as ce}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.049405bf.js";function Xt(ot){let h,ye,ue,Me,U,Te,J,he,b,pt='This tutorial demonstrates how to continuously pre-train the <a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">Llama 3.2 1B</a> model using the Hugging Face <a href="https://huggingface.co/docs/optimum-neuron/index" rel="nofollow">Optimum Neuron</a> library on <a href="https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-hyperpod.html" rel="nofollow">Amazon SageMaker Hyperpod</a>. We leverage several performance optimizations such as tensor parallelism, sequence parallelism, and ZeRO-1 to efficiently train large language models on Trainium-powered instances.',ge,C,ct="One of the key benefits of using SageMaker Hyperpod is the ability to leverage the pre-built Optimum Neuron containers provided by Hugging Face. These containers come with all the necessary libraries and dependencies pre-installed, making it easy to get started with training on AWS Trainium instances.",fe,w,ut="By using the SageMaker pre-built containers, you can avoid the hassle of manually setting up the environment and focus on the core training and fine-tuning tasks. The containers are optimized for performance and include various optimization techniques, such as tensor parallelism and selective checkpointing, to efficiently train large language models like Llama 3.2 1B.",Ue,j,mt="You will learn how to:",Je,v,dt='<li><a href="#continuous-pretraining-of-llama-32-1b-on-sagemaker-hyperpod-with-pre-built-containers">Continuous Pretraining of Llama 3.2 1B on SageMaker Hyperpod with Pre-built Containers</a> <ul><li><a href="#1-setup-aws-environment">1. Setup AWS Environment</a></li> <li><a href="#2-prepare-the-training-environment">2. Prepare the Training Environment</a></li> <li><a href="#3-configure-the-training-job">3. Configure the Training Job</a></li> <li><a href="#4-launch-training-on-sagemaker-hyperpod">4. Launch Training on SageMaker Hyperpod</a></li> <li><a href="#5-monitor-and-validate-training">5. Monitor and Validate Training</a></li></ul></li>',be,$,Ce,R,yt="Before starting this tutorial, you need to set up your AWS environment:",we,M,se,Mt='Create an AWS SageMaker Hyperpod cluster with at least one <code>trn1.32xlarge</code> instance. You can follow the <a href="https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/own-account" rel="nofollow">Hyperpod EKS workshop</a> to set up the cluster.',at,re,Tt='Since Llama 3.2 is a gated model users have to register in Hugging Face and obtain an <a href="https://huggingface.co/docs/hub/en/security-tokens" rel="nofollow">access token</a> before running this example. You will also need to review and accept the license agreement on the <a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">meta-llama/Llama-3.2-1B</a> model page.',it,f,st,oe,ht="aws configure",rt,S,je,I,ve,_,gt="Set up your training environment with the necessary dependencies:",$e,N,Re,W,ft="Login to ECR and pull the <code>huggingface-pytorch-training-neuronx</code> image:",Se,k,Ie,V,Ut="Build and push the Docker image to your ECR registry:",_e,x,Ne,E,Jt="Push the image to your private registry:",We,A,ke,G,Ve,B,bt=`Next, you will generate the script to be used by the pre-training job. Begin by logging into Hugging Face using your access token mentioned in the prerequisite steps.
Modify the <code>generate-jobspec.sh</code> script to include the Hugging Face access token before running it:`,xe,F,Ee,L,Ct="Generate the Kubernetes job specification by executing <code>generate-jobspec.sh</code>. This will create a deployment manifest called <code>llama_train.yaml</code> for the Amazon SageMaker Hyperpod EKS cluster.",Ae,Q,Ge,Z,Be,X,wt="Deploy the training job to your Kubernetes cluster:",Fe,H,Le,Y,jt='The manifest runs the training script on the cluster using torchrun for distributed training. You can explore the complete training script at <a href="https://github.com/huggingface/optimum-neuron/blob/main/examples/language-modeling/run_clm.py" rel="nofollow">run_clm.py</a>.',Qe,z,vt="You will use the following distributed training techniques in this script:",Ze,P,$t="<li>Distributed Training: Uses torchrun with 8 processes per node for efficient multi-device training</li> <li>Model Parallelism: Implements both tensor parallelism (TP=8) and pipeline parallelism (PP=1)</li> <li>Mixed Precision: Utilizes BFloat16 for improved training efficiency</li> <li>Gradient Checkpointing: Enables memory-efficient training</li>",Xe,O,Rt="The manifest runs the following command on the cluster. The environment variables are set when creating the manifest in <code>generate-jobspec.sh</code>.",He,q,Ye,D,St="The training job will now start running on the SageMaker Hyperpod cluster.",ze,K,It="This uses a pre-built script from Optimum-neuron. The script uses the Trainer class from the Optimum Neuron library, which is a specialized version of the Hugging Face Trainer optimized for training on AWS Trainium instances.",Pe,ee,_t="Here’s an overview of the main components in the script:",Oe,te,Nt="<li><p>Model Loading: The model is loaded using <code>AutoModelForCausalLM.from_pretrained()</code> with lazy loading for parallelism.</p></li> <li><p>Data Processing: The dataset is tokenized and processed into chunks suitable for language modeling.</p></li> <li><p>Training Arguments: The script uses <code>NeuronTrainingArguments</code> to configure training hyperparameters, including options for tensor parallelism and pipeline parallelism.</p></li> <li><p>Trainer Setup: A Trainer instance <code>[optimum.neuron.NeuronTrainer]</code> is created with the model, training arguments, datasets, and other necessary components.</p></li> <li><p>Training Loop: The <code>trainer.train()</code> method is called to start the continuous pretraining process.</p></li>",qe,le,De,ne,Wt="You can monitor the progress through Kubernetes logs:",Ke,ae,et,ie,kt="Once the pretraining is complete, you can fine-tune the model for specific tasks using the techniques covered in the previous tutorials. Congrats on pre-training Llama on AWS Trainium!",tt,me,lt;return U=new Zt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),J=new ce({props:{title:"🚀 Continuous Pretraining of Llama 3.2 1B on SageMaker Hyperpod with Pre-built Containers",local:"-continuous-pretraining-of-llama-32-1b-on-sagemaker-hyperpod-with-pre-built-containers",headingTag:"h1"}}),$=new ce({props:{title:"1. Setup AWS Environment",local:"1-setup-aws-environment",headingTag:"h2"}}),S=new T({props:{code:"YXdzJTIwY29uZmlndXJlJTBBQVdTJTIwQWNjZXNzJTIwS2V5JTIwSUQlMjAlNUJOb25lJTVEJTNBJTIwWU9VUl9BQ0NFU1NfS0VZJTBBQVdTJTIwU2VjcmV0JTIwQWNjZXNzJTIwS2V5JTIwJTVCTm9uZSU1RCUzQSUyMFlPVVJfU0VDUkVUX0tFWSUwQURlZmF1bHQlMjByZWdpb24lMjBuYW1lJTIwJTVCTm9uZSU1RCUzQSUyMFlPVVJfUkVHSU9OJTBBRGVmYXVsdCUyMG91dHB1dCUyMGZvcm1hdCUyMCU1Qk5vbmUlNUQlM0ElMjBqc29u",highlighted:`aws configure
AWS Access Key ID [None]: YOUR_ACCESS_KEY
AWS Secret Access Key [None]: YOUR_SECRET_KEY
Default region name [None]: YOUR_REGION
Default output format [None]: json`,wrap:!1}}),I=new ce({props:{title:"2. Prepare the Training Environment",local:"2-prepare-the-training-environment",headingTag:"h2"}}),N=new T({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRm9wdGltdW0tbmV1cm9uLmdpdCUwQW1rZGlyJTIwfiUyRnByZS10cmFpbmluZyUwQWNkJTIwcHJlLXRyYWluaW5nJTBBJTBBY3AlMjAtciUyMC4uJTJGb3B0aW11bS1uZXVyb24lMkZkb2NzJTJGc291cmNlJTJGdHJhaW5pbmdfdHV0b3JpYWxzJTJGYW1hem9uX2VrcyUyMC4lMEFjZCUyMGFtYXpvbl9la3M=",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/huggingface/optimum-neuron.git
<span class="hljs-built_in">mkdir</span> ~/pre-training
<span class="hljs-built_in">cd</span> pre-training

<span class="hljs-built_in">cp</span> -r ../optimum-neuron/docs/source/training_tutorials/amazon_eks .
<span class="hljs-built_in">cd</span> amazon_eks`,wrap:!1}}),k=new T({props:{code:"cmVnaW9uJTNEdXMtZWFzdC0xJTBBZGxjX2FjY291bnRfaWQlM0QqKioqKioqKioqKiolMEFhd3MlMjBlY3IlMjBnZXQtbG9naW4tcGFzc3dvcmQlMjAtLXJlZ2lvbiUyMCUyNHJlZ2lvbiUyMCU3QyUyMGRvY2tlciUyMGxvZ2luJTIwLS11c2VybmFtZSUyMEFXUyUyMC0tcGFzc3dvcmQtc3RkaW4lMjAlMjRkbGNfYWNjb3VudF9pZC5ka3IuZWNyLiUyNHJlZ2lvbi5hbWF6b25hd3MuY29tJTBBJTBBZG9ja2VyJTIwcHVsbCUyMCUyNCU3QmRsY19hY2NvdW50X2lkJTdELmRrci5lY3IuJTI0JTdCcmVnaW9uJTdELmFtYXpvbmF3cy5jb20lMkZodWdnaW5nZmFjZS1weXRvcmNoLXRyYWluaW5nLW5ldXJvbnglM0EyLjEuMi10cmFuc2Zvcm1lcnM0LjQzLjItbmV1cm9ueC1weTMxMC1zZGsyLjIwLjAtdWJ1bnR1MjAuMDQtdjEuMA==",highlighted:`region=us-east-1
dlc_account_id=************
aws ecr get-login-password --region <span class="hljs-variable">$region</span> | docker login --username AWS --password-stdin <span class="hljs-variable">$dlc_account_id</span>.dkr.ecr.<span class="hljs-variable">$region</span>.amazonaws.com

docker pull <span class="hljs-variable">\${dlc_account_id}</span>.dkr.ecr.<span class="hljs-variable">\${region}</span>.amazonaws.com/huggingface-pytorch-training-neuronx:2.1.2-transformers4.43.2-neuronx-py310-sdk2.20.0-ubuntu20.04-v1.0`,wrap:!1}}),x=new T({props:{code:"ZXhwb3J0JTIwQVdTX1JFR0lPTiUzRCUyNChhd3MlMjBlYzIlMjBkZXNjcmliZS1hdmFpbGFiaWxpdHktem9uZXMlMjAtLW91dHB1dCUyMHRleHQlMjAtLXF1ZXJ5JTIwJ0F2YWlsYWJpbGl0eVpvbmVzJTVCMCU1RC4lNUJSZWdpb25OYW1lJTVEJyklMEFleHBvcnQlMjBBQ0NPVU5UJTNEJTI0KGF3cyUyMHN0cyUyMGdldC1jYWxsZXItaWRlbnRpdHklMjAtLXF1ZXJ5JTIwQWNjb3VudCUyMC0tb3V0cHV0JTIwdGV4dCklMEFleHBvcnQlMjBSRUdJU1RSWSUzRCUyNCU3QkFDQ09VTlQlN0QuZGtyLmVjci4lMjQlN0JBV1NfUkVHSU9OJTdELmFtYXpvbmF3cy5jb20lMkYlMEFleHBvcnQlMjBJTUFHRSUzRG9wdGltdW0tbmV1cm9uLWxsYW1hLXByZXRyYWluaW5nJTBBZXhwb3J0JTIwVEFHJTNEJTNBbGF0ZXN0JTBBJTBBZG9ja2VyJTIwYnVpbGQlMjAtdCUyMCUyNCU3QlJFR0lTVFJZJTdEJTI0JTdCSU1BR0UlN0QlMjQlN0JUQUclN0QlMjAu",highlighted:`<span class="hljs-built_in">export</span> AWS_REGION=$(aws ec2 describe-availability-zones --output text --query <span class="hljs-string">&#x27;AvailabilityZones[0].[RegionName]&#x27;</span>)
<span class="hljs-built_in">export</span> ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
<span class="hljs-built_in">export</span> REGISTRY=<span class="hljs-variable">\${ACCOUNT}</span>.dkr.ecr.<span class="hljs-variable">\${AWS_REGION}</span>.amazonaws.com/
<span class="hljs-built_in">export</span> IMAGE=optimum-neuron-llama-pretraining
<span class="hljs-built_in">export</span> TAG=:latest

docker build -t <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span><span class="hljs-variable">\${TAG}</span> .`,wrap:!1}}),A=new T({props:{code:"JTIzJTIwQ3JlYXRlJTIwcmVnaXN0cnklMjBpZiUyMG5lZWRlZCUwQWV4cG9ydCUyMFJFR0lTVFJZX0NPVU5UJTNEJTI0KGF3cyUyMGVjciUyMGRlc2NyaWJlLXJlcG9zaXRvcmllcyUyMCU3QyUyMGdyZXAlMjAlNUMlMjIlMjQlN0JJTUFHRSU3RCU1QyUyMiUyMCU3QyUyMHdjJTIwLWwpJTBBaWYlMjAlNUIlMjAlMjIlMjQlN0JSRUdJU1RSWV9DT1VOVCUyRiUyRiU1QiEwLTklNUQlMkYlN0QlMjIlMjAlM0QlM0QlMjAlMjIwJTIyJTIwJTVEJTNCJTIwdGhlbiUwQSUyMCUyMCUyMGVjaG8lMjAlMjJDcmVhdGluZyUyMHJlcG9zaXRvcnklMjAlMjQlN0JSRUdJU1RSWSU3RCUyNCU3QklNQUdFJTdEJTIwLi4uJTIyJTBBJTIwJTIwJTIwYXdzJTIwZWNyJTIwY3JlYXRlLXJlcG9zaXRvcnklMjAtLXJlcG9zaXRvcnktbmFtZSUyMCUyNCU3QklNQUdFJTdEJTBBZWxzZSUwQSUyMCUyMCUyMGVjaG8lMjAlMjJSZXBvc2l0b3J5JTIwJTI0JTdCUkVHSVNUUlklN0QlMjQlN0JJTUFHRSU3RCUyMGFscmVhZHklMjBleGlzdHMlMjIlMEFmaSUwQSUwQSUyMyUyMExvZ2luJTIwdG8lMjByZWdpc3RyeSUwQWVjaG8lMjAlMjJMb2dnaW5nJTIwaW4lMjB0byUyMCUyNFJFR0lTVFJZJTIwLi4uJTIyJTBBYXdzJTIwZWNyJTIwZ2V0LWxvZ2luLXBhc3N3b3JkJTIwJTdDJTIwZG9ja2VyJTIwbG9naW4lMjAtLXVzZXJuYW1lJTIwQVdTJTIwLS1wYXNzd29yZC1zdGRpbiUyMCUyNFJFR0lTVFJZJTBBJTBBJTIzJTIwUHVzaCUyMGltYWdlJTIwdG8lMjByZWdpc3RyeSUwQWRvY2tlciUyMGltYWdlJTIwcHVzaCUyMCUyNCU3QlJFR0lTVFJZJTdEJTI0JTdCSU1BR0UlN0QlMjQlN0JUQUclN0Q=",highlighted:`<span class="hljs-comment"># Create registry if needed</span>
<span class="hljs-built_in">export</span> REGISTRY_COUNT=$(aws ecr describe-repositories | grep \\&quot;<span class="hljs-variable">\${IMAGE}</span>\\&quot; | <span class="hljs-built_in">wc</span> -l)
<span class="hljs-keyword">if</span> [ <span class="hljs-string">&quot;<span class="hljs-variable">\${REGISTRY_COUNT//[!0-9]/}</span>&quot;</span> == <span class="hljs-string">&quot;0&quot;</span> ]; <span class="hljs-keyword">then</span>
   <span class="hljs-built_in">echo</span> <span class="hljs-string">&quot;Creating repository <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span> ...&quot;</span>
   aws ecr create-repository --repository-name <span class="hljs-variable">\${IMAGE}</span>
<span class="hljs-keyword">else</span>
   <span class="hljs-built_in">echo</span> <span class="hljs-string">&quot;Repository <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span> already exists&quot;</span>
<span class="hljs-keyword">fi</span>

<span class="hljs-comment"># Login to registry</span>
<span class="hljs-built_in">echo</span> <span class="hljs-string">&quot;Logging in to <span class="hljs-variable">$REGISTRY</span> ...&quot;</span>
aws ecr get-login-password | docker login --username AWS --password-stdin <span class="hljs-variable">$REGISTRY</span>

<span class="hljs-comment"># Push image to registry</span>
docker image push <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span><span class="hljs-variable">\${TAG}</span>`,wrap:!1}}),G=new ce({props:{title:"3. Configure the Training Job",local:"3-configure-the-training-job",headingTag:"h2"}}),F=new T({props:{code:"ZXhwb3J0JTIwSEZfQUNDRVNTX1RPS0VOJTNEJTIyJTNDeW91cl9IRl90b2tlbl9oZXJlJTNFJTIy",highlighted:'<span class="hljs-built_in">export</span> HF_ACCESS_TOKEN=<span class="hljs-string">&quot;&lt;your_HF_token_here&gt;&quot;</span>',wrap:!1}}),Q=new T({props:{code:"LiUyRmdlbmVyYXRlLWpvYnNwZWMuc2g=",highlighted:"./generate-jobspec.sh",wrap:!1}}),Z=new ce({props:{title:"4. Launch Training on SageMaker Hyperpod",local:"4-launch-training-on-sagemaker-hyperpod",headingTag:"h2"}}),H=new T({props:{code:"a3ViZWN0bCUyMGFwcGx5JTIwLWYlMjBsbGFtYV90cmFpbi55YW1s",highlighted:"kubectl apply -f llama_train.yaml",wrap:!1}}),q=new T({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jX3Blcl9ub2RlJTNEOCUyMC0tbm5vZGVzJTNEJTI0JTdCTlVNX05PREVTJTdEJTIwcnVuX2NsbS5weSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTNEJTI0JTdCSEZfTU9ERUxfTkFNRSU3RCUwQSUyMCUyMCUyMCUyMC0tdG9rZW4lM0QlMjQlN0JIRl9BQ0NFU1NfVE9LRU4lN0QlMEElMjAlMjAlMjAlMjAtLWRhdGFzZXRfbmFtZSUzRCUyNCU3QkRBVEFTRVRfTkFNRSU3RCUwQSUyMCUyMCUyMCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUzRCUyNCU3QkRBVEFTRVRfQ09ORklHX05BTUUlN0QlMEElMjAlMjAlMjAlMjAtLXN0cmVhbWluZyUzRFRydWUlMEElMjAlMjAlMjAlMjAtLWNhY2hlX2RpciUzRCUyNCU3QlRPS0VOSVpFRF9EQVRBX1BBVEglN0QlMEElMjAlMjAlMjAlMjAtLW51bV90cmFpbl9lcG9jaHMlM0QxJTBBJTIwJTIwJTIwJTIwLS1kb190cmFpbiUwQSUyMCUyMCUyMCUyMC0tbGVhcm5pbmdfcmF0ZSUzRDFlLTQlMEElMjAlMjAlMjAlMjAtLW1heF9zdGVwcyUzRCUyNCU3Qk1BWF9TVEVQUyU3RCUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEJTI0JTdCQkFUQ0hfU0laRSU3RCUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0Q0JTBBJTIwJTIwJTIwJTIwLS1ncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0QxJTBBJTIwJTIwJTIwJTIwLS1ncmFkaWVudF9jaGVja3BvaW50aW5nJTBBJTIwJTIwJTIwJTIwLS1ibG9ja19zaXplJTNENDA5NiUwQSUyMCUyMCUyMCUyMC0tYmYxNiUwQSUyMCUyMCUyMCUyMC0tbWF4X2dyYWRfbm9ybSUzRDEuMCUwQSUyMCUyMCUyMCUyMC0tbHJfc2NoZWR1bGVyX3R5cGUlM0RsaW5lYXIlMEElMjAlMjAlMjAlMjAtLXRlbnNvcl9wYXJhbGxlbF9zaXplJTNEOCUwQSUyMCUyMCUyMCUyMC0tcGlwZWxpbmVfcGFyYWxsZWxfc2l6ZSUzRDElMEElMjAlMjAlMjAlMjAtLWxvZ2dpbmdfc3RlcHMlM0QxJTBBJTIwJTIwJTIwJTIwLS1zYXZlX3RvdGFsX2xpbWl0JTNEMSUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUzRCUyNCU3QkNIRUNLUE9JTlRfRElSJTdEJTBBJTIwJTIwJTIwJTIwLS1vdmVyd3JpdGVfb3V0cHV0X2Rpcg==",highlighted:`torchrun --nproc_per_node=8 --nnodes=<span class="hljs-variable">\${NUM_NODES}</span> run_clm.py \\
    --model_name_or_path=<span class="hljs-variable">\${HF_MODEL_NAME}</span>
    --token=<span class="hljs-variable">\${HF_ACCESS_TOKEN}</span>
    --dataset_name=<span class="hljs-variable">\${DATASET_NAME}</span>
    --dataset_config_name=<span class="hljs-variable">\${DATASET_CONFIG_NAME}</span>
    --streaming=True
    --cache_dir=<span class="hljs-variable">\${TOKENIZED_DATA_PATH}</span>
    --num_train_epochs=1
    --do_train
    --learning_rate=1e-4
    --max_steps=<span class="hljs-variable">\${MAX_STEPS}</span>
    --per_device_train_batch_size=<span class="hljs-variable">\${BATCH_SIZE}</span>
    --per_device_eval_batch_size=4
    --gradient_accumulation_steps=1
    --gradient_checkpointing
    --block_size=4096
    --bf16
    --max_grad_norm=1.0
    --lr_scheduler_type=linear
    --tensor_parallel_size=8
    --pipeline_parallel_size=1
    --logging_steps=1
    --save_total_limit=1
    --output_dir=<span class="hljs-variable">\${CHECKPOINT_DIR}</span>
    --overwrite_output_dir`,wrap:!1}}),le=new ce({props:{title:"5. Monitor and Validate Training",local:"5-monitor-and-validate-training",headingTag:"h2"}}),ae=new T({props:{code:"JTIzJTIwTW9uaXRvciUyMHRyYWluaW5nJTIwbG9ncyUwQWt1YmVjdGwlMjBsb2dzJTIwLWYlMjAtbiUyMGt1YmVmbG93JTIwbGxhbWEtdHJhaW5pbmctZWtzLXdvcmtlci0wJTBBJTBBJTIzJTIwVmFsaWRhdGUlMjBzYXZlZCUyMGNoZWNrcG9pbnRzJTBBa3ViZWN0bCUyMGV4ZWMlMjAtaXQlMjBsbGFtYS10cmFpbmluZy1la3Mtd29ya2VyLTAlMjAtLSUyMGxzJTIwLWwlMjAlMkZmc3glMkZvdXRwdXQ=",highlighted:`<span class="hljs-comment"># Monitor training logs</span>
kubectl logs -f -n kubeflow llama-training-eks-worker-0

<span class="hljs-comment"># Validate saved checkpoints</span>
kubectl <span class="hljs-built_in">exec</span> -it llama-training-eks-worker-0 -- <span class="hljs-built_in">ls</span> -l /fsx/output`,wrap:!1}}),{c(){h=s("meta"),ye=a(),ue=s("p"),Me=a(),p(U.$$.fragment),Te=a(),p(J.$$.fragment),he=a(),b=s("p"),b.innerHTML=pt,ge=a(),C=s("p"),C.textContent=ct,fe=a(),w=s("p"),w.textContent=ut,Ue=a(),j=s("p"),j.textContent=mt,Je=a(),v=s("ul"),v.innerHTML=dt,be=a(),p($.$$.fragment),Ce=a(),R=s("p"),R.textContent=yt,we=a(),M=s("ol"),se=s("li"),se.innerHTML=Mt,at=a(),re=s("li"),re.innerHTML=Tt,it=a(),f=s("li"),st=Vt("Configure your AWS credentials. If you haven’t already set up your AWS credentials, you can do this by installing the AWS CLI and running "),oe=s("code"),oe.textContent=ht,rt=Vt(`. You’ll need to enter your AWS Access Key ID, Secret Access Key, default region, and output format.

	`),p(S.$$.fragment),je=a(),p(I.$$.fragment),ve=a(),_=s("p"),_.textContent=gt,$e=a(),p(N.$$.fragment),Re=a(),W=s("p"),W.innerHTML=ft,Se=a(),p(k.$$.fragment),Ie=a(),V=s("p"),V.textContent=Ut,_e=a(),p(x.$$.fragment),Ne=a(),E=s("p"),E.textContent=Jt,We=a(),p(A.$$.fragment),ke=a(),p(G.$$.fragment),Ve=a(),B=s("p"),B.innerHTML=bt,xe=a(),p(F.$$.fragment),Ee=a(),L=s("p"),L.innerHTML=Ct,Ae=a(),p(Q.$$.fragment),Ge=a(),p(Z.$$.fragment),Be=a(),X=s("p"),X.textContent=wt,Fe=a(),p(H.$$.fragment),Le=a(),Y=s("p"),Y.innerHTML=jt,Qe=a(),z=s("p"),z.textContent=vt,Ze=a(),P=s("ul"),P.innerHTML=$t,Xe=a(),O=s("p"),O.innerHTML=Rt,He=a(),p(q.$$.fragment),Ye=a(),D=s("p"),D.textContent=St,ze=a(),K=s("p"),K.textContent=It,Pe=a(),ee=s("p"),ee.textContent=_t,Oe=a(),te=s("ul"),te.innerHTML=Nt,qe=a(),p(le.$$.fragment),De=a(),ne=s("p"),ne.textContent=Wt,Ke=a(),p(ae.$$.fragment),et=a(),ie=s("p"),ie.textContent=kt,tt=a(),me=s("p"),this.h()},l(e){const t=Qt("svelte-u9bgzb",document.head);h=r(t,"META",{name:!0,content:!0}),t.forEach(l),ye=i(e),ue=r(e,"P",{}),nt(ue).forEach(l),Me=i(e),c(U.$$.fragment,e),Te=i(e),c(J.$$.fragment,e),he=i(e),b=r(e,"P",{"data-svelte-h":!0}),o(b)!=="svelte-umkoc6"&&(b.innerHTML=pt),ge=i(e),C=r(e,"P",{"data-svelte-h":!0}),o(C)!=="svelte-stv42r"&&(C.textContent=ct),fe=i(e),w=r(e,"P",{"data-svelte-h":!0}),o(w)!=="svelte-vecog1"&&(w.textContent=ut),Ue=i(e),j=r(e,"P",{"data-svelte-h":!0}),o(j)!=="svelte-1hahfn0"&&(j.textContent=mt),Je=i(e),v=r(e,"UL",{"data-svelte-h":!0}),o(v)!=="svelte-1rpf5h0"&&(v.innerHTML=dt),be=i(e),c($.$$.fragment,e),Ce=i(e),R=r(e,"P",{"data-svelte-h":!0}),o(R)!=="svelte-y2j4y7"&&(R.textContent=yt),we=i(e),M=r(e,"OL",{});var pe=nt(M);se=r(pe,"LI",{"data-svelte-h":!0}),o(se)!=="svelte-1syvdtd"&&(se.innerHTML=Mt),at=i(pe),re=r(pe,"LI",{"data-svelte-h":!0}),o(re)!=="svelte-11uf0xx"&&(re.innerHTML=Tt),it=i(pe),f=r(pe,"LI",{});var de=nt(f);st=xt(de,"Configure your AWS credentials. If you haven’t already set up your AWS credentials, you can do this by installing the AWS CLI and running "),oe=r(de,"CODE",{"data-svelte-h":!0}),o(oe)!=="svelte-1d1e3tl"&&(oe.textContent=ht),rt=xt(de,`. You’ll need to enter your AWS Access Key ID, Secret Access Key, default region, and output format.

	`),c(S.$$.fragment,de),de.forEach(l),pe.forEach(l),je=i(e),c(I.$$.fragment,e),ve=i(e),_=r(e,"P",{"data-svelte-h":!0}),o(_)!=="svelte-umwl4"&&(_.textContent=gt),$e=i(e),c(N.$$.fragment,e),Re=i(e),W=r(e,"P",{"data-svelte-h":!0}),o(W)!=="svelte-1dk8o0a"&&(W.innerHTML=ft),Se=i(e),c(k.$$.fragment,e),Ie=i(e),V=r(e,"P",{"data-svelte-h":!0}),o(V)!=="svelte-1m60kbs"&&(V.textContent=Ut),_e=i(e),c(x.$$.fragment,e),Ne=i(e),E=r(e,"P",{"data-svelte-h":!0}),o(E)!=="svelte-8aefpq"&&(E.textContent=Jt),We=i(e),c(A.$$.fragment,e),ke=i(e),c(G.$$.fragment,e),Ve=i(e),B=r(e,"P",{"data-svelte-h":!0}),o(B)!=="svelte-nm6jck"&&(B.innerHTML=bt),xe=i(e),c(F.$$.fragment,e),Ee=i(e),L=r(e,"P",{"data-svelte-h":!0}),o(L)!=="svelte-1r8nykb"&&(L.innerHTML=Ct),Ae=i(e),c(Q.$$.fragment,e),Ge=i(e),c(Z.$$.fragment,e),Be=i(e),X=r(e,"P",{"data-svelte-h":!0}),o(X)!=="svelte-1yxf8j9"&&(X.textContent=wt),Fe=i(e),c(H.$$.fragment,e),Le=i(e),Y=r(e,"P",{"data-svelte-h":!0}),o(Y)!=="svelte-15nk6hk"&&(Y.innerHTML=jt),Qe=i(e),z=r(e,"P",{"data-svelte-h":!0}),o(z)!=="svelte-1fh7d8c"&&(z.textContent=vt),Ze=i(e),P=r(e,"UL",{"data-svelte-h":!0}),o(P)!=="svelte-1mkk3sr"&&(P.innerHTML=$t),Xe=i(e),O=r(e,"P",{"data-svelte-h":!0}),o(O)!=="svelte-cy8ews"&&(O.innerHTML=Rt),He=i(e),c(q.$$.fragment,e),Ye=i(e),D=r(e,"P",{"data-svelte-h":!0}),o(D)!=="svelte-6gn68c"&&(D.textContent=St),ze=i(e),K=r(e,"P",{"data-svelte-h":!0}),o(K)!=="svelte-1exj7sy"&&(K.textContent=It),Pe=i(e),ee=r(e,"P",{"data-svelte-h":!0}),o(ee)!=="svelte-1h422fc"&&(ee.textContent=_t),Oe=i(e),te=r(e,"UL",{"data-svelte-h":!0}),o(te)!=="svelte-9cyjwx"&&(te.innerHTML=Nt),qe=i(e),c(le.$$.fragment,e),De=i(e),ne=r(e,"P",{"data-svelte-h":!0}),o(ne)!=="svelte-j415kd"&&(ne.textContent=Wt),Ke=i(e),c(ae.$$.fragment,e),et=i(e),ie=r(e,"P",{"data-svelte-h":!0}),o(ie)!=="svelte-rdd17m"&&(ie.textContent=kt),tt=i(e),me=r(e,"P",{}),nt(me).forEach(l),this.h()},h(){Et(h,"name","hf:doc:metadata"),Et(h,"content",Ht)},m(e,t){g(document.head,h),n(e,ye,t),n(e,ue,t),n(e,Me,t),u(U,e,t),n(e,Te,t),u(J,e,t),n(e,he,t),n(e,b,t),n(e,ge,t),n(e,C,t),n(e,fe,t),n(e,w,t),n(e,Ue,t),n(e,j,t),n(e,Je,t),n(e,v,t),n(e,be,t),u($,e,t),n(e,Ce,t),n(e,R,t),n(e,we,t),n(e,M,t),g(M,se),g(M,at),g(M,re),g(M,it),g(M,f),g(f,st),g(f,oe),g(f,rt),u(S,f,null),n(e,je,t),u(I,e,t),n(e,ve,t),n(e,_,t),n(e,$e,t),u(N,e,t),n(e,Re,t),n(e,W,t),n(e,Se,t),u(k,e,t),n(e,Ie,t),n(e,V,t),n(e,_e,t),u(x,e,t),n(e,Ne,t),n(e,E,t),n(e,We,t),u(A,e,t),n(e,ke,t),u(G,e,t),n(e,Ve,t),n(e,B,t),n(e,xe,t),u(F,e,t),n(e,Ee,t),n(e,L,t),n(e,Ae,t),u(Q,e,t),n(e,Ge,t),u(Z,e,t),n(e,Be,t),n(e,X,t),n(e,Fe,t),u(H,e,t),n(e,Le,t),n(e,Y,t),n(e,Qe,t),n(e,z,t),n(e,Ze,t),n(e,P,t),n(e,Xe,t),n(e,O,t),n(e,He,t),u(q,e,t),n(e,Ye,t),n(e,D,t),n(e,ze,t),n(e,K,t),n(e,Pe,t),n(e,ee,t),n(e,Oe,t),n(e,te,t),n(e,qe,t),u(le,e,t),n(e,De,t),n(e,ne,t),n(e,Ke,t),u(ae,e,t),n(e,et,t),n(e,ie,t),n(e,tt,t),n(e,me,t),lt=!0},p:Gt,i(e){lt||(m(U.$$.fragment,e),m(J.$$.fragment,e),m($.$$.fragment,e),m(S.$$.fragment,e),m(I.$$.fragment,e),m(N.$$.fragment,e),m(k.$$.fragment,e),m(x.$$.fragment,e),m(A.$$.fragment,e),m(G.$$.fragment,e),m(F.$$.fragment,e),m(Q.$$.fragment,e),m(Z.$$.fragment,e),m(H.$$.fragment,e),m(q.$$.fragment,e),m(le.$$.fragment,e),m(ae.$$.fragment,e),lt=!0)},o(e){d(U.$$.fragment,e),d(J.$$.fragment,e),d($.$$.fragment,e),d(S.$$.fragment,e),d(I.$$.fragment,e),d(N.$$.fragment,e),d(k.$$.fragment,e),d(x.$$.fragment,e),d(A.$$.fragment,e),d(G.$$.fragment,e),d(F.$$.fragment,e),d(Q.$$.fragment,e),d(Z.$$.fragment,e),d(H.$$.fragment,e),d(q.$$.fragment,e),d(le.$$.fragment,e),d(ae.$$.fragment,e),lt=!1},d(e){e&&(l(ye),l(ue),l(Me),l(Te),l(he),l(b),l(ge),l(C),l(fe),l(w),l(Ue),l(j),l(Je),l(v),l(be),l(Ce),l(R),l(we),l(M),l(je),l(ve),l(_),l($e),l(Re),l(W),l(Se),l(Ie),l(V),l(_e),l(Ne),l(E),l(We),l(ke),l(Ve),l(B),l(xe),l(Ee),l(L),l(Ae),l(Ge),l(Be),l(X),l(Fe),l(Le),l(Y),l(Qe),l(z),l(Ze),l(P),l(Xe),l(O),l(He),l(Ye),l(D),l(ze),l(K),l(Pe),l(ee),l(Oe),l(te),l(qe),l(De),l(ne),l(Ke),l(et),l(ie),l(tt),l(me)),l(h),y(U,e),y(J,e),y($,e),y(S),y(I,e),y(N,e),y(k,e),y(x,e),y(A,e),y(G,e),y(F,e),y(Q,e),y(Z,e),y(H,e),y(q,e),y(le,e),y(ae,e)}}}const Ht='{"title":"🚀 Continuous Pretraining of Llama 3.2 1B on SageMaker Hyperpod with Pre-built Containers","local":"-continuous-pretraining-of-llama-32-1b-on-sagemaker-hyperpod-with-pre-built-containers","sections":[{"title":"1. Setup AWS Environment","local":"1-setup-aws-environment","sections":[],"depth":2},{"title":"2. Prepare the Training Environment","local":"2-prepare-the-training-environment","sections":[],"depth":2},{"title":"3. Configure the Training Job","local":"3-configure-the-training-job","sections":[],"depth":2},{"title":"4. Launch Training on SageMaker Hyperpod","local":"4-launch-training-on-sagemaker-hyperpod","sections":[],"depth":2},{"title":"5. Monitor and Validate Training","local":"5-monitor-and-validate-training","sections":[],"depth":2}],"depth":1}';function Yt(ot){return Bt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Kt extends Ft{constructor(h){super(),Lt(this,h,Yt,Xt,At,{})}}export{Kt as component};