optimum-neuron/v0.3.0/en/_app/immutable/nodes/47.46ed1d62.js

import{s as At,n as Et,o as xt}from"../chunks/scheduler.85c25b89.js";import{S as Gt,i as Bt,g as s,s as a,r as p,m as Wt,A as Ft,h as r,f as l,c as i,j as tt,u as c,x as o,n as kt,k as Vt,y as g,a as n,v as u,d as m,t as d,w as y}from"../chunks/index.c9bcf812.js";import{C as h}from"../chunks/CodeBlock.c004bd26.js";import{H as pe}from"../chunks/getInferenceSnippets.5ea0a804.js";function Qt(st){let T,de,ce,ye,f,Me,J,rt='This tutorial demonstrates how to continuously pre-train the <a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">Llama 3.2 1B</a> model using the Hugging Face <a href="https://huggingface.co/docs/optimum-neuron/index" rel="nofollow">Optimum Neuron</a> library on <a href="https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-hyperpod.html" rel="nofollow">Amazon SageMaker Hyperpod</a>. We leverage several performance optimizations such as tensor parallelism, sequence parallelism, and ZeRO-1 to efficiently train large language models on Trainium-powered instances.',he,b,ot="One of the key benefits of using SageMaker Hyperpod is the ability to leverage the pre-built Optimum Neuron containers provided by Hugging Face. These containers come with all the necessary libraries and dependencies pre-installed, making it easy to get started with training on AWS Trainium instances.",Te,C,pt="By using the SageMaker pre-built containers, you can avoid the hassle of manually setting up the environment and focus on the core training and fine-tuning tasks. The containers are optimized for performance and include various optimization techniques, such as tensor parallelism and selective checkpointing, to efficiently train large language models like Llama 3.2 1B.",ge,w,ct="You will learn how to:",Ue,j,ut='<li><a href="#continuous-pretraining-of-llama-32-1b-on-sagemaker-hyperpod-with-pre-built-containers">Continuous Pretraining of Llama 3.2 1B on SageMaker Hyperpod with Pre-built Containers</a><ul><li><a href="#1-setup-aws-environment">1. Setup AWS Environment</a></li> <li><a href="#2-prepare-the-training-environment">2. Prepare the Training Environment</a></li> <li><a href="#3-configure-the-training-job">3. Configure the Training Job</a></li> <li><a href="#4-launch-training-on-sagemaker-hyperpod">4. Launch Training on SageMaker Hyperpod</a></li> <li><a href="#5-monitor-and-validate-training">5. Monitor and Validate Training</a></li></ul></li>',fe,v,Je,$,mt="Before starting this tutorial, you need to set up your AWS environment:",be,M,ie,dt='Create an AWS SageMaker Hyperpod cluster with at least one <code>trn1.32xlarge</code> instance. You can follow the <a href="https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/own-account" rel="nofollow">Hyperpod EKS workshop</a> to set up the cluster.',lt,se,yt='Since Llama 3.2 is a gated model users have to register in Hugging Face and obtain an <a href="https://huggingface.co/docs/hub/en/security-tokens" rel="nofollow">access token</a> before running this example. You will also need to review and accept the license agreement on the <a href="https://huggingface.co/meta-llama/Llama-3.2-1B" rel="nofollow">meta-llama/Llama-3.2-1B</a> model page.',nt,U,at,re,Mt="aws configure",it,R,Ce,S,we,I,ht="Set up your training environment with the necessary dependencies:",je,_,ve,N,Tt="Login to ECR and pull the <code>huggingface-pytorch-training-neuronx</code> image:",$e,W,Re,k,gt="Build and push the Docker image to your ECR registry:",Se,V,Ie,A,Ut="Push the image to your private registry:",_e,E,Ne,x,We,G,ft=`Next, you will generate the script to be used by the pre-training job. Begin by logging into Hugging Face using your access token mentioned in the prerequisite steps.
Modify the <code>generate-jobspec.sh</code> script to include the Hugging Face access token before running it:`,ke,B,Ve,F,Jt="Generate the Kubernetes job specification by executing <code>generate-jobspec.sh</code>. This will create a deployment manifest called <code>llama_train.yaml</code> for the Amazon SageMaker Hyperpod EKS cluster.",Ae,Q,Ee,Z,xe,L,bt="Deploy the training job to your Kubernetes cluster:",Ge,X,Be,H,Ct='The manifest runs the training script on the cluster using torchrun for distributed training. You can explore the complete training script at <a href="https://github.com/huggingface/optimum-neuron/blob/main/examples/language-modeling/run_clm.py" rel="nofollow">run_clm.py</a>.',Fe,Y,wt="You will use the following distributed training techniques in this script:",Qe,z,jt="<li>Distributed Training: Uses torchrun with 8 processes per node for efficient multi-device training</li> <li>Model Parallelism: Implements both tensor parallelism (TP=8) and pipeline parallelism (PP=1)</li> <li>Mixed Precision: Utilizes BFloat16 for improved training efficiency</li> <li>Gradient Checkpointing: Enables memory-efficient training</li>",Ze,P,vt="The manifest runs the following command on the cluster. The environment variables are set when creating the manifest in <code>generate-jobspec.sh</code>.",Le,O,Xe,q,$t="The training job will now start running on the SageMaker Hyperpod cluster.",He,D,Rt="This uses a pre-built script from Optimum-neuron. The script uses the Trainer class from the Optimum Neuron library, which is a specialized version of the Hugging Face Trainer optimized for training on AWS Trainium instances.",Ye,K,St="Here’s an overview of the main components in the script:",ze,ee,It="<li><p>Model Loading: The model is loaded using <code>AutoModelForCausalLM.from_pretrained()</code> with lazy loading for parallelism.</p></li> <li><p>Data Processing: The dataset is tokenized and processed into chunks suitable for language modeling.</p></li> <li><p>Training Arguments: The script uses <code>NeuronTrainingArguments</code> to configure training hyperparameters, including options for tensor parallelism and pipeline parallelism.</p></li> <li><p>Trainer Setup: A Trainer instance <code>[optimum.neuron.NeuronTrainer]</code> is created with the model, training arguments, datasets, and other necessary components.</p></li> <li><p>Training Loop: The <code>trainer.train()</code> method is called to start the continuous pretraining process.</p></li>",Pe,te,Oe,le,_t="You can monitor the progress through Kubernetes logs:",qe,ne,De,ae,Nt="Once the pretraining is complete, you can fine-tune the model for specific tasks using the techniques covered in the previous tutorials. Congrats on pre-training Llama on AWS Trainium!",Ke,ue,et;return f=new pe({props:{title:"Continuous Pretraining of Llama 3.2 1B on SageMaker Hyperpod with Pre-built Containers",local:"continuous-pretraining-of-llama-32-1b-on-sagemaker-hyperpod-with-pre-built-containers",headingTag:"h1"}}),v=new pe({props:{title:"1. Setup AWS Environment",local:"1-setup-aws-environment",headingTag:"h2"}}),R=new h({props:{code:"YXdzJTIwY29uZmlndXJlJTBBQVdTJTIwQWNjZXNzJTIwS2V5JTIwSUQlMjAlNUJOb25lJTVEJTNBJTIwWU9VUl9BQ0NFU1NfS0VZJTBBQVdTJTIwU2VjcmV0JTIwQWNjZXNzJTIwS2V5JTIwJTVCTm9uZSU1RCUzQSUyMFlPVVJfU0VDUkVUX0tFWSUwQURlZmF1bHQlMjByZWdpb24lMjBuYW1lJTIwJTVCTm9uZSU1RCUzQSUyMFlPVVJfUkVHSU9OJTBBRGVmYXVsdCUyMG91dHB1dCUyMGZvcm1hdCUyMCU1Qk5vbmUlNUQlM0ElMjBqc29u",highlighted:`aws configure
AWS Access Key ID [None]: YOUR_ACCESS_KEY
AWS Secret Access Key [None]: YOUR_SECRET_KEY
Default region name [None]: YOUR_REGION
Default output format [None]: json`,wrap:!1}}),S=new pe({props:{title:"2. Prepare the Training Environment",local:"2-prepare-the-training-environment",headingTag:"h2"}}),_=new h({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRm9wdGltdW0tbmV1cm9uLmdpdCUwQW1rZGlyJTIwfiUyRnByZS10cmFpbmluZyUwQWNkJTIwcHJlLXRyYWluaW5nJTBBJTBBY3AlMjAtciUyMC4uJTJGb3B0aW11bS1uZXVyb24lMkZkb2NzJTJGc291cmNlJTJGdHJhaW5pbmdfdHV0b3JpYWxzJTJGYW1hem9uX2VrcyUyMC4lMEFjZCUyMGFtYXpvbl9la3M=",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/huggingface/optimum-neuron.git
<span class="hljs-built_in">mkdir</span> ~/pre-training
<span class="hljs-built_in">cd</span> pre-training

<span class="hljs-built_in">cp</span> -r ../optimum-neuron/docs/source/training_tutorials/amazon_eks .
<span class="hljs-built_in">cd</span> amazon_eks`,wrap:!1}}),W=new h({props:{code:"cmVnaW9uJTNEdXMtZWFzdC0xJTBBZGxjX2FjY291bnRfaWQlM0QqKioqKioqKioqKiolMEFhd3MlMjBlY3IlMjBnZXQtbG9naW4tcGFzc3dvcmQlMjAtLXJlZ2lvbiUyMCUyNHJlZ2lvbiUyMCU3QyUyMGRvY2tlciUyMGxvZ2luJTIwLS11c2VybmFtZSUyMEFXUyUyMC0tcGFzc3dvcmQtc3RkaW4lMjAlMjRkbGNfYWNjb3VudF9pZC5ka3IuZWNyLiUyNHJlZ2lvbi5hbWF6b25hd3MuY29tJTBBJTBBZG9ja2VyJTIwcHVsbCUyMCUyNCU3QmRsY19hY2NvdW50X2lkJTdELmRrci5lY3IuJTI0JTdCcmVnaW9uJTdELmFtYXpvbmF3cy5jb20lMkZodWdnaW5nZmFjZS1weXRvcmNoLXRyYWluaW5nLW5ldXJvbnglM0EyLjEuMi10cmFuc2Zvcm1lcnM0LjQzLjItbmV1cm9ueC1weTMxMC1zZGsyLjIwLjAtdWJ1bnR1MjAuMDQtdjEuMA==",highlighted:`region=us-east-1
dlc_account_id=************
aws ecr get-login-password --region <span class="hljs-variable">$region</span> | docker login --username AWS --password-stdin <span class="hljs-variable">$dlc_account_id</span>.dkr.ecr.<span class="hljs-variable">$region</span>.amazonaws.com

docker pull <span class="hljs-variable">\${dlc_account_id}</span>.dkr.ecr.<span class="hljs-variable">\${region}</span>.amazonaws.com/huggingface-pytorch-training-neuronx:2.1.2-transformers4.43.2-neuronx-py310-sdk2.20.0-ubuntu20.04-v1.0`,wrap:!1}}),V=new h({props:{code:"ZXhwb3J0JTIwQVdTX1JFR0lPTiUzRCUyNChhd3MlMjBlYzIlMjBkZXNjcmliZS1hdmFpbGFiaWxpdHktem9uZXMlMjAtLW91dHB1dCUyMHRleHQlMjAtLXF1ZXJ5JTIwJ0F2YWlsYWJpbGl0eVpvbmVzJTVCMCU1RC4lNUJSZWdpb25OYW1lJTVEJyklMEFleHBvcnQlMjBBQ0NPVU5UJTNEJTI0KGF3cyUyMHN0cyUyMGdldC1jYWxsZXItaWRlbnRpdHklMjAtLXF1ZXJ5JTIwQWNjb3VudCUyMC0tb3V0cHV0JTIwdGV4dCklMEFleHBvcnQlMjBSRUdJU1RSWSUzRCUyNCU3QkFDQ09VTlQlN0QuZGtyLmVjci4lMjQlN0JBV1NfUkVHSU9OJTdELmFtYXpvbmF3cy5jb20lMkYlMEFleHBvcnQlMjBJTUFHRSUzRG9wdGltdW0tbmV1cm9uLWxsYW1hLXByZXRyYWluaW5nJTBBZXhwb3J0JTIwVEFHJTNEJTNBbGF0ZXN0JTBBJTBBZG9ja2VyJTIwYnVpbGQlMjAtdCUyMCUyNCU3QlJFR0lTVFJZJTdEJTI0JTdCSU1BR0UlN0QlMjQlN0JUQUclN0QlMjAu",highlighted:`<span class="hljs-built_in">export</span> AWS_REGION=$(aws ec2 describe-availability-zones --output text --query <span class="hljs-string">&#x27;AvailabilityZones[0].[RegionName]&#x27;</span>)
<span class="hljs-built_in">export</span> ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
<span class="hljs-built_in">export</span> REGISTRY=<span class="hljs-variable">\${ACCOUNT}</span>.dkr.ecr.<span class="hljs-variable">\${AWS_REGION}</span>.amazonaws.com/
<span class="hljs-built_in">export</span> IMAGE=optimum-neuron-llama-pretraining
<span class="hljs-built_in">export</span> TAG=:latest

docker build -t <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span><span class="hljs-variable">\${TAG}</span> .`,wrap:!1}}),E=new h({props:{code:"JTIzJTIwQ3JlYXRlJTIwcmVnaXN0cnklMjBpZiUyMG5lZWRlZCUwQWV4cG9ydCUyMFJFR0lTVFJZX0NPVU5UJTNEJTI0KGF3cyUyMGVjciUyMGRlc2NyaWJlLXJlcG9zaXRvcmllcyUyMCU3QyUyMGdyZXAlMjAlNUMlMjIlMjQlN0JJTUFHRSU3RCU1QyUyMiUyMCU3QyUyMHdjJTIwLWwpJTBBaWYlMjAlNUIlMjAlMjIlMjQlN0JSRUdJU1RSWV9DT1VOVCUyRiUyRiU1QiEwLTklNUQlMkYlN0QlMjIlMjAlM0QlM0QlMjAlMjIwJTIyJTIwJTVEJTNCJTIwdGhlbiUwQSUyMCUyMCUyMGVjaG8lMjAlMjJDcmVhdGluZyUyMHJlcG9zaXRvcnklMjAlMjQlN0JSRUdJU1RSWSU3RCUyNCU3QklNQUdFJTdEJTIwLi4uJTIyJTBBJTIwJTIwJTIwYXdzJTIwZWNyJTIwY3JlYXRlLXJlcG9zaXRvcnklMjAtLXJlcG9zaXRvcnktbmFtZSUyMCUyNCU3QklNQUdFJTdEJTBBZWxzZSUwQSUyMCUyMCUyMGVjaG8lMjAlMjJSZXBvc2l0b3J5JTIwJTI0JTdCUkVHSVNUUlklN0QlMjQlN0JJTUFHRSU3RCUyMGFscmVhZHklMjBleGlzdHMlMjIlMEFmaSUwQSUwQSUyMyUyMExvZ2luJTIwdG8lMjByZWdpc3RyeSUwQWVjaG8lMjAlMjJMb2dnaW5nJTIwaW4lMjB0byUyMCUyNFJFR0lTVFJZJTIwLi4uJTIyJTBBYXdzJTIwZWNyJTIwZ2V0LWxvZ2luLXBhc3N3b3JkJTIwJTdDJTIwZG9ja2VyJTIwbG9naW4lMjAtLXVzZXJuYW1lJTIwQVdTJTIwLS1wYXNzd29yZC1zdGRpbiUyMCUyNFJFR0lTVFJZJTBBJTBBJTIzJTIwUHVzaCUyMGltYWdlJTIwdG8lMjByZWdpc3RyeSUwQWRvY2tlciUyMGltYWdlJTIwcHVzaCUyMCUyNCU3QlJFR0lTVFJZJTdEJTI0JTdCSU1BR0UlN0QlMjQlN0JUQUclN0Q=",highlighted:`<span class="hljs-comment"># Create registry if needed</span>
<span class="hljs-built_in">export</span> REGISTRY_COUNT=$(aws ecr describe-repositories | grep \\&quot;<span class="hljs-variable">\${IMAGE}</span>\\&quot; | <span class="hljs-built_in">wc</span> -l)
<span class="hljs-keyword">if</span> [ <span class="hljs-string">&quot;<span class="hljs-variable">\${REGISTRY_COUNT//[!0-9]/}</span>&quot;</span> == <span class="hljs-string">&quot;0&quot;</span> ]; <span class="hljs-keyword">then</span>
   <span class="hljs-built_in">echo</span> <span class="hljs-string">&quot;Creating repository <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span> ...&quot;</span>
   aws ecr create-repository --repository-name <span class="hljs-variable">\${IMAGE}</span>
<span class="hljs-keyword">else</span>
   <span class="hljs-built_in">echo</span> <span class="hljs-string">&quot;Repository <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span> already exists&quot;</span>
<span class="hljs-keyword">fi</span>

<span class="hljs-comment"># Login to registry</span>
<span class="hljs-built_in">echo</span> <span class="hljs-string">&quot;Logging in to <span class="hljs-variable">$REGISTRY</span> ...&quot;</span>
aws ecr get-login-password | docker login --username AWS --password-stdin <span class="hljs-variable">$REGISTRY</span>

<span class="hljs-comment"># Push image to registry</span>
docker image push <span class="hljs-variable">\${REGISTRY}</span><span class="hljs-variable">\${IMAGE}</span><span class="hljs-variable">\${TAG}</span>`,wrap:!1}}),x=new pe({props:{title:"3. Configure the Training Job",local:"3-configure-the-training-job",headingTag:"h2"}}),B=new h({props:{code:"ZXhwb3J0JTIwSEZfQUNDRVNTX1RPS0VOJTNEJTIyJTNDeW91cl9IRl90b2tlbl9oZXJlJTNFJTIy",highlighted:'<span class="hljs-built_in">export</span> HF_ACCESS_TOKEN=<span class="hljs-string">&quot;&lt;your_HF_token_here&gt;&quot;</span>',wrap:!1}}),Q=new h({props:{code:"LiUyRmdlbmVyYXRlLWpvYnNwZWMuc2g=",highlighted:"./generate-jobspec.sh",wrap:!1}}),Z=new pe({props:{title:"4. Launch Training on SageMaker Hyperpod",local:"4-launch-training-on-sagemaker-hyperpod",headingTag:"h2"}}),X=new h({props:{code:"a3ViZWN0bCUyMGFwcGx5JTIwLWYlMjBsbGFtYV90cmFpbi55YW1s",highlighted:"kubectl apply -f llama_train.yaml",wrap:!1}}),O=new h({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jX3Blcl9ub2RlJTNEOCUyMC0tbm5vZGVzJTNEJTI0JTdCTlVNX05PREVTJTdEJTIwcnVuX2NsbS5weSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTNEJTI0JTdCSEZfTU9ERUxfTkFNRSU3RCUwQSUyMCUyMCUyMCUyMC0tdG9rZW4lM0QlMjQlN0JIRl9BQ0NFU1NfVE9LRU4lN0QlMEElMjAlMjAlMjAlMjAtLWRhdGFzZXRfbmFtZSUzRCUyNCU3QkRBVEFTRVRfTkFNRSU3RCUwQSUyMCUyMCUyMCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUzRCUyNCU3QkRBVEFTRVRfQ09ORklHX05BTUUlN0QlMEElMjAlMjAlMjAlMjAtLXN0cmVhbWluZyUzRFRydWUlMEElMjAlMjAlMjAlMjAtLWNhY2hlX2RpciUzRCUyNCU3QlRPS0VOSVpFRF9EQVRBX1BBVEglN0QlMEElMjAlMjAlMjAlMjAtLW51bV90cmFpbl9lcG9jaHMlM0QxJTBBJTIwJTIwJTIwJTIwLS1kb190cmFpbiUwQSUyMCUyMCUyMCUyMC0tbGVhcm5pbmdfcmF0ZSUzRDFlLTQlMEElMjAlMjAlMjAlMjAtLW1heF9zdGVwcyUzRCUyNCU3Qk1BWF9TVEVQUyU3RCUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEJTI0JTdCQkFUQ0hfU0laRSU3RCUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0Q0JTBBJTIwJTIwJTIwJTIwLS1ncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlM0QxJTBBJTIwJTIwJTIwJTIwLS1ncmFkaWVudF9jaGVja3BvaW50aW5nJTBBJTIwJTIwJTIwJTIwLS1ibG9ja19zaXplJTNENDA5NiUwQSUyMCUyMCUyMCUyMC0tYmYxNiUwQSUyMCUyMCUyMCUyMC0tbWF4X2dyYWRfbm9ybSUzRDEuMCUwQSUyMCUyMCUyMCUyMC0tbHJfc2NoZWR1bGVyX3R5cGUlM0RsaW5lYXIlMEElMjAlMjAlMjAlMjAtLXRlbnNvcl9wYXJhbGxlbF9zaXplJTNEOCUwQSUyMCUyMCUyMCUyMC0tcGlwZWxpbmVfcGFyYWxsZWxfc2l6ZSUzRDElMEElMjAlMjAlMjAlMjAtLWxvZ2dpbmdfc3RlcHMlM0QxJTBBJTIwJTIwJTIwJTIwLS1zYXZlX3RvdGFsX2xpbWl0JTNEMSUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUzRCUyNCU3QkNIRUNLUE9JTlRfRElSJTdEJTBBJTIwJTIwJTIwJTIwLS1vdmVyd3JpdGVfb3V0cHV0X2Rpcg==",highlighted:`torchrun --nproc_per_node=8 --nnodes=<span class="hljs-variable">\${NUM_NODES}</span> run_clm.py \\
    --model_name_or_path=<span class="hljs-variable">\${HF_MODEL_NAME}</span>
    --token=<span class="hljs-variable">\${HF_ACCESS_TOKEN}</span>
    --dataset_name=<span class="hljs-variable">\${DATASET_NAME}</span>
    --dataset_config_name=<span class="hljs-variable">\${DATASET_CONFIG_NAME}</span>
    --streaming=True
    --cache_dir=<span class="hljs-variable">\${TOKENIZED_DATA_PATH}</span>
    --num_train_epochs=1
    --do_train
    --learning_rate=1e-4
    --max_steps=<span class="hljs-variable">\${MAX_STEPS}</span>
    --per_device_train_batch_size=<span class="hljs-variable">\${BATCH_SIZE}</span>
    --per_device_eval_batch_size=4
    --gradient_accumulation_steps=1
    --gradient_checkpointing
    --block_size=4096
    --bf16
    --max_grad_norm=1.0
    --lr_scheduler_type=linear
    --tensor_parallel_size=8
    --pipeline_parallel_size=1
    --logging_steps=1
    --save_total_limit=1
    --output_dir=<span class="hljs-variable">\${CHECKPOINT_DIR}</span>
    --overwrite_output_dir`,wrap:!1}}),te=new pe({props:{title:"5. Monitor and Validate Training",local:"5-monitor-and-validate-training",headingTag:"h2"}}),ne=new h({props:{code:"JTIzJTIwTW9uaXRvciUyMHRyYWluaW5nJTIwbG9ncyUwQWt1YmVjdGwlMjBsb2dzJTIwLWYlMjAtbiUyMGt1YmVmbG93JTIwbGxhbWEtdHJhaW5pbmctZWtzLXdvcmtlci0wJTBBJTBBJTIzJTIwVmFsaWRhdGUlMjBzYXZlZCUyMGNoZWNrcG9pbnRzJTBBa3ViZWN0bCUyMGV4ZWMlMjAtaXQlMjBsbGFtYS10cmFpbmluZy1la3Mtd29ya2VyLTAlMjAtLSUyMGxzJTIwLWwlMjAlMkZmc3glMkZvdXRwdXQ=",highlighted:`<span class="hljs-comment"># Monitor training logs</span>
kubectl logs -f -n kubeflow llama-training-eks-worker-0

<span class="hljs-comment"># Validate saved checkpoints</span>
kubectl <span class="hljs-built_in">exec</span> -it llama-training-eks-worker-0 -- <span class="hljs-built_in">ls</span> -l /fsx/output`,wrap:!1}}),{c(){T=s("meta"),de=a(),ce=s("p"),ye=a(),p(f.$$.fragment),Me=a(),J=s("p"),J.innerHTML=rt,he=a(),b=s("p"),b.textContent=ot,Te=a(),C=s("p"),C.textContent=pt,ge=a(),w=s("p"),w.textContent=ct,Ue=a(),j=s("ul"),j.innerHTML=ut,fe=a(),p(v.$$.fragment),Je=a(),$=s("p"),$.textContent=mt,be=a(),M=s("ol"),ie=s("li"),ie.innerHTML=dt,lt=a(),se=s("li"),se.innerHTML=yt,nt=a(),U=s("li"),at=Wt("Configure your AWS credentials. If you haven’t already set up your AWS credentials, you can do this by installing the AWS CLI and running "),re=s("code"),re.textContent=Mt,it=Wt(`. You’ll need to enter your AWS Access Key ID, Secret Access Key, default region, and output format. 
	`),p(R.$$.fragment),Ce=a(),p(S.$$.fragment),we=a(),I=s("p"),I.textContent=ht,je=a(),p(_.$$.fragment),ve=a(),N=s("p"),N.innerHTML=Tt,$e=a(),p(W.$$.fragment),Re=a(),k=s("p"),k.textContent=gt,Se=a(),p(V.$$.fragment),Ie=a(),A=s("p"),A.textContent=Ut,_e=a(),p(E.$$.fragment),Ne=a(),p(x.$$.fragment),We=a(),G=s("p"),G.innerHTML=ft,ke=a(),p(B.$$.fragment),Ve=a(),F=s("p"),F.innerHTML=Jt,Ae=a(),p(Q.$$.fragment),Ee=a(),p(Z.$$.fragment),xe=a(),L=s("p"),L.textContent=bt,Ge=a(),p(X.$$.fragment),Be=a(),H=s("p"),H.innerHTML=Ct,Fe=a(),Y=s("p"),Y.textContent=wt,Qe=a(),z=s("ul"),z.innerHTML=jt,Ze=a(),P=s("p"),P.innerHTML=vt,Le=a(),p(O.$$.fragment),Xe=a(),q=s("p"),q.textContent=$t,He=a(),D=s("p"),D.textContent=Rt,Ye=a(),K=s("p"),K.textContent=St,ze=a(),ee=s("ul"),ee.innerHTML=It,Pe=a(),p(te.$$.fragment),Oe=a(),le=s("p"),le.textContent=_t,qe=a(),p(ne.$$.fragment),De=a(),ae=s("p"),ae.textContent=Nt,Ke=a(),ue=s("p"),this.h()},l(e){const t=Ft("svelte-u9bgzb",document.head);T=r(t,"META",{name:!0,content:!0}),t.forEach(l),de=i(e),ce=r(e,"P",{}),tt(ce).forEach(l),ye=i(e),c(f.$$.fragment,e),Me=i(e),J=r(e,"P",{"data-svelte-h":!0}),o(J)!=="svelte-umkoc6"&&(J.innerHTML=rt),he=i(e),b=r(e,"P",{"data-svelte-h":!0}),o(b)!=="svelte-stv42r"&&(b.textContent=ot),Te=i(e),C=r(e,"P",{"data-svelte-h":!0}),o(C)!=="svelte-vecog1"&&(C.textContent=pt),ge=i(e),w=r(e,"P",{"data-svelte-h":!0}),o(w)!=="svelte-1hahfn0"&&(w.textContent=ct),Ue=i(e),j=r(e,"UL",{"data-svelte-h":!0}),o(j)!=="svelte-1pgq9r8"&&(j.innerHTML=ut),fe=i(e),c(v.$$.fragment,e),Je=i(e),$=r(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-y2j4y7"&&($.textContent=mt),be=i(e),M=r(e,"OL",{});var oe=tt(M);ie=r(oe,"LI",{"data-svelte-h":!0}),o(ie)!=="svelte-1syvdtd"&&(ie.innerHTML=dt),lt=i(oe),se=r(oe,"LI",{"data-svelte-h":!0}),o(se)!=="svelte-11uf0xx"&&(se.innerHTML=yt),nt=i(oe),U=r(oe,"LI",{});var me=tt(U);at=kt(me,"Configure your AWS credentials. If you haven’t already set up your AWS credentials, you can do this by installing the AWS CLI and running "),re=r(me,"CODE",{"data-svelte-h":!0}),o(re)!=="svelte-1d1e3tl"&&(re.textContent=Mt),it=kt(me,`. You’ll need to enter your AWS Access Key ID, Secret Access Key, default region, and output format. 
	`),c(R.$$.fragment,me),me.forEach(l),oe.forEach(l),Ce=i(e),c(S.$$.fragment,e),we=i(e),I=r(e,"P",{"data-svelte-h":!0}),o(I)!=="svelte-umwl4"&&(I.textContent=ht),je=i(e),c(_.$$.fragment,e),ve=i(e),N=r(e,"P",{"data-svelte-h":!0}),o(N)!=="svelte-1dk8o0a"&&(N.innerHTML=Tt),$e=i(e),c(W.$$.fragment,e),Re=i(e),k=r(e,"P",{"data-svelte-h":!0}),o(k)!=="svelte-1m60kbs"&&(k.textContent=gt),Se=i(e),c(V.$$.fragment,e),Ie=i(e),A=r(e,"P",{"data-svelte-h":!0}),o(A)!=="svelte-8aefpq"&&(A.textContent=Ut),_e=i(e),c(E.$$.fragment,e),Ne=i(e),c(x.$$.fragment,e),We=i(e),G=r(e,"P",{"data-svelte-h":!0}),o(G)!=="svelte-nm6jck"&&(G.innerHTML=ft),ke=i(e),c(B.$$.fragment,e),Ve=i(e),F=r(e,"P",{"data-svelte-h":!0}),o(F)!=="svelte-1r8nykb"&&(F.innerHTML=Jt),Ae=i(e),c(Q.$$.fragment,e),Ee=i(e),c(Z.$$.fragment,e),xe=i(e),L=r(e,"P",{"data-svelte-h":!0}),o(L)!=="svelte-1yxf8j9"&&(L.textContent=bt),Ge=i(e),c(X.$$.fragment,e),Be=i(e),H=r(e,"P",{"data-svelte-h":!0}),o(H)!=="svelte-15nk6hk"&&(H.innerHTML=Ct),Fe=i(e),Y=r(e,"P",{"data-svelte-h":!0}),o(Y)!=="svelte-1fh7d8c"&&(Y.textContent=wt),Qe=i(e),z=r(e,"UL",{"data-svelte-h":!0}),o(z)!=="svelte-1mkk3sr"&&(z.innerHTML=jt),Ze=i(e),P=r(e,"P",{"data-svelte-h":!0}),o(P)!=="svelte-6h6qxc"&&(P.innerHTML=vt),Le=i(e),c(O.$$.fragment,e),Xe=i(e),q=r(e,"P",{"data-svelte-h":!0}),o(q)!=="svelte-1w6rwnk"&&(q.textContent=$t),He=i(e),D=r(e,"P",{"data-svelte-h":!0}),o(D)!=="svelte-1exj7sy"&&(D.textContent=Rt),Ye=i(e),K=r(e,"P",{"data-svelte-h":!0}),o(K)!=="svelte-1h422fc"&&(K.textContent=St),ze=i(e),ee=r(e,"UL",{"data-svelte-h":!0}),o(ee)!=="svelte-9cyjwx"&&(ee.innerHTML=It),Pe=i(e),c(te.$$.fragment,e),Oe=i(e),le=r(e,"P",{"data-svelte-h":!0}),o(le)!=="svelte-j415kd"&&(le.textContent=_t),qe=i(e),c(ne.$$.fragment,e),De=i(e),ae=r(e,"P",{"data-svelte-h":!0}),o(ae)!=="svelte-rdd17m"&&(ae.textContent=Nt),Ke=i(e),ue=r(e,"P",{}),tt(ue).forEach(l),this.h()},h(){Vt(T,"name","hf:doc:metadata"),Vt(T,"content",Zt)},m(e,t){g(document.head,T),n(e,de,t),n(e,ce,t),n(e,ye,t),u(f,e,t),n(e,Me,t),n(e,J,t),n(e,he,t),n(e,b,t),n(e,Te,t),n(e,C,t),n(e,ge,t),n(e,w,t),n(e,Ue,t),n(e,j,t),n(e,fe,t),u(v,e,t),n(e,Je,t),n(e,$,t),n(e,be,t),n(e,M,t),g(M,ie),g(M,lt),g(M,se),g(M,nt),g(M,U),g(U,at),g(U,re),g(U,it),u(R,U,null),n(e,Ce,t),u(S,e,t),n(e,we,t),n(e,I,t),n(e,je,t),u(_,e,t),n(e,ve,t),n(e,N,t),n(e,$e,t),u(W,e,t),n(e,Re,t),n(e,k,t),n(e,Se,t),u(V,e,t),n(e,Ie,t),n(e,A,t),n(e,_e,t),u(E,e,t),n(e,Ne,t),u(x,e,t),n(e,We,t),n(e,G,t),n(e,ke,t),u(B,e,t),n(e,Ve,t),n(e,F,t),n(e,Ae,t),u(Q,e,t),n(e,Ee,t),u(Z,e,t),n(e,xe,t),n(e,L,t),n(e,Ge,t),u(X,e,t),n(e,Be,t),n(e,H,t),n(e,Fe,t),n(e,Y,t),n(e,Qe,t),n(e,z,t),n(e,Ze,t),n(e,P,t),n(e,Le,t),u(O,e,t),n(e,Xe,t),n(e,q,t),n(e,He,t),n(e,D,t),n(e,Ye,t),n(e,K,t),n(e,ze,t),n(e,ee,t),n(e,Pe,t),u(te,e,t),n(e,Oe,t),n(e,le,t),n(e,qe,t),u(ne,e,t),n(e,De,t),n(e,ae,t),n(e,Ke,t),n(e,ue,t),et=!0},p:Et,i(e){et||(m(f.$$.fragment,e),m(v.$$.fragment,e),m(R.$$.fragment,e),m(S.$$.fragment,e),m(_.$$.fragment,e),m(W.$$.fragment,e),m(V.$$.fragment,e),m(E.$$.fragment,e),m(x.$$.fragment,e),m(B.$$.fragment,e),m(Q.$$.fragment,e),m(Z.$$.fragment,e),m(X.$$.fragment,e),m(O.$$.fragment,e),m(te.$$.fragment,e),m(ne.$$.fragment,e),et=!0)},o(e){d(f.$$.fragment,e),d(v.$$.fragment,e),d(R.$$.fragment,e),d(S.$$.fragment,e),d(_.$$.fragment,e),d(W.$$.fragment,e),d(V.$$.fragment,e),d(E.$$.fragment,e),d(x.$$.fragment,e),d(B.$$.fragment,e),d(Q.$$.fragment,e),d(Z.$$.fragment,e),d(X.$$.fragment,e),d(O.$$.fragment,e),d(te.$$.fragment,e),d(ne.$$.fragment,e),et=!1},d(e){e&&(l(de),l(ce),l(ye),l(Me),l(J),l(he),l(b),l(Te),l(C),l(ge),l(w),l(Ue),l(j),l(fe),l(Je),l($),l(be),l(M),l(Ce),l(we),l(I),l(je),l(ve),l(N),l($e),l(Re),l(k),l(Se),l(Ie),l(A),l(_e),l(Ne),l(We),l(G),l(ke),l(Ve),l(F),l(Ae),l(Ee),l(xe),l(L),l(Ge),l(Be),l(H),l(Fe),l(Y),l(Qe),l(z),l(Ze),l(P),l(Le),l(Xe),l(q),l(He),l(D),l(Ye),l(K),l(ze),l(ee),l(Pe),l(Oe),l(le),l(qe),l(De),l(ae),l(Ke),l(ue)),l(T),y(f,e),y(v,e),y(R),y(S,e),y(_,e),y(W,e),y(V,e),y(E,e),y(x,e),y(B,e),y(Q,e),y(Z,e),y(X,e),y(O,e),y(te,e),y(ne,e)}}}const Zt='{"title":"Continuous Pretraining of Llama 3.2 1B on SageMaker Hyperpod with Pre-built Containers","local":"continuous-pretraining-of-llama-32-1b-on-sagemaker-hyperpod-with-pre-built-containers","sections":[{"title":"1. Setup AWS Environment","local":"1-setup-aws-environment","sections":[],"depth":2},{"title":"2. Prepare the Training Environment","local":"2-prepare-the-training-environment","sections":[],"depth":2},{"title":"3. Configure the Training Job","local":"3-configure-the-training-job","sections":[],"depth":2},{"title":"4. Launch Training on SageMaker Hyperpod","local":"4-launch-training-on-sagemaker-hyperpod","sections":[],"depth":2},{"title":"5. Monitor and Validate Training","local":"5-monitor-and-validate-training","sections":[],"depth":2}],"depth":1}';function Lt(st){return xt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends Gt{constructor(T){super(),Bt(this,T,Lt,Qt,At,{})}}export{Pt as component};