Upload folder using huggingface_hub

nanochat_tokenizer.py

@@ -32,16 +32,26 @@ class NanochatTokenizer(PreTrainedTokenizer):
                 if os.path.isfile(local_file):
                     vocab_file = local_file
                 else:
+                    # Extract repo ID from cache path if needed
+                    repo_id = self.name_or_path
+                    if "models--" in str(repo_id):
+                        # Cache path format: .../models--namespace--repo_name/snapshots/...
+                        parts = str(repo_id).split("models--")
+                        if len(parts) > 1:
+                            # Get the models--namespace--repo_name part
+                            repo_part = parts[1].split("/")[0]
+                            repo_id = repo_part.replace("--", "/")
+
                     # Download from HF Hub
                     try:
                         vocab_file = hf_hub_download(
-                            repo_id=self.name_or_path,
+                            repo_id=repo_id,
                             filename="tokenizer.pkl",
                             repo_type="model"
                         )
                     except Exception as e:
                         raise ValueError(
-                            f"Could not find or download tokenizer.pkl for {self.name_or_path}: {e}"
+                            f"Could not find or download tokenizer.pkl for {repo_id}: {e}"
                         )
 
         if vocab_file is None or not os.path.isfile(vocab_file):