Initial upload of tiny GPT-2 model

README.md

@@ -1,36 +1,51 @@
-
 ---
 tags:
-- language-model
-- transformer-decoder
-- tiny-shakespeare
+  - language-model
+  - gpt-2
+  - fine-tuned
+  - tiny-shakespeare
 license: mit
 datasets:
-- tiny_shakespeare
-model_description: |
-    This is a small autoregressive language model based on the Transformer architecture trained on the Tiny Shakespeare dataset.
-
-    ## Model Description
-  The model is a custom implementation of a TransformerDecoderModel, which uses a decoder-only architecture similar to GPT-2. 
-  It was trained on the Tiny Shakespeare dataset to generate text in the style of William Shakespeare.
-
-  ## Training Details
-  The model was trained and tracked using [Weights & Biases](https://wandb.ai/honcharova-de-hannover/LanguageModel_Project?nw=nwuserhoncharovade).
-
-  ## How to Use
-  To generate text with this model, you can load it and the tokenizer as follows:
-
-    ```python
-    from transformers import AutoTokenizer
-    from transformers import GPT2LMHeadModel
-
-    # Load the model and tokenizer
-    model = GPT2LMHeadModel.from_pretrained('NataliaH/gpt2-tiny-shakespeare')
-    tokenizer = AutoTokenizer.from_pretrained('NataliaH/gpt2-tiny-shakespeare')
-
-    # Provide input text and generate output
-    input_text = 'To be or not to be'
-    inputs = tokenizer(input_text, return_tensors='pt')
-    outputs = model.generate(**inputs)
-    print(tokenizer.decode(outputs[0], skip_special_tokens=True))
-    ```
+  - tiny_shakespeare
+---
+
+# GPT-2 Tiny Shakespeare Model
+
+This is a small autoregressive language model based on the Transformer architecture trained on the Tiny Shakespeare dataset.
+
+## Model Description
+The model is a custom implementation of a TransformerDecoderModel, which uses a decoder-only architecture similar to GPT-2. It was trained on the Tiny Shakespeare dataset to generate text in the style of William Shakespeare.
+
+## How to Use
+To generate text with this model, you can load it and the tokenizer as follows:
+
+```python
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+
+model = GPT2LMHeadModel.from_pretrained('NataliaH/gpt2-tiny-shakespeare')
+tokenizer = GPT2Tokenizer.from_pretrained('NataliaH/gpt2-tiny-shakespeare')
+
+input_text = 'To be or not to be'
+inputs = tokenizer(input_text, return_tensors='pt')
+outputs = model.generate(**inputs)
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```
+
+## Tags
+- Transformer
+- GPT-2
+- Tiny Shakespeare
+- Language Model
+- Text Generation
+- Autoregressive
+
+## Training Details
+
+- **Epochs**: 3
+- **Batch size**: 4
+- **Learning Rate**: 5e-5
+- **Loss Function**: Cross-Entropy Loss
+- **Optimizer**: AdamW
+
+## License
+This model is licensed under the MIT license.

config.json

@@ -1,39 +1 @@
-{
-  "activation_function": "gelu_new",
-  "architectures": [
-    "GPT2LMHeadModel"
-  ],
-  "attn_pdrop": 0.1,
-  "bos_token_id": 50256,
-  "embd_pdrop": 0.1,
-  "eos_token_id": 50256,
-  "initializer_range": 0.02,
-  "layer_norm_epsilon": 1e-05,
-  "model_type": "gpt2",
-  "n_ctx": 1024,
-  "n_embd": 768,
-  "n_head": 12,
-  "n_inner": null,
-  "n_layer": 12,
-  "n_positions": 1024,
-  "pad_token_id": 50256,
-  "reorder_and_upcast_attn": false,
-  "resid_pdrop": 0.1,
-  "scale_attn_by_inverse_layer_idx": false,
-  "scale_attn_weights": true,
-  "summary_activation": null,
-  "summary_first_dropout": 0.1,
-  "summary_proj_to_labels": true,
-  "summary_type": "cls_index",
-  "summary_use_proj": true,
-  "task_specific_params": {
-    "text-generation": {
-      "do_sample": true,
-      "max_length": 50
-    }
-  },
-  "torch_dtype": "float32",
-  "transformers_version": "4.51.3",
-  "use_cache": true,
-  "vocab_size": 50257
-}
+{"architectures": ["TransformerDecoderModel"], "vocab_size": 50257}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:902374a1f9aef83539bb5392218d589d97ff1ce05f76edb74fbb8914679cb28f
-size 120143250
+oid sha256:4b2922ff630927e80f0e2261ea775429909dd615ce8947dacb59054d75c00b36
+size 497813466

special_tokens_map.json

@@ -1,6 +1,23 @@
 {
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_bos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
     "50256": {
@@ -13,9 +14,10 @@
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
+  "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 1024,
-  "pad_token": "<|endoftext|>",
+  "pad_token": null,
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"
 }