Upload 2 files

app.py

@@ -0,0 +1,162 @@
+import torch
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+import gradio as gr
+import pandas as pd
+from collections import Counter, defaultdict
+import os
+from huggingface_hub import login
+
+# Get the token from the environment variable
+api_token = os.getenv('HF_TOKEN')
+
+# Load pre-trained model and tokenizer
+model_name = "gpt2"
+model = GPT2LMHeadModel.from_pretrained(model_name)
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+model.eval()
+
+def create_ngrams(tokens, n):
+    return [tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1)]
+
+def calculate_probabilities(four_gram_counts, three_gram_counts):
+    probabilities = defaultdict(lambda: defaultdict(float))
+    for four_gram, count in four_gram_counts.items():
+        three_gram = four_gram[:-1]
+        probabilities[three_gram][four_gram[-1]] = count / three_gram_counts[three_gram]
+    return probabilities
+
+def kneser_ney_smoothing(ngram_counts, lower_order_counts, discount=0.75):
+    continuation_counts = Counter()
+    lower_counts = Counter()
+
+    for ngram in ngram_counts:
+        lower_counts[ngram[1:]] += 1
+        continuation_counts[ngram[1:]] += 1
+
+    def continuation_probability(word):
+        return continuation_counts[word] / sum(continuation_counts.values())
+
+    probabilities = defaultdict(lambda: defaultdict(float))
+    for ngram, count in ngram_counts.items():
+        lower_ngram = ngram[:-1]
+        discounted_count = max(count - discount, 0)
+        lambda_factor = (discount / lower_order_counts[lower_ngram]) * len(continuation_counts)
+        probabilities[lower_ngram][ngram[-1]] = (discounted_count / lower_order_counts[lower_ngram]) + lambda_factor * continuation_probability(ngram[-1])
+
+    return probabilities
+
+def generate_text_with_probs(initial_context, top_p, max_length, top_k):
+    input_ids = tokenizer.encode(initial_context, return_tensors="pt")
+    generated_text = initial_context
+    token_tables = []
+
+    with torch.no_grad():
+        for _ in range(max_length):
+            outputs = model(input_ids=input_ids)
+            next_token_logits = outputs.logits[:, -1, :]
+
+            # Apply top-p (nucleus) sampling
+            sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
+            cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
+            sorted_indices_to_remove = cumulative_probs > top_p
+            sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+            sorted_indices_to_remove[..., 0] = 0
+
+            # Convert boolean mask to indices to set logits to -inf
+            indices_to_remove = sorted_indices[sorted_indices_to_remove]
+            next_token_logits[:, indices_to_remove] = -float('Inf')
+
+            # Compute probabilities
+            probabilities = torch.softmax(next_token_logits, dim=-1)
+
+            # Get the next token using multinomial sampling
+            next_token = torch.multinomial(probabilities, num_samples=1)
+
+            # Get next token and its probability
+            next_token_prob = probabilities[0, next_token].item()
+            next_token_text = tokenizer.decode(next_token.item())
+
+            # Get top tokens and their probabilities
+            top_tokens = sorted_indices[0, :top_k]  # Get top k tokens
+            top_probs = probabilities[0, top_tokens]
+            top_token_probs = [(tokenizer.decode([token.item()]), prob.item()) for token, prob in zip(top_tokens, top_probs)]
+
+            # Create DataFrame for current token's top-k probabilities
+            df = pd.DataFrame(top_token_probs, columns=["Token", "Probability"])
+            df.index = df.index + 1  # Add numbering to the DataFrame
+            token_tables.append((f"Next token: {next_token_text} (Probability: {next_token_prob:.4f})", df))
+
+            # Add the next token to the input_ids
+            input_ids = torch.cat([input_ids, next_token], dim=-1)
+
+            if next_token.item() == tokenizer.eos_token_id:
+                break
+
+    # Decode the generated text
+    generated_text = tokenizer.decode(input_ids[0], skip_special_tokens=True)
+
+    return generated_text, token_tables
+
+def predict_next_token_ngram(input_text, context_text, max_length):
+    context_tokens = tokenizer.tokenize(context_text)
+    four_grams = create_ngrams(context_tokens, 4)
+    four_gram_counts = Counter(four_grams)
+    three_gram_counts = Counter([gram[:-1] for gram in four_grams])
+    probabilities = calculate_probabilities(four_gram_counts, three_gram_counts)
+    probs = kneser_ney_smoothing(four_gram_counts, three_gram_counts)
+
+    input_tokens = tokenizer.tokenize(input_text)
+    generated_text = input_text
+    token_tables = []
+
+    if len(input_tokens) >= max_length:
+        generated_text = tokenizer.convert_tokens_to_string(input_tokens)
+        return generated_text, token_tables
+
+    while len(input_tokens) < max_length:
+        input_3_gram = tuple(input_tokens[-3:])
+        next_token_probs = probs.get(input_3_gram, {})
+        if not next_token_probs:
+            break
+        next_token = max(next_token_probs, key=next_token_probs.get)
+        input_tokens.append(next_token)
+
+        # Get top tokens and their probabilities
+        top_k = 4
+        top_k_tokens = sorted(next_token_probs.items(), key=lambda x: x[1], reverse=True)[:top_k]
+        top_k_tokens_df = pd.DataFrame(top_k_tokens, columns=["Token", "Probability"])
+        top_k_tokens_df.index = top_k_tokens_df.index + 1  # Add numbering to the DataFrame
+        top_k_tokens_df["Token"] = top_k_tokens_df["Token"].apply(lambda x: tokenizer.convert_tokens_to_string([x]))
+
+        token_tables.append((f"Next token: {next_token} (Predicted)", top_k_tokens_df))
+
+    generated_text = tokenizer.convert_tokens_to_string(input_tokens)
+    return generated_text, token_tables
+
+def combined_model_predictions(context_text, initial_context, top_p, max_length, top_k):
+    generated_text, token_tables = generate_text_with_probs(initial_context, top_p, max_length, top_k)
+    ngram_generated_text, ngram_token_tables = predict_next_token_ngram(initial_context, context_text, max_length)
+
+    return generated_text, token_tables, ngram_generated_text, ngram_token_tables
+
+iface = gr.Interface(
+    fn=combined_model_predictions,
+    inputs=[
+        gr.Textbox(lines=4, placeholder="Enter context for N-gram model..."),
+        gr.Textbox(lines=2, placeholder="Enter initial context here..."),
+        gr.Slider(0, 1, step=0.01, value=0.9, label="Top-p (nucleus) sampling"),
+        gr.Slider(1, 100, step=1, value=50, label="Max length"),
+        gr.Slider(1, 50, step=1, value=10, label="Top-k"),  # Added Top-k slider
+    ],
+    outputs=[
+        gr.Textbox(label="Generated Text"),
+        gr.Dataframe(label="LLM Token Probabilities"),  # Display DataFrame as output
+        gr.Textbox(label="N-gram Generated Text"),
+        gr.Dataframe(label="N-gram Token Predictions"),  # Display N-gram model predictions
+    ],
+    title="Next Token Visualizer (GPT-2 - 124M param.)",
+    description="Generate text using GPT-2 with top-p (nucleus) sampling and see the probabilities of generated tokens in tables, along with N-gram model predictions.",
+)
+
+# Launch the Gradio app
+iface.launch()

requirements.txt

@@ -0,0 +1,207 @@
+absl-py==2.1.0
+aiofiles==23.2.1
+annotated-types==0.5.0
+anyio @ file:///home/conda/feedstock_root/build_artifacts/anyio_1688651106312/work/dist
+appnope @ file:///home/conda/feedstock_root/build_artifacts/appnope_1649077682618/work
+argon2-cffi @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi_1640817743617/work
+argon2-cffi-bindings @ file:///Users/runner/miniforge3/conda-bld/argon2-cffi-bindings_1666850912616/work
+arrow @ file:///home/conda/feedstock_root/build_artifacts/arrow_1662382474514/work
+asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1670263926556/work
+async-lru @ file:///home/conda/feedstock_root/build_artifacts/async-lru_1690563019058/work
+attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1683424013410/work
+Babel @ file:///home/conda/feedstock_root/build_artifacts/babel_1677767029043/work
+backcall @ file:///home/conda/feedstock_root/build_artifacts/backcall_1592338393461/work
+backports.functools-lru-cache @ file:///home/conda/feedstock_root/build_artifacts/backports.functools_lru_cache_1687772187254/work
+beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1680888073205/work
+bitsandbytes==0.42.0
+bleach @ file:///home/conda/feedstock_root/build_artifacts/bleach_1674535352125/work
+blis==0.7.10
+Brotli @ file:///Users/runner/miniforge3/conda-bld/brotli-split_1687884292427/work
+cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work
+catalogue==2.0.9
+certifi==2023.7.22
+cffi @ file:///Users/runner/miniforge3/conda-bld/cffi_1671179605388/work
+charset-normalizer @ file:///home/conda/feedstock_root/build_artifacts/charset-normalizer_1688813409104/work
+chex==0.1.7
+click==8.1.7
+colorama @ file:///home/conda/feedstock_root/build_artifacts/colorama_1666700638685/work
+comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1691044910542/work
+confection==0.1.3
+contourpy @ file:///Users/runner/miniforge3/conda-bld/contourpy_1686734165903/work
+cycler @ file:///home/conda/feedstock_root/build_artifacts/cycler_1635519461629/work
+cymem==2.0.8
+debugpy @ file:///Users/runner/miniforge3/conda-bld/debugpy_1691021385546/work
+decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work
+defusedxml @ file:///home/conda/feedstock_root/build_artifacts/defusedxml_1615232257335/work
+dm-tree==0.1.8
+dnspython==2.6.1
+email_validator==2.2.0
+entrypoints @ file:///home/conda/feedstock_root/build_artifacts/entrypoints_1643888246732/work
+etils==1.3.0
+exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1688381075899/work
+executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1667317341051/work
+faiss-cpu==1.7.4
+fastapi==0.111.1
+fastapi-cli==0.0.4
+fastjsonschema @ file:///home/conda/feedstock_root/build_artifacts/python-fastjsonschema_1690055433477/work/dist
+ffmpy==0.4.0
+filelock==3.12.2
+flax==0.7.2
+flit_core @ file:///home/conda/feedstock_root/build_artifacts/flit-core_1684084314667/work/source/flit_core
+fonttools @ file:///Users/runner/miniforge3/conda-bld/fonttools_1691006833604/work
+fqdn @ file:///home/conda/feedstock_root/build_artifacts/fqdn_1638810296540/work/dist
+fsspec==2023.9.1
+gradio==4.39.0
+gradio_client==1.1.1
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.24.2
+idna @ file:///home/conda/feedstock_root/build_artifacts/idna_1663625384323/work
+importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1688754491823/work
+importlib-resources @ file:///home/conda/feedstock_root/build_artifacts/importlib_resources_1691408075105/work
+ipykernel @ file:///Users/runner/miniforge3/conda-bld/ipykernel_1691424478312/work
+ipython @ file:///Users/runner/miniforge3/conda-bld/ipython_1683289207929/work
+ipywidgets @ file:///home/conda/feedstock_root/build_artifacts/ipywidgets_1690877070294/work
+isoduration @ file:///home/conda/feedstock_root/build_artifacts/isoduration_1638811571363/work/dist
+jax==0.4.13
+jaxlib==0.4.13
+jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1690896916983/work
+Jinja2 @ file:///home/conda/feedstock_root/build_artifacts/jinja2_1654302431367/work
+joblib @ file:///home/conda/feedstock_root/build_artifacts/joblib_1691577114857/work
+json5 @ file:///home/conda/feedstock_root/build_artifacts/json5_1688248289187/work
+jsonpointer==2.0
+jsonschema @ file:///home/conda/feedstock_root/build_artifacts/jsonschema-meta_1691407964752/work
+jsonschema-specifications @ file:///home/conda/feedstock_root/build_artifacts/jsonschema-specifications_1689701150890/work
+jupyter @ file:///Users/runner/miniforge3/conda-bld/jupyter_1670249744855/work
+jupyter-console @ file:///home/conda/feedstock_root/build_artifacts/jupyter_console_1678118109161/work
+jupyter-events @ file:///home/conda/feedstock_root/build_artifacts/jupyter_events_1691505939576/work
+jupyter-lsp @ file:///home/conda/feedstock_root/build_artifacts/jupyter-lsp-meta_1685453365113/work/jupyter-lsp
+jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1687700988094/work
+jupyter_core @ file:///Users/runner/miniforge3/conda-bld/jupyter_core_1686775732472/work
+jupyter_server @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_1687869799272/work
+jupyter_server_terminals @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_terminals_1673491454549/work
+jupyterlab @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_1691085900640/work
+jupyterlab-pygments @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_pygments_1649936611996/work
+jupyterlab-widgets @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_widgets_1688489450369/work
+jupyterlab_server @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_server_1690205927615/work
+kiwisolver @ file:///Users/runner/miniforge3/conda-bld/kiwisolver_1666805848360/work
+langcodes==3.3.0
+markdown-it-py==3.0.0
+MarkupSafe @ file:///Users/runner/miniforge3/conda-bld/markupsafe_1685769224358/work
+matplotlib @ file:///Users/runner/miniforge3/conda-bld/matplotlib-suite_1688684899624/work
+matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1660814786464/work
+mdurl==0.1.2
+micrograd==0.1.0
+mistune @ file:///home/conda/feedstock_root/build_artifacts/mistune_1686313613819/work/dist
+ml-dtypes==0.2.0
+mpmath==1.3.0
+msgpack==1.0.8
+munkres==1.1.4
+murmurhash==1.0.10
+nbclient @ file:///home/conda/feedstock_root/build_artifacts/nbclient_1684790896106/work
+nbconvert @ file:///home/conda/feedstock_root/build_artifacts/nbconvert-meta_1690313290323/work
+nbformat @ file:///home/conda/feedstock_root/build_artifacts/nbformat_1690814868471/work
+nest-asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1664684991461/work
+networkx==3.1
+nltk==3.8.1
+notebook @ file:///home/conda/feedstock_root/build_artifacts/notebook_1691139191280/work
+notebook_shim @ file:///home/conda/feedstock_root/build_artifacts/notebook-shim_1682360583588/work
+numpy @ file:///Users/runner/miniforge3/conda-bld/numpy_1687808433287/work
+opt-einsum==3.3.0
+optax==0.1.8
+orbax-checkpoint==0.2.3
+orjson==3.10.6
+overrides @ file:///home/conda/feedstock_root/build_artifacts/overrides_1691338815398/work
+packaging @ file:///home/conda/feedstock_root/build_artifacts/packaging_1681337016113/work
+pandas @ file:///Users/runner/miniforge3/conda-bld/pandas_1688740593370/work
+pandocfilters @ file:///home/conda/feedstock_root/build_artifacts/pandocfilters_1631603243851/work
+parso @ file:///home/conda/feedstock_root/build_artifacts/parso_1638334955874/work
+pathy==0.10.2
+pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1667297516076/work
+pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1602536217715/work
+Pillow @ file:///Users/runner/miniforge3/conda-bld/pillow_1688255913792/work
+pkgutil_resolve_name @ file:///home/conda/feedstock_root/build_artifacts/pkgutil-resolve-name_1633981968097/work
+platformdirs @ file:///home/conda/feedstock_root/build_artifacts/platformdirs_1690813113769/work
+pooch @ file:///home/conda/feedstock_root/build_artifacts/pooch_1679580333621/work
+preshed==3.0.9
+prometheus-client @ file:///home/conda/feedstock_root/build_artifacts/prometheus_client_1689032443210/work
+prompt-toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1688565951714/work
+psutil @ file:///Users/runner/miniforge3/conda-bld/psutil_1681775366577/work
+ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1609419310487/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
+pure-eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1642875951954/work
+pycparser @ file:///home/conda/feedstock_root/build_artifacts/pycparser_1636257122734/work
+pydantic==2.3.0
+pydantic_core==2.6.3
+pydub==0.25.1
+Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1691408637400/work
+pyparsing @ file:///home/conda/feedstock_root/build_artifacts/pyparsing_1652235407899/work
+PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work
+python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/python-dateutil_1626286286081/work
+python-dotenv==1.0.1
+python-json-logger @ file:///home/conda/feedstock_root/build_artifacts/python-json-logger_1677079630776/work
+python-multipart==0.0.9
+pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1680088766131/work
+PyYAML @ file:///Users/runner/miniforge3/conda-bld/pyyaml_1666772523114/work
+pyzmq @ file:///Users/runner/miniforge3/conda-bld/pyzmq_1691667741827/work
+referencing @ file:///home/conda/feedstock_root/build_artifacts/referencing_1691337268233/work
+regex==2023.8.8
+requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1684774241324/work
+rfc3339-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3339-validator_1638811747357/work
+rfc3986-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3986-validator_1598024191506/work
+rich==13.7.1
+rpds-py @ file:///Users/runner/miniforge3/conda-bld/rpds-py_1689705310488/work
+ruff==0.5.5
+safetensors==0.4.3
+scikit-learn @ file:///Users/runner/miniforge3/conda-bld/scikit-learn_1688116469994/work
+scipy @ file:///Users/runner/miniforge3/conda-bld/scipy-split_1683900453702/work/base/dist/scipy-1.10.1-cp38-cp38-macosx_11_0_arm64.whl#sha256=06d649e33fd400c8608c1083a7bb6f490989227277eae7f65577d925d87848e8
+semantic-version==2.10.0
+Send2Trash @ file:///home/conda/feedstock_root/build_artifacts/send2trash_1628511208346/work
+sentencepiece==0.2.0
+shellingham==1.5.4
+six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work
+smart-open==6.4.0
+sniffio @ file:///home/conda/feedstock_root/build_artifacts/sniffio_1662051266223/work
+soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1658207591808/work
+spacy==3.6.1
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+srsly==2.4.7
+stack-data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1669632077133/work
+starlette==0.37.2
+sympy==1.12
+tensorstore==0.1.45
+terminado @ file:///Users/runner/miniforge3/conda-bld/terminado_1670254106711/work
+thinc==8.1.12
+threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1689261241048/work
+tinycss2 @ file:///home/conda/feedstock_root/build_artifacts/tinycss2_1666100256010/work
+tokenizers==0.19.1
+tomli @ file:///home/conda/feedstock_root/build_artifacts/tomli_1644342247877/work
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.0.1
+torchaudio==2.0.2
+torchvision==0.15.2
+tornado @ file:///Users/runner/miniforge3/conda-bld/tornado_1684150249057/work
+tqdm @ file:///home/conda/feedstock_root/build_artifacts/tqdm_1691671248568/work
+traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1675110562325/work
+transformers==4.43.1
+typer==0.12.3
+typing-utils @ file:///home/conda/feedstock_root/build_artifacts/typing_utils_1622899189314/work
+typing_extensions==4.12.2
+tzdata @ file:///home/conda/feedstock_root/build_artifacts/python-tzdata_1680081134351/work
+unicodedata2 @ file:///Users/runner/miniforge3/conda-bld/unicodedata2_1667240193675/work
+uri-template @ file:///home/conda/feedstock_root/build_artifacts/uri-template_1688655812972/work/dist
+urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1689789803562/work
+uvicorn==0.30.3
+uvloop==0.19.0
+wasabi==1.1.2
+watchfiles==0.22.0
+wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1673864653149/work
+webcolors @ file:///home/conda/feedstock_root/build_artifacts/webcolors_1679900785843/work
+webencodings==0.5.1
+websocket-client @ file:///home/conda/feedstock_root/build_artifacts/websocket-client_1687789148259/work
+websockets==11.0.3
+widgetsnbextension @ file:///home/conda/feedstock_root/build_artifacts/widgetsnbextension_1688504439014/work
+zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1689374466814/work