Spaces:
Runtime error
Runtime error
taka-yamakoshi commited on
Commit ·
6751661
1
Parent(s): ef0b5c6
fix a minor bug
Browse files
app.py
CHANGED
|
@@ -95,7 +95,7 @@ if __name__=='__main__':
|
|
| 95 |
|
| 96 |
# Title
|
| 97 |
st.markdown(generate_markdown('WordPiece Explorer',size=32), unsafe_allow_html=True)
|
| 98 |
-
st.markdown(generate_markdown('quick and easy way to explore how tokenizers work',size=24), unsafe_allow_html=True)
|
| 99 |
|
| 100 |
# Select and load the tokenizer
|
| 101 |
st.sidebar.write('1. Choose the tokenizer from below')
|
|
@@ -139,7 +139,7 @@ if __name__=='__main__':
|
|
| 139 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids']
|
| 140 |
else:
|
| 141 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids'][1:-1]
|
| 142 |
-
sentence = st.text_input(f'Tokenized IDs',value=' '.join(default_tokens))
|
| 143 |
num_tokens = DeTokenizeText(sentence)
|
| 144 |
else:
|
| 145 |
sentence = st.text_input(f'Text',value='Tokenizers decompose bigger words into smaller tokens')
|
|
|
|
| 95 |
|
| 96 |
# Title
|
| 97 |
st.markdown(generate_markdown('WordPiece Explorer',size=32), unsafe_allow_html=True)
|
| 98 |
+
st.markdown(generate_markdown('- quick and easy way to explore how tokenizers work -',size=24), unsafe_allow_html=True)
|
| 99 |
|
| 100 |
# Select and load the tokenizer
|
| 101 |
st.sidebar.write('1. Choose the tokenizer from below')
|
|
|
|
| 139 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids']
|
| 140 |
else:
|
| 141 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids'][1:-1]
|
| 142 |
+
sentence = st.text_input(f'Tokenized IDs',value=' '.join([str(token) for token in default_tokens]))
|
| 143 |
num_tokens = DeTokenizeText(sentence)
|
| 144 |
else:
|
| 145 |
sentence = st.text_input(f'Text',value='Tokenizers decompose bigger words into smaller tokens')
|