app.py

# coding=utf-8
# author: xusong
# time: 2022/8/23 16:06

"""

plots

table

## related demo
http://text-processing.com/demo/tokenize/

## 可视化

[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
"""

import json
import pandas as pd
import gradio as gr

from vocab import all_tokenizers, load_tokener

# 显示空格：https://blog.csdn.net/liuxiao723846/article/details/118994673
# 隐藏legend：
css = """
.space-show {white-space: pre-wrap;}
.cell-wrap {white-space: pre-wrap;}
.category-legend {display: none !important}
"""

example_text = """中文测试：华为智能音箱发布：华为Sound X。維基百科由非營利組織──維基媒體基金會負責維持
标点测试：，。！？；
空格测试：  2个空格        8个空格
数字测试：(10086 + 98) = 100184"""


def tokenize(text, tokenizer_type):
    print(text, tokenizer_type)
    pos_tokens = []
    tokenizer = load_tokener(tokenizer_type)
    encoding = tokenizer.encode(text)

    table = []

    for idx, token_id in enumerate(encoding):
        decode_text = tokenizer.decode([token_id])  # 特殊字符解码后会统一变成 �，对应 "\ufffd"
        pos_tokens.extend([(decode_text, str(idx % 3))])

        # token  "Byte":  # 这是 utf-8编码吧？
        token = tokenizer.convert_ids_to_tokens([token_id])[0]
        if isinstance(token, bytes):
            token_str = token.decode("utf-8")
            token_bytes = token
            json_dumps = json.dumps(token_str)
        elif isinstance(token, str):
            token_str = token
            token_bytes = bytes(token_str, "utf-8")
            json_dumps = json.dumps(token_str)
        else:
            return

        table.append(
            {"TokenID": token_id,
             "Token": token_str,  # utf-8解码后的字符串，为什么有些是 <0xE7>，表示什么？比如llama
             "Text": decode_text,  #
             # "Bytes": token_bytes,  # bytes类型在gradio前端页面被解码成字符串，比如   b'\xe4\xb8\xad' 仍然显示成 "中"。因此 str(token_bytes)
             "Bytes": str(token_bytes),
             # "Unicode": json_dumps  # unicode, 如果是ascii码，就直接显示。如果不是ascii码，就显示unicode
             }
        )

    table_df = pd.DataFrame(table)
    print(table)
    print(table_df)

    return pos_tokens, table_df


def test_coding():
    bytes1 = b'\xe4\xb8\xad'
    print(bytes1)  # b'\xe4\xb8\xad'


with gr.Blocks(css=css) as demo:
    gr.HTML("""<h1 align="center">Tokenizer Arena</h1>""")
    # links: https://www.coderstool.com/utf8-encoding-decoding
    #


    user_input = gr.Textbox(
        value=example_text,
        lines=5
    )  # placeholder="Enter sentence here..."

    # submitBtn = gr.Button("生成回复", variant="primary")

    # TODO: 图 表 压缩率
    # llama chatglm gpt_nexo_20b baichuan  baichuan_7b
    with gr.Row():
        with gr.Column():
            tokenizer_type_1 = gr.Dropdown(
                all_tokenizers, value="llama", label="tokenizer"
            )
            token_counter_1 = None  # 计数器
            output_text_1 = gr.Highlightedtext(
                label="Tokenization",
                show_legend=True,
                elem_classes="space-show"
            )

            output_table_1 = gr.Dataframe(
                headers=["TokenID", "Byte", "Text"],
                datatype=["str", "str", "str"],
                #elem_classes="space-show",   # 给整个Dataframe加这个css不起作用，因此直接修改cell-wrap
            )

        with gr.Column():
            tokenizer_type_2 = gr.Dropdown(
                all_tokenizers, value="baichuan_7b", label="tokenizer"
            )
            token_counter_2 = None  # 计数器
            output_text_2 = gr.Highlightedtext(
                label="Tokenization",
                show_legend=True,
                elem_classes="space-show"
            )

            output_table_2 = gr.Dataframe(
                headers=["TokenID", "Token", "Text"],
                datatype=["str", "str", "str"],
            )

        user_input.change(tokenize,
                          [user_input, tokenizer_type_1],
                          [output_text_1, output_table_1])
        tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1], [output_text_1, output_table_1])

        user_input.change(tokenize,
                          [user_input, tokenizer_type_2],
                          [output_text_2, output_table_2])
        tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2], [output_text_2, output_table_2])

    # submitBtn.click(tokenize, [user_input, tokenizer_type], outputs,
    #                 show_progress=True)

    # examples=[
    #     ["What a beautiful morning for a walk!"],
    #     ["It was the best of times, it was the worst of times."],
    #     ["多个空格    It  ss  was the best of times, it was the worst of times."],
    # ]

if __name__ == "__main__":
    demo.launch()