cli/halueval-upload-cli.py

#!/usr/bin/env python3

import random
import requests

from datasets import load_dataset, Dataset, DatasetDict


path = "pminervini/HaluEval"

API_URL = f"https://datasets-server.huggingface.co/splits?dataset={path}"
response = requests.get(API_URL)
res_json = response.json()

gold_splits = {"dialogue", "qa", "summarization", "general"}

available_splits = {split["config"] for split in res_json["splits"]} if "splits" in res_json else set()

name_to_ds = dict()

for name in gold_splits:
    ds = load_dataset("json", data_files={"data": f"data/{name}_data.json"})
    name_to_ds[name] = ds
    # if name not in available_splits:
    ds.push_to_hub(path, config_name=name)


def list_to_dict(lst: list) -> dict:
    res = dict()
    for entry in lst:
        for k, v in entry.items():
            if k not in res:
                res[k] = []
            res[k] += [v]
    return res


for name in gold_splits - {"general"}:
    random.seed(42)
    ds = name_to_ds[name]
    new_entry_lst = []

    for entry in ds["data"]:
        is_hallucinated = random.random() > 0.5
        new_entry = None
        if name in {"qa"}:
            new_entry = {
                "knowledge": entry["knowledge"],
                "question": entry["question"],
                "answer": entry[f'{"hallucinated" if is_hallucinated else "right"}_answer'],
                "hallucination": "yes" if is_hallucinated else "no",
            }
        if name in {"dialogue"}:
            new_entry = {
                "knowledge": entry["knowledge"],
                "dialogue_history": entry["dialogue_history"],
                "response": entry[f'{"hallucinated" if is_hallucinated else "right"}_response'],
                "hallucination": "yes" if is_hallucinated else "no",
            }
        if name in {"summarization"}:
            new_entry = {
                "document": entry["document"],
                "summary": entry[f'{"hallucinated" if is_hallucinated else "right"}_summary'],
                "hallucination": "yes" if is_hallucinated else "no",
            }
        assert new_entry is not None
        new_entry_lst += [new_entry]
    new_ds_map = list_to_dict(new_entry_lst)
    new_ds = Dataset.from_dict(new_ds_map)
    new_dsd = DatasetDict({"data": new_ds})

    new_dsd.push_to_hub(path, config_name=f"{name}_samples")