merge_datasets.py

import datasets
import pandas as pd

dfs = []

for i in range(10):
    dfs.append(
        datasets.ArrowReader.read_table(f'segment_{i}/dataset.arrow').to_pandas()
    )

full_df = pd.concat(dfs, ignore_index=True)
full_df.to_csv('dataset.csv')