Untitled

mail@pastecode.io avatar
unknown
plain_text
a year ago
374 B
3
Indexable
Never
def load_dataset_from_file(file_path):
    with open(file_path, 'r') as file:
        lines = file.readlines()

    texts = []
    ids = np.array([])

    for line in lines:
        data = json.loads(line)
        texts.append(data['text'])
        ids = np.append(ids, data['id']) #o tą linijkę chodzi
        
    return {'text': (texts), 'input_ids': (ids)}