参考地址
在paddlehub中的例子需要构建数据集。demo中的代码如下
dataset = hub.dataset.ChnSentiCorp()
当替换为自定义数据集时,首先需要将自己的数据集转换为如下形式。
- train.tsv
- dev.tsv
- test.tsv
然后,定义自己的数据集函数进行转换
dataset = DemoDataset(dataset_dir=model_path)
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from collections import namedtuple
import codecs
import os
import csv
from paddlehub.dataset import InputExample, HubDataset
class DemoDataset(HubDataset):
"""DemoDataset"""
def __init__(self,dataset_dir ):
self.dataset_dir = dataset_dir #模型路径
self._load_train_examples()
self._load_test_examples()
self._load_dev_examples()
def _load_train_examples(self):
self.train_file = os.path.join(self.dataset_dir, "train.tsv")
self.train_examples = self._read_tsv(self.train_file)
def _load_dev_examples(self):
self.dev_file = os.path.join(self.dataset_dir, "dev.tsv")
self.dev_examples = self._read_tsv(self.dev_file)
def _load_test_examples(self):
self.test_file = os.path.join(self.dataset_dir, "test.tsv")
self.test_examples = self._read_tsv(self.test_file)
def get_train_examples(self):
return self.train_examples
def get_dev_examples(self):
return self.dev_examples
def get_test_examples(self):
return self.test_examples
def get_labels(self):# 确定标签,根据自己的数据集lable进行定义
"""define it according the real dataset"""
return ["0", "1"]
@property
def num_labels(self):
"""
Return the number of labels in the dataset.
"""
return len(self.get_labels())
def _read_tsv(self, input_file, quotechar=None):
"""Reads a tab separated value file."""
with codecs.open(input_file, "r", encoding="UTF-8") as f:
reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
examples = []
seq_id = 0
header = next(reader) # skip header
for line in reader:
example = InputExample(
guid=seq_id, label=line[0], text_a=line[1])
seq_id += 1
examples.append(example)
return examples