Paddle.fluid之loader

self-defination reader

def make_reader(txt_file, is_train = True):
    def reader():
        with open(txt_file, 'r') as f:
            lines = f.readlines()
        f.close()
        if is_train:
            np.random.shuffle(lines)
            # do somehting to balance the samples
        for line in lines:
            yield preprocess(line, is_train)  # self-defination pre-processing 
    return paddle.reader.buffered(reader,size=256)

• for-loop style

db_test = make_reader(test_file, is_train=False)
for i, data in enumerate(db_test()):
    # do something meaningful

• paddle.batch(reader, batch_size)

my_reader = make_reader(test_file, is_train=False)
test_reader = paddle.batch(my_reader, batch_size=bz) # iterator
# case-1
for idx, data in enumerate(test_reader()):
    # type(data) = list, len(data) = batch_size

# case-2
test_py_reader = fluid.layers.py_reader(
    capacity=256, 
    shapes=((-1, 3, 224, 224), (-1, 1)),
    lod_levels=[0, 0], 
    dtypes=('float32', 'int64'),
    use_double_buffer=True)
test_py_reader.decorate_paddle_reader(test_reader)

test_py_reader.start()
try:
    while True:
        test_exe.run(fetch_list=[test_loss.name])
except fluid.core.EOFException:
    test_py_reader.reset()

fluid.io.PyReader

1