Danbooru preprocess

获取所有的文件名列表

 ls portraits/ > portraits_filenames.txt
less portraits_filenames.txt

10000310.jpg
10000330.jpg
1000050.jpg
10000610.jpg
10000760.jpg
10000990.jpg
10001010.jpg
100010.jpg
*****

python:

def load_data_filter(args):
    """
    loads and yields image data from metadata files
    """
    data = []
    metadata_paths = find_metadata_files(
        os.path.join(args.directory, args.metadata_dir)
    )
    i = 0
    for path in metadata_paths:
        with open(path, "r") as f:
            for line in f:
                #if i >= args.max_examples:
                    #break
                example = json.loads(line)
                if tag_check(
                    {y['name']
                     for y in example['tags']},
                    example['rating'],
                    example['score'],
                    sc