python 处理nlp_chinese_corpus中baike2018qa的数据

最近需要使用baike2018qa的数据，数据的地址为： https://github.com/brightmart/nlp_chinese_corpus

发现处理起来并不简单，这里我把我处理的方法分享出来，我需要title和desc字段拼接，然后category当成类别，然后划分数据集，下面我的处理流程，有需要的可以进行修改哈：

import pandas as pd
from sklearn.model_selection import train_test_split


file_path = './data/baike_qa_valid.json'
output_files = 'data/train.txt'
data = pd.read_json(file_path, orient='records', lines=True)
print(data.head())
print(data.shape)
print(data.keys())
# data['text']=data['title']+data['desc']
data['content']=data['title']+data['desc']
print(data.shape)
data['content']=data['content'].apply(lambda x:x.replace('\n','').replace('\t',"").replace('\r',""))
# data = data.dropna(subset=['category'])
data['category']=data['category'].apply(lambda x:x.replace('-',","))
data = data[~(data['category'].isin(['']))]
print(data.shape)

data.to_csv(output_files, columns=['content','category'], header=None, sep='\t', index=False)

X_train, X_test = train_test_split(data,test_size=0.1,random_state=42)

X_train.to_csv('data/train.txt',columns=['content','category'],header=None,sep='\t',index=False)
X_test.to_csv('data/dev.txt',columns=['content','category'],header=None,sep='\t',index=False)

print(data['category'].value_counts())
labels = data['category'].unique()
print(len(labels))

with open('data/label.txt','w') as f:
    for label in labels:
        f.write(label+'\n')