使用pgloader加载数据到postgresql中
替换conf文件的日期
sed -i “s/[0-9]{4}-[0-9]{2}-[0-9]{2}/${YESTERDAY}/” /export/loganaly/pgloader-stat-trace-pgdata3.conf
执行加载数据
python /usr/bin/pgloader -c /export/loganalyse/pgdata3.conf
相关配置文件
[jiao@l-jiao.s.cn6 ~]$ cat pgloader-stat-trace-pgdata3.conf
[pgsql]
host = l-xxx.xx.cn5
port = 5432
base = data_sas
user = pgdba
pass = XXXXXXXXXXXXXXXXXXXXXX
log_file = /var/pgdata3-trace-2015-12-01.log
log_min_messages = ERROR
client_min_messages = WARNING
lc_messages = C
client_encoding = 'utf-8'
#input_encoding = 'utf-8'
copy_every = 20000
commit_every = 20000
#copy_delimiter = %
null = "NULL"
empty_string = ""
max_parallel_sections = 4
[pgloader]
table = wl_data.stat_trace
format = text
filename = /export/data/trace.2015-12-01.log
field_sep = \t
columns = report_date,requesttime,source,mode,uid,pids,env_type,query,pt,act,ts,platform,vid,ntitle,ntitlelen,nimg,nimglen,userid,gid
reject_log = /export/loganalyse/tuan/pgloader-conf/log/trace-reject-2015-12-01.log
reject_data = /export/log/trace-statistics-reject-2015-12-01.data
quotechar = ^A
escapechar = ^A