ulimit -n
65535
root soft nofile 65536
root hard nofile 65536
* soft nofile 65536
* hard nofile 65536
net.core.somaxconn = 1024
net.core.netdev_max_backlog = 5000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.ipv4.tcp_rmem = 4096 12582912 16777216
net.ipv4.tcp_max_syn_backlog = 8096
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240 65535
sysctl -p
# td-agent 4
$ curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent4.sh | sh
# td-agent 3
$ curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent3.sh | sh
记得修改读取目标文件以及pos文件的权限为td-agent
chown -R td-agent:td-agent /var/log/containers
gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
vi /etc/hosts
199.232.28.133 raw.githubusercontent.com
curl -sSL https://get.rvm.io | bash -s stable
source /etc/profile.d/rvm.sh
rvm list known
rvm install 2.4.1
#查看版本
ruby -v
gem -v
yum install ruby
gem sources --add https://gems.ruby-china.com/ --remove https://rubygems.org/
gem sources -l
#更新缓存
gem sources -u
# 安装
gem install fluentd --no-doc
fluentd --setup ./fluent
# 启动
fluentd -c ./fluent/fluent.conf -vv &
# 测试
echo '{"json":"message"}' | fluent-cat debug.test
# 停止
pkill -f fluentd
curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent3.sh | sh
systemctl start td-agent
curl -X POST -d 'json={"json":"message"}' http://localhost:8888/debug.test
tail /var/log/td-agent/td-agent.log
gem install fluent-plugin-elasticsearch
echo "测试" >> /etc/td-agent/huitest.log
####
## Output descriptions:
##
# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
<match td.*.*>
@type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path /var/log/td-agent/buffer/td
<secondary>
@type file
path /var/log/td-agent/failed_records
</secondary>
</match>
## match tag=debug.** and dump to console
<match debug.**>
@type stdout
</match>
<source>
@type tail
path /etc/td-agent/huitest.log
# pos_file /etc/td-agent/huitest.log.pos
tag hui
# refresh_interval 10
<parse>
@type none
</parse>
</source>
<match **>
@type elasticsearch
host 172.28.6.119
port 9200
# user elastic
# password 123456
# time_slice_format %Y%m%d
index_name ${tag}.%Y%m%d
logstash_format true
logstash_dateformat %Y%m%d
include_tag_key true
type_name testlog
# buffer_chunk_limit 2M
# buffer_queue_limit 32
# max_retry_wait 30
# disable_retry_limit
# num_threads 8
logstash_prefix hui
flush_interval 1s
<buffer tag, time>
timekey 1h
timekey_wait 1s
flush_interval 1s
</buffer>
</match>
#<match hui>
# @type file
# path /ect/td-agent/generaltest.log
# time_slice_format %Y%m%d
# time_slice_wait 10m
# comperss gzip
#</match>
####
## Source descriptions:
##
## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
<source>
@type forward
</source>
## built-in UNIX socket input
#<source>
# @type unix
#</source>
# HTTP input
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
<source>
@type http
bind 127.0.0.1
port 8888
# tag debug.log
</source>
## live debugging agent
<source>
@type debug_agent
bind 127.0.0.1
port 24230
</source>
####
## Examples:
##
## File input
## read apache logs continuously and tags td.apache.access
#<source>
# @type tail
# format apache
# path /var/log/httpd-access.log
# tag td.apache.access
#</source>
## File output
## match tag=local.** and write to file
#<match local.**>
# @type file
# path /var/log/td-agent/access
#</match>
## Forwarding
## match tag=system.** and forward to another td-agent server
#<match system.**>
# @type forward
# host 192.168.0.11
# # secondary host is optional
# <secondary>
# host 192.168.0.12
# </secondary>
#</match>
## Multiple output
## match tag=td.*.* and output to Treasure Data AND file
#<match td.*.*>
# @type copy
# <store>
# @type tdlog
# apikey API_KEY
# auto_create_table
# buffer_type file
# buffer_path /var/log/td-agent/buffer/td
# </store>
# <store>
# @type file
# path /var/log/td-agent/td-%Y-%m-%d/%H.log
# </store>
#</match>
https://archive.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz