flume至hbase-学习记录

环境准备(jdk已有):
hadoop:

1. wget http://archive.apache.org/dist/hadoop/core/hadoop-2.7.5/hadoop-2.7.5-src.tar.gz
2. tar -zxvf hadoop-2.7.5-src.tar.gz
3. mv hadoop-2.7.5 hadoop
4. 环境变量加入

export HADOOP_HOME=/home/soft/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

5. 启动:hadoop/sbin: sh start-all.sh

zookeeper:

1. 下载wget https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper- 3.4.14/zookeeper-3.4.14.tar.gz
2. tar -zxvf zookeeper-3.4.14.tar.gz
3. mv zookeeper-3.4.14 zookeeper
4. cd zookeeper/conf
5. mv zoo_sample.cfg zoo.cfg
6. zoo.cfg 加入代码(设置存储及log目录):

dataDir=/home/data/zookeeper/data
dataLogDir=/home/data/zookeeper/log

7. 启动zk: bin 目录下 sh zkServer.sh start (stop)

hbase:

1. wget https://mirror.bit.edu.cn/apache/hbase/2.2.5/hbase-2.2.5-bin.tar.gz
2. tar -zxvf hbase-2.2.5-bin hbase
3. conf目录下hbase-env.sh加入以下代码(jdk环境及禁用hbase自带的zk):

 export HBASE_MANAGES_ZK=false
 export JAVA_HOME=/usr/java/jdk1.8.0_131/

1. hbase-site.xml 配置

<configuration>
<!-- 如果hbase配置HBASE_MANAGES_ZK=false,则必须配置此项否则hbase还是会启动自带zk -->
<property>
    <name>hbase.cluster.distributed</name>
    <value>true</value> 
</property>
<!-- hbase存放数据目录 -->
  <property>
    <name>hbase.rootdir</name>
    <value>file:///home/data/hbase</value>
  </property>
  <!-- ZooKeeper数据文件路径 -->
  <property>
    <name>hbase.zookeeper.property.dataDir</name>
    <value>/home/zookeeper/data</value>
  </property>
  <property>
    <name>hbase.unsafe.stream.capability.enforce</name>
    <value>false</value>
    <description>
      Controls whether HBase will check for stream capabilities (hflush/hsync).
    </description>
  </property>
</configuration>

5. 环境变量加入

#hbase
export HBASE_HOME=/home/soft/hbase
export PATH=$HBASE_HOME/bin:$PATH

6. 启动hbase：./bin/start-hbase.sh
7. 执行命令:hbase sell
8. 在hbase中创建表为 userinfo,列簇为 info,为flume写入做准备,create 'userinfo', 'info'
9. 执行jps 可见 HMaster ,说明hbase启动

hvie　　
(flume skin为hbase时用到hive的jar,需要把hive/lib目录下的hbase开头的jar复制到flume/lib下,否则无法使用)

1. wget http://archive.apache.org/dist/hive/hive-2.3.7/apache-hive-2.3.7-bin.tar.gz
2. tar -zxvf apache-hive-2.3.7-bin.tar.gz
3. mv apache-hive-2.3.7 hive
4. cp /home/soft/hive/lib/hbase* /home/soft/flume/lib

flume:

1. 下载: wget https://mirrors.bfsu.edu.cn/apache/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz
2. tar -zxvf apache-flume-1.9.0-bin.tar.gz
3. mv apache-flume-1.9.0-bin flume
4. 环境变量加入

#flume
export FLUME_HOME=/home/soft/flume
export PATH=$PATH:$FLUME_HOME/bin

5. flume配置(conf目录):
   1. cp flume-conf.properties.template flume-conf.properties
   2. cp flume-env.sh.template flume-env.sh
   3. touch flumeHbase.conf 写入内容:

a1.sources = r1
a1.sinks = k1
a1.channels = c1

# 配置源
a1.sources.r1.type = avro
a1.sources.r1.port = 44444
a1.sources.r1.bind = localhost
# channel 为内存
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1
#sinks 为hbase
a1.sinks.k1.type = org.apache.flume.sink.hbase.HBaseSink
#hbase表名
a1.sinks.k1.table = userinfo
#hbase列簇
a1.sinks.k1.columnFamily =info
#通配符(数据格式为1#小明#18 )
a1.sinks.k1.serializer.regex =(.*)#(.*)#(.*)
a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
a1.sinks.k1.channel = c1
#列名
a1.sinks.k1.serializer.colNames = ROW_KEY,name,age
# 索引为0，即第一列
a1.sinks.k1.serializer.rowKeyIndex = 0

6. 启动flume:

flume-ng agent --conf conf --conf-file conf/flumeHbase.conf  --name a1  -Dflume.root.logger=INFO,console

开始测试
用RPC客户端写入数据到flmue:
1.依赖

<dependency>
	<groupId>org.apache.flume</groupId>
	<artifactId>flume-ng-core</artifactId>
	<version>1.9.0</version>
</dependency>

2.工具类

public class FlumeUtils {
    private static RpcClient client;
    private static String hostname="127.0.0.1";
    private static int port=44444;

    public static void sendDataToFlume(String data) {
        if(client==null || !client.isActive()){
            client = RpcClientFactory.getDefaultInstance(hostname, port);
        }
        Event event = EventBuilder.withBody(data, Charset.forName("UTF-8"));
        try {
            client.append(event);
        } catch (EventDeliveryException e) {
            client.close();
            client = null;
            client = RpcClientFactory.getDefaultInstance(hostname, port);
        }
    }
    public static void sendDataToFlumeList(List<String> list){
        if(client==null || !client.isActive()){
            client = RpcClientFactory.getDefaultInstance(hostname, port);
        }
        List<Event> events=new ArrayList<>();
        for(String data:list){
            Event event = EventBuilder.withBody(data, Charset.forName("UTF-8"));
            events.add(event);
        }
        try {
            client.appendBatch(events);
        } catch (EventDeliveryException e) {
            client.close();
            client = null;
            client = RpcClientFactory.getDefaultInstance(hostname, port);
        }
    }

    public static void cleanUp() {
        client.close();
    }
}

3.请求action

 @GetMapping("test")
    public String test(){
        List<String> list=new ArrayList<>();
        int key=1001;
        for (int i = 0; i < 200; i++) {
            key+=i;
            list.add(JSON.toJSONString(key+"#"+(name+i)+"#"+i));
        }
        FlumeUtils.sendDataToFlumeList(list);
        return "success";
    }

查看是否写入成功
1.hbase shell
2 scan ‘userinfo’ 可见已写入hbase