storm-on-yarn配置
充浩波
2023-12-01
1, 安装mvn
wget http://mirror.symnds.com/software/Apache/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz
tar –zxvf apache-maven-3.1.1-bin.tar.gz
export MVN_HOME=/home/bigdata/mvn/apache-maven-3.1.1
2, 下载storm on yanr
https://github.com/yahoo/storm-yarn
下载zip,上传到服务器,解压:unzip storm-yarn-master.zip
mvn package -DskipTests
hadoop fs -copyFromLocal storm.zip /lib/storm/0.9.0-wip21
export STORM_WORK=/home/bigdata/storm_eco/storm-yarn-master
export STORM_HOME=$STORM_WORK
export PATH=$PATH:$STORM_WORK/storm-yarn-master/bin:$STORM_WORK/storm-0.9.0-wip21/bin
hadoop fs -mkdir -p /lib/storm/0.9.0-wip21
hadoop fs -copyFromLocal $STORM_HOME/storm-yarn-master/lib/storm.zip /lib/storm/0.9.0-wip21
storm-yarn launch $STORM_HOME/storm-0.9.0-wip21/conf/storm.yaml
http://zh.hortonworks.com/kb/storm-on-yarn-install-on-hdp2-beta-cluster/
scp -r bigdata@gs-server-v-127:/home/bigdata/storm_eco/storm-0.9.0-wip21 bigdata@gs-server-v-128:~/storm_eco/
scp -r bigdata@gs-server-v-127:/home/bigdata/storm_eco/storm-0.9.0-wip21 bigdata@gs-server-v-129:~/storm_eco/
scp -r bigdata@gs-server-v-127:/home/bigdata/storm_eco/storm-yarn-master bigdata@gs-server-v-128:~/storm_eco/
scp -r bigdata@gs-server-v-127:/home/bigdata/storm_eco/storm-yarn-master bigdata@gs-server-v-129:~/storm_eco/
/home/bigdata/storm_eco/storm-yarn-master
http://blog.csdn.net/xcytec/article/details/21695461
https://issues.apache.org/jira/browse/YARN-1473
<property>
<name>yarn.application.classpath</name>
<value>$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*</value>
</property>
sudo vi /opt/cloudera/parcels/CDH/lib/hadoop/etc/hadoop/yarn-site.xml
CM设置
yarn.application.classpath
$HADOOP_CLIENT_CONF_DIR
$HADOOP_CONF_DIR
$HADOOP_COMMON_HOME/*
$HADOOP_COMMON_HOME/lib/*
$HADOOP_HDFS_HOME/*
$HADOOP_HDFS_HOME/lib/*
$HADOOP_YARN_HOME/*
$HADOOP_YARN_HOME/lib/*