集群安装HDP3.5.1和ambari和dolphinscheduler的预处理流程

命令行界面
systemctl set-default multi-user.target
图形界面
systemctl set-default graphical.target

装java1.8 oracle

#wget http://[ip]:8080/jdk-8u202-linux-x64.rpm
rpm -e --nodeps java-1.7.0-openjdk-headless-1.7.0.191-2.6.15.5.el7.x86_64 java-1.8.0-openjdk-headless-1.8.0.181-7.b13.el7.x86_64 java-1.8.0-openjdk-1.8.0.181-7.b13.el7.x86_64 javapackages-tools-3.4.1-11.el7.noarch java-1.7.0-openjdk-1.7.0.191-2.6.15.5.el7.x86_64
rpm -ivh jdk-8u202-linux-x64.rpm
echo "export JAVA_HOME=/usr/java/default" >> /etc/profile
echo "export PATH=$JAVA_HOME/bin:$PATH" >> /etc/profile
echo "export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar" >> /etc/profile
source /etc/profile
java -version

换yum源

echo "nameserver 114.114.114.114" >> /etc/resolv.conf
minorver=7.6.1810
sed -e "s|^mirrorlist=|#mirrorlist=|g" \
         -e "s|^#baseurl=http://mirror.centos.org/centos/\$releasever|baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/$minorver|g" \
         -i.bak \
         /etc/yum.repos.d/CentOS-*.repo
yum clean all
yum makecache

加swap

mkdir /usr/swap
cd /usr/swap
dd if=/dev/zero of=swapfile bs=100M count=100
mkswap /usr/swap/swapfile
swapon /usr/swap/swapfile

安装maven

wget http://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz

#wget http://[ip]:8000/apache-maven-3.6.3-bin.tar.gz
tar -xf apache-maven-3.6.3-bin.tar.gz -C /usr/local/
mv /usr/local/apache-maven-3.6.3/ /usr/local/maven3
echo "export PATH=$PATH:/usr/local/maven3/bin" >> /etc/profile

node换阿里源

npm config set registry https://registry.npmmirror.com

生成ds.rpm
克隆代码
需要yum安装rpmbuilder
mvn -U clean install -Prpmbuild -Dmaven.test.skip=true -X
生成项目在:dolphinscheduler-dist/target/rpm/apache-dolphinscheduler/RPMS/noarch
所有机器都安装rpm文件
在ambari-server上
移动 ambari_plugin/common-services/DOLPHIN to ambari-server/resources/common-services/
移动 ambari_plugin/statcks/DOLPHIN to ambari-server/resources/stacks/HDP/2.6/services/–stack version
mysql创建用户

CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
create user 'dolphin'@'%' identified by '123456';
GRANT all on dolphinscheduler.* TO 'dolphin'@'%';
GRANT all on dolphinscheduler.* TO 'dolphin'@'localhost';
flush privileges;

ambari上安装

装ds.rpm

cd /opt/soft/
rm -rf dolphinscheduler
rm -rf run
rpm -e apache-dolphinscheduler-2.0.0-1.noarch
wget http://192.168.1.201:10001/apache-dolphinscheduler-2.0.0-1.noarch.rpm
rpm -ivh apache-dolphinscheduler-2.0.0-1.noarch.rpm

需要把mysql驱动复制到/opt/soft/dolphinscheduler/lib
需要把mysql驱动复制到/opt/soft/dolphinscheduler/lib/plugin/task/sql
datax需要替换mysql驱动(/opt/soft/datax/plugin/writer/mysqlwriter和/opt/soft/datax/plugin/reader/mysqlreader/libs)

本地存储需要修改:

fs.defaultFS=file:/// 
resource.storage.type=HDFS
这样创建的文件就在本机了
resource.upload.path=/dolphinscheduler
指定本机的文件存储位置,需要有相关权限

免密登录 (需要sudo一次否则无法执行)

ssh-keygen -t rsa
ssh-copy-id [服务器ip]
sudo ssh-copy-id [服务器ip]