加载raid驱动的luster内核编译
加载raid驱动的luster内核编译
一、安装流程：
安装OFEB（IB）驱动-->安装lustre内核-->把RAID驱动继承到lustre内核中去-->重新安装OFEB（IB）驱动-->安装lustre工具包-->挂载lustre
二、安装OFEB驱动
1：查看内核版本
[root@localhostopt]# rpm -qa |grep kerne
kernel-2.6.18-194.el5
kernel-devel-2.6.18-194.el5
kernel-headers-2.6.18-194.el5
2：安装OFED (ib驱动)
[root@localhostopt]# ls
libevent-devel-1.4.13-1.x86_64.rpm nfs-utils-lib-devel-1.0.8-7.6.el5.x86_64.rpm OFED-1.5.2-rc2 tcl-devel-8.4.13-4.el5.x86_64.rpm
[root@localhostopt]# rpm -ivh libevent-devel-1.4.13-1.x86_64.rpmnfs-utils-lib-devel-1.0.8-7.6.el5.x86_64.rpm tcl-devel-8.4.13-4.el5.x86_64.rpm
warning:libevent-devel-1.4.13-1.x86_64.rpm: Header V3 DSAsignature: NOKEY, key ID 37017186
Preparing... ########################################### [100%]
1:tcl-devel ###########################################[ 33%]
2:libevent-devel ########################################### [ 67%]
3:nfs-utils-lib-devel ###########################################[100%]
[root@localhostopt]# cd OFED-1.5.2-rc2/
[root@localhostOFED-1.5.2-rc2]# ls
BUILD_ID docs in1.sh install.pl LICENSE ofed.conf ofed.conf.ok README.txt RPMS SRPMS un0.sh uninstall.exp uninstall.sh
[root@localhostOFED-1.5.2-rc2]# ./un0.sh
[root@localhostOFED-1.5.2-rc2]# ./in1.sh
三、安装lustre内核
[root@localhostlustre1.8.5]# rpm -ivh --forcekernel-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.x86_64.rpmkernel-headers-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.x86_64.rpmkernel-devel-2.6.18-194.17.1.el5_lustre.1.8.5.x86_64.rpm
Preparing... ###########################################[100%]
1:kernel-devel ########################################### [ 33%]
2:kernel ########################################### [ 67%]
WARNING:No module megasr found for kernel 2.6.18-194.17.1.0.1.el5_lustre.1.8.5,continuing anyway
WARNING:No module megasr found for kernel 2.6.18-194.17.1.0.1.el5_lustre.1.8.5,continuing anyway
WARNING:No module megasr found for kernel 2.6.18-194.17.1.0.1.el5_lustre.1.8.5,continuing anyway
3:kernel-headers ########################################### [100%]
[root@localhostlustre1.8.5]# rpm -qa |grep lustre
kernel-devel-2.6.18-194.17.1.el5_lustre.1.8.5
kernel-headers-2.6.18-194.17.1.0.1.el5_lustre.1.8.5
kernel-2.6.18-194.17.1.0.1.el5_lustre.1.8.5
四、把RAID驱动继承到lustre内核中去
1.编译RAID驱动
[root@localhostopt]# tar zxvf 13.10.0708.2009_Linux_SWR_Driver_R5_Univ_for_2.6.30_x86_64.tar.gz
[root@localhostshim]# patch -p0 < megasr_open_source.patch
patchingfile open_source/osl/lin/linux_osl.c
patchingfile open_source/osl/lin/linux_osl.h
此步骤需要执行两次，原因不知道
[root@localhostshim]# cd
library/ megasr_open_source.patch open_source/
[root@localhostshim]# cd library/rhel5-64/
[root@localhostrhel5-64]# make -C /usr/src/kernels/2.6.18-194.17.1.el5_lustre.1.8.5-x86_64/M=$(pwd)
[root@localhostrhel5-64]# ls
built-in.o megasr.ko megasr.mod.c megasr.o Module.symvers
Makefile megasr-lib.a megasr.mod.o Module.markers
（编译成功，会生成megasr.ko驱动模块）
2：把RAID驱动集成到lustre内核中
[root@localhost~]#mkdir initrd
[root@localhost~]# cd initrd/
[root@localhostinitrd]# cp /boot/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.
[root@localhostinitrd]# mv initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.imginitrd-2.6.
.18-194.17.1.0.1.el5_lustre.1.8.5.img.gz
[root@localhostinitrd]# gunzip initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.gz
[root@localhostinitrd]# cpio -div < initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img
[root@localhostinitrd]# ls
bin etc initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img proc sys
dev init lib sbin sysroot
[root@localhostinitrd]# cp /opt/13.10.0708.2009_Linux_SWR_Driver_R5_Univ_for_2.6.30_x8
86_64/shim/library/rhel5-64/megasr.kolib
[root@localhostlib]# chmod 600 megasr.ko
[root@localhostlib]# modinfo megasr.ko |grep vermagic
vermagic: 2.6.18-194.17.1.el5_lustre.1.8.5SMP mod_unload gcc-4.1 （查看是否为lustre内核编译上驱动）
[root@localhostinitrd]# vi init
"init"
"init"93L, 2469C#!/bin/nash
mount -tproc /proc /proc
setquiet
echoMounting proc filesystem
echoMounting sysfs filesystem
mount -tsysfs /sys /sys
echoCreating /dev
mount -omode=0755 -t tmpfs /dev /dev
mkdir/dev/pts
mount -tdevpts -o gid=5,mode=620 /dev/pts /dev/pts
mkdir/dev/shm
mkdir/dev/mapper
echoCreating initial device nodes
mknod/dev/null c 1 3
mknod/dev/zero c 1 5
mknod/dev/urandom c 1 9
mknod/dev/systty c 4 0
mknod/dev/tty c 5 0
mknod/dev/console c 5 1
mknod/dev/ptmx c 5 2
mknod/dev/rtc c 10 135
mknod/dev/tty0 c 4 0
mknod/dev/tty1 c 4 1
mknod/dev/tty2 c 4 2
mknod/dev/tty3 c 4 3
mknod/dev/tty4 c 4 4
mknod/dev/tty5 c 4 5
mknod/dev/tty6 c 4 6
mknod/dev/tty7 c 4 7
mknod/dev/tty8 c 4 8
mknod/dev/tty9 c 4 9
mknod/dev/tty10 c 4 10
mknod/dev/tty11 c 4 11
mknod/dev/tty12 c 4 12
mknod/dev/ttyS0 c 4 64
mknod/dev/ttyS1 c 4 65
mknod/dev/ttyS2 c 4 66
mknod/dev/ttyS3 c 4 67
echoSetting up hotplug.
hotplug
echoCreating block device nodes.
mkblkdevs
echo"Loading ehci-hcd.ko module"
insmod/lib/ehci-hcd.ko
echo"Loading ohci-hcd.ko module"
insmod/lib/ohci-hcd.ko
echo"Loading uhci-hcd.ko module"
insmod/lib/uhci-hcd.ko
mount -tusbfs /proc/bus/usb /proc/bus/usb
echo"Loading jbd.ko module"
insmod/lib/jbd.ko
echo"Loading ext3.ko module"
insmod/lib/ext3.ko
echo"Loading shpchp.ko module"
insmod/lib/shpchp.ko
echo"Loading scsi_mod.ko module"
insmod/lib/scsi_mod.ko
echo"Loading sd_mod.ko module"
insmod/lib/sd_mod.ko
echo"Loading libata.ko module"
insmod/lib/libata.ko
echo"Loading ata_piix.ko module"
insmod/lib/ata_piix.ko
echo"Loading scsi_transport_fc.ko module"
insmod/lib/scsi_transport_fc.ko
echo"Loading lpfc.ko module"
insmod/lib/lpfc.ko
echo"Loading dm-mem-cache.ko module"
insmod/lib/scsi_transport_fc.ko-- INSERT --- INSERT --echo "Loading megasr.komodule"-- INSERT --insmod /lib/megasr.komkblkdevs
echo"Loading ehci-hcd.ko module"
insmod/lib/ehci-hcd.ko
echo"Loading ohci-hcd.ko module"
insmod/lib/ohci-hcd.ko
echo"Loading uhci-hcd.ko module"
insmod/lib/uhci-hcd.ko
mount -tusbfs /proc/bus/usb /proc/bus/usb
echo"Loading jbd.ko module"
insmod/lib/jbd.ko
echo"Loading ext3.ko module"
insmod/lib/ext3.ko
echo"Loading shpchp.ko module"
insmod/lib/shpchp.ko
echo"Loading scsi_mod.ko module"
insmod/lib/scsi_mod.ko
echo "Loading megasr.ko module"------------------------（在此位置添加红色字体）
insmod /lib/megasr.ko
echo"Loading sd_mod.ko module"
insmod/lib/sd_mod.ko
echo"Loading libata.ko module"
insmod/lib/libata.ko
echo"Loading ata_piix.ko module"
insmod/lib/ata_piix.ko
echo"Loading scsi_transport_fc.ko module"
insmod/lib/scsi_transport_fc.ko
echo"Loading lpfc.ko module"
insmod/lib/lpfc.ko
echo"Loading dm-mem-cache.ko module":wq
"init"
"init"95L, 2523C written
将内核重新压缩，并修改grub配置文件
[root@localhostinitrd]# rm -fr initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img
[root@localhostinitrd]# find ./ | cpio -H newc -o > /boot/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new
[root@localhostinitrd]# cd /boot/
[root@localhostboot]# gzip initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new
[root@localhostboot]# mv initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new.gzinitrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new
[root@localhostboot]# vi grub/grub.conf
"grub/grub.conf"
"grub/grub.conf"21L, 819C# grub.conf generated by anaconda
#
# Notethat you do not have to rerun grub after making changes to this file
#NOTICE: You have a /boot partition. This means that
#allkernel and initrd paths are relative to /boot/, eg.
#root(hd0,0)
#kernel/vmlinuz-version ro root=/dev/sda3
#initrd/initrd-version.img
#boot=/dev/sda
default=0
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title RedHat Enterprise Linux Server (2.6.18-194.17.1.0.1.el5_lustre.1.8.5)root(hd0,0)kernel /vmlinuz-2.6.18-194.17.1.0.1.el5_lustre.1.8.5ro root=LABEL=/ quietinitrd/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new (用新内核启动系统)
title RedHat Enterprise Linux Server (2.6.18-194.el5)root(hd0,0)kernel /vmlinuz-2.6.18-194.el5 ro root=LABEL=/ quietinitrd /initrd-2.6.18-194.el5.img
~ ~ initrd/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.im--INSERT --.new:wq
"grub/grub.conf"
"grub/grub.conf"21L, 823C written
五、重新安装OFED驱动
六、安装lustre工具包
1.自己生成rpm安装包
[root@login3tmp]# tar -zxvf lustre-1.8.5.tar.gz 生成 lustre-1.8.5 文件夹.
[root@login3tmp]# cd lustre-1.8.5
[root@login3lustre-1.8.5]#./configure--with-linux=/usr/src/kernel/<LUSTRE-KERNEL> --with=/usr/src/ofa_kernel
[root@login3lustre-1.8.5]#makerpms 会在/usr/src/redhat/RPMS/x86_64目录下生成需要安装的rpm文件
[root@login3 x86_64]# pwd
/usr/src/redhat/RPMS/x86_64
[root@login3 x86_64]# ls
lustre-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm
lustre-debuginfo-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm
lustre-ldiskfs-3.1.4-2.6.18_194.17.1.el5_lustre.1.8.5_201204080650.x86_64.rpm
lustre-ldiskfs-debuginfo-3.1.4-2.6.18_194.17.1.el5_lustre.1.8.5_201204080650.x86_64.rpm
lustre-modules-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm
lustre-source-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm
lustre-tests-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm
2.安装自己生成的rpm包
[root@login3 x86_64]# rpm -Uvh --force*.rpm. 注意如果此步安装不成功，可能是需要 expect软件包。可以到网上去下，或者是从系统的镜像文件系统中提取，为了和系统兼容，个人建议还是从镜像文件中提取比较好。
六、挂载lustre系统
1、将options lnet networks=o2ib 加到/etc/modprob.conf
[root@login3 ~]# vim /etc/modprobe.conf
alias eth0 igb
alias eth1 igb
alias scsi_hostadapter megasr
alias scsi_hostadapter1 ata_piix
alias scsi_hostadapter2 usb-storage
options lnetnetworks=o2ib
2、启动IB网模块
[root@login3 ~]# modprobe lnet
[root@login3 ~]# lctl network up
LNET configured
3、这步不成功有可能是：
1、之前source过 /etc/rc.local 文件，解决办法如下：
lctlnetwork down;
lctlnetwork unconfigure;
lustre_rmmod;
modprobelnet;
lctlnetwork up;
2、重启机器
4、查看是否启动成功
[root@login3 ~]# lctl list_nids
10.0.5.3@o2ib
5、启动完毕，如果成功，则挂载即可，挂载方法是编辑/etc/rc.local
[root@login3 ~]# vi /etc/rc.local
#!/bin/sh
#
# This script will be executed *after* allthe other init scripts.
# You can put your own initialization stuffin here if you don't
# want to do the full Sys V style initstuff.
touch /var/lock/subsys/local
##for lustre##
sleep 20
modprobe lnet
lctl network up
sleep 50
mount.lustre 10.0.4.1@o2ib:10.0.4.2@o2ib:/jdhpc/lustre/
[root@login3 ~]# source /etc/rc.local
6、用mount命令查看lustre是否被挂载
[root@login3 ~]# mount
/dev/mapper/VolGroup00-LogVol01 on / typeext3 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
/dev/sda1 on /boot type ext3 (rw)
tmpfs on /dev/shm type tmpfs (rw)
none on /proc/sys/fs/binfmt_misc typebinfmt_misc (rw)
sunrpc on /var/lib/nfs/rpc_pipefs typerpc_pipefs (rw)
10.0.4.1@o2ib:10.0.4.2@o2ib:/jdhpc on /lustre type lustre (0)
可以看到lustre已经被挂载成功。