使用Vagrant搭建openstack或ovn环境(by quqi99)

作者：张华 发表于：2021-08-27
版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明
(http://blog.csdn.net/quqi99 )

LXD在LXC的基础上包装了一层让容器更易用，Vagrant类似包了一层让虚机更易用，这样Vagrant非常适合搭建多机的测试环境，如openstack环境，如ovn组网环境。

Vagrant用法

#sudo apt install -y virt-manager vagrant vagrant-libvirt -y
sudo apt install curl xml2 bridge-utils cpu-checker -y
sudo apt install libguestfs-tools libvirt-clients libvirt-daemon-system libvirt-dev qemu-kvm virt-manager -y
#install vagrant and vagrant-libvirt plugin
#we should not install vagrant via apt-get to avoid some errors
curl -O https://releases.hashicorp.com/vagrant/2.2.19/vagrant_2.2.19_x86_64.deb
sudo dpkg -i ./vagrant_2.2.19_x86_64.deb
sudo sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list && sudo apt update
sudo apt-get build-dep vagrant ruby-libvirt -y
sudo apt-get install dnsmasq-base ebtables libvirt-dev libxml2-dev libxslt1-dev qemu ruby-dev zlib1g-dev -y
proxychains sudo gem update
gem sources --add https://gems.ruby-china.com/ --remove https://rubygems.org/
sudo gem sources --add https://gems.ruby-china.com/ --remove https://rubygems.org/
gem sources -l
sudo gem sources -l
#vagrant plugin uninstall vagrant-libvirt
sudo vagrant plugin install vagrant-libvirt --plugin-clean-sources --plugin-source https://gems.ruby-china.com/
sudo vagrant plugin list
#test vagrant box
sudo vagrant box add generic/ubuntu2004 --provider libvirt
sudo vagrant box list
mkdir -p ~/vagrant-vms && cd ~/vagrant-vms
cat <<EOF | tee Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :
ENV['VAGRANT_DEFAULT_PROVIDER'] = 'libvirt'
Vagrant.configure("2") do |config|
  config.vm.define "test" do |config|
  config.vm.hostname = "test"
  config.vm.box = "generic/ubuntu2004"
  config.vm.box_check_update = false
  config.vm.network "private_network", ip: "192.168.18.9"
  config.vm.provider :libvirt do |v|
    v.memory = 1024
    end
  end
end
EOF
sudo vagrant up --provider libvirt
sudo vagrant up --provider libvirt --provision
sudo vagrant ssh test
sudo vagrant ssh-config
sudo vagrant halt
sudo vagrant destroy -f
ssh -i /home/hua/vagrant-vms/.vagrant/machines/cent-01/libvirt/private_key vagrant@192.168.121.6

Vagrant openstack

注意修改池名，否则会报：No usable storage pool found

sudo apt install -y virt-manager vagrant vagrant-libvirt -y
git clone https://github.com/zhhuabj/maas-sandbox.git
cd maas-sandbox && ssh-keygen -q -t rsa -f ./id_rsa -N "" && cat id_rsa.pub >> ~/.ssh/authorized_keys
#No usable storage pool found
sed -i -i 's/images/default/g' ./Vagrantfile
sudo virsh pool-destroy default
sudo virsh pool-undefine default
sudo mkdir -p /images && sudo chown -R $USER /images
sudo virsh pool-define-as --name default --type dir --target /images
sudo virsh pool-autostart default
sudo virsh pool-start default
sudo virsh pool-list
VAGRANT_LOG=debug sudo vagrant up --provider libvirt
$ ls .vagrant/machines/
maas  node01  node02  node03  node04  node05

此实验都未完成。原因是使用国内机器做测试存在特色网络问题（MAAS is still importing images, waiting 30s，此处极慢），使用国外机器可能是因为多网卡的原因，在创建的虚机里面存在dns问题，systemd-resolved --status会看到有多个nameservers, 这些来自netplan配置，正确的应该是192.168.121.1，在/etc/resolve.conf改成192.168.121.1就好了。但尚不清楚如何在vagrant里配置。当然想继续实验，只能暂时将机器创建好了，改了resolve.conf之后，再手动执行vagrantfile中的脚本了。

20210831更新 - 解决dns问题

上面恶心的dns问题终于解决：

• 通过修改netplan去掉4.2.2.1, 4.2.2.2, 208.67.220.220，然后再通过netplan生成systemd-resolved项
• 使用192.168.121.1还不行，还得使用它的上游10.230.56.2才行。

ubuntu@saha:~/vagrant-vms$ cat Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :
ENV['VAGRANT_DEFAULT_PROVIDER'] = 'libvirt'
Vagrant.configure("2") do |config|
  config.vm.define "test" do |config|
  config.vm.hostname = "test"
  config.vm.box = "generic/ubuntu2004"
  config.vm.box_check_update = false
  config.vm.network "private_network", ip: "192.168.18.9", nameservers: "192.168.18.1"
  config.vm.provider :libvirt do |v|
    v.memory = 1024
#    v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
#    v.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
    end
  end
  config.vm.provision "shell", run: "always", inline: <<-SHELL
  cat <<-EOF > /etc/netplan/01-netcfg.yaml
network:
  version: 2
  renderer: networkd
  ethernets:
    eth0:
      dhcp4: true
      dhcp6: false
      optional: true
      nameservers:
        addresses: [10.230.56.2]
EOF
  netplan generate
  systemctl restart systemd-networkd.service
  systemctl restart ifplugd.service
  SHELL
end

为什么会多出4.2.2.1, 4.2.2.2, 208.67.220.220这些DNS出来了，见：
https://github.com/lavabit/robox/issues/54
https://www.reddit.com/r/linuxquestions/comments/l8p3d2/networkd_how_to_change_dns_in_ubuntu_2010_server/

Vagrant OVN

因为我的实验机器有物理上的防火墙限制，只能访问少数如git和ubuntu的一些站名，不能访问redhat等其他网站。所以使用了附录中的diff将centos改成了ubuntu

git clone https://github.com/danalsan/vagrants.git
cd /bak/work/vagrants/ovn-external-ports
sudo vagrant up --provider=libvirt
sudo vagrant ssh <vm-name>
hua@node1:/bak/work/vagrants/ovn-external-ports$ ls .vagrant/machines/
central  gw1  gw2  host1  worker1  worker2
sudo vagrant ssh maas

附录

ubuntu@saha:~/vagrants/ovn-external-ports$ git diff
diff --git a/ovn-external-ports/Vagrantfile b/ovn-external-ports/Vagrantfile
index fa6c027..d0839f9 100644
--- a/ovn-external-ports/Vagrantfile
+++ b/ovn-external-ports/Vagrantfile
@@ -23,12 +23,12 @@ Vagrant.configure(2) do |config|
         lb.memory = vm_memory
         lb.cpus = vm_cpus
         lb.suspend_mode = 'managedsave'
-        lb.storage_pool_name = 'images'
+        lb.storage_pool_name = 'default'
     end
 
     config.ssh.forward_agent = true
     config.vm.hostname = "ovnhost"
-    config.vm.box = "centos/7"
+    config.vm.box = "generic/ubuntu2004"
     config.vm.synced_folder './', '/vagrant', type: 'rsync'
     config.vm.synced_folder '../utils', '/vagrant/utils', type: 'rsync'
 
diff --git a/utils/common-functions b/utils/common-functions
index 8b9a245..b688d63 100644
--- a/utils/common-functions
+++ b/utils/common-functions
@@ -1,13 +1,19 @@
 function centos_setup {
-    sudo setenforce 0
-    sudo sed -i 's/^SELINUX=.*/SELINUX=permissive/g' /etc/selinux/config
-
-    sudo yum group install "Development Tools" -y
-    sudo yum install net-tools tcpdump -y
-    sudo yum install epel-release -y
-    sudo yum install python3-devel python-six openssl-devel python36-six -y
-    sudo yum install kernel-devel
-    sudo yum install bmon -y
+    #sudo setenforce 0
+    #sudo sed -i 's/^SELINUX=.*/SELINUX=permissive/g' /etc/selinux/config
+
+    #sudo yum group install "Development Tools" -y
+    #sudo yum install net-tools tcpdump -y
+    #sudo yum install epel-release -y
+    #sudo yum install python3-devel python-six openssl-devel python36-six -y
+    #sudo yum install kernel-devel
+    #sudo yum install bmon -y
+
+   sudo apt install build-essential -y
+   sudo apt install net-tools tcpdump -y
+   sudo apt install python3-dev python-six libssl-dev python36-six -y
+   sudo apt install linux-headers-$(uname -r)
+   sudo apt install bmon -y
 }
 
 function install_ovs {

20220527 - vagrant-lxd

首先根据文档安装lxd - https://blog.csdn.net/quqi99/article/details/52131486
然后vagrant-lxd,

# https://gitlab.com/catalyst-it/devtools/vagrant-lxd
curl -O https://releases.hashicorp.com/vagrant/2.2.19/vagrant_2.2.19_x86_64.deb
sudo dpkg -i vagrant_2.2.19_x86_64.deb
sudo vagrant plugin install vagrant-lxd
sudo vagrant plugin list
# configure lxd to accept HTTPS connections from your machine
lxc config set core.https_address 127.0.0.1
lxc config get core.https_address
curl --insecure https://127.0.0.1:8443
# configure LXD to allow requests from this machine
sudo /snap/bin/lxc config trust add /root/.vagrant.d/data/lxd/client.crt

这样使用它：

# https://app.vagrantup.com/boxes/search?provider=lxc
# /root/.vagrant.d/gems/2.7.4/gems/vagrant-lxd-0.5.6/lib/vagrant-lxd/driver.rb
echo root:$(id -u):1 | sudo tee -a /etc/subuid
echo root:$(id -g):1 | sudo tee -a /etc/subgid
cat <<EOF | tee Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :
ENV['VAGRANT_DEFAULT_PROVIDER'] = 'lxd'
Vagrant.configure('2') do |config|
  config.vm.define "test1" do |config|
  config.vm.hostname = "test1"
  config.vm.box = 'hibox/focal64'
  config.vm.provider 'lxd' do |lxd|
    lxd.nesting = true
    lxd.privileged = true
    lxd.profiles = ['juju-default-eth0']
    end
  end
end
EOF
sudo vagrant up --provider lxd
sudo vagrant ssh test1
sudo vagrant halt
sudo vagrant destroy -f
sudo vagrant ssh-config
ssh -i .vagrant/machines/test1/lxd//private_key vagrant@192.168.122.100

但要注意一点，容器内网卡名字必须使用eth0, 因为代码里写死了(/root/.vagrant.d/gems/2.7.4/gems/vagrant-lxd-0.5.6/lib/vagrant-lxd/driver.rb)

cat << EOF | tee ./lxd-profile.yaml
config:
  boot.autostart: "true"
  linux.kernel_modules: openvswitch,nbd,ip_tables,ip6_tables
  security.nesting: "true"
  security.privileged: "true"
description: ""
devices:
  eth0:
    mtu: "9000"
    name: eth0
    nictype: bridged
    parent: lxdbr0
    type: nic
  eth1:
    mtu: "9000"
    name: eth1
    nictype: bridged
    parent: lxdbr0
    type: nic
  kvm:
    path: /dev/kvm
    type: unix-char
  mem:
    path: /dev/mem
    type: unix-char
  root:
    path: /
    pool: default
    type: disk
  tun:
    path: /dev/net/tun
    type: unix-char
name: juju-default-eth0
used_by: []
EOF
lxc profile create juju-default-eth0 2>/dev/null || echo "juju-default-eth0 profile already exists"
cat ./lxd-profile.yaml |lxc profile edit juju-default-eth0
#lxc profile device set juju-default root pool=default
lxc profile show juju-default-eth0

另外，不清楚怎么固定IP啊

20221230 - Try multipath

ubuntu有一个更方便创建虚机的工具叫multipath， 试了一下，但虚机的cpu总是高达500%

#but multipass cause high cpu usage so give up
sudo snap install multipass
#multipass stop --all
#multipass set local.driver=qemu                                     
#multipass start --all
#multipass get local.driver
multipass find |grep jammy
multipass launch 22.04 --name maas -c 1 --disk 24G --mem 4G
multipass list
multipass exec maas -- bash