RBD on KVM
If you PXE boot from an interface other than the first interface, you may need to replace the pxe image supplied with your KVM (qemu) installation.
To do this...
- Go to rom-o-matic.net
- Select a release under the Etherboot Releases section
- Select your NIC type
- Select the .zrom image
- Click configure
- Locate BOOT_INDEX and select the appropriate interface as per your setup
- Click Get ROM
- Replace <quemu-prefix>/share/qemu/pxe-<iface-driver>.bin with this image
To confirm that your image is correct...
$ file pxe-e1000.bin pxe-e1000.bin: BIOS (ia32) ROM Ext. (64*512)
All our tests on KVM have been using the second interface, and hence using ROMs from this site, please see the Support Matrix.
Setting up the KVM Host for Hotwire
You will need to select a KVM home, user and group - we chose /var/lib/kvm, kvm, and kvm respectively.
In the user's home directory, we have...
kvm@epenguin1:~$ cat Makefile kvm@epenguin1:~$ ls -l total 28 -rw-r--r-- 1 kvm kvm 0 2009-04-16 13:59 disc.conf -rw-r--r-- 1 kvm kvm 66 2009-03-13 19:26 disc.conf.no-more-iso-disk drwxr-xr-x 10 kvm kvm 117 2009-03-13 17:57 hugetlbfs drwxr-xr-x 3 kvm kvm 4096 2009-04-20 09:47 images -rw-r--r-- 1 kvm kvm 711 2009-03-13 18:26 mac.conf -rw-r--r-- 1 kvm kvm 322 2009-03-13 16:48 mac-ultimo.conf -rw-r--r-- 1 kvm kvm 5004 2009-04-20 09:47 Makefile -rwxr-xr-x 1 kvm kvm 733 2009-03-13 18:03 prep
The images folder hosts all the built KVM images, while the configuration files define some aspects of the hardware.
First, we'll start with the Makefile...
DEBIAN = debian-testing-amd64-netinst.iso export http_proxy=http://vproxydet.cache.det.nsw.edu.au:80/ #PAGE_SIZE = 2 #HUGETLBFS = `du -sm $(GUEST)|awk '{ print $$1*1.2/$(PAGE_SIZE) }'` #echo $(HUGETLBFS) > /proc/sys/vm/nr_hugepages help: @echo "Usage:" @echo " * To see active guests on this host..." @echo "" @echo " GUEST=ALL make status" @echo "" @echo " * To Hotwire a guests..." @echo " GUEST=$(GUEST) make BUILD" @echo "" @echo " * To see active guests on this host..." @echo "" @echo " GUEST=$(GUEST) make on (Ineractive)" @echo " GUEST=$(GUEST) make ON (Daemonize)" @echo "" @echo " * To POWEROFF an active guest..." @echo "" @echo " GUEST=$(GUEST) make kill" watch: watch -n1 '( /usr/sbin/brctl show; echo; /sbin/ifconfig|grep ^tap; echo; grep Huge /proc/meminfo )' ifdef GUEST #$(warning $(GUEST) $(origin GUEST)) #. x86_64 Supported NICs: i82551 i82557b i82559er ne2k_pci pcnet rtl8139 e1000 virtio NIC_MODEL := e1000 MAC_0 = $(strip $(shell awk -F'|' '$$1 ~ /$(GUEST)-eth0/ { print $$2 }' mac.conf)) MAC_1 = $(strip $(shell awk -F'|' '$$1 ~ /$(GUEST)-eth1/ { print $$2 }' mac.conf)) TAP_0 = $(strip $(shell awk -F'|' '$$1 ~ /$(GUEST)-eth0/ { print $$3 }' mac.conf)) TAP_1 = $(strip $(shell awk -F'|' '$$1 ~ /$(GUEST)-eth1/ { print $$3 }' mac.conf)) DISC_DEV = $(strip $(shell awk -F'|' '$$1 ~ /$(GUEST)/ { print $$2 }' disc.conf)) DISC_IMG = $(strip $(shell awk -F'|' '$$1 ~ /$(GUEST)/ { print $$3 }' disc.conf)) #MOUNTED = $(strip $(wildcard hugetlbfs/$(GUEST))) #ifeq ($(MOUNTED),) #$(error "Missing $(MOUNTED)") #endif conf: @echo "Settings for guest $(GUEST) are..." @echo " * MAC_0 : $(MAC_0)" @echo " * MAC_1 : $(MAC_1)" @echo " * TAP_0 : $(TAP_0)" @echo " * TAP_1 : $(TAP_1)" @echo " * DISC_DEV : $(DISC_DEV)" @echo " * DISC_IMG : $(DISC_IMG)" ################################################################################ #. Guest Networking... netstart: @echo -n " * Starting network for $(GUEST)..." @test -f .$(GUEST).ON && echo "Refused" || ( \ sleep 1; \ sudo /sbin/ifup $(TAP_0) > /dev/null 2>&1; \ sudo /sbin/ifup $(TAP_1) > /dev/null 2>&1; \ echo "Done"; \ ) netstop: @echo -n " * Stopping network for $(GUEST)..." @test -f .$(GUEST).ON && echo "Refused" || ( \ sleep 1; \ sudo /sbin/ifdown $(TAP_0) > /dev/null 2>&1; \ sudo /sbin/ifdown $(TAP_1) > /dev/null 2>&1; \ echo "Done"; \ ) .PHONY: netstart netstop ################################################################################ #. Daemonized Guest... ON: netstart POWERON netstop POWERON: .$(GUEST).ON MOUNT = ifneq ($(DISC_DEV),) .$(GUEST).ON: MOUNT = -$(DISC_DEV) $(DISC_IMG) endif .$(GUEST).ON: images/$(GUEST).img hugetlbfs/$(GUEST) @echo " * Starting $(GUEST)..." @/opt/bin/qemu-system-x86_64 \ -name $(GUEST) \ -m 384 \ -mem-path hugetlbfs/$(GUEST) \ -boot c \ -daemonize \ -net nic,vlan=0,macaddr=$(MAC_0),model=$(NIC_MODEL) \ -net tap,vlan=0,ifname=$(TAP_0),script=no \ -net nic,vlan=1,macaddr=$(MAC_1),model=$(NIC_MODEL) \ -net tap,vlan=1,ifname=$(TAP_1),script=no \ $(MOUNT) $< && touch $@ .PHONY: ON POWERON ################################################################################ #. Interactive Guest... on: netstart poweron netstop MOUNT = ifneq ($(DISC_DEV),) poweron: MOUNT = -$(DISC_DEV) $(DISC_IMG) endif poweron: images/$(GUEST).img hugetlbfs/$(GUEST) /opt/bin/qemu-system-x86_64 \ -name $(GUEST) \ -m 384 \ -mem-path hugetlbfs/$(GUEST) \ -boot c \ -curses \ -net nic,vlan=1,macaddr=$(MAC_0),model=$(NIC_MODEL) \ -net tap,vlan=1,ifname=$(TAP_0),script=no \ -net nic,vlan=0,macaddr=$(MAC_1),model=$(NIC_MODEL) \ -net tap,vlan=0,ifname=$(TAP_1),script=no \ $(MOUNT) $<; poweroff: ps -C qemu-system-x86_64 -o '%P' --noheader ifeq ($(GUEST),ALL) status: @echo "Active KVM guests on this host..." @ps -C qemu-system-x86_64 -o '%P %a' --noheader|awk '{print $$4, $$1}' else kill: PID := $(shell ps -C qemu-system-x86_64 -o '%P %a' --noheader|awk '$$4=="$(GUEST)"{print $$1}') kill: ps -C qemu-system-x86_64 -o '%P %a' --noheader|awk '$$4=="$(GUEST)"{print $$1}' kill $(PID) endif .PHONY: on poweron ################################################################################ #. Building Guest... BUILD: netstart images/$(GUEST).img netstop images/%.img: @/opt/bin/qemu-img create -f raw $@ 10G @/opt/bin/qemu-system-x86_64 \ -name $(GUEST) \ -no-acpi \ -m 384 \ -mem-path hugetlbfs/$(GUEST) \ -no-shutdown \ -no-reboot \ -curses \ -hda $@ \ -boot n \ -net nic,vlan=0,macaddr=$(MAC_0),model=$(NIC_MODEL) \ -net tap,vlan=0,ifname=$(TAP_0),script=no \ -net nic,vlan=1,macaddr=$(MAC_1),model=$(NIC_MODEL) \ -net tap,vlan=1,ifname=$(TAP_1),script=no images/$(DEBIAN): wget 'http://cdimage.debian.org/cdimage/daily-builds/daily/arch-latest/amd64/iso-cd/$@' -O $@ else $(warning "You need to set the GUEST environment variable first. For example, `GUEST=thror make [on|ON]'") endif
The preparation script prep...
#!/bin/sh
umount /var/lib/kvm/hugetlbfs/teispes 2>/dev/null
umount /var/lib/kvm/hugetlbfs/cyrus 2>/dev/null
umount /var/lib/kvm/hugetlbfs/smerdis 2>/dev/null
umount /var/lib/kvm/hugetlbfs/darius 2>/dev/null
umount /var/lib/kvm/hugetlbfs/ochus 2>/dev/null
umount /var/lib/kvm/hugetlbfs/artaxerxes 2>/dev/null
umount /var/lib/kvm/hugetlbfs/xerxes 2>/dev/null
umount /var/lib/kvm/hugetlbfs/sogdianus 2>/dev/null
mount /var/lib/kvm/hugetlbfs/teispes
mount /var/lib/kvm/hugetlbfs/cyrus
mount /var/lib/kvm/hugetlbfs/smerdis
mount /var/lib/kvm/hugetlbfs/darius
mount /var/lib/kvm/hugetlbfs/ochus
mount /var/lib/kvm/hugetlbfs/artaxerxes
mount /var/lib/kvm/hugetlbfs/xerxes
mount /var/lib/kvm/hugetlbfs/sogdianus
chown -R kvm:kvm hugetlbfs
The hugelbfs directory hosts the images as defined by fstab...
hugetlbfs /var/lib/kvm/hugetlbfs/teispes hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/cyrus hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/smerdis hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/darius hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/ochus hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/artaxerxes hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/xerxes hugetlbfs defaults 0 0 hugetlbfs /var/lib/kvm/hugetlbfs/sogdianus hugetlbfs defaults 0 0
The configuration files in turn are as follows...
disc.conf
smerdis | hdb | /srv/hotwire/iso.d/hotwire-xfs-iso-repository.img
mac.conf
teispes-eth0 | ba:db:00:b5:00:01 | tap0 cyrus-eth0 | ba:db:00:b5:00:02 | tap2 smerdis-eth0 | ba:db:00:b5:00:03 | tap4 darius-eth0 | ba:db:00:b5:00:04 | tap6 ochus-eth0 | ba:db:00:b5:00:05 | tap8 artaxerxes-eth0 | ba:db:00:b5:00:06 | tap10 xerxes-eth0 | ba:db:00:b5:00:07 | tap12 sogdianus-eth0 | ba:db:00:b5:00:08 | tap14 teispes-eth1 | ba:db:00:b5:80:01 | tap1 cyrus-eth1 | ba:db:00:b5:80:02 | tap3 smerdis-eth1 | ba:db:00:b5:80:03 | tap5 darius-eth1 | ba:db:00:b5:80:04 | tap7 ochus-eth1 | ba:db:00:b5:80:05 | tap9 artaxerxes-eth1 | ba:db:00:b5:80:06 | tap11 xerxes-eth1 | ba:db:00:b5:80:07 | tap13 sogdianus-eth1 | ba:db:00:b5:80:08 | tap15
