root@pve-2:/etc/ceph# pvecm status Quorum information ------------------ Date: Mon Sep 24 19:11:19 2018 Quorum provider: corosync_votequorum Nodes: 3 Node ID: 0x00000002 Ring ID: 1/12 Quorate: Yes Votequorum information ---------------------- Expected votes: 3 Highest expected: 3 Total votes: 3 Quorum: 2 Flags: Quorate Membership information ---------------------- Nodeid Votes Name 0x00000001 1 192.168.1.51 0x00000002 1 192.168.1.52 (local) 0x00000003 1 192.168.1.53
此时,随便登陆3个节点中的任何一台的web界面,都可以管理整个集群,如下图:
四.集群使用ceph存储
使用外部ceph集群里创建的存储池 ceph1,登陆web管理界面,如下图:
注意,ceph-external这个pool就是上面创建的ceph池。
创建ceph认证文件:
在任意一个节点上运行:
mkdir /etc/pve/priv/ceph cp /etc/ceph/ceph.client.admin.keyring /etc/pve/priv/ceph/ceph1.keyring
注意:ceph1.keyring 中的ceph1 是指存储的ID号ceph1
由于建立了集群,使用了proxmoxfs集群文件系统,/etc/pve目录属于集群资源,3个节点共用,因此只需要在任何一个节点执行上面的命令即可。
接下来,上传ISO镜像,生成虚拟机,使用ceph存储,如下图:
生成虚拟机之后,启动虚拟机,启动失败,报这个错误:
<> "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS".
应该是我的测试环境ProxmoxVE的问题,查了一下,应该是和虚拟化嵌套开启有关,参考了这篇文章:
https://www.linuxidc.com/Linux/2018-12/15576870.htm
具体就不多解释了,在ProxmoxVE物理机上(注意,不是上面3个pve-1/pve-2/pve-3)执行下面的命令:
root@pve:~# modprobe -r kvm_intel root@pve:~# modprobe kvm_intel nested=1 root@pve:~# cat /sys/module/kvm_intel/parameters/nested Y root@pve:~# echo "options kvm_intel nested=1" >> /etc/modprobe.d/modprobe.conf root@pve:~# qm showcmd 111 /usr/bin/kvm -id 111 -name pve-1 -chardev 'socket,id=qmp,path=/var/run/qemu-server/111.qmp,server,nowait' -mon 'chardev=qmp,mode=control' -pidfile /var/run/qemu-server/111.pid -daemonize -smbios 'type=1,uuid=d9eb0729-f0ee-4176-836d-308b70d13754' -smp '3,sockets=1,cores=3,maxcpus=3' -nodefaults -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' -vga std -vnc unix:/var/run/qemu-server/111.vnc,x509,password -cpu kvm64,+lahf_lm,+sep,+kvm_pv_unhalt,+kvm_pv_eoi,enforce -m 12000 -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' -iscsi 'initiator-name=iqn.1993-08.org.debian:01:b48afece2d1' -drive 'file=/var/lib/vz/template/iso/proxmox-ve_5.2-1.iso,if=none,id=drive-ide2,media=cdrom,aio=threads' -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' -drive 'file=/dev/pvevg2/vm-111-disk-7,if=none,id=drive-scsi0,format=raw,cache=none,aio=native,detect-zeroes=on' -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' -drive 'file=/dev/pvevg2/vm-111-disk-2,if=none,id=drive-scsi1,format=raw,cache=none,aio=native,detect-zeroes=on' -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1' -drive 'file=/dev/pvevg2/vm-111-disk-3,if=none,id=drive-scsi2,format=raw,cache=none,aio=native,detect-zeroes=on' -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=2,drive=drive-scsi2,id=scsi2' -drive 'file=/dev/pvevg2/vm-111-disk-6,if=none,id=drive-scsi3,format=raw,cache=none,aio=native,detect-zeroes=on' -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=3,drive=drive-scsi3,id=scsi3' -netdev 'type=tap,id=net0,ifname=tap111i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=76:60:17:9D:6A:FF,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' -netdev 'type=tap,id=net1,ifname=tap111i1,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=6A:93:EB:0E:A8:84,netdev=net1,bus=pci.0,addr=0x13,id=net1,bootindex=301' 找到“-cpu kvm64,+lahf_lm,+sep,+kvm_pv_unhalt,+kvm_pv_eoi,enforce”这部分,在enforce前面加“+vmx,”参数如下: -cpu kvm64,+lahf_lm,+sep,+kvm_pv_unhalt,+kvm_pv_eoi,+vmx,enforce 然后执行:qm stop 111 停止虚拟机后,再执行上面修改后的命令。 虚拟机启动后,ssh进入,执行 grep vmx /proc/cpuinfo 查看是不是有输出,如下: root@pve-1:~# grep vmx /proc/cpuinfo flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx lm constant_tsc nopl xtopology cpuid pni vmx cx16 x2apic hypervisor lahf_lm cpuid_fault pti tpr_shadow vnmi flexpriority ept vpid flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx lm constant_tsc nopl xtopology cpuid pni vmx cx16 x2apic hypervisor lahf_lm cpuid_fault pti tpr_shadow vnmi flexpriority ept vpid flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx lm constant_tsc nopl xtopology cpuid pni vmx cx16 x2apic hypervisor lahf_lm cpuid_fault pti tpr_shadow vnmi flexpriority ept vpid 表示已经支持虚拟嵌套。
再重新启动上面创建的虚拟机,成功!
安装过程中观察ceph -w 实时输出: