1
0
Fork 0
Browse Source

Added libvirt playbooks

master
Jakub Filo 3 years ago
parent
commit
e9125e93de
  1. 6
      libvirthost.yml
  2. 15
      roles/crypto/tasks/main.yml
  3. 15
      roles/libvirthost/files/etc/default/grub.d/vfio.cfg
  4. 8
      roles/libvirthost/files/etc/libvirt/hooks/qemu
  5. 4
      roles/libvirthost/files/etc/modprobe.d/kvm-intel.conf
  6. 2
      roles/libvirthost/files/etc/modprobe.d/vfio-pci.conf
  7. 4
      roles/libvirthost/files/etc/modules-load.d/vfio.conf
  8. 2
      roles/libvirthost/files/etc/network/interfaces.d/eno1
  9. 7
      roles/libvirthost/files/etc/network/interfaces.d/virbr0
  10. 11
      roles/libvirthost/files/etc/systemd/system/kvm-tuning.service
  11. 11
      roles/libvirthost/files/etc/systemd/system/restore_rift.service
  12. 29
      roles/libvirthost/files/usr/local/bin/kvm-tuning.sh
  13. 85
      roles/libvirthost/tasks/main.yml

6
libvirthost.yml

@ -0,0 +1,6 @@
---
- hosts: libvirthost
roles:
- common
- crypto
- libvirthost

15
roles/crypto/tasks/main.yml

@ -0,0 +1,15 @@
---
- name: Cryptsetup part of this tasklist
shell: |
mkdir -p -m0700 /srv/common/luks
( umask 0077 && dd if=/dev/urandom bs=1 count=4096 of=/srv/common/luks/crypt_dev_md1 conv=fsync )
yes fai | cryptsetup luksAddKey /dev/md1 /srv/common/luks/crypt_dev_md1
echo "crypt_dev_md1 /dev/md1 /srv/common/luks/crypt_dev_md1 luks,discard,key-slot=0" > /etc/crypttab
echo "KEYFILE_PATTERN=*" >> /etc/cryptsetup-initramfs/conf-hook
echo "UMASK=0077" >> /etc/initramfs-tools/conf.d/umask.conf
update-initramfs -u
args:
executable: /bin/sh
creates: /srv/common/luks
become: yes

15
roles/libvirthost/files/etc/default/grub.d/vfio.cfg

@ -0,0 +1,15 @@
GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt rd.driver.pre=vfio-pci isolcpus=1,2,3,4,5,6,7 noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off rcu_nocb_poll rcu-nocbs=2-7 default_hugepagesz=1G"
#GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt rd.driver.pre=vfio-pci vfio-pci.ids=10de:1b06,10de:10ef vfio_iommu_type1.allow_unsafe_interrupts=1 isolcpus=2,3,4,5,6,7"
#00:00.0 Host bridge [0600]: Intel Corporation 8th Gen Core 8-core Desktop Processor Host Bridge/DRAM Registers [Coffee Lake S] [8086:3e30] (rev 0d)
#00:01.0 PCI bridge [0604]: Intel Corporation Skylake PCIe Controller (x16) [8086:1901] (rev 0d)
#00:02.0 VGA compatible controller [0300]: Intel Corporation Device [8086:3e98] (rev 02)
#00:12.0 Signal processing controller [1180]: Intel Corporation Cannon Lake PCH Thermal Controller [8086:a379] (rev 10)
#00:14.0 USB controller [0c03]: Intel Corporation Cannon Lake PCH USB 3.1 xHCI Host Controller [8086:a36d] (rev 10)
#00:14.2 RAM memory [0500]: Intel Corporation Cannon Lake PCH Shared SRAM [8086:a36f] (rev 10)
#00:16.0 Communication controller [0780]: Intel Corporation Cannon Lake PCH HECI Controller [8086:a360] (rev 10)
#00:17.0 RAID bus controller [0104]: Intel Corporation SATA Controller [RAID mode] [8086:2822] (rev 10)
#00:1c.0 PCI bridge [0604]: Intel Corporation Cannon Lake PCH PCI Express Root Port [8086:a338] (rev f0)
#00:1f.0 ISA bridge [0601]: Intel Corporation Z390 Chipset LPC/eSPI Controller [8086:a305] (rev 10)
#00:1f.4 SMBus [0c05]: Intel Corporation Cannon Lake PCH SMBus Controller [8086:a323] (rev 10)
#00:1f.5 Serial bus controller [0c80]: Intel Corporation Cannon Lake PCH SPI Controller [8086:a324] (rev 10)
#00:1f.6 Ethernet controller [0200]: Intel Corporation Ethernet Connection (7) I219-V [8086:15bc] (rev 10)

8
roles/libvirthost/files/etc/libvirt/hooks/qemu

@ -0,0 +1,8 @@
#!/bin/sh
GUEST_NAME="$1"
STATE_NAME="$2"
if [ "$STATE_NAME" = "stopped" ] && [ "$GUEST_NAME" = "rift" ]; then
systemctl suspend;
fi

4
roles/libvirthost/files/etc/modprobe.d/kvm-intel.conf

@ -0,0 +1,4 @@
options kvm-intel nested=1
options kvm-intel enable_shadow_vmcs=1
options kvm-intel enable_apicv=1
options kvm-intel ept=1

2
roles/libvirthost/files/etc/modprobe.d/vfio-pci.conf

@ -0,0 +1,2 @@
options vfio-pci ids=10de:1b06,10de:10ef,8086:a36d,8086:a36f

4
roles/libvirthost/files/etc/modules-load.d/vfio.conf

@ -0,0 +1,4 @@
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd

2
roles/libvirthost/files/etc/network/interfaces.d/eno1

@ -0,0 +1,2 @@
iface eno1 manual

7
roles/libvirthost/files/etc/network/interfaces.d/virbr0

@ -0,0 +1,7 @@
auto virbr0
iface virbr0 inet static
address 10.91.1.3
gateway 10.91.1.1
netmask 255.255.255.0
bridge_ports eno1

11
roles/libvirthost/files/etc/systemd/system/kvm-tuning.service

@ -0,0 +1,11 @@
[Unit]
Description=Tune some system parameters for better KVM performance
[Service]
Type=simple
RemainAfterExit=yes
ExecStart=/usr/local/bin/kvm-tuning.sh
[Install]
WantedBy=multi-user.target

11
roles/libvirthost/files/etc/systemd/system/restore_rift.service

@ -0,0 +1,11 @@
[Unit]
Description=Restore rift
After=suspend.target hibernate.target hybrid-sleep.target
[Service]
ExecStart=/usr/bin/virsh start rift
#User=my_user_name
#Environment=DISPLAY=:0
[Install]
WantedBy=suspend.target hibernate.target hybrid-sleep.target

29
roles/libvirthost/files/usr/local/bin/kvm-tuning.sh

@ -0,0 +1,29 @@
#!/usr/bin/env bash
TOTAL_CORES='0-7'
HOST_CORES='0-1' # Cores reserved for host
HOST_CORES_MASK=C0 # 0-1,6-7, bitmask 0b000011000011
VIRT_CORES='2-7' # Cores reserved for virtual machine(s)
# Stretch package contains an old version of cset which doesn't work (or never worked)
#cset set -c $TOTAL_CORES -s machine.slice
#cset shield --kthread on --cpu $VIRT_CORES
# Reduce VM jitter: https://www.kernel.org/doc/Documentation/kernel-per-CPU-kthreads.txt
echo 3 > /proc/sys/vm/drop_caches
echo 1 > /proc/sys/vm/compact_memory
# the kernel's dirty page writeback mechanism uses kthread workers. They introduce
# massive arbitrary latencies when doing disk writes on the host and aren't
# migrated by cset. Restrict the workqueue to use only cpu 0.
echo $HOST_CORES_MASK > /sys/bus/workqueue/devices/writeback/cpumask
echo $HOST_CORES_MASK > /sys/devices/virtual/workqueue/cpumask
echo "C0" | tee /sys/devices/virtual/workqueue/*/cpumask
# THP can allegedly result in jitter. Better keep it off.
echo never > /sys/kernel/mm/transparent_hugepage/enabled
# Force P-states to P0
echo "Setting governor on all cores:"
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
echo 0 > /sys/bus/workqueue/devices/writeback/numa
echo 0 > /proc/sys/kernel/watchdog
echo 300 > /proc/sys/vm/stat_interval

85
roles/libvirthost/tasks/main.yml

@ -0,0 +1,85 @@
---
- name: install packages
apt:
state: latest
pkg:
- libvirt-daemon
- libvirt-daemon-system
- qemu-system-x86
- bridge-utils
- ovmf
- nvme-cli
- lm-sensors
- libhugetlbfs-bin
- iotop
install_recommends: no
force_apt_get: yes
become: yes
- name: copy config files, scripts and services
copy:
src: "{{ item }}"
dest: "/{{ item }}"
force: yes
mode: preserve
become: yes
with_items:
- etc/default/grub.d/
- etc/modules-load.d/
- etc/modprobe.d/
- etc/network/interfaces.d/
- etc/systemd/system/
- etc/libvirt/hooks/
- usr/local/bin/
- name: hugepages config
lineinfile:
state: present
path: /etc/fstab
regexp: "^hugetlbfs"
line: "hugetlbfs /dev/hugepages hugetlbfs defaults 0 0"
become: yes
- name: add user to grups
user:
name: plantroon
groups: libvirt,kvm
append: yes
become: yes
- name: kernel param tuning
sysctl:
state: present
name: "{{ item.name }}"
value: "{{ item.value }}"
sysctl_file: /etc/sysctl.d/vfio.conf
reload: yes
become: yes
with_items:
#TOTAL_RAM/2048kB (size in proc/meminfo) 13824 for 32 gig
- { name: 'vm.nr_hugepages', value: '32' }
- { name: 'vm.stat_interval', value: '120' }
- { name: 'kernel.watchdog', value: '0' }
- name: enable tuning service
systemd:
name: kvm-tuning
state: started
enabled: yes
become: yes
- name: enable restore_rift service
systemd:
name: restore_rift
state: started
enabled: yes
become: yes
- name: update grub configs to reflect the above changes
command: update-grub
become: yes
- name: update initramfs
command: "update-initramfs -u"
become: yes
Loading…
Cancel
Save