Added libvirt playbooks

libvirthost.yml

@@ -0,0 +1,6 @@
+---
+- hosts: libvirthost
+  roles:
+    - common
+    - crypto
+    - libvirthost

roles/crypto/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+- name: Cryptsetup part of this tasklist
+  shell: |
+    mkdir -p -m0700 /srv/common/luks
+    ( umask 0077 && dd if=/dev/urandom bs=1 count=4096 of=/srv/common/luks/crypt_dev_md1 conv=fsync )
+    yes fai | cryptsetup luksAddKey /dev/md1 /srv/common/luks/crypt_dev_md1
+    echo "crypt_dev_md1 /dev/md1 /srv/common/luks/crypt_dev_md1 luks,discard,key-slot=0" > /etc/crypttab
+    echo "KEYFILE_PATTERN=*" >> /etc/cryptsetup-initramfs/conf-hook
+    echo "UMASK=0077" >> /etc/initramfs-tools/conf.d/umask.conf
+    update-initramfs -u
+  args:
+    executable: /bin/sh
+    creates: /srv/common/luks
+  become: yes
+

roles/libvirthost/files/etc/default/grub.d/vfio.cfg

@@ -0,0 +1,15 @@
+GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt rd.driver.pre=vfio-pci isolcpus=1,2,3,4,5,6,7 noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off rcu_nocb_poll rcu-nocbs=2-7 default_hugepagesz=1G"
+#GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt rd.driver.pre=vfio-pci vfio-pci.ids=10de:1b06,10de:10ef vfio_iommu_type1.allow_unsafe_interrupts=1 isolcpus=2,3,4,5,6,7"
+#00:00.0 Host bridge [0600]: Intel Corporation 8th Gen Core 8-core Desktop Processor Host Bridge/DRAM Registers [Coffee Lake S] [8086:3e30] (rev 0d)
+#00:01.0 PCI bridge [0604]: Intel Corporation Skylake PCIe Controller (x16) [8086:1901] (rev 0d)
+#00:02.0 VGA compatible controller [0300]: Intel Corporation Device [8086:3e98] (rev 02)
+#00:12.0 Signal processing controller [1180]: Intel Corporation Cannon Lake PCH Thermal Controller [8086:a379] (rev 10)
+#00:14.0 USB controller [0c03]: Intel Corporation Cannon Lake PCH USB 3.1 xHCI Host Controller [8086:a36d] (rev 10)
+#00:14.2 RAM memory [0500]: Intel Corporation Cannon Lake PCH Shared SRAM [8086:a36f] (rev 10)
+#00:16.0 Communication controller [0780]: Intel Corporation Cannon Lake PCH HECI Controller [8086:a360] (rev 10)
+#00:17.0 RAID bus controller [0104]: Intel Corporation SATA Controller [RAID mode] [8086:2822] (rev 10)
+#00:1c.0 PCI bridge [0604]: Intel Corporation Cannon Lake PCH PCI Express Root Port [8086:a338] (rev f0)
+#00:1f.0 ISA bridge [0601]: Intel Corporation Z390 Chipset LPC/eSPI Controller [8086:a305] (rev 10)
+#00:1f.4 SMBus [0c05]: Intel Corporation Cannon Lake PCH SMBus Controller [8086:a323] (rev 10)
+#00:1f.5 Serial bus controller [0c80]: Intel Corporation Cannon Lake PCH SPI Controller [8086:a324] (rev 10)
+#00:1f.6 Ethernet controller [0200]: Intel Corporation Ethernet Connection (7) I219-V [8086:15bc] (rev 10)

roles/libvirthost/files/etc/libvirt/hooks/qemu

@@ -0,0 +1,8 @@
+#!/bin/sh
+GUEST_NAME="$1"
+STATE_NAME="$2"
+
+if [ "$STATE_NAME" = "stopped" ] && [ "$GUEST_NAME" = "rift" ]; then
+    systemctl suspend;
+fi
+

roles/libvirthost/files/etc/modprobe.d/kvm-intel.conf

@@ -0,0 +1,4 @@
+options kvm-intel nested=1
+options kvm-intel enable_shadow_vmcs=1
+options kvm-intel enable_apicv=1
+options kvm-intel ept=1

roles/libvirthost/files/etc/modprobe.d/vfio-pci.conf

@@ -0,0 +1,2 @@
+options vfio-pci ids=10de:1b06,10de:10ef,8086:a36d,8086:a36f
+

roles/libvirthost/files/etc/modules-load.d/vfio.conf

@@ -0,0 +1,4 @@
+vfio
+vfio_iommu_type1
+vfio_pci
+vfio_virqfd

roles/libvirthost/files/etc/network/interfaces.d/eno1

@@ -0,0 +1,2 @@
+iface eno1 manual
+

roles/libvirthost/files/etc/network/interfaces.d/virbr0

@@ -0,0 +1,7 @@
+auto virbr0
+iface virbr0 inet static
+  address 10.91.1.3
+  gateway 10.91.1.1
+  netmask 255.255.255.0
+  bridge_ports eno1
+  

roles/libvirthost/files/etc/systemd/system/kvm-tuning.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Tune some system parameters for better KVM performance
+
+[Service]
+Type=simple
+RemainAfterExit=yes
+ExecStart=/usr/local/bin/kvm-tuning.sh
+
+[Install]
+WantedBy=multi-user.target
+

roles/libvirthost/files/etc/systemd/system/restore_rift.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Restore rift
+After=suspend.target hibernate.target hybrid-sleep.target
+
+[Service]
+ExecStart=/usr/bin/virsh start rift
+#User=my_user_name
+#Environment=DISPLAY=:0
+
+[Install]
+WantedBy=suspend.target hibernate.target hybrid-sleep.target

roles/libvirthost/files/usr/local/bin/kvm-tuning.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+TOTAL_CORES='0-7'
+HOST_CORES='0-1'            # Cores reserved for host
+HOST_CORES_MASK=C0             # 0-1,6-7, bitmask 0b000011000011
+VIRT_CORES='2-7'           # Cores reserved for virtual machine(s)
+
+# Stretch package contains an old version of cset which doesn't work (or never worked)
+#cset set -c $TOTAL_CORES -s machine.slice
+#cset shield --kthread on --cpu $VIRT_CORES
+# Reduce VM jitter: https://www.kernel.org/doc/Documentation/kernel-per-CPU-kthreads.txt
+echo 3 > /proc/sys/vm/drop_caches
+echo 1 > /proc/sys/vm/compact_memory
+# the kernel's dirty page writeback mechanism uses kthread workers. They introduce
+# massive arbitrary latencies when doing disk writes on the host and aren't
+# migrated by cset. Restrict the workqueue to use only cpu 0.
+echo $HOST_CORES_MASK > /sys/bus/workqueue/devices/writeback/cpumask
+echo $HOST_CORES_MASK > /sys/devices/virtual/workqueue/cpumask
+echo "C0" | tee /sys/devices/virtual/workqueue/*/cpumask
+# THP can allegedly result in jitter. Better keep it off.
+echo never > /sys/kernel/mm/transparent_hugepage/enabled
+# Force P-states to P0
+echo "Setting governor on all cores:"
+echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+echo 0 > /sys/bus/workqueue/devices/writeback/numa
+echo 0 > /proc/sys/kernel/watchdog
+echo 300 > /proc/sys/vm/stat_interval
+
+

roles/libvirthost/tasks/main.yml

@@ -0,0 +1,85 @@
+---
+- name: install packages
+  apt:
+    state: latest
+    pkg: 
+      - libvirt-daemon
+      - libvirt-daemon-system
+      - qemu-system-x86
+      - bridge-utils
+      - ovmf
+      - nvme-cli
+      - lm-sensors
+      - libhugetlbfs-bin
+      - iotop
+    install_recommends: no
+    force_apt_get: yes
+  become: yes
+
+- name: copy config files, scripts and services
+  copy:
+    src: "{{ item }}"
+    dest: "/{{ item }}"
+    force: yes
+    mode: preserve
+  become: yes
+  with_items:
+    - etc/default/grub.d/
+    - etc/modules-load.d/
+    - etc/modprobe.d/
+    - etc/network/interfaces.d/
+    - etc/systemd/system/
+    - etc/libvirt/hooks/
+    - usr/local/bin/
+
+- name: hugepages config
+  lineinfile:
+    state: present
+    path: /etc/fstab
+    regexp: "^hugetlbfs"
+    line: "hugetlbfs /dev/hugepages hugetlbfs defaults 0 0"
+  become: yes
+
+- name: add user to grups
+  user:
+    name: plantroon
+    groups: libvirt,kvm
+    append: yes
+  become: yes
+
+- name: kernel param tuning
+  sysctl:
+    state: present
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    sysctl_file: /etc/sysctl.d/vfio.conf
+    reload: yes
+  become: yes
+  with_items:
+      #TOTAL_RAM/2048kB (size in proc/meminfo) 13824 for 32 gig
+    - { name: 'vm.nr_hugepages', value: '32' }
+    - { name: 'vm.stat_interval', value: '120' }
+    - { name: 'kernel.watchdog', value: '0' }
+
+- name: enable tuning service
+  systemd:
+    name: kvm-tuning
+    state: started
+    enabled: yes
+  become: yes
+
+- name: enable restore_rift service
+  systemd:
+    name: restore_rift
+    state: started
+    enabled: yes
+  become: yes
+
+- name: update grub configs to reflect the above changes
+  command: update-grub
+  become: yes
+
+- name: update initramfs
+  command: "update-initramfs -u"
+  become: yes
+