# Nested ESXi Setup on KVM

You can find [instructions on how to run ESXi nested on ESXi](https://williamlam.com/nested-virtualization). This is a good solution vor a VMWare based (Home) Lab setup.

It is also possible to install ESXi on a libvirt/KVM based virtualization host. The hardware that works is quite strange. Performance problems are minor though as it is virtual hardware.

## Known Problems

* There was a "[security patch](https://bugs.launchpad.net/ubuntu/+source/rustc-1.62/+bug/1993183)" for kvm_amd nested that broke nested ESXi.
* ESXi 8.0 / VMWare Workstation 17 [needs flushbyasid](https://communities.vmware.com/t5/VMware-Workstation-Pro/VMware-nested-under-KVM-on-Ryzen-stopped-working-on-5-19-kernel/td-p/2956504) and that is not supported when nested virtualization is used.

## Prerequisites

Get the [install ISO for ESXi 7.0.3 from VMWare](https://customerconnect.vmware.com/en/evalcenter?p=free-esxi7). Check back before a new install unless you want to practice patching.
([Install ISO for ESXi 8.0.1](https://customerconnect.vmware.com/en/evalcenter?p=free-esxi8) but that is broken)

## VM settings

All Disks and CDROM: SATA  
Network: Intel e1000 Pro

## Example domain definition

```XML
<domain type="kvm">
  <name>esxi2a</name>
  <uuid>8e94bab6-6a97-470b-8313-b3ce6c2cbc7b</uuid>
  <memory unit="KiB">29360128</memory>
  <currentMemory unit="KiB">29360128</currentMemory>
  <vcpu placement="static">16</vcpu>
  <os>
    <type arch="x86_64" machine="pc-q35-7.2">hvm</type>
    <loader readonly="yes" type="pflash">/usr/share/OVMF/OVMF_CODE_4M.ms.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/esxi2a_VARS.fd</nvram>
  </os>
  <features>
    <acpi/>
    <apic/>
    <vmport state="off"/>
  </features>
  <cpu mode="host-passthrough" check="none" migratable="off"/>
  <clock offset="utc">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native" discard="unmap"/>
      <source dev="/dev/VM-SSD-Disks/esxi-sys"/>
      <target dev="sda" bus="sata" rotation_rate="1"/>
      <boot order="1"/>
      <address type="drive" controller="0" bus="0" target="0" unit="0"/>
    </disk>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native" discard="unmap"/>
      <source dev="/dev/VM-SSD-Disks/esxi-vmfs"/>
      <target dev="sdb" bus="sata" rotation_rate="1"/>
      <address type="drive" controller="0" bus="0" target="0" unit="1"/>
    </disk>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native" discard="unmap"/>
      <source dev="/dev/VM-HDD-Disks/esxi-hdd"/>
      <target dev="sdc" bus="sata"/>
      <address type="drive" controller="0" bus="0" target="0" unit="2"/>
    </disk>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native" discard="unmap"/>
      <source dev="/dev/VM-SSD-Disks/esxi-ssd"/>
      <target dev="sdd" bus="sata" rotation_rate="1"/>
      <address type="drive" controller="0" bus="0" target="0" unit="3"/>
    </disk>
    <disk type="file" device="cdrom">
      <driver name="qemu" type="raw"/>
      <source file="/home/ISOs/VMware-VMvisor-Installer-7.0U3n-21930508.x86_64.iso"/>
      <target dev="sde" bus="sata"/>
      <readonly/>
      <boot order="2"/>
      <address type="drive" controller="0" bus="0" target="0" unit="4"/>
    </disk>
    <controller type="usb" index="0" model="ich9-ehci1">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x7"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci1">
      <master startport="0"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x0" multifunction="on"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci2">
      <master startport="2"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x1"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci3">
      <master startport="4"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x2"/>
    </controller>
    <controller type="sata" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pcie-root"/>
    <controller type="pci" index="1" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="1" port="0x10"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="2" model="pcie-to-pci-bridge">
      <model name="pcie-pci-bridge"/>
      <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
    </controller>
    <controller type="pci" index="3" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="3" port="0x11"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
    </controller>
    <controller type="pci" index="4" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="4" port="0x12"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
    </controller>
    <controller type="pci" index="5" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="5" port="0x13"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
    </controller>
    <controller type="virtio-serial" index="0">
      <address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
    </controller>
    <interface type="network">
      <mac address="52:54:00:e5:1b:ab"/>
      <source network="inetbridge"/>
      <model type="vmxnet3"/>
      <address type="pci" domain="0x0000" bus="0x02" slot="0x01" function="0x0"/>
    </interface>
    <interface type="network">
      <mac address="52:54:00:2a:b1:d6"/>
      <source network="inetbridge"/>
      <model type="vmxnet3"/>
      <address type="pci" domain="0x0000" bus="0x02" slot="0x02" function="0x0"/>
    </interface>
    <interface type="network">
      <mac address="52:54:00:00:2c:28"/>
      <source network="sanbridge"/>
      <model type="vmxnet3"/>
      <address type="pci" domain="0x0000" bus="0x02" slot="0x03" function="0x0"/>
    </interface>
    <serial type="pty">
      <target type="isa-serial" port="0">
        <model name="isa-serial"/>
      </target>
    </serial>
    <console type="pty">
      <target type="serial" port="0"/>
    </console>
    <channel type="spicevmc">
      <target type="virtio" name="com.redhat.spice.0"/>
      <address type="virtio-serial" controller="0" bus="0" port="1"/>
    </channel>
    <input type="tablet" bus="usb">
      <address type="usb" bus="0" port="1"/>
    </input>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <graphics type="spice" autoport="yes" listen="192.168.1.241">
      <listen type="address" address="192.168.1.241"/>
    </graphics>
    <audio id="1" type="spice"/>
    <video>
      <model type="qxl" ram="65536" vram="65536" vgamem="16384" heads="1" primary="yes"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"/>
    </video>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="2"/>
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="3"/>
    </redirdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
    </memballoon>
  </devices>
</domain>
```