Can't boot from default initramfs (LUKS2, LVM)

smirky · 2023-02-12 09:58:49

Hello!

I have a USB flash drive which I keep around with Arch on it, fully encrypted (including /boot) with EFI, LUKS2, LVM (unnecessary but I prefer to have it).
For some reason, only the fallback initramfs can get me to successfully boot, but the default initramfs can't and drops me to the emergency shell without keyboard support:

Starting systemd-udevd version 252.5-1-arch
ERROR: device 'UUID=311df3f1-0568-4b52-86ed-d9e48915b572' not found. Skipping fsck.
mount: /new_root: can't find UUID=311df3f1-0568-4b52-86ed-d9e48915b572.
You are now being dropped into an emergency shell.
sh: can't access tty: job control turned off
[rootfs ]#

What's even more strange is that my Desktop PC can't boot the default initramfs, but QEMU can!

$ qemu-system-x86_64 -hdb /dev/sdd -m 4G -bios /usr/share/ovmf/x64/OVMF.fd -vga qxl

You may find my configuration below:

# /boot/grub/grub.cfg
#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub-mkconfig using templates
# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
insmod luks2
insmod cryptodisk
insmod lvm
insmod part_gpt
insmod part_msdos
if [ -s $prefix/grubenv ]; then
  load_env
fi
if [ "${next_entry}" ] ; then
   set default="${next_entry}"
   set next_entry=
   save_env next_entry
   set boot_once=true
else
   set default="0"
fi

if [ x"${feature_menuentry_id}" = xy ]; then
  menuentry_id_option="--id"
else
  menuentry_id_option=""
fi

export menuentry_id_option

if [ "${prev_saved_entry}" ]; then
  set saved_entry="${prev_saved_entry}"
  save_env saved_entry
  set prev_saved_entry=
  save_env prev_saved_entry
  set boot_once=true
fi

function savedefault {
  if [ -z "${boot_once}" ]; then
    saved_entry="${chosen}"
    save_env saved_entry
  fi
}

function load_video {
  if [ x$feature_all_video_module = xy ]; then
    insmod all_video
  else
    insmod efi_gop
    insmod efi_uga
    insmod ieee1275_fb
    insmod vbe
    insmod vga
    insmod video_bochs
    insmod video_cirrus
  fi
}

if [ x$feature_default_font_path = xy ] ; then
   font=unicode
else
insmod part_gpt
insmod cryptodisk
insmod luks2
insmod gcry_rijndael
insmod gcry_rijndael
insmod gcry_sha512
insmod lvm
insmod ext2
cryptomount -u bbd2dd1042094879a1e25ee1eff8ff5c
set root='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'
if [ x$feature_platform_search_hint = xy ]; then
  search --no-floppy --fs-uuid --set=root --hint='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'  311df3f1-0568-4b52-86ed-d9e48915b572
else
  search --no-floppy --fs-uuid --set=root 311df3f1-0568-4b52-86ed-d9e48915b572
fi
    font="/usr/share/grub/unicode.pf2"
fi

if loadfont $font ; then
  set gfxmode=auto
  load_video
  insmod gfxterm
  set locale_dir=$prefix/locale
  set lang=en_US
  insmod gettext
fi
terminal_input console
terminal_output gfxterm
if [ x$feature_timeout_style = xy ] ; then
  set timeout_style=menu
  set timeout=5
# Fallback normal timeout code in case the timeout_style feature is
# unavailable.
else
  set timeout=5
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/10_linux ###
menuentry 'Arch Linux' --class arch --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-311df3f1-0568-4b52-86ed-d9e48915b572' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_gpt
        insmod cryptodisk
        insmod luks2
        insmod gcry_rijndael
        insmod gcry_rijndael
        insmod gcry_sha512
        insmod lvm
        insmod ext2
        cryptomount -u bbd2dd1042094879a1e25ee1eff8ff5c
        set root='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'  311df3f1-0568-4b52-86ed-d9e48915b572
        else
          search --no-floppy --fs-uuid --set=root 311df3f1-0568-4b52-86ed-d9e48915b572
        fi
        echo    'Loading Linux linux ...'
        linux   /boot/vmlinuz-linux root=/dev/mapper/adata-root rw cryptdevice=UUID=bbd2dd10-4209-4879-a1e2-5ee1eff8ff5c:cryptlvm root=UUID=311df3f1-0568-4b52-86ed-d9e48915b572 cryptkey=rootfs:/root/secrets/crypto_keyfile.bin loglevel=3 quiet
        echo    'Loading initial ramdisk ...'
        initrd  /boot/intel-ucode.img /boot/initramfs-linux.img
}
submenu 'Advanced options for Arch Linux' $menuentry_id_option 'gnulinux-advanced-311df3f1-0568-4b52-86ed-d9e48915b572' {
        menuentry 'Arch Linux, with Linux linux' --class arch --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-linux-advanced-311df3f1-0568-4b52-86ed-d9e48915b572' {
                load_video
                set gfxpayload=keep
                insmod gzio
                insmod part_gpt
                insmod cryptodisk
                insmod luks2
                insmod gcry_rijndael
                insmod gcry_rijndael
                insmod gcry_sha512
                insmod lvm
                insmod ext2
                cryptomount -u bbd2dd1042094879a1e25ee1eff8ff5c
                set root='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'
                if [ x$feature_platform_search_hint = xy ]; then
                  search --no-floppy --fs-uuid --set=root --hint='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'  311df3f1-0568-4b52-86ed-d9e48915b572
                else
                  search --no-floppy --fs-uuid --set=root 311df3f1-0568-4b52-86ed-d9e48915b572
                fi
                echo    'Loading Linux linux ...'
                linux   /boot/vmlinuz-linux root=/dev/mapper/adata-root rw cryptdevice=UUID=bbd2dd10-4209-4879-a1e2-5ee1eff8ff5c:cryptlvm root=UUID=311df3f1-0568-4b52-86ed-d9e48915b572 cryptkey=rootfs:/root/secrets/crypto_keyfile.bin loglevel=3 quiet
                echo    'Loading initial ramdisk ...'
                initrd  /boot/intel-ucode.img /boot/initramfs-linux.img
        }
        menuentry 'Arch Linux, with Linux linux (fallback initramfs)' --class arch --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-linux-fallback-311df3f1-0568-4b52-86ed-d9e48915b572' {
                load_video
                set gfxpayload=keep
                insmod gzio
                insmod part_gpt
                insmod cryptodisk
                insmod luks2
                insmod gcry_rijndael
                insmod gcry_rijndael
                insmod gcry_sha512
                insmod lvm
                insmod ext2
                cryptomount -u bbd2dd1042094879a1e25ee1eff8ff5c
                set root='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'
                if [ x$feature_platform_search_hint = xy ]; then
                  search --no-floppy --fs-uuid --set=root --hint='lvmid/0JJxqj-EiA1-0j2c-agVu-PQBc-esVl-OoN5g3/Es7wO4-fLFO-dTj3-D5Ty-diEi-BAQm-clYOLH'  311df3f1-0568-4b52-86ed-d9e48915b572
                else
                  search --no-floppy --fs-uuid --set=root 311df3f1-0568-4b52-86ed-d9e48915b572
                fi
                echo    'Loading Linux linux ...'
                linux   /boot/vmlinuz-linux root=/dev/mapper/adata-root rw cryptdevice=UUID=bbd2dd10-4209-4879-a1e2-5ee1eff8ff5c:cryptlvm root=UUID=311df3f1-0568-4b52-86ed-d9e48915b572 cryptkey=rootfs:/root/secrets/crypto_keyfile.bin loglevel=3 quiet
                echo    'Loading initial ramdisk ...'
                initrd  /boot/initramfs-linux-fallback.img
        }
}

### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###
### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/30_uefi-firmware ###
menuentry 'UEFI Firmware Settings' $menuentry_id_option 'uefi-firmware' {
        fwsetup
}
### END /etc/grub.d/30_uefi-firmware ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries.  Simply type the
# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f  ${config_directory}/custom.cfg ]; then
  source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f  $prefix/custom.cfg ]; then
  source $prefix/custom.cfg
fi
### END /etc/grub.d/41_custom ###

# /etc/mkinitcpio.conf
MODULES=(usbhid xhci_hcd)
BINARIES=()
FILES=(/root/secrets/crypto_keyfile.bin)
HOOKS=(base udev autodetect keyboard modconf block encrypt lvm2 filesystems fsck)

# /etc/default/grub
GRUB_DEFAULT=0
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="Arch"
GRUB_CMDLINE_LINUX_DEFAULT="loglevel=3 quiet audit=0"
GRUB_CMDLINE_LINUX="resume=UUID=44254836-80fb-49e3-bbc6-24821946fea4 amdgpu.ppfeaturemask=0xffffffff"
GRUB_PRELOAD_MODULES="part_gpt part_msdos lvm"
GRUB_TIMEOUT_STYLE=menu
GRUB_TERMINAL_INPUT=console
GRUB_GFXMODE=auto
GRUB_GFXPAYLOAD_LINUX=keep
GRUB_DISABLE_RECOVERY=true

# /etc/fstab
# <file system> <dir> <type> <options> <dump> <pass>
UUID=311df3f1-0568-4b52-86ed-d9e48915b572       /               ext4            rw,relatime     0 1

UUID=29C0-453E                                  /efi            vfat            rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,utf8,errors=remount-ro  0 2

 # `lsblk -f`
NAME             FSTYPE      FSVER    LABEL UUID                                   FSAVAIL FSUSE% MOUNTPOINTS
fd0                                                                                               
sda              crypto_LUKS 2              0d92b43f-5dd6-4a21-bc23-644f24061892                  
├─sda1                                                                                            
├─sda2           vfat        FAT32          29C0-453E                               548.2M     0% /efi
└─sda3           crypto_LUKS 2              bbd2dd10-4209-4879-a1e2-5ee1eff8ff5c                  
  └─cryptlvm     LVM2_member LVM2 001       EJR1qt-Zk1x-4bcb-pdC9-Xk5P-CFM5-dV5Iut                
    └─adata-root ext4        1.0            311df3f1-0568-4b52-86ed-d9e48915b572      8.5G    34% /
sr0

I found a similar topic to my problem:
https://bbs.archlinux.org/viewtopic.php?id=247333
Unfortunately, I think his problem was that he didn't configure the grub default file to handle the cryptdevice, but I did and I still have this problem.
Any advice will be appreciated!

frostschutz · 2023-02-12 11:07:43

If fallback works, but regular doesn't, that's usually a case of a missing module / driver for something. Same story if the keyboard doesn't work.

Since you mention Qemu, the hardware in Qemu might be different so if you build your Initramfs from within Qemu, it might be built without modules necessary for the bare metal system.

You have to determine which modules are required and put them into the MODULES variable of your mkinitcpio.conf so they're always included.

Alternatively just stick to the fallback image (or try without autodetect hook).

smirky · 2023-02-12 16:18:10

Yes, if I recall correctly, I did the installation via QEMU, because I didn't want to shutdown my PC entirely and QEMU fits in quite nicely.
Are you saying that if I try to rebuild the initramfs with the USB booted on the bare-metal it may actually fix my problem?

Scimmia · 2023-02-12 16:22:29

fix this problem, yes, but then it may or may not boot in qemu. Again, see the autodetect hook.

smirky · 2023-02-12 21:32:04

After booting on bare metal I ran only:

mkinitcpio -P

I was able to boot on the default initramfs afterwards. I didn't remove the autodetect module.
Now QEMU is not able to boot from the default initramfs, but boots from the fallback. Very strange...

Scimmia · 2023-02-12 21:40:06

Not strange at all, completely expected. In fact, I already told you that would probably happen.Look up what autodetect does.

Arch Linux

#1 2023-02-12 09:58:49

Can't boot from default initramfs (LUKS2, LVM)

#2 2023-02-12 11:07:43

Re: Can't boot from default initramfs (LUKS2, LVM)

#3 2023-02-12 16:18:10

Re: Can't boot from default initramfs (LUKS2, LVM)

#4 2023-02-12 16:22:29

Re: Can't boot from default initramfs (LUKS2, LVM)

#5 2023-02-12 21:32:04

Re: Can't boot from default initramfs (LUKS2, LVM)

#6 2023-02-12 21:40:06

Re: Can't boot from default initramfs (LUKS2, LVM)

Board footer