[SOLVED] /boot not mounted on boot

schard · 2021-07-16 11:56:42

I have a server running on a RAID-1 mdraid on two SSDs with / encrypted.
After booting the machine, /boot, which is also the EFI partition is not being mounted automatically, which it should be.
State after a regular boot:

$ lsblk 
NAME       MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINTS
sda          8:0    0 931,5G  0 disk  
├─sda1       8:1    0   500M  0 part  
│ └─md127    9:127  0 499,9M  0 raid1 
└─sda2       8:2    0   930G  0 part  
  └─md126    9:126  0 929,9G  0 raid1 
    └─root 253:0    0 929,9G  0 crypt /
sdb          8:16   0 931,5G  0 disk  
├─sdb1       8:17   0   500M  0 part  
│ └─md127    9:127  0 499,9M  0 raid1 
└─sdb2       8:18   0   930G  0 part  
  └─md126    9:126  0 929,9G  0 raid1 
    └─root 253:0    0 929,9G  0 crypt /

$ cat /proc/cmdline 
initrd=\intel-ucode.img initrd=\initramfs-linux.img root=UUID=2b9dc231-ed74-442c-841d-8254d3d7e91a rw rd.luks.options=bf4b8b61-5907-408c-8fcb-65f77e34cc86=keyfile-timeout=10s rd.luks.key=bf4b8b61-5907-408c-8fcb-65f77e34cc86=/keyfile:UUID=B332-1C72 rd.luks.name=bf4b8b61-5907-408c-8fcb-65f77e34cc86=root init=/usr/lib/systemd/systemd quiet

$ sudo mdadm --detail /dev/md126
/dev/md126:
           Version : 1.2
     Creation Time : Fri Nov 13 12:32:44 2020
        Raid Level : raid1
        Array Size : 975043584 (929.87 GiB 998.44 GB)
     Used Dev Size : 975043584 (929.87 GiB 998.44 GB)
      Raid Devices : 2
     Total Devices : 2
       Persistence : Superblock is persistent

     Intent Bitmap : Internal

       Update Time : Fri Jul 16 13:50:39 2021
             State : clean 
    Active Devices : 2
   Working Devices : 2
    Failed Devices : 0
     Spare Devices : 0

Consistency Policy : bitmap

              Name : archiso:cryptroot
              UUID : e4353e5f:188eaaa7:afbb8df6:a1a914bb
            Events : 5329

    Number   Major   Minor   RaidDevice State
       0       8        2        0      active sync   /dev/sda2
       2       8       18        1      active sync   /dev/sdb2
$ sudo mdadm --detail /dev/md127
/dev/md127:
           Version : 1.0
     Creation Time : Fri Nov 13 15:41:39 2020
        Raid Level : raid1
        Array Size : 511936 (499.94 MiB 524.22 MB)
     Used Dev Size : 511936 (499.94 MiB 524.22 MB)
      Raid Devices : 2
     Total Devices : 2
       Persistence : Superblock is persistent

       Update Time : Fri Jul 16 13:46:13 2021
             State : clean 
    Active Devices : 2
   Working Devices : 2
    Failed Devices : 0
     Spare Devices : 0

Consistency Policy : resync

              Name : archiso:efi
              UUID : 457d2b2d:e0f98963:65fb2d50:43d742ad
            Events : 150

    Number   Major   Minor   RaidDevice State
       0       8        1        0      active sync   /dev/sda1
       2       8       17        1      active sync   /dev/sdb1

$ mount
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
devtmpfs on /dev type devtmpfs (rw,nosuid,size=4007800k,nr_inodes=1001950,mode=755,inode64)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev,inode64)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,size=1613448k,nr_inodes=819200,mode=755,inode64)
cgroup2 on /sys/fs/cgroup type cgroup2 (rw,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
efivarfs on /sys/firmware/efi/efivars type efivarfs (rw,nosuid,nodev,noexec,relatime)
none on /sys/fs/bpf type bpf (rw,nosuid,nodev,noexec,relatime,mode=700)
/dev/mapper/root on / type ext4 (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=31,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13578)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,pagesize=2M)
mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime)
tmpfs on /tmp type tmpfs (rw,nosuid,nodev,nr_inodes=409600,inode64)
tracefs on /sys/kernel/tracing type tracefs (rw,nosuid,nodev,noexec,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,nosuid,nodev,noexec,relatime)
configfs on /sys/kernel/config type configfs (rw,nosuid,nodev,noexec,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,nosuid,nodev,noexec,relatime)
none on /run/credentials/systemd-sysusers.service type ramfs (ro,nosuid,nodev,noexec,relatime,mode=700)
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=806720k,nr_inodes=201680,mode=700,uid=1000,gid=1001,inode64)

$ cat /etc/fstab
# /dev/mapper/root LABEL=root
UUID=2b9dc231-ed74-442c-841d-8254d3d7e91a	/         	ext4      	rw,relatime	0 1

# tracefs
tracefs             	/sys/kernel/tracing	tracefs   	rw,nosuid,nodev,noexec	0 0

# /dev/md127 LABEL=EFI
UUID=B332-1C72      	/boot     	vfat      	rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,utf8,errors=remount-ro	0 2

$ cat /etc/mkinitcpio.conf
# vim:set ft=sh
# MODULES
# The following modules are loaded before any boot hooks are
# run.  Advanced users may wish to specify all system modules
# in this array.  For instance:
#     MODULES="piix ide_disk reiserfs"
MODULES=(vfat)

# BINARIES
# This setting includes any additional binaries a given user may
# wish into the CPIO image.  This is run last, so it may be used to
# override the actual binaries included by a given hook
# BINARIES are dependency parsed, so you may safely ignore libraries
BINARIES=()

# FILES
# This setting is similar to BINARIES above, however, files are added
# as-is and are not parsed in any way.  This is useful for config files.
FILES=()

# HOOKS
# This is the most important setting in this file.  The HOOKS control the
# modules and scripts added to the image, and what happens at boot time.
# Order is important, and it is recommended that you do not change the
# order in which HOOKS are added.  Run 'mkinitcpio -H <hook name>' for
# help on a given hook.
# 'base' is _required_ unless you know precisely what you are doing.
# 'udev' is _required_ in order to automatically load modules
# 'filesystems' is _required_ unless you specify your fs modules in MODULES
# Examples:
##   This setup specifies all modules in the MODULES setting above.
##   No raid, lvm2, or encrypted root is needed.
#    HOOKS="base"
#
##   This setup will autodetect all modules for your system and should
##   work as a sane default
#    HOOKS="base udev autodetect block filesystems"
#
##   This setup will generate a 'full' image which supports most systems.
##   No autodetection is done.
#    HOOKS="base udev block filesystems"
#
##   This setup assembles a pata mdadm array with an encrypted root FS.
##   Note: See 'mkinitcpio -H mdadm' for more information on raid devices.
#    HOOKS="base udev block mdadm encrypt filesystems"
#
##   This setup loads an lvm2 volume group on a usb device.
#    HOOKS="base udev block lvm2 filesystems"
#
##   NOTE: If you have /usr on a separate partition, you MUST include the
#    usr, fsck and shutdown hooks.
HOOKS=(base systemd autodetect keyboard sd-vconsole modconf block mdadm_udev sd-encrypt filesystems fsck)

# COMPRESSION
# Use this to compress the initramfs image. By default, gzip compression
# is used. Use 'cat' to create an uncompressed image.
#COMPRESSION="gzip"
#COMPRESSION="bzip2"
#COMPRESSION="lzma"
#COMPRESSION="xz"
#COMPRESSION="lzop"
#COMPRESSION="lz4"
COMPRESSION="cat"

# COMPRESSION_OPTIONS
# Additional options for the compressor
#COMPRESSION_OPTIONS=""

$ sudo find /run/systemd/ -name *.mount
/run/systemd/generator/boot.mount
/run/systemd/generator/sys-kernel-tracing.mount
/run/systemd/generator/local-fs.target.requires/boot.mount
/run/systemd/generator/local-fs.target.requires/sys-kernel-tracing.mount
/run/systemd/generator/local-fs.target.requires/-.mount
/run/systemd/generator/-.mount
/run/systemd/units/invocation:sys-fs-fuse-connections.mount
/run/systemd/units/invocation:sys-kernel-config.mount
/run/systemd/units/invocation:tmp.mount
/run/systemd/units/invocation:sys-kernel-tracing.mount
/run/systemd/units/invocation:sys-kernel-debug.mount
/run/systemd/units/invocation:dev-mqueue.mount
/run/systemd/units/invocation:dev-hugepages.mount
/run/systemd/units/invocation:sysroot.mount

$ sudo cat /run/systemd/generator/boot.mount
# Automatically generated by systemd-fstab-generator

[Unit]
Documentation=man:fstab(5) man:systemd-fstab-generator(8)
SourcePath=/etc/fstab
Before=local-fs.target
After=blockdev@dev-disk-by\x2duuid-B332\x2d1C72.target

[Mount]
Where=/boot
What=/dev/disk/by-uuid/B332-1C72
Type=vfat
Options=rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,utf8,errors=remount-ro

$ cat /run/systemd/generator/local-fs.target.requires/boot.mount
# Automatically generated by systemd-fstab-generator

[Unit]
Documentation=man:fstab(5) man:systemd-fstab-generator(8)
SourcePath=/etc/fstab
Before=local-fs.target
After=blockdev@dev-disk-by\x2duuid-B332\x2d1C72.target

[Mount]
Where=/boot
What=/dev/disk/by-uuid/B332-1C72
Type=vfat
Options=rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,utf8,errors=remount-ro

dmesg
journal

Why is /boot not mounted automatically?

Edit
Due to lack of answers, I delegated this to SO: https://unix.stackexchange.com/question … ed-on-boot

Solution
After some more tinkering, it turned out, that the filesystem had a minor issue that I detected and repaired using dosfstools:

$ sudo dosfsck /dev/md126
fsck.fat 4.2 (2021-01-31)
There are differences between boot sector and its backup.
This is mostly harmless. Differences: (offset:original/backup)
  65:01/00
1) Copy original to backup
2) Copy backup to original
3) No action
[123?q]? 1
Dirty bit is set. Fs was not properly unmounted and some data may be corrupt.
1) Remove dirty bit
2) No action
[12?q]? 1

*** Filesystem was changed ***
The changes have not yet been written, you can still choose to leave the
filesystem unmodified:
1) Write changes
2) Leave filesystem unchanged
[12?q]? 1
/dev/md126: 17 files, 11536/127730 clusters

After that, /boot is mounted automatically (tested with several reboots).
It's a pity, that systemd's corresponding mount unit did not log/report this issue, so I had no idea what was going on.

Last edited by schard (2021-08-05 10:05:54)

Arch Linux

#1 2021-07-16 11:56:42

[SOLVED] /boot not mounted on boot

Board footer