1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
ppc: 0.884
vnc: 0.884
TCG: 0.880
graphic: 0.879
i386: 0.873
VMM: 0.872
KVM: 0.871
peripherals: 0.861
permissions: 0.857
performance: 0.842
boot: 0.839
x86: 0.839
register: 0.817
architecture: 0.807
semantic: 0.806
files: 0.804
user-level: 0.803
assembly: 0.801
device: 0.794
debug: 0.780
arm: 0.779
virtual: 0.779
hypervisor: 0.769
PID: 0.743
socket: 0.729
kernel: 0.729
risc-v: 0.720
network: 0.703
mistranslation: 0.695
CloudLinux / CageFS - guest-fsfreeze hangs system
Description of problem:
Since CloudLinux provides CageFS (virtualized file system), each time guest-fsfreeze for Proxmox backup, the system is hangs up and become unavailable. It's caused by cagefs-skeleton mount points:
```
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime,gid=1002,hidepid=2)
devtmpfs on /dev type devtmpfs (rw,nosuid,size=3993656k,nr_inodes=998414,mode=755)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
configfs on /sys/kernel/config type configfs (rw,relatime)
/dev/sda2 on / type ext4 (rw,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=34,pipe_ino=10005,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=10005)
mqueue on /dev/mqueue type mqueue (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
/dev/sda1 on /boot type ext4 (rw,relatime,data=ordered)
/usr/tmpDSK on /tmp type ext4 (rw,nosuid,noexec,relatime,discard,data=ordered)
/usr/tmpDSK on /var/tmp type ext4 (rw,nosuid,noexec,relatime,discard,data=ordered)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,relatime,freezer)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,relatime,devices)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,relatime,cpuacct,cpu)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,relatime,cpuset)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,relatime,memory)
/dev/sda2 on /usr/share/cagefs-skeleton type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
devpts on /usr/share/cagefs-skeleton/dev/pts type devpts (rw,nosuid,relatime,gid=5,mode=620,ptmxmode=000)
/dev/sda2 on /usr/share/cagefs-skeleton/lib type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/lib64 type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/include type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/lib type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/lib64 type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/apache/domlogs type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/3rdparty/bin type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/3rdparty/lib type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/3rdparty/lib64 type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/3rdparty/perl type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/3rdparty/php type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/3rdparty/share type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/Cpanel type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/Whostmgr type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/base type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/cgi-priv type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/cpaddons type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/etc type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/hooks type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/htdocs type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/img-sys type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/install type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/lang type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/lib type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/libexec type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/locale type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/php type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/scripts type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/share type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/shared type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/sys_cpanel/boxtrapper-message type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/var type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/whostmgr type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/local/cpanel/whostmgr/docroot/cgi/softaculous type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/l.v.e-manager/cl.nodejs type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/l.v.e-manager/cl.python type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/locale type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/man type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/terminfo type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/vim type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/usr/share/zoneinfo type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/cpanel/ea4 type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/lib/mysql type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/lib/proxyexec/cagefs.sock type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/lib/spamassassin type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
tmpfs on /usr/share/cagefs-skeleton/run/dbus type tmpfs (rw,nosuid,mode=755)
tmpfs on /usr/share/cagefs-skeleton/run/nscd type tmpfs (rw,nosuid,mode=755)
/dev/sda2 on /usr/share/cagefs-skeleton/var/softaculous type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/spool/at type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/www/cgi-bin type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/www/html type ext4 (rw,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/suphp/sbin type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/cpanel/ea-php73/root/usr/bin type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/cpanel/ea-php73/root/etc type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/cpanel/ea-php74/root/usr/bin type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/cpanel/ea-php74/root/etc type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/cpanel/ea-php80/root/usr/bin type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/opt/cpanel/ea-php80/root/etc type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
/dev/sda2 on /usr/share/cagefs-skeleton/var/lve/lveinfo.ver.cagefs type ext4 (ro,nosuid,relatime,data=ordered,jqfmt=vfsv1,usrjquota=quota.user)
proc on /usr/share/cagefs-skeleton/proc type proc (rw,nosuid,relatime,gid=1002,hidepid=2)
systemd-1 on /usr/share/cagefs-skeleton/proc/sys/fs/binfmt_misc type autofs (rw,nosuid,relatime,fd=34,pipe_ino=10005,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=10005)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=800880k,mode=700)
```
Is there anyway qemu-guest-agent can handle this? I saw a lot of users faced this problem since CloudLinux becomes more popular after RHEL halted future CentOS releases.
CloudLinux has an option to umount/mount cagefs-skeleton, maybe it's something can be implemented - https://docs.cloudlinux.com/command-line_tools/
The same issue happens when JailShell is enabled on cPanel servers (OVH reported about) - https://docs.ovh.com/ca/en/vps/cpanel_auto_backup/
Thank you for your time.
Steps to reproduce:
1. Manually start backup for the VM with qemu-agent enabled.
2. The backup process stuck at "INFO: issuing guest-agent 'fs-freeze' command"
3. The VM become unavailable, you can only unlock it and force reset.
|