systemd SEGV when reloading - ubuntu 22.04

[mhkool]

On Ubuntu server 22.04 (not 24.04 as previously stated) systemd has crashed with a SEGV signal on two consecutive days at the same time.
early in the morning there is a job "Daily apt upgrade and clean activities..." - it does various things and ends with systemd reload that fails:

Sep 18 06:09:30 srv038 systemd[1]: Starting Daily apt upgrade and clean activities...
Sep 18 06:09:36 srv038 kernel: [64023.277390] audit: type=1400 audit(1726632576.266:84): apparmor="STATUS" operation="profile_replace" info="same as current profile, skipping" profile="unconfined" name="/usr/bin/freshclam" pid=23962 comm="apparmor_parser"
Sep 18 06:09:36 srv038 systemd[1]: Reloading.
Sep 18 06:09:36 srv038 kernel: [64023.418952] systemd[1]: segfault at 55d94f18cbc0 ip 000055d907d74d73 sp 00007ffd4b305660 error 4 in systemd[55d907cee000+e0000]
Sep 18 06:09:36 srv038 kernel: [64023.419797] Code: 00 ba 35 03 00 00 48 8d 35 2c d9 05 00 48 8d 3d 8a 41 08 00 e8 3e e1 f7 ff 66 0f 1f 44 00 00 48 63 57 08 48 8d 05 4d b4 0b 00 <48> 8b 04 d0 48 8b 80 d8 00 00 00 48 83 c408 ff e0 66 66 2e 0f 1f
Sep 18 06:09:36 srv038 systemd[1]: Caught <SEGV>, dumped core as pid 24166.
Sep 18 06:09:36 srv038 systemd[1]: Freezing execution.

On Sep 17 the same thing happened with almost identical messages:
Sep 17 06:28:46 srv038 systemd[1]: Reloading.
Sep 17 06:28:46 srv038 kernel: [6032205.051966] systemd[1]: segfault at 746e756f6d00 ip 00007faad8238149 sp 00007ffe615fb430 error 4 in libsystemd-shared-249.so[7faad815d000+1a9000]
Sep 17 06:28:46 srv038 kernel: [6032205.052764] Code: 48 89 44 24 38 31 c0 f6 47 2f 04 48 8d 05 af a0 18 00 48 89 e5 48 0f 44 f0 48 89 ef e8 50 99 01 00 48 8b 03 48 89 ee 4c 89 e7 <ff> 10 48 89 ef e8 4d 9c 01 00 0f b6 53 2f f6 c2 04 74 24 8b 4b 24
Sep 17 06:29:01 srv038 clamd[724347]: Tue Sep 17 06:29:01 2024 -> ^File path check failure on: /var/amavis/tmp/amavis-20240917T000117-432993-oihZIiju/parts
Sep 17 06:29:01 srv038 kernel: [6032220.281106] audit: type=1400 audit(1726547341.718:2774): apparmor="DENIED" operation="open" profile="/usr/sbin/clamd" name="/var/amavis/tmp/amavis-20240917T000117-432993-oihZIiju/parts/" pid=724347 comm="clamd" requested_mask="r" denied_mask="r" fsuid=119 ouid=124
Sep 17 06:29:28 srv038 earlyoom[2336]: mem avail: 28290 of 128717 MiB (21.98%), swap free: 32715 of 32767 MiB (99.84%)
Sep 17 06:30:17 srv038 systemd[1]: Caught <SEGV>, dumped core as pid 733084.
Sep 17 06:30:17 srv038 systemd[1]: Freezing execution.

This server has lots of software and runs fine otherwise. I did a apt update + upgrade and fixed the apparmor message for clamd on Sep 17 so the Sep 18 crash was with latest updates.

Any ideas on how to resolve this?
Thanks

[mhkool]

gdb bt full of the systemd core dump seems to indicate that systemd chokes on a unit file. I guess it is clamav-freshclam since apt complains about it.

Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".Core was generated by `/sbin/init'.
Program terminated with signal SIGSEGV, Segmentation fault.


warning: Section `.reg-xstate/24166' in core file too small.
#0 0x00007fec4596675b in kill () at ../sysdeps/unix/syscall-template.S:120
120 ../sysdeps/unix/syscall-template.S: No such file or directory.
[Current thread is 1 (LWP 24166)]
(gdb) bt full
#0 0x00007fec4596675b in kill () at ../sysdeps/unix/syscall-template.S:120
No locals.
#1 0x000055d907cfab1c in crash (sig=11) at ../src/core/main.c:257
sa = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0 <repeats 16 times>}},
sa_flags = 0, sa_restorer = 0x0}
pid = <optimized out>
__func__ = "crash"
__PRETTY_FUNCTION__ = "crash"
#2 <signal handler called>
No locals.
#3 0x000055d907d74d73 in unit_active_state (u=<optimized out>) at ../src/core/unit.c:830
__PRETTY_FUNCTION__ = "unit_active_state"
#4 0x000055d907d84d86 in unit_may_gc (u=0x55d908ddfa00) at ../src/core/unit.c:392
state = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
#5 0x000055d907d739fe in unit_may_gc (u=0x55d908ddfa00) at ../src/core/unit.c:389
state = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
_found = <optimized out>
__assert_in_set = <optimized out>
_error = <optimized out>
_u = <optimized out>
_l = <optimized out>
#6 unit_add_to_gc_queue (u=0x55d908ddfa00) at ../src/core/unit.c:468
__PRETTY_FUNCTION__ = "unit_add_to_gc_queue"
--Type <RET> for more, q to quit, c to continue without paging--
#7 0x000055d907d85ab1 in unit_clear_dependencies (u=0x55d908da3000) at ../src/core/unit.c:555
other_deps = 0x0
other = 0x55d908ddfa00
deps = 0x55d908dc4878
__PRETTY_FUNCTION__ = <optimized out>
#8 unit_free (u=0x55d908da3000) at ../src/core/unit.c:698
slice = 0x55d908cb92a0
t = 0x0
__PRETTY_FUNCTION__ = <optimized out>
#9 0x000055d907d45d05 in unit_free (u=<optimized out>) at ../src/core/unit.c:654
slice = <optimized out>
t = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__unique_prefix_i15 = <optimized out>
j = <optimized out>
j = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
--Type <RET> for more, q to quit, c to continue without paging--
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
#10 manager_clear_jobs_and_units (m=m@entry=0x55d908cb78f0) at ../src/core/manager.c:1421
u = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
#11 0x000055d907d4b01a in manager_clear_jobs_and_units (m=0x55d908cb78f0) at ../src/core/manager.c:1418
u = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
#12 manager_reload (m=0x55d908cb78f0) at ../src/core/manager.c:3826
reloading = 0x0
fds = 0x7fec45112ba8
f = 0x55d908f6c940
r = <optimized out>
__PRETTY_FUNCTION__ = "manager_reload"
__func__ = "manager_reload"
#13 0x000055d907cf9a7b in invoke_main_loop (ret_error_message=0x7ffd4b3057f0, ret_switch_root_init=<synthetic pointer>,
ret_switch_root_dir=<synthetic pointer>, ret_fds=0x7ffd4b3057e0, ret_shutdown_verb=<synthetic pointer>,
ret_retval=<synthetic pointer>, ret_reexecute=<synthetic pointer>, saved_rlimit_memlock=0x7ffd4b305820,
saved_rlimit_nofile=0x7ffd4b305830, m=0x55d908cb78f0) at ../src/core/main.c:1934
saved_log_target = <optimized out>
--Type <RET> for more, q to quit, c to continue without paging--
saved_log_level = <optimized out>
r = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
_error = <optimized out>
_level = <optimized out>
_e = <optimized out>
saved_log_target = <optimized out>
saved_log_level = <optimized out>
_level = <optimized out>
_e = <optimized out>
_level = <optimized out>
_e = <optimized out>
_level = <optimized out>
_e = <optimized out>
_ptr_ = <optimized out>
_ptr_ = <optimized out>
_level = <optimized out>
_e = <optimized out>
table = <optimized out>
_level = <optimized out>
_e = <optimized out>
#14 main (argc=1, argv=0x7ffd4b305ab8) at ../src/core/main.c:2910
initrd_timestamp = {realtime = 0, monotonic = 0}
userspace_timestamp = {realtime = 1726568591653978, monotonic = 37908672}
kernel_timestamp = {realtime = 1726568553745306, monotonic = 0}
security_start_timestamp = {realtime = 1726568591655330, monotonic = 37910024}
security_finish_timestamp = {realtime = 1726568591655914, monotonic = 37910609}
--Type <RET> for more, q to quit, c to continue without paging--
saved_rlimit_nofile = {rlim_cur = 1024, rlim_max = 4096}
saved_rlimit_memlock = {rlim_cur = 65536, rlim_max = 65536}
skip_setup = <optimized out>
loaded_policy = false
queue_default_job = <optimized out>
first_boot = <optimized out>
reexecute = false
switch_root_dir = 0x0
switch_root_init = 0x0
before_startup = <optimized out>
after_startup = <optimized out>
timespan = "\000\000\000\000\000\000\000\000\000\303t\255 \211[\367\r\000\000\000\000\000\000\000\000\310\375\377 \377\377\377\377\377\002", '\000' <repeats 15 times>, "\002\000\000\000\000\000\000\000\006\000\000\000\ 000\000\000\200"
shutdown_verb = 0x0
error_message = 0x0
r = <optimized out>
retval = 1
m = 0x55d908cb78f0
fds = 0x0
finish = <optimized out>
systemd = "systemd"
__func__ = "main"
__PRETTY_FUNCTION__ = "main"