Xenomai 3 and Xenomai 4 do not boot on Raspi CM4 due to kernel panic caused by pcie_brcmstb

I have a Raspberry Pi Compute Module 4 with an IO Board. I want to run Xenomai 3 and 4 on it. PCIe should work, so it recognizes my network adapter. I have built some Xenomai Kernels activated the Broadcom Brcmstb PCIe controller, but due to that I get a Kernel Panic during boot If I blacklist pcie_brcmstb it starts Xenomai normally, but if I type lspci there is no output. I am not sure if I am missing something... This is the output during boot:

[    5.948202] lr : pci_generic_config_read+0x24/0xe0
[    5.948205] sp : ffff800010c134c0
[    5.948208] x29: ffff800010c135f0 x28: ffff3cb4429ec880
[    5.948218] x27: 0000000000000001 x26: 0000000000000000
[    5.948224] x25: 0000000000000001 x24: ffffa446dc4d5540
[    5.948230] x23: 0000000020000005 x22: ffffa446dab93c9c
[    5.948236] x21: ffff800010c13610 x20: 0000ffffffffffff
[    5.948242] x19: 0000000000000004 x18: ffffffffffffffff
[    5.948247] x17: 0000000000000005 x16: 00008a71ea304cb0
[    5.948253] x15: ffff3cb44305da1c x14: ffffa446dc4605a0
[    5.948259] x13: 0000000000000040 x12: 0000000000000000
[    5.948265] x11: 0000000000000000 x10: 0000000000000000
[    5.948270] x9 : 0000000000000228 x8 : 0000000000000000
[    5.948276] x7 : ffff3cb442b25540 x6 : ffffa446dc4633c0
[    5.948282] x5 : ffff800010b30000 x4 : ffff3cb4457fa380
[    5.948288] x3 : 0000000000000000 x2 : 0000000000008000
[    5.948294] x1 : 00000000deaddead x0 : ffff800010b38000
[    5.948300] Kernel panic - not syncing: Asynchronous SError Interrupt
[    5.948304] CPU: 0 PID: 242 Comm: (udev-worker) Not tainted 5.10.209xeno3-00256-ge2e46a0e4e4b-dirty #4
[    5.948307] Hardware name: Raspberry Pi Compute Module 4 Rev 1.1 (DT)
[    5.948309] IRQ stage: Linux
[    5.948311] Call trace:
[    5.948314]  dump_backtrace+0x0/0x1b0
[    5.948316]  show_stack+0x18/0x40
[    5.948318]  dump_stack+0xf4/0x124
[    5.948321]  panic+0x19c/0x36c
[    5.948323]  add_taint+0x0/0xc0
[    5.948325]  arm64_serror_panic+0x78/0x84
[    5.948328]  do_serror+0x38/0xac
[    5.948330]  el1_error+0x90/0x110
[    5.948332]  el1_irq+0x84/0x1c0
[    5.948335]  pci_generic_config_read+0x3c/0xe0
[    5.948338]  pci_bus_read_config_dword+0x7c/0xd0
[    5.948341]  pci_bus_generic_read_dev_vendor_id+0x34/0x1b0
[    5.948343]  pci_scan_single_device+0xa0/0x150
[    5.948346]  pci_scan_slot+0x40/0x120
[    5.948348]  pci_scan_child_bus_extend+0x54/0x2a0
[    5.948351]  pci_scan_bridge_extend+0x148/0x5c4
[    5.948353]  pci_scan_child_bus_extend+0x138/0x2a0
[    5.948356]  pci_scan_root_bus_bridge+0x64/0xdc
[    5.948358]  pci_host_probe+0x18/0xc4
[    5.948361]  brcm_pcie_probe+0x1dc/0x4e4 [pcie_brcmstb]
[    5.948364]  platform_drv_probe+0x54/0xac
[    5.948366]  really_probe+0xec/0x4e0
[    5.948369]  driver_probe_device+0x58/0xec
[    5.948371]  device_driver_attach+0xc0/0xd0
[    5.948373]  __driver_attach+0x68/0x130
[    5.948376]  bus_for_each_dev+0x70/0xd0
[    5.948378]  driver_attach+0x24/0x30
[    5.948381]  bus_add_driver+0x108/0x1fc
[    5.948383]  driver_register+0x78/0x130
[    5.948386]  __platform_driver_register+0x48/0x54
[    5.948389]  brcm_pcie_driver_init+0x24/0x1000 [pcie_brcmstb]
[    5.948391]  do_one_initcall+0x50/0x1c0
[    5.948393]  do_init_module+0x44/0x230
[    5.948396]  load_module+0x1f98/0x26f0
[    5.948398]  __do_sys_finit_module+0xa4/0xf0
[    5.948401]  __arm64_sys_finit_module+0x20/0x30
[    5.948404]  el0_svc_common.constprop.0+0xfc/0x214
[    5.948406]  do_el0_svc+0x28/0xac
[    5.948408]  el0_svc+0x1c/0x2c
[    5.948411]  el0_sync_handler+0xa4/0x12c
[    5.948413]  el0_sync+0x180/0x1c0
[    5.948449] SMP: stopping secondary CPUs
[    5.948452] Kernel Offset: 0x2446ca600000 from 0xffff800010000000
[    5.948455] PHYS_OFFSET: 0xffffc34c00000000
[    5.948457] CPU features: 0x28240022,61806000
[    5.948459] Memory Limit: none

If you need further informations feel free to ask, thanks already.