From 64d3c7b5150c8556361cede3d12b4fd6a6ff546e Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Tue, 14 Jan 2025 20:24:00 +0000 Subject: grub/xhci: Add xHCI non-root-hub fixes from Nitrokey See: https://github.com/Nitrokey/nethsm-grub/commits/nethsm-z790?since=2025-01-13&until=2025-01-13 And more generally, see branch: https://github.com/Nitrokey/nethsm-grub/commits/nethsm-z790 This brings in a few minor fixes, and also a not-so-minor fix: Add TT (transaction translation) handling for non-SuperSpeed devices in xhci.c More generally, this patchset will improve non-root hub support in the xHCI code. There is also a patch to work around a quirk on the MSI Z790-P mainboard, which I'm planning to add to Libreboot at a later date. Signed-off-by: Leah Rowe --- ...ub-s-missing-characters-for-borders-arrow.patch | 4 +- ...2-say-the-name-libreboot-in-the-grub-menu.patch | 6 +- .../grub/xhci/patches/0003-Add-CC0-license.patch | 4 +- .../xhci/patches/0004-Define-GRUB_UINT32_MAX.patch | 4 +- .../xhci/patches/0005-Add-Argon2-algorithm.patch | 4 +- ...0006-Error-on-missing-Argon2id-parameters.patch | 4 +- .../0007-Compile-with-Argon2id-support.patch | 4 +- .../0008-Make-grub-install-work-with-Argon2.patch | 4 +- ...board-coreboot-force-scancodes2-translate.patch | 4 +- ...eylayouts-don-t-print-Unknown-key-message.patch | 4 +- ...print-missing-prefix-errors-on-the-screen.patch | 4 +- ...012-don-t-print-error-if-module-not-found.patch | 4 +- .../0013-don-t-print-empty-error-messages.patch | 4 +- ...us-usb-Parse-SuperSpeed-companion-descrip.patch | 4 +- .../xhci/patches/0015-usb-Add-enum-for-xHCI.patch | 6 +- ...-usbtrans-Set-default-maximum-packet-size.patch | 6 +- ...us-usb-Add-function-pointer-for-attach-de.patch | 4 +- ...us-usb-usbhub-Add-new-private-fields-for-.patch | 6 +- .../0019-grub-core-bus-usb-Add-xhci-support.patch | 13 +- ...us-usb-usbhub-Add-xHCI-non-root-hub-suppo.patch | 6 +- .../patches/0021-Fix-compilation-on-x86_64.patch | 90 -- .../0021-xHCI-also-accept-SBRN-0x31-and-0x32.patch | 26 + ...2-Add-native-NVMe-driver-based-on-SeaBIOS.patch | 1075 -------------------- .../xhci/patches/0022-xhci-fix-port-indexing.patch | 43 + ...karound-z790-non-root-hub-speed-detection.patch | 28 + .../0024-xhci-configure-TT-for-non-root-hubs.patch | 98 ++ .../patches/0025-Fix-compilation-on-x86_64.patch | 90 ++ ...6-Add-native-NVMe-driver-based-on-SeaBIOS.patch | 1074 +++++++++++++++++++ 28 files changed, 1405 insertions(+), 1218 deletions(-) delete mode 100644 config/grub/xhci/patches/0021-Fix-compilation-on-x86_64.patch create mode 100644 config/grub/xhci/patches/0021-xHCI-also-accept-SBRN-0x31-and-0x32.patch delete mode 100644 config/grub/xhci/patches/0022-Add-native-NVMe-driver-based-on-SeaBIOS.patch create mode 100644 config/grub/xhci/patches/0022-xhci-fix-port-indexing.patch create mode 100644 config/grub/xhci/patches/0023-xhci-workaround-z790-non-root-hub-speed-detection.patch create mode 100644 config/grub/xhci/patches/0024-xhci-configure-TT-for-non-root-hubs.patch create mode 100644 config/grub/xhci/patches/0025-Fix-compilation-on-x86_64.patch create mode 100644 config/grub/xhci/patches/0026-Add-native-NVMe-driver-based-on-SeaBIOS.patch (limited to 'config/grub/xhci/patches') diff --git a/config/grub/xhci/patches/0001-mitigate-grub-s-missing-characters-for-borders-arrow.patch b/config/grub/xhci/patches/0001-mitigate-grub-s-missing-characters-for-borders-arrow.patch index 757c8e03..a5ddd68f 100644 --- a/config/grub/xhci/patches/0001-mitigate-grub-s-missing-characters-for-borders-arrow.patch +++ b/config/grub/xhci/patches/0001-mitigate-grub-s-missing-characters-for-borders-arrow.patch @@ -1,7 +1,7 @@ -From c94b9ea258feb53773bab06bc2a6c6e66a44f867 Mon Sep 17 00:00:00 2001 +From 98b66bcfe063a4035f851a542e57dc2c4bd51a57 Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Sun, 31 Oct 2021 03:47:05 +0000 -Subject: [PATCH 01/22] mitigate grub's missing characters for borders/arrow +Subject: [PATCH 01/26] mitigate grub's missing characters for borders/arrow characters This cleans up the display on the main screen in GRUB. diff --git a/config/grub/xhci/patches/0002-say-the-name-libreboot-in-the-grub-menu.patch b/config/grub/xhci/patches/0002-say-the-name-libreboot-in-the-grub-menu.patch index 23f84212..f262b303 100644 --- a/config/grub/xhci/patches/0002-say-the-name-libreboot-in-the-grub-menu.patch +++ b/config/grub/xhci/patches/0002-say-the-name-libreboot-in-the-grub-menu.patch @@ -1,14 +1,14 @@ -From da775eafbe79e3100ca2262bde49e3e65978e83f Mon Sep 17 00:00:00 2001 +From 71b146159e0aad33b267d062885ca0635aa23283 Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Sat, 19 Nov 2022 16:30:24 +0000 -Subject: [PATCH 02/22] say the name libreboot, in the grub menu +Subject: [PATCH 02/26] say the name libreboot, in the grub menu --- grub-core/normal/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grub-core/normal/main.c b/grub-core/normal/main.c -index bd4431000..ff16e0f2e 100644 +index bd4431000..74cfc49c0 100644 --- a/grub-core/normal/main.c +++ b/grub-core/normal/main.c @@ -209,7 +209,7 @@ grub_normal_init_page (struct grub_term_output *term, diff --git a/config/grub/xhci/patches/0003-Add-CC0-license.patch b/config/grub/xhci/patches/0003-Add-CC0-license.patch index 44d27eb5..e0a0cd63 100644 --- a/config/grub/xhci/patches/0003-Add-CC0-license.patch +++ b/config/grub/xhci/patches/0003-Add-CC0-license.patch @@ -1,7 +1,7 @@ -From b91e5cafa7863b5a61e97d483f98dcece374721a Mon Sep 17 00:00:00 2001 +From ba1b693f60ab68af207d8f39a5d6e28f74872031 Mon Sep 17 00:00:00 2001 From: Ax333l Date: Thu, 17 Aug 2023 00:00:00 +0000 -Subject: [PATCH 03/22] Add CC0 license +Subject: [PATCH 03/26] Add CC0 license Signed-off-by: Nicholas Johnson --- diff --git a/config/grub/xhci/patches/0004-Define-GRUB_UINT32_MAX.patch b/config/grub/xhci/patches/0004-Define-GRUB_UINT32_MAX.patch index 2d830d43..6fd54747 100644 --- a/config/grub/xhci/patches/0004-Define-GRUB_UINT32_MAX.patch +++ b/config/grub/xhci/patches/0004-Define-GRUB_UINT32_MAX.patch @@ -1,7 +1,7 @@ -From 58e0d7b7bcec2392b81dda613d070804f7c68cf5 Mon Sep 17 00:00:00 2001 +From a3c13eb165cb375017a76b219f1de1604a9dc156 Mon Sep 17 00:00:00 2001 From: Ax333l Date: Thu, 17 Aug 2023 00:00:00 +0000 -Subject: [PATCH 04/22] Define GRUB_UINT32_MAX +Subject: [PATCH 04/26] Define GRUB_UINT32_MAX Signed-off-by: Nicholas Johnson --- diff --git a/config/grub/xhci/patches/0005-Add-Argon2-algorithm.patch b/config/grub/xhci/patches/0005-Add-Argon2-algorithm.patch index ff2dc74e..4900eed2 100644 --- a/config/grub/xhci/patches/0005-Add-Argon2-algorithm.patch +++ b/config/grub/xhci/patches/0005-Add-Argon2-algorithm.patch @@ -1,7 +1,7 @@ -From 65a9b252b79d5d1fa022e3da9ef91459ad8efb94 Mon Sep 17 00:00:00 2001 +From 176751e11424f75155ce64311923acf48fc4ec3c Mon Sep 17 00:00:00 2001 From: Ax333l Date: Thu, 17 Aug 2023 00:00:00 +0000 -Subject: [PATCH 05/22] Add Argon2 algorithm +Subject: [PATCH 05/26] Add Argon2 algorithm Signed-off-by: Nicholas Johnson --- diff --git a/config/grub/xhci/patches/0006-Error-on-missing-Argon2id-parameters.patch b/config/grub/xhci/patches/0006-Error-on-missing-Argon2id-parameters.patch index 22553006..559080cf 100644 --- a/config/grub/xhci/patches/0006-Error-on-missing-Argon2id-parameters.patch +++ b/config/grub/xhci/patches/0006-Error-on-missing-Argon2id-parameters.patch @@ -1,7 +1,7 @@ -From 809908898f9d67979885a651b295ca1fa066039c Mon Sep 17 00:00:00 2001 +From a779f27023bee409d06f6b9023cd907d2f3b959d Mon Sep 17 00:00:00 2001 From: Ax333l Date: Thu, 17 Aug 2023 00:00:00 +0000 -Subject: [PATCH 06/22] Error on missing Argon2id parameters +Subject: [PATCH 06/26] Error on missing Argon2id parameters Signed-off-by: Nicholas Johnson --- diff --git a/config/grub/xhci/patches/0007-Compile-with-Argon2id-support.patch b/config/grub/xhci/patches/0007-Compile-with-Argon2id-support.patch index b77a5751..95fda3c6 100644 --- a/config/grub/xhci/patches/0007-Compile-with-Argon2id-support.patch +++ b/config/grub/xhci/patches/0007-Compile-with-Argon2id-support.patch @@ -1,7 +1,7 @@ -From 26ae5da9ed8382869b699a653988e6c8f32bd3d1 Mon Sep 17 00:00:00 2001 +From 3ab4a66104c772241e42b1f4f42a5890f3059164 Mon Sep 17 00:00:00 2001 From: Ax333l Date: Thu, 17 Aug 2023 00:00:00 +0000 -Subject: [PATCH 07/22] Compile with Argon2id support +Subject: [PATCH 07/26] Compile with Argon2id support Signed-off-by: Nicholas Johnson --- diff --git a/config/grub/xhci/patches/0008-Make-grub-install-work-with-Argon2.patch b/config/grub/xhci/patches/0008-Make-grub-install-work-with-Argon2.patch index 00f94eef..37de98c5 100644 --- a/config/grub/xhci/patches/0008-Make-grub-install-work-with-Argon2.patch +++ b/config/grub/xhci/patches/0008-Make-grub-install-work-with-Argon2.patch @@ -1,7 +1,7 @@ -From 6ce0b0629aa1377237c19290f33008ff81969962 Mon Sep 17 00:00:00 2001 +From 94a89aea5118294173d29bdf49e41f7801e3df48 Mon Sep 17 00:00:00 2001 From: Ax333l Date: Thu, 17 Aug 2023 00:00:00 +0000 -Subject: [PATCH 08/22] Make grub-install work with Argon2 +Subject: [PATCH 08/26] Make grub-install work with Argon2 Signed-off-by: Nicholas Johnson --- diff --git a/config/grub/xhci/patches/0009-at_keyboard-coreboot-force-scancodes2-translate.patch b/config/grub/xhci/patches/0009-at_keyboard-coreboot-force-scancodes2-translate.patch index 31798205..847d2291 100644 --- a/config/grub/xhci/patches/0009-at_keyboard-coreboot-force-scancodes2-translate.patch +++ b/config/grub/xhci/patches/0009-at_keyboard-coreboot-force-scancodes2-translate.patch @@ -1,7 +1,7 @@ -From 3f69e2c35ccf2a37fd35f1821c7b85cc6df07ca7 Mon Sep 17 00:00:00 2001 +From ddd9c783a8b6dec9e116930f2a8ccfac354d7e29 Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Mon, 30 Oct 2023 22:19:21 +0000 -Subject: [PATCH 09/22] at_keyboard coreboot: force scancodes2+translate +Subject: [PATCH 09/26] at_keyboard coreboot: force scancodes2+translate Scan code set 2 with translation should be assumed in every case, as the default starting position. diff --git a/config/grub/xhci/patches/0010-keylayouts-don-t-print-Unknown-key-message.patch b/config/grub/xhci/patches/0010-keylayouts-don-t-print-Unknown-key-message.patch index 742aae38..c2cec5d4 100644 --- a/config/grub/xhci/patches/0010-keylayouts-don-t-print-Unknown-key-message.patch +++ b/config/grub/xhci/patches/0010-keylayouts-don-t-print-Unknown-key-message.patch @@ -1,7 +1,7 @@ -From 44befdaa1ee588591f46c13d057fdef56b87218c Mon Sep 17 00:00:00 2001 +From 1391cfd14088e90e01a477269e2892c89ba2536d Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Tue, 31 Oct 2023 10:33:28 +0000 -Subject: [PATCH 10/22] keylayouts: don't print "Unknown key" message +Subject: [PATCH 10/26] keylayouts: don't print "Unknown key" message on keyboards with stuck keys, this results in GRUB just spewing it repeatedly, preventing use of GRUB. diff --git a/config/grub/xhci/patches/0011-don-t-print-missing-prefix-errors-on-the-screen.patch b/config/grub/xhci/patches/0011-don-t-print-missing-prefix-errors-on-the-screen.patch index 960c693d..b1877070 100644 --- a/config/grub/xhci/patches/0011-don-t-print-missing-prefix-errors-on-the-screen.patch +++ b/config/grub/xhci/patches/0011-don-t-print-missing-prefix-errors-on-the-screen.patch @@ -1,7 +1,7 @@ -From bfecf6d86ccf82fb85af93600931cee589891089 Mon Sep 17 00:00:00 2001 +From 3f1eb3f472d2c92e5a2a14c7c3469154ae450df6 Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Sun, 5 Nov 2023 16:14:58 +0000 -Subject: [PATCH 11/22] don't print missing prefix errors on the screen +Subject: [PATCH 11/26] don't print missing prefix errors on the screen we do actually set the prefix. this patch modifies grub to still set grub_errno and return accordingly, diff --git a/config/grub/xhci/patches/0012-don-t-print-error-if-module-not-found.patch b/config/grub/xhci/patches/0012-don-t-print-error-if-module-not-found.patch index c098f7e8..24a1ce4b 100644 --- a/config/grub/xhci/patches/0012-don-t-print-error-if-module-not-found.patch +++ b/config/grub/xhci/patches/0012-don-t-print-error-if-module-not-found.patch @@ -1,7 +1,7 @@ -From f44fc56f06559313841de2b53637020fbe139803 Mon Sep 17 00:00:00 2001 +From 5a2700d29f8145d94886aa369e4c68d57295d44a Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Sun, 5 Nov 2023 16:36:22 +0000 -Subject: [PATCH 12/22] don't print error if module not found +Subject: [PATCH 12/26] don't print error if module not found still set grub_errno accordingly, and otherwise behave the same. in libreboot, we remove a lot of diff --git a/config/grub/xhci/patches/0013-don-t-print-empty-error-messages.patch b/config/grub/xhci/patches/0013-don-t-print-empty-error-messages.patch index 23672773..d7a52bce 100644 --- a/config/grub/xhci/patches/0013-don-t-print-empty-error-messages.patch +++ b/config/grub/xhci/patches/0013-don-t-print-empty-error-messages.patch @@ -1,7 +1,7 @@ -From e459ed55902003644e7bcd76361cc43247231a99 Mon Sep 17 00:00:00 2001 +From 376f1dbf2c56623e9b9bda3a4079b4b970ab2ebd Mon Sep 17 00:00:00 2001 From: Leah Rowe Date: Sun, 5 Nov 2023 17:25:20 +0000 -Subject: [PATCH 13/22] don't print empty error messages +Subject: [PATCH 13/26] don't print empty error messages this is part two of the quest to kill the prefix error message. after i disabled prefix-related diff --git a/config/grub/xhci/patches/0014-grub-core-bus-usb-Parse-SuperSpeed-companion-descrip.patch b/config/grub/xhci/patches/0014-grub-core-bus-usb-Parse-SuperSpeed-companion-descrip.patch index bc18cb13..d8e1c766 100644 --- a/config/grub/xhci/patches/0014-grub-core-bus-usb-Parse-SuperSpeed-companion-descrip.patch +++ b/config/grub/xhci/patches/0014-grub-core-bus-usb-Parse-SuperSpeed-companion-descrip.patch @@ -1,7 +1,7 @@ -From 85fdf29d37b58be96042d6df24f8bc08bba9906d Mon Sep 17 00:00:00 2001 +From 8094d43abecf758617aec1d4f8c9b1099cb4e67e Mon Sep 17 00:00:00 2001 From: Patrick Rudolph Date: Sun, 15 Nov 2020 19:00:27 +0100 -Subject: [PATCH 14/22] grub-core/bus/usb: Parse SuperSpeed companion +Subject: [PATCH 14/26] grub-core/bus/usb: Parse SuperSpeed companion descriptors Parse the SS_ENDPOINT_COMPANION descriptor, which is only present on USB 3.0 diff --git a/config/grub/xhci/patches/0015-usb-Add-enum-for-xHCI.patch b/config/grub/xhci/patches/0015-usb-Add-enum-for-xHCI.patch index d8330e59..30133543 100644 --- a/config/grub/xhci/patches/0015-usb-Add-enum-for-xHCI.patch +++ b/config/grub/xhci/patches/0015-usb-Add-enum-for-xHCI.patch @@ -1,7 +1,7 @@ -From 26a5ed135c94b295b9a0037f5e7201eb5d38fde4 Mon Sep 17 00:00:00 2001 +From fd92646af5b9a5cc027ce0bd6afa5cc7bc8f46a3 Mon Sep 17 00:00:00 2001 From: Patrick Rudolph -Date: Sun, 15 Nov 2020 19:47:06 +0100 -Subject: [PATCH 15/22] usb: Add enum for xHCI +Date: Mon, 7 Dec 2020 08:41:22 +0100 +Subject: [PATCH 15/26] usb: Add enum for xHCI Will be used in future patches. diff --git a/config/grub/xhci/patches/0016-usbtrans-Set-default-maximum-packet-size.patch b/config/grub/xhci/patches/0016-usbtrans-Set-default-maximum-packet-size.patch index 27402534..a4abb713 100644 --- a/config/grub/xhci/patches/0016-usbtrans-Set-default-maximum-packet-size.patch +++ b/config/grub/xhci/patches/0016-usbtrans-Set-default-maximum-packet-size.patch @@ -1,7 +1,7 @@ -From 13bc84d9f43328ba47273cbcf87c517398dcfc93 Mon Sep 17 00:00:00 2001 +From f7c7c25e274b79a7c8e62250fa9bc49f81bd4456 Mon Sep 17 00:00:00 2001 From: Patrick Rudolph -Date: Sun, 15 Nov 2020 19:48:03 +0100 -Subject: [PATCH 16/22] usbtrans: Set default maximum packet size +Date: Mon, 7 Dec 2020 08:41:23 +0100 +Subject: [PATCH 16/26] usbtrans: Set default maximum packet size Set the maximum packet size to 512 for SuperSpeed devices. diff --git a/config/grub/xhci/patches/0017-grub-core-bus-usb-Add-function-pointer-for-attach-de.patch b/config/grub/xhci/patches/0017-grub-core-bus-usb-Add-function-pointer-for-attach-de.patch index eede3791..6d4ed855 100644 --- a/config/grub/xhci/patches/0017-grub-core-bus-usb-Add-function-pointer-for-attach-de.patch +++ b/config/grub/xhci/patches/0017-grub-core-bus-usb-Add-function-pointer-for-attach-de.patch @@ -1,7 +1,7 @@ -From 1518589ff286b54a27d4b11f017cb43c90cbf566 Mon Sep 17 00:00:00 2001 +From c3d70163d472953b705ff1250e3c4bfec8cb52ad Mon Sep 17 00:00:00 2001 From: Patrick Rudolph Date: Sun, 15 Nov 2020 19:51:42 +0100 -Subject: [PATCH 17/22] grub-core/bus/usb: Add function pointer for +Subject: [PATCH 17/26] grub-core/bus/usb: Add function pointer for attach/detach events The xHCI code needs to be called for attaching or detaching a device. diff --git a/config/grub/xhci/patches/0018-grub-core-bus-usb-usbhub-Add-new-private-fields-for-.patch b/config/grub/xhci/patches/0018-grub-core-bus-usb-usbhub-Add-new-private-fields-for-.patch index 099ef79d..e68985b0 100644 --- a/config/grub/xhci/patches/0018-grub-core-bus-usb-usbhub-Add-new-private-fields-for-.patch +++ b/config/grub/xhci/patches/0018-grub-core-bus-usb-usbhub-Add-new-private-fields-for-.patch @@ -1,7 +1,7 @@ -From 8dd981a95cc7731b63b2d79ce6f98ce2d6b5a993 Mon Sep 17 00:00:00 2001 +From 4b21796042db28db773534d5f0cf83738a72b929 Mon Sep 17 00:00:00 2001 From: Patrick Rudolph -Date: Sun, 15 Nov 2020 19:54:40 +0100 -Subject: [PATCH 18/22] grub-core/bus/usb/usbhub: Add new private fields for +Date: Mon, 7 Dec 2020 08:41:25 +0100 +Subject: [PATCH 18/26] grub-core/bus/usb/usbhub: Add new private fields for xHCI controller Store the root port number, the route, consisting out of the port ID diff --git a/config/grub/xhci/patches/0019-grub-core-bus-usb-Add-xhci-support.patch b/config/grub/xhci/patches/0019-grub-core-bus-usb-Add-xhci-support.patch index aaabb3da..0cf2bda7 100644 --- a/config/grub/xhci/patches/0019-grub-core-bus-usb-Add-xhci-support.patch +++ b/config/grub/xhci/patches/0019-grub-core-bus-usb-Add-xhci-support.patch @@ -1,7 +1,7 @@ -From dde187c07a8f2a1d3357aa362ffcc0fe8dab72d7 Mon Sep 17 00:00:00 2001 +From 75b3d347ea5f3c51615f8ccb670b9f2033f8594f Mon Sep 17 00:00:00 2001 From: Patrick Rudolph -Date: Sun, 15 Nov 2020 19:59:25 +0100 -Subject: [PATCH 19/22] grub-core/bus/usb: Add xhci support +Date: Mon, 7 Dec 2020 08:41:26 +0100 +Subject: [PATCH 19/26] grub-core/bus/usb: Add xhci support Add support for xHCI USB controllers. The code is based on seabios implementation, but has been heavily @@ -41,13 +41,6 @@ TODO: * Test on USB3 hubs * Support for USB 3.1 and USB 3.2 controllers -Tested on qemu using coreboot and grub as payload: - -qemu-system-x86_64 -M q35 -bios $firmware -device qemu-xhci,id=xhci -accel kvm -m 1024M \ - -device usb-storage,drive=thumbdrive,bus=xhci.0,port=3 \ - -drive if=none,format=raw,id=thumbdrive,file=ubuntu-20.04.1-desktop-amd64.iso \ - -device usb-kbd,bus=xhci.0 - Signed-off-by: Patrick Rudolph Signed-off-by: sylv --- diff --git a/config/grub/xhci/patches/0020-grub-core-bus-usb-usbhub-Add-xHCI-non-root-hub-suppo.patch b/config/grub/xhci/patches/0020-grub-core-bus-usb-usbhub-Add-xHCI-non-root-hub-suppo.patch index 6a3266b2..f27e5428 100644 --- a/config/grub/xhci/patches/0020-grub-core-bus-usb-usbhub-Add-xHCI-non-root-hub-suppo.patch +++ b/config/grub/xhci/patches/0020-grub-core-bus-usb-usbhub-Add-xHCI-non-root-hub-suppo.patch @@ -1,7 +1,7 @@ -From 28856bd0646e0396b3d5249a99b731f2743aec5a Mon Sep 17 00:00:00 2001 +From 6d05a393bf192bb5ae3ba1f7d653617b7ab9bb2e Mon Sep 17 00:00:00 2001 From: Patrick Rudolph -Date: Thu, 3 Dec 2020 13:44:55 +0100 -Subject: [PATCH 20/22] grub-core/bus/usb/usbhub: Add xHCI non root hub support +Date: Mon, 7 Dec 2020 08:41:27 +0100 +Subject: [PATCH 20/26] grub-core/bus/usb/usbhub: Add xHCI non root hub support Tested on Intel PCH C246, the USB3 hub can be configured by grub. diff --git a/config/grub/xhci/patches/0021-Fix-compilation-on-x86_64.patch b/config/grub/xhci/patches/0021-Fix-compilation-on-x86_64.patch deleted file mode 100644 index 8f80664c..00000000 --- a/config/grub/xhci/patches/0021-Fix-compilation-on-x86_64.patch +++ /dev/null @@ -1,90 +0,0 @@ -From 040942842dc05c9b94c9be21f01b08da3e25fe6d Mon Sep 17 00:00:00 2001 -From: Patrick Rudolph -Date: Wed, 24 Feb 2021 08:25:41 +0100 -Subject: [PATCH 21/22] Fix compilation on x86_64 - -Signed-off-by: Patrick Rudolph ---- - grub-core/bus/usb/xhci.c | 24 ++++++++++++++++-------- - 1 file changed, 16 insertions(+), 8 deletions(-) - -diff --git a/grub-core/bus/usb/xhci.c b/grub-core/bus/usb/xhci.c -index f4591ffb5..3495bb919 100644 ---- a/grub-core/bus/usb/xhci.c -+++ b/grub-core/bus/usb/xhci.c -@@ -184,7 +184,7 @@ enum - * then we can get it from a trb pointer (provided by evt ring). - */ - #define XHCI_RING(_trb) \ -- ((struct grub_xhci_ring*)((grub_uint32_t)(_trb) & ~(GRUB_XHCI_RING_SIZE-1))) -+ ((struct grub_xhci_ring*)((grub_addr_t)(_trb) & ~(GRUB_XHCI_RING_SIZE-1))) - - /* slot context */ - struct grub_xhci_slotctx { -@@ -495,6 +495,14 @@ grub_xhci_read8(volatile void *addr) { - return (*((volatile grub_uint32_t *)addr)); - } - -+static inline void * -+grub_xhci_read_etrb_ptr(volatile struct grub_xhci_trb *trb) { -+ grub_uint64_t tmp; -+ tmp = (grub_uint64_t)grub_xhci_read32(&trb->ptr_low); -+ tmp |= ((grub_uint64_t)grub_xhci_read32(&trb->ptr_high)) << 32; -+ return (void *)(grub_addr_t)tmp; -+} -+ - static inline grub_uint32_t - grub_xhci_port_read (struct grub_xhci *x, grub_uint32_t port) - { -@@ -664,7 +672,7 @@ static void xhci_process_events(struct grub_xhci *x) - case ER_TRANSFER: - case ER_COMMAND_COMPLETE: - { -- struct grub_xhci_trb *rtrb = (void*)grub_xhci_read32(&etrb->ptr_low); -+ struct grub_xhci_trb *rtrb = grub_xhci_read_etrb_ptr(etrb); - struct grub_xhci_ring *ring = XHCI_RING(rtrb); - volatile struct grub_xhci_trb *evt = &ring->evt; - grub_uint32_t eidx = rtrb - ring->ring + 1; -@@ -697,9 +705,9 @@ static void xhci_process_events(struct grub_xhci *x) - } - grub_xhci_write32(&evts->nidx, nidx); - volatile struct grub_xhci_ir *ir = x->ir; -- grub_uint32_t erdp = (grub_uint32_t)(evts->ring + nidx); -- grub_xhci_write32(&ir->erdp_low, erdp); -- grub_xhci_write32(&ir->erdp_high, 0); -+ grub_uint64_t erdp = (grub_addr_t)(void *)(&evts->ring[nidx]); -+ grub_xhci_write32(&ir->erdp_low, erdp & 0xffffffff); -+ grub_xhci_write32(&ir->erdp_high, erdp >> 32); - } - } - -@@ -800,7 +808,7 @@ static void xhci_trb_queue(volatile struct grub_xhci_ring *ring, - grub_uint32_t xferlen, grub_uint32_t flags) - { - grub_dprintf("xhci", "%s: ring %p data %llx len %d flags 0x%x remain 0x%x\n", __func__, -- ring, data_or_addr, xferlen & 0x1ffff, flags, xferlen >> 17); -+ ring, (unsigned long long)data_or_addr, xferlen & 0x1ffff, flags, xferlen >> 17); - - if (xhci_ring_full(ring)) - { -@@ -1907,7 +1915,7 @@ grub_xhci_setup_transfer (grub_usb_controller_t dev, - if (transfer->type == GRUB_USB_TRANSACTION_TYPE_CONTROL) - { - volatile struct grub_usb_packet_setup *setupdata; -- setupdata = (void *)transfer->transactions[0].data; -+ setupdata = (void *)(grub_addr_t)transfer->transactions[0].data; - grub_dprintf("xhci", "%s: CONTROLL TRANS req %d\n", __func__, setupdata->request); - grub_dprintf("xhci", "%s: CONTROLL TRANS length %d\n", __func__, setupdata->length); - -@@ -1974,7 +1982,7 @@ grub_xhci_setup_transfer (grub_usb_controller_t dev, - /* Assume the ring has enough free space for all TRBs */ - if (flags & TRB_TR_IDT && tr->size <= (int)sizeof(inline_data)) - { -- grub_memcpy(&inline_data, (void *)tr->data, tr->size); -+ grub_memcpy(&inline_data, (void *)(grub_addr_t)tr->data, tr->size); - xhci_trb_queue(reqs, inline_data, tr->size, flags); - } - else --- -2.39.5 - diff --git a/config/grub/xhci/patches/0021-xHCI-also-accept-SBRN-0x31-and-0x32.patch b/config/grub/xhci/patches/0021-xHCI-also-accept-SBRN-0x31-and-0x32.patch new file mode 100644 index 00000000..3bc5f0b2 --- /dev/null +++ b/config/grub/xhci/patches/0021-xHCI-also-accept-SBRN-0x31-and-0x32.patch @@ -0,0 +1,26 @@ +From 3fa33c091866b13c452b7a39ecee291e5901ff22 Mon Sep 17 00:00:00 2001 +From: Sven Anderson +Date: Sat, 28 May 2022 21:39:23 +0200 +Subject: [PATCH 21/26] xHCI: also accept SBRN 0x31 and 0x32 + +Signed-off-by: Sven Anderson +--- + grub-core/bus/usb/xhci-pci.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/grub-core/bus/usb/xhci-pci.c b/grub-core/bus/usb/xhci-pci.c +index a5bd3c97d..cde21f57a 100644 +--- a/grub-core/bus/usb/xhci-pci.c ++++ b/grub-core/bus/usb/xhci-pci.c +@@ -76,7 +76,7 @@ grub_xhci_pci_iter (grub_pci_device_t dev, grub_pci_id_t pciid, + /* Check Serial Bus Release Number */ + addr = grub_pci_make_address (dev, GRUB_XHCI_PCI_SBRN_REG); + release = grub_pci_read_byte (addr); +- if (release != 0x30) ++ if (release != 0x30 && release != 0x31 &&release != 0x32) + { + grub_dprintf ("xhci", "XHCI grub_xhci_pci_iter: Wrong SBRN: %0x\n", + release); +-- +2.39.5 + diff --git a/config/grub/xhci/patches/0022-Add-native-NVMe-driver-based-on-SeaBIOS.patch b/config/grub/xhci/patches/0022-Add-native-NVMe-driver-based-on-SeaBIOS.patch deleted file mode 100644 index c105dd19..00000000 --- a/config/grub/xhci/patches/0022-Add-native-NVMe-driver-based-on-SeaBIOS.patch +++ /dev/null @@ -1,1075 +0,0 @@ -From 2fb0de8881bd2fe637d05bec0ae887f380434b7d Mon Sep 17 00:00:00 2001 -From: Mate Kukri -Date: Mon, 20 May 2024 11:43:35 +0100 -Subject: [PATCH 22/22] Add native NVMe driver based on SeaBIOS - -Tested to successfully boot Debian on QEMU and OptiPlex 3050. - -Signed-off-by: Mate Kukri ---- - Makefile.am | 2 +- - grub-core/Makefile.core.def | 6 + - grub-core/commands/nativedisk.c | 1 + - grub-core/disk/nvme-int.h | 208 +++++++++ - grub-core/disk/nvme.c | 781 ++++++++++++++++++++++++++++++++ - include/grub/disk.h | 1 + - 6 files changed, 998 insertions(+), 1 deletion(-) - create mode 100644 grub-core/disk/nvme-int.h - create mode 100644 grub-core/disk/nvme.c - -diff --git a/Makefile.am b/Makefile.am -index 65016f856..7bc0866ba 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -434,7 +434,7 @@ if COND_i386_coreboot - FS_PAYLOAD_MODULES ?= $(shell cat grub-core/fs.lst) - default_payload.elf: grub-mkstandalone grub-mkimage FORCE - test -f $@ && rm $@ || true -- pkgdatadir=. ./grub-mkstandalone --grub-mkimage=./grub-mkimage -O i386-coreboot -o $@ --modules='ahci pata xhci ehci uhci ohci usb_keyboard usbms part_msdos ext2 fat at_keyboard part_gpt usbserial_usbdebug cbfs' --install-modules='ls linux search configfile normal cbtime cbls memrw iorw minicmd lsmmap lspci halt reboot hexdump pcidump regexp setpci lsacpi chain test serial multiboot cbmemc linux16 gzio echo help syslinuxcfg xnu $(FS_PAYLOAD_MODULES) password_pbkdf2 $(EXTRA_PAYLOAD_MODULES)' --fonts= --themes= --locales= -d grub-core/ /boot/grub/grub.cfg=$(srcdir)/coreboot.cfg -+ pkgdatadir=. ./grub-mkstandalone --grub-mkimage=./grub-mkimage -O i386-coreboot -o $@ --modules='ahci pata nvme xhci ehci uhci ohci usb_keyboard usbms part_msdos ext2 fat at_keyboard part_gpt usbserial_usbdebug cbfs' --install-modules='ls linux search configfile normal cbtime cbls memrw iorw minicmd lsmmap lspci halt reboot hexdump pcidump regexp setpci lsacpi chain test serial multiboot cbmemc linux16 gzio echo help syslinuxcfg xnu $(FS_PAYLOAD_MODULES) password_pbkdf2 $(EXTRA_PAYLOAD_MODULES)' --fonts= --themes= --locales= -d grub-core/ /boot/grub/grub.cfg=$(srcdir)/coreboot.cfg - endif - - endif -diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def -index fda723f0c..27a707fda 100644 ---- a/grub-core/Makefile.core.def -+++ b/grub-core/Makefile.core.def -@@ -2683,4 +2683,10 @@ module = { - common = tests/asn1/asn1_test.c; - cflags = '-Wno-uninitialized'; - cppflags = '-I$(srcdir)/lib/libtasn1-grub -I$(srcdir)/tests/asn1/'; -+}; -+ -+module = { -+ name = nvme; -+ common = disk/nvme.c; -+ enable = pci; - }; -diff --git a/grub-core/commands/nativedisk.c b/grub-core/commands/nativedisk.c -index 6806bff9c..fd68a513e 100644 ---- a/grub-core/commands/nativedisk.c -+++ b/grub-core/commands/nativedisk.c -@@ -78,6 +78,7 @@ get_uuid (const char *name, char **uuid, int getnative) - case GRUB_DISK_DEVICE_ATA_ID: - case GRUB_DISK_DEVICE_SCSI_ID: - case GRUB_DISK_DEVICE_XEN: -+ case GRUB_DISK_DEVICE_NVME_ID: - if (getnative) - break; - /* FALLTHROUGH */ -diff --git a/grub-core/disk/nvme-int.h b/grub-core/disk/nvme-int.h -new file mode 100644 -index 000000000..1295b58aa ---- /dev/null -+++ b/grub-core/disk/nvme-int.h -@@ -0,0 +1,208 @@ -+// NVMe datastructures and constants -+// -+// Copyright 2017 Amazon.com, Inc. or its affiliates. -+// -+// This file may be distributed under the terms of the GNU LGPLv3 license. -+ -+#ifndef __NVME_INT_H -+#define __NVME_INT_H -+ -+#include -+ -+/* Data structures */ -+ -+/* The register file of a NVMe host controller. This struct follows the naming -+ scheme in the NVMe specification. */ -+struct nvme_reg { -+ grub_uint64_t cap; /* controller capabilities */ -+ grub_uint32_t vs; /* version */ -+ grub_uint32_t intms; /* interrupt mask set */ -+ grub_uint32_t intmc; /* interrupt mask clear */ -+ grub_uint32_t cc; /* controller configuration */ -+ grub_uint32_t _res0; -+ grub_uint32_t csts; /* controller status */ -+ grub_uint32_t _res1; -+ grub_uint32_t aqa; /* admin queue attributes */ -+ grub_uint64_t asq; /* admin submission queue base address */ -+ grub_uint64_t acq; /* admin completion queue base address */ -+}; -+ -+/* Submission queue entry */ -+struct nvme_sqe { -+ union { -+ grub_uint32_t dword[16]; -+ struct { -+ grub_uint32_t cdw0; /* Command DWORD 0 */ -+ grub_uint32_t nsid; /* Namespace ID */ -+ grub_uint64_t _res0; -+ grub_uint64_t mptr; /* metadata ptr */ -+ -+ grub_uint64_t dptr_prp1; -+ grub_uint64_t dptr_prp2; -+ }; -+ }; -+}; -+ -+/* Completion queue entry */ -+struct nvme_cqe { -+ union { -+ grub_uint32_t dword[4]; -+ struct { -+ grub_uint32_t cdw0; -+ grub_uint32_t _res0; -+ grub_uint16_t sq_head; -+ grub_uint16_t sq_id; -+ grub_uint16_t cid; -+ grub_uint16_t status; -+ }; -+ }; -+}; -+ -+/* The common part of every submission or completion queue. */ -+struct nvme_queue { -+ grub_uint32_t *dbl; /* doorbell */ -+ grub_uint16_t mask; /* length - 1 */ -+}; -+ -+struct nvme_cq { -+ struct nvme_queue common; -+ struct nvme_cqe *cqe; -+ -+ /* We have read upto (but not including) this entry in the queue. */ -+ grub_uint16_t head; -+ -+ /* The current phase bit the controller uses to indicate that it has written -+ a new entry. This is inverted after each wrap. */ -+ unsigned phase : 1; -+}; -+ -+struct nvme_sq { -+ struct nvme_queue common; -+ struct nvme_sqe *sqe; -+ -+ /* Corresponding completion queue. We only support a single SQ per CQ. */ -+ struct nvme_cq *cq; -+ -+ /* The last entry the controller has fetched. */ -+ grub_uint16_t head; -+ -+ /* The last value we have written to the tail doorbell. */ -+ grub_uint16_t tail; -+}; -+ -+struct nvme_ctrl { -+ grub_pci_device_t pci; -+ struct nvme_reg volatile *reg; -+ -+ grub_uint32_t ctrlnum; -+ -+ grub_uint32_t doorbell_stride; /* in bytes */ -+ -+ struct nvme_sq admin_sq; -+ struct nvme_cq admin_cq; -+ -+ grub_uint32_t ns_count; -+ -+ struct nvme_sq io_sq; -+ struct nvme_cq io_cq; -+}; -+ -+struct nvme_namespace { -+ struct nvme_namespace *next; -+ struct nvme_namespace **prev; -+ -+ char *devname; -+ -+ grub_uint32_t nsnum; -+ -+ struct nvme_ctrl *ctrl; -+ -+ grub_uint32_t ns_id; -+ -+ grub_uint64_t lba_count; /* The total amount of sectors. */ -+ -+ grub_uint32_t block_size; -+ grub_uint32_t metadata_size; -+ grub_uint32_t max_req_size; -+}; -+ -+/* Data structures for NVMe admin identify commands */ -+ -+struct nvme_identify_ctrl { -+ grub_uint16_t vid; -+ grub_uint16_t ssvid; -+ char sn[20]; -+ char mn[40]; -+ char fr[8]; -+ -+ grub_uint8_t rab; -+ grub_uint8_t ieee[3]; -+ grub_uint8_t cmic; -+ grub_uint8_t mdts; -+ -+ char _boring[516 - 78]; -+ -+ grub_uint32_t nn; /* number of namespaces */ -+}; -+ -+struct nvme_identify_ns_list { -+ grub_uint32_t ns_id[1024]; -+}; -+ -+struct nvme_lba_format { -+ grub_uint16_t ms; -+ grub_uint8_t lbads; -+ grub_uint8_t rp; -+}; -+ -+struct nvme_identify_ns { -+ grub_uint64_t nsze; -+ grub_uint64_t ncap; -+ grub_uint64_t nuse; -+ grub_uint8_t nsfeat; -+ grub_uint8_t nlbaf; -+ grub_uint8_t flbas; -+ -+ char _boring[128 - 27]; -+ -+ struct nvme_lba_format lbaf[16]; -+}; -+ -+union nvme_identify { -+ struct nvme_identify_ns ns; -+ struct nvme_identify_ctrl ctrl; -+ struct nvme_identify_ns_list ns_list; -+}; -+ -+/* NVMe constants */ -+ -+#define NVME_CAP_CSS_NVME (1ULL << 37) -+ -+#define NVME_CSTS_FATAL (1U << 1) -+#define NVME_CSTS_RDY (1U << 0) -+ -+#define NVME_CC_EN (1U << 0) -+ -+#define NVME_SQE_OPC_ADMIN_CREATE_IO_SQ 1U -+#define NVME_SQE_OPC_ADMIN_CREATE_IO_CQ 5U -+#define NVME_SQE_OPC_ADMIN_IDENTIFY 6U -+ -+#define NVME_SQE_OPC_IO_WRITE 1U -+#define NVME_SQE_OPC_IO_READ 2U -+ -+#define NVME_ADMIN_IDENTIFY_CNS_ID_NS 0U -+#define NVME_ADMIN_IDENTIFY_CNS_ID_CTRL 1U -+#define NVME_ADMIN_IDENTIFY_CNS_GET_NS_LIST 2U -+ -+#define NVME_CQE_DW3_P (1U << 16) -+ -+#define NVME_PAGE_SIZE 4096 -+#define NVME_PAGE_MASK ~(NVME_PAGE_SIZE - 1) -+ -+/* Length for the queue entries. */ -+#define NVME_SQE_SIZE_LOG 6 -+#define NVME_CQE_SIZE_LOG 4 -+ -+#endif -+ -+/* EOF */ -diff --git a/grub-core/disk/nvme.c b/grub-core/disk/nvme.c -new file mode 100644 -index 000000000..093237c70 ---- /dev/null -+++ b/grub-core/disk/nvme.c -@@ -0,0 +1,781 @@ -+// Low level NVMe disk access -+// -+// Based on SeaBIOS NVMe driver - Copyright 2017 Amazon.com, Inc. or its affiliates. -+// Port to GRUB2 done by Mate Kukri -+// -+// This file may be distributed under the terms of the GNU LGPLv3 license. -+ -+#include -+#include -+#include -+#include "nvme-int.h" -+ -+GRUB_MOD_LICENSE ("GPLv3"); /* LGPLv3 in reality but it is GPLv3 compatible */ -+ -+static grub_uint32_t grub_nvme_ctrlcnt; -+static grub_uint32_t grub_nvme_nscnt; -+ -+static struct nvme_namespace *grub_nvme_namespaces; -+ -+// Page aligned "dma bounce buffer" of size NVME_PAGE_SIZE -+static void *nvme_dma_buffer; -+ -+static void * -+zalloc_page_aligned(grub_uint32_t size) -+{ -+ void *res = grub_memalign(NVME_PAGE_SIZE, size); -+ if (res) grub_memset(res, 0, size); -+ return res; -+} -+ -+static void -+nvme_init_queue_common(struct nvme_ctrl *ctrl, struct nvme_queue *q, grub_uint16_t q_idx, -+ grub_uint16_t length) -+{ -+ grub_memset(q, 0, sizeof(*q)); -+ q->dbl = (grub_uint32_t *)((char *)ctrl->reg + 0x1000 + q_idx * ctrl->doorbell_stride); -+ grub_dprintf("nvme", " q %p q_idx %u dbl %p\n", q, q_idx, q->dbl); -+ q->mask = length - 1; -+} -+ -+static int -+nvme_init_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq, grub_uint16_t q_idx, grub_uint16_t length, -+ struct nvme_cq *cq) -+{ -+ nvme_init_queue_common(ctrl, &sq->common, q_idx, length); -+ sq->sqe = zalloc_page_aligned(sizeof(*sq->sqe) * length); -+ -+ if (!sq->sqe) { -+ return -1; -+ } -+ -+ grub_dprintf("nvme", "sq %p q_idx %u sqe %p\n", sq, q_idx, sq->sqe); -+ sq->cq = cq; -+ sq->head = 0; -+ sq->tail = 0; -+ -+ return 0; -+} -+ -+static int -+nvme_init_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq, grub_uint16_t q_idx, grub_uint16_t length) -+{ -+ nvme_init_queue_common(ctrl, &cq->common, q_idx, length); -+ cq->cqe = zalloc_page_aligned(sizeof(*cq->cqe) * length); -+ if (!cq->cqe) { -+ return -1; -+ } -+ -+ cq->head = 0; -+ -+ /* All CQE phase bits are initialized to zero. This means initially we wait -+ for the host controller to set these to 1. */ -+ cq->phase = 1; -+ -+ return 0; -+} -+ -+static int -+nvme_poll_cq(struct nvme_cq *cq) -+{ -+ grub_uint32_t dw3 = *(volatile grub_uint32_t *) &cq->cqe[cq->head].dword[3]; -+ return (!!(dw3 & NVME_CQE_DW3_P) == cq->phase); -+} -+ -+static int -+nvme_is_cqe_success(struct nvme_cqe const *cqe) -+{ -+ return ((cqe->status >> 1) & 0xFF) == 0; -+} -+ -+static struct nvme_cqe -+nvme_error_cqe(void) -+{ -+ struct nvme_cqe r; -+ -+ /* 0xFF is a vendor specific status code != success. Should be okay for -+ indicating failure. */ -+ grub_memset(&r, 0xFF, sizeof(r)); -+ return r; -+} -+ -+static struct nvme_cqe -+nvme_consume_cqe(struct nvme_sq *sq) -+{ -+ struct nvme_cq *cq = sq->cq; -+ -+ if (!nvme_poll_cq(cq)) { -+ /* Cannot consume a completion queue entry, if there is none ready. */ -+ return nvme_error_cqe(); -+ } -+ -+ struct nvme_cqe *cqe = &cq->cqe[cq->head]; -+ grub_uint16_t cq_next_head = (cq->head + 1) & cq->common.mask; -+ grub_dprintf("nvme", "cq %p head %u -> %u\n", cq, cq->head, cq_next_head); -+ if (cq_next_head < cq->head) { -+ grub_dprintf("nvme", "cq %p wrap\n", cq); -+ cq->phase = ~cq->phase; -+ } -+ cq->head = cq_next_head; -+ -+ /* Update the submission queue head. */ -+ if (cqe->sq_head != sq->head) { -+ sq->head = cqe->sq_head; -+ grub_dprintf("nvme", "sq %p advanced to %u\n", sq, cqe->sq_head); -+ } -+ -+ /* Tell the controller that we consumed the completion. */ -+ *(volatile grub_uint32_t *) cq->common.dbl = cq->head; -+ -+ return *cqe; -+} -+ -+static struct nvme_cqe -+nvme_wait(struct nvme_sq *sq) -+{ -+ // static const unsigned nvme_timeout = 5000 /* ms */; -+ // grub_uint32_t to = timer_calc(nvme_timeout); -+ while (!nvme_poll_cq(sq->cq)) { -+ /* FIXME -+ yield(); -+ -+ if (timer_check(to)) { -+ warn_timeout(); -+ return nvme_error_cqe(); -+ }*/ -+ } -+ -+ return nvme_consume_cqe(sq); -+} -+ -+/* Returns the next submission queue entry (or NULL if the queue is full). It -+ also fills out Command Dword 0 and clears the rest. */ -+static struct nvme_sqe * -+nvme_get_next_sqe(struct nvme_sq *sq, grub_uint8_t opc, void *metadata, void *data, void *data2) -+{ -+ if (((sq->head + 1) & sq->common.mask) == sq->tail) { -+ grub_dprintf("nvme", "submission queue is full\n"); -+ return NULL; -+ } -+ -+ struct nvme_sqe *sqe = &sq->sqe[sq->tail]; -+ grub_dprintf("nvme", "sq %p next_sqe %u\n", sq, sq->tail); -+ -+ grub_memset(sqe, 0, sizeof(*sqe)); -+ sqe->cdw0 = opc | (sq->tail << 16 /* CID */); -+ sqe->mptr = (grub_uint32_t)metadata; -+ sqe->dptr_prp1 = (grub_uint32_t)data; -+ sqe->dptr_prp2 = (grub_uint32_t)data2; -+ -+ return sqe; -+} -+ -+/* Call this after you've filled out an sqe that you've got from nvme_get_next_sqe. */ -+static void -+nvme_commit_sqe(struct nvme_sq *sq) -+{ -+ grub_dprintf("nvme", "sq %p commit_sqe %u\n", sq, sq->tail); -+ sq->tail = (sq->tail + 1) & sq->common.mask; -+ *(volatile grub_uint32_t *) sq->common.dbl = sq->tail; -+} -+ -+/* Perform an identify command on the admin queue and return the resulting -+ buffer. This may be a NULL pointer, if something failed. This function -+ cannot be used after initialization, because it uses buffers in tmp zone. */ -+static union nvme_identify * -+nvme_admin_identify(struct nvme_ctrl *ctrl, grub_uint8_t cns, grub_uint32_t nsid) -+{ -+ union nvme_identify *identify_buf = zalloc_page_aligned(4096); -+ if (!identify_buf) -+ return NULL; -+ -+ struct nvme_sqe *cmd_identify; -+ cmd_identify = nvme_get_next_sqe(&ctrl->admin_sq, -+ NVME_SQE_OPC_ADMIN_IDENTIFY, NULL, -+ identify_buf, NULL); -+ if (!cmd_identify) -+ goto error; -+ -+ cmd_identify->nsid = nsid; -+ cmd_identify->dword[10] = cns; -+ -+ nvme_commit_sqe(&ctrl->admin_sq); -+ -+ struct nvme_cqe cqe = nvme_wait(&ctrl->admin_sq); -+ -+ if (!nvme_is_cqe_success(&cqe)) { -+ goto error; -+ } -+ -+ return identify_buf; -+ error: -+ grub_free(identify_buf); -+ return NULL; -+} -+ -+static struct nvme_identify_ctrl * -+nvme_admin_identify_ctrl(struct nvme_ctrl *ctrl) -+{ -+ return &nvme_admin_identify(ctrl, NVME_ADMIN_IDENTIFY_CNS_ID_CTRL, 0)->ctrl; -+} -+ -+static struct nvme_identify_ns * -+nvme_admin_identify_ns(struct nvme_ctrl *ctrl, grub_uint32_t ns_id) -+{ -+ return &nvme_admin_identify(ctrl, NVME_ADMIN_IDENTIFY_CNS_ID_NS, -+ ns_id)->ns; -+} -+ -+static void -+nvme_probe_ns(struct nvme_ctrl *ctrl, grub_uint32_t ns_idx, grub_uint8_t mdts) -+{ -+ grub_uint32_t ns_id = ns_idx + 1; -+ -+ struct nvme_identify_ns *id = nvme_admin_identify_ns(ctrl, ns_id); -+ if (!id) { -+ grub_dprintf("nvme", "NVMe couldn't identify namespace %u.\n", ns_id); -+ goto free_buffer; -+ } -+ -+ grub_uint8_t current_lba_format = id->flbas & 0xF; -+ if (current_lba_format > id->nlbaf) { -+ grub_dprintf("nvme", "NVMe NS %u: current LBA format %u is beyond what the " -+ " namespace supports (%u)?\n", -+ ns_id, current_lba_format, id->nlbaf + 1); -+ goto free_buffer; -+ } -+ -+ if (!id->nsze) { -+ grub_dprintf("nvme", "NVMe NS %u is inactive.\n", ns_id); -+ goto free_buffer; -+ } -+ -+ if (!nvme_dma_buffer) { -+ nvme_dma_buffer = zalloc_page_aligned(NVME_PAGE_SIZE); -+ if (!nvme_dma_buffer) { -+ goto free_buffer; -+ } -+ } -+ -+ struct nvme_namespace *ns = grub_malloc(sizeof(*ns)); -+ if (!ns) { -+ goto free_buffer; -+ } -+ grub_memset(ns, 0, sizeof(*ns)); -+ ns->ctrl = ctrl; -+ ns->ns_id = ns_id; -+ ns->lba_count = id->nsze; -+ -+ struct nvme_lba_format *fmt = &id->lbaf[current_lba_format]; -+ -+ ns->block_size = 1U << fmt->lbads; -+ ns->metadata_size = fmt->ms; -+ -+ if (ns->block_size > NVME_PAGE_SIZE) { -+ /* If we see devices that trigger this path, we need to increase our -+ buffer size. */ -+ grub_free(ns); -+ goto free_buffer; -+ } -+ -+ if (mdts) { -+ ns->max_req_size = ((1U << mdts) * NVME_PAGE_SIZE) / ns->block_size; -+ grub_dprintf("nvme", "NVME NS %u max request size: %d sectors\n", -+ ns_id, ns->max_req_size); -+ } else { -+ ns->max_req_size = -1U; -+ } -+ -+ ns->devname = grub_xasprintf("nvme%un%u", ctrl->ctrlnum, ns_id); -+ ns->nsnum = grub_nvme_nscnt++; -+ -+ grub_list_push (GRUB_AS_LIST_P (&grub_nvme_namespaces), GRUB_AS_LIST (ns)); -+ -+free_buffer: -+ grub_free(id); -+} -+ -+ -+/* Release memory allocated for a completion queue */ -+static void -+nvme_destroy_cq(struct nvme_cq *cq) -+{ -+ grub_free(cq->cqe); -+ cq->cqe = NULL; -+} -+ -+/* Release memory allocated for a submission queue */ -+static void -+nvme_destroy_sq(struct nvme_sq *sq) -+{ -+ grub_free(sq->sqe); -+ sq->sqe = NULL; -+} -+ -+/* Returns 0 on success. */ -+static int -+nvme_create_io_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq, grub_uint16_t q_idx) -+{ -+ int rc; -+ struct nvme_sqe *cmd_create_cq; -+ grub_uint32_t length = 1 + (ctrl->reg->cap & 0xffff); -+ if (length > NVME_PAGE_SIZE / sizeof(struct nvme_cqe)) -+ length = NVME_PAGE_SIZE / sizeof(struct nvme_cqe); -+ -+ rc = nvme_init_cq(ctrl, cq, q_idx, length); -+ if (rc) { -+ goto err; -+ } -+ -+ cmd_create_cq = nvme_get_next_sqe(&ctrl->admin_sq, -+ NVME_SQE_OPC_ADMIN_CREATE_IO_CQ, NULL, -+ cq->cqe, NULL); -+ if (!cmd_create_cq) { -+ goto err_destroy_cq; -+ } -+ -+ cmd_create_cq->dword[10] = (cq->common.mask << 16) | (q_idx >> 1); -+ cmd_create_cq->dword[11] = 1 /* physically contiguous */; -+ -+ nvme_commit_sqe(&ctrl->admin_sq); -+ -+ struct nvme_cqe cqe = nvme_wait(&ctrl->admin_sq); -+ -+ if (!nvme_is_cqe_success(&cqe)) { -+ grub_dprintf("nvme", "create io cq failed: %08x %08x %08x %08x\n", -+ cqe.dword[0], cqe.dword[1], cqe.dword[2], cqe.dword[3]); -+ -+ goto err_destroy_cq; -+ } -+ -+ return 0; -+ -+err_destroy_cq: -+ nvme_destroy_cq(cq); -+err: -+ return -1; -+} -+ -+/* Returns 0 on success. */ -+static int -+nvme_create_io_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq, grub_uint16_t q_idx, struct nvme_cq *cq) -+{ -+ int rc; -+ struct nvme_sqe *cmd_create_sq; -+ grub_uint32_t length = 1 + (ctrl->reg->cap & 0xffff); -+ if (length > NVME_PAGE_SIZE / sizeof(struct nvme_cqe)) -+ length = NVME_PAGE_SIZE / sizeof(struct nvme_cqe); -+ -+ rc = nvme_init_sq(ctrl, sq, q_idx, length, cq); -+ if (rc) { -+ goto err; -+ } -+ -+ cmd_create_sq = nvme_get_next_sqe(&ctrl->admin_sq, -+ NVME_SQE_OPC_ADMIN_CREATE_IO_SQ, NULL, -+ sq->sqe, NULL); -+ if (!cmd_create_sq) { -+ goto err_destroy_sq; -+ } -+ -+ cmd_create_sq->dword[10] = (sq->common.mask << 16) | (q_idx >> 1); -+ cmd_create_sq->dword[11] = (q_idx >> 1) << 16 | 1 /* contiguous */; -+ grub_dprintf("nvme", "sq %p create dword10 %08x dword11 %08x\n", sq, -+ cmd_create_sq->dword[10], cmd_create_sq->dword[11]); -+ -+ nvme_commit_sqe(&ctrl->admin_sq); -+ -+ struct nvme_cqe cqe = nvme_wait(&ctrl->admin_sq); -+ -+ if (!nvme_is_cqe_success(&cqe)) { -+ grub_dprintf("nvme", "create io sq failed: %08x %08x %08x %08x\n", -+ cqe.dword[0], cqe.dword[1], cqe.dword[2], cqe.dword[3]); -+ goto err_destroy_sq; -+ } -+ -+ return 0; -+ -+err_destroy_sq: -+ nvme_destroy_sq(sq); -+err: -+ return -1; -+} -+ -+/* Reads count sectors into buf. The buffer cannot cross page boundaries. */ -+static int -+nvme_io_xfer(struct nvme_namespace *ns, grub_uint64_t lba, void *prp1, void *prp2, -+ grub_uint16_t count, int write) -+{ -+ if (((grub_uint32_t)prp1 & 0x3) || ((grub_uint32_t)prp2 & 0x3)) { -+ /* Buffer is misaligned */ -+ return -1; -+ } -+ -+ struct nvme_sqe *io_read = nvme_get_next_sqe(&ns->ctrl->io_sq, -+ write ? NVME_SQE_OPC_IO_WRITE -+ : NVME_SQE_OPC_IO_READ, -+ NULL, prp1, prp2); -+ io_read->nsid = ns->ns_id; -+ io_read->dword[10] = (grub_uint32_t)lba; -+ io_read->dword[11] = (grub_uint32_t)(lba >> 32); -+ io_read->dword[12] = (1U << 31 /* limited retry */) | (count - 1); -+ -+ nvme_commit_sqe(&ns->ctrl->io_sq); -+ -+ struct nvme_cqe cqe = nvme_wait(&ns->ctrl->io_sq); -+ -+ if (!nvme_is_cqe_success(&cqe)) { -+ grub_dprintf("nvme", "read io: %08x %08x %08x %08x\n", -+ cqe.dword[0], cqe.dword[1], cqe.dword[2], cqe.dword[3]); -+ -+ return -1; -+ } -+ -+ grub_dprintf("nvme", "ns %u %s lba %llu+%u\n", ns->ns_id, write ? "write" : "read", -+ lba, count); -+ return count; -+} -+ -+// Transfer up to one page of data using the internal dma bounce buffer -+static int -+nvme_bounce_xfer(struct nvme_namespace *ns, grub_uint64_t lba, void *buf, grub_uint16_t count, -+ int write) -+{ -+ grub_uint16_t const max_blocks = NVME_PAGE_SIZE / ns->block_size; -+ grub_uint16_t blocks = count < max_blocks ? count : max_blocks; -+ -+ if (write) -+ grub_memcpy(nvme_dma_buffer, buf, blocks * ns->block_size); -+ -+ int res = nvme_io_xfer(ns, lba, nvme_dma_buffer, NULL, blocks, write); -+ -+ if (!write && res >= 0) -+ grub_memcpy(buf, nvme_dma_buffer, res * ns->block_size); -+ -+ return res; -+} -+ -+#define NVME_MAX_PRPL_ENTRIES 15 /* Allows requests up to 64kb */ -+ -+// Transfer data using page list (if applicable) -+static int -+nvme_prpl_xfer(struct nvme_namespace *ns, grub_uint64_t lba, void *buf, grub_uint16_t count, -+ int write) -+{ -+ grub_uint32_t base = (long)buf; -+ grub_int32_t size; -+ -+ if (count > ns->max_req_size) -+ count = ns->max_req_size; -+ -+ size = count * ns->block_size; -+ /* Special case for transfers that fit into PRP1, but are unaligned */ -+ if (((size + (base & ~NVME_PAGE_MASK)) <= NVME_PAGE_SIZE)) -+ goto single; -+ -+ /* Every request has to be page aligned */ -+ if (base & ~NVME_PAGE_MASK) -+ goto bounce; -+ -+ /* Make sure a full block fits into the last chunk */ -+ if (size & (ns->block_size - 1ULL)) -+ goto bounce; -+ -+ /* Build PRP list if we need to describe more than 2 pages */ -+ if ((ns->block_size * count) > (NVME_PAGE_SIZE * 2)) { -+ grub_uint32_t prpl_len = 0; -+ grub_uint64_t *prpl = nvme_dma_buffer; -+ int first_page = 1; -+ for (; size > 0; base += NVME_PAGE_SIZE, size -= NVME_PAGE_SIZE) { -+ if (first_page) { -+ /* First page is special */ -+ first_page = 0; -+ continue; -+ } -+ if (prpl_len >= NVME_MAX_PRPL_ENTRIES) -+ goto bounce; -+ prpl[prpl_len++] = base; -+ } -+ return nvme_io_xfer(ns, lba, buf, prpl, count, write); -+ } -+ -+ /* Directly embed the 2nd page if we only need 2 pages */ -+ if ((ns->block_size * count) > NVME_PAGE_SIZE) -+ return nvme_io_xfer(ns, lba, buf, (char *) buf + NVME_PAGE_SIZE, count, write); -+ -+single: -+ /* One page is enough, don't expose anything else */ -+ return nvme_io_xfer(ns, lba, buf, NULL, count, write); -+ -+bounce: -+ /* Use bounce buffer to make transfer */ -+ return nvme_bounce_xfer(ns, lba, buf, count, write); -+} -+ -+static int -+nvme_create_io_queues(struct nvme_ctrl *ctrl) -+{ -+ if (nvme_create_io_cq(ctrl, &ctrl->io_cq, 3)) -+ goto err; -+ -+ if (nvme_create_io_sq(ctrl, &ctrl->io_sq, 2, &ctrl->io_cq)) -+ goto err_free_cq; -+ -+ return 0; -+ -+ err_free_cq: -+ nvme_destroy_cq(&ctrl->io_cq); -+ err: -+ return -1; -+} -+ -+/* Waits for CSTS.RDY to match rdy. Returns 0 on success. */ -+static int -+nvme_wait_csts_rdy(struct nvme_ctrl *ctrl, unsigned rdy) -+{ -+ // grub_uint32_t const max_to = 500 /* ms */ * ((ctrl->reg->cap >> 24) & 0xFFU); -+ // grub_uint32_t to = timer_calc(max_to); -+ grub_uint32_t csts; -+ -+ while (rdy != ((csts = ctrl->reg->csts) & NVME_CSTS_RDY)) { -+ // FIXME -+ //yield(); -+ -+ if (csts & NVME_CSTS_FATAL) { -+ grub_dprintf("nvme", "NVMe fatal error during controller shutdown\n"); -+ return -1; -+ } -+ -+ /* -+ if (timer_check(to)) { -+ warn_timeout(); -+ return -1; -+ }*/ -+ } -+ -+ return 0; -+} -+ -+/* Returns 0 on success. */ -+static int grub_nvme_controller_enable(struct nvme_ctrl *ctrl) -+{ -+ grub_pci_address_t addr; -+ int rc; -+ -+ addr = grub_pci_make_address (ctrl->pci, GRUB_PCI_REG_COMMAND); -+ grub_pci_write_word (addr, grub_pci_read_word (addr) | GRUB_PCI_COMMAND_BUS_MASTER); -+ -+ /* Turn the controller off. */ -+ ctrl->reg->cc = 0; -+ if (nvme_wait_csts_rdy(ctrl, 0)) { -+ grub_dprintf("nvme", "NVMe fatal error during controller shutdown\n"); -+ return -1; -+ } -+ -+ ctrl->doorbell_stride = 4U << ((ctrl->reg->cap >> 32) & 0xF); -+ -+ rc = nvme_init_cq(ctrl, &ctrl->admin_cq, 1, -+ NVME_PAGE_SIZE / sizeof(struct nvme_cqe)); -+ if (rc) { -+ return -1; -+ } -+ -+ rc = nvme_init_sq(ctrl, &ctrl->admin_sq, 0, -+ NVME_PAGE_SIZE / sizeof(struct nvme_sqe), &ctrl->admin_cq); -+ if (rc) { -+ goto err_destroy_admin_cq; -+ } -+ -+ ctrl->reg->aqa = ctrl->admin_cq.common.mask << 16 -+ | ctrl->admin_sq.common.mask; -+ -+ ctrl->reg->asq = (grub_uint32_t)ctrl->admin_sq.sqe; -+ ctrl->reg->acq = (grub_uint32_t)ctrl->admin_cq.cqe; -+ -+ grub_dprintf("nvme", " admin submission queue: %p\n", ctrl->admin_sq.sqe); -+ grub_dprintf("nvme", " admin completion queue: %p\n", ctrl->admin_cq.cqe); -+ -+ ctrl->reg->cc = NVME_CC_EN | (NVME_CQE_SIZE_LOG << 20) -+ | (NVME_SQE_SIZE_LOG << 16 /* IOSQES */); -+ -+ if (nvme_wait_csts_rdy(ctrl, 1)) { -+ grub_dprintf("nvme", "NVMe fatal error while enabling controller\n"); -+ goto err_destroy_admin_sq; -+ } -+ -+ /* The admin queue is set up and the controller is ready. Let's figure out -+ what namespaces we have. */ -+ -+ struct nvme_identify_ctrl *identify = nvme_admin_identify_ctrl(ctrl); -+ -+ if (!identify) { -+ grub_dprintf("nvme", "NVMe couldn't identify controller.\n"); -+ goto err_destroy_admin_sq; -+ } -+ -+ grub_dprintf("nvme", "NVMe has %u namespace%s.\n", -+ identify->nn, (identify->nn == 1) ? "" : "s"); -+ -+ ctrl->ns_count = identify->nn; -+ grub_uint8_t mdts = identify->mdts; -+ grub_free(identify); -+ -+ if ((ctrl->ns_count == 0) || nvme_create_io_queues(ctrl)) { -+ /* No point to continue, if the controller says it doesn't have -+ namespaces or we couldn't create I/O queues. */ -+ goto err_destroy_admin_sq; -+ } -+ -+ /* Give the controller a global number */ -+ ctrl->ctrlnum = grub_nvme_ctrlcnt++; -+ -+ /* Populate namespace IDs */ -+ for (grub_uint32_t ns_idx = 0; ns_idx < ctrl->ns_count; ns_idx++) { -+ nvme_probe_ns(ctrl, ns_idx, mdts); -+ } -+ -+ grub_dprintf("nvme", "NVMe initialization complete!\n"); -+ return 0; -+ -+ err_destroy_admin_sq: -+ nvme_destroy_sq(&ctrl->admin_sq); -+ err_destroy_admin_cq: -+ nvme_destroy_cq(&ctrl->admin_cq); -+ return -1; -+} -+ -+static int grub_nvme_pci_probe(grub_pci_device_t dev, grub_pci_id_t pciid __attribute__ ((unused)), void *data __attribute__ ((unused))) -+{ -+ grub_pci_address_t addr; -+ grub_uint32_t class, bar, version; -+ struct nvme_reg volatile *reg; -+ -+ class = grub_pci_read (grub_pci_make_address (dev, GRUB_PCI_REG_CLASS)); -+ if (class >> 16 != 0x0108) -+ return 0; -+ if ((class >> 8 & 0xff) != 2) { /* as of NVM 1.0e */ -+ grub_dprintf("nvme", "Found incompatble NVMe: prog-if=%02x\n", class >> 8 & 0xff); -+ return 0; -+ } -+ -+ bar = grub_pci_read (grub_pci_make_address (dev, GRUB_PCI_REG_ADDRESS_REG0)); -+ reg = grub_pci_device_map_range (dev, bar & GRUB_PCI_ADDR_MEM_MASK, sizeof (*reg)); -+ -+ addr = grub_pci_make_address (dev, GRUB_PCI_REG_COMMAND); -+ grub_pci_write_word (addr, grub_pci_read_word (addr) | GRUB_PCI_COMMAND_MEM_ENABLED); -+ -+ version = reg->vs; -+ grub_dprintf("nvme", "Found NVMe controller with version %u.%u.%u.\n", version >> 16, (version >> 8) & 0xFF, version & 0xFF); -+ grub_dprintf("nvme", " Capabilities %016llx\n", reg->cap); -+ -+ if (~reg->cap & NVME_CAP_CSS_NVME) { -+ grub_dprintf("nvme", "Controller doesn't speak NVMe command set. Skipping.\n"); -+ goto err; -+ } -+ -+ struct nvme_ctrl *ctrl = grub_malloc(sizeof(*ctrl)); -+ if (!ctrl) -+ goto err; -+ -+ grub_memset(ctrl, 0, sizeof(*ctrl)); -+ -+ ctrl->reg = reg; -+ ctrl->pci = dev; -+ -+ if (grub_nvme_controller_enable(ctrl)) -+ goto err_free_ctrl; -+ -+ return 0; -+ -+ err_free_ctrl: -+ grub_free(ctrl); -+ err: -+ grub_dprintf("nvme", "Failed to enable NVMe controller.\n"); -+ return 0; -+} -+ -+static int -+grub_nvme_iterate (grub_disk_dev_iterate_hook_t hook, void *hook_data, grub_disk_pull_t pull) -+{ -+ struct nvme_namespace *ns; -+ -+ if (pull != GRUB_DISK_PULL_NONE) -+ return 0; -+ -+ FOR_LIST_ELEMENTS(ns, grub_nvme_namespaces) -+ if (hook (ns->devname, hook_data)) -+ return 1; -+ -+ return 0; -+} -+ -+static grub_err_t -+grub_nvme_open (const char *name __attribute ((unused)), grub_disk_t disk __attribute ((unused))) -+{ -+ struct nvme_namespace *ns; -+ -+ FOR_LIST_ELEMENTS(ns, grub_nvme_namespaces) -+ if (grub_strcmp (ns->devname, name) == 0) -+ break; -+ -+ if (! ns) -+ return grub_error (GRUB_ERR_UNKNOWN_DEVICE, "can't open device"); -+ -+ disk->total_sectors = ns->lba_count; -+ disk->max_agglomerate = ns->max_req_size; -+ -+ disk->id = ns->nsnum; /* global id of the namespace */ -+ -+ disk->data = ns; -+ -+ return 0; -+} -+ -+static grub_err_t -+nvme_readwrite(struct nvme_namespace *ns, grub_disk_addr_t sector, grub_size_t num_sectors, char *buf, int write) -+{ -+ for (int i = 0; i < num_sectors;) { -+ grub_uint16_t blocks_remaining = num_sectors - i; -+ char *op_buf = buf + i * ns->block_size; -+ int blocks = nvme_prpl_xfer(ns, sector + i, op_buf, blocks_remaining, write); -+ if (blocks < 0) -+ return GRUB_ERR_IO; -+ i += blocks; -+ } -+ return GRUB_ERR_NONE; -+} -+ -+static grub_err_t -+grub_nvme_read (grub_disk_t disk, grub_disk_addr_t sector, grub_size_t num_sectors, char *buf) -+{ -+ return nvme_readwrite((struct nvme_namespace *) disk->data, sector, num_sectors, buf, 0); -+} -+ -+static grub_err_t -+grub_nvme_write (grub_disk_t disk, grub_disk_addr_t sector, grub_size_t num_sectors, const char *buf) -+{ -+ return nvme_readwrite((struct nvme_namespace *) disk->data, sector, num_sectors, buf, 1); -+} -+ -+static struct grub_disk_dev grub_nvme_dev = -+ { -+ .name = "nvme", -+ .id = GRUB_DISK_DEVICE_NVME_ID, -+ .disk_iterate = grub_nvme_iterate, -+ .disk_open = grub_nvme_open, -+ .disk_read = grub_nvme_read, -+ .disk_write = grub_nvme_write, -+ .next = 0 -+ }; -+ -+GRUB_MOD_INIT(nvme) -+{ -+ grub_stop_disk_firmware (); -+ grub_pci_iterate (grub_nvme_pci_probe, NULL); -+ grub_disk_dev_register (&grub_nvme_dev); -+} -+ -+GRUB_MOD_FINI(nvme) -+{ -+ grub_disk_dev_unregister (&grub_nvme_dev); -+} -diff --git a/include/grub/disk.h b/include/grub/disk.h -index fbf23df7f..186e76f0b 100644 ---- a/include/grub/disk.h -+++ b/include/grub/disk.h -@@ -52,6 +52,7 @@ enum grub_disk_dev_id - GRUB_DISK_DEVICE_UBOOTDISK_ID, - GRUB_DISK_DEVICE_XEN, - GRUB_DISK_DEVICE_OBDISK_ID, -+ GRUB_DISK_DEVICE_NVME_ID - }; - - struct grub_disk; --- -2.39.5 - diff --git a/config/grub/xhci/patches/0022-xhci-fix-port-indexing.patch b/config/grub/xhci/patches/0022-xhci-fix-port-indexing.patch new file mode 100644 index 00000000..83427fcc --- /dev/null +++ b/config/grub/xhci/patches/0022-xhci-fix-port-indexing.patch @@ -0,0 +1,43 @@ +From c455091ce40ffd8218802aaadeacf2039fea5036 Mon Sep 17 00:00:00 2001 +From: Sven Anderson +Date: Mon, 13 Jan 2025 19:51:41 +0100 +Subject: [PATCH 22/26] xhci: fix port indexing + +--- + grub-core/bus/usb/xhci.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/grub-core/bus/usb/xhci.c b/grub-core/bus/usb/xhci.c +index f4591ffb5..dc89b9619 100644 +--- a/grub-core/bus/usb/xhci.c ++++ b/grub-core/bus/usb/xhci.c +@@ -2250,7 +2250,7 @@ grub_xhci_detect_dev (grub_usb_controller_t dev, int port, int *changed) + + *changed = 0; + grub_dprintf("xhci", "%s: dev=%p USB%d_%d port %d\n", __func__, dev, +- x->psids[port-1].major, x->psids[port-1].minor, port); ++ x->psids[port].major, x->psids[port].minor, port); + + /* On shutdown advertise all ports as disconnected. This will trigger + * a gracefull detatch. */ +@@ -2285,13 +2285,13 @@ grub_xhci_detect_dev (grub_usb_controller_t dev, int port, int *changed) + if (!(portsc & GRUB_XHCI_PORTSC_CCS)) + return GRUB_USB_SPEED_NONE; + +- for (grub_uint8_t i = 0; i < 16 && x->psids[port-1].psids[i].id > 0; i++) ++ for (grub_uint8_t i = 0; i < 16 && x->psids[port].psids[i].id > 0; i++) + { +- if (x->psids[port-1].psids[i].id == speed) ++ if (x->psids[port].psids[i].id == speed) + { + grub_dprintf("xhci", "%s: grub_usb_speed = %d\n", __func__, +- x->psids[port-1].psids[i].grub_usb_speed ); +- return x->psids[port-1].psids[i].grub_usb_speed; ++ x->psids[port].psids[i].grub_usb_speed ); ++ return x->psids[port].psids[i].grub_usb_speed; + } + } + +-- +2.39.5 + diff --git a/config/grub/xhci/patches/0023-xhci-workaround-z790-non-root-hub-speed-detection.patch b/config/grub/xhci/patches/0023-xhci-workaround-z790-non-root-hub-speed-detection.patch new file mode 100644 index 00000000..0181f3bd --- /dev/null +++ b/config/grub/xhci/patches/0023-xhci-workaround-z790-non-root-hub-speed-detection.patch @@ -0,0 +1,28 @@ +From b033ef3b4b2384df4165ae7829155cafd4ee3488 Mon Sep 17 00:00:00 2001 +From: Sven Anderson +Date: Mon, 13 Jan 2025 19:55:15 +0100 +Subject: [PATCH 23/26] xhci: workaround z790 non-root-hub speed detection + +--- + grub-core/bus/usb/xhci.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/grub-core/bus/usb/xhci.c b/grub-core/bus/usb/xhci.c +index dc89b9619..d13a7c39d 100644 +--- a/grub-core/bus/usb/xhci.c ++++ b/grub-core/bus/usb/xhci.c +@@ -2285,6 +2285,11 @@ grub_xhci_detect_dev (grub_usb_controller_t dev, int port, int *changed) + if (!(portsc & GRUB_XHCI_PORTSC_CCS)) + return GRUB_USB_SPEED_NONE; + ++ if (port == 4 || port == 12 && speed < XHCI_USB_HIGHSPEED) { // inital hub speed detection on Z790 is too low ++ grub_dprintf("xhci", "%s: setting internal hub speed to high\n", __func__); ++ return GRUB_USB_SPEED_HIGH; ++ } ++ + for (grub_uint8_t i = 0; i < 16 && x->psids[port].psids[i].id > 0; i++) + { + if (x->psids[port].psids[i].id == speed) +-- +2.39.5 + diff --git a/config/grub/xhci/patches/0024-xhci-configure-TT-for-non-root-hubs.patch b/config/grub/xhci/patches/0024-xhci-configure-TT-for-non-root-hubs.patch new file mode 100644 index 00000000..5f612885 --- /dev/null +++ b/config/grub/xhci/patches/0024-xhci-configure-TT-for-non-root-hubs.patch @@ -0,0 +1,98 @@ +From 7748bcc0e7cef17dbf626f83cc03e56a17075330 Mon Sep 17 00:00:00 2001 +From: Sven Anderson +Date: Mon, 13 Jan 2025 20:26:32 +0100 +Subject: [PATCH 24/26] xhci: configure TT for non-root-hubs + +--- + grub-core/bus/usb/usbhub.c | 6 +++++ + grub-core/bus/usb/xhci.c | 45 +++++++++++++++++++++++++++++++++----- + include/grub/usb.h | 2 ++ + 3 files changed, 47 insertions(+), 6 deletions(-) + +diff --git a/grub-core/bus/usb/usbhub.c b/grub-core/bus/usb/usbhub.c +index e96505aa9..629b3ed53 100644 +--- a/grub-core/bus/usb/usbhub.c ++++ b/grub-core/bus/usb/usbhub.c +@@ -818,3 +818,9 @@ grub_usb_iterate (grub_usb_iterate_hook_t hook, void *hook_data) + + return 0; + } ++ ++grub_usb_device_t ++grub_usb_get_dev (int addr) ++{ ++ return grub_usb_devs[addr]; ++} +diff --git a/grub-core/bus/usb/xhci.c b/grub-core/bus/usb/xhci.c +index d13a7c39d..8ad2a10f9 100644 +--- a/grub-core/bus/usb/xhci.c ++++ b/grub-core/bus/usb/xhci.c +@@ -623,13 +623,46 @@ grub_xhci_alloc_inctx(struct grub_xhci *x, int maxepid, + break; + } + +- /* Route is greater zero on devices that are connected to a non root hub */ +- if (dev->route) +- { +- /* FIXME: Implement this code for non SuperSpeed hub devices */ ++ /* Set routing string */ ++ slot->ctx[0] |= dev->route; ++ ++ /* Set root hub port number */ ++ slot->ctx[1] |= (dev->root_port + 1) << 16; ++ ++ if (dev->split_hubaddr && (dev->speed == GRUB_USB_SPEED_LOW || ++ dev->speed == GRUB_USB_SPEED_FULL)) { ++ ++ grub_usb_device_t hubdev = grub_usb_get_dev(dev->split_hubaddr); ++ ++ if (!hubdev || hubdev->descdev.class != GRUB_USB_CLASS_HUB) { ++ grub_dprintf("xhci", "Invalid hub device at addr %d!\n", dev->split_hubaddr); ++ return NULL; ++ } ++ ++ struct grub_xhci_priv *hub_priv = hubdev->xhci_priv; ++ if (!hub_priv) { ++ grub_dprintf("xhci", "Hub has no xhci_priv!\n"); ++ return NULL; ++ } ++ ++ if (hubdev->speed == GRUB_USB_SPEED_HIGH) { ++ /* Direct connection to high-speed hub - set up TT */ ++ grub_dprintf("xhci", "Direct high-speed hub connection - configuring TT with " ++ "hub slot %d port %d\n", hub_priv->slotid, dev->split_hubport); ++ slot->ctx[2] |= hub_priv->slotid; ++ slot->ctx[2] |= dev->split_hubport << 8; + } +- slot->ctx[0] |= dev->route; +- slot->ctx[1] |= (dev->root_port+1) << 16; ++ else { ++ /* Hub is not high-speed, inherit TT settings from parent */ ++ volatile struct grub_xhci_slotctx *hubslot; ++ grub_dprintf("xhci", "Non high-speed hub - inheriting TT settings from parent\n"); ++ hubslot = grub_dma_phys2virt(x->devs[hub_priv->slotid].ptr_low, x->devs_dma); ++ slot->ctx[2] = hubslot->ctx[2]; ++ } ++ } ++ ++ grub_dprintf("xhci", "Slot context: ctx[0]=0x%08x ctx[1]=0x%08x ctx[2]=0x%08x\n", ++ slot->ctx[0], slot->ctx[1], slot->ctx[2]); + + grub_arch_sync_dma_caches(in, size); + +diff --git a/include/grub/usb.h b/include/grub/usb.h +index eb71fa1c7..df97a60cc 100644 +--- a/include/grub/usb.h ++++ b/include/grub/usb.h +@@ -62,6 +62,8 @@ typedef int (*grub_usb_controller_iterate_hook_t) (grub_usb_controller_t dev, + /* Call HOOK with each device, until HOOK returns non-zero. */ + int grub_usb_iterate (grub_usb_iterate_hook_t hook, void *hook_data); + ++grub_usb_device_t grub_usb_get_dev (int addr); ++ + grub_usb_err_t grub_usb_device_initialize (grub_usb_device_t dev); + + grub_usb_err_t grub_usb_get_descriptor (grub_usb_device_t dev, +-- +2.39.5 + diff --git a/config/grub/xhci/patches/0025-Fix-compilation-on-x86_64.patch b/config/grub/xhci/patches/0025-Fix-compilation-on-x86_64.patch new file mode 100644 index 00000000..409f16e5 --- /dev/null +++ b/config/grub/xhci/patches/0025-Fix-compilation-on-x86_64.patch @@ -0,0 +1,90 @@ +From 915d9b148cb57e22bb58505f72bc81add2479a73 Mon Sep 17 00:00:00 2001 +From: Patrick Rudolph +Date: Wed, 24 Feb 2021 08:25:41 +0100 +Subject: [PATCH 25/26] Fix compilation on x86_64 + +Signed-off-by: Patrick Rudolph +--- + grub-core/bus/usb/xhci.c | 24 ++++++++++++++++-------- + 1 file changed, 16 insertions(+), 8 deletions(-) + +diff --git a/grub-core/bus/usb/xhci.c b/grub-core/bus/usb/xhci.c +index 8ad2a10f9..ceb57c9b3 100644 +--- a/grub-core/bus/usb/xhci.c ++++ b/grub-core/bus/usb/xhci.c +@@ -184,7 +184,7 @@ enum + * then we can get it from a trb pointer (provided by evt ring). + */ + #define XHCI_RING(_trb) \ +- ((struct grub_xhci_ring*)((grub_uint32_t)(_trb) & ~(GRUB_XHCI_RING_SIZE-1))) ++ ((struct grub_xhci_ring*)((grub_addr_t)(_trb) & ~(GRUB_XHCI_RING_SIZE-1))) + + /* slot context */ + struct grub_xhci_slotctx { +@@ -495,6 +495,14 @@ grub_xhci_read8(volatile void *addr) { + return (*((volatile grub_uint32_t *)addr)); + } + ++static inline void * ++grub_xhci_read_etrb_ptr(volatile struct grub_xhci_trb *trb) { ++ grub_uint64_t tmp; ++ tmp = (grub_uint64_t)grub_xhci_read32(&trb->ptr_low); ++ tmp |= ((grub_uint64_t)grub_xhci_read32(&trb->ptr_high)) << 32; ++ return (void *)(grub_addr_t)tmp; ++} ++ + static inline grub_uint32_t + grub_xhci_port_read (struct grub_xhci *x, grub_uint32_t port) + { +@@ -697,7 +705,7 @@ static void xhci_process_events(struct grub_xhci *x) + case ER_TRANSFER: + case ER_COMMAND_COMPLETE: + { +- struct grub_xhci_trb *rtrb = (void*)grub_xhci_read32(&etrb->ptr_low); ++ struct grub_xhci_trb *rtrb = grub_xhci_read_etrb_ptr(etrb); + struct grub_xhci_ring *ring = XHCI_RING(rtrb); + volatile struct grub_xhci_trb *evt = &ring->evt; + grub_uint32_t eidx = rtrb - ring->ring + 1; +@@ -730,9 +738,9 @@ static void xhci_process_events(struct grub_xhci *x) + } + grub_xhci_write32(&evts->nidx, nidx); + volatile struct grub_xhci_ir *ir = x->ir; +- grub_uint32_t erdp = (grub_uint32_t)(evts->ring + nidx); +- grub_xhci_write32(&ir->erdp_low, erdp); +- grub_xhci_write32(&ir->erdp_high, 0); ++ grub_uint64_t erdp = (grub_addr_t)(void *)(&evts->ring[nidx]); ++ grub_xhci_write32(&ir->erdp_low, erdp & 0xffffffff); ++ grub_xhci_write32(&ir->erdp_high, erdp >> 32); + } + } + +@@ -833,7 +841,7 @@ static void xhci_trb_queue(volatile struct grub_xhci_ring *ring, + grub_uint32_t xferlen, grub_uint32_t flags) + { + grub_dprintf("xhci", "%s: ring %p data %llx len %d flags 0x%x remain 0x%x\n", __func__, +- ring, data_or_addr, xferlen & 0x1ffff, flags, xferlen >> 17); ++ ring, (unsigned long long)data_or_addr, xferlen & 0x1ffff, flags, xferlen >> 17); + + if (xhci_ring_full(ring)) + { +@@ -1940,7 +1948,7 @@ grub_xhci_setup_transfer (grub_usb_controller_t dev, + if (transfer->type == GRUB_USB_TRANSACTION_TYPE_CONTROL) + { + volatile struct grub_usb_packet_setup *setupdata; +- setupdata = (void *)transfer->transactions[0].data; ++ setupdata = (void *)(grub_addr_t)transfer->transactions[0].data; + grub_dprintf("xhci", "%s: CONTROLL TRANS req %d\n", __func__, setupdata->request); + grub_dprintf("xhci", "%s: CONTROLL TRANS length %d\n", __func__, setupdata->length); + +@@ -2007,7 +2015,7 @@ grub_xhci_setup_transfer (grub_usb_controller_t dev, + /* Assume the ring has enough free space for all TRBs */ + if (flags & TRB_TR_IDT && tr->size <= (int)sizeof(inline_data)) + { +- grub_memcpy(&inline_data, (void *)tr->data, tr->size); ++ grub_memcpy(&inline_data, (void *)(grub_addr_t)tr->data, tr->size); + xhci_trb_queue(reqs, inline_data, tr->size, flags); + } + else +-- +2.39.5 + diff --git a/config/grub/xhci/patches/0026-Add-native-NVMe-driver-based-on-SeaBIOS.patch b/config/grub/xhci/patches/0026-Add-native-NVMe-driver-based-on-SeaBIOS.patch new file mode 100644 index 00000000..ffb1a8a4 --- /dev/null +++ b/config/grub/xhci/patches/0026-Add-native-NVMe-driver-based-on-SeaBIOS.patch @@ -0,0 +1,1074 @@ +From a4e50d731ed687420850a10a84415b9cb9c6cf13 Mon Sep 17 00:00:00 2001 +From: Mate Kukri +Date: Mon, 20 May 2024 11:43:35 +0100 +Subject: [PATCH 26/26] Add native NVMe driver based on SeaBIOS + +Tested to successfully boot Debian on QEMU and OptiPlex 3050. + +Signed-off-by: Mate Kukri +--- + Makefile.am | 2 +- + grub-core/Makefile.core.def | 6 + + grub-core/commands/nativedisk.c | 1 + + grub-core/disk/nvme-int.h | 208 +++++++++ + grub-core/disk/nvme.c | 781 ++++++++++++++++++++++++++++++++ + include/grub/disk.h | 1 + + 6 files changed, 998 insertions(+), 1 deletion(-) + create mode 100644 grub-core/disk/nvme-int.h + create mode 100644 grub-core/disk/nvme.c + +diff --git a/Makefile.am b/Makefile.am +index 65016f856..7bc0866ba 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -434,7 +434,7 @@ if COND_i386_coreboot + FS_PAYLOAD_MODULES ?= $(shell cat grub-core/fs.lst) + default_payload.elf: grub-mkstandalone grub-mkimage FORCE + test -f $@ && rm $@ || true +- pkgdatadir=. ./grub-mkstandalone --grub-mkimage=./grub-mkimage -O i386-coreboot -o $@ --modules='ahci pata xhci ehci uhci ohci usb_keyboard usbms part_msdos ext2 fat at_keyboard part_gpt usbserial_usbdebug cbfs' --install-modules='ls linux search configfile normal cbtime cbls memrw iorw minicmd lsmmap lspci halt reboot hexdump pcidump regexp setpci lsacpi chain test serial multiboot cbmemc linux16 gzio echo help syslinuxcfg xnu $(FS_PAYLOAD_MODULES) password_pbkdf2 $(EXTRA_PAYLOAD_MODULES)' --fonts= --themes= --locales= -d grub-core/ /boot/grub/grub.cfg=$(srcdir)/coreboot.cfg ++ pkgdatadir=. ./grub-mkstandalone --grub-mkimage=./grub-mkimage -O i386-coreboot -o $@ --modules='ahci pata nvme xhci ehci uhci ohci usb_keyboard usbms part_msdos ext2 fat at_keyboard part_gpt usbserial_usbdebug cbfs' --install-modules='ls linux search configfile normal cbtime cbls memrw iorw minicmd lsmmap lspci halt reboot hexdump pcidump regexp setpci lsacpi chain test serial multiboot cbmemc linux16 gzio echo help syslinuxcfg xnu $(FS_PAYLOAD_MODULES) password_pbkdf2 $(EXTRA_PAYLOAD_MODULES)' --fonts= --themes= --locales= -d grub-core/ /boot/grub/grub.cfg=$(srcdir)/coreboot.cfg + endif + + endif +diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def +index fda723f0c..367e4b5e6 100644 +--- a/grub-core/Makefile.core.def ++++ b/grub-core/Makefile.core.def +@@ -2684,3 +2684,9 @@ module = { + cflags = '-Wno-uninitialized'; + cppflags = '-I$(srcdir)/lib/libtasn1-grub -I$(srcdir)/tests/asn1/'; + }; ++ ++module = { ++ name = nvme; ++ common = disk/nvme.c; ++ enable = pci; ++}; +diff --git a/grub-core/commands/nativedisk.c b/grub-core/commands/nativedisk.c +index 6806bff9c..fd68a513e 100644 +--- a/grub-core/commands/nativedisk.c ++++ b/grub-core/commands/nativedisk.c +@@ -78,6 +78,7 @@ get_uuid (const char *name, char **uuid, int getnative) + case GRUB_DISK_DEVICE_ATA_ID: + case GRUB_DISK_DEVICE_SCSI_ID: + case GRUB_DISK_DEVICE_XEN: ++ case GRUB_DISK_DEVICE_NVME_ID: + if (getnative) + break; + /* FALLTHROUGH */ +diff --git a/grub-core/disk/nvme-int.h b/grub-core/disk/nvme-int.h +new file mode 100644 +index 000000000..1295b58aa +--- /dev/null ++++ b/grub-core/disk/nvme-int.h +@@ -0,0 +1,208 @@ ++// NVMe datastructures and constants ++// ++// Copyright 2017 Amazon.com, Inc. or its affiliates. ++// ++// This file may be distributed under the terms of the GNU LGPLv3 license. ++ ++#ifndef __NVME_INT_H ++#define __NVME_INT_H ++ ++#include ++ ++/* Data structures */ ++ ++/* The register file of a NVMe host controller. This struct follows the naming ++ scheme in the NVMe specification. */ ++struct nvme_reg { ++ grub_uint64_t cap; /* controller capabilities */ ++ grub_uint32_t vs; /* version */ ++ grub_uint32_t intms; /* interrupt mask set */ ++ grub_uint32_t intmc; /* interrupt mask clear */ ++ grub_uint32_t cc; /* controller configuration */ ++ grub_uint32_t _res0; ++ grub_uint32_t csts; /* controller status */ ++ grub_uint32_t _res1; ++ grub_uint32_t aqa; /* admin queue attributes */ ++ grub_uint64_t asq; /* admin submission queue base address */ ++ grub_uint64_t acq; /* admin completion queue base address */ ++}; ++ ++/* Submission queue entry */ ++struct nvme_sqe { ++ union { ++ grub_uint32_t dword[16]; ++ struct { ++ grub_uint32_t cdw0; /* Command DWORD 0 */ ++ grub_uint32_t nsid; /* Namespace ID */ ++ grub_uint64_t _res0; ++ grub_uint64_t mptr; /* metadata ptr */ ++ ++ grub_uint64_t dptr_prp1; ++ grub_uint64_t dptr_prp2; ++ }; ++ }; ++}; ++ ++/* Completion queue entry */ ++struct nvme_cqe { ++ union { ++ grub_uint32_t dword[4]; ++ struct { ++ grub_uint32_t cdw0; ++ grub_uint32_t _res0; ++ grub_uint16_t sq_head; ++ grub_uint16_t sq_id; ++ grub_uint16_t cid; ++ grub_uint16_t status; ++ }; ++ }; ++}; ++ ++/* The common part of every submission or completion queue. */ ++struct nvme_queue { ++ grub_uint32_t *dbl; /* doorbell */ ++ grub_uint16_t mask; /* length - 1 */ ++}; ++ ++struct nvme_cq { ++ struct nvme_queue common; ++ struct nvme_cqe *cqe; ++ ++ /* We have read upto (but not including) this entry in the queue. */ ++ grub_uint16_t head; ++ ++ /* The current phase bit the controller uses to indicate that it has written ++ a new entry. This is inverted after each wrap. */ ++ unsigned phase : 1; ++}; ++ ++struct nvme_sq { ++ struct nvme_queue common; ++ struct nvme_sqe *sqe; ++ ++ /* Corresponding completion queue. We only support a single SQ per CQ. */ ++ struct nvme_cq *cq; ++ ++ /* The last entry the controller has fetched. */ ++ grub_uint16_t head; ++ ++ /* The last value we have written to the tail doorbell. */ ++ grub_uint16_t tail; ++}; ++ ++struct nvme_ctrl { ++ grub_pci_device_t pci; ++ struct nvme_reg volatile *reg; ++ ++ grub_uint32_t ctrlnum; ++ ++ grub_uint32_t doorbell_stride; /* in bytes */ ++ ++ struct nvme_sq admin_sq; ++ struct nvme_cq admin_cq; ++ ++ grub_uint32_t ns_count; ++ ++ struct nvme_sq io_sq; ++ struct nvme_cq io_cq; ++}; ++ ++struct nvme_namespace { ++ struct nvme_namespace *next; ++ struct nvme_namespace **prev; ++ ++ char *devname; ++ ++ grub_uint32_t nsnum; ++ ++ struct nvme_ctrl *ctrl; ++ ++ grub_uint32_t ns_id; ++ ++ grub_uint64_t lba_count; /* The total amount of sectors. */ ++ ++ grub_uint32_t block_size; ++ grub_uint32_t metadata_size; ++ grub_uint32_t max_req_size; ++}; ++ ++/* Data structures for NVMe admin identify commands */ ++ ++struct nvme_identify_ctrl { ++ grub_uint16_t vid; ++ grub_uint16_t ssvid; ++ char sn[20]; ++ char mn[40]; ++ char fr[8]; ++ ++ grub_uint8_t rab; ++ grub_uint8_t ieee[3]; ++ grub_uint8_t cmic; ++ grub_uint8_t mdts; ++ ++ char _boring[516 - 78]; ++ ++ grub_uint32_t nn; /* number of namespaces */ ++}; ++ ++struct nvme_identify_ns_list { ++ grub_uint32_t ns_id[1024]; ++}; ++ ++struct nvme_lba_format { ++ grub_uint16_t ms; ++ grub_uint8_t lbads; ++ grub_uint8_t rp; ++}; ++ ++struct nvme_identify_ns { ++ grub_uint64_t nsze; ++ grub_uint64_t ncap; ++ grub_uint64_t nuse; ++ grub_uint8_t nsfeat; ++ grub_uint8_t nlbaf; ++ grub_uint8_t flbas; ++ ++ char _boring[128 - 27]; ++ ++ struct nvme_lba_format lbaf[16]; ++}; ++ ++union nvme_identify { ++ struct nvme_identify_ns ns; ++ struct nvme_identify_ctrl ctrl; ++ struct nvme_identify_ns_list ns_list; ++}; ++ ++/* NVMe constants */ ++ ++#define NVME_CAP_CSS_NVME (1ULL << 37) ++ ++#define NVME_CSTS_FATAL (1U << 1) ++#define NVME_CSTS_RDY (1U << 0) ++ ++#define NVME_CC_EN (1U << 0) ++ ++#define NVME_SQE_OPC_ADMIN_CREATE_IO_SQ 1U ++#define NVME_SQE_OPC_ADMIN_CREATE_IO_CQ 5U ++#define NVME_SQE_OPC_ADMIN_IDENTIFY 6U ++ ++#define NVME_SQE_OPC_IO_WRITE 1U ++#define NVME_SQE_OPC_IO_READ 2U ++ ++#define NVME_ADMIN_IDENTIFY_CNS_ID_NS 0U ++#define NVME_ADMIN_IDENTIFY_CNS_ID_CTRL 1U ++#define NVME_ADMIN_IDENTIFY_CNS_GET_NS_LIST 2U ++ ++#define NVME_CQE_DW3_P (1U << 16) ++ ++#define NVME_PAGE_SIZE 4096 ++#define NVME_PAGE_MASK ~(NVME_PAGE_SIZE - 1) ++ ++/* Length for the queue entries. */ ++#define NVME_SQE_SIZE_LOG 6 ++#define NVME_CQE_SIZE_LOG 4 ++ ++#endif ++ ++/* EOF */ +diff --git a/grub-core/disk/nvme.c b/grub-core/disk/nvme.c +new file mode 100644 +index 000000000..093237c70 +--- /dev/null ++++ b/grub-core/disk/nvme.c +@@ -0,0 +1,781 @@ ++// Low level NVMe disk access ++// ++// Based on SeaBIOS NVMe driver - Copyright 2017 Amazon.com, Inc. or its affiliates. ++// Port to GRUB2 done by Mate Kukri ++// ++// This file may be distributed under the terms of the GNU LGPLv3 license. ++ ++#include ++#include ++#include ++#include "nvme-int.h" ++ ++GRUB_MOD_LICENSE ("GPLv3"); /* LGPLv3 in reality but it is GPLv3 compatible */ ++ ++static grub_uint32_t grub_nvme_ctrlcnt; ++static grub_uint32_t grub_nvme_nscnt; ++ ++static struct nvme_namespace *grub_nvme_namespaces; ++ ++// Page aligned "dma bounce buffer" of size NVME_PAGE_SIZE ++static void *nvme_dma_buffer; ++ ++static void * ++zalloc_page_aligned(grub_uint32_t size) ++{ ++ void *res = grub_memalign(NVME_PAGE_SIZE, size); ++ if (res) grub_memset(res, 0, size); ++ return res; ++} ++ ++static void ++nvme_init_queue_common(struct nvme_ctrl *ctrl, struct nvme_queue *q, grub_uint16_t q_idx, ++ grub_uint16_t length) ++{ ++ grub_memset(q, 0, sizeof(*q)); ++ q->dbl = (grub_uint32_t *)((char *)ctrl->reg + 0x1000 + q_idx * ctrl->doorbell_stride); ++ grub_dprintf("nvme", " q %p q_idx %u dbl %p\n", q, q_idx, q->dbl); ++ q->mask = length - 1; ++} ++ ++static int ++nvme_init_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq, grub_uint16_t q_idx, grub_uint16_t length, ++ struct nvme_cq *cq) ++{ ++ nvme_init_queue_common(ctrl, &sq->common, q_idx, length); ++ sq->sqe = zalloc_page_aligned(sizeof(*sq->sqe) * length); ++ ++ if (!sq->sqe) { ++ return -1; ++ } ++ ++ grub_dprintf("nvme", "sq %p q_idx %u sqe %p\n", sq, q_idx, sq->sqe); ++ sq->cq = cq; ++ sq->head = 0; ++ sq->tail = 0; ++ ++ return 0; ++} ++ ++static int ++nvme_init_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq, grub_uint16_t q_idx, grub_uint16_t length) ++{ ++ nvme_init_queue_common(ctrl, &cq->common, q_idx, length); ++ cq->cqe = zalloc_page_aligned(sizeof(*cq->cqe) * length); ++ if (!cq->cqe) { ++ return -1; ++ } ++ ++ cq->head = 0; ++ ++ /* All CQE phase bits are initialized to zero. This means initially we wait ++ for the host controller to set these to 1. */ ++ cq->phase = 1; ++ ++ return 0; ++} ++ ++static int ++nvme_poll_cq(struct nvme_cq *cq) ++{ ++ grub_uint32_t dw3 = *(volatile grub_uint32_t *) &cq->cqe[cq->head].dword[3]; ++ return (!!(dw3 & NVME_CQE_DW3_P) == cq->phase); ++} ++ ++static int ++nvme_is_cqe_success(struct nvme_cqe const *cqe) ++{ ++ return ((cqe->status >> 1) & 0xFF) == 0; ++} ++ ++static struct nvme_cqe ++nvme_error_cqe(void) ++{ ++ struct nvme_cqe r; ++ ++ /* 0xFF is a vendor specific status code != success. Should be okay for ++ indicating failure. */ ++ grub_memset(&r, 0xFF, sizeof(r)); ++ return r; ++} ++ ++static struct nvme_cqe ++nvme_consume_cqe(struct nvme_sq *sq) ++{ ++ struct nvme_cq *cq = sq->cq; ++ ++ if (!nvme_poll_cq(cq)) { ++ /* Cannot consume a completion queue entry, if there is none ready. */ ++ return nvme_error_cqe(); ++ } ++ ++ struct nvme_cqe *cqe = &cq->cqe[cq->head]; ++ grub_uint16_t cq_next_head = (cq->head + 1) & cq->common.mask; ++ grub_dprintf("nvme", "cq %p head %u -> %u\n", cq, cq->head, cq_next_head); ++ if (cq_next_head < cq->head) { ++ grub_dprintf("nvme", "cq %p wrap\n", cq); ++ cq->phase = ~cq->phase; ++ } ++ cq->head = cq_next_head; ++ ++ /* Update the submission queue head. */ ++ if (cqe->sq_head != sq->head) { ++ sq->head = cqe->sq_head; ++ grub_dprintf("nvme", "sq %p advanced to %u\n", sq, cqe->sq_head); ++ } ++ ++ /* Tell the controller that we consumed the completion. */ ++ *(volatile grub_uint32_t *) cq->common.dbl = cq->head; ++ ++ return *cqe; ++} ++ ++static struct nvme_cqe ++nvme_wait(struct nvme_sq *sq) ++{ ++ // static const unsigned nvme_timeout = 5000 /* ms */; ++ // grub_uint32_t to = timer_calc(nvme_timeout); ++ while (!nvme_poll_cq(sq->cq)) { ++ /* FIXME ++ yield(); ++ ++ if (timer_check(to)) { ++ warn_timeout(); ++ return nvme_error_cqe(); ++ }*/ ++ } ++ ++ return nvme_consume_cqe(sq); ++} ++ ++/* Returns the next submission queue entry (or NULL if the queue is full). It ++ also fills out Command Dword 0 and clears the rest. */ ++static struct nvme_sqe * ++nvme_get_next_sqe(struct nvme_sq *sq, grub_uint8_t opc, void *metadata, void *data, void *data2) ++{ ++ if (((sq->head + 1) & sq->common.mask) == sq->tail) { ++ grub_dprintf("nvme", "submission queue is full\n"); ++ return NULL; ++ } ++ ++ struct nvme_sqe *sqe = &sq->sqe[sq->tail]; ++ grub_dprintf("nvme", "sq %p next_sqe %u\n", sq, sq->tail); ++ ++ grub_memset(sqe, 0, sizeof(*sqe)); ++ sqe->cdw0 = opc | (sq->tail << 16 /* CID */); ++ sqe->mptr = (grub_uint32_t)metadata; ++ sqe->dptr_prp1 = (grub_uint32_t)data; ++ sqe->dptr_prp2 = (grub_uint32_t)data2; ++ ++ return sqe; ++} ++ ++/* Call this after you've filled out an sqe that you've got from nvme_get_next_sqe. */ ++static void ++nvme_commit_sqe(struct nvme_sq *sq) ++{ ++ grub_dprintf("nvme", "sq %p commit_sqe %u\n", sq, sq->tail); ++ sq->tail = (sq->tail + 1) & sq->common.mask; ++ *(volatile grub_uint32_t *) sq->common.dbl = sq->tail; ++} ++ ++/* Perform an identify command on the admin queue and return the resulting ++ buffer. This may be a NULL pointer, if something failed. This function ++ cannot be used after initialization, because it uses buffers in tmp zone. */ ++static union nvme_identify * ++nvme_admin_identify(struct nvme_ctrl *ctrl, grub_uint8_t cns, grub_uint32_t nsid) ++{ ++ union nvme_identify *identify_buf = zalloc_page_aligned(4096); ++ if (!identify_buf) ++ return NULL; ++ ++ struct nvme_sqe *cmd_identify; ++ cmd_identify = nvme_get_next_sqe(&ctrl->admin_sq, ++ NVME_SQE_OPC_ADMIN_IDENTIFY, NULL, ++ identify_buf, NULL); ++ if (!cmd_identify) ++ goto error; ++ ++ cmd_identify->nsid = nsid; ++ cmd_identify->dword[10] = cns; ++ ++ nvme_commit_sqe(&ctrl->admin_sq); ++ ++ struct nvme_cqe cqe = nvme_wait(&ctrl->admin_sq); ++ ++ if (!nvme_is_cqe_success(&cqe)) { ++ goto error; ++ } ++ ++ return identify_buf; ++ error: ++ grub_free(identify_buf); ++ return NULL; ++} ++ ++static struct nvme_identify_ctrl * ++nvme_admin_identify_ctrl(struct nvme_ctrl *ctrl) ++{ ++ return &nvme_admin_identify(ctrl, NVME_ADMIN_IDENTIFY_CNS_ID_CTRL, 0)->ctrl; ++} ++ ++static struct nvme_identify_ns * ++nvme_admin_identify_ns(struct nvme_ctrl *ctrl, grub_uint32_t ns_id) ++{ ++ return &nvme_admin_identify(ctrl, NVME_ADMIN_IDENTIFY_CNS_ID_NS, ++ ns_id)->ns; ++} ++ ++static void ++nvme_probe_ns(struct nvme_ctrl *ctrl, grub_uint32_t ns_idx, grub_uint8_t mdts) ++{ ++ grub_uint32_t ns_id = ns_idx + 1; ++ ++ struct nvme_identify_ns *id = nvme_admin_identify_ns(ctrl, ns_id); ++ if (!id) { ++ grub_dprintf("nvme", "NVMe couldn't identify namespace %u.\n", ns_id); ++ goto free_buffer; ++ } ++ ++ grub_uint8_t current_lba_format = id->flbas & 0xF; ++ if (current_lba_format > id->nlbaf) { ++ grub_dprintf("nvme", "NVMe NS %u: current LBA format %u is beyond what the " ++ " namespace supports (%u)?\n", ++ ns_id, current_lba_format, id->nlbaf + 1); ++ goto free_buffer; ++ } ++ ++ if (!id->nsze) { ++ grub_dprintf("nvme", "NVMe NS %u is inactive.\n", ns_id); ++ goto free_buffer; ++ } ++ ++ if (!nvme_dma_buffer) { ++ nvme_dma_buffer = zalloc_page_aligned(NVME_PAGE_SIZE); ++ if (!nvme_dma_buffer) { ++ goto free_buffer; ++ } ++ } ++ ++ struct nvme_namespace *ns = grub_malloc(sizeof(*ns)); ++ if (!ns) { ++ goto free_buffer; ++ } ++ grub_memset(ns, 0, sizeof(*ns)); ++ ns->ctrl = ctrl; ++ ns->ns_id = ns_id; ++ ns->lba_count = id->nsze; ++ ++ struct nvme_lba_format *fmt = &id->lbaf[current_lba_format]; ++ ++ ns->block_size = 1U << fmt->lbads; ++ ns->metadata_size = fmt->ms; ++ ++ if (ns->block_size > NVME_PAGE_SIZE) { ++ /* If we see devices that trigger this path, we need to increase our ++ buffer size. */ ++ grub_free(ns); ++ goto free_buffer; ++ } ++ ++ if (mdts) { ++ ns->max_req_size = ((1U << mdts) * NVME_PAGE_SIZE) / ns->block_size; ++ grub_dprintf("nvme", "NVME NS %u max request size: %d sectors\n", ++ ns_id, ns->max_req_size); ++ } else { ++ ns->max_req_size = -1U; ++ } ++ ++ ns->devname = grub_xasprintf("nvme%un%u", ctrl->ctrlnum, ns_id); ++ ns->nsnum = grub_nvme_nscnt++; ++ ++ grub_list_push (GRUB_AS_LIST_P (&grub_nvme_namespaces), GRUB_AS_LIST (ns)); ++ ++free_buffer: ++ grub_free(id); ++} ++ ++ ++/* Release memory allocated for a completion queue */ ++static void ++nvme_destroy_cq(struct nvme_cq *cq) ++{ ++ grub_free(cq->cqe); ++ cq->cqe = NULL; ++} ++ ++/* Release memory allocated for a submission queue */ ++static void ++nvme_destroy_sq(struct nvme_sq *sq) ++{ ++ grub_free(sq->sqe); ++ sq->sqe = NULL; ++} ++ ++/* Returns 0 on success. */ ++static int ++nvme_create_io_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq, grub_uint16_t q_idx) ++{ ++ int rc; ++ struct nvme_sqe *cmd_create_cq; ++ grub_uint32_t length = 1 + (ctrl->reg->cap & 0xffff); ++ if (length > NVME_PAGE_SIZE / sizeof(struct nvme_cqe)) ++ length = NVME_PAGE_SIZE / sizeof(struct nvme_cqe); ++ ++ rc = nvme_init_cq(ctrl, cq, q_idx, length); ++ if (rc) { ++ goto err; ++ } ++ ++ cmd_create_cq = nvme_get_next_sqe(&ctrl->admin_sq, ++ NVME_SQE_OPC_ADMIN_CREATE_IO_CQ, NULL, ++ cq->cqe, NULL); ++ if (!cmd_create_cq) { ++ goto err_destroy_cq; ++ } ++ ++ cmd_create_cq->dword[10] = (cq->common.mask << 16) | (q_idx >> 1); ++ cmd_create_cq->dword[11] = 1 /* physically contiguous */; ++ ++ nvme_commit_sqe(&ctrl->admin_sq); ++ ++ struct nvme_cqe cqe = nvme_wait(&ctrl->admin_sq); ++ ++ if (!nvme_is_cqe_success(&cqe)) { ++ grub_dprintf("nvme", "create io cq failed: %08x %08x %08x %08x\n", ++ cqe.dword[0], cqe.dword[1], cqe.dword[2], cqe.dword[3]); ++ ++ goto err_destroy_cq; ++ } ++ ++ return 0; ++ ++err_destroy_cq: ++ nvme_destroy_cq(cq); ++err: ++ return -1; ++} ++ ++/* Returns 0 on success. */ ++static int ++nvme_create_io_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq, grub_uint16_t q_idx, struct nvme_cq *cq) ++{ ++ int rc; ++ struct nvme_sqe *cmd_create_sq; ++ grub_uint32_t length = 1 + (ctrl->reg->cap & 0xffff); ++ if (length > NVME_PAGE_SIZE / sizeof(struct nvme_cqe)) ++ length = NVME_PAGE_SIZE / sizeof(struct nvme_cqe); ++ ++ rc = nvme_init_sq(ctrl, sq, q_idx, length, cq); ++ if (rc) { ++ goto err; ++ } ++ ++ cmd_create_sq = nvme_get_next_sqe(&ctrl->admin_sq, ++ NVME_SQE_OPC_ADMIN_CREATE_IO_SQ, NULL, ++ sq->sqe, NULL); ++ if (!cmd_create_sq) { ++ goto err_destroy_sq; ++ } ++ ++ cmd_create_sq->dword[10] = (sq->common.mask << 16) | (q_idx >> 1); ++ cmd_create_sq->dword[11] = (q_idx >> 1) << 16 | 1 /* contiguous */; ++ grub_dprintf("nvme", "sq %p create dword10 %08x dword11 %08x\n", sq, ++ cmd_create_sq->dword[10], cmd_create_sq->dword[11]); ++ ++ nvme_commit_sqe(&ctrl->admin_sq); ++ ++ struct nvme_cqe cqe = nvme_wait(&ctrl->admin_sq); ++ ++ if (!nvme_is_cqe_success(&cqe)) { ++ grub_dprintf("nvme", "create io sq failed: %08x %08x %08x %08x\n", ++ cqe.dword[0], cqe.dword[1], cqe.dword[2], cqe.dword[3]); ++ goto err_destroy_sq; ++ } ++ ++ return 0; ++ ++err_destroy_sq: ++ nvme_destroy_sq(sq); ++err: ++ return -1; ++} ++ ++/* Reads count sectors into buf. The buffer cannot cross page boundaries. */ ++static int ++nvme_io_xfer(struct nvme_namespace *ns, grub_uint64_t lba, void *prp1, void *prp2, ++ grub_uint16_t count, int write) ++{ ++ if (((grub_uint32_t)prp1 & 0x3) || ((grub_uint32_t)prp2 & 0x3)) { ++ /* Buffer is misaligned */ ++ return -1; ++ } ++ ++ struct nvme_sqe *io_read = nvme_get_next_sqe(&ns->ctrl->io_sq, ++ write ? NVME_SQE_OPC_IO_WRITE ++ : NVME_SQE_OPC_IO_READ, ++ NULL, prp1, prp2); ++ io_read->nsid = ns->ns_id; ++ io_read->dword[10] = (grub_uint32_t)lba; ++ io_read->dword[11] = (grub_uint32_t)(lba >> 32); ++ io_read->dword[12] = (1U << 31 /* limited retry */) | (count - 1); ++ ++ nvme_commit_sqe(&ns->ctrl->io_sq); ++ ++ struct nvme_cqe cqe = nvme_wait(&ns->ctrl->io_sq); ++ ++ if (!nvme_is_cqe_success(&cqe)) { ++ grub_dprintf("nvme", "read io: %08x %08x %08x %08x\n", ++ cqe.dword[0], cqe.dword[1], cqe.dword[2], cqe.dword[3]); ++ ++ return -1; ++ } ++ ++ grub_dprintf("nvme", "ns %u %s lba %llu+%u\n", ns->ns_id, write ? "write" : "read", ++ lba, count); ++ return count; ++} ++ ++// Transfer up to one page of data using the internal dma bounce buffer ++static int ++nvme_bounce_xfer(struct nvme_namespace *ns, grub_uint64_t lba, void *buf, grub_uint16_t count, ++ int write) ++{ ++ grub_uint16_t const max_blocks = NVME_PAGE_SIZE / ns->block_size; ++ grub_uint16_t blocks = count < max_blocks ? count : max_blocks; ++ ++ if (write) ++ grub_memcpy(nvme_dma_buffer, buf, blocks * ns->block_size); ++ ++ int res = nvme_io_xfer(ns, lba, nvme_dma_buffer, NULL, blocks, write); ++ ++ if (!write && res >= 0) ++ grub_memcpy(buf, nvme_dma_buffer, res * ns->block_size); ++ ++ return res; ++} ++ ++#define NVME_MAX_PRPL_ENTRIES 15 /* Allows requests up to 64kb */ ++ ++// Transfer data using page list (if applicable) ++static int ++nvme_prpl_xfer(struct nvme_namespace *ns, grub_uint64_t lba, void *buf, grub_uint16_t count, ++ int write) ++{ ++ grub_uint32_t base = (long)buf; ++ grub_int32_t size; ++ ++ if (count > ns->max_req_size) ++ count = ns->max_req_size; ++ ++ size = count * ns->block_size; ++ /* Special case for transfers that fit into PRP1, but are unaligned */ ++ if (((size + (base & ~NVME_PAGE_MASK)) <= NVME_PAGE_SIZE)) ++ goto single; ++ ++ /* Every request has to be page aligned */ ++ if (base & ~NVME_PAGE_MASK) ++ goto bounce; ++ ++ /* Make sure a full block fits into the last chunk */ ++ if (size & (ns->block_size - 1ULL)) ++ goto bounce; ++ ++ /* Build PRP list if we need to describe more than 2 pages */ ++ if ((ns->block_size * count) > (NVME_PAGE_SIZE * 2)) { ++ grub_uint32_t prpl_len = 0; ++ grub_uint64_t *prpl = nvme_dma_buffer; ++ int first_page = 1; ++ for (; size > 0; base += NVME_PAGE_SIZE, size -= NVME_PAGE_SIZE) { ++ if (first_page) { ++ /* First page is special */ ++ first_page = 0; ++ continue; ++ } ++ if (prpl_len >= NVME_MAX_PRPL_ENTRIES) ++ goto bounce; ++ prpl[prpl_len++] = base; ++ } ++ return nvme_io_xfer(ns, lba, buf, prpl, count, write); ++ } ++ ++ /* Directly embed the 2nd page if we only need 2 pages */ ++ if ((ns->block_size * count) > NVME_PAGE_SIZE) ++ return nvme_io_xfer(ns, lba, buf, (char *) buf + NVME_PAGE_SIZE, count, write); ++ ++single: ++ /* One page is enough, don't expose anything else */ ++ return nvme_io_xfer(ns, lba, buf, NULL, count, write); ++ ++bounce: ++ /* Use bounce buffer to make transfer */ ++ return nvme_bounce_xfer(ns, lba, buf, count, write); ++} ++ ++static int ++nvme_create_io_queues(struct nvme_ctrl *ctrl) ++{ ++ if (nvme_create_io_cq(ctrl, &ctrl->io_cq, 3)) ++ goto err; ++ ++ if (nvme_create_io_sq(ctrl, &ctrl->io_sq, 2, &ctrl->io_cq)) ++ goto err_free_cq; ++ ++ return 0; ++ ++ err_free_cq: ++ nvme_destroy_cq(&ctrl->io_cq); ++ err: ++ return -1; ++} ++ ++/* Waits for CSTS.RDY to match rdy. Returns 0 on success. */ ++static int ++nvme_wait_csts_rdy(struct nvme_ctrl *ctrl, unsigned rdy) ++{ ++ // grub_uint32_t const max_to = 500 /* ms */ * ((ctrl->reg->cap >> 24) & 0xFFU); ++ // grub_uint32_t to = timer_calc(max_to); ++ grub_uint32_t csts; ++ ++ while (rdy != ((csts = ctrl->reg->csts) & NVME_CSTS_RDY)) { ++ // FIXME ++ //yield(); ++ ++ if (csts & NVME_CSTS_FATAL) { ++ grub_dprintf("nvme", "NVMe fatal error during controller shutdown\n"); ++ return -1; ++ } ++ ++ /* ++ if (timer_check(to)) { ++ warn_timeout(); ++ return -1; ++ }*/ ++ } ++ ++ return 0; ++} ++ ++/* Returns 0 on success. */ ++static int grub_nvme_controller_enable(struct nvme_ctrl *ctrl) ++{ ++ grub_pci_address_t addr; ++ int rc; ++ ++ addr = grub_pci_make_address (ctrl->pci, GRUB_PCI_REG_COMMAND); ++ grub_pci_write_word (addr, grub_pci_read_word (addr) | GRUB_PCI_COMMAND_BUS_MASTER); ++ ++ /* Turn the controller off. */ ++ ctrl->reg->cc = 0; ++ if (nvme_wait_csts_rdy(ctrl, 0)) { ++ grub_dprintf("nvme", "NVMe fatal error during controller shutdown\n"); ++ return -1; ++ } ++ ++ ctrl->doorbell_stride = 4U << ((ctrl->reg->cap >> 32) & 0xF); ++ ++ rc = nvme_init_cq(ctrl, &ctrl->admin_cq, 1, ++ NVME_PAGE_SIZE / sizeof(struct nvme_cqe)); ++ if (rc) { ++ return -1; ++ } ++ ++ rc = nvme_init_sq(ctrl, &ctrl->admin_sq, 0, ++ NVME_PAGE_SIZE / sizeof(struct nvme_sqe), &ctrl->admin_cq); ++ if (rc) { ++ goto err_destroy_admin_cq; ++ } ++ ++ ctrl->reg->aqa = ctrl->admin_cq.common.mask << 16 ++ | ctrl->admin_sq.common.mask; ++ ++ ctrl->reg->asq = (grub_uint32_t)ctrl->admin_sq.sqe; ++ ctrl->reg->acq = (grub_uint32_t)ctrl->admin_cq.cqe; ++ ++ grub_dprintf("nvme", " admin submission queue: %p\n", ctrl->admin_sq.sqe); ++ grub_dprintf("nvme", " admin completion queue: %p\n", ctrl->admin_cq.cqe); ++ ++ ctrl->reg->cc = NVME_CC_EN | (NVME_CQE_SIZE_LOG << 20) ++ | (NVME_SQE_SIZE_LOG << 16 /* IOSQES */); ++ ++ if (nvme_wait_csts_rdy(ctrl, 1)) { ++ grub_dprintf("nvme", "NVMe fatal error while enabling controller\n"); ++ goto err_destroy_admin_sq; ++ } ++ ++ /* The admin queue is set up and the controller is ready. Let's figure out ++ what namespaces we have. */ ++ ++ struct nvme_identify_ctrl *identify = nvme_admin_identify_ctrl(ctrl); ++ ++ if (!identify) { ++ grub_dprintf("nvme", "NVMe couldn't identify controller.\n"); ++ goto err_destroy_admin_sq; ++ } ++ ++ grub_dprintf("nvme", "NVMe has %u namespace%s.\n", ++ identify->nn, (identify->nn == 1) ? "" : "s"); ++ ++ ctrl->ns_count = identify->nn; ++ grub_uint8_t mdts = identify->mdts; ++ grub_free(identify); ++ ++ if ((ctrl->ns_count == 0) || nvme_create_io_queues(ctrl)) { ++ /* No point to continue, if the controller says it doesn't have ++ namespaces or we couldn't create I/O queues. */ ++ goto err_destroy_admin_sq; ++ } ++ ++ /* Give the controller a global number */ ++ ctrl->ctrlnum = grub_nvme_ctrlcnt++; ++ ++ /* Populate namespace IDs */ ++ for (grub_uint32_t ns_idx = 0; ns_idx < ctrl->ns_count; ns_idx++) { ++ nvme_probe_ns(ctrl, ns_idx, mdts); ++ } ++ ++ grub_dprintf("nvme", "NVMe initialization complete!\n"); ++ return 0; ++ ++ err_destroy_admin_sq: ++ nvme_destroy_sq(&ctrl->admin_sq); ++ err_destroy_admin_cq: ++ nvme_destroy_cq(&ctrl->admin_cq); ++ return -1; ++} ++ ++static int grub_nvme_pci_probe(grub_pci_device_t dev, grub_pci_id_t pciid __attribute__ ((unused)), void *data __attribute__ ((unused))) ++{ ++ grub_pci_address_t addr; ++ grub_uint32_t class, bar, version; ++ struct nvme_reg volatile *reg; ++ ++ class = grub_pci_read (grub_pci_make_address (dev, GRUB_PCI_REG_CLASS)); ++ if (class >> 16 != 0x0108) ++ return 0; ++ if ((class >> 8 & 0xff) != 2) { /* as of NVM 1.0e */ ++ grub_dprintf("nvme", "Found incompatble NVMe: prog-if=%02x\n", class >> 8 & 0xff); ++ return 0; ++ } ++ ++ bar = grub_pci_read (grub_pci_make_address (dev, GRUB_PCI_REG_ADDRESS_REG0)); ++ reg = grub_pci_device_map_range (dev, bar & GRUB_PCI_ADDR_MEM_MASK, sizeof (*reg)); ++ ++ addr = grub_pci_make_address (dev, GRUB_PCI_REG_COMMAND); ++ grub_pci_write_word (addr, grub_pci_read_word (addr) | GRUB_PCI_COMMAND_MEM_ENABLED); ++ ++ version = reg->vs; ++ grub_dprintf("nvme", "Found NVMe controller with version %u.%u.%u.\n", version >> 16, (version >> 8) & 0xFF, version & 0xFF); ++ grub_dprintf("nvme", " Capabilities %016llx\n", reg->cap); ++ ++ if (~reg->cap & NVME_CAP_CSS_NVME) { ++ grub_dprintf("nvme", "Controller doesn't speak NVMe command set. Skipping.\n"); ++ goto err; ++ } ++ ++ struct nvme_ctrl *ctrl = grub_malloc(sizeof(*ctrl)); ++ if (!ctrl) ++ goto err; ++ ++ grub_memset(ctrl, 0, sizeof(*ctrl)); ++ ++ ctrl->reg = reg; ++ ctrl->pci = dev; ++ ++ if (grub_nvme_controller_enable(ctrl)) ++ goto err_free_ctrl; ++ ++ return 0; ++ ++ err_free_ctrl: ++ grub_free(ctrl); ++ err: ++ grub_dprintf("nvme", "Failed to enable NVMe controller.\n"); ++ return 0; ++} ++ ++static int ++grub_nvme_iterate (grub_disk_dev_iterate_hook_t hook, void *hook_data, grub_disk_pull_t pull) ++{ ++ struct nvme_namespace *ns; ++ ++ if (pull != GRUB_DISK_PULL_NONE) ++ return 0; ++ ++ FOR_LIST_ELEMENTS(ns, grub_nvme_namespaces) ++ if (hook (ns->devname, hook_data)) ++ return 1; ++ ++ return 0; ++} ++ ++static grub_err_t ++grub_nvme_open (const char *name __attribute ((unused)), grub_disk_t disk __attribute ((unused))) ++{ ++ struct nvme_namespace *ns; ++ ++ FOR_LIST_ELEMENTS(ns, grub_nvme_namespaces) ++ if (grub_strcmp (ns->devname, name) == 0) ++ break; ++ ++ if (! ns) ++ return grub_error (GRUB_ERR_UNKNOWN_DEVICE, "can't open device"); ++ ++ disk->total_sectors = ns->lba_count; ++ disk->max_agglomerate = ns->max_req_size; ++ ++ disk->id = ns->nsnum; /* global id of the namespace */ ++ ++ disk->data = ns; ++ ++ return 0; ++} ++ ++static grub_err_t ++nvme_readwrite(struct nvme_namespace *ns, grub_disk_addr_t sector, grub_size_t num_sectors, char *buf, int write) ++{ ++ for (int i = 0; i < num_sectors;) { ++ grub_uint16_t blocks_remaining = num_sectors - i; ++ char *op_buf = buf + i * ns->block_size; ++ int blocks = nvme_prpl_xfer(ns, sector + i, op_buf, blocks_remaining, write); ++ if (blocks < 0) ++ return GRUB_ERR_IO; ++ i += blocks; ++ } ++ return GRUB_ERR_NONE; ++} ++ ++static grub_err_t ++grub_nvme_read (grub_disk_t disk, grub_disk_addr_t sector, grub_size_t num_sectors, char *buf) ++{ ++ return nvme_readwrite((struct nvme_namespace *) disk->data, sector, num_sectors, buf, 0); ++} ++ ++static grub_err_t ++grub_nvme_write (grub_disk_t disk, grub_disk_addr_t sector, grub_size_t num_sectors, const char *buf) ++{ ++ return nvme_readwrite((struct nvme_namespace *) disk->data, sector, num_sectors, buf, 1); ++} ++ ++static struct grub_disk_dev grub_nvme_dev = ++ { ++ .name = "nvme", ++ .id = GRUB_DISK_DEVICE_NVME_ID, ++ .disk_iterate = grub_nvme_iterate, ++ .disk_open = grub_nvme_open, ++ .disk_read = grub_nvme_read, ++ .disk_write = grub_nvme_write, ++ .next = 0 ++ }; ++ ++GRUB_MOD_INIT(nvme) ++{ ++ grub_stop_disk_firmware (); ++ grub_pci_iterate (grub_nvme_pci_probe, NULL); ++ grub_disk_dev_register (&grub_nvme_dev); ++} ++ ++GRUB_MOD_FINI(nvme) ++{ ++ grub_disk_dev_unregister (&grub_nvme_dev); ++} +diff --git a/include/grub/disk.h b/include/grub/disk.h +index fbf23df7f..186e76f0b 100644 +--- a/include/grub/disk.h ++++ b/include/grub/disk.h +@@ -52,6 +52,7 @@ enum grub_disk_dev_id + GRUB_DISK_DEVICE_UBOOTDISK_ID, + GRUB_DISK_DEVICE_XEN, + GRUB_DISK_DEVICE_OBDISK_ID, ++ GRUB_DISK_DEVICE_NVME_ID + }; + + struct grub_disk; +-- +2.39.5 + -- cgit v1.2.1