// SPDX-License-Identifier: GPL-2.0 use kernel::{device, devres::Devres, error::code::*, pci, prelude::*, sync::Arc}; use crate::driver::Bar0;use crate::falcon::{gsp::Gsp, sec2::Sec2, Falcon};use crate::fb::FbLayout;use crate::fb::SysmemFlush;use crate::firmware::fwsec::{FwsecCommand, FwsecFirmware};use crate::firmware::{Firmware, FIRMWARE_VERSION};use crate::gfw;use crate::regs;use crate::util;use crate::vbios::Vbios;use core::fmt; macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => { /// Enum representation of the GPU chipset. #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] pub(crate) enum Chipset { $($variant = $value),*, } impl Chipset { pub(crate) const ALL: &'static [Chipset] = &[ $( Chipset::$variant, )* ]; pub(crate) const NAMES: [&'static str; Self::ALL.len()] = [ $( util::const_bytes_to_str( util::to_lowercase_bytes::<{ stringify!($variant).len() }>( stringify!($variant) ).as_slice() ), )* ]; } // TODO[FPRI]: replace with something like derive(FromPrimitive) impl TryFrom<u32> for Chipset { type Error = kernel::error::Error; fn try_from(value: u32) -> Result<Self, Self::Error> { match value { $( $value => Ok(Chipset::$variant), )* _ => Err(ENODEV), } } } }} define_chipset!({ // Turing TU102 = 0x162, TU104 = 0x164, TU106 = 0x166, TU117 = 0x167, TU116 = 0x168, // Ampere GA100 = 0x170, GA102 = 0x172, GA103 = 0x173, GA104 = 0x174, GA106 = 0x176, GA107 = 0x177, // Ada AD102 = 0x192, AD103 = 0x193, AD104 = 0x194, AD106 = 0x196, AD107 = 0x197,}); impl Chipset { pub(crate) fn arch(&self) -> Architecture { match self { Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => { Architecture::Turing } Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => { Architecture::Ampere } Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => { Architecture::Ada } } }} // TODO//// The resulting strings are used to generate firmware paths, hence the// generated strings have to be stable.//// Hence, replace with something like strum_macros derive(Display).//// For now, redirect to fmt::Debug for convenience.impl fmt::Display for Chipset { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self:?}") }} /// Enum representation of the GPU generation.#[derive(fmt::Debug)]pub(crate) enum Architecture { Turing = 0x16, Ampere = 0x17, Ada = 0x19,} impl TryFrom<u8> for Architecture { type Error = Error; fn try_from(value: u8) -> Result<Self> { match value { 0x16 => Ok(Self::Turing), 0x17 => Ok(Self::Ampere), 0x19 => Ok(Self::Ada), _ => Err(ENODEV), } }} pub(crate) struct Revision { major: u8, minor: u8,} impl Revision { fn from_boot0(boot0: regs::NV_PMC_BOOT_0) -> Self { Self { major: boot0.major_revision(), minor: boot0.minor_revision(), } }} impl fmt::Display for Revision { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:x}.{:x}", self.major, self.minor) }} /// Structure holding the metadata of the GPU.pub(crate) struct Spec { chipset: Chipset, /// The revision of the chipset. revision: Revision,} impl Spec { fn new(bar: &Bar0) -> Result<Spec> { let boot0 = regs::NV_PMC_BOOT_0::read(bar); Ok(Self { chipset: boot0.chipset()?, revision: Revision::from_boot0(boot0), }) }} /// Structure holding the resources required to operate the GPU.#[pin_data(PinnedDrop)]pub(crate) struct Gpu { spec: Spec, /// MMIO mapping of PCI BAR 0 bar: Arc<Devres<Bar0>>, fw: Firmware, /// System memory page required for flushing all pending GPU-side memory writes done through /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). sysmem_flush: SysmemFlush,} #[pinned_drop]impl PinnedDrop for Gpu { fn drop(self: Pin<&mut Self>) { // Unregister the sysmem flush page before we release it. self.bar .try_access_with(|b| self.sysmem_flush.unregister(b)); }} impl Gpu { /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly /// created the WPR2 region. /// /// TODO: this needs to be moved into a larger type responsible for booting the whole GSP /// (`GspBooter`?). fn run_fwsec_frts( dev: &device::Device<device::Bound>, falcon: &Falcon<Gsp>, bar: &Bar0, bios: &Vbios, fb_layout: &FbLayout, ) -> Result<()> { // Check that the WPR2 region does not already exists - if it does, we cannot run // FWSEC-FRTS until the GPU is reset. if regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound() != 0 { dev_err!( dev, "WPR2 region already exists - GPU needs to be reset to proceed\n" ); return Err(EBUSY); } let fwsec_frts = FwsecFirmware::new( dev, falcon, bar, bios, FwsecCommand::Frts { frts_addr: fb_layout.frts.start, frts_size: fb_layout.frts.end - fb_layout.frts.start, }, )?; // Run FWSEC-FRTS to create the WPR2 region. fwsec_frts.run(dev, falcon, bar)?; // SCRATCH_E contains the error code for FWSEC-FRTS. let frts_status = regs::NV_PBUS_SW_SCRATCH_0E::read(bar).frts_err_code(); if frts_status != 0 { dev_err!( dev, "FWSEC-FRTS returned with error code {:#x}", frts_status ); return Err(EIO); } // Check that the WPR2 region has been created as we requested. let (wpr2_lo, wpr2_hi) = ( regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO::read(bar).lower_bound(), regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound(), ); match (wpr2_lo, wpr2_hi) { (_, 0) => { dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); Err(EIO) } (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { dev_err!( dev, "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", wpr2_lo, fb_layout.frts.start, ); Err(EIO) } (wpr2_lo, wpr2_hi) => { dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); dev_dbg!(dev, "GPU instance built\n"); Ok(()) } } } pub(crate) fn new( pdev: &pci::Device<device::Bound>, devres_bar: Arc<Devres<Bar0>>, ) -> Result<impl PinInit<Self>> { let bar = devres_bar.access(pdev.as_ref())?; let spec = Spec::new(bar)?; let fw = Firmware::new(pdev.as_ref(), spec.chipset, FIRMWARE_VERSION)?; dev_info!( pdev.as_ref(), "NVIDIA (Chipset: {}, Architecture: {:?}, Revision: {})\n", spec.chipset, spec.chipset.arch(), spec.revision ); // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. gfw::wait_gfw_boot_completion(bar) .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?; let sysmem_flush = SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?; let gsp_falcon = Falcon::<Gsp>::new( pdev.as_ref(), spec.chipset, bar, spec.chipset > Chipset::GA100, )?; gsp_falcon.clear_swgen0_intr(bar); let _sec2_falcon = Falcon::<Sec2>::new(pdev.as_ref(), spec.chipset, bar, true)?; let fb_layout = FbLayout::new(spec.chipset, bar)?; dev_dbg!(pdev.as_ref(), "{:#x?}\n", fb_layout); let bios = Vbios::new(pdev, bar)?; Self::run_fwsec_frts(pdev.as_ref(), &gsp_falcon, bar, &bios, &fb_layout)?; Ok(pin_init!(Self { spec, bar: devres_bar, fw, sysmem_flush, })) }}
// SPDX-License-Identifier: GPL-2.0
use kernel::{device, devres::Devres, error::code::*, pci, prelude::*, sync::Arc};
use crate::driver::Bar0;
use crate::falcon::{gsp::Gsp, sec2::Sec2, Falcon};
use crate::fb::FbLayout;
use crate::fb::SysmemFlush;
use crate::firmware::fwsec::{FwsecCommand, FwsecFirmware};
use crate::firmware::{Firmware, FIRMWARE_VERSION};
use crate::gfw;
use crate::regs;
use crate::util;
use crate::vbios::Vbios;
use core::fmt;
macro_rules! define_chipset {
({ $($variant:ident = $value:expr),* $(,)* }) =>
{
/// Enum representation of the GPU chipset.
#[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)]
pub(crate) enum Chipset {
$($variant = $value),*,
}
impl Chipset {
pub(crate) const ALL: &'static [Chipset] = &[
$( Chipset::$variant, )*
];
pub(crate) const NAMES: [&'static str; Self::ALL.len()] = [
$( util::const_bytes_to_str(
util::to_lowercase_bytes::<{ stringify!($variant).len() }>(
stringify!($variant)
).as_slice()
), )*
];
}
// TODO[FPRI]: replace with something like derive(FromPrimitive)
impl TryFrom<u32> for Chipset {
type Error = kernel::error::Error;
fn try_from(value: u32) -> Result<Self, Self::Error> {
match value {
$( $value => Ok(Chipset::$variant), )*
_ => Err(ENODEV),
}
}
}
}
}
define_chipset!({
// Turing
TU102 = 0x162,
TU104 = 0x164,
TU106 = 0x166,
TU117 = 0x167,
TU116 = 0x168,
// Ampere
GA100 = 0x170,
GA102 = 0x172,
GA103 = 0x173,
GA104 = 0x174,
GA106 = 0x176,
GA107 = 0x177,
// Ada
AD102 = 0x192,
AD103 = 0x193,
AD104 = 0x194,
AD106 = 0x196,
AD107 = 0x197,
});
impl Chipset {
pub(crate) fn arch(&self) -> Architecture {
match self {
Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => {
Architecture::Turing
}
Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => {
Architecture::Ampere
}
Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => {
Architecture::Ada
}
}
}
}
// TODO
//
// The resulting strings are used to generate firmware paths, hence the
// generated strings have to be stable.
//
// Hence, replace with something like strum_macros derive(Display).
//
// For now, redirect to fmt::Debug for convenience.
impl fmt::Display for Chipset {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
/// Enum representation of the GPU generation.
#[derive(fmt::Debug)]
pub(crate) enum Architecture {
Turing = 0x16,
Ampere = 0x17,
Ada = 0x19,
}
impl TryFrom<u8> for Architecture {
type Error = Error;
fn try_from(value: u8) -> Result<Self> {
match value {
0x16 => Ok(Self::Turing),
0x17 => Ok(Self::Ampere),
0x19 => Ok(Self::Ada),
_ => Err(ENODEV),
}
}
}
pub(crate) struct Revision {
major: u8,
minor: u8,
}
impl Revision {
fn from_boot0(boot0: regs::NV_PMC_BOOT_0) -> Self {
Self {
major: boot0.major_revision(),
minor: boot0.minor_revision(),
}
}
}
impl fmt::Display for Revision {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:x}.{:x}", self.major, self.minor)
}
}
/// Structure holding the metadata of the GPU.
pub(crate) struct Spec {
chipset: Chipset,
/// The revision of the chipset.
revision: Revision,
}
impl Spec {
fn new(bar: &Bar0) -> Result<Spec> {
let boot0 = regs::NV_PMC_BOOT_0::read(bar);
Ok(Self {
chipset: boot0.chipset()?,
revision: Revision::from_boot0(boot0),
})
}
}
/// Structure holding the resources required to operate the GPU.
#[pin_data(PinnedDrop)]
pub(crate) struct Gpu {
spec: Spec,
/// MMIO mapping of PCI BAR 0
bar: Arc<Devres<Bar0>>,
fw: Firmware,
/// System memory page required for flushing all pending GPU-side memory writes done through
/// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation).
sysmem_flush: SysmemFlush,
}
#[pinned_drop]
impl PinnedDrop for Gpu {
fn drop(self: Pin<&mut Self>) {
// Unregister the sysmem flush page before we release it.
self.bar
.try_access_with(|b| self.sysmem_flush.unregister(b));
}
}
impl Gpu {
/// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly
/// created the WPR2 region.
///
/// TODO: this needs to be moved into a larger type responsible for booting the whole GSP
/// (`GspBooter`?).
fn run_fwsec_frts(
dev: &device::Device<device::Bound>,
falcon: &Falcon<Gsp>,
bar: &Bar0,
bios: &Vbios,
fb_layout: &FbLayout,
) -> Result<()> {
// Check that the WPR2 region does not already exists - if it does, we cannot run
// FWSEC-FRTS until the GPU is reset.
if regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound() != 0 {
dev_err!(
dev,
"WPR2 region already exists - GPU needs to be reset to proceed\n"
);
return Err(EBUSY);
}
let fwsec_frts = FwsecFirmware::new(
dev,
falcon,
bar,
bios,
FwsecCommand::Frts {
frts_addr: fb_layout.frts.start,
frts_size: fb_layout.frts.end - fb_layout.frts.start,
},
)?;
// Run FWSEC-FRTS to create the WPR2 region.
fwsec_frts.run(dev, falcon, bar)?;
// SCRATCH_E contains the error code for FWSEC-FRTS.
let frts_status = regs::NV_PBUS_SW_SCRATCH_0E::read(bar).frts_err_code();
if frts_status != 0 {
dev_err!(
dev,
"FWSEC-FRTS returned with error code {:#x}",
frts_status
);
return Err(EIO);
}
// Check that the WPR2 region has been created as we requested.
let (wpr2_lo, wpr2_hi) = (
regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO::read(bar).lower_bound(),
regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI::read(bar).higher_bound(),
);
match (wpr2_lo, wpr2_hi) {
(_, 0) => {
dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n");
Err(EIO)
}
(wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => {
dev_err!(
dev,
"WPR2 region created at unexpected address {:#x}; expected {:#x}\n",
wpr2_lo,
fb_layout.frts.start,
);
Err(EIO)
}
(wpr2_lo, wpr2_hi) => {
dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi);
dev_dbg!(dev, "GPU instance built\n");
Ok(())
}
}
}
pub(crate) fn new(
pdev: &pci::Device<device::Bound>,
devres_bar: Arc<Devres<Bar0>>,
) -> Result<impl PinInit<Self>> {
let bar = devres_bar.access(pdev.as_ref())?;
let spec = Spec::new(bar)?;
let fw = Firmware::new(pdev.as_ref(), spec.chipset, FIRMWARE_VERSION)?;
dev_info!(
pdev.as_ref(),
"NVIDIA (Chipset: {}, Architecture: {:?}, Revision: {})\n",
spec.chipset,
spec.chipset.arch(),
spec.revision
);
// We must wait for GFW_BOOT completion before doing any significant setup on the GPU.
gfw::wait_gfw_boot_completion(bar)
.inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?;
let sysmem_flush = SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?;
let gsp_falcon = Falcon::<Gsp>::new(
pdev.as_ref(),
spec.chipset,
bar,
spec.chipset > Chipset::GA100,
)?;
gsp_falcon.clear_swgen0_intr(bar);
let _sec2_falcon = Falcon::<Sec2>::new(pdev.as_ref(), spec.chipset, bar, true)?;
let fb_layout = FbLayout::new(spec.chipset, bar)?;
dev_dbg!(pdev.as_ref(), "{:#x?}\n", fb_layout);
let bios = Vbios::new(pdev, bar)?;
Self::run_fwsec_frts(pdev.as_ref(), &gsp_falcon, bar, &bios, &fb_layout)?;
Ok(pin_init!(Self {
spec,
bar: devres_bar,
fw,
sysmem_flush,
}))
}
}