// SPDX-License-Identifier: GPL-2.0 //! Falcon microprocessor base support use hal::FalconHal; use kernel::{    device,    dma::{        Coherent,        CoherentBox,        DmaAddress, //    },    io::{        io_project,        poll::read_poll_timeout,        register::{            RegisterBase,            WithBase, //        },        Io,    },    prelude::*,    time::Delta,}; use crate::{    bounded_enum,    driver::Bar0,    falcon::hal::LoadMethod,    gpu::Chipset,    num::{        self,        FromSafeCast, //    },    regs,}; pub(crate) mod fsp;pub(crate) mod gsp;mod hal;pub(crate) mod sec2; /// Alignment (in bytes) of falcon memory blocks.pub(crate) const MEM_BLOCK_ALIGNMENT: usize = 256; bounded_enum! {    /// Revision number of a falcon core, used in the [`crate::regs::NV_PFALCON_FALCON_HWCFG1`]    /// register.    #[derive(Debug, Copy, Clone)]    pub(crate) enum FalconCoreRev with TryFrom<Bounded<u32, 4>> {        Rev1 = 1,        Rev2 = 2,        Rev3 = 3,        Rev4 = 4,        Rev5 = 5,        Rev6 = 6,        Rev7 = 7,    }} bounded_enum! {    /// Revision subversion number of a falcon core, used in the    /// [`crate::regs::NV_PFALCON_FALCON_HWCFG1`] register.    #[derive(Debug, Copy, Clone)]    pub(crate) enum FalconCoreRevSubversion with From<Bounded<u32, 2>> {        Subversion0 = 0,        Subversion1 = 1,        Subversion2 = 2,        Subversion3 = 3,    }} bounded_enum! {    /// Security mode of the Falcon microprocessor.    ///    /// See `falcon.rst` for more details.    #[derive(Debug, Copy, Clone)]    pub(crate) enum FalconSecurityModel with TryFrom<Bounded<u32, 2>> {        /// Non-Secure: runs unsigned code without privileges.        None = 0,        /// Light-Secured (LS): Runs signed code with some privileges.        /// Entry into this mode is only possible from 'Heavy-secure' mode, which verifies the        /// code's signature.        ///        /// Also known as Low-Secure, Privilege Level 2 or PL2.        Light = 2,        /// Heavy-Secured (HS): Runs signed code with full privileges.        /// The code's signature is verified by the Falcon Boot ROM (BROM).        ///        /// Also known as High-Secure, Privilege Level 3 or PL3.        Heavy = 3,    }} bounded_enum! {    /// Signing algorithm for a given firmware, used in the    /// [`crate::regs::NV_PFALCON2_FALCON_MOD_SEL`] register. It is passed to the Falcon Boot ROM    /// (BROM) as a parameter.    #[derive(Debug, Copy, Clone)]    pub(crate) enum FalconModSelAlgo with TryFrom<Bounded<u32, 8>> {        /// AES.        Aes = 0,        /// RSA3K.        Rsa3k = 1,    }} bounded_enum! {    /// Valid values for the `size` field of the [`crate::regs::NV_PFALCON_FALCON_DMATRFCMD`]    /// register.    #[derive(Debug, Copy, Clone)]    pub(crate) enum DmaTrfCmdSize with TryFrom<Bounded<u32, 3>> {        /// 256 bytes transfer.        Size256B = 0x6,    }} bounded_enum! {    /// Currently active core on a dual falcon/riscv (Peregrine) controller.    #[derive(Debug, Copy, Clone, PartialEq, Eq)]    pub(crate) enum PeregrineCoreSelect with From<Bounded<u32, 1>> {        /// Falcon core is active.        Falcon = 0,        /// RISC-V core is active.        Riscv = 1,    }} /// Different types of memory present in a falcon core.#[derive(Debug, Copy, Clone, PartialEq, Eq)]pub(crate) enum FalconMem {    /// Secure Instruction Memory.    ImemSecure,    /// Non-Secure Instruction Memory.    #[expect(unused)]    ImemNonSecure,    /// Data Memory.    Dmem,} bounded_enum! {    /// Defines the Framebuffer Interface (FBIF) aperture type.    /// This determines the memory type for external memory access during a DMA transfer, which is    /// performed by the Falcon's Framebuffer DMA (FBDMA) engine. See falcon.rst for more details.    #[derive(Debug, Copy, Clone)]    pub(crate) enum FalconFbifTarget with TryFrom<Bounded<u32, 2>> {        /// Local Framebuffer (GPU's VRAM memory).        LocalFb = 0,        /// Coherent system memory (System DRAM).        CoherentSysmem = 1,        /// Non-coherent system memory (System DRAM).        NoncoherentSysmem = 2,    }} bounded_enum! {    /// Type of memory addresses to use.    #[derive(Debug, Copy, Clone)]    pub(crate) enum FalconFbifMemType with From<Bounded<u32, 1>> {        /// Virtual memory addresses.        Virtual = 0,        /// Physical memory addresses.        Physical = 1,    }} /// Type used to represent the `PFALCON` registers address base for a given falcon engine.pub(crate) struct PFalconBase(()); /// Type used to represent the `PFALCON2` registers address base for a given falcon engine.pub(crate) struct PFalcon2Base(()); /// Trait defining the parameters of a given Falcon engine.////// Each engine provides one base for `PFALCON` and `PFALCON2` registers.pub(crate) trait FalconEngine:    Send + Sync + RegisterBase<PFalconBase> + RegisterBase<PFalcon2Base> + Sized{} /// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM)/// using DMA.#[derive(Debug, Clone)]pub(crate) struct FalconDmaLoadTarget {    /// Offset from the start of the source object to copy from.    pub(crate) src_start: u32,    /// Offset from the start of the destination memory to copy into.    pub(crate) dst_start: u32,    /// Number of bytes to copy.    pub(crate) len: u32,} /// Parameters for the falcon boot ROM.#[derive(Debug, Clone)]pub(crate) struct FalconBromParams {    /// Offset in `DMEM`` of the firmware's signature.    pub(crate) pkc_data_offset: u32,    /// Mask of engines valid for this firmware.    pub(crate) engine_id_mask: u16,    /// ID of the ucode used to infer a fuse register to validate the signature.    pub(crate) ucode_id: u8,} /// Trait implemented by falcon firmwares that can be loaded using DMA.pub(crate) trait FalconDmaLoadable {    /// Returns the firmware data as a slice of bytes.    fn as_slice(&self) -> &[u8];     /// Returns the load parameters for Secure `IMEM`.    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget;     /// Returns the load parameters for Non-Secure `IMEM`,    /// used only on Turing and GA100.    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget>;     /// Returns the load parameters for `DMEM`.    fn dmem_load_params(&self) -> FalconDmaLoadTarget;     /// Returns an adapter that provides the required parameter to load this firmware using PIO.    ///    /// This can only fail if some `u32` fields cannot be converted to `u16`, or if the indices in    /// the headers are invalid.    fn try_as_pio_loadable(&self) -> Result<FalconDmaFirmwarePioAdapter<'_, Self>> {        let new_pio_imem = |params: FalconDmaLoadTarget, secure| {            let start = usize::from_safe_cast(params.src_start);            let end = start + usize::from_safe_cast(params.len);            let data = self.as_slice().get(start..end).ok_or(EINVAL)?;             let dst_start = u16::try_from(params.dst_start).map_err(|_| EINVAL)?;             Ok::<_, Error>(FalconPioImemLoadTarget {                data,                dst_start,                secure,                start_tag: dst_start >> 8,            })        };         let imem_sec = new_pio_imem(self.imem_sec_load_params(), true)?;         let imem_ns = if let Some(params) = self.imem_ns_load_params() {            Some(new_pio_imem(params, false)?)        } else {            None        };         let dmem = {            let params = self.dmem_load_params();            let start = usize::from_safe_cast(params.src_start);            let end = start + usize::from_safe_cast(params.len);            let data = self.as_slice().get(start..end).ok_or(EINVAL)?;             let dst_start = u16::try_from(params.dst_start).map_err(|_| EINVAL)?;             FalconPioDmemLoadTarget { data, dst_start }        };         Ok(FalconDmaFirmwarePioAdapter {            fw: self,            imem_sec,            imem_ns,            dmem,        })    }} /// Represents a portion of the firmware to be loaded into IMEM using PIO.#[derive(Clone)]pub(crate) struct FalconPioImemLoadTarget<'a> {    pub(crate) data: &'a [u8],    pub(crate) dst_start: u16,    pub(crate) secure: bool,    pub(crate) start_tag: u16,} /// Represents a portion of the firmware to be loaded into DMEM using PIO.#[derive(Clone)]pub(crate) struct FalconPioDmemLoadTarget<'a> {    pub(crate) data: &'a [u8],    pub(crate) dst_start: u16,} /// Trait for providing PIO load parameters of falcon firmwares.pub(crate) trait FalconPioLoadable {    /// Returns the load parameters for Secure `IMEM`, if any.    fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>>;     /// Returns the load parameters for Non-Secure `IMEM`, if any.    fn imem_ns_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>>;     /// Returns the load parameters for `DMEM`.    fn dmem_load_params(&self) -> FalconPioDmemLoadTarget<'_>;} /// Adapter type that makes any DMA-loadable firmware also loadable via PIO.////// Created using [`FalconDmaLoadable::try_as_pio_loadable`].pub(crate) struct FalconDmaFirmwarePioAdapter<'a, T: FalconDmaLoadable + ?Sized> {    /// Reference to the DMA firmware.    fw: &'a T,    /// Validated secure IMEM parameters.    imem_sec: FalconPioImemLoadTarget<'a>,    /// Validated non-secure IMEM parameters.    imem_ns: Option<FalconPioImemLoadTarget<'a>>,    /// Validated DMEM parameters.    dmem: FalconPioDmemLoadTarget<'a>,} impl<'a, T> FalconPioLoadable for FalconDmaFirmwarePioAdapter<'a, T>where    T: FalconDmaLoadable + ?Sized,{    fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {        Some(self.imem_sec.clone())    }     fn imem_ns_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {        self.imem_ns.clone()    }     fn dmem_load_params(&self) -> FalconPioDmemLoadTarget<'_> {        self.dmem.clone()    }} impl<'a, T> FalconFirmware for FalconDmaFirmwarePioAdapter<'a, T>where    T: FalconDmaLoadable + FalconFirmware + ?Sized,{    type Target = <T as FalconFirmware>::Target;     fn brom_params(&self) -> FalconBromParams {        self.fw.brom_params()    }     fn boot_addr(&self) -> u32 {        self.fw.boot_addr()    }} /// Trait for a falcon firmware.////// A falcon firmware can be loaded on a given engine.pub(crate) trait FalconFirmware {    /// Engine on which this firmware is to be loaded.    type Target: FalconEngine;     /// Returns the parameters to write into the BROM registers.    fn brom_params(&self) -> FalconBromParams;     /// Returns the start address of the firmware.    fn boot_addr(&self) -> u32;} /// Contains the base parameters common to all Falcon instances.pub(crate) struct Falcon<'a, E: FalconEngine> {    hal: KBox<dyn FalconHal<E>>,    dev: &'a device::Device<device::Bound>,    bar: Bar0<'a>,} impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {    /// Create a new falcon instance.    pub(crate) fn new(        dev: &'a device::Device<device::Bound>,        chipset: Chipset,        bar: Bar0<'a>,    ) -> Result<Self> {        Ok(Self {            hal: hal::falcon_hal(chipset)?,            dev,            bar,        })    }     /// Resets DMA-related registers.    pub(crate) fn dma_reset(&self) {        self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {            v.with_allow_phys_no_ctx(true)        });         self.bar.write(            WithBase::of::<E>(),            regs::NV_PFALCON_FALCON_DMACTL::zeroed(),        );    }     /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete.    pub(crate) fn reset(&self) -> Result {        self.hal.reset_eng(self)?;        self.hal.select_core(self)?;        self.hal.reset_wait_mem_scrubbing(self)?;         self.bar.write(            WithBase::of::<E>(),            regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()),        );         Ok(())    }     /// Falcons supports up to four ports, but we only ever use one, so just hard-code it.    const PIO_PORT: usize = 0;     /// Write a slice to Falcon IMEM memory using programmed I/O (PIO).    ///    /// Returns `EINVAL` if `img.len()` is not a multiple of 4.    fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result {        // Rejecting misaligned images here allows us to avoid checking        // inside the loops.        if load_offsets.data.len() % 4 != 0 {            return Err(EINVAL);        }         self.bar.write(            WithBase::of::<E>().at(Self::PIO_PORT),            regs::NV_PFALCON_FALCON_IMEMC::zeroed()                .with_secure(load_offsets.secure)                .with_aincw(true)                .with_offs(load_offsets.dst_start),        );         for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() {            let n = u16::try_from(n)?;            let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?;            self.bar.write(                WithBase::of::<E>().at(Self::PIO_PORT),                regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag),            );            for word in block.chunks_exact(4) {                let w = [word[0], word[1], word[2], word[3]];                self.bar.write(                    WithBase::of::<E>().at(Self::PIO_PORT),                    regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)),                );            }        }         Ok(())    }     /// Write a slice to Falcon DMEM memory using programmed I/O (PIO).    ///    /// Returns `EINVAL` if `img.len()` is not a multiple of 4.    fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result {        // Rejecting misaligned images here allows us to avoid checking        // inside the loops.        if load_offsets.data.len() % 4 != 0 {            return Err(EINVAL);        }         self.bar.write(            WithBase::of::<E>().at(Self::PIO_PORT),            regs::NV_PFALCON_FALCON_DMEMC::zeroed()                .with_aincw(true)                .with_offs(load_offsets.dst_start),        );         for word in load_offsets.data.chunks_exact(4) {            let w = [word[0], word[1], word[2], word[3]];            self.bar.write(                WithBase::of::<E>().at(Self::PIO_PORT),                regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)),            );        }         Ok(())    }     /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it.    pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(        &self,        fw: &F,    ) -> Result {        self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {            v.with_allow_phys_no_ctx(true)        });         self.bar.write(            WithBase::of::<E>(),            regs::NV_PFALCON_FALCON_DMACTL::zeroed(),        );         if let Some(imem_ns) = fw.imem_ns_load_params() {            self.pio_wr_imem_slice(imem_ns)?;        }        if let Some(imem_sec) = fw.imem_sec_load_params() {            self.pio_wr_imem_slice(imem_sec)?;        }        self.pio_wr_dmem_slice(fw.dmem_load_params())?;         self.hal.program_brom(self, &fw.brom_params());         self.bar.write(            WithBase::of::<E>(),            regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),        );         Ok(())

Lines 1–500 of 768. Load more