// SPDX-License-Identifier: GPL-2.0 use core::ops::{ Deref, Range, //}; use kernel::{ device, dma::CoherentHandle, fmt, io::Io, prelude::*, ptr::{ Alignable, Alignment, // }, sizes::*, //}; use crate::{ driver::Bar0, firmware::gsp::GspFirmware, gpu::Chipset, gsp, num::FromSafeCast, vgpu::VgpuState, //}; mod hal;mod regs; /// Type holding the sysmem flush memory page, a page of memory to be written into the/// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR*` registers and used to maintain memory coherency.////// A system memory page is required for `sysmembar`, which is a GPU-initiated hardware/// memory-barrier operation that flushes all pending GPU-side memory writes that were done through/// PCIE to system memory. It is required for falcons to be reset as the reset operation involves a/// reset handshake. When the falcon acknowledges a reset, it writes into system memory. To ensure/// this write is visible to the host and prevent driver timeouts, the falcon must perform a/// sysmembar operation to flush its writes.////// Because of this, the sysmem flush memory page must be registered as early as possible during/// driver initialization, and before any falcon is reset.///pub(crate) struct SysmemFlush<'sys> { /// Chipset we are operating on. chipset: Chipset, device: &'sys device::Device, bar: Bar0<'sys>, /// Keep the page alive as long as we need it. page: CoherentHandle,} impl<'sys> SysmemFlush<'sys> { /// Allocate a memory page and register it as the sysmem flush page. pub(crate) fn register( dev: &'sys device::Device<device::Bound>, bar: Bar0<'sys>, chipset: Chipset, ) -> Result<Self> { let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?; hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_address())?; Ok(Self { chipset, device: dev, bar, page, }) }} impl Drop for SysmemFlush<'_> { fn drop(&mut self) { let hal = hal::fb_hal(self.chipset); if hal.read_sysmem_flush_page(self.bar) == self.page.dma_address() { let _ = hal.write_sysmem_flush_page(self.bar, 0).inspect_err(|e| { dev_warn!( &self.device, "failed to unregister sysmem flush page: {:?}\n", e ) }); } else { // Another page has been registered after us for some reason - warn as this is a bug. dev_warn!( &self.device, "attempt to unregister a sysmem flush page that is not active\n" ); } }} pub(crate) struct FbRange(Range<u64>); impl FbRange { pub(crate) fn len(&self) -> u64 { self.0.end - self.0.start }} impl From<Range<u64>> for FbRange { fn from(range: Range<u64>) -> Self { Self(range) }} impl Deref for FbRange { type Target = Range<u64>; fn deref(&self) -> &Self::Target { &self.0 }} impl fmt::Debug for FbRange { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Use alternate format ({:#?}) to include size, compact format ({:?}) for just the range. if f.alternate() { let size = self.len(); if size < u64::SZ_1M { let size_kib = size / u64::SZ_1K; f.write_fmt(fmt!( "{:#x}..{:#x} ({} KiB)", self.0.start, self.0.end, size_kib )) } else { let size_mib = size / u64::SZ_1M; f.write_fmt(fmt!( "{:#x}..{:#x} ({} MiB)", self.0.start, self.0.end, size_mib )) } } else { f.write_fmt(fmt!("{:#x}..{:#x}", self.0.start, self.0.end)) } }} /// Layout of the GPU framebuffer memory.////// Contains ranges of GPU memory reserved for a given purpose during the GSP boot process.#[derive(Debug)]pub(crate) struct FbRanges { /// Range of the framebuffer. Starts at `0`. pub(crate) fb: FbRange, /// VGA workspace, small area of reserved memory at the end of the framebuffer. pub(crate) vga_workspace: FbRange, /// FRTS range. pub(crate) frts: FbRange, /// Memory area containing the GSP bootloader image. pub(crate) boot: FbRange, /// Memory area containing the GSP firmware image. pub(crate) elf: FbRange, /// WPR2 heap. pub(crate) wpr2_heap: FbRange, /// WPR2 region range, starting with an instance of `GspFwWprMeta`. pub(crate) wpr2: FbRange, /// Non-WPR heap, located just below WPR2. pub(crate) non_wpr_heap: FbRange, /// Number of VF partitions. pub(crate) vf_partition_count: u8, /// PMU reserved memory size, in bytes. pub(crate) pmu_reserved_size: u32,} impl FbRanges { /// Computes concrete framebuffer ranges required on non-FSP booting architectures. pub(crate) fn new( chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware, vgpu_state: VgpuState, ) -> Result<Self> { let hal = hal::fb_hal(chipset); let fb = { let fb_size = hal.vidmem_size(bar); FbRange(0..fb_size) }; let vga_workspace = { let vga_base = { const NV_PRAMIN_SIZE: u64 = u64::SZ_1M; let base = fb.end - NV_PRAMIN_SIZE; if hal.supports_display(bar) { match bar .read(regs::NV_PDISP_VGA_WORKSPACE_BASE) .vga_workspace_addr() { Some(addr) => { if addr < base { const VBIOS_WORKSPACE_SIZE: u64 = u64::SZ_128K; // Point workspace address to end of framebuffer. fb.end - VBIOS_WORKSPACE_SIZE } else { addr } } None => base, } } else { base } }; FbRange(vga_base..fb.end) }; let frts = { const FRTS_DOWN_ALIGN: Alignment = Alignment::new::<SZ_128K>(); let frts_size: u64 = hal.frts_size(); let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - frts_size; FbRange(frts_base..frts_base + frts_size) }; let boot = { const BOOTLOADER_DOWN_ALIGN: Alignment = Alignment::new::<SZ_4K>(); let bootloader_size = u64::from_safe_cast(gsp_fw.bootloader.ucode.size()); let bootloader_base = (frts.start - bootloader_size).align_down(BOOTLOADER_DOWN_ALIGN); FbRange(bootloader_base..bootloader_base + bootloader_size) }; let elf = { const ELF_DOWN_ALIGN: Alignment = Alignment::new::<SZ_64K>(); let elf_size = u64::from_safe_cast(gsp_fw.size); let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN); FbRange(elf_addr..elf_addr + elf_size) }; let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?; let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>(); let wpr2_heap_addr = elf .start .checked_sub(wpr2_heap_size) .ok_or(EOVERFLOW)? .align_down(WPR2_HEAP_DOWN_ALIGN); FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)) }; let wpr2 = { const WPR2_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>(); let wpr2_addr = (wpr2_heap.start - u64::from_safe_cast(size_of::<gsp::GspFwWprMeta>())) .align_down(WPR2_DOWN_ALIGN); FbRange(wpr2_addr..frts.end) }; let non_wpr_heap = { let non_wpr_heap_size = hal.non_wpr_heap_size(); FbRange(wpr2.start - non_wpr_heap_size..wpr2.start) }; Ok(Self { fb, vga_workspace, frts, boot, elf, wpr2_heap, wpr2, non_wpr_heap, vf_partition_count, pmu_reserved_size: hal.pmu_reserved_size(), }) }} /// Reads the WPR2 memory region registers and returns the range if set./// Returns `None` if the WPR2 region is not set.pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option<Range<u64>> { let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); if !wpr2_hi.is_wpr2_set() { return None; } let wpr2_lo = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO); Some(wpr2_lo.lower_bound()..wpr2_hi.higher_bound())} /// Computes the number of VF partitions and the WPR2 heap size from the vGPU state.fn wpr2_heap_params(chipset: Chipset, vgpu_state: VgpuState, fb_size: u64) -> Result<(u8, u64)> { Ok(match vgpu_state { VgpuState::Disabled => ( 0, gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb_size)?, ), VgpuState::Enabled { total_vfs } => ( u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?, gsp::LibosParams::vgpu_wpr_heap_size(), ), })} /// Framebuffer region sizes needed for GSP-FMC boot.#[derive(Debug)]pub(crate) struct FbSizes { /// FRTS size, in bytes. pub(crate) frts_size: u64, /// WPR2 heap size, in bytes. pub(crate) wpr2_heap_size: u64, /// Non-WPR heap size, in bytes. pub(crate) non_wpr_heap_size: u64, /// PMU reserved memory size, in bytes. pub(crate) pmu_reserved_size: u32, /// Number of VF partitions. pub(crate) vf_partition_count: u8,} impl FbSizes { /// Computes the framebuffer region sizes for GSP-FMC boot. pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, vgpu_state: VgpuState) -> Result<Self> { let hal = hal::fb_hal(chipset); let fb_size = hal.vidmem_size(bar); let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb_size)?; Ok(Self { frts_size: hal.frts_size(), wpr2_heap_size, non_wpr_heap_size: hal.non_wpr_heap_size(), pmu_reserved_size: hal.pmu_reserved_size(), vf_partition_count, }) }}
// SPDX-License-Identifier: GPL-2.0
use core::ops::{
Deref,
Range, //
};
use kernel::{
device,
dma::CoherentHandle,
fmt,
io::Io,
prelude::*,
ptr::{
Alignable,
Alignment, //
},
sizes::*, //
};
use crate::{
driver::Bar0,
firmware::gsp::GspFirmware,
gpu::Chipset,
gsp,
num::FromSafeCast,
vgpu::VgpuState, //
};
mod hal;
mod regs;
/// Type holding the sysmem flush memory page, a page of memory to be written into the
/// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR*` registers and used to maintain memory coherency.
///
/// A system memory page is required for `sysmembar`, which is a GPU-initiated hardware
/// memory-barrier operation that flushes all pending GPU-side memory writes that were done through
/// PCIE to system memory. It is required for falcons to be reset as the reset operation involves a
/// reset handshake. When the falcon acknowledges a reset, it writes into system memory. To ensure
/// this write is visible to the host and prevent driver timeouts, the falcon must perform a
/// sysmembar operation to flush its writes.
///
/// Because of this, the sysmem flush memory page must be registered as early as possible during
/// driver initialization, and before any falcon is reset.
///
pub(crate) struct SysmemFlush<'sys> {
/// Chipset we are operating on.
chipset: Chipset,
device: &'sys device::Device,
bar: Bar0<'sys>,
/// Keep the page alive as long as we need it.
page: CoherentHandle,
}
impl<'sys> SysmemFlush<'sys> {
/// Allocate a memory page and register it as the sysmem flush page.
pub(crate) fn register(
dev: &'sys device::Device<device::Bound>,
bar: Bar0<'sys>,
chipset: Chipset,
) -> Result<Self> {
let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?;
hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_address())?;
Ok(Self {
chipset,
device: dev,
bar,
page,
})
}
}
impl Drop for SysmemFlush<'_> {
fn drop(&mut self) {
let hal = hal::fb_hal(self.chipset);
if hal.read_sysmem_flush_page(self.bar) == self.page.dma_address() {
let _ = hal.write_sysmem_flush_page(self.bar, 0).inspect_err(|e| {
dev_warn!(
&self.device,
"failed to unregister sysmem flush page: {:?}\n",
e
)
});
} else {
// Another page has been registered after us for some reason - warn as this is a bug.
dev_warn!(
&self.device,
"attempt to unregister a sysmem flush page that is not active\n"
);
}
}
}
pub(crate) struct FbRange(Range<u64>);
impl FbRange {
pub(crate) fn len(&self) -> u64 {
self.0.end - self.0.start
}
}
impl From<Range<u64>> for FbRange {
fn from(range: Range<u64>) -> Self {
Self(range)
}
}
impl Deref for FbRange {
type Target = Range<u64>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl fmt::Debug for FbRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Use alternate format ({:#?}) to include size, compact format ({:?}) for just the range.
if f.alternate() {
let size = self.len();
if size < u64::SZ_1M {
let size_kib = size / u64::SZ_1K;
f.write_fmt(fmt!(
"{:#x}..{:#x} ({} KiB)",
self.0.start,
self.0.end,
size_kib
))
} else {
let size_mib = size / u64::SZ_1M;
f.write_fmt(fmt!(
"{:#x}..{:#x} ({} MiB)",
self.0.start,
self.0.end,
size_mib
))
}
} else {
f.write_fmt(fmt!("{:#x}..{:#x}", self.0.start, self.0.end))
}
}
}
/// Layout of the GPU framebuffer memory.
///
/// Contains ranges of GPU memory reserved for a given purpose during the GSP boot process.
#[derive(Debug)]
pub(crate) struct FbRanges {
/// Range of the framebuffer. Starts at `0`.
pub(crate) fb: FbRange,
/// VGA workspace, small area of reserved memory at the end of the framebuffer.
pub(crate) vga_workspace: FbRange,
/// FRTS range.
pub(crate) frts: FbRange,
/// Memory area containing the GSP bootloader image.
pub(crate) boot: FbRange,
/// Memory area containing the GSP firmware image.
pub(crate) elf: FbRange,
/// WPR2 heap.
pub(crate) wpr2_heap: FbRange,
/// WPR2 region range, starting with an instance of `GspFwWprMeta`.
pub(crate) wpr2: FbRange,
/// Non-WPR heap, located just below WPR2.
pub(crate) non_wpr_heap: FbRange,
/// Number of VF partitions.
pub(crate) vf_partition_count: u8,
/// PMU reserved memory size, in bytes.
pub(crate) pmu_reserved_size: u32,
}
impl FbRanges {
/// Computes concrete framebuffer ranges required on non-FSP booting architectures.
pub(crate) fn new(
chipset: Chipset,
bar: Bar0<'_>,
gsp_fw: &GspFirmware,
vgpu_state: VgpuState,
) -> Result<Self> {
let hal = hal::fb_hal(chipset);
let fb = {
let fb_size = hal.vidmem_size(bar);
FbRange(0..fb_size)
};
let vga_workspace = {
let vga_base = {
const NV_PRAMIN_SIZE: u64 = u64::SZ_1M;
let base = fb.end - NV_PRAMIN_SIZE;
if hal.supports_display(bar) {
match bar
.read(regs::NV_PDISP_VGA_WORKSPACE_BASE)
.vga_workspace_addr()
{
Some(addr) => {
if addr < base {
const VBIOS_WORKSPACE_SIZE: u64 = u64::SZ_128K;
// Point workspace address to end of framebuffer.
fb.end - VBIOS_WORKSPACE_SIZE
} else {
addr
}
}
None => base,
}
} else {
base
}
};
FbRange(vga_base..fb.end)
};
let frts = {
const FRTS_DOWN_ALIGN: Alignment = Alignment::new::<SZ_128K>();
let frts_size: u64 = hal.frts_size();
let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - frts_size;
FbRange(frts_base..frts_base + frts_size)
};
let boot = {
const BOOTLOADER_DOWN_ALIGN: Alignment = Alignment::new::<SZ_4K>();
let bootloader_size = u64::from_safe_cast(gsp_fw.bootloader.ucode.size());
let bootloader_base = (frts.start - bootloader_size).align_down(BOOTLOADER_DOWN_ALIGN);
FbRange(bootloader_base..bootloader_base + bootloader_size)
};
let elf = {
const ELF_DOWN_ALIGN: Alignment = Alignment::new::<SZ_64K>();
let elf_size = u64::from_safe_cast(gsp_fw.size);
let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN);
FbRange(elf_addr..elf_addr + elf_size)
};
let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?;
let wpr2_heap = {
const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>();
let wpr2_heap_addr = elf
.start
.checked_sub(wpr2_heap_size)
.ok_or(EOVERFLOW)?
.align_down(WPR2_HEAP_DOWN_ALIGN);
FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN))
};
let wpr2 = {
const WPR2_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>();
let wpr2_addr = (wpr2_heap.start - u64::from_safe_cast(size_of::<gsp::GspFwWprMeta>()))
.align_down(WPR2_DOWN_ALIGN);
FbRange(wpr2_addr..frts.end)
};
let non_wpr_heap = {
let non_wpr_heap_size = hal.non_wpr_heap_size();
FbRange(wpr2.start - non_wpr_heap_size..wpr2.start)
};
Ok(Self {
fb,
vga_workspace,
frts,
boot,
elf,
wpr2_heap,
wpr2,
non_wpr_heap,
vf_partition_count,
pmu_reserved_size: hal.pmu_reserved_size(),
})
}
}
/// Reads the WPR2 memory region registers and returns the range if set.
/// Returns `None` if the WPR2 region is not set.
pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option<Range<u64>> {
let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI);
if !wpr2_hi.is_wpr2_set() {
return None;
}
let wpr2_lo = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO);
Some(wpr2_lo.lower_bound()..wpr2_hi.higher_bound())
}
/// Computes the number of VF partitions and the WPR2 heap size from the vGPU state.
fn wpr2_heap_params(chipset: Chipset, vgpu_state: VgpuState, fb_size: u64) -> Result<(u8, u64)> {
Ok(match vgpu_state {
VgpuState::Disabled => (
0,
gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb_size)?,
),
VgpuState::Enabled { total_vfs } => (
u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?,
gsp::LibosParams::vgpu_wpr_heap_size(),
),
})
}
/// Framebuffer region sizes needed for GSP-FMC boot.
#[derive(Debug)]
pub(crate) struct FbSizes {
/// FRTS size, in bytes.
pub(crate) frts_size: u64,
/// WPR2 heap size, in bytes.
pub(crate) wpr2_heap_size: u64,
/// Non-WPR heap size, in bytes.
pub(crate) non_wpr_heap_size: u64,
/// PMU reserved memory size, in bytes.
pub(crate) pmu_reserved_size: u32,
/// Number of VF partitions.
pub(crate) vf_partition_count: u8,
}
impl FbSizes {
/// Computes the framebuffer region sizes for GSP-FMC boot.
pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, vgpu_state: VgpuState) -> Result<Self> {
let hal = hal::fb_hal(chipset);
let fb_size = hal.vidmem_size(bar);
let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb_size)?;
Ok(Self {
frts_size: hal.frts_size(),
wpr2_heap_size,
non_wpr_heap_size: hal.non_wpr_heap_size(),
pmu_reserved_size: hal.pmu_reserved_size(),
vf_partition_count,
})
}
}