#![allow(clippy::items_after_test_module)]
use common::{LazyPagesExecutionContext, LazyPagesRuntimeContext};
use gear_backend_common::lazy_pages::{GlobalsAccessConfig, Status};
use pages::{PageNumber, WasmPageNumber};
use sp_std::vec::Vec;
use std::{cell::RefCell, convert::TryInto, num::NonZeroU32};
mod common;
mod globals;
mod host_func;
mod init_flag;
mod mprotect;
mod pages;
mod process;
mod signal;
mod sys;
mod utils;
use crate::{
common::{
ContextError, GlobalNames, LazyPagesContext, PagePrefix, PageSizes, WeightNo, Weights,
},
globals::{GlobalNo, GlobalsContext},
init_flag::InitializationFlag,
pages::{PageDynSize, PageSizeNo},
};
#[cfg(test)]
mod tests;
pub use common::LazyPagesVersion;
pub use host_func::pre_process_memory_accesses;
use mprotect::MprotectError;
use signal::{DefaultUserSignalHandler, UserSignalHandler};
static LAZY_PAGES_INITIALIZED: InitializationFlag = InitializationFlag::new();
thread_local! {
static LAZY_PAGES_CONTEXT: RefCell<LazyPagesContext> = RefCell::new(Default::default());
}
#[derive(Debug, derive_more::Display, derive_more::From)]
pub enum Error {
#[display(fmt = "WASM memory native address {_0:#x} is not aligned to the native page size")]
WasmMemAddrIsNotAligned(usize),
#[display(fmt = "{_0}")]
#[from]
Mprotect(MprotectError),
#[display(fmt = "Wasm memory end addr is out of usize: begin addr = {_0:#x}, size = {_1:#x}")]
WasmMemoryEndAddrOverflow(usize, u32),
#[display(fmt = "Prefix of storage with memory pages was not set")]
MemoryPagesPrefixNotSet,
#[display(fmt = "Memory size must be null when memory host addr is not set")]
MemorySizeIsNotNull,
#[display(fmt = "Wasm mem size is too big")]
WasmMemSizeOverflow,
#[display(fmt = "Stack end offset cannot be bigger than memory size")]
StackEndBiggerThanMemSize,
#[display(fmt = "Stack end offset is too big")]
StackEndOverflow,
#[display(fmt = "Wasm addr and size are not changed, so host func call is needless")]
NothingToChange,
#[display(fmt = "Wasm memory addr must be set, when trying to change something in lazy pages")]
WasmMemAddrIsNotSet,
#[display(fmt = "{_0}")]
#[from]
GlobalContext(ContextError),
#[display(fmt = "Wrong weights amount: get {_0}, must be {_1}")]
WrongWeightsAmount(usize, usize),
}
fn check_memory_interval(addr: usize, size: u32) -> Result<(), Error> {
addr.checked_add(size as usize)
.ok_or(Error::WasmMemoryEndAddrOverflow(addr, size))
.map(|_| ())
}
pub fn initialize_for_program(
wasm_mem_addr: Option<usize>,
wasm_mem_size: u32,
stack_end: Option<u32>,
program_id: Vec<u8>,
globals_config: Option<GlobalsAccessConfig>,
weights: Vec<u64>,
) -> Result<(), Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
let mut ctx = ctx.borrow_mut();
let runtime_ctx = ctx.runtime_context_mut()?;
if let Some(addr) = wasm_mem_addr {
if addr % region::page::size() != 0 {
return Err(Error::WasmMemAddrIsNotAligned(addr));
}
}
let stack_end = stack_end.unwrap_or_default();
if wasm_mem_size < stack_end {
return Err(Error::StackEndBiggerThanMemSize);
}
let wasm_mem_size =
WasmPageNumber::new(wasm_mem_size, runtime_ctx).ok_or(Error::WasmMemSizeOverflow)?;
let wasm_mem_size_in_bytes = wasm_mem_size.offset(runtime_ctx);
if let Some(addr) = wasm_mem_addr {
check_memory_interval(addr, wasm_mem_size_in_bytes)?;
} else if wasm_mem_size_in_bytes != 0 {
return Err(Error::MemorySizeIsNotNull);
}
let stack_end =
WasmPageNumber::new(stack_end, runtime_ctx).ok_or(Error::StackEndOverflow)?;
let weights: Weights = weights.try_into().map_err(|ws: Vec<u64>| {
Error::WrongWeightsAmount(ws.len(), WeightNo::Amount as usize)
})?;
let execution_ctx = LazyPagesExecutionContext {
page_sizes: runtime_ctx.page_sizes,
weights,
wasm_mem_addr,
wasm_mem_size,
program_storage_prefix: PagePrefix::new_from_program_prefix(
runtime_ctx
.pages_storage_prefix
.iter()
.chain(program_id.iter())
.copied()
.collect(),
),
accessed_pages: Default::default(),
write_accessed_pages: Default::default(),
stack_end,
globals_context: globals_config.map(|cfg| GlobalsContext {
names: runtime_ctx.global_names.clone(),
access_ptr: cfg.access_ptr,
access_mod: cfg.access_mod,
}),
status: Status::Normal,
};
if let Some(addr) = wasm_mem_addr {
let stack_end_offset = execution_ctx.stack_end.offset(&execution_ctx);
log::trace!("{addr:#x} {stack_end_offset:#x}");
let addr = addr + stack_end_offset as usize;
let size = wasm_mem_size_in_bytes - stack_end_offset;
if size != 0 {
mprotect::mprotect_interval(addr, size as usize, false, false)?;
}
}
ctx.set_execution_context(execution_ctx);
log::trace!("Initialize lazy-pages for current program: {:?}", ctx);
Ok(())
})
}
pub fn set_lazy_pages_protection() -> Result<(), Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
let ctx = ctx.borrow();
let ctx = ctx.execution_context()?;
let mem_addr = ctx.wasm_mem_addr.ok_or(Error::WasmMemAddrIsNotSet)?;
let start_offset = ctx.stack_end.offset(ctx);
let mem_size = ctx.wasm_mem_size.offset(ctx);
mprotect::mprotect_mem_interval_except_pages(
mem_addr,
start_offset as usize,
mem_size as usize,
ctx.write_accessed_pages.iter().copied(),
ctx,
false,
false,
)?;
let read_only_pages = ctx
.accessed_pages
.iter()
.filter(|&&page| !ctx.write_accessed_pages.contains(&page))
.copied();
mprotect::mprotect_pages(mem_addr, read_only_pages, ctx, true, false)?;
Ok(())
})
}
pub fn unset_lazy_pages_protection() -> Result<(), Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
let ctx = ctx.borrow();
let ctx = ctx.execution_context()?;
let addr = ctx.wasm_mem_addr.ok_or(Error::WasmMemAddrIsNotSet)?;
let size = ctx.wasm_mem_size.offset(ctx);
mprotect::mprotect_interval(addr, size as usize, true, true)?;
Ok(())
})
}
pub fn change_wasm_mem_addr_and_size(addr: Option<usize>, size: Option<u32>) -> Result<(), Error> {
if matches!((addr, size), (None, None)) {
return Err(Error::NothingToChange);
}
LAZY_PAGES_CONTEXT.with(|ctx| {
let mut ctx = ctx.borrow_mut();
let ctx = ctx.execution_context_mut()?;
let addr = match addr {
Some(addr) => match addr % region::page::size() {
0 => addr,
_ => return Err(Error::WasmMemAddrIsNotAligned(addr)),
},
None => match ctx.wasm_mem_addr {
Some(addr) => addr,
None => return Err(Error::WasmMemAddrIsNotSet),
},
};
let size = match size {
Some(size) => WasmPageNumber::new(size, ctx).ok_or(Error::WasmMemSizeOverflow)?,
None => ctx.wasm_mem_size,
};
check_memory_interval(addr, size.offset(ctx))?;
ctx.wasm_mem_addr = Some(addr);
ctx.wasm_mem_size = size;
Ok(())
})
}
pub fn write_accessed_pages() -> Result<Vec<u32>, Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
ctx.borrow()
.execution_context()
.map(|ctx| ctx.write_accessed_pages.iter().map(|p| p.raw()).collect())
.map_err(Into::into)
})
}
pub fn status() -> Result<Status, Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
ctx.borrow()
.execution_context()
.map(|ctx| ctx.status)
.map_err(Into::into)
})
}
#[derive(Debug, Clone, derive_more::Display)]
pub enum InitError {
#[display(fmt = "Wrong page sizes amount: get {_0}, must be {_1}")]
WrongSizesAmount(usize, usize),
#[display(fmt = "Wrong global names amount: get {_0}, must be {_1}")]
WrongGlobalNamesAmount(usize, usize),
#[display(fmt = "Not suitable page sizes")]
NotSuitablePageSizes,
#[display(fmt = "Can not set signal handler: {_0}")]
CanNotSetUpSignalHandler(String),
#[display(fmt = "Failed to init for thread: {_0}")]
InitForThread(String),
#[display(fmt = "Provided by runtime memory page size cannot be zero")]
ZeroPageSize,
}
unsafe fn init_for_process<H: UserSignalHandler>() -> Result<(), InitError> {
use InitError::*;
#[cfg(target_vendor = "apple")]
{
use mach::{
exception_types::*, kern_return::*, mach_types::*, port::*, thread_status::*, traps::*,
};
extern "C" {
fn task_set_exception_ports(
task: task_t,
exception_mask: exception_mask_t,
new_port: mach_port_t,
behavior: exception_behavior_t,
new_flavor: thread_state_flavor_t,
) -> kern_return_t;
}
#[cfg(target_arch = "x86_64")]
static MACHINE_THREAD_STATE: i32 = x86_THREAD_STATE64 as i32;
#[cfg(target_arch = "aarch64")]
static MACHINE_THREAD_STATE: i32 = 6;
task_set_exception_ports(
mach_task_self(),
EXC_MASK_BAD_ACCESS,
MACH_PORT_NULL,
EXCEPTION_STATE_IDENTITY as exception_behavior_t,
MACHINE_THREAD_STATE,
);
}
LAZY_PAGES_INITIALIZED.get_or_init(|| {
if let Err(err) = sys::setup_signal_handler::<H>() {
return Err(CanNotSetUpSignalHandler(err.to_string()));
}
log::trace!("Successfully initialize lazy-pages for process");
Ok(())
})
}
#[cfg(test)]
pub(crate) fn reset_init_flag() {
LAZY_PAGES_INITIALIZED.reset();
}
fn init_with_handler<H: UserSignalHandler>(
_version: LazyPagesVersion,
page_sizes: Vec<u32>,
global_names: Vec<String>,
pages_storage_prefix: Vec<u8>,
) -> Result<(), InitError> {
use InitError::*;
let page_sizes = page_sizes
.into_iter()
.map(TryInto::<NonZeroU32>::try_into)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| ZeroPageSize)?;
let page_sizes: PageSizes = match page_sizes.try_into() {
Ok(sizes) => sizes,
Err(sizes) => return Err(WrongSizesAmount(sizes.len(), PageSizeNo::Amount as usize)),
};
let wasm_page_size = page_sizes[PageSizeNo::WasmSizeNo as usize];
let gear_page_size = page_sizes[PageSizeNo::GearSizeNo as usize];
let native_page_size = region::page::size();
if wasm_page_size < gear_page_size
|| (gear_page_size.get() as usize) < native_page_size
|| !u32::is_power_of_two(wasm_page_size.get())
|| !u32::is_power_of_two(gear_page_size.get())
|| !usize::is_power_of_two(native_page_size)
{
return Err(NotSuitablePageSizes);
}
let global_names: GlobalNames = match global_names.try_into() {
Ok(names) => names,
Err(names) => {
return Err(WrongGlobalNamesAmount(
names.len(),
GlobalNo::Amount as usize,
))
}
};
LAZY_PAGES_CONTEXT.with(|ctx| {
ctx.borrow_mut()
.set_runtime_context(LazyPagesRuntimeContext {
page_sizes,
global_names,
pages_storage_prefix,
})
});
unsafe { init_for_process::<H>()? }
unsafe { sys::init_for_thread().map_err(InitForThread)? }
Ok(())
}
pub fn init(
version: LazyPagesVersion,
page_sizes: Vec<u32>,
global_names: Vec<String>,
pages_storage_prefix: Vec<u8>,
) -> Result<(), InitError> {
init_with_handler::<DefaultUserSignalHandler>(
version,
page_sizes,
global_names,
pages_storage_prefix,
)
}