#![allow(clippy::items_after_test_module)]
#![doc(html_logo_url = "https://docs.gear.rs/logo.svg")]
#![doc(html_favicon_url = "https://gear-tech.io/favicons/favicon.ico")]
mod common;
mod globals;
mod host_func;
mod init_flag;
mod mprotect;
mod process;
mod signal;
mod sys;
mod utils;
#[cfg(test)]
mod tests;
pub use crate::common::LazyPagesStorage;
pub use common::{LazyPagesVersion, PageSizes};
pub use host_func::pre_process_memory_accesses;
use crate::{
common::{ContextError, LazyPagesContext, PagePrefix, WeightNo, Weights},
globals::{GlobalNo, GlobalsContext},
init_flag::InitializationFlag,
};
use common::{LazyPagesExecutionContext, LazyPagesRuntimeContext};
use gear_core::pages::{PageDynSize, PageNumber, PageSizeNo, WasmPage};
use gear_lazy_pages_common::{GlobalsAccessConfig, LazyPagesInitContext, Status};
use mprotect::MprotectError;
use signal::{DefaultUserSignalHandler, UserSignalHandler};
use std::{cell::RefCell, convert::TryInto, num::NonZeroU32};
static LAZY_PAGES_INITIALIZED: InitializationFlag = InitializationFlag::new();
thread_local! {
static LAZY_PAGES_CONTEXT: RefCell<LazyPagesContext> = RefCell::new(Default::default());
}
#[derive(Debug, derive_more::Display, derive_more::From)]
pub enum Error {
#[display(fmt = "WASM memory native address {_0:#x} is not aligned to the native page size")]
WasmMemAddrIsNotAligned(usize),
#[display(fmt = "{_0}")]
#[from]
Mprotect(MprotectError),
#[display(fmt = "Wasm memory end addr is out of usize: begin addr = {_0:#x}, size = {_1:#x}")]
WasmMemoryEndAddrOverflow(usize, u32),
#[display(fmt = "Prefix of storage with memory pages was not set")]
MemoryPagesPrefixNotSet,
#[display(fmt = "Memory size must be null when memory host addr is not set")]
MemorySizeIsNotNull,
#[display(fmt = "Wasm mem size is too big")]
WasmMemSizeOverflow,
#[display(fmt = "Stack end offset cannot be bigger than memory size")]
StackEndBiggerThanMemSize,
#[display(fmt = "Stack end offset is too big")]
StackEndOverflow,
#[display(fmt = "Wasm addr and size are not changed, so host func call is needless")]
NothingToChange,
#[display(fmt = "Wasm memory addr must be set, when trying to change something in lazy pages")]
WasmMemAddrIsNotSet,
#[display(fmt = "{_0}")]
#[from]
GlobalContext(ContextError),
#[display(fmt = "Wrong weights amount: get {_0}, must be {_1}")]
WrongWeightsAmount(usize, usize),
}
fn check_memory_interval(addr: usize, size: u32) -> Result<(), Error> {
addr.checked_add(size as usize)
.ok_or(Error::WasmMemoryEndAddrOverflow(addr, size))
.map(|_| ())
}
pub fn initialize_for_program(
wasm_mem_addr: Option<usize>,
wasm_mem_size: u32,
stack_end: Option<u32>,
program_key: Vec<u8>,
globals_config: Option<GlobalsAccessConfig>,
weights: Vec<u64>,
) -> Result<(), Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
let mut ctx = ctx.borrow_mut();
let runtime_ctx = ctx.runtime_context_mut()?;
if let Some(addr) = wasm_mem_addr {
if addr % region::page::size() != 0 {
return Err(Error::WasmMemAddrIsNotAligned(addr));
}
}
let stack_end = stack_end.unwrap_or_default();
if wasm_mem_size < stack_end {
return Err(Error::StackEndBiggerThanMemSize);
}
let wasm_mem_size =
WasmPage::new(wasm_mem_size, runtime_ctx).ok_or(Error::WasmMemSizeOverflow)?;
let wasm_mem_size_in_bytes = wasm_mem_size.offset(runtime_ctx);
if let Some(addr) = wasm_mem_addr {
check_memory_interval(addr, wasm_mem_size_in_bytes)?;
} else if wasm_mem_size_in_bytes != 0 {
return Err(Error::MemorySizeIsNotNull);
}
let stack_end = WasmPage::new(stack_end, runtime_ctx).ok_or(Error::StackEndOverflow)?;
let weights: Weights = weights.try_into().map_err(|ws: Vec<u64>| {
Error::WrongWeightsAmount(ws.len(), WeightNo::Amount as usize)
})?;
let execution_ctx = LazyPagesExecutionContext {
weights,
wasm_mem_addr,
wasm_mem_size,
program_storage_prefix: PagePrefix::new_from_program_prefix(
[runtime_ctx.pages_storage_prefix.as_slice(), &program_key].concat(),
),
accessed_pages: Default::default(),
write_accessed_pages: Default::default(),
stack_end,
globals_context: globals_config.map(|cfg| GlobalsContext {
names: runtime_ctx.global_names.clone(),
access_ptr: cfg.access_ptr,
access_mod: cfg.access_mod,
}),
status: Status::Normal,
};
if let Some(addr) = wasm_mem_addr {
let stack_end_offset = execution_ctx.stack_end.offset(runtime_ctx);
log::trace!("{addr:#x} {stack_end_offset:#x}");
let addr = addr + stack_end_offset as usize;
let size = wasm_mem_size_in_bytes - stack_end_offset;
if size != 0 {
mprotect::mprotect_interval(addr, size as usize, false, false)?;
}
}
ctx.set_execution_context(execution_ctx);
log::trace!("Initialize lazy-pages for current program: {:?}", ctx);
Ok(())
})
}
pub fn set_lazy_pages_protection() -> Result<(), Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
let ctx = ctx.borrow();
let (rt_ctx, exec_ctx) = ctx.contexts()?;
let mem_addr = exec_ctx.wasm_mem_addr.ok_or(Error::WasmMemAddrIsNotSet)?;
let start_offset = exec_ctx.stack_end.offset(rt_ctx);
let mem_size = exec_ctx.wasm_mem_size.offset(rt_ctx);
mprotect::mprotect_mem_interval_except_pages(
mem_addr,
start_offset as usize,
mem_size as usize,
exec_ctx.write_accessed_pages.iter().copied(),
rt_ctx,
false,
false,
)?;
let read_only_pages = exec_ctx
.accessed_pages
.iter()
.filter(|&&page| !exec_ctx.write_accessed_pages.contains(&page))
.copied();
mprotect::mprotect_pages(mem_addr, read_only_pages, rt_ctx, true, false)?;
Ok(())
})
}
pub fn unset_lazy_pages_protection() -> Result<(), Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
let ctx = ctx.borrow();
let (rt_ctx, exec_ctx) = ctx.contexts()?;
let addr = exec_ctx.wasm_mem_addr.ok_or(Error::WasmMemAddrIsNotSet)?;
let size = exec_ctx.wasm_mem_size.offset(rt_ctx);
mprotect::mprotect_interval(addr, size as usize, true, true)?;
Ok(())
})
}
pub fn change_wasm_mem_addr_and_size(addr: Option<usize>, size: Option<u32>) -> Result<(), Error> {
if matches!((addr, size), (None, None)) {
return Err(Error::NothingToChange);
}
LAZY_PAGES_CONTEXT.with(|ctx| {
let mut ctx = ctx.borrow_mut();
let (rt_ctx, exec_ctx) = ctx.contexts_mut()?;
let addr = match addr {
Some(addr) => match addr % region::page::size() {
0 => addr,
_ => return Err(Error::WasmMemAddrIsNotAligned(addr)),
},
None => match exec_ctx.wasm_mem_addr {
Some(addr) => addr,
None => return Err(Error::WasmMemAddrIsNotSet),
},
};
let size = match size {
Some(size) => WasmPage::new(size, rt_ctx).ok_or(Error::WasmMemSizeOverflow)?,
None => exec_ctx.wasm_mem_size,
};
check_memory_interval(addr, size.offset(rt_ctx))?;
exec_ctx.wasm_mem_addr = Some(addr);
exec_ctx.wasm_mem_size = size;
Ok(())
})
}
pub fn write_accessed_pages() -> Result<Vec<u32>, Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
ctx.borrow()
.execution_context()
.map(|ctx| ctx.write_accessed_pages.iter().map(|p| p.raw()).collect())
.map_err(Into::into)
})
}
pub fn status() -> Result<Status, Error> {
LAZY_PAGES_CONTEXT.with(|ctx| {
ctx.borrow()
.execution_context()
.map(|ctx| ctx.status)
.map_err(Into::into)
})
}
#[derive(Debug, Clone, derive_more::Display)]
pub enum InitError {
#[display(fmt = "Wrong page sizes amount: get {_0}, must be {_1}")]
WrongSizesAmount(usize, usize),
#[display(fmt = "Wrong global names: expected {_0}, found {_1}")]
WrongGlobalNames(String, String),
#[display(fmt = "Not suitable page sizes")]
NotSuitablePageSizes,
#[display(fmt = "Can not set signal handler: {_0}")]
CanNotSetUpSignalHandler(String),
#[display(fmt = "Failed to init for thread: {_0}")]
InitForThread(String),
#[display(fmt = "Provided by runtime memory page size cannot be zero")]
ZeroPageSize,
}
unsafe fn init_for_process<H: UserSignalHandler>() -> Result<(), InitError> {
use InitError::*;
#[cfg(target_vendor = "apple")]
{
use mach::{
exception_types::*, kern_return::*, mach_types::*, port::*, thread_status::*, traps::*,
};
extern "C" {
fn task_set_exception_ports(
task: task_t,
exception_mask: exception_mask_t,
new_port: mach_port_t,
behavior: exception_behavior_t,
new_flavor: thread_state_flavor_t,
) -> kern_return_t;
}
#[cfg(target_arch = "x86_64")]
static MACHINE_THREAD_STATE: i32 = x86_THREAD_STATE64 as i32;
#[cfg(target_arch = "aarch64")]
static MACHINE_THREAD_STATE: i32 = 6;
task_set_exception_ports(
mach_task_self(),
EXC_MASK_BAD_ACCESS,
MACH_PORT_NULL,
EXCEPTION_STATE_IDENTITY as exception_behavior_t,
MACHINE_THREAD_STATE,
);
}
LAZY_PAGES_INITIALIZED.get_or_init(|| {
if let Err(err) = sys::setup_signal_handler::<H>() {
return Err(CanNotSetUpSignalHandler(err.to_string()));
}
log::trace!("Successfully initialize lazy-pages for process");
Ok(())
})
}
#[cfg(test)]
pub(crate) fn reset_init_flag() {
LAZY_PAGES_INITIALIZED.reset();
}
fn init_with_handler<H: UserSignalHandler, S: LazyPagesStorage + 'static>(
_version: LazyPagesVersion,
ctx: LazyPagesInitContext,
pages_storage: S,
) -> Result<(), InitError> {
use InitError::*;
let LazyPagesInitContext {
page_sizes,
global_names,
pages_storage_prefix,
} = ctx;
let page_sizes = page_sizes
.into_iter()
.map(TryInto::<NonZeroU32>::try_into)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| ZeroPageSize)?;
let page_sizes: PageSizes = match page_sizes.try_into() {
Ok(sizes) => sizes,
Err(sizes) => return Err(WrongSizesAmount(sizes.len(), PageSizeNo::Amount as usize)),
};
let wasm_page_size = page_sizes[PageSizeNo::WasmSizeNo as usize];
let gear_page_size = page_sizes[PageSizeNo::GearSizeNo as usize];
let native_page_size = region::page::size();
if wasm_page_size < gear_page_size
|| (gear_page_size.get() as usize) < native_page_size
|| !u32::is_power_of_two(wasm_page_size.get())
|| !u32::is_power_of_two(gear_page_size.get())
|| !usize::is_power_of_two(native_page_size)
{
return Err(NotSuitablePageSizes);
}
if global_names[GlobalNo::Gas as usize].as_str() != "gear_gas" {
return Err(WrongGlobalNames(
"gear_gas".to_string(),
global_names[GlobalNo::Gas as usize].to_string(),
));
}
LAZY_PAGES_CONTEXT.with(|ctx| {
ctx.borrow_mut()
.set_runtime_context(LazyPagesRuntimeContext {
page_sizes,
global_names,
pages_storage_prefix,
program_storage: Box::new(pages_storage),
})
});
unsafe { init_for_process::<H>()? }
unsafe { sys::init_for_thread().map_err(InitForThread)? }
Ok(())
}
pub fn init<S: LazyPagesStorage + 'static>(
version: LazyPagesVersion,
ctx: LazyPagesInitContext,
pages_storage: S,
) -> Result<(), InitError> {
init_with_handler::<DefaultUserSignalHandler, S>(version, ctx, pages_storage)
}