extern crate libc;
use std::fs::File;
use std::mem::ManuallyDrop;
use std::os::unix::io::{FromRawFd, RawFd};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{io, ptr};
#[cfg(any(
all(target_os = "linux", not(target_arch = "mips")),
target_os = "freebsd",
target_os = "android"
))]
const MAP_STACK: libc::c_int = libc::MAP_STACK;
#[cfg(not(any(
all(target_os = "linux", not(target_arch = "mips")),
target_os = "freebsd",
target_os = "android"
)))]
const MAP_STACK: libc::c_int = 0;
#[cfg(any(target_os = "linux", target_os = "android"))]
const MAP_POPULATE: libc::c_int = libc::MAP_POPULATE;
#[cfg(not(any(target_os = "linux", target_os = "android")))]
const MAP_POPULATE: libc::c_int = 0;
#[cfg(any(target_os = "linux", target_os = "android"))]
const MAP_HUGETLB: libc::c_int = libc::MAP_HUGETLB;
#[cfg(target_os = "linux")]
const MAP_HUGE_MASK: libc::c_int = libc::MAP_HUGE_MASK;
#[cfg(any(target_os = "linux", target_os = "android"))]
const MAP_HUGE_SHIFT: libc::c_int = libc::MAP_HUGE_SHIFT;
#[cfg(not(any(target_os = "linux", target_os = "android")))]
const MAP_HUGETLB: libc::c_int = 0;
#[cfg(not(target_os = "linux"))]
const MAP_HUGE_MASK: libc::c_int = 0;
#[cfg(not(any(target_os = "linux", target_os = "android")))]
const MAP_HUGE_SHIFT: libc::c_int = 0;
#[cfg(any(
target_os = "android",
all(target_os = "linux", not(target_env = "musl"))
))]
use libc::{mmap64 as mmap, off64_t as off_t};
#[cfg(not(any(
target_os = "android",
all(target_os = "linux", not(target_env = "musl"))
)))]
use libc::{mmap, off_t};
pub struct MmapInner {
ptr: *mut libc::c_void,
len: usize,
}
impl MmapInner {
/// Creates a new `MmapInner`.
///
/// This is a thin wrapper around the `mmap` system call.
fn new(
len: usize,
prot: libc::c_int,
flags: libc::c_int,
file: RawFd,
offset: u64,
) -> io::Result<MmapInner> {
let alignment = offset % page_size() as u64;
let aligned_offset = offset - alignment;
let (map_len, map_offset) = Self::adjust_mmap_params(len, alignment as usize)?;
unsafe {
let ptr = mmap(
ptr::null_mut(),
map_len as libc::size_t,
prot,
flags,
file,
aligned_offset as off_t,
);
if ptr == libc::MAP_FAILED {
Err(io::Error::last_os_error())
} else {
Ok(Self::from_raw_parts(ptr, len, map_offset))
}
}
}
fn adjust_mmap_params(len: usize, alignment: usize) -> io::Result<(usize, usize)> {
use std::isize;
// Rust's slice cannot be larger than isize::MAX.
// See
https://doc.rust-lang.org/std/slice/fn.from_raw_parts.html
//
// This is not a problem on 64-bit targets, but on 32-bit one
// having a file or an anonymous mapping larger than 2GB is quite normal
// and we have to prevent it.
//
// The code below is essentially the same as in Rust's std:
//
https://github.com/rust-lang/rust/blob/db78ab70a88a0a5e89031d7ee4eccec835dcdbde/library/alloc/src/raw_vec.rs#L495
if std::mem::size_of::<usize>() < 8 && len > isize::MAX as usize {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"memory map length overflows isize",
));
}
let map_len = len + alignment;
let map_offset = alignment;
// `libc::mmap` does not support zero-size mappings. POSIX defines:
//
//
https://pubs.opengroup.org/onlinepubs/9699919799/functions/mmap.html
// > If `len` is zero, `mmap()` shall fail and no mapping shall be established.
//
// So if we would create such a mapping, crate a one-byte mapping instead:
let map_len = map_len.max(1);
// Note that in that case `MmapInner::len` is still set to zero,
// and `Mmap` will still dereferences to an empty slice.
//
// If this mapping is backed by an empty file, we create a mapping larger than the file.
// This is unusual but well-defined. On the same man page, POSIX further defines:
//
// > The `mmap()` function can be used to map a region of memory that is larger
// > than the current size of the object.
//
// (The object here is the file.)
//
// > Memory access within the mapping but beyond the current end of the underlying
// > objects may result in SIGBUS signals being sent to the process. The reason for this
// > is that the size of the object can be manipulated by other processes and can change
// > at any moment. The implementation should tell the application that a memory reference
// > is outside the object where this can be detected; otherwise, written data may be lost
// > and read data may not reflect actual data in the object.
//
// Because `MmapInner::len` is not incremented, this increment of `aligned_len`
// will not allow accesses past the end of the file and will not cause SIGBUS.
//
// (SIGBUS is still possible by mapping a non-empty file and then truncating it
// to a shorter size, but that is unrelated to this handling of empty files.)
Ok((map_len, map_offset))
}
/// Get the current memory mapping as a `(ptr, map_len, offset)` tuple.
///
/// Note that `map_len` is the length of the memory mapping itself and
/// _not_ the one that would be passed to `from_raw_parts`.
fn as_mmap_params(&self) -> (*mut libc::c_void, usize, usize) {
let offset = self.ptr as usize % page_size();
let len = self.len + offset;
// There are two possible memory layouts we could have, depending on
// the length and offset passed when constructing this instance:
//
// 1. The "normal" memory layout looks like this:
//
// |<------------------>|<---------------------->|
// mmap ptr offset ptr public slice
//
// That is, we have
// - The start of the page-aligned memory mapping returned by mmap,
// followed by,
// - Some number of bytes that are memory mapped but ignored since
// they are before the byte offset requested by the user, followed
// by,
// - The actual memory mapped slice requested by the user.
//
// This maps cleanly to a (ptr, len, offset) tuple.
//
// 2. Then, we have the case where the user requested a zero-length
// memory mapping. mmap(2) does not support zero-length mappings so
// this crate works around that by actually making a mapping of
// length one. This means that we have
// - A length zero slice, followed by,
// - A single memory mapped byte
//
// Note that this only happens if the offset within the page is also
// zero. Otherwise, we have a memory map of offset bytes and not a
// zero-length memory map.
//
// This doesn't fit cleanly into a (ptr, len, offset) tuple. Instead,
// we fudge it slightly: a zero-length memory map turns into a
// mapping of length one and can't be told apart outside of this
// method without knowing the original length.
if len == 0 {
(self.ptr, 1, 0)
} else {
(unsafe { self.ptr.offset(-(offset as isize)) }, len, offset)
}
}
/// Construct this `MmapInner` from its raw components
///
/// # Safety
///
/// - `ptr` must point to the start of memory mapping that can be freed
/// using `munmap(2)` (i.e. returned by `mmap(2)` or `mremap(2)`)
/// - The memory mapping at `ptr` must have a length of `len + offset`.
/// - If `len + offset == 0` then the memory mapping must be of length 1.
/// - `offset` must be less than the current page size.
unsafe fn from_raw_parts(ptr: *mut libc::c_void, len: usize, offset: usize) -> Self {
debug_assert_eq!(ptr as usize % page_size(), 0, "ptr not page-aligned");
debug_assert!(offset < page_size(), "offset larger than page size");
Self {
ptr: ptr.add(offset),
len,
}
}
pub fn map(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
let populate = if populate { MAP_POPULATE } else { 0 };
MmapInner::new(
len,
libc::PROT_READ,
libc::MAP_SHARED | populate,
file,
offset,
)
}
pub fn map_exec(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner>
{
let populate = if populate { MAP_POPULATE } else { 0 };
MmapInner::new(
len,
libc::PROT_READ | libc::PROT_EXEC,
libc::MAP_SHARED | populate,
file,
offset,
)
}
pub fn map_mut(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
let populate = if populate { MAP_POPULATE } else { 0 };
MmapInner::new(
len,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_SHARED | populate,
file,
offset,
)
}
pub fn map_copy(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
let populate = if populate { MAP_POPULATE } else { 0 };
MmapInner::new(
len,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_PRIVATE | populate,
file,
offset,
)
}
pub fn map_copy_read_only(
len: usize,
file: RawFd,
offset: u64,
populate: bool,
) -> io::Result<MmapInner> {
let populate = if populate { MAP_POPULATE } else { 0 };
MmapInner::new(
len,
libc::PROT_READ,
libc::MAP_PRIVATE | populate,
file,
offset,
)
}
/// Open an anonymous memory map.
pub fn map_anon(
len: usize,
stack: bool,
populate: bool,
huge: Option<u8>,
) -> io::Result<MmapInner> {
let stack = if stack { MAP_STACK } else { 0 };
let populate = if populate { MAP_POPULATE } else { 0 };
let hugetlb = if huge.is_some() { MAP_HUGETLB } else { 0 };
let offset = huge
.map(|mask| ((mask as u64) & (MAP_HUGE_MASK as u64)) << MAP_HUGE_SHIFT)
.unwrap_or(0);
MmapInner::new(
len,
libc::PROT_READ | libc::PROT_WRITE,
libc::MAP_PRIVATE | libc::MAP_ANON | stack | populate | hugetlb,
-1,
offset,
)
}
pub fn flush(&self, offset: usize, len: usize) -> io::Result<()> {
let alignment = (self.ptr as usize + offset) % page_size();
let offset = offset as isize - alignment as isize;
let len = len + alignment;
let result =
unsafe { libc::msync(self.ptr.offset(offset), len as libc::size_t, libc::MS_SYNC) };
if result == 0 {
Ok(())
} else {
Err(io::Error::last_os_error())
}
}
pub fn flush_async(&self, offset: usize, len: usize) -> io::Result<()> {
let alignment = (self.ptr as usize + offset) % page_size();
let offset = offset as isize - alignment as isize;
let len = len + alignment;
let result =
unsafe { libc::msync(self.ptr.offset(offset), len as libc::size_t, libc::MS_ASYNC) };
if result == 0 {
Ok(())
} else {
Err(io::Error::last_os_error())
}
}
fn mprotect(&mut self, prot: libc::c_int) -> io::Result<()> {
unsafe {
let alignment = self.ptr as usize % page_size();
let ptr = self.ptr.offset(-(alignment as isize));
let len = self.len + alignment;
let len = len.max(1);
if libc::mprotect(ptr, len, prot) == 0 {
Ok(())
} else {
Err(io::Error::last_os_error())
}
}
}
pub fn make_read_only(&mut self) -> io::Result<()> {
self.mprotect(libc::PROT_READ)
}
pub fn make_exec(&mut self) -> io::Result<()> {
self.mprotect(libc::PROT_READ | libc::PROT_EXEC)
}
pub fn make_mut(&mut self) -> io::Result<()> {
self.mprotect(libc::PROT_READ | libc::PROT_WRITE)
}
#[inline]
pub fn ptr(&self) -> *const u8 {
self.ptr as *const u8
}
#[inline]
pub fn mut_ptr(&mut self) -> *mut u8 {
self.ptr as *mut u8
}
#[inline]
pub fn len(&self) -> usize {
self.len
}
pub fn advise(&self, advice: libc::c_int, offset: usize, len: usize) -> io::Result<()> {
let alignment = (self.ptr as usize + offset) % page_size();
let offset = offset as isize - alignment as isize;
let len = len + alignment;
unsafe {
if libc::madvise(self.ptr.offset(offset), len, advice) != 0 {
Err(io::Error::last_os_error())
} else {
Ok(())
}
}
}
#[cfg(target_os = "linux")]
pub fn remap(&mut self, new_len: usize, options: crate::RemapOptions) -> io::Result<()> {
let (old_ptr, old_len, offset) = self.as_mmap_params();
let (map_len, offset) = Self::adjust_mmap_params(new_len, offset)?;
unsafe {
let new_ptr = libc::mremap(old_ptr, old_len, map_len, options.into_flags());
if new_ptr == libc::MAP_FAILED {
Err(io::Error::last_os_error())
} else {
// We explicitly don't drop self since the pointer within is no longer valid.
ptr::write(self, Self::from_raw_parts(new_ptr, new_len, offset));
Ok(())
}
}
}
pub fn lock(&self) -> io::Result<()> {
unsafe {
if libc::mlock(self.ptr, self.len) != 0 {
Err(io::Error::last_os_error())
} else {
Ok(())
}
}
}
pub fn unlock(&self) -> io::Result<()> {
unsafe {
if libc::munlock(self.ptr, self.len) != 0 {
Err(io::Error::last_os_error())
} else {
Ok(())
}
}
}
}
impl Drop for MmapInner {
fn drop(&mut self) {
let (ptr, len, _) = self.as_mmap_params();
// Any errors during unmapping/closing are ignored as the only way
// to report them would be through panicking which is highly discouraged
// in Drop impls, c.f. https://github.com/rust-lang/lang-team/issues/97
unsafe { libc::munmap(ptr, len as libc::size_t) };
}
}
unsafe impl Sync for MmapInner {}
unsafe impl Send for MmapInner {}
fn page_size() -> usize {
static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
match PAGE_SIZE.load(Ordering::Relaxed) {
0 => {
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize };
PAGE_SIZE.store(page_size, Ordering::Relaxed);
page_size
}
page_size => page_size,
}
}
pub fn file_len(file: RawFd) -> io::Result<u64> {
// SAFETY: We must not close the passed-in fd by dropping the File we create,
// we ensure this by immediately wrapping it in a ManuallyDrop.
unsafe {
let file = ManuallyDrop::new(File::from_raw_fd(file));
Ok(file.metadata()?.len())
}
}
